00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015 #include "httpClient.h"
00016 #include "httpChannel.h"
00017 #include "config_downloader.h"
00018 #include "filename.h"
00019 #include "config_express.h"
00020 #include "virtualFileSystem.h"
00021 #include "executionEnvironment.h"
00022 #include "httpBasicAuthorization.h"
00023 #include "httpDigestAuthorization.h"
00024 #include "globPattern.h"
00025
00026 #ifdef HAVE_OPENSSL
00027
00028 PT(HTTPClient) HTTPClient::_global_ptr;
00029
00030
00031
00032
00033
00034 static string
00035 trim_blanks(const string &str) {
00036 size_t start = 0;
00037 while (start < str.length() && isspace(str[start])) {
00038 start++;
00039 }
00040
00041 size_t end = str.length();
00042 while (end > start && isspace(str[end - 1])) {
00043 end--;
00044 }
00045
00046 return str.substr(start, end - start);
00047 }
00048
00049
00050
00051
00052
00053
00054
00055
00056
00057
00058
00059
00060
00061 static void
00062 tokenize(const string &str, vector_string &words, const string &delimiters) {
00063 size_t p = 0;
00064 while (p < str.length()) {
00065 size_t q = str.find_first_of(delimiters, p);
00066 if (q == string::npos) {
00067 words.push_back(str.substr(p));
00068 return;
00069 }
00070 words.push_back(str.substr(p, q - p));
00071 p = q + 1;
00072 }
00073 words.push_back(string());
00074 }
00075
00076
00077
00078
00079
00080
00081 HTTPClient::
00082 HTTPClient() {
00083 ConfigVariableBool verify_ssl
00084 ("verify-ssl", true,
00085 PRC_DESC("Configure this true (the default) to insist on verifying all SSL "
00086 "(e.g. https) servers against a known certificate, or false to allow "
00087 "an unverified connection. This controls the default behavior; the "
00088 "specific behavior for a particular HTTPClient can be adjusted at "
00089 "runtime with set_verify_ssl()."));
00090
00091 ConfigVariableString ssl_cipher_list
00092 ("ssl-cipher-list", "DEFAULT",
00093 PRC_DESC("This is the default value for HTTPClient::set_cipher_list()."));
00094
00095 ConfigVariableString http_proxy
00096 ("http-proxy", "",
00097 PRC_DESC("This specifies the default value for HTTPClient::set_proxy_spec(). "
00098 "It is a semicolon-delimited list of proxies that we use to contact "
00099 "all HTTP hosts that don't specify otherwise. See "
00100 "set_proxy_spec() for more information."));
00101
00102 ConfigVariableString http_direct_hosts
00103 ("http-direct-hosts", "",
00104 PRC_DESC("This specifies the default value for HTTPClient::set_direct_host_spec(). "
00105 "It is a semicolon-delimited list of host names that do not require a "
00106 "proxy. See set_direct_host_spec() for more information."));
00107
00108 ConfigVariableBool http_try_all_direct
00109 ("http-try-all-direct", true,
00110 PRC_DESC("This specifies the default value for HTTPClient::set_try_all_direct(). "
00111 "If this is true, a direct connection will always be attempted after an "
00112 "attempt to connect through a proxy fails."));
00113
00114 ConfigVariableString http_proxy_username
00115 ("http-proxy-username", "",
00116 PRC_DESC("This specifies a default username:password to pass to the proxy."));
00117
00118 ConfigVariableList http_username
00119 ("http-username",
00120 PRC_DESC("Adds one or more username/password pairs to all HTTP clients. The client "
00121 "will present this username/password when asked to authenticate a request "
00122 "for a particular server and/or realm. The username is of the form "
00123 "server:realm:username:password, where either or both of server and "
00124 "realm may be empty, or just realm:username:password or username:password. "
00125 "If the server or realm is empty, they will match anything."));
00126
00127 ConfigVariableFilename http_client_certificate_filename
00128 ("http-client-certificate-filename", "",
00129 PRC_DESC("This provides a default client certificate to offer up should an "
00130 "SSL server demand one. The file names a PEM-formatted file "
00131 "that includes a public and private key specification. A "
00132 "connection-specific certificate may also be specified at runtime on "
00133 "the HTTPClient object, but this will require having a different "
00134 "HTTPClient object for each differently-certificated connection."));
00135
00136 ConfigVariableString http_client_certificate_passphrase
00137 ("http-client-certificate-passphrase", "",
00138 PRC_DESC("This specifies the passphrase to use to decode the certificate named "
00139 "by http-client-certificate-filename."));
00140
00141
00142 ConfigVariableList http_preapproved_server_certificate_filename
00143 ("http-preapproved-server-certificate-filename",
00144 PRC_DESC("This specifies a server hostname:port combination, followed by "
00145 "at least one space, and the name of a PEM file that contains "
00146 "an SSL certificate that we expect this server to offer. If "
00147 "it does, we will accept the cert without further question, "
00148 "even if the cert does not match any known certificate "
00149 "authorities. This option may appear multiple times."));
00150
00151 ConfigVariableList http_preapproved_server_certificate_name
00152 ("http-preapproved-server-certificate-name",
00153 PRC_DESC("This specifies a server hostname:port combination, followed by "
00154 "at least one space, and a string describing the subject name "
00155 "of an SSL certificate we expect this server to offer. If it "
00156 "it does, we will accept the cert, but only if it also matches "
00157 "a known certificate authority. This option may appear "
00158 "multiple times."));
00159
00160 _http_version = HTTPEnum::HV_11;
00161 _verify_ssl = verify_ssl ? VS_normal : VS_no_verify;
00162 _ssl_ctx = (SSL_CTX *)NULL;
00163
00164 set_proxy_spec(http_proxy);
00165 set_direct_host_spec(http_direct_hosts);
00166 _try_all_direct = http_try_all_direct;
00167
00168 if (!http_proxy_username.empty()) {
00169 set_username("*proxy", "", http_proxy_username);
00170 }
00171
00172 set_cipher_list(ssl_cipher_list);
00173
00174 {
00175
00176 int num_unique_values = http_username.get_num_unique_values();
00177 for (int i = 0; i < num_unique_values; i++) {
00178 string username = http_username.get_unique_value(i);
00179 add_http_username(username);
00180 }
00181 }
00182
00183 _client_certificate_filename = http_client_certificate_filename;
00184 _client_certificate_passphrase = http_client_certificate_passphrase;
00185
00186 _client_certificate_loaded = false;
00187 _client_certificate_pub = NULL;
00188 _client_certificate_priv = NULL;
00189
00190 int num_server_certs = http_preapproved_server_certificate_filename.get_num_unique_values();
00191 int si;
00192 for (si = 0; si < num_server_certs; si++) {
00193 string cert_line = http_preapproved_server_certificate_filename.get_unique_value(si);
00194 string a, b;
00195 split_whitespace(a, b, cert_line);
00196 add_preapproved_server_certificate_filename(URLSpec(a, true), b);
00197 }
00198
00199 num_server_certs = http_preapproved_server_certificate_name.get_num_unique_values();
00200 for (si = 0; si < num_server_certs; si++) {
00201 string cert_line = http_preapproved_server_certificate_name.get_unique_value(si);
00202 string a, b;
00203 split_whitespace(a, b, cert_line);
00204 add_preapproved_server_certificate_name(URLSpec(a, true), b);
00205 }
00206
00207
00208
00209 OpenSSLWrapper::get_global_ptr();
00210 }
00211
00212
00213
00214
00215
00216
00217 HTTPClient::
00218 HTTPClient(const HTTPClient ©) {
00219 _ssl_ctx = (SSL_CTX *)NULL;
00220
00221 (*this) = copy;
00222 }
00223
00224
00225
00226
00227
00228
00229 void HTTPClient::
00230 operator = (const HTTPClient ©) {
00231 _proxies_by_scheme = copy._proxies_by_scheme;
00232 _direct_hosts = copy._direct_hosts;
00233 _try_all_direct = copy._try_all_direct;
00234 _http_version = copy._http_version;
00235 _verify_ssl = copy._verify_ssl;
00236 _usernames = copy._usernames;
00237 _cookies = copy._cookies;
00238 }
00239
00240
00241
00242
00243
00244
00245 HTTPClient::
00246 ~HTTPClient() {
00247
00248
00249
00250 if (_ssl_ctx != (SSL_CTX *)NULL) {
00251 _ssl_ctx->cert_store = NULL;
00252 SSL_CTX_free(_ssl_ctx);
00253 }
00254
00255 unload_client_certificate();
00256 }
00257
00258
00259
00260
00261
00262
00263
00264
00265
00266
00267
00268
00269
00270
00271
00272
00273
00274 void HTTPClient::
00275 init_random_seed() {
00276
00277
00278 OpenSSLWrapper::get_global_ptr();
00279 }
00280
00281
00282
00283
00284
00285
00286
00287
00288
00289
00290
00291
00292
00293
00294
00295 void HTTPClient::
00296 set_proxy_spec(const string &proxy_spec) {
00297 clear_proxy();
00298
00299 string trim_proxy_spec = trim_blanks(proxy_spec);
00300
00301
00302 if (!trim_proxy_spec.empty()) {
00303 vector_string proxies;
00304 tokenize(trim_proxy_spec, proxies, ";");
00305
00306 for (vector_string::const_iterator pi = proxies.begin();
00307 pi != proxies.end();
00308 ++pi) {
00309 const string &spec = (*pi);
00310
00311
00312 string scheme;
00313 string proxy;
00314 size_t equals = spec.find('=');
00315 if (equals == string::npos) {
00316 scheme = "";
00317 proxy = trim_blanks(spec);
00318 } else {
00319 scheme = trim_blanks(spec.substr(0, equals));
00320 proxy = trim_blanks(spec.substr(equals + 1));
00321 }
00322
00323 if (proxy == "DIRECT" || proxy.empty()) {
00324 add_proxy(scheme, URLSpec());
00325 } else {
00326 add_proxy(scheme, URLSpec(proxy, true));
00327 }
00328 }
00329 }
00330 }
00331
00332
00333
00334
00335
00336
00337
00338
00339
00340
00341
00342
00343 string HTTPClient::
00344 get_proxy_spec() const {
00345 string result;
00346
00347 ProxiesByScheme::const_iterator si;
00348 for (si = _proxies_by_scheme.begin(); si != _proxies_by_scheme.end(); ++si) {
00349 const string &scheme = (*si).first;
00350 const Proxies &proxies = (*si).second;
00351 Proxies::const_iterator pi;
00352 for (pi = proxies.begin(); pi != proxies.end(); ++pi) {
00353 const URLSpec &url = (*pi);
00354 if (!result.empty()) {
00355 result += ";";
00356 }
00357 if (!scheme.empty()) {
00358 result += scheme;
00359 result += "=";
00360 }
00361 if (url.empty()) {
00362 result += "DIRECT";
00363 } else {
00364 result += url.get_url();
00365 }
00366 }
00367 }
00368
00369 return result;
00370 }
00371
00372
00373
00374
00375
00376
00377
00378
00379
00380 void HTTPClient::
00381 set_direct_host_spec(const string &direct_host_spec) {
00382 clear_direct_host();
00383
00384
00385 vector_string hosts;
00386 tokenize(direct_host_spec, hosts, ";");
00387
00388 for (vector_string::const_iterator hi = hosts.begin();
00389 hi != hosts.end();
00390 ++hi) {
00391 string spec = trim_blanks(*hi);
00392
00393
00394
00395
00396 if (!spec.empty()) {
00397 add_direct_host(spec);
00398 }
00399 }
00400 }
00401
00402
00403
00404
00405
00406
00407
00408
00409
00410 string HTTPClient::
00411 get_direct_host_spec() const {
00412 string result;
00413
00414 DirectHosts::const_iterator si;
00415 for (si = _direct_hosts.begin(); si != _direct_hosts.end(); ++si) {
00416 const GlobPattern &host = (*si);
00417
00418 if (!result.empty()) {
00419 result += ";";
00420 }
00421 result += host.get_pattern();
00422 }
00423
00424 return result;
00425 }
00426
00427
00428
00429
00430
00431
00432
00433
00434 void HTTPClient::
00435 clear_proxy() {
00436 _proxies_by_scheme.clear();
00437 }
00438
00439
00440
00441
00442
00443
00444
00445
00446
00447
00448 void HTTPClient::
00449 add_proxy(const string &scheme, const URLSpec &proxy) {
00450 URLSpec proxy_url(proxy);
00451
00452
00453 string lc_scheme;
00454 lc_scheme.reserve(scheme.length());
00455 string::const_iterator si;
00456 for (si = scheme.begin(); si != scheme.end(); ++si) {
00457 lc_scheme += tolower(*si);
00458 }
00459
00460
00461 if (!lc_scheme.empty() && lc_scheme[lc_scheme.length() - 1] == ':') {
00462 lc_scheme = lc_scheme.substr(0, lc_scheme.length() - 1);
00463 }
00464
00465 if (!proxy_url.empty()) {
00466
00467
00468
00469
00470
00471 if (lc_scheme == "socks") {
00472
00473
00474 proxy_url.set_scheme("socks");
00475
00476 } else if (!proxy_url.has_scheme()) {
00477
00478
00479 proxy_url.set_scheme("http");
00480 }
00481 }
00482
00483 _proxies_by_scheme[lc_scheme].push_back(proxy_url);
00484 }
00485
00486
00487
00488
00489
00490
00491
00492
00493
00494 void HTTPClient::
00495 clear_direct_host() {
00496 _direct_hosts.clear();
00497 }
00498
00499
00500
00501
00502
00503
00504
00505
00506
00507 void HTTPClient::
00508 add_direct_host(const string &hostname) {
00509
00510 string lc_hostname;
00511 lc_hostname.reserve(hostname.length());
00512 for (string::const_iterator si = hostname.begin();
00513 si != hostname.end();
00514 ++si) {
00515 lc_hostname += tolower(*si);
00516 }
00517
00518 _direct_hosts.push_back(GlobPattern(lc_hostname));
00519 }
00520
00521
00522
00523
00524
00525
00526
00527
00528
00529
00530
00531
00532
00533
00534 void HTTPClient::
00535 get_proxies_for_url(const URLSpec &url, pvector<URLSpec> &proxies) const {
00536
00537 string hostname = url.get_server();
00538
00539
00540
00541 if (!hostname.empty()) {
00542 DirectHosts::const_iterator si;
00543 for (si = _direct_hosts.begin(); si != _direct_hosts.end(); ++si) {
00544 if ((*si).matches(hostname)) {
00545
00546 proxies.push_back(URLSpec());
00547 return;
00548 }
00549 }
00550 }
00551
00552
00553
00554 pvector<URLSpec> temp_list;
00555
00556
00557 string scheme = url.get_scheme();
00558 bool got_any = false;
00559
00560 if (!scheme.empty()) {
00561
00562 if (get_proxies_for_scheme(scheme, temp_list)) {
00563 got_any = true;
00564 }
00565 }
00566
00567 if (!got_any && (scheme.empty() || url.is_ssl())) {
00568
00569
00570
00571
00572 if (get_proxies_for_scheme("socks", temp_list)) {
00573 got_any = true;
00574 }
00575 if (get_proxies_for_scheme("https", temp_list)) {
00576 got_any = true;
00577 }
00578 }
00579
00580 if (!got_any) {
00581
00582
00583 if (get_proxies_for_scheme("", temp_list)) {
00584 got_any = true;
00585 }
00586 }
00587
00588 if (_try_all_direct) {
00589
00590 temp_list.push_back(URLSpec());
00591 }
00592
00593
00594 if (!got_any) {
00595 get_proxies_for_scheme("http", temp_list);
00596 }
00597
00598
00599 pvector<URLSpec>::iterator pi;
00600 pset<URLSpec> used;
00601 for (pi = temp_list.begin(); pi != temp_list.end(); ++pi) {
00602 if (used.insert(*pi).second) {
00603
00604 proxies.push_back(*pi);
00605 }
00606 }
00607 }
00608
00609
00610
00611
00612
00613
00614
00615
00616
00617 string HTTPClient::
00618 get_proxies_for_url(const URLSpec &url) const {
00619 pvector<URLSpec> proxies;
00620 get_proxies_for_url(url, proxies);
00621
00622 string result;
00623 if (!proxies.empty()) {
00624 pvector<URLSpec>::const_iterator pi = proxies.begin();
00625 if ((*pi).get_url().empty()) {
00626 result += "DIRECT";
00627 } else {
00628 result += (*pi).get_url();
00629 }
00630 ++pi;
00631
00632 while (pi != proxies.end()) {
00633 result += ";";
00634 if ((*pi).get_url().empty()) {
00635 result += "DIRECT";
00636 } else {
00637 result += (*pi).get_url();
00638 }
00639 ++pi;
00640 }
00641 }
00642
00643 return result;
00644 }
00645
00646
00647
00648
00649
00650
00651
00652
00653
00654
00655
00656
00657
00658
00659
00660 void HTTPClient::
00661 set_username(const string &server, const string &realm, const string &username) {
00662 string key = server + ":" + realm;
00663 if (username.empty()) {
00664 _usernames.erase(key);
00665 } else {
00666 _usernames[key] = username;
00667 }
00668 }
00669
00670
00671
00672
00673
00674
00675
00676
00677 string HTTPClient::
00678 get_username(const string &server, const string &realm) const {
00679 string key = server + ":" + realm;
00680 Usernames::const_iterator ui;
00681 ui = _usernames.find(key);
00682 if (ui != _usernames.end()) {
00683 return (*ui).second;
00684 }
00685 return string();
00686 }
00687
00688
00689
00690
00691
00692
00693
00694 void HTTPClient::
00695 set_cookie(const HTTPCookie &cookie) {
00696 if (cookie.is_expired()) {
00697 clear_cookie(cookie);
00698
00699 } else {
00700 pair<Cookies::iterator, bool> result = _cookies.insert(cookie);
00701 if (!result.second) {
00702
00703
00704 const HTTPCookie &orig_cookie = *result.first;
00705 ((HTTPCookie &)orig_cookie).update_from(cookie);
00706 }
00707 }
00708 }
00709
00710
00711
00712
00713
00714
00715
00716
00717 bool HTTPClient::
00718 clear_cookie(const HTTPCookie &cookie) {
00719 Cookies::iterator ci = _cookies.find(cookie);
00720 if (ci != _cookies.end()) {
00721 _cookies.erase(ci);
00722 return true;
00723 }
00724
00725 return false;
00726 }
00727
00728
00729
00730
00731
00732
00733 void HTTPClient::
00734 clear_all_cookies() {
00735 _cookies.clear();
00736 }
00737
00738
00739
00740
00741
00742
00743
00744
00745 bool HTTPClient::
00746 has_cookie(const HTTPCookie &cookie) const {
00747 Cookies::const_iterator ci = _cookies.find(cookie);
00748 return (ci != _cookies.end());
00749 }
00750
00751
00752
00753
00754
00755
00756
00757
00758 HTTPCookie HTTPClient::
00759 get_cookie(const HTTPCookie &cookie) const {
00760 Cookies::const_iterator ci = _cookies.find(cookie);
00761 if (ci != _cookies.end()) {
00762 return (*ci);
00763 }
00764
00765 return HTTPCookie();
00766 }
00767
00768
00769
00770
00771
00772
00773
00774
00775
00776 void HTTPClient::
00777 copy_cookies_from(const HTTPClient &other) {
00778 Cookies::const_iterator ci;
00779 for (ci = other._cookies.begin(); ci != other._cookies.end(); ++ci) {
00780 set_cookie(*ci);
00781 }
00782 }
00783
00784
00785
00786
00787
00788
00789
00790
00791
00792 void HTTPClient::
00793 write_cookies(ostream &out) const {
00794 Cookies::const_iterator ci;
00795 for (ci = _cookies.begin(); ci != _cookies.end(); ++ci) {
00796 out << *ci << "\n";
00797 }
00798 }
00799
00800
00801
00802
00803
00804
00805
00806
00807
00808 void HTTPClient::
00809 send_cookies(ostream &out, const URLSpec &url) {
00810 HTTPDate now = HTTPDate::now();
00811 bool any_expired = false;
00812 bool first_cookie = true;
00813
00814 Cookies::const_iterator ci;
00815 for (ci = _cookies.begin(); ci != _cookies.end(); ++ci) {
00816 const HTTPCookie &cookie = (*ci);
00817 if (cookie.is_expired(now)) {
00818 any_expired = true;
00819
00820 } else if (cookie.matches_url(url)) {
00821 if (first_cookie) {
00822 out << "Cookie: ";
00823 first_cookie = false;
00824 } else {
00825 out << "; ";
00826 }
00827 out << cookie.get_name() << "=" << cookie.get_value();
00828 }
00829 }
00830
00831 if (!first_cookie) {
00832 out << "\r\n";
00833 }
00834
00835 if (any_expired) {
00836 Cookies new_cookies;
00837 Cookies::const_iterator ci;
00838 for (ci = _cookies.begin(); ci != _cookies.end(); ++ci) {
00839 const HTTPCookie &cookie = (*ci);
00840 if (!cookie.is_expired(now)) {
00841 new_cookies.insert(new_cookies.end(), cookie);
00842 }
00843 }
00844 _cookies.swap(new_cookies);
00845 }
00846 }
00847
00848
00849
00850
00851
00852
00853
00854
00855
00856
00857
00858
00859
00860 bool HTTPClient::
00861 load_client_certificate() {
00862 if (!_client_certificate_loaded) {
00863 _client_certificate_loaded = true;
00864
00865 if (!_client_certificate_filename.empty()) {
00866 _client_certificate_filename.set_text();
00867
00868
00869 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00870
00871 if (!vfs->read_file(_client_certificate_filename,
00872 _client_certificate_pem, true)) {
00873
00874 downloader_cat.warning()
00875 << "Could not read " << _client_certificate_filename << ".\n";
00876 return false;
00877 }
00878 }
00879
00880 if (!_client_certificate_pem.empty()) {
00881
00882
00883
00884 BIO *mbio = BIO_new_mem_buf((void *)_client_certificate_pem.data(),
00885 _client_certificate_pem.length());
00886
00887 ERR_clear_error();
00888 _client_certificate_priv =
00889 PEM_read_bio_PrivateKey(mbio, NULL, NULL,
00890 (char *)_client_certificate_passphrase.c_str());
00891
00892
00893
00894 BIO_reset(mbio);
00895
00896 ERR_clear_error();
00897 _client_certificate_pub =
00898 PEM_read_bio_X509(mbio, NULL, NULL, NULL);
00899
00900 BIO_free(mbio);
00901
00902
00903 NotifySeverity sev = NS_debug;
00904 string source = "memory";
00905 if (!_client_certificate_filename.empty()) {
00906
00907
00908
00909
00910 sev = NS_info;
00911 source = _client_certificate_filename;
00912 }
00913
00914 if (downloader_cat.is_on(sev)) {
00915 if (_client_certificate_priv != (EVP_PKEY *)NULL &&
00916 _client_certificate_pub != (X509 *)NULL) {
00917 downloader_cat.out(sev)
00918 << "Read client certificate from " << source << "\n";
00919
00920 } else {
00921 if (_client_certificate_priv == (EVP_PKEY *)NULL) {
00922 downloader_cat.out(sev)
00923 << "Could not read private key from " << source << "\n";
00924 }
00925
00926 if (_client_certificate_pub == (X509 *)NULL) {
00927 downloader_cat.out(sev)
00928 << "Could not read public key from " << source << "\n";
00929 }
00930 }
00931 }
00932 }
00933 }
00934
00935 return (_client_certificate_priv != (EVP_PKEY *)NULL &&
00936 _client_certificate_pub != (X509 *)NULL);
00937 }
00938
00939
00940
00941
00942
00943
00944
00945
00946
00947
00948
00949
00950
00951
00952
00953
00954
00955
00956
00957 bool HTTPClient::
00958 add_preapproved_server_certificate_filename(const URLSpec &url, const Filename &filename) {
00959 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00960 string pem;
00961
00962 if (!vfs->read_file(filename, pem, true)) {
00963
00964 downloader_cat.warning()
00965 << "Could not read " << filename << ".\n";
00966 return false;
00967 }
00968
00969 return add_preapproved_server_certificate_pem(url, pem);
00970 }
00971
00972
00973
00974
00975
00976
00977
00978
00979
00980
00981
00982
00983
00984
00985
00986
00987
00988
00989
00990 bool HTTPClient::
00991 add_preapproved_server_certificate_pem(const URLSpec &url, const string &pem) {
00992
00993
00994
00995 BIO *mbio = BIO_new_mem_buf((void *)pem.data(), pem.length());
00996
00997 ERR_clear_error();
00998 X509 *cert = PEM_read_bio_X509(mbio, NULL, NULL, NULL);
00999 BIO_free(mbio);
01000
01001 if (cert == NULL) {
01002 downloader_cat.warning()
01003 << "Could not parse PEM data\n";
01004 return false;
01005 }
01006
01007 string server_and_port = url.get_server_and_port();
01008 PreapprovedServerCerts::iterator psci =
01009 _preapproved_server_certs.insert(PreapprovedServerCerts::value_type(server_and_port, PreapprovedServerCert())).first;
01010
01011 PreapprovedServerCert &psc = (*psci).second;
01012 psc._certs.push_back(cert);
01013
01014 return true;
01015 }
01016
01017
01018
01019
01020
01021
01022
01023
01024
01025
01026
01027
01028
01029
01030
01031
01032
01033
01034
01035
01036
01037
01038
01039
01040
01041 bool HTTPClient::
01042 add_preapproved_server_certificate_name(const URLSpec &url, const string &name) {
01043 X509_NAME *cert_name = parse_x509_name(name);
01044 if (cert_name == NULL) {
01045 downloader_cat.warning()
01046 << "Could not parse certificate name " << name << "\n";
01047 return false;
01048 }
01049
01050 string server_and_port = url.get_server_and_port();
01051 PreapprovedServerCerts::iterator psci =
01052 _preapproved_server_certs.insert(PreapprovedServerCerts::value_type(server_and_port, PreapprovedServerCert())).first;
01053
01054 PreapprovedServerCert &psc = (*psci).second;
01055 psc._cert_names.push_back(cert_name);
01056
01057 return true;
01058 }
01059
01060
01061
01062
01063
01064
01065
01066 void HTTPClient::
01067 clear_preapproved_server_certificates(const URLSpec &url) {
01068 string server_and_port = url.get_server_and_port();
01069 _preapproved_server_certs.erase(server_and_port);
01070 }
01071
01072
01073
01074
01075
01076
01077
01078 void HTTPClient::
01079 clear_all_preapproved_server_certificates() {
01080 _preapproved_server_certs.clear();
01081 }
01082
01083
01084
01085
01086
01087
01088
01089 string HTTPClient::
01090 get_http_version_string() const {
01091 switch (_http_version) {
01092 case HTTPEnum::HV_09:
01093 return "HTTP/0.9";
01094
01095 case HTTPEnum::HV_10:
01096 return "HTTP/1.0";
01097
01098 case HTTPEnum::HV_11:
01099 return "HTTP/1.1";
01100
01101 case HTTPEnum::HV_other:
01102
01103 return "HTTP/1.1";
01104 }
01105
01106 return "unknown";
01107 }
01108
01109
01110
01111
01112
01113
01114
01115
01116
01117 HTTPEnum::HTTPVersion HTTPClient::
01118 parse_http_version_string(const string &version) {
01119 if (version == "HTTP/1.0") {
01120 return HTTPEnum::HV_10;
01121 } else if (version == "HTTP/1.1") {
01122 return HTTPEnum::HV_11;
01123 } else if (version.substr(0, 6) == "HTTP/0") {
01124 return HTTPEnum::HV_09;
01125 } else {
01126 return HTTPEnum::HV_other;
01127 }
01128 }
01129
01130
01131
01132
01133
01134
01135
01136
01137
01138
01139 bool HTTPClient::
01140 load_certificates(const Filename &filename) {
01141 OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
01142 return (sslw->load_certificates(filename) != 0);
01143 }
01144
01145
01146
01147
01148
01149
01150
01151
01152
01153
01154
01155
01156
01157
01158
01159
01160
01161
01162
01163
01164
01165
01166 PT(HTTPChannel) HTTPClient::
01167 make_channel(bool persistent_connection) {
01168 PT(HTTPChannel) doc = new HTTPChannel(this);
01169 doc->set_persistent_connection(persistent_connection);
01170 return doc;
01171 }
01172
01173
01174
01175
01176
01177
01178
01179
01180
01181
01182 PT(HTTPChannel) HTTPClient::
01183 post_form(const URLSpec &url, const string &body) {
01184 PT(HTTPChannel) doc = new HTTPChannel(this);
01185 doc->post_form(url, body);
01186 return doc;
01187 }
01188
01189
01190
01191
01192
01193
01194
01195
01196
01197
01198 PT(HTTPChannel) HTTPClient::
01199 get_document(const URLSpec &url) {
01200 PT(HTTPChannel) doc = new HTTPChannel(this);
01201 doc->get_document(url);
01202 return doc;
01203 }
01204
01205
01206
01207
01208
01209
01210
01211
01212
01213
01214 PT(HTTPChannel) HTTPClient::
01215 get_header(const URLSpec &url) {
01216 PT(HTTPChannel) doc = new HTTPChannel(this);
01217 doc->get_header(url);
01218 return doc;
01219 }
01220
01221
01222
01223
01224
01225
01226 HTTPClient *HTTPClient::
01227 get_global_ptr() {
01228 if (_global_ptr == NULL) {
01229 _global_ptr = new HTTPClient;
01230 }
01231 return _global_ptr;
01232 }
01233
01234
01235
01236
01237
01238
01239
01240
01241 SSL_CTX *HTTPClient::
01242 get_ssl_ctx() {
01243 if (_ssl_ctx != (SSL_CTX *)NULL) {
01244 return _ssl_ctx;
01245 }
01246
01247 init_random_seed();
01248
01249 _ssl_ctx = SSL_CTX_new(SSLv23_client_method());
01250
01251 #if defined(SSL_097) && !defined(NDEBUG)
01252
01253
01254 if (downloader_cat.is_debug()) {
01255 SSL_CTX_set_msg_callback(_ssl_ctx, ssl_msg_callback);
01256 }
01257 #endif
01258
01259
01260 OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
01261 sslw->notify_ssl_errors();
01262
01263 SSL_CTX_set_cert_store(_ssl_ctx, sslw->get_x509_store());
01264
01265 return _ssl_ctx;
01266 }
01267
01268
01269
01270
01271
01272
01273
01274
01275
01276
01277
01278
01279
01280
01281
01282
01283
01284
01285
01286
01287
01288 void HTTPClient::
01289 check_preapproved_server_certificate(const URLSpec &url, X509 *cert,
01290 bool &cert_preapproved, bool &cert_name_preapproved) const {
01291 cert_preapproved = false;
01292 cert_name_preapproved = false;
01293
01294 string server_and_port = url.get_server_and_port();
01295 PreapprovedServerCerts::const_iterator psci =
01296 _preapproved_server_certs.find(server_and_port);
01297
01298 if (psci == _preapproved_server_certs.end()) {
01299
01300 return;
01301 }
01302
01303 const PreapprovedServerCert &psc = (*psci).second;
01304
01305
01306 ServerCerts::const_iterator sci;
01307 for (sci = psc._certs.begin(); sci != psc._certs.end(); ++sci) {
01308 if (X509_cmp(cert, *sci) == 0) {
01309 cert_preapproved = true;
01310 cert_name_preapproved = true;
01311 downloader_cat.info()
01312 << "Server certificate is pre-approved.\n";
01313 return;
01314 }
01315 }
01316
01317
01318 X509_NAME *subject = X509_get_subject_name(cert);
01319 ServerCertNames::const_iterator scni;
01320 for (scni = psc._cert_names.begin(); scni != psc._cert_names.end(); ++scni) {
01321 X509_NAME *cert_name = (*scni);
01322 if (x509_name_subset(cert_name, subject)) {
01323 downloader_cat.info()
01324 << "Server certificate name is pre-approved.\n";
01325 cert_name_preapproved = true;
01326 return;
01327 }
01328 }
01329
01330
01331 return;
01332 }
01333
01334
01335
01336
01337
01338
01339
01340
01341 bool HTTPClient::
01342 get_proxies_for_scheme(const string &scheme, pvector<URLSpec> &proxies) const {
01343 ProxiesByScheme::const_iterator si = _proxies_by_scheme.find(scheme);
01344 if (si == _proxies_by_scheme.end()) {
01345 return false;
01346 }
01347 const Proxies &scheme_proxies = (*si).second;
01348 if (scheme_proxies.empty()) {
01349 return false;
01350 }
01351
01352 Proxies::const_iterator pi;
01353 for (pi = scheme_proxies.begin(); pi != scheme_proxies.end(); ++pi) {
01354 proxies.push_back(*pi);
01355 }
01356
01357 return true;
01358 }
01359
01360
01361
01362
01363
01364
01365
01366
01367
01368 void HTTPClient::
01369 add_http_username(const string &http_username) {
01370 size_t c1 = http_username.find(':');
01371 if (c1 != string::npos) {
01372 size_t c2 = http_username.find(':', c1 + 1);
01373 if (c2 != string::npos) {
01374 size_t c3 = http_username.find(':', c2 + 1);
01375 if (c3 != string::npos) {
01376 size_t c4 = http_username.find(':', c3 + 1);
01377 if (c4 != string::npos) {
01378
01379 downloader_cat.error()
01380 << "Invalid http-username " << http_username << "\n";
01381
01382 } else {
01383
01384 set_username(http_username.substr(0, c1),
01385 http_username.substr(c1 + 1, c2 - (c1 + 1)),
01386 http_username.substr(c2 + 1));
01387 }
01388
01389 } else {
01390
01391 set_username(string(),
01392 http_username.substr(0, c1),
01393 http_username.substr(c1 + 1));
01394 }
01395 } else {
01396
01397 set_username(string(), string(), http_username);
01398 }
01399 } else {
01400
01401 downloader_cat.error()
01402 << "Invalid http-username " << http_username << "\n";
01403 }
01404 }
01405
01406
01407
01408
01409
01410
01411
01412 string HTTPClient::
01413 select_username(const URLSpec &url, bool is_proxy, const string &realm) const {
01414 string username;
01415
01416
01417
01418
01419
01420 if (url.has_username() && !is_proxy) {
01421 username = url.get_username();
01422 }
01423
01424
01425 if (is_proxy) {
01426 if (username.empty()) {
01427
01428 username = get_username("*proxy", realm);
01429 }
01430 if (username.empty()) {
01431
01432 username = get_username("*proxy", string());
01433 }
01434 }
01435 if (username.empty()) {
01436
01437 username = get_username(url.get_server(), realm);
01438 }
01439 if (username.empty()) {
01440
01441 username = get_username(url.get_server(), string());
01442 }
01443 if (username.empty()) {
01444
01445 username = get_username(string(), realm);
01446 }
01447 if (username.empty()) {
01448
01449 username = get_username(string(), string());
01450 }
01451
01452 return username;
01453 }
01454
01455
01456
01457
01458
01459
01460
01461
01462 HTTPAuthorization *HTTPClient::
01463 select_auth(const URLSpec &url, bool is_proxy, const string &last_realm) {
01464 Domains &domains = is_proxy ? _proxy_domains : _www_domains;
01465 string canon = HTTPAuthorization::get_canonical_url(url).get_url();
01466
01467
01468
01469 Domains::const_iterator best_di = domains.end();
01470 size_t longest_length = 0;
01471 Domains::const_iterator di;
01472 for (di = domains.begin(); di != domains.end(); ++di) {
01473 const string &domain = (*di).first;
01474 size_t length = domain.length();
01475 if (domain == canon.substr(0, length)) {
01476
01477 if (length > longest_length) {
01478 best_di = di;
01479 longest_length = length;
01480 }
01481 }
01482 }
01483
01484 if (best_di != domains.end()) {
01485
01486 if (downloader_cat.is_spam()) {
01487 downloader_cat.spam()
01488 << "Choosing domain " << (*best_di).first << " for " << url << "\n";
01489 }
01490 const Realms &realms = (*best_di).second._realms;
01491
01492 Realms::const_iterator ri;
01493 ri = realms.find(last_realm);
01494 if (ri != realms.end()) {
01495 return (*ri).second;
01496 }
01497
01498 if (!realms.empty()) {
01499
01500 return (*realms.begin()).second;
01501 }
01502 }
01503
01504
01505 return NULL;
01506 }
01507
01508
01509
01510
01511
01512
01513
01514
01515
01516
01517
01518 PT(HTTPAuthorization) HTTPClient::
01519 generate_auth(const URLSpec &url, bool is_proxy, const string &challenge) {
01520 HTTPAuthorization::AuthenticationSchemes schemes;
01521 HTTPAuthorization::parse_authentication_schemes(schemes, challenge);
01522
01523 PT(HTTPAuthorization) auth;
01524 HTTPAuthorization::AuthenticationSchemes::iterator si;
01525
01526 si = schemes.find("digest");
01527 if (si != schemes.end()) {
01528 auth = new HTTPDigestAuthorization((*si).second, url, is_proxy);
01529 }
01530
01531 if (auth == (HTTPAuthorization *)NULL || !auth->is_valid()) {
01532 si = schemes.find("basic");
01533 if (si != schemes.end()) {
01534 auth = new HTTPBasicAuthorization((*si).second, url, is_proxy);
01535 }
01536 }
01537
01538 if (auth == (HTTPAuthorization *)NULL || !auth->is_valid()) {
01539 downloader_cat.warning()
01540 << "Don't know how to use any of the server's available authorization schemes:\n";
01541 for (si = schemes.begin(); si != schemes.end(); ++si) {
01542 downloader_cat.warning() << (*si).first << "\n";
01543 }
01544
01545 } else {
01546
01547
01548 Domains &domains = is_proxy ? _proxy_domains : _www_domains;
01549 const vector_string &domain = auth->get_domain();
01550 vector_string::const_iterator si;
01551 for (si = domain.begin(); si != domain.end(); ++si) {
01552 domains[(*si)]._realms[auth->get_realm()] = auth;
01553 }
01554 }
01555
01556 return auth;
01557 }
01558
01559
01560
01561
01562
01563
01564
01565
01566 void HTTPClient::
01567 unload_client_certificate() {
01568 if (_client_certificate_priv != (EVP_PKEY *)NULL) {
01569 EVP_PKEY_free(_client_certificate_priv);
01570 _client_certificate_priv = NULL;
01571 }
01572
01573 if (_client_certificate_pub != (X509 *)NULL) {
01574 X509_free(_client_certificate_pub);
01575 _client_certificate_pub = NULL;
01576 }
01577
01578 _client_certificate_loaded = false;
01579 }
01580
01581
01582
01583
01584
01585
01586
01587
01588
01589 X509_NAME *HTTPClient::
01590 parse_x509_name(const string &source) {
01591 X509_NAME *result = NULL;
01592
01593 result = X509_NAME_new();
01594 bool added_any = false;
01595
01596 string::const_iterator si;
01597 si = source.begin();
01598 while (si != source.end()) {
01599 if ((*si) == '/') {
01600
01601 ++si;
01602 } else {
01603 string type;
01604 while (si != source.end() && (*si) != '=' && (*si) != '/') {
01605 if ((*si) == '\\') {
01606 ++si;
01607 if (si != source.end()) {
01608 type += (*si);
01609 ++si;
01610 }
01611 } else {
01612 type += (*si);
01613 ++si;
01614 }
01615 }
01616
01617 int nid = OBJ_txt2nid((char *)type.c_str());
01618 if (nid == NID_undef) {
01619 downloader_cat.info()
01620 << "Unknown type " << type << " in X509 name: " << source
01621 << "\n";
01622 X509_NAME_free(result);
01623 return NULL;
01624 }
01625
01626 string value;
01627
01628 if (si != source.end() && (*si) == '=') {
01629 ++si;
01630 while (si != source.end() && (*si) != '/') {
01631 if ((*si) == '\\') {
01632 ++si;
01633 if (si != source.end()) {
01634 value += (*si);
01635 ++si;
01636 }
01637 } else {
01638 value += (*si);
01639 ++si;
01640 }
01641 }
01642 }
01643
01644 if (!value.empty()) {
01645 int add_result =
01646 X509_NAME_add_entry_by_NID(result, nid, V_ASN1_APP_CHOOSE,
01647 (unsigned char *)value.c_str(), -1, -1, 0);
01648 if (!add_result) {
01649 downloader_cat.info()
01650 << "Unable to add " << type << "=" << value << " in X509 name: "
01651 << source << "\n";
01652 X509_NAME_free(result);
01653 return NULL;
01654 }
01655 added_any = true;
01656 }
01657 }
01658 }
01659
01660 if (!added_any) {
01661 downloader_cat.info()
01662 << "Invalid empty X509 name: " << source << "\n";
01663 X509_NAME_free(result);
01664 return NULL;
01665 }
01666
01667 return result;
01668 }
01669
01670
01671
01672
01673
01674
01675
01676
01677 bool HTTPClient::
01678 x509_name_subset(X509_NAME *name_a, X509_NAME *name_b) {
01679 int count_a = X509_NAME_entry_count(name_a);
01680 for (int ai = 0; ai < count_a; ai++) {
01681 X509_NAME_ENTRY *na = X509_NAME_get_entry(name_a, ai);
01682
01683 int bi = X509_NAME_get_index_by_OBJ(name_b, na->object, -1);
01684 if (bi < 0) {
01685
01686 return false;
01687 }
01688
01689 X509_NAME_ENTRY *nb = X509_NAME_get_entry(name_b, bi);
01690 if (na->value->length != nb->value->length ||
01691 memcmp(na->value->data, nb->value->data, na->value->length) != 0) {
01692
01693 return false;
01694 }
01695 }
01696 return true;
01697 }
01698
01699
01700
01701
01702
01703
01704
01705 void HTTPClient::
01706 split_whitespace(string &a, string &b, const string &c) {
01707 size_t p = 0;
01708
01709
01710 while (p < c.length() && isspace(c[p])) {
01711 ++p;
01712 }
01713
01714
01715 size_t q = p;
01716 while (p < c.length() && !isspace(c[p])) {
01717 ++p;
01718 }
01719
01720 a = c.substr(q, p - q);
01721
01722
01723 while (p < c.length() && isspace(c[p])) {
01724 ++p;
01725 }
01726 b = c.substr(p);
01727 }
01728
01729 #if defined(SSL_097) && !defined(NDEBUG)
01730
01731
01732
01733
01734
01735
01736 void HTTPClient::
01737 ssl_msg_callback(int write_p, int version, int content_type,
01738 const void *, size_t len, SSL *, void *) {
01739 ostringstream describe;
01740 if (write_p) {
01741 describe << "sent ";
01742 } else {
01743 describe << "received ";
01744 }
01745 switch (version) {
01746 case SSL2_VERSION:
01747 describe << "SSL 2.0 ";
01748 break;
01749
01750 case SSL3_VERSION:
01751 describe << "SSL 3.0 ";
01752 break;
01753
01754 case TLS1_VERSION:
01755 describe << "TLS 1.0 ";
01756 break;
01757
01758 default:
01759 describe << "unknown protocol ";
01760 }
01761
01762 describe << "message: ";
01763
01764 if (version != SSL2_VERSION) {
01765 switch (content_type) {
01766 case 20:
01767 describe << "change cipher spec, ";
01768 break;
01769
01770 case 21:
01771 describe << "alert, ";
01772 break;
01773
01774 case 22:
01775 describe << "handshake, ";
01776 break;
01777
01778 case 23:
01779 describe << "application data, ";
01780 break;
01781
01782 default:
01783 describe << "unknown content type, ";
01784 }
01785 }
01786
01787 describe << len << " bytes.\n";
01788
01789 downloader_cat.debug() << describe.str();
01790 }
01791 #endif // defined(SSL_097) && !defined(NDEBUG)
01792
01793
01794
01795
01796
01797
01798 HTTPClient::PreapprovedServerCert::
01799 ~PreapprovedServerCert() {
01800 ServerCerts::const_iterator sci;
01801 for (sci = _certs.begin(); sci != _certs.end(); ++sci) {
01802 X509_free(*sci);
01803 }
01804
01805 ServerCertNames::const_iterator scni;
01806 for (scni = _cert_names.begin(); scni != _cert_names.end(); ++scni) {
01807 X509_NAME_free(*scni);
01808 }
01809 }
01810
01811 #endif // HAVE_OPENSSL