Panda3D
 All Classes Functions Variables Enumerations
httpClient.cxx
1 // Filename: httpClient.cxx
2 // Created by: drose (24Sep02)
3 //
4 ////////////////////////////////////////////////////////////////////
5 //
6 // PANDA 3D SOFTWARE
7 // Copyright (c) Carnegie Mellon University. All rights reserved.
8 //
9 // All use of this software is subject to the terms of the revised BSD
10 // license. You should have received a copy of this license along
11 // with this source code in a file named "LICENSE."
12 //
13 ////////////////////////////////////////////////////////////////////
14 
15 #include "httpClient.h"
16 #include "httpChannel.h"
17 #include "config_downloader.h"
18 #include "filename.h"
19 #include "config_express.h"
20 #include "virtualFileSystem.h"
21 #include "executionEnvironment.h"
22 #include "httpBasicAuthorization.h"
23 #include "httpDigestAuthorization.h"
24 #include "globPattern.h"
25 
26 #ifdef HAVE_OPENSSL
27 
28 PT(HTTPClient) HTTPClient::_global_ptr;
29 
30 ////////////////////////////////////////////////////////////////////
31 // Function: trim_blanks
32 // Description:
33 ////////////////////////////////////////////////////////////////////
34 static string
35 trim_blanks(const string &str) {
36  size_t start = 0;
37  while (start < str.length() && isspace(str[start])) {
38  start++;
39  }
40 
41  size_t end = str.length();
42  while (end > start && isspace(str[end - 1])) {
43  end--;
44  }
45 
46  return str.substr(start, end - start);
47 }
48 
49 ////////////////////////////////////////////////////////////////////
50 // Function: tokenize
51 // Description: Chops the source string up into pieces delimited by
52 // any of the characters specified in delimiters.
53 // Repeated delimiter characters represent zero-length
54 // tokens.
55 //
56 // It is the user's responsibility to ensure the output
57 // vector is cleared before calling this function; the
58 // results will simply be appended to the end of the
59 // vector.
60 ////////////////////////////////////////////////////////////////////
61 static void
62 tokenize(const string &str, vector_string &words, const string &delimiters) {
63  size_t p = 0;
64  while (p < str.length()) {
65  size_t q = str.find_first_of(delimiters, p);
66  if (q == string::npos) {
67  words.push_back(str.substr(p));
68  return;
69  }
70  words.push_back(str.substr(p, q - p));
71  p = q + 1;
72  }
73  words.push_back(string());
74 }
75 
76 ////////////////////////////////////////////////////////////////////
77 // Function: HTTPClient::Constructor
78 // Access: Published
79 // Description:
80 ////////////////////////////////////////////////////////////////////
81 HTTPClient::
82 HTTPClient() {
83  ConfigVariableBool verify_ssl
84  ("verify-ssl", true,
85  PRC_DESC("Configure this true (the default) to insist on verifying all SSL "
86  "(e.g. https) servers against a known certificate, or false to allow "
87  "an unverified connection. This controls the default behavior; the "
88  "specific behavior for a particular HTTPClient can be adjusted at "
89  "runtime with set_verify_ssl()."));
90 
91  ConfigVariableString ssl_cipher_list
92  ("ssl-cipher-list", "DEFAULT",
93  PRC_DESC("This is the default value for HTTPClient::set_cipher_list()."));
94 
95  ConfigVariableString http_proxy
96  ("http-proxy", "",
97  PRC_DESC("This specifies the default value for HTTPClient::set_proxy_spec(). "
98  "It is a semicolon-delimited list of proxies that we use to contact "
99  "all HTTP hosts that don't specify otherwise. See "
100  "set_proxy_spec() for more information."));
101 
102  ConfigVariableString http_direct_hosts
103  ("http-direct-hosts", "",
104  PRC_DESC("This specifies the default value for HTTPClient::set_direct_host_spec(). "
105  "It is a semicolon-delimited list of host names that do not require a "
106  "proxy. See set_direct_host_spec() for more information."));
107 
108  ConfigVariableBool http_try_all_direct
109  ("http-try-all-direct", true,
110  PRC_DESC("This specifies the default value for HTTPClient::set_try_all_direct(). "
111  "If this is true, a direct connection will always be attempted after an "
112  "attempt to connect through a proxy fails."));
113 
114  ConfigVariableString http_proxy_username
115  ("http-proxy-username", "",
116  PRC_DESC("This specifies a default username:password to pass to the proxy."));
117 
118  ConfigVariableList http_username
119  ("http-username",
120  PRC_DESC("Adds one or more username/password pairs to all HTTP clients. The client "
121  "will present this username/password when asked to authenticate a request "
122  "for a particular server and/or realm. The username is of the form "
123  "server:realm:username:password, where either or both of server and "
124  "realm may be empty, or just realm:username:password or username:password. "
125  "If the server or realm is empty, they will match anything."));
126 
127  ConfigVariableFilename http_client_certificate_filename
128  ("http-client-certificate-filename", "",
129  PRC_DESC("This provides a default client certificate to offer up should an "
130  "SSL server demand one. The file names a PEM-formatted file "
131  "that includes a public and private key specification. A "
132  "connection-specific certificate may also be specified at runtime on "
133  "the HTTPClient object, but this will require having a different "
134  "HTTPClient object for each differently-certificated connection."));
135 
136  ConfigVariableString http_client_certificate_passphrase
137  ("http-client-certificate-passphrase", "",
138  PRC_DESC("This specifies the passphrase to use to decode the certificate named "
139  "by http-client-certificate-filename."));
140 
141 
142  ConfigVariableList http_preapproved_server_certificate_filename
143  ("http-preapproved-server-certificate-filename",
144  PRC_DESC("This specifies a server hostname:port combination, followed by "
145  "at least one space, and the name of a PEM file that contains "
146  "an SSL certificate that we expect this server to offer. If "
147  "it does, we will accept the cert without further question, "
148  "even if the cert does not match any known certificate "
149  "authorities. This option may appear multiple times."));
150 
151  ConfigVariableList http_preapproved_server_certificate_name
152  ("http-preapproved-server-certificate-name",
153  PRC_DESC("This specifies a server hostname:port combination, followed by "
154  "at least one space, and a string describing the subject name "
155  "of an SSL certificate we expect this server to offer. If it "
156  "it does, we will accept the cert, but only if it also matches "
157  "a known certificate authority. This option may appear "
158  "multiple times."));
159 
160  _http_version = HTTPEnum::HV_11;
161  _verify_ssl = verify_ssl ? VS_normal : VS_no_verify;
162  _ssl_ctx = (SSL_CTX *)NULL;
163 
164  set_proxy_spec(http_proxy);
165  set_direct_host_spec(http_direct_hosts);
166  _try_all_direct = http_try_all_direct;
167 
168  if (!http_proxy_username.empty()) {
169  set_username("*proxy", "", http_proxy_username);
170  }
171 
172  set_cipher_list(ssl_cipher_list);
173 
174  {
175  // Also load in the general usernames.
176  int num_unique_values = http_username.get_num_unique_values();
177  for (int i = 0; i < num_unique_values; i++) {
178  string username = http_username.get_unique_value(i);
179  add_http_username(username);
180  }
181  }
182 
183  _client_certificate_filename = http_client_certificate_filename;
184  _client_certificate_passphrase = http_client_certificate_passphrase;
185 
186  _client_certificate_loaded = false;
187  _client_certificate_pub = NULL;
188  _client_certificate_priv = NULL;
189 
190  int num_server_certs = http_preapproved_server_certificate_filename.get_num_unique_values();
191  int si;
192  for (si = 0; si < num_server_certs; si++) {
193  string cert_line = http_preapproved_server_certificate_filename.get_unique_value(si);
194  string a, b;
195  split_whitespace(a, b, cert_line);
196  add_preapproved_server_certificate_filename(URLSpec(a, true), b);
197  }
198 
199  num_server_certs = http_preapproved_server_certificate_name.get_num_unique_values();
200  for (si = 0; si < num_server_certs; si++) {
201  string cert_line = http_preapproved_server_certificate_name.get_unique_value(si);
202  string a, b;
203  split_whitespace(a, b, cert_line);
204  add_preapproved_server_certificate_name(URLSpec(a, true), b);
205  }
206 
207  // The first time we create an HTTPClient, we must initialize the
208  // OpenSSL library. The OpenSSLWrapper object does that.
209  OpenSSLWrapper::get_global_ptr();
210 }
211 
212 ////////////////////////////////////////////////////////////////////
213 // Function: HTTPClient::Copy Constructor
214 // Access: Published
215 // Description:
216 ////////////////////////////////////////////////////////////////////
217 HTTPClient::
218 HTTPClient(const HTTPClient &copy) {
219  _ssl_ctx = (SSL_CTX *)NULL;
220 
221  (*this) = copy;
222 }
223 
224 ////////////////////////////////////////////////////////////////////
225 // Function: HTTPClient::Copy Assignment Operator
226 // Access: Published
227 // Description:
228 ////////////////////////////////////////////////////////////////////
229 void HTTPClient::
230 operator = (const HTTPClient &copy) {
231  _proxies_by_scheme = copy._proxies_by_scheme;
232  _direct_hosts = copy._direct_hosts;
233  _try_all_direct = copy._try_all_direct;
234  _http_version = copy._http_version;
235  _verify_ssl = copy._verify_ssl;
236  _usernames = copy._usernames;
237  _cookies = copy._cookies;
238 }
239 
240 ////////////////////////////////////////////////////////////////////
241 // Function: HTTPClient::Destructor
242 // Access: Published
243 // Description:
244 ////////////////////////////////////////////////////////////////////
245 HTTPClient::
246 ~HTTPClient() {
247  // Before we can free the context, we must remove the X509_STORE
248  // pointer from it, so it won't be destroyed along with it (this
249  // object is shared among all contexts).
250  if (_ssl_ctx != (SSL_CTX *)NULL) {
251  _ssl_ctx->cert_store = NULL;
252  SSL_CTX_free(_ssl_ctx);
253  }
254 
255  unload_client_certificate();
256 }
257 
258 ////////////////////////////////////////////////////////////////////
259 // Function: HTTPClient::init_random_seed
260 // Access: Published, Static
261 // Description: This may be called once, presumably at the beginning
262 // of an application, to initialize OpenSSL's random
263 // seed. On Windows, it is particularly important to
264 // call this at startup if you are going to be performing
265 // any https operations or otherwise use encryption,
266 // since the Windows algorithm for getting a random seed
267 // takes 2-3 seconds at startup, but can take 30 seconds
268 // or more after you have opened a 3-D graphics window
269 // and started rendering.
270 //
271 // There is no harm in calling this method multiple
272 // times, or in not calling it at all.
273 ////////////////////////////////////////////////////////////////////
274 void HTTPClient::
275 init_random_seed() {
276  // Creating the global OpenSSLWrapper object is nowadays sufficient
277  // to ensure that OpenSSL and its random seed have been initialized.
278  OpenSSLWrapper::get_global_ptr();
279 }
280 
281 ////////////////////////////////////////////////////////////////////
282 // Function: HTTPClient::set_proxy_spec
283 // Access: Published
284 // Description: Specifies the complete set of proxies to use for all
285 // schemes. This is either a semicolon-delimited set of
286 // hostname:ports, or a semicolon-delimited set of pairs
287 // of the form "scheme=hostname:port", or a combination.
288 // Use the keyword DIRECT, or an empty string, to
289 // represent a direct connection. A particular scheme
290 // and/or proxy host may be listed more than once. This
291 // is a convenience function that can be used in place
292 // of explicit calls to add_proxy() for each
293 // scheme/proxy pair.
294 ////////////////////////////////////////////////////////////////////
295 void HTTPClient::
296 set_proxy_spec(const string &proxy_spec) {
297  clear_proxy();
298 
299  string trim_proxy_spec = trim_blanks(proxy_spec);
300 
301  // Tokenize the string based on the semicolons.
302  if (!trim_proxy_spec.empty()) {
303  vector_string proxies;
304  tokenize(trim_proxy_spec, proxies, ";");
305 
306  for (vector_string::const_iterator pi = proxies.begin();
307  pi != proxies.end();
308  ++pi) {
309  const string &spec = (*pi);
310 
311  // Divide out the scheme and the hostname.
312  string scheme;
313  string proxy;
314  size_t equals = spec.find('=');
315  if (equals == string::npos) {
316  scheme = "";
317  proxy = trim_blanks(spec);
318  } else {
319  scheme = trim_blanks(spec.substr(0, equals));
320  proxy = trim_blanks(spec.substr(equals + 1));
321  }
322 
323  if (proxy == "DIRECT" || proxy.empty()) {
324  add_proxy(scheme, URLSpec());
325  } else {
326  add_proxy(scheme, URLSpec(proxy, true));
327  }
328  }
329  }
330 }
331 
332 ////////////////////////////////////////////////////////////////////
333 // Function: HTTPClient::get_proxy_spec
334 // Access: Published
335 // Description: Returns the complete set of proxies to use for all
336 // schemes. This is a string of the form specified by
337 // set_proxy_spec(), above. Note that the string
338 // returned by this function may not be exactly the same
339 // as the string passed into set_proxy_spec(), since the
340 // string is regenerated from the internal storage
341 // structures and may therefore be reordered.
342 ////////////////////////////////////////////////////////////////////
343 string HTTPClient::
344 get_proxy_spec() const {
345  string result;
346 
347  ProxiesByScheme::const_iterator si;
348  for (si = _proxies_by_scheme.begin(); si != _proxies_by_scheme.end(); ++si) {
349  const string &scheme = (*si).first;
350  const Proxies &proxies = (*si).second;
351  Proxies::const_iterator pi;
352  for (pi = proxies.begin(); pi != proxies.end(); ++pi) {
353  const URLSpec &url = (*pi);
354  if (!result.empty()) {
355  result += ";";
356  }
357  if (!scheme.empty()) {
358  result += scheme;
359  result += "=";
360  }
361  if (url.empty()) {
362  result += "DIRECT";
363  } else {
364  result += url.get_url();
365  }
366  }
367  }
368 
369  return result;
370 }
371 
372 ////////////////////////////////////////////////////////////////////
373 // Function: HTTPClient::set_direct_host_spec
374 // Access: Published
375 // Description: Specifies the set of hosts that should be connected
376 // to directly, without using a proxy. This is a
377 // semicolon-separated list of hostnames that may
378 // contain wildcard characters ("*").
379 ////////////////////////////////////////////////////////////////////
380 void HTTPClient::
381 set_direct_host_spec(const string &direct_host_spec) {
382  clear_direct_host();
383 
384  // Tokenize the string based on the semicolons.
385  vector_string hosts;
386  tokenize(direct_host_spec, hosts, ";");
387 
388  for (vector_string::const_iterator hi = hosts.begin();
389  hi != hosts.end();
390  ++hi) {
391  string spec = trim_blanks(*hi);
392 
393  // We should be careful to avoid adding any empty hostnames to the
394  // list. In particular, we will get one empty hostname if the
395  // direct_host_spec is empty.
396  if (!spec.empty()) {
397  add_direct_host(spec);
398  }
399  }
400 }
401 
402 ////////////////////////////////////////////////////////////////////
403 // Function: HTTPClient::get_direct_host_spec
404 // Access: Published
405 // Description: Returns the set of hosts that should be connected
406 // to directly, without using a proxy, as a
407 // semicolon-separated list of hostnames that may
408 // contain wildcard characters ("*").
409 ////////////////////////////////////////////////////////////////////
410 string HTTPClient::
411 get_direct_host_spec() const {
412  string result;
413 
414  DirectHosts::const_iterator si;
415  for (si = _direct_hosts.begin(); si != _direct_hosts.end(); ++si) {
416  const GlobPattern &host = (*si);
417 
418  if (!result.empty()) {
419  result += ";";
420  }
421  result += host.get_pattern();
422  }
423 
424  return result;
425 }
426 
427 ////////////////////////////////////////////////////////////////////
428 // Function: HTTPClient::clear_proxy
429 // Access: Published
430 // Description: Resets the proxy spec to empty. Subsequent calls to
431 // add_proxy() may be made to build up the set of proxy
432 // servers.
433 ////////////////////////////////////////////////////////////////////
434 void HTTPClient::
435 clear_proxy() {
436  _proxies_by_scheme.clear();
437 }
438 
439 ////////////////////////////////////////////////////////////////////
440 // Function: HTTPClient::add_proxy
441 // Access: Published
442 // Description: Adds the indicated proxy host as a proxy for
443 // communications on the given scheme. Usually the
444 // scheme is "http" or "https". It may be the empty
445 // string to indicate a general proxy. The proxy string
446 // may be the empty URL to indicate a direct connection.
447 ////////////////////////////////////////////////////////////////////
448 void HTTPClient::
449 add_proxy(const string &scheme, const URLSpec &proxy) {
450  URLSpec proxy_url(proxy);
451 
452  // The scheme is always converted to lowercase.
453  string lc_scheme;
454  lc_scheme.reserve(scheme.length());
455  string::const_iterator si;
456  for (si = scheme.begin(); si != scheme.end(); ++si) {
457  lc_scheme += tolower(*si);
458  }
459 
460  // Remove the trailing colon, if there is one.
461  if (!lc_scheme.empty() && lc_scheme[lc_scheme.length() - 1] == ':') {
462  lc_scheme = lc_scheme.substr(0, lc_scheme.length() - 1);
463  }
464 
465  if (!proxy_url.empty()) {
466  // Enforce the scheme that we use to communicate to the proxy
467  // itself. This is not the same as lc_scheme, which is the scheme
468  // of the requested connection. Generally, all proxies speak
469  // HTTP, except for Socks proxies.
470 
471  if (lc_scheme == "socks") {
472  // Scheme "socks" implies we talk to the proxy via the "socks"
473  // scheme, no matter what scheme the user actually specified.
474  proxy_url.set_scheme("socks");
475 
476  } else if (!proxy_url.has_scheme()) {
477  // Otherwise, if the user didn't specify a scheme to talk to the
478  // proxy, the default is "http".
479  proxy_url.set_scheme("http");
480  }
481  }
482 
483  _proxies_by_scheme[lc_scheme].push_back(proxy_url);
484 }
485 
486 ////////////////////////////////////////////////////////////////////
487 // Function: HTTPClient::clear_direct_host
488 // Access: Published
489 // Description: Resets the set of direct hosts to empty. Subsequent
490 // calls to add_direct_host() may be made to build up
491 // the list of hosts that do not require a proxy
492 // connection.
493 ////////////////////////////////////////////////////////////////////
494 void HTTPClient::
495 clear_direct_host() {
496  _direct_hosts.clear();
497 }
498 
499 ////////////////////////////////////////////////////////////////////
500 // Function: HTTPClient::add_direct_host
501 // Access: Published
502 // Description: Adds the indicated name to the set of hostnames that
503 // are connected to directly, without using a proxy.
504 // This name may be either a DNS name or an IP address,
505 // and it may include the * as a wildcard character.
506 ////////////////////////////////////////////////////////////////////
507 void HTTPClient::
508 add_direct_host(const string &hostname) {
509  // The hostname is always converted to lowercase.
510  string lc_hostname;
511  lc_hostname.reserve(hostname.length());
512  for (string::const_iterator si = hostname.begin();
513  si != hostname.end();
514  ++si) {
515  lc_hostname += tolower(*si);
516  }
517 
518  _direct_hosts.push_back(GlobPattern(lc_hostname));
519 }
520 
521 ////////////////////////////////////////////////////////////////////
522 // Function: HTTPClient::get_proxies_for_url
523 // Access: Published
524 // Description: Fills up the indicated vector with the list of
525 // URLSpec objects, in the order in which they should be
526 // tried, that are appropriate proxies to try for the
527 // indicated URL. The empty URL is returned for a
528 // direct connection.
529 //
530 // It is the user's responsibility to empty this vector
531 // before calling this method; otherwise, the proxy
532 // URL's will simply be appended to the existing list.
533 ////////////////////////////////////////////////////////////////////
534 void HTTPClient::
535 get_proxies_for_url(const URLSpec &url, pvector<URLSpec> &proxies) const {
536  // First, check if the hostname matches any listed in direct_hosts.
537  string hostname = url.get_server();
538 
539  // If the hostname is empty, treat it as a special case: we don't
540  // match any of the hostnames listed in direct_hosts (even "*").
541  if (!hostname.empty()) {
542  DirectHosts::const_iterator si;
543  for (si = _direct_hosts.begin(); si != _direct_hosts.end(); ++si) {
544  if ((*si).matches(hostname)) {
545  // It matches, so don't use any proxies.
546  proxies.push_back(URLSpec());
547  return;
548  }
549  }
550  }
551 
552  // Build our list of proxies into a temporary vector, so we can pull
553  // out duplicates later.
554  pvector<URLSpec> temp_list;
555 
556  // Now choose the appropriate proxy based on the scheme.
557  string scheme = url.get_scheme();
558  bool got_any = false;
559 
560  if (!scheme.empty()) {
561  // If we have a scheme, try to match it.
562  if (get_proxies_for_scheme(scheme, temp_list)) {
563  got_any = true;
564  }
565  }
566 
567  if (!got_any && (scheme.empty() || url.is_ssl())) {
568  // An empty scheme (or an ssl-style scheme) implies we will need
569  // to make a direct connection, so fallback to a socks-style
570  // and/or https-style scheme.
571 
572  if (get_proxies_for_scheme("socks", temp_list)) {
573  got_any = true;
574  }
575  if (get_proxies_for_scheme("https", temp_list)) {
576  got_any = true;
577  }
578  }
579 
580  if (!got_any) {
581  // If we didn't find our scheme of choice, fall back to the default
582  // proxy type, if we've got one.
583  if (get_proxies_for_scheme("", temp_list)) {
584  got_any = true;
585  }
586  }
587 
588  if (_try_all_direct) {
589  // We may try a direct connection if all else fails.
590  temp_list.push_back(URLSpec());
591  }
592 
593  // Finally, as a very last resort, fall back to the HTTP proxy.
594  if (!got_any) {
595  get_proxies_for_scheme("http", temp_list);
596  }
597 
598  // Now remove duplicate entries as we copy the resulting list out.
600  pset<URLSpec> used;
601  for (pi = temp_list.begin(); pi != temp_list.end(); ++pi) {
602  if (used.insert(*pi).second) {
603  // This is a unique one.
604  proxies.push_back(*pi);
605  }
606  }
607 }
608 
609 ////////////////////////////////////////////////////////////////////
610 // Function: HTTPClient::get_proxies_for_url
611 // Access: Published
612 // Description: Returns a semicolon-delimited list of proxies, in the
613 // order in which they should be tried, that are
614 // appropriate for the indicated URL. The keyword
615 // DIRECT indicates a direct connection should be tried.
616 ////////////////////////////////////////////////////////////////////
617 string HTTPClient::
618 get_proxies_for_url(const URLSpec &url) const {
619  pvector<URLSpec> proxies;
620  get_proxies_for_url(url, proxies);
621 
622  string result;
623  if (!proxies.empty()) {
624  pvector<URLSpec>::const_iterator pi = proxies.begin();
625  if ((*pi).get_url().empty()) {
626  result += "DIRECT";
627  } else {
628  result += (*pi).get_url();
629  }
630  ++pi;
631 
632  while (pi != proxies.end()) {
633  result += ";";
634  if ((*pi).get_url().empty()) {
635  result += "DIRECT";
636  } else {
637  result += (*pi).get_url();
638  }
639  ++pi;
640  }
641  }
642 
643  return result;
644 }
645 
646 ////////////////////////////////////////////////////////////////////
647 // Function: HTTPClient::set_username
648 // Access: Published
649 // Description: Specifies the username:password string corresponding
650 // to a particular server and/or realm, when demanded by
651 // the server. Either or both of the server or realm
652 // may be empty; if so, they match anything. Also, the
653 // server may be set to the special string "*proxy",
654 // which will match any proxy server.
655 //
656 // If the username is set to the empty string, this
657 // clears the password for the particular server/realm
658 // pair.
659 ////////////////////////////////////////////////////////////////////
660 void HTTPClient::
661 set_username(const string &server, const string &realm, const string &username) {
662  string key = server + ":" + realm;
663  if (username.empty()) {
664  _usernames.erase(key);
665  } else {
666  _usernames[key] = username;
667  }
668 }
669 
670 ////////////////////////////////////////////////////////////////////
671 // Function: HTTPClient::get_username
672 // Access: Published
673 // Description: Returns the username:password string set for this
674 // server/realm pair, or empty string if nothing has
675 // been set. See set_username().
676 ////////////////////////////////////////////////////////////////////
677 string HTTPClient::
678 get_username(const string &server, const string &realm) const {
679  string key = server + ":" + realm;
680  Usernames::const_iterator ui;
681  ui = _usernames.find(key);
682  if (ui != _usernames.end()) {
683  return (*ui).second;
684  }
685  return string();
686 }
687 
688 ////////////////////////////////////////////////////////////////////
689 // Function: HTTPClient::set_cookie
690 // Access: Published
691 // Description: Stores the indicated cookie in the client's list of
692 // cookies, as if it had been received from a server.
693 ////////////////////////////////////////////////////////////////////
694 void HTTPClient::
695 set_cookie(const HTTPCookie &cookie) {
696  if (cookie.is_expired()) {
697  clear_cookie(cookie);
698 
699  } else {
700  pair<Cookies::iterator, bool> result = _cookies.insert(cookie);
701  if (!result.second) {
702  // We already had a cookie matching the supplied domain/path/name,
703  // so replace it.
704  const HTTPCookie &orig_cookie = *result.first;
705  ((HTTPCookie &)orig_cookie).update_from(cookie);
706  }
707  }
708 }
709 
710 ////////////////////////////////////////////////////////////////////
711 // Function: HTTPClient::clear_cookie
712 // Access: Published
713 // Description: Removes the cookie with the matching domain/path/name
714 // from the client's list of cookies. Returns true if
715 // it was removed, false if the cookie was not matched.
716 ////////////////////////////////////////////////////////////////////
717 bool HTTPClient::
718 clear_cookie(const HTTPCookie &cookie) {
719  Cookies::iterator ci = _cookies.find(cookie);
720  if (ci != _cookies.end()) {
721  _cookies.erase(ci);
722  return true;
723  }
724 
725  return false;
726 }
727 
728 ////////////////////////////////////////////////////////////////////
729 // Function: HTTPClient::clear_all_cookies
730 // Access: Published
731 // Description: Removes the all stored cookies from the client.
732 ////////////////////////////////////////////////////////////////////
733 void HTTPClient::
734 clear_all_cookies() {
735  _cookies.clear();
736 }
737 
738 ////////////////////////////////////////////////////////////////////
739 // Function: HTTPClient::has_cookie
740 // Access: Published
741 // Description: Returns true if there is a cookie in the client
742 // matching the given cookie's domain/path/name, false
743 // otherwise.
744 ////////////////////////////////////////////////////////////////////
745 bool HTTPClient::
746 has_cookie(const HTTPCookie &cookie) const {
747  Cookies::const_iterator ci = _cookies.find(cookie);
748  return (ci != _cookies.end());
749 }
750 
751 ////////////////////////////////////////////////////////////////////
752 // Function: HTTPClient::get_cookie
753 // Access: Published
754 // Description: Looks up and returns the cookie in the client
755 // matching the given cookie's domain/path/name. If
756 // there is no matching cookie, returns an empty cookie.
757 ////////////////////////////////////////////////////////////////////
758 HTTPCookie HTTPClient::
759 get_cookie(const HTTPCookie &cookie) const {
760  Cookies::const_iterator ci = _cookies.find(cookie);
761  if (ci != _cookies.end()) {
762  return (*ci);
763  }
764 
765  return HTTPCookie();
766 }
767 
768 ////////////////////////////////////////////////////////////////////
769 // Function: HTTPClient::copy_cookies_from
770 // Access: Published
771 // Description: Copies all the cookies from the indicated HTTPClient
772 // into this one. Existing cookies in this client are
773 // not affected, unless they are shadowed by the new
774 // cookies.
775 ////////////////////////////////////////////////////////////////////
776 void HTTPClient::
777 copy_cookies_from(const HTTPClient &other) {
778  Cookies::const_iterator ci;
779  for (ci = other._cookies.begin(); ci != other._cookies.end(); ++ci) {
780  set_cookie(*ci);
781  }
782 }
783 
784 ////////////////////////////////////////////////////////////////////
785 // Function: HTTPClient::write_cookies
786 // Access: Published
787 // Description: Outputs the complete list of cookies stored on the
788 // client, for all domains, including the expired
789 // cookies (which will normally not be sent back to a
790 // host).
791 ////////////////////////////////////////////////////////////////////
792 void HTTPClient::
793 write_cookies(ostream &out) const {
794  Cookies::const_iterator ci;
795  for (ci = _cookies.begin(); ci != _cookies.end(); ++ci) {
796  out << *ci << "\n";
797  }
798 }
799 
800 ////////////////////////////////////////////////////////////////////
801 // Function: HTTPClient::send_cookies
802 // Access: Published
803 // Description: Writes to the indicated ostream a "Cookie" header
804 // line for sending the cookies appropriate to the
805 // indicated URL along with an HTTP request. This also
806 // removes expired cookies.
807 ////////////////////////////////////////////////////////////////////
808 void HTTPClient::
809 send_cookies(ostream &out, const URLSpec &url) {
810  HTTPDate now = HTTPDate::now();
811  bool any_expired = false;
812  bool first_cookie = true;
813 
814  Cookies::const_iterator ci;
815  for (ci = _cookies.begin(); ci != _cookies.end(); ++ci) {
816  const HTTPCookie &cookie = (*ci);
817  if (cookie.is_expired(now)) {
818  any_expired = true;
819 
820  } else if (cookie.matches_url(url)) {
821  if (first_cookie) {
822  out << "Cookie: ";
823  first_cookie = false;
824  } else {
825  out << "; ";
826  }
827  out << cookie.get_name() << "=" << cookie.get_value();
828  }
829  }
830 
831  if (!first_cookie) {
832  out << "\r\n";
833  }
834 
835  if (any_expired) {
836  Cookies new_cookies;
837  Cookies::const_iterator ci;
838  for (ci = _cookies.begin(); ci != _cookies.end(); ++ci) {
839  const HTTPCookie &cookie = (*ci);
840  if (!cookie.is_expired(now)) {
841  new_cookies.insert(new_cookies.end(), cookie);
842  }
843  }
844  _cookies.swap(new_cookies);
845  }
846 }
847 
848 ////////////////////////////////////////////////////////////////////
849 // Function: HTTPClient::load_client_certificate
850 // Access: Published
851 // Description: Attempts to load the certificate named by
852 // set_client_certificate_filename() immediately, and
853 // returns true if successful, false otherwise.
854 //
855 // Normally this need not be explicitly called, since it
856 // will be called automatically if the server requests a
857 // certificate, but it may be useful to determine ahead
858 // of time if the certificate can be loaded correctly.
859 ////////////////////////////////////////////////////////////////////
860 bool HTTPClient::
861 load_client_certificate() {
862  if (!_client_certificate_loaded) {
863  _client_certificate_loaded = true;
864 
865  if (!_client_certificate_filename.empty()) {
866  _client_certificate_filename.set_text();
867 
868  // First, read the complete file into memory.
870 
871  if (!vfs->read_file(_client_certificate_filename,
872  _client_certificate_pem, true)) {
873  // Could not find or read file.
874  downloader_cat.warning()
875  << "Could not read " << _client_certificate_filename << ".\n";
876  return false;
877  }
878  }
879 
880  if (!_client_certificate_pem.empty()) {
881  // Create an in-memory BIO to read the "file" from the memory
882  // buffer, and call the low-level routines to read the
883  // keys from the BIO.
884  BIO *mbio = BIO_new_mem_buf((void *)_client_certificate_pem.data(),
885  _client_certificate_pem.length());
886 
887  ERR_clear_error();
888  _client_certificate_priv =
889  PEM_read_bio_PrivateKey(mbio, NULL, NULL,
890  (char *)_client_certificate_passphrase.c_str());
891 
892  // Rewind the "file" to the beginning in order to read the public
893  // key (which might appear first in the file).
894  BIO_reset(mbio);
895 
896  ERR_clear_error();
897  _client_certificate_pub =
898  PEM_read_bio_X509(mbio, NULL, NULL, NULL);
899 
900  BIO_free(mbio);
901 
902 
903  NotifySeverity sev = NS_debug;
904  string source = "memory";
905  if (!_client_certificate_filename.empty()) {
906  // Only report status to "info" severity if we have read the
907  // certificate from a file. If it came from an in-memory image,
908  // a failure will presumably be handled by whoever set the
909  // image.
910  sev = NS_info;
911  source = _client_certificate_filename;
912  }
913 
914  if (downloader_cat.is_on(sev)) {
915  if (_client_certificate_priv != (EVP_PKEY *)NULL &&
916  _client_certificate_pub != (X509 *)NULL) {
917  downloader_cat.out(sev)
918  << "Read client certificate from " << source << "\n";
919 
920  } else {
921  if (_client_certificate_priv == (EVP_PKEY *)NULL) {
922  downloader_cat.out(sev)
923  << "Could not read private key from " << source << "\n";
924  }
925 
926  if (_client_certificate_pub == (X509 *)NULL) {
927  downloader_cat.out(sev)
928  << "Could not read public key from " << source << "\n";
929  }
930  }
931  }
932  }
933  }
934 
935  return (_client_certificate_priv != (EVP_PKEY *)NULL &&
936  _client_certificate_pub != (X509 *)NULL);
937 }
938 
939 ////////////////////////////////////////////////////////////////////
940 // Function: HTTPClient::add_preapproved_server_certificate_filename
941 // Access: Published
942 // Description: Adds the certificate defined in the indicated PEM
943 // filename as a "pre-approved" certificate for the
944 // indicated server, defined by the hostname and port
945 // (only) from the given URL.
946 //
947 // If the server offers this particular certificate on a
948 // secure connection, it will be accepted without
949 // question. This is particularly useful for
950 // communicating with a server using a known self-signed
951 // certificate.
952 //
953 // See also the similar
954 // add_preapproved_server_certificate_pem(), and the
955 // weaker add_preapproved_server_certificate_name().
956 ////////////////////////////////////////////////////////////////////
957 bool HTTPClient::
958 add_preapproved_server_certificate_filename(const URLSpec &url, const Filename &filename) {
960  string pem;
961 
962  if (!vfs->read_file(filename, pem, true)) {
963  // Could not find or read file.
964  downloader_cat.warning()
965  << "Could not read " << filename << ".\n";
966  return false;
967  }
968 
969  return add_preapproved_server_certificate_pem(url, pem);
970 }
971 
972 ////////////////////////////////////////////////////////////////////
973 // Function: HTTPClient::add_preapproved_server_certificate_pem
974 // Access: Published
975 // Description: Adds the certificate defined in the indicated data
976 // string, formatted as a PEM block, as a "pre-approved"
977 // certificate for the indicated server, defined by the
978 // hostname and port (only) from the given URL.
979 //
980 // If the server offers this particular certificate on a
981 // secure connection, it will be accepted without
982 // question. This is particularly useful for
983 // communicating with a server using a known self-signed
984 // certificate.
985 //
986 // See also the similar
987 // add_preapproved_server_certificate_filename(), and
988 // the weaker add_preapproved_server_certificate_name().
989 ////////////////////////////////////////////////////////////////////
990 bool HTTPClient::
991 add_preapproved_server_certificate_pem(const URLSpec &url, const string &pem) {
992  // Create an in-memory BIO to read the "file" from the memory
993  // buffer, and call the low-level routine to read the
994  // cert from the BIO.
995  BIO *mbio = BIO_new_mem_buf((void *)pem.data(), pem.length());
996 
997  ERR_clear_error();
998  X509 *cert = PEM_read_bio_X509(mbio, NULL, NULL, NULL);
999  BIO_free(mbio);
1000 
1001  if (cert == NULL) {
1002  downloader_cat.warning()
1003  << "Could not parse PEM data\n";
1004  return false;
1005  }
1006 
1007  string server_and_port = url.get_server_and_port();
1008  PreapprovedServerCerts::iterator psci =
1009  _preapproved_server_certs.insert(PreapprovedServerCerts::value_type(server_and_port, PreapprovedServerCert())).first;
1010 
1011  PreapprovedServerCert &psc = (*psci).second;
1012  psc._certs.push_back(cert);
1013 
1014  return true;
1015 }
1016 
1017 ////////////////////////////////////////////////////////////////////
1018 // Function: HTTPClient::add_preapproved_server_certificate_name
1019 // Access: Published
1020 // Description: Adds the certificate *name* only, as a "pre-approved"
1021 // certificate name for the indicated server, defined by
1022 // the hostname and port (only) from the given URL.
1023 //
1024 // This is a weaker function than
1025 // add_preapproved_server_certificate_filename(). This
1026 // checks only the subject name of the certificate,
1027 // without checking for a particular certificate by key.
1028 // This means that a variety of server certificates may
1029 // match the indicated name.
1030 //
1031 // Because this is a weaker verification, it only
1032 // applies to server certificates that are signed by a
1033 // recognized certificate authority. Thus, it cannot be
1034 // used to pre-approve self-signed certificates, but it
1035 // can be used to accept a server certificate offered by
1036 // a different hostname than the one in the cert itself.
1037 //
1038 // The certificate name should be formatted in the form
1039 // /type0=value0/type1=value1/type2=...
1040 ////////////////////////////////////////////////////////////////////
1041 bool HTTPClient::
1042 add_preapproved_server_certificate_name(const URLSpec &url, const string &name) {
1043  X509_NAME *cert_name = parse_x509_name(name);
1044  if (cert_name == NULL) {
1045  downloader_cat.warning()
1046  << "Could not parse certificate name " << name << "\n";
1047  return false;
1048  }
1049 
1050  string server_and_port = url.get_server_and_port();
1051  PreapprovedServerCerts::iterator psci =
1052  _preapproved_server_certs.insert(PreapprovedServerCerts::value_type(server_and_port, PreapprovedServerCert())).first;
1053 
1054  PreapprovedServerCert &psc = (*psci).second;
1055  psc._cert_names.push_back(cert_name);
1056 
1057  return true;
1058 }
1059 
1060 ////////////////////////////////////////////////////////////////////
1061 // Function: HTTPClient::clear_preapproved_server_certificates
1062 // Access: Published
1063 // Description: Removes all preapproved server certificates for the
1064 // indicated server and port.
1065 ////////////////////////////////////////////////////////////////////
1066 void HTTPClient::
1067 clear_preapproved_server_certificates(const URLSpec &url) {
1068  string server_and_port = url.get_server_and_port();
1069  _preapproved_server_certs.erase(server_and_port);
1070 }
1071 
1072 ////////////////////////////////////////////////////////////////////
1073 // Function: HTTPClient::clear_all_preapproved_server_certificates
1074 // Access: Published
1075 // Description: Removes all preapproved server certificates for all
1076 // servers.
1077 ////////////////////////////////////////////////////////////////////
1078 void HTTPClient::
1079 clear_all_preapproved_server_certificates() {
1080  _preapproved_server_certs.clear();
1081 }
1082 
1083 ////////////////////////////////////////////////////////////////////
1084 // Function: HTTPClient::get_http_version_string
1085 // Access: Published
1086 // Description: Returns the current HTTP version setting as a string,
1087 // e.g. "HTTP/1.0" or "HTTP/1.1".
1088 ////////////////////////////////////////////////////////////////////
1089 string HTTPClient::
1090 get_http_version_string() const {
1091  switch (_http_version) {
1092  case HTTPEnum::HV_09:
1093  return "HTTP/0.9";
1094 
1095  case HTTPEnum::HV_10:
1096  return "HTTP/1.0";
1097 
1098  case HTTPEnum::HV_11:
1099  return "HTTP/1.1";
1100 
1101  case HTTPEnum::HV_other:
1102  // Report the best we can do.
1103  return "HTTP/1.1";
1104  }
1105 
1106  return "unknown";
1107 }
1108 
1109 ////////////////////////////////////////////////////////////////////
1110 // Function: HTTPClient::parse_http_version_string
1111 // Access: Published
1112 // Description: Matches the string representing a particular HTTP
1113 // version against any of the known versions and returns
1114 // the appropriate enumerated value, or HV_other if the
1115 // version is unknown.
1116 ////////////////////////////////////////////////////////////////////
1117 HTTPEnum::HTTPVersion HTTPClient::
1118 parse_http_version_string(const string &version) {
1119  if (version == "HTTP/1.0") {
1120  return HTTPEnum::HV_10;
1121  } else if (version == "HTTP/1.1") {
1122  return HTTPEnum::HV_11;
1123  } else if (version.substr(0, 6) == "HTTP/0") {
1124  return HTTPEnum::HV_09;
1125  } else {
1126  return HTTPEnum::HV_other;
1127  }
1128 }
1129 
1130 ////////////////////////////////////////////////////////////////////
1131 // Function: HTTPClient::load_certificates
1132 // Access: Published
1133 // Description: Reads the certificate(s) (delimited by -----BEGIN
1134 // CERTIFICATE----- and -----END CERTIFICATE-----) from
1135 // the indicated file and makes them known as trusted
1136 // public keys for validating future connections.
1137 // Returns true on success, false otherwise.
1138 ////////////////////////////////////////////////////////////////////
1139 bool HTTPClient::
1140 load_certificates(const Filename &filename) {
1141  OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
1142  return (sslw->load_certificates(filename) != 0);
1143 }
1144 
1145 ////////////////////////////////////////////////////////////////////
1146 // Function: HTTPClient::make_channel
1147 // Access: Published
1148 // Description: Returns a new HTTPChannel object that may be used
1149 // for reading multiple documents using the same
1150 // connection, for greater network efficiency than
1151 // calling HTTPClient::get_document() repeatedly (which
1152 // would force a new connection for each document).
1153 //
1154 // Also, HTTPChannel has some additional, less common
1155 // interface methods than the basic interface methods
1156 // that exist on HTTPClient; if you wish to call any of
1157 // these methods you must first obtain an HTTPChannel.
1158 //
1159 // Pass true for persistent_connection to gain this
1160 // network efficiency. If, on the other hand, your
1161 // intention is to use the channel to retrieve only one
1162 // document, then pass false to inform the server that
1163 // we will be dropping the connection after the first
1164 // document.
1165 ////////////////////////////////////////////////////////////////////
1166 PT(HTTPChannel) HTTPClient::
1167 make_channel(bool persistent_connection) {
1168  PT(HTTPChannel) doc = new HTTPChannel(this);
1169  doc->set_persistent_connection(persistent_connection);
1170  return doc;
1171 }
1172 
1173 ////////////////////////////////////////////////////////////////////
1174 // Function: HTTPClient::post_form
1175 // Access: Published
1176 // Description: Posts form data to a particular URL and retrieves the
1177 // response. Returns a new HTTPChannel object whether
1178 // the document is successfully read or not; you can
1179 // test is_valid() and get_return_code() to determine
1180 // whether the document was retrieved.
1181 ////////////////////////////////////////////////////////////////////
1182 PT(HTTPChannel) HTTPClient::
1183 post_form(const URLSpec &url, const string &body) {
1184  PT(HTTPChannel) doc = new HTTPChannel(this);
1185  doc->post_form(url, body);
1186  return doc;
1187 }
1188 
1189 ////////////////////////////////////////////////////////////////////
1190 // Function: HTTPClient::get_document
1191 // Access: Published
1192 // Description: Opens the named document for reading. Returns a new
1193 // HTTPChannel object whether the document is
1194 // successfully read or not; you can test is_valid() and
1195 // get_return_code() to determine whether the document
1196 // was retrieved.
1197 ////////////////////////////////////////////////////////////////////
1198 PT(HTTPChannel) HTTPClient::
1199 get_document(const URLSpec &url) {
1200  PT(HTTPChannel) doc = new HTTPChannel(this);
1201  doc->get_document(url);
1202  return doc;
1203 }
1204 
1205 ////////////////////////////////////////////////////////////////////
1206 // Function: HTTPClient::get_header
1207 // Access: Published
1208 // Description: Like get_document(), except only the header
1209 // associated with the document is retrieved. This may
1210 // be used to test for existence of the document; it
1211 // might also return the size of the document (if the
1212 // server gives us this information).
1213 ////////////////////////////////////////////////////////////////////
1214 PT(HTTPChannel) HTTPClient::
1215 get_header(const URLSpec &url) {
1216  PT(HTTPChannel) doc = new HTTPChannel(this);
1217  doc->get_header(url);
1218  return doc;
1219 }
1220 
1221 ////////////////////////////////////////////////////////////////////
1222 // Function: HTTPClient::get_global_ptr
1223 // Access: Published, Static
1224 // Description: Returns the default global HTTPClient.
1225 ////////////////////////////////////////////////////////////////////
1226 HTTPClient *HTTPClient::
1227 get_global_ptr() {
1228  if (_global_ptr == NULL) {
1229  _global_ptr = new HTTPClient;
1230  }
1231  return _global_ptr;
1232 }
1233 
1234 
1235 ////////////////////////////////////////////////////////////////////
1236 // Function: HTTPClient::get_ssl_ctx
1237 // Access: Public
1238 // Description: Returns the OpenSSL context object, creating it first
1239 // if needed.
1240 ////////////////////////////////////////////////////////////////////
1241 SSL_CTX *HTTPClient::
1242 get_ssl_ctx() {
1243  if (_ssl_ctx != (SSL_CTX *)NULL) {
1244  return _ssl_ctx;
1245  }
1246 
1247  init_random_seed();
1248 
1249  _ssl_ctx = SSL_CTX_new(SSLv23_client_method());
1250 
1251 #if defined(SSL_097) && !defined(NDEBUG)
1252  // If we have debugging enabled, set a callback that allows us to
1253  // report the SSL messages as they are sent and received.
1254  if (downloader_cat.is_debug()) {
1255  SSL_CTX_set_msg_callback(_ssl_ctx, ssl_msg_callback);
1256  }
1257 #endif
1258 
1259  // Make sure the error strings are loaded.
1260  OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
1261  sslw->notify_ssl_errors();
1262 
1263  SSL_CTX_set_cert_store(_ssl_ctx, sslw->get_x509_store());
1264 
1265  return _ssl_ctx;
1266 }
1267 
1268 ////////////////////////////////////////////////////////////////////
1269 // Function: HTTPClient::check_preapproved_server_certificate
1270 // Access: Private
1271 // Description: Checks to see if the indicated certificate is on the
1272 // pre-approved list for the indicated server's URL.
1273 //
1274 // If the full cert itself (including its key) is on the
1275 // pre-approved list, sets both cert_preapproved and
1276 // cert_name_preapproved to true.
1277 //
1278 // If the full cert is not on the pre-approved list, but
1279 // its name matches a name on the pre-approved list,
1280 // sets cert_name_preapproved to true, and
1281 // cert_preapproved to false.
1282 //
1283 // Otherwise, sets both values to false. This doesn't
1284 // mean the cert is necessarily invalid, just that it
1285 // wasn't on the pre-approved list (which is usually
1286 // empty anyway).
1287 ////////////////////////////////////////////////////////////////////
1288 void HTTPClient::
1289 check_preapproved_server_certificate(const URLSpec &url, X509 *cert,
1290  bool &cert_preapproved, bool &cert_name_preapproved) const {
1291  cert_preapproved = false;
1292  cert_name_preapproved = false;
1293 
1294  string server_and_port = url.get_server_and_port();
1295  PreapprovedServerCerts::const_iterator psci =
1296  _preapproved_server_certs.find(server_and_port);
1297 
1298  if (psci == _preapproved_server_certs.end()) {
1299  // No preapproved certs defined for this server.
1300  return;
1301  }
1302 
1303  const PreapprovedServerCert &psc = (*psci).second;
1304 
1305  // See if we have an exact match on the cert itself.
1306  ServerCerts::const_iterator sci;
1307  for (sci = psc._certs.begin(); sci != psc._certs.end(); ++sci) {
1308  if (X509_cmp(cert, *sci) == 0) {
1309  cert_preapproved = true;
1310  cert_name_preapproved = true;
1311  downloader_cat.info()
1312  << "Server certificate is pre-approved.\n";
1313  return;
1314  }
1315  }
1316 
1317  // OK, look for a match on the cert's name only.
1318  X509_NAME *subject = X509_get_subject_name(cert);
1319  ServerCertNames::const_iterator scni;
1320  for (scni = psc._cert_names.begin(); scni != psc._cert_names.end(); ++scni) {
1321  X509_NAME *cert_name = (*scni);
1322  if (x509_name_subset(cert_name, subject)) {
1323  downloader_cat.info()
1324  << "Server certificate name is pre-approved.\n";
1325  cert_name_preapproved = true;
1326  return;
1327  }
1328  }
1329 
1330  // Didn't find any match.
1331  return;
1332 }
1333 
1334 ////////////////////////////////////////////////////////////////////
1335 // Function: HTTPClient::get_proxies_for_scheme
1336 // Access: Private
1337 // Description: Adds the proxy servers associated with the indicated
1338 // scheme, if any, to the list. Returns true if any
1339 // were added, false otherwise.
1340 ////////////////////////////////////////////////////////////////////
1341 bool HTTPClient::
1342 get_proxies_for_scheme(const string &scheme, pvector<URLSpec> &proxies) const {
1343  ProxiesByScheme::const_iterator si = _proxies_by_scheme.find(scheme);
1344  if (si == _proxies_by_scheme.end()) {
1345  return false;
1346  }
1347  const Proxies &scheme_proxies = (*si).second;
1348  if (scheme_proxies.empty()) {
1349  return false;
1350  }
1351 
1352  Proxies::const_iterator pi;
1353  for (pi = scheme_proxies.begin(); pi != scheme_proxies.end(); ++pi) {
1354  proxies.push_back(*pi);
1355  }
1356 
1357  return true;
1358 }
1359 
1360 ////////////////////////////////////////////////////////////////////
1361 // Function: HTTPClient::add_http_username
1362 // Access: Private
1363 // Description: Handles a Config definition for http-username as
1364 // server:realm:username:password, where either or both
1365 // of server and realm may be empty, or just
1366 // server:username:password or username:password.
1367 ////////////////////////////////////////////////////////////////////
1368 void HTTPClient::
1369 add_http_username(const string &http_username) {
1370  size_t c1 = http_username.find(':');
1371  if (c1 != string::npos) {
1372  size_t c2 = http_username.find(':', c1 + 1);
1373  if (c2 != string::npos) {
1374  size_t c3 = http_username.find(':', c2 + 1);
1375  if (c3 != string::npos) {
1376  size_t c4 = http_username.find(':', c3 + 1);
1377  if (c4 != string::npos) {
1378  // Oops, we have five? Problem.
1379  downloader_cat.error()
1380  << "Invalid http-username " << http_username << "\n";
1381 
1382  } else {
1383  // Ok, we have four.
1384  set_username(http_username.substr(0, c1),
1385  http_username.substr(c1 + 1, c2 - (c1 + 1)),
1386  http_username.substr(c2 + 1));
1387  }
1388 
1389  } else {
1390  // We have only three.
1391  set_username(string(),
1392  http_username.substr(0, c1),
1393  http_username.substr(c1 + 1));
1394  }
1395  } else {
1396  // We have only two.
1397  set_username(string(), string(), http_username);
1398  }
1399  } else {
1400  // We have only one? Problem.
1401  downloader_cat.error()
1402  << "Invalid http-username " << http_username << "\n";
1403  }
1404 }
1405 
1406 ////////////////////////////////////////////////////////////////////
1407 // Function: HTTPClient::select_username
1408 // Access: Private
1409 // Description: Chooses a suitable username:password string for the
1410 // given URL and realm.
1411 ////////////////////////////////////////////////////////////////////
1412 string HTTPClient::
1413 select_username(const URLSpec &url, bool is_proxy, const string &realm) const {
1414  string username;
1415 
1416  // Look in several places in order to find the matching username.
1417 
1418  // Fist, if there's a username on the URL, that always wins (except
1419  // when we are looking for a proxy username).
1420  if (url.has_username() && !is_proxy) {
1421  username = url.get_username();
1422  }
1423 
1424  // Otherwise, start looking on the HTTPClient.
1425  if (is_proxy) {
1426  if (username.empty()) {
1427  // Try the *proxy/realm.
1428  username = get_username("*proxy", realm);
1429  }
1430  if (username.empty()) {
1431  // Then, try *proxy/any realm.
1432  username = get_username("*proxy", string());
1433  }
1434  }
1435  if (username.empty()) {
1436  // Try the specific server/realm.
1437  username = get_username(url.get_server(), realm);
1438  }
1439  if (username.empty()) {
1440  // Then, try the specific server/any realm.
1441  username = get_username(url.get_server(), string());
1442  }
1443  if (username.empty()) {
1444  // Then, try any server with this realm.
1445  username = get_username(string(), realm);
1446  }
1447  if (username.empty()) {
1448  // Then, take the general password.
1449  username = get_username(string(), string());
1450  }
1451 
1452  return username;
1453 }
1454 
1455 ////////////////////////////////////////////////////////////////////
1456 // Function: HTTPClient::select_auth
1457 // Access: Private
1458 // Description: Chooses a suitable pre-computed authorization for the
1459 // indicated URL. Returns NULL if no authorization
1460 // matches.
1461 ////////////////////////////////////////////////////////////////////
1462 HTTPAuthorization *HTTPClient::
1463 select_auth(const URLSpec &url, bool is_proxy, const string &last_realm) {
1464  Domains &domains = is_proxy ? _proxy_domains : _www_domains;
1465  string canon = HTTPAuthorization::get_canonical_url(url).get_url();
1466 
1467  // Look for the longest domain string that is a prefix of our
1468  // canonical URL. We have to make a linear scan through the list.
1469  Domains::const_iterator best_di = domains.end();
1470  size_t longest_length = 0;
1471  Domains::const_iterator di;
1472  for (di = domains.begin(); di != domains.end(); ++di) {
1473  const string &domain = (*di).first;
1474  size_t length = domain.length();
1475  if (domain == canon.substr(0, length)) {
1476  // This domain string matches. Is it the longest?
1477  if (length > longest_length) {
1478  best_di = di;
1479  longest_length = length;
1480  }
1481  }
1482  }
1483 
1484  if (best_di != domains.end()) {
1485  // Ok, we found a matching domain. Use it.
1486  if (downloader_cat.is_spam()) {
1487  downloader_cat.spam()
1488  << "Choosing domain " << (*best_di).first << " for " << url << "\n";
1489  }
1490  const Realms &realms = (*best_di).second._realms;
1491  // First, try our last realm.
1492  Realms::const_iterator ri;
1493  ri = realms.find(last_realm);
1494  if (ri != realms.end()) {
1495  return (*ri).second;
1496  }
1497 
1498  if (!realms.empty()) {
1499  // Oh well, just return the first realm.
1500  return (*realms.begin()).second;
1501  }
1502  }
1503 
1504  // No matching domains.
1505  return NULL;
1506 }
1507 
1508 ////////////////////////////////////////////////////////////////////
1509 // Function: HTTPClient::generate_auth
1510 // Access: Private
1511 // Description: Generates a new authorization entry in response to a
1512 // 401 or 407 challenge from the server or proxy. The
1513 // new authorization entry is stored for future
1514 // connections to the same server (or, more precisely,
1515 // the same domain, which may be a subset of the server,
1516 // or it may include multiple servers).
1517 ////////////////////////////////////////////////////////////////////
1518 PT(HTTPAuthorization) HTTPClient::
1519 generate_auth(const URLSpec &url, bool is_proxy, const string &challenge) {
1520  HTTPAuthorization::AuthenticationSchemes schemes;
1521  HTTPAuthorization::parse_authentication_schemes(schemes, challenge);
1522 
1523  PT(HTTPAuthorization) auth;
1524  HTTPAuthorization::AuthenticationSchemes::iterator si;
1525 
1526  si = schemes.find("digest");
1527  if (si != schemes.end()) {
1528  auth = new HTTPDigestAuthorization((*si).second, url, is_proxy);
1529  }
1530 
1531  if (auth == (HTTPAuthorization *)NULL || !auth->is_valid()) {
1532  si = schemes.find("basic");
1533  if (si != schemes.end()) {
1534  auth = new HTTPBasicAuthorization((*si).second, url, is_proxy);
1535  }
1536  }
1537 
1538  if (auth == (HTTPAuthorization *)NULL || !auth->is_valid()) {
1539  downloader_cat.warning()
1540  << "Don't know how to use any of the server's available authorization schemes:\n";
1541  for (si = schemes.begin(); si != schemes.end(); ++si) {
1542  downloader_cat.warning() << (*si).first << "\n";
1543  }
1544 
1545  } else {
1546  // Now that we've got an authorization, store it under under each
1547  // of its suggested domains for future use.
1548  Domains &domains = is_proxy ? _proxy_domains : _www_domains;
1549  const vector_string &domain = auth->get_domain();
1550  vector_string::const_iterator si;
1551  for (si = domain.begin(); si != domain.end(); ++si) {
1552  domains[(*si)]._realms[auth->get_realm()] = auth;
1553  }
1554  }
1555 
1556  return auth;
1557 }
1558 
1559 ////////////////////////////////////////////////////////////////////
1560 // Function: HTTPClient::unload_client_certificate
1561 // Access: Private
1562 // Description: Frees the resources allocated by a previous call to
1563 // load_client_certificate(), and marks the certificate
1564 // unloaded.
1565 ////////////////////////////////////////////////////////////////////
1566 void HTTPClient::
1567 unload_client_certificate() {
1568  if (_client_certificate_priv != (EVP_PKEY *)NULL) {
1569  EVP_PKEY_free(_client_certificate_priv);
1570  _client_certificate_priv = NULL;
1571  }
1572 
1573  if (_client_certificate_pub != (X509 *)NULL) {
1574  X509_free(_client_certificate_pub);
1575  _client_certificate_pub = NULL;
1576  }
1577 
1578  _client_certificate_loaded = false;
1579 }
1580 
1581 ////////////////////////////////////////////////////////////////////
1582 // Function: HTTPClient::parse_x509_name
1583 // Access: Private, Static
1584 // Description: Parses a string of the form
1585 // /type0=value0/type1=value1/type2=... into a newly
1586 // allocated X509_NAME object. Returns NULL if the
1587 // string is invalid.
1588 ////////////////////////////////////////////////////////////////////
1589 X509_NAME *HTTPClient::
1590 parse_x509_name(const string &source) {
1591  X509_NAME *result = NULL;
1592 
1593  result = X509_NAME_new();
1594  bool added_any = false;
1595 
1596  string::const_iterator si;
1597  si = source.begin();
1598  while (si != source.end()) {
1599  if ((*si) == '/') {
1600  // Skip a slash delimiter.
1601  ++si;
1602  } else {
1603  string type;
1604  while (si != source.end() && (*si) != '=' && (*si) != '/') {
1605  if ((*si) == '\\') {
1606  ++si;
1607  if (si != source.end()) {
1608  type += (*si);
1609  ++si;
1610  }
1611  } else {
1612  type += (*si);
1613  ++si;
1614  }
1615  }
1616 
1617  int nid = OBJ_txt2nid((char *)type.c_str());
1618  if (nid == NID_undef) {
1619  downloader_cat.info()
1620  << "Unknown type " << type << " in X509 name: " << source
1621  << "\n";
1622  X509_NAME_free(result);
1623  return NULL;
1624  }
1625 
1626  string value;
1627 
1628  if (si != source.end() && (*si) == '=') {
1629  ++si;
1630  while (si != source.end() && (*si) != '/') {
1631  if ((*si) == '\\') {
1632  ++si;
1633  if (si != source.end()) {
1634  value += (*si);
1635  ++si;
1636  }
1637  } else {
1638  value += (*si);
1639  ++si;
1640  }
1641  }
1642  }
1643 
1644  if (!value.empty()) {
1645  int add_result =
1646  X509_NAME_add_entry_by_NID(result, nid, V_ASN1_APP_CHOOSE,
1647  (unsigned char *)value.c_str(), -1, -1, 0);
1648  if (!add_result) {
1649  downloader_cat.info()
1650  << "Unable to add " << type << "=" << value << " in X509 name: "
1651  << source << "\n";
1652  X509_NAME_free(result);
1653  return NULL;
1654  }
1655  added_any = true;
1656  }
1657  }
1658  }
1659 
1660  if (!added_any) {
1661  downloader_cat.info()
1662  << "Invalid empty X509 name: " << source << "\n";
1663  X509_NAME_free(result);
1664  return NULL;
1665  }
1666 
1667  return result;
1668 }
1669 
1670 ////////////////////////////////////////////////////////////////////
1671 // Function: HTTPClient::x509_name_subset
1672 // Access: Private, Static
1673 // Description: Returns true if name_a is a subset of name_b: each
1674 // property of name_a is defined in name_b, and the
1675 // defined value is equivalent to that of name_a.
1676 ////////////////////////////////////////////////////////////////////
1677 bool HTTPClient::
1678 x509_name_subset(X509_NAME *name_a, X509_NAME *name_b) {
1679  int count_a = X509_NAME_entry_count(name_a);
1680  for (int ai = 0; ai < count_a; ai++) {
1681  X509_NAME_ENTRY *na = X509_NAME_get_entry(name_a, ai);
1682 
1683  int bi = X509_NAME_get_index_by_OBJ(name_b, na->object, -1);
1684  if (bi < 0) {
1685  // This entry in name_a is not defined in name_b.
1686  return false;
1687  }
1688 
1689  X509_NAME_ENTRY *nb = X509_NAME_get_entry(name_b, bi);
1690  if (na->value->length != nb->value->length ||
1691  memcmp(na->value->data, nb->value->data, na->value->length) != 0) {
1692  // This entry in name_a doesn't match that of name_b.
1693  return false;
1694  }
1695  }
1696  return true;
1697 }
1698 
1699 ////////////////////////////////////////////////////////////////////
1700 // Function: HTTPClient::split_whitespace
1701 // Access: Private, Static
1702 // Description: Puts the first word of c into a, and the remainder
1703 // into b.
1704 ////////////////////////////////////////////////////////////////////
1705 void HTTPClient::
1706 split_whitespace(string &a, string &b, const string &c) {
1707  size_t p = 0;
1708 
1709  // Skip initial whitespace
1710  while (p < c.length() && isspace(c[p])) {
1711  ++p;
1712  }
1713 
1714  // Find the length of the first word
1715  size_t q = p;
1716  while (p < c.length() && !isspace(c[p])) {
1717  ++p;
1718  }
1719 
1720  a = c.substr(q, p - q);
1721 
1722  // Skip whitespace between words
1723  while (p < c.length() && isspace(c[p])) {
1724  ++p;
1725  }
1726  b = c.substr(p);
1727 }
1728 
1729 #if defined(SSL_097) && !defined(NDEBUG)
1730 ////////////////////////////////////////////////////////////////////
1731 // Function: HTTPClient::ssl_msg_callback
1732 // Access: Private, Static
1733 // Description: This method is attached as a callback for SSL
1734 // messages only when debug output is enabled.
1735 ////////////////////////////////////////////////////////////////////
1736 void HTTPClient::
1737 ssl_msg_callback(int write_p, int version, int content_type,
1738  const void *, size_t len, SSL *, void *) {
1739  ostringstream describe;
1740  if (write_p) {
1741  describe << "sent ";
1742  } else {
1743  describe << "received ";
1744  }
1745  switch (version) {
1746  case SSL2_VERSION:
1747  describe << "SSL 2.0 ";
1748  break;
1749 
1750  case SSL3_VERSION:
1751  describe << "SSL 3.0 ";
1752  break;
1753 
1754  case TLS1_VERSION:
1755  describe << "TLS 1.0 ";
1756  break;
1757 
1758  default:
1759  describe << "unknown protocol ";
1760  }
1761 
1762  describe << "message: ";
1763 
1764  if (version != SSL2_VERSION) {
1765  switch (content_type) {
1766  case 20:
1767  describe << "change cipher spec, ";
1768  break;
1769 
1770  case 21:
1771  describe << "alert, ";
1772  break;
1773 
1774  case 22:
1775  describe << "handshake, ";
1776  break;
1777 
1778  case 23:
1779  describe << "application data, ";
1780  break;
1781 
1782  default:
1783  describe << "unknown content type, ";
1784  }
1785  }
1786 
1787  describe << len << " bytes.\n";
1788 
1789  downloader_cat.debug() << describe.str();
1790 }
1791 #endif // defined(SSL_097) && !defined(NDEBUG)
1792 
1793 ////////////////////////////////////////////////////////////////////
1794 // Function: HTTPClient::PreapprovedServerCert::Destructor
1795 // Access: Public
1796 // Description:
1797 ////////////////////////////////////////////////////////////////////
1798 HTTPClient::PreapprovedServerCert::
1799 ~PreapprovedServerCert() {
1800  ServerCerts::const_iterator sci;
1801  for (sci = _certs.begin(); sci != _certs.end(); ++sci) {
1802  X509_free(*sci);
1803  }
1804 
1805  ServerCertNames::const_iterator scni;
1806  for (scni = _cert_names.begin(); scni != _cert_names.end(); ++scni) {
1807  X509_NAME_free(*scni);
1808  }
1809 }
1810 
1811 #endif // HAVE_OPENSSL
bool has_username() const
Returns true if the URL specifies a username (and/or password), false otherwise.
Definition: urlSpec.I:118
A container for a URL, e.g.
Definition: urlSpec.h:29
This is a convenience class to specialize ConfigVariable as a Filename type.
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS&#39;s file system.
This is a convenience class to specialize ConfigVariable as a boolean type.
bool is_ssl() const
Returns true if the URL&#39;s scheme specifies an SSL-secured protocol such as https, or false otherwise...
Definition: urlSpec.I:234
const string & get_pattern() const
Returns the pattern string that the GlobPattern object matches.
Definition: globPattern.I:100
string read_file(const Filename &filename, bool auto_unwrap) const
Convenience function; returns the entire contents of the indicated file as a string.
This class is similar to ConfigVariable, but it reports its value as a list of strings.
This is our own Panda specialization on the default STL vector.
Definition: pvector.h:39
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:44
This is a convenience class to specialize ConfigVariable as a string type.
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
A container for an HTTP-legal time/date indication.
Definition: httpDate.h:30
string get_scheme() const
Returns the scheme specified by the URL, or empty string if no scheme is specified.
Definition: urlSpec.cxx:70
static HTTPDate now()
Returns an HTTPDate that represents the current time and date.
Definition: httpDate.I:60
const string & get_url() const
Returns the complete URL specification.
Definition: urlSpec.I:254
string get_server() const
Returns the server name specified by the URL, if any.
Definition: urlSpec.I:198
string get_username() const
Returns the username specified by the URL, if any.
Definition: urlSpec.I:188
string get_server_and_port() const
Returns a string consisting of the server name, followed by a colon, followed by the port number...
Definition: urlSpec.cxx:137
This is our own Panda specialization on the default STL set.
Definition: pset.h:52
This class can be used to test for string matches against standard Unix-shell filename globbing conve...
Definition: globPattern.h:37