Panda3D
 All Classes Functions Variables Enumerations
httpChannel.cxx
1 // Filename: httpChannel.cxx
2 // Created by: drose (24Sep02)
3 //
4 ////////////////////////////////////////////////////////////////////
5 //
6 // PANDA 3D SOFTWARE
7 // Copyright (c) Carnegie Mellon University. All rights reserved.
8 //
9 // All use of this software is subject to the terms of the revised BSD
10 // license. You should have received a copy of this license along
11 // with this source code in a file named "LICENSE."
12 //
13 ////////////////////////////////////////////////////////////////////
14 
15 #include "httpChannel.h"
16 #include "httpClient.h"
17 #include "httpCookie.h"
18 #include "bioStream.h"
19 #include "chunkedStream.h"
20 #include "identityStream.h"
21 #include "config_downloader.h"
22 #include "virtualFileSystem.h"
23 #include "virtualFileMountHTTP.h"
24 #include "ramfile.h"
25 #include "globPattern.h"
26 
27 #include <stdio.h>
28 
29 #ifdef HAVE_OPENSSL
30 
31 #if defined(WIN32_VC) || defined(WIN64_VC)
32  #include <WinSock2.h>
33  #include <windows.h> // for select()
34  #undef X509_NAME
35 #endif // WIN32_VC
36 
37 TypeHandle HTTPChannel::_type_handle;
38 
39 #define _NOTIFY_HTTP_CHANNEL_ID "[" << this << "] "
40 
41 ////////////////////////////////////////////////////////////////////
42 // Function: HTTPChannel::Constructor
43 // Access: Private
44 // Description:
45 ////////////////////////////////////////////////////////////////////
46 HTTPChannel::
47 HTTPChannel(HTTPClient *client) :
48  _client(client)
49 {
50  if (downloader_cat.is_debug()) {
51  downloader_cat.debug()
52  << _NOTIFY_HTTP_CHANNEL_ID
53  << "created.\n";
54  }
55 
56  ConfigVariableDouble extra_ssl_handshake_time
57  ("extra-ssl-handshake-time", 0.0,
58  PRC_DESC("This specifies how much extra time to try to establish"
59  "the ssl handshake before we bail."));
60  _extra_ssl_handshake_time = extra_ssl_handshake_time;
61  _proxy_next_index = 0;
62  _persistent_connection = false;
63  _allow_proxy = true;
64  _proxy_tunnel = http_proxy_tunnel;
65  _connect_timeout = http_connect_timeout;
66  _http_timeout = http_timeout;
67  _skip_body_size = http_skip_body_size;
68  _idle_timeout = http_idle_timeout;
69  _blocking_connect = false;
70  _download_throttle = download_throttle;
71  _max_bytes_per_second = downloader_byte_rate;
72  _seconds_per_update = downloader_frequency;
73  _max_updates_per_second = 1.0f / _seconds_per_update;
74  _bytes_per_update = int(_max_bytes_per_second * _seconds_per_update);
75 
76  // _nonblocking is true if the socket is actually in non-blocking mode.
77  _nonblocking = false;
78 
79  // _wanted_nonblocking is true if the user specifically requested
80  // one of the non-blocking interfaces. It is false if the socket is
81  // only incidentally non-blocking (for instance, because
82  // SIMPLE_THREADS is on).
83  _wanted_nonblocking = false;
84 
85  _want_ssl = false;
86  _proxy_serves_document = false;
87  _proxy_tunnel_now = false;
88  _first_byte_requested = 0;
89  _last_byte_requested = 0;
90  _first_byte_delivered = 0;
91  _last_byte_delivered = 0;
92  _read_index = 0;
93  _expected_file_size = 0;
94  _file_size = 0;
95  _transfer_file_size = 0;
96  _got_expected_file_size = false;
97  _got_file_size = false;
98  _got_transfer_file_size = false;
99  _bytes_downloaded = 0;
100  _bytes_requested = 0;
101  _status_entry = StatusEntry();
102  _response_type = RT_none;
103  _http_version = _client->get_http_version();
104  _http_version_string = _client->get_http_version_string();
105  _state = S_new;
106  _done_state = S_new;
107  _started_download = false;
108  _sent_so_far = 0;
109  _body_stream = NULL;
110  _owns_body_stream = false;
111  _sbio = NULL;
112  _cipher_list = _client->get_cipher_list();
113  _last_status_code = 0;
114  _last_run_time = 0.0f;
115  _download_to_ramfile = NULL;
116  _download_to_stream = NULL;
117 }
118 
119 ////////////////////////////////////////////////////////////////////
120 // Function: HTTPChannel::Destructor
121 // Access: Public, Virtual
122 // Description:
123 ////////////////////////////////////////////////////////////////////
124 HTTPChannel::
125 ~HTTPChannel() {
126  if (downloader_cat.is_debug()) {
127  downloader_cat.debug()
128  << _NOTIFY_HTTP_CHANNEL_ID
129  << "destroyed.\n";
130  }
131 
132  close_connection();
133  reset_download_to();
134 }
135 
136 ////////////////////////////////////////////////////////////////////
137 // Function: HTTPChannel::get_status_string
138 // Access: Published
139 // Description: Returns the string as returned by the server
140 // describing the status code for humans. This may or
141 // may not be meaningful.
142 ////////////////////////////////////////////////////////////////////
143 string HTTPChannel::
144 get_status_string() const {
145  switch (_status_entry._status_code) {
146  case SC_incomplete:
147  return "Connection in progress";
148 
149  case SC_internal_error:
150  return "Internal error";
151 
152  case SC_no_connection:
153  return "No connection";
154 
155  case SC_timeout:
156  return "Timeout on connection";
157 
158  case SC_lost_connection:
159  return "Lost connection";
160 
161  case SC_non_http_response:
162  return "Non-HTTP response";
163 
164  case SC_invalid_http:
165  return "Could not understand HTTP response";
166 
167  case SC_socks_invalid_version:
168  return "Unsupported SOCKS version";
169 
170  case SC_socks_no_acceptable_login_method:
171  return "No acceptable SOCKS login method";
172 
173  case SC_socks_refused:
174  return "SOCKS proxy refused connection";
175 
176  case SC_socks_no_connection:
177  return "SOCKS proxy unable to connect";
178 
179  case SC_ssl_internal_failure:
180  return "SSL internal failure";
181 
182  case SC_ssl_no_handshake:
183  return "No SSL handshake";
184 
185  case SC_http_error_watermark:
186  // This shouldn't be triggered.
187  return "Internal error";
188 
189  case SC_ssl_invalid_server_certificate:
190  return "SSL invalid server certificate";
191 
192  case SC_ssl_unexpected_server:
193  return "Unexpected SSL server";
194 
195  case SC_download_open_error:
196  return "Error opening file";
197 
198  case SC_download_write_error:
199  return "Error writing to disk";
200 
201  case SC_download_invalid_range:
202  return "Invalid subrange requested";
203  }
204 
205  return _status_entry._status_string;
206 }
207 
208 ////////////////////////////////////////////////////////////////////
209 // Function: HTTPChannel::get_header_value
210 // Access: Published
211 // Description: Returns the HTML header value associated with the
212 // indicated key, or empty string if the key was not
213 // defined in the message returned by the server.
214 ////////////////////////////////////////////////////////////////////
215 string HTTPChannel::
216 get_header_value(const string &key) const {
217  Headers::const_iterator hi = _headers.find(downcase(key));
218  if (hi != _headers.end()) {
219  return (*hi).second;
220  }
221  return string();
222 }
223 
224 ////////////////////////////////////////////////////////////////////
225 // Function: HTTPChannel::will_close_connection
226 // Access: Published
227 // Description: Returns true if the server has indicated it will
228 // close the connection after this document has been
229 // read, or false if it will remain open (and future
230 // documents may be requested on the same connection).
231 ////////////////////////////////////////////////////////////////////
232 bool HTTPChannel::
233 will_close_connection() const {
234  if (get_http_version() < HTTPEnum::HV_11) {
235  // pre-HTTP 1.1 always closes.
236  return true;
237  }
238 
239  string connection = get_header_value("Connection");
240  if (downcase(connection) == "close") {
241  // The server says it will close.
242  return true;
243  }
244 
245  if (connection.empty() && !get_persistent_connection()) {
246  // The server didn't say, but we asked it to close.
247  return true;
248  }
249 
250  // Assume the server will keep it open.
251  return false;
252 }
253 
254 ////////////////////////////////////////////////////////////////////
255 // Function: HTTPChannel::get_file_size
256 // Access: Published, Virtual
257 // Description: Returns the size of the file, if it is known.
258 // Returns the value set by set_expected_file_size() if
259 // the file size is not known, or 0 if this value was
260 // not set.
261 //
262 // If the file is dynamically generated, the size may
263 // not be available until a read has started
264 // (e.g. open_read_body() has been called); and even
265 // then it may increase as more of the file is read due
266 // to the nature of HTTP/1.1 requests which can change
267 // their minds midstream about how much data they're
268 // sending you.
269 ////////////////////////////////////////////////////////////////////
270 streamsize HTTPChannel::
271 get_file_size() const {
272  if (_got_file_size) {
273  return _file_size;
274  } else if (_got_transfer_file_size) {
275  return _transfer_file_size;
276  } else if (_got_expected_file_size) {
277  return _expected_file_size;
278  } else {
279  return 0;
280  }
281 }
282 
283 ////////////////////////////////////////////////////////////////////
284 // Function: HTTPChannel::write_headers
285 // Access: Published
286 // Description: Outputs a list of all headers defined by the server
287 // to the indicated output stream.
288 ////////////////////////////////////////////////////////////////////
289 void HTTPChannel::
290 write_headers(ostream &out) const {
291  Headers::const_iterator hi;
292  for (hi = _headers.begin(); hi != _headers.end(); ++hi) {
293  out << (*hi).first << ": " << (*hi).second << "\n";
294  }
295 }
296 
297 ////////////////////////////////////////////////////////////////////
298 // Function: HTTPChannel::run
299 // Access: Published
300 // Description: This must be called from time to time when
301 // non-blocking I/O is in use. It checks for data
302 // coming in on the socket and writes data out to the
303 // socket when possible, and does whatever processing is
304 // required towards completing the current task.
305 //
306 // The return value is true if the task is still pending
307 // (and run() will need to be called again in the
308 // future), or false if the current task is complete.
309 ////////////////////////////////////////////////////////////////////
310 bool HTTPChannel::
311 run() {
312  if (downloader_cat.is_spam()) {
313  downloader_cat.spam()
314  << _NOTIFY_HTTP_CHANNEL_ID
315  << "run().\n";
316  }
317 
318  if (_state == _done_state || _state == S_failure) {
319  clear_extra_headers();
320  if (!reached_done_state()) {
321  return false;
322  }
323  }
324 
325  if (_started_download) {
326  if (_wanted_nonblocking && _download_throttle) {
327  double now = TrueClock::get_global_ptr()->get_short_time();
328  double elapsed = now - _last_run_time;
329  if (elapsed < _seconds_per_update) {
330  // Come back later.
331  thread_yield();
332  return true;
333  }
334  int num_potential_updates = (int)(elapsed / _seconds_per_update);
335  _last_run_time = now;
336  _bytes_requested += _bytes_per_update * num_potential_updates;
337  if (downloader_cat.is_spam()) {
338  downloader_cat.spam()
339  << _NOTIFY_HTTP_CHANNEL_ID
340  << "elapsed = " << elapsed << " num_potential_updates = "
341  << num_potential_updates << " bytes_requested = "
342  << _bytes_requested << "\n";
343  }
344  }
345 
346  bool repeat_later = false;
347  switch (_download_dest) {
348  case DD_none:
349  // We're done.
350  break;
351 
352  case DD_file:
353  repeat_later = run_download_to_file();
354  break;
355 
356  case DD_ram:
357  repeat_later = run_download_to_ram();
358  break;
359 
360  case DD_stream:
361  repeat_later = run_download_to_stream();
362  break;
363  }
364  if (repeat_later) {
365  thread_yield();
366  }
367  return repeat_later;
368  }
369 
370  /*
371  if (downloader_cat.is_spam()) {
372  downloader_cat.spam()
373  << _NOTIFY_HTTP_CHANNEL_ID
374  << "begin run(), _state = " << _state << ", _done_state = "
375  << _done_state << "\n";
376  }
377  */
378 
379  if (_state == _done_state) {
380  return reached_done_state();
381  }
382 
383  bool repeat_later;
384  do {
385  // If we're in a state that expects to have a connection already
386  // (that is, any state other that S_try_next_proxy), then
387  // reestablish the connection if it has been dropped.
388  if (_bio.is_null() && _state != S_try_next_proxy) {
389  if (_connect_count > http_max_connect_count) {
390  // Too many connection attempts; just give up. We should
391  // never trigger this failsafe, since the code in each
392  // individual case has similar logic to prevent more than two
393  // consecutive lost connections.
394  downloader_cat.warning()
395  << _NOTIFY_HTTP_CHANNEL_ID
396  << "Too many lost connections, giving up.\n";
397  _status_entry._status_code = SC_lost_connection;
398  _state = S_failure;
399  return false;
400  }
401 
402  // No connection. Attempt to establish one.
403  if (_proxy.empty()) {
404  _bio = new BioPtr(_request.get_url());
405  } else {
406  _bio = new BioPtr(_proxy);
407  }
408  _source = new BioStreamPtr(new BioStream(_bio));
409  if (_nonblocking) {
410  BIO_set_nbio(*_bio, 1);
411  }
412 
413  if (downloader_cat.is_debug()) {
414  if (_connect_count > 0) {
415  downloader_cat.debug()
416  << _NOTIFY_HTTP_CHANNEL_ID
417  << "Reconnecting to " << _bio->get_server_name() << ":"
418  << _bio->get_port() << "\n";
419  } else {
420  downloader_cat.debug()
421  << _NOTIFY_HTTP_CHANNEL_ID
422  << "Connecting to " << _bio->get_server_name() << ":"
423  << _bio->get_port() << "\n";
424  }
425  }
426 
427  _state = S_connecting;
428  _started_connecting_time =
429  TrueClock::get_global_ptr()->get_short_time();
430  _connect_count++;
431  }
432 
433  /*
434  if (downloader_cat.is_spam()) {
435  downloader_cat.spam()
436  << _NOTIFY_HTTP_CHANNEL_ID
437  << "continue run(), _state = " << _state << "\n";
438  }
439  */
440 
441  switch (_state) {
442  case S_try_next_proxy:
443  repeat_later = run_try_next_proxy();
444  break;
445 
446  case S_connecting:
447  repeat_later = run_connecting();
448  break;
449 
450  case S_connecting_wait:
451  repeat_later = run_connecting_wait();
452  break;
453 
454  case S_http_proxy_ready:
455  repeat_later = run_http_proxy_ready();
456  break;
457 
458  case S_http_proxy_request_sent:
459  repeat_later = run_http_proxy_request_sent();
460  break;
461 
462  case S_http_proxy_reading_header:
463  repeat_later = run_http_proxy_reading_header();
464  break;
465 
466  case S_socks_proxy_greet:
467  repeat_later = run_socks_proxy_greet();
468  break;
469 
470  case S_socks_proxy_greet_reply:
471  repeat_later = run_socks_proxy_greet_reply();
472  break;
473 
474  case S_socks_proxy_connect:
475  repeat_later = run_socks_proxy_connect();
476  break;
477 
478  case S_socks_proxy_connect_reply:
479  repeat_later = run_socks_proxy_connect_reply();
480  break;
481 
482  case S_setup_ssl:
483  repeat_later = run_setup_ssl();
484  break;
485 
486  case S_ssl_handshake:
487  repeat_later = run_ssl_handshake();
488  break;
489 
490  case S_ready:
491  repeat_later = run_ready();
492  break;
493 
494  case S_request_sent:
495  repeat_later = run_request_sent();
496  break;
497 
498  case S_reading_header:
499  repeat_later = run_reading_header();
500  break;
501 
502  case S_start_direct_file_read:
503  repeat_later = run_start_direct_file_read();
504  break;
505 
506  case S_read_header:
507  repeat_later = run_read_header();
508  break;
509 
510  case S_begin_body:
511  repeat_later = run_begin_body();
512  break;
513 
514  case S_reading_body:
515  repeat_later = run_reading_body();
516  break;
517 
518  case S_read_body:
519  repeat_later = run_read_body();
520  break;
521 
522  case S_read_trailer:
523  repeat_later = run_read_trailer();
524  break;
525 
526  default:
527  downloader_cat.warning()
528  << _NOTIFY_HTTP_CHANNEL_ID
529  << "Unhandled state " << _state << "\n";
530  return false;
531  }
532 
533  if (_state == _done_state || _state == S_failure) {
534  clear_extra_headers();
535  // We've reached our terminal state.
536  return reached_done_state();
537  }
538  thread_consider_yield();
539  } while (!repeat_later || _bio.is_null());
540 
541  /*
542  if (downloader_cat.is_spam()) {
543  downloader_cat.spam()
544  << _NOTIFY_HTTP_CHANNEL_ID
545  << "later run(), _state = " << _state
546  << ", _done_state = " << _done_state << "\n";
547  }
548  */
549 
550  thread_yield();
551  return true;
552 }
553 
554 ////////////////////////////////////////////////////////////////////
555 // Function: HTTPChannel::open_read_body
556 // Access: Published
557 // Description: Returns a newly-allocated istream suitable for
558 // reading the body of the document. This may only be
559 // called immediately after a call to get_document() or
560 // post_form(), or after a call to run() has returned
561 // false.
562 //
563 // Note that, in nonblocking mode, the returned stream
564 // may report an early EOF, even before the actual end
565 // of file. When this happens, you should call
566 // stream->is_closed() to determine whether you should
567 // attempt to read some more later.
568 //
569 // The user is responsible for passing the returned
570 // istream to close_read_body() later.
571 ////////////////////////////////////////////////////////////////////
572 ISocketStream *HTTPChannel::
573 open_read_body() {
574  reset_body_stream();
575 
576  if ((_state != S_read_header && _state != S_begin_body) || _source.is_null()) {
577  return NULL;
578  }
579 
580  string transfer_coding = downcase(get_header_value("Transfer-Encoding"));
581 
582  ISocketStream *result;
583  if (transfer_coding == "chunked") {
584  // "chunked" transfer encoding. This means we will have to decode
585  // the length of the file as we read it in chunks. The
586  // IChunkedStream does this.
587  _state = S_reading_body;
588  _read_index++;
589  result = new IChunkedStream(_source, this);
590 
591  } else {
592  // If the transfer encoding is anything else, assume "identity".
593  // This is just the literal characters following the header, up
594  // until _file_size bytes have been read (if content-length was
595  // specified), or till end of file otherwise.
596  _state = S_reading_body;
597  _read_index++;
598  result = new IIdentityStream(_source, this, _got_file_size, _file_size);
599  }
600 
601  result->_channel = this;
602  _body_stream = result;
603  _owns_body_stream = false;
604 
605  return result;
606 }
607 
608 ////////////////////////////////////////////////////////////////////
609 // Function: HTTPChannel::close_read_body
610 // Access: Public
611 // Description: Closes a file opened by a previous call to
612 // open_read_body(). This really just deletes the
613 // istream pointer, but it is recommended to use this
614 // interface instead of deleting it explicitly, to help
615 // work around compiler issues.
616 ////////////////////////////////////////////////////////////////////
617 void HTTPChannel::
618 close_read_body(istream *stream) const {
619  if (stream != (istream *)NULL) {
620  // For some reason--compiler bug in gcc 3.2?--explicitly deleting
621  // the stream pointer does not call the appropriate global delete
622  // function; instead apparently calling the system delete
623  // function. So we call the delete function by hand instead.
624 #if !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW)
625  stream->~istream();
626  (*global_operator_delete)(stream);
627 #else
628  delete stream;
629 #endif
630  }
631 }
632 
633 ////////////////////////////////////////////////////////////////////
634 // Function: HTTPChannel::download_to_file
635 // Access: Published
636 // Description: Specifies the name of a file to download the
637 // resulting document to. This should be called
638 // immediately after get_document() or
639 // begin_get_document() or related functions.
640 //
641 // In the case of the blocking I/O methods like
642 // get_document(), this function will download the
643 // entire document to the file and return true if it was
644 // successfully downloaded, false otherwise.
645 //
646 // In the case of non-blocking I/O methods like
647 // begin_get_document(), this function simply indicates an
648 // intention to download to the indicated file. It
649 // returns true if the file can be opened for writing,
650 // false otherwise, but the contents will not be
651 // completely downloaded until run() has returned false.
652 // At this time, it is possible that a communications
653 // error will have left a partial file, so
654 // is_download_complete() may be called to test this.
655 //
656 // If subdocument_resumes is true and the document in
657 // question was previously requested as a subdocument
658 // (i.e. get_subdocument() with a first_byte value
659 // greater than zero), this will automatically seek to
660 // the appropriate byte within the file for writing the
661 // output. In this case, the file must already exist
662 // and must have at least first_byte bytes in it. If
663 // subdocument_resumes is false, a subdocument will
664 // always be downloaded beginning at the first byte of
665 // the file.
666 ////////////////////////////////////////////////////////////////////
667 bool HTTPChannel::
668 download_to_file(const Filename &filename, bool subdocument_resumes) {
669  reset_download_to();
670  _download_to_filename = filename;
671  _download_to_filename.set_binary();
672  _subdocument_resumes = subdocument_resumes;
673 
674  _download_dest = DD_file;
675 
676  if (_wanted_nonblocking && _state != S_read_header) {
677  // In nonblocking mode, we can't start the download yet; that will
678  // be done later as run() is called.
679  return true;
680  }
681 
682  // In normal, blocking mode, go ahead and do the download.
683  if (!open_download_file()) {
684  reset_download_to();
685  return false;
686  }
687 
688  while (run()) {
689  }
690  return is_download_complete() && is_valid();
691 }
692 
693 ////////////////////////////////////////////////////////////////////
694 // Function: HTTPChannel::download_to_ram
695 // Access: Published
696 // Description: Specifies a Ramfile object to download the
697 // resulting document to. This should be called
698 // immediately after get_document() or
699 // begin_get_document() or related functions.
700 //
701 // In the case of the blocking I/O methods like
702 // get_document(), this function will download the
703 // entire document to the Ramfile and return true if it
704 // was successfully downloaded, false otherwise.
705 //
706 // In the case of non-blocking I/O methods like
707 // begin_get_document(), this function simply indicates an
708 // intention to download to the indicated Ramfile. It
709 // returns true if the file can be opened for writing,
710 // false otherwise, but the contents will not be
711 // completely downloaded until run() has returned false.
712 // At this time, it is possible that a communications
713 // error will have left a partial file, so
714 // is_download_complete() may be called to test this.
715 //
716 // If subdocument_resumes is true and the document in
717 // question was previously requested as a subdocument
718 // (i.e. get_subdocument() with a first_byte value
719 // greater than zero), this will automatically seek to
720 // the appropriate byte within the Ramfile for writing
721 // the output. In this case, the Ramfile must already
722 // have at least first_byte bytes in it.
723 ////////////////////////////////////////////////////////////////////
724 bool HTTPChannel::
725 download_to_ram(Ramfile *ramfile, bool subdocument_resumes) {
726  nassertr(ramfile != (Ramfile *)NULL, false);
727  reset_download_to();
728  ramfile->_pos = 0;
729  _download_to_ramfile = ramfile;
730  _download_dest = DD_ram;
731  _subdocument_resumes = (subdocument_resumes && _first_byte_delivered != 0);
732 
733  if (_wanted_nonblocking && _state != S_read_header) {
734  // In nonblocking mode, we can't start the download yet; that will
735  // be done later as run() is called.
736  return true;
737  }
738 
739  // In normal, blocking mode, go ahead and do the download.
740  if (!open_download_file()) {
741  reset_download_to();
742  return false;
743  }
744 
745  while (run()) {
746  }
747  return is_download_complete() && is_valid();
748 }
749 
750 ////////////////////////////////////////////////////////////////////
751 // Function: HTTPChannel::download_to_stream
752 // Access: Published
753 // Description: Specifies the name of an ostream to download the
754 // resulting document to. This should be called
755 // immediately after get_document() or
756 // begin_get_document() or related functions.
757 //
758 // In the case of the blocking I/O methods like
759 // get_document(), this function will download the
760 // entire document to the file and return true if it was
761 // successfully downloaded, false otherwise.
762 //
763 // In the case of non-blocking I/O methods like
764 // begin_get_document(), this function simply indicates an
765 // intention to download to the indicated file. It
766 // returns true if the file can be opened for writing,
767 // false otherwise, but the contents will not be
768 // completely downloaded until run() has returned false.
769 // At this time, it is possible that a communications
770 // error will have left a partial file, so
771 // is_download_complete() may be called to test this.
772 //
773 // If subdocument_resumes is true and the document in
774 // question was previously requested as a subdocument
775 // (i.e. get_subdocument() with a first_byte value
776 // greater than zero), this will automatically seek to
777 // the appropriate byte within the file for writing the
778 // output. In this case, the file must already exist
779 // and must have at least first_byte bytes in it. If
780 // subdocument_resumes is false, a subdocument will
781 // always be downloaded beginning at the first byte of
782 // the file.
783 ////////////////////////////////////////////////////////////////////
784 bool HTTPChannel::
785 download_to_stream(ostream *strm, bool subdocument_resumes) {
786  reset_download_to();
787  _download_to_stream = strm;
788  _download_to_stream->clear();
789  _subdocument_resumes = subdocument_resumes;
790 
791  _download_dest = DD_stream;
792 
793  if (_wanted_nonblocking && _state != S_read_header) {
794  // In nonblocking mode, we can't start the download yet; that will
795  // be done later as run() is called.
796  return true;
797  }
798 
799  // In normal, blocking mode, go ahead and do the download.
800  if (!open_download_file()) {
801  reset_download_to();
802  return false;
803  }
804 
805  while (run()) {
806  }
807  return is_download_complete() && is_valid();
808 }
809 
810 ////////////////////////////////////////////////////////////////////
811 // Function: HTTPChannel::get_connection
812 // Access: Published
813 // Description: Returns the connection that was established via a
814 // previous call to connect_to() or begin_connect_to(),
815 // or NULL if the connection attempt failed or if those
816 // methods have not recently been called.
817 //
818 // This stream has been allocated from the free store.
819 // It is the user's responsibility to delete this
820 // pointer when finished with it.
821 ////////////////////////////////////////////////////////////////////
822 SocketStream *HTTPChannel::
823 get_connection() {
824  if (!is_connection_ready()) {
825  return NULL;
826  }
827 
828  BioStream *stream = _source->get_stream();
829  _source->set_stream(NULL);
830 
831  // We're now passing ownership of the connection to the caller.
832  if (downloader_cat.is_debug()) {
833  downloader_cat.debug()
834  << _NOTIFY_HTTP_CHANNEL_ID
835  << "passing ownership of connection to caller.\n";
836  }
837  reset_to_new();
838 
839  return stream;
840 }
841 
842 ////////////////////////////////////////////////////////////////////
843 // Function: HTTPChannel::downcase
844 // Access: Public, Static
845 // Description: Returns the input string with all uppercase letters
846 // converted to lowercase.
847 ////////////////////////////////////////////////////////////////////
848 string HTTPChannel::
849 downcase(const string &s) {
850  string result;
851  result.reserve(s.size());
852  string::const_iterator p;
853  for (p = s.begin(); p != s.end(); ++p) {
854  result += tolower(*p);
855  }
856  return result;
857 }
858 
859 ////////////////////////////////////////////////////////////////////
860 // Function: HTTPChannel::body_stream_destructs
861 // Access: Public
862 // Description: Called by ISocketStream destructor when _body_stream
863 // is destructing.
864 ////////////////////////////////////////////////////////////////////
865 void HTTPChannel::
866 body_stream_destructs(ISocketStream *stream) {
867  if (stream == _body_stream) {
868  if (_state == S_reading_body) {
869  switch (_body_stream->get_read_state()) {
870  case ISocketStream::RS_complete:
871  finished_body(false);
872  break;
873 
874  case ISocketStream::RS_error:
875  _state = HTTPChannel::S_failure;
876  _status_entry._status_code = HTTPChannel::SC_lost_connection;
877  break;
878 
879  default:
880  break;
881  }
882  }
883  _body_stream = NULL;
884  _owns_body_stream = false;
885  }
886 }
887 
888 
889 ////////////////////////////////////////////////////////////////////
890 // Function: HTTPChannel::reached_done_state
891 // Access: Private
892 // Description: Called by run() after it reaches the done state, this
893 // simply checks to see if a download was requested, and
894 // begins the download if it has been.
895 ////////////////////////////////////////////////////////////////////
896 bool HTTPChannel::
897 reached_done_state() {
898  /*
899  if (downloader_cat.is_spam()) {
900  downloader_cat.spam()
901  << _NOTIFY_HTTP_CHANNEL_ID
902  << "terminating run(), _state = " << _state
903  << ", _done_state = " << _done_state << "\n";
904  }
905  */
906 
907  if (_state == S_failure) {
908  // We had to give up. Each proxy we tried, in sequence, failed.
909  // But maybe the last attempt didn't give us the most informative
910  // response; go back and find the best one.
911  if (!_status_list.empty()) {
912  _status_list.push_back(_status_entry);
913  if (downloader_cat.is_debug()) {
914  downloader_cat.debug()
915  << _NOTIFY_HTTP_CHANNEL_ID
916  << "Reexamining failure responses.\n";
917  }
918  size_t best_i = 0;
919  if (downloader_cat.is_debug()) {
920  downloader_cat.debug()
921  << _NOTIFY_HTTP_CHANNEL_ID
922  << " " << 0 << ". " << _status_list[0]._status_code << " "
923  << _status_list[0]._status_string << "\n";
924  }
925  for (size_t i = 1; i < _status_list.size(); i++) {
926  if (downloader_cat.is_debug()) {
927  downloader_cat.debug()
928  << _NOTIFY_HTTP_CHANNEL_ID
929  << " " << i << ". " << _status_list[i]._status_code << " "
930  << _status_list[i]._status_string << "\n";
931  }
932  if (more_useful_status_code(_status_list[i]._status_code,
933  _status_list[best_i]._status_code)) {
934  best_i = i;
935  }
936  }
937  if (downloader_cat.is_debug()) {
938  downloader_cat.debug()
939  << _NOTIFY_HTTP_CHANNEL_ID
940  << "chose index " << best_i << ", above.\n";
941  }
942  _status_entry = _status_list[best_i];
943  _status_list.clear();
944  }
945 
946  return false;
947  }
948 
949  // We don't need the list of previous failures any more--we've connected.
950  _status_list.clear();
951 
952  if (_download_dest == DD_none) {
953  // All done.
954  return false;
955 
956  } else {
957  // Oops, we have to download the body now.
958  open_read_body();
959  if (_body_stream == (ISocketStream *)NULL) {
960  if (downloader_cat.is_debug()) {
961  downloader_cat.debug()
962  << _NOTIFY_HTTP_CHANNEL_ID
963  << "Unable to download body: " << _request.get_url() << "\n";
964  }
965  return false;
966 
967  } else {
968  _owns_body_stream = true;
969  if (_state != S_reading_body) {
970  reset_body_stream();
971  }
972  _started_download = true;
973 
974  _done_state = S_read_trailer;
975  _last_run_time = TrueClock::get_global_ptr()->get_short_time();
976  return true;
977  }
978  }
979 }
980 
981 ////////////////////////////////////////////////////////////////////
982 // Function: HTTPChannel::run_try_next_proxy
983 // Access: Private
984 // Description: This state is reached when a previous connection
985 // attempt fails. If we have multiple proxies in line
986 // to try, it sets us up for the next proxy and tries to
987 // connect again; otherwise, it sets the state to
988 // S_failure.
989 ////////////////////////////////////////////////////////////////////
990 bool HTTPChannel::
991 run_try_next_proxy() {
992  if (_proxy_next_index < _proxies.size()) {
993  // Record the previous proxy's status entry, so we can come back
994  // to it later if we get nonsense from the remaining proxies.
995  _status_list.push_back(_status_entry);
996  _status_entry = StatusEntry();
997 
998  // Now try the next proxy in sequence.
999  _proxy = _proxies[_proxy_next_index];
1000  _proxy_auth = (HTTPAuthorization *)NULL;
1001  _proxy_next_index++;
1002  close_connection();
1003  reconsider_proxy();
1004  _state = S_connecting;
1005 
1006  return false;
1007  }
1008 
1009  // No more proxies to try, or we're not using a proxy.
1010  _state = S_failure;
1011  return false;
1012 }
1013 
1014 ////////////////////////////////////////////////////////////////////
1015 // Function: HTTPChannel::run_connecting
1016 // Access: Private
1017 // Description: In this state, we have not yet established a
1018 // network connection to the server (or proxy).
1019 ////////////////////////////////////////////////////////////////////
1020 bool HTTPChannel::
1021 run_connecting() {
1022  _status_entry = StatusEntry();
1023 
1024  if (BIO_do_connect(*_bio) <= 0) {
1025  if (BIO_should_retry(*_bio)) {
1026  _state = S_connecting_wait;
1027  return false;
1028  }
1029  downloader_cat.info()
1030  << _NOTIFY_HTTP_CHANNEL_ID
1031  << "Could not connect to " << _bio->get_server_name() << ":"
1032  << _bio->get_port() << "\n";
1033  OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
1034  _status_entry._status_code = SC_no_connection;
1035  _state = S_try_next_proxy;
1036  return false;
1037  }
1038 
1039  if (downloader_cat.is_debug()) {
1040  downloader_cat.debug()
1041  << _NOTIFY_HTTP_CHANNEL_ID
1042  << "Connected to " << _bio->get_server_name() << ":"
1043  << _bio->get_port() << "\n";
1044  }
1045 
1046  if (_proxy_tunnel_now) {
1047  if (_proxy.get_scheme() == "socks") {
1048  _state = S_socks_proxy_greet;
1049  } else {
1050  _state = S_http_proxy_ready;
1051  }
1052 
1053  } else {
1054  if (_want_ssl) {
1055  _state = S_setup_ssl;
1056  } else {
1057  _state = S_ready;
1058  }
1059  }
1060  return false;
1061 }
1062 
1063 ////////////////////////////////////////////////////////////////////
1064 // Function: HTTPChannel::run_connecting_wait
1065 // Access: Private
1066 // Description: Here we have begun to establish a nonblocking
1067 // connection, but we got a come-back-later message, so
1068 // we are waiting for the socket to finish connecting.
1069 ////////////////////////////////////////////////////////////////////
1070 bool HTTPChannel::
1071 run_connecting_wait() {
1072  int fd = -1;
1073  BIO_get_fd(*_bio, &fd);
1074  if (fd < 0) {
1075  downloader_cat.warning()
1076  << _NOTIFY_HTTP_CHANNEL_ID
1077  << "nonblocking socket BIO has no file descriptor.\n";
1078  // This shouldn't be possible.
1079  _status_entry._status_code = SC_internal_error;
1080  _state = S_try_next_proxy;
1081  return false;
1082  }
1083 
1084  if (downloader_cat.is_spam()) {
1085  downloader_cat.spam()
1086  << _NOTIFY_HTTP_CHANNEL_ID
1087  << "waiting to connect to " << _request.get_url().get_server_and_port() << ".\n";
1088  }
1089  fd_set wset;
1090  FD_ZERO(&wset);
1091  FD_SET(fd, &wset);
1092  struct timeval tv;
1093  if (get_blocking_connect()) {
1094  // Since we'll be blocking on this connect, fill in the timeout
1095  // into the structure.
1096  tv.tv_sec = (int)_connect_timeout;
1097  tv.tv_usec = (int)((_connect_timeout - tv.tv_sec) * 1000000.0);
1098  } else {
1099  // We won't block on this connect, so select() for 0 time.
1100  tv.tv_sec = 0;
1101  tv.tv_usec = 0;
1102  }
1103  int errcode = select(fd + 1, NULL, &wset, NULL, &tv);
1104  if (errcode < 0) {
1105  downloader_cat.warning()
1106  << _NOTIFY_HTTP_CHANNEL_ID
1107  << "Error in select.\n";
1108  // This shouldn't be possible.
1109  _status_entry._status_code = SC_internal_error;
1110  _state = S_try_next_proxy;
1111  return false;
1112  }
1113 
1114  if (errcode == 0) {
1115  // Nothing's happened so far; come back later.
1116  if (get_blocking_connect() ||
1117  (TrueClock::get_global_ptr()->get_short_time() -
1118  _started_connecting_time > get_connect_timeout())) {
1119  // Time to give up.
1120  downloader_cat.info()
1121  << _NOTIFY_HTTP_CHANNEL_ID
1122  << "Timeout connecting to "
1123  << _request.get_url().get_server_and_port()
1124  << " for " << _request.get_url()
1125  << ".\n";
1126  _status_entry._status_code = SC_timeout;
1127  _state = S_try_next_proxy;
1128  return false;
1129  }
1130  return true;
1131  }
1132 
1133  // The socket is now ready for writing.
1134  _state = S_connecting;
1135  return false;
1136 }
1137 
1138 
1139 ////////////////////////////////////////////////////////////////////
1140 // Function: HTTPChannel::run_http_proxy_ready
1141 // Access: Private
1142 // Description: This state is reached only after first establishing a
1143 // connection to the proxy, if a proxy is in use and we
1144 // are tunneling through it via a CONNECT command.
1145 ////////////////////////////////////////////////////////////////////
1146 bool HTTPChannel::
1147 run_http_proxy_ready() {
1148  // If there's a request to be sent to the proxy, send it now.
1149  nassertr(!_proxy_request_text.empty(), false);
1150  if (!server_send(_proxy_request_text, false)) {
1151  return true;
1152  }
1153 
1154  // All done sending request.
1155  _state = S_http_proxy_request_sent;
1156  _sent_request_time = TrueClock::get_global_ptr()->get_short_time();
1157  return false;
1158 }
1159 
1160 ////////////////////////////////////////////////////////////////////
1161 // Function: HTTPChannel::run_http_proxy_request_sent
1162 // Access: Private
1163 // Description: This state is reached only after we have sent a
1164 // special message to the proxy and we are waiting for
1165 // the proxy's response. It is not used in the normal
1166 // http-over-proxy case, which does not require a
1167 // special message to the proxy.
1168 ////////////////////////////////////////////////////////////////////
1169 bool HTTPChannel::
1170 run_http_proxy_request_sent() {
1171  // Wait for the first line to come back from the server.
1172  string line;
1173  if (!server_getline_failsafe(line)) {
1174  return true;
1175  }
1176 
1177  // Skip unexpected blank lines. We're just being generous here.
1178  while (line.empty()) {
1179  if (!server_getline_failsafe(line)) {
1180  return true;
1181  }
1182  }
1183 
1184  if (!parse_http_response(line)) {
1185  return false;
1186  }
1187 
1188  _state = S_http_proxy_reading_header;
1189  _current_field_name = string();
1190  _current_field_value = string();
1191  _headers.clear();
1192  _got_file_size = false;
1193  _got_transfer_file_size = false;
1194  return false;
1195 }
1196 
1197 ////////////////////////////////////////////////////////////////////
1198 // Function: HTTPChannel::run_http_proxy_reading_header
1199 // Access: Private
1200 // Description: In this state we are reading the header lines from
1201 // the proxy's response to our special message.
1202 ////////////////////////////////////////////////////////////////////
1203 bool HTTPChannel::
1204 run_http_proxy_reading_header() {
1205  if (parse_http_header()) {
1206  return true;
1207  }
1208 
1209  _redirect = get_header_value("Location");
1210  // We can take the proxy's word for it that this is the actual URL
1211  // for the redirect.
1212 
1213  _server_response_has_no_body =
1214  (get_status_code() / 100 == 1 ||
1215  get_status_code() == 204 ||
1216  get_status_code() == 304);
1217 
1218  int last_status = _last_status_code;
1219  _last_status_code = get_status_code();
1220 
1221  if (get_status_code() == 407 && last_status != 407 && !_proxy.empty()) {
1222  // 407: not authorized to proxy. Try to get the authorization.
1223  string authenticate_request = get_header_value("Proxy-Authenticate");
1224  _proxy_auth = _client->generate_auth(_proxy, true, authenticate_request);
1225  if (_proxy_auth != (HTTPAuthorization *)NULL) {
1226  _proxy_realm = _proxy_auth->get_realm();
1227  _proxy_username = _client->select_username(_proxy, true, _proxy_realm);
1228  if (!_proxy_username.empty()) {
1229  make_proxy_request_text();
1230 
1231  // Roll the state forward to force a new request.
1232  _state = S_begin_body;
1233  return false;
1234  }
1235  }
1236  }
1237 
1238  if (!is_valid()) {
1239  // Proxy wouldn't open connection.
1240 
1241  // Change some of the status codes a proxy might return to
1242  // differentiate them from similar status codes the destination
1243  // server might have returned.
1244  if (get_status_code() != 407) {
1245  _status_entry._status_code += 1000;
1246  }
1247 
1248  _state = S_try_next_proxy;
1249  return false;
1250  }
1251 
1252  // Now we have a tunnel opened through the proxy.
1253  make_request_text();
1254 
1255  if (_want_ssl) {
1256  _state = S_setup_ssl;
1257  } else {
1258  _state = S_ready;
1259  }
1260 
1261  return false;
1262 }
1263 
1264 ////////////////////////////////////////////////////////////////////
1265 // Function: HTTPChannel::run_socks_proxy_greet
1266 // Access: Private
1267 // Description: This state is reached only after first establishing a
1268 // connection to a SOCKS proxy, with which we now have
1269 // to negotiate a connection.
1270 ////////////////////////////////////////////////////////////////////
1271 bool HTTPChannel::
1272 run_socks_proxy_greet() {
1273  static const char socks_greeting[] = {
1274  0x05, // Socks version 5
1275  0x01, // Number of supported login methods
1276  0x00, // Login method 0: no authentication
1277  /*
1278  0x01, // Login method 1: GSSAPI
1279  0x02 // Login method 2: username/password
1280  */
1281  };
1282  static const int socks_greeting_len = sizeof(socks_greeting);
1283  if (!server_send(string(socks_greeting, socks_greeting_len), true)) {
1284  return true;
1285  }
1286  _sent_request_time = TrueClock::get_global_ptr()->get_short_time();
1287 
1288  // All done sending request.
1289  _state = S_socks_proxy_greet_reply;
1290  return false;
1291 }
1292 
1293 ////////////////////////////////////////////////////////////////////
1294 // Function: HTTPChannel::run_socks_proxy_greet_reply
1295 // Access: Private
1296 // Description: We are waiting for the SOCKS proxy to respond to our
1297 // greeting.
1298 ////////////////////////////////////////////////////////////////////
1299 bool HTTPChannel::
1300 run_socks_proxy_greet_reply() {
1301  string reply;
1302 
1303  // Get the two-byte reply from the SOCKS server.
1304  if (!server_get_failsafe(reply, 2)) {
1305  return true;
1306  }
1307 
1308  if (reply[0] != 0x05) {
1309  // We only speak Socks5.
1310  downloader_cat.info()
1311  << _NOTIFY_HTTP_CHANNEL_ID
1312  << "Rejecting Socks version " << (int)reply[0] << "\n";
1313  _status_entry._status_code = SC_socks_invalid_version;
1314  _state = S_try_next_proxy;
1315  return false;
1316  }
1317 
1318  if (reply[1] == (char)0xff) {
1319  downloader_cat.info()
1320  << _NOTIFY_HTTP_CHANNEL_ID
1321  << "Socks server does not accept our available login methods.\n";
1322  _status_entry._status_code = SC_socks_no_acceptable_login_method;
1323  _state = S_try_next_proxy;
1324  return false;
1325  }
1326 
1327  if (reply[1] == 0x00) {
1328  // No login method required. Proceed directly to the connect
1329  // message.
1330  _state = S_socks_proxy_connect;
1331  return false;
1332  }
1333 
1334  // The server accepted a login method we didn't offer!
1335  downloader_cat.info()
1336  << _NOTIFY_HTTP_CHANNEL_ID
1337  << "Socks server accepted unrequested login method "
1338  << (int)reply[1] << "\n";
1339  _status_entry._status_code = SC_socks_no_acceptable_login_method;
1340  _state = S_try_next_proxy;
1341  return false;
1342 }
1343 
1344 ////////////////////////////////////////////////////////////////////
1345 // Function: HTTPChannel::run_socks_proxy_connect
1346 // Access: Private
1347 // Description: The SOCKS proxy has accepted us, and now we may issue
1348 // the connect request.
1349 ////////////////////////////////////////////////////////////////////
1350 bool HTTPChannel::
1351 run_socks_proxy_connect() {
1352  static const char socks_connect[] = {
1353  0x05, // Socks version 5
1354  0x01, // Command 1: connect
1355  0x00, // reserved
1356  0x03, // DNS name
1357  };
1358  static const int socks_connect_len = sizeof(socks_connect);
1359 
1360  string hostname = _request.get_url().get_server();
1361  int port = _request.get_url().get_port();
1362 
1363  if (downloader_cat.is_debug()) {
1364  downloader_cat.debug()
1365  << _NOTIFY_HTTP_CHANNEL_ID
1366  << "Requesting SOCKS5 connection to "
1367  << _request.get_url().get_server_and_port() << "\n";
1368  }
1369 
1370  string connect =
1371  string(socks_connect, socks_connect_len) +
1372  string(1, (char)hostname.length()) +
1373  hostname +
1374  string(1, (char)((port >> 8) & 0xff)) +
1375  string(1, (char)(port & 0xff));
1376 
1377  if (!server_send(connect, true)) {
1378  return true;
1379  }
1380  _sent_request_time = TrueClock::get_global_ptr()->get_short_time();
1381 
1382  _state = S_socks_proxy_connect_reply;
1383  return false;
1384 }
1385 
1386 ////////////////////////////////////////////////////////////////////
1387 // Function: HTTPChannel::run_socks_proxy_connect_reply
1388 // Access: Private
1389 // Description: We are waiting for the SOCKS proxy to honor our
1390 // connect request.
1391 ////////////////////////////////////////////////////////////////////
1392 bool HTTPChannel::
1393 run_socks_proxy_connect_reply() {
1394  string reply;
1395 
1396  // Get the first two bytes of the connect reply.
1397  if (!server_get_failsafe(reply, 2)) {
1398  return true;
1399  }
1400 
1401  if (reply[0] != 0x05) {
1402  // We only speak Socks5.
1403  downloader_cat.info()
1404  << _NOTIFY_HTTP_CHANNEL_ID
1405  << "Rejecting Socks version " << (int)reply[0] << "\n";
1406  close_connection(); // connection is now bad.
1407  _status_entry._status_code = SC_socks_invalid_version;
1408  _state = S_try_next_proxy;
1409  return false;
1410  }
1411 
1412  if (reply[1] != 0x00) {
1413  downloader_cat.info()
1414  << _NOTIFY_HTTP_CHANNEL_ID
1415  << "Connection refused, SOCKS code " << (int)reply[1] << "\n";
1416  /*
1417  Socks error codes (from RFC1928):
1418  o X'00' succeeded
1419  o X'01' general SOCKS server failure
1420  o X'02' connection not allowed by ruleset
1421  o X'03' Network unreachable
1422  o X'04' Host unreachable
1423  o X'05' Connection refused
1424  o X'06' TTL expired
1425  o X'07' Command not supported
1426  o X'08' Address type not supported
1427  o X'09' to X'FF' unassigned
1428  */
1429 
1430  switch (reply[1]) {
1431  case 0x03:
1432  case 0x04:
1433  case 0x05:
1434  case 0x06:
1435  // These generally mean the same thing: the SOCKS proxy tried,
1436  // but couldn't reach the host.
1437  _status_entry._status_code = SC_socks_no_connection;
1438  break;
1439 
1440  default:
1441  _status_entry._status_code = SC_socks_refused;
1442  }
1443 
1444  close_connection(); // connection is now bad.
1445  _state = S_try_next_proxy;
1446  return false;
1447  }
1448 
1449  // Now put those bytes back, and get five bytes of the reply.
1450  _working_get = reply;
1451  if (!server_get_failsafe(reply, 5)) {
1452  return true;
1453  }
1454 
1455  // Figure out how many bytes total we will expect for the reply.
1456  int total_bytes = 6;
1457 
1458  switch (reply[3]) {
1459  case 0x01: // IPv4
1460  total_bytes += 4;
1461  break;
1462 
1463  case 0x03: // DNS
1464  total_bytes += (unsigned int)reply[4];
1465  break;
1466 
1467  default:
1468  downloader_cat.info()
1469  << _NOTIFY_HTTP_CHANNEL_ID
1470  << "Unsupported SOCKS address type: " << (int)reply[3] << "\n";
1471  _status_entry._status_code = SC_socks_invalid_version;
1472  _state = S_try_next_proxy;
1473  return false;
1474  }
1475 
1476  // Now put back the bytes we've read so far, and get the rest of
1477  // them.
1478  _working_get = reply;
1479  if (!server_get_failsafe(reply, total_bytes)) {
1480  return true;
1481  }
1482 
1483  if (downloader_cat.is_debug()) {
1484  // Finally, we can decode the whole thing.
1485  string connect_host;
1486 
1487  switch (reply[3]) {
1488  case 0x01: // IPv4
1489  {
1490  ostringstream strm;
1491  strm << (unsigned int)(unsigned char)reply[4] << "."
1492  << (unsigned int)(unsigned char)reply[5] << "."
1493  << (unsigned int)(unsigned char)reply[6] << "."
1494  << (unsigned int)(unsigned char)reply[7];
1495  connect_host = strm.str();
1496  }
1497  break;
1498 
1499  case 0x03: // DNS
1500  connect_host = string(&reply[5], (unsigned int)reply[4]);
1501  break;
1502  }
1503 
1504  int connect_port =
1505  (((unsigned int)(unsigned char)reply[total_bytes - 2]) << 8) |
1506  ((unsigned int)(unsigned char)reply[total_bytes - 1]);
1507 
1508  downloader_cat.debug()
1509  << _NOTIFY_HTTP_CHANNEL_ID
1510  << _proxy << " directed us to " << connect_host << ":"
1511  << connect_port << "\n";
1512  }
1513 
1514  if (_want_ssl) {
1515  _state = S_setup_ssl;
1516  } else {
1517  _state = S_ready;
1518  }
1519 
1520  return false;
1521 }
1522 
1523 ////////////////////////////////////////////////////////////////////
1524 // Function: HTTPChannel::run_setup_ssl
1525 // Access: Private
1526 // Description: This state begins elevating our existing, unsecure
1527 // connection to a secure, SSL connection.
1528 ////////////////////////////////////////////////////////////////////
1529 bool HTTPChannel::
1530 run_setup_ssl() {
1531  _sbio = BIO_new_ssl(_client->get_ssl_ctx(), true);
1532  BIO_push(_sbio, *_bio);
1533 
1534  SSL *ssl = NULL;
1535  BIO_get_ssl(_sbio, &ssl);
1536  nassertr(ssl != (SSL *)NULL, false);
1537 
1538  // We only take one word at a time from the _cipher_list. If that
1539  // connection fails, then we take the next word.
1540  string cipher_list = _cipher_list;
1541  if (!cipher_list.empty()) {
1542  size_t space = cipher_list.find(" ");
1543  if (space != string::npos) {
1544  cipher_list = cipher_list.substr(0, space);
1545  }
1546  }
1547 
1548  if (downloader_cat.is_debug()) {
1549  downloader_cat.debug()
1550  << _NOTIFY_HTTP_CHANNEL_ID
1551  << "Setting ssl-cipher-list '" << cipher_list << "'\n";
1552  }
1553  int result = SSL_set_cipher_list(ssl, cipher_list.c_str());
1554  if (result == 0) {
1555  downloader_cat.error()
1556  << _NOTIFY_HTTP_CHANNEL_ID
1557  << "Invalid cipher list: '" << cipher_list << "'\n";
1558  OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
1559  _status_entry._status_code = SC_ssl_internal_failure;
1560  _state = S_failure;
1561  return false;
1562  }
1563 
1564  // It would be nice to use something like SSL_set_client_cert_cb()
1565  // here to set a callback to provide the certificate should it be
1566  // requested, or even to potentially provide any of a number of
1567  // certificates according to the server's CA presented, but that
1568  // interface as provided by OpenSSL is broken since there's no way
1569  // to pass additional data to the callback function (and hence no
1570  // way to tie it back to the HTTPChannel object, other than by
1571  // building a messy mapping of SSL pointers back to HTTPChannel
1572  // pointers).
1573  if (_client->load_client_certificate()) {
1574  SSL_use_certificate(ssl, _client->_client_certificate_pub);
1575  SSL_use_PrivateKey(ssl, _client->_client_certificate_priv);
1576  if (!SSL_check_private_key(ssl)) {
1577  downloader_cat.warning()
1578  << _NOTIFY_HTTP_CHANNEL_ID
1579  << "Client private key does not match public key!\n";
1580  }
1581  }
1582 
1583  if (downloader_cat.is_spam()) {
1584  downloader_cat.spam()
1585  << _NOTIFY_HTTP_CHANNEL_ID
1586  << "SSL Ciphers available:\n";
1587  const char *name;
1588  int pri = 0;
1589  name = SSL_get_cipher_list(ssl, pri);
1590  while (name != NULL) {
1591  downloader_cat.spam()
1592  << _NOTIFY_HTTP_CHANNEL_ID
1593  << " " << pri + 1 << ". " << name << "\n";
1594  pri++;
1595  name = SSL_get_cipher_list(ssl, pri);
1596  }
1597  }
1598 
1599  if (downloader_cat.is_debug()) {
1600  downloader_cat.debug()
1601  << _NOTIFY_HTTP_CHANNEL_ID
1602  << "performing SSL handshake\n";
1603  }
1604  _state = S_ssl_handshake;
1605 
1606  // We start the connect timer over again when we reach the SSL
1607  // handshake.
1608  _started_connecting_time =
1609  TrueClock::get_global_ptr()->get_short_time();
1610 
1611  return false;
1612 }
1613 
1614 ////////////////////////////////////////////////////////////////////
1615 // Function: HTTPChannel::run_ssl_handshake
1616 // Access: Private
1617 // Description: This state performs the SSL handshake with the
1618 // server, and also verifies the server's identity when
1619 // the handshake has successfully completed.
1620 ////////////////////////////////////////////////////////////////////
1621 bool HTTPChannel::
1622 run_ssl_handshake() {
1623  if (BIO_do_handshake(_sbio) <= 0) {
1624  if (BIO_should_retry(_sbio)) {
1625  double elapsed =
1626  TrueClock::get_global_ptr()->get_short_time() -
1627  _started_connecting_time;
1628  if (elapsed <= get_connect_timeout() + _extra_ssl_handshake_time) {
1629  // Keep trying.
1630  return true;
1631  }
1632  // Time to give up on the handshake.
1633  }
1634 
1635  downloader_cat.info()
1636  << _NOTIFY_HTTP_CHANNEL_ID
1637  << "Could not establish SSL handshake with "
1638  << _request.get_url().get_server_and_port() << "\n";
1639  OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
1640 
1641  // It seems to be an error to free sbio at this point; perhaps
1642  // it's already been freed?
1643 
1644  if (!_cipher_list.empty()) {
1645  // If we've got another cipher to try, do so.
1646  size_t space = _cipher_list.find(" ");
1647  if (space != string::npos) {
1648  while (space < _cipher_list.length() && _cipher_list[space] == ' ') {
1649  ++space;
1650  }
1651  _cipher_list = _cipher_list.substr(space);
1652  if (!_cipher_list.empty()) {
1653  close_connection();
1654  reconsider_proxy();
1655  _state = S_connecting;
1656  return false;
1657  }
1658  }
1659  }
1660 
1661  // All done trying ciphers; they all failed.
1662  _cipher_list = _client->get_cipher_list();
1663  _status_entry._status_code = SC_ssl_no_handshake;
1664  _state = S_failure;
1665  return false;
1666  }
1667 
1668  SSL *ssl = NULL;
1669  BIO_get_ssl(_sbio, &ssl);
1670  nassertr(ssl != (SSL *)NULL, false);
1671 
1672  if (!_nonblocking) {
1673  SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
1674  }
1675 
1676  const SSL_CIPHER *cipher = SSL_get_current_cipher(ssl);
1677  if (cipher == (const SSL_CIPHER *)NULL) {
1678  downloader_cat.warning()
1679  << _NOTIFY_HTTP_CHANNEL_ID
1680  << "No current cipher on SSL connection.\n";
1681  } else {
1682  if (downloader_cat.is_debug()) {
1683  downloader_cat.debug()
1684  << _NOTIFY_HTTP_CHANNEL_ID
1685  << "Using cipher " << SSL_CIPHER_get_name((SSL_CIPHER *) cipher) << "\n";
1686  }
1687  }
1688 
1689  // Now that we've made an SSL handshake, we can use the SSL bio to
1690  // do all of our communication henceforth.
1691  _bio->set_bio(_sbio);
1692  _sbio = NULL;
1693 
1694  X509 *cert = SSL_get_peer_certificate(ssl);
1695  if (cert == (X509 *)NULL) {
1696  downloader_cat.info()
1697  << _NOTIFY_HTTP_CHANNEL_ID
1698  << "No certificate was presented by server.\n";
1699 
1700  // This shouldn't be possible, per the SSL specs.
1701  _status_entry._status_code = SC_ssl_invalid_server_certificate;
1702  _state = S_failure;
1703  return false;
1704  }
1705 
1706  X509_NAME *subject = X509_get_subject_name(cert);
1707  if (downloader_cat.is_debug()) {
1708  string org_name = get_x509_name_component(subject, NID_organizationName);
1709  string org_unit_name = get_x509_name_component(subject, NID_organizationalUnitName);
1710  string common_name = get_x509_name_component(subject, NID_commonName);
1711 
1712  downloader_cat.debug()
1713  << _NOTIFY_HTTP_CHANNEL_ID
1714  << "Server is " << common_name << " from " << org_unit_name
1715  << " / " << org_name << "\n";
1716 
1717  if (downloader_cat.is_spam()) {
1718  downloader_cat.spam()
1719  << _NOTIFY_HTTP_CHANNEL_ID
1720  << "Received certificate from server:\n" << flush;
1721  X509_print_fp(stderr, cert);
1722  fflush(stderr);
1723  }
1724  }
1725 
1726  bool cert_preapproved = false;
1727  bool cert_name_preapproved = false;
1728  check_preapproved_server_certificate(cert, cert_preapproved, cert_name_preapproved);
1729 
1730  // Now verify the server certificate is valid.
1731  long verify_result = SSL_get_verify_result(ssl);
1732  bool cert_valid = true;
1733 
1734  if (verify_result == X509_V_ERR_CERT_HAS_EXPIRED) {
1735  downloader_cat.info()
1736  << _NOTIFY_HTTP_CHANNEL_ID
1737  << "Expired certificate from " << _request.get_url().get_server_and_port() << "\n";
1738  if (_client->get_verify_ssl() == HTTPClient::VS_normal && !cert_preapproved) {
1739  cert_valid = false;
1740  }
1741 
1742  } else if (verify_result == X509_V_ERR_CERT_NOT_YET_VALID) {
1743  downloader_cat.info()
1744  << _NOTIFY_HTTP_CHANNEL_ID
1745  << "Premature certificate from " << _request.get_url().get_server_and_port() << "\n";
1746  if (_client->get_verify_ssl() == HTTPClient::VS_normal && !cert_preapproved) {
1747  cert_valid = false;
1748  }
1749 
1750  } else if (verify_result == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT ||
1751  verify_result == X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN) {
1752  downloader_cat.info()
1753  << _NOTIFY_HTTP_CHANNEL_ID
1754  << "Self-signed certificate from " << _request.get_url().get_server_and_port() << "\n";
1755  if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_preapproved) {
1756  cert_valid = false;
1757  }
1758 
1759  } else if (verify_result != X509_V_OK) {
1760  downloader_cat.info()
1761  << _NOTIFY_HTTP_CHANNEL_ID
1762  << "Unable to verify identity of " << _request.get_url().get_server_and_port()
1763  << ", verify error code " << verify_result << "\n";
1764  if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_preapproved) {
1765  cert_valid = false;
1766  }
1767  }
1768 
1769  if (!cert_valid) {
1770  _status_entry._status_code = SC_ssl_invalid_server_certificate;
1771  _state = S_failure;
1772  return false;
1773  }
1774 
1775  if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_name_preapproved) {
1776  // Check that the server is someone we expected to be talking
1777  // to.
1778  if (!validate_server_name(cert)) {
1779  _status_entry._status_code = SC_ssl_unexpected_server;
1780  _state = S_failure;
1781  return false;
1782  }
1783  }
1784 
1785  X509_free(cert);
1786 
1787  _state = S_ready;
1788  return false;
1789 }
1790 
1791 ////////////////////////////////////////////////////////////////////
1792 // Function: HTTPChannel::run_ready
1793 // Access: Private
1794 // Description: This is the main "ready" state. In this state, we
1795 // have established a (possibly secure) connection to
1796 // the server (or proxy), and the server (or proxy) is
1797 // idle and waiting for us to send a request.
1798 //
1799 // If persistent_connection is true, we will generally
1800 // come back to this state after finishing each request
1801 // on a given connection.
1802 ////////////////////////////////////////////////////////////////////
1803 bool HTTPChannel::
1804 run_ready() {
1805  // If there's a request to be sent upstream, send it now.
1806  if (!_request_text.empty()) {
1807  if (!server_send(_request_text, false)) {
1808  return true;
1809  }
1810  }
1811 
1812  // All done sending request.
1813  _state = S_request_sent;
1814  _sent_request_time = TrueClock::get_global_ptr()->get_short_time();
1815  return false;
1816 }
1817 
1818 ////////////////////////////////////////////////////////////////////
1819 // Function: HTTPChannel::run_request_sent
1820 // Access: Private
1821 // Description: In this state we have sent our request to the server
1822 // (or proxy) and we are waiting for a response.
1823 ////////////////////////////////////////////////////////////////////
1824 bool HTTPChannel::
1825 run_request_sent() {
1826  // Wait for the first line to come back from the server.
1827  string line;
1828  if (!server_getline_failsafe(line)) {
1829  return true;
1830  }
1831 
1832  // Skip unexpected blank lines. We're just being generous here.
1833  while (line.empty()) {
1834  if (!server_getline_failsafe(line)) {
1835  return true;
1836  }
1837  }
1838 
1839  if (!parse_http_response(line)) {
1840  // Not an HTTP response. _state is already set appropriately.
1841  return false;
1842  }
1843 
1844  _state = S_reading_header;
1845  _current_field_name = string();
1846  _current_field_value = string();
1847  _headers.clear();
1848  _got_file_size = false;
1849  _got_transfer_file_size = false;
1850  return false;
1851 }
1852 
1853 ////////////////////////////////////////////////////////////////////
1854 // Function: HTTPChannel::run_reading_header
1855 // Access: Private
1856 // Description: In this state we have received the first response to
1857 // our request from the server (or proxy) and we are
1858 // reading the set of header lines preceding the
1859 // requested document.
1860 ////////////////////////////////////////////////////////////////////
1861 bool HTTPChannel::
1862 run_reading_header() {
1863  if (parse_http_header()) {
1864  if (_bio.is_null()) {
1865  downloader_cat.info()
1866  << _NOTIFY_HTTP_CHANNEL_ID
1867  << "Connection lost while reading HTTP response.\n";
1868  if (_response_type == RT_http_hangup) {
1869  // This was our second hangup in a row. Give up.
1870  _status_entry._status_code = SC_lost_connection;
1871  _state = S_try_next_proxy;
1872 
1873  } else {
1874  // Try again, once.
1875  _response_type = RT_http_hangup;
1876  }
1877 
1878  } else {
1879  double elapsed =
1880  TrueClock::get_global_ptr()->get_short_time() -
1881  _sent_request_time;
1882  if (elapsed > get_http_timeout()) {
1883  // Time to give up.
1884  downloader_cat.info()
1885  << _NOTIFY_HTTP_CHANNEL_ID
1886  << "Timeout waiting for "
1887  << _request.get_url().get_server_and_port()
1888  << " in run_reading_header (" << elapsed
1889  << " seconds elapsed).\n";
1890  _status_entry._status_code = SC_timeout;
1891  _state = S_try_next_proxy;
1892  }
1893  }
1894  return true;
1895  }
1896  _response_type = RT_http_complete;
1897 
1898  // Ok, we've established an HTTP connection to the server. Our
1899  // extra send headers have done their job; clear them for next time.
1900  clear_extra_headers();
1901 
1902  _server_response_has_no_body =
1903  (get_status_code() / 100 == 1 ||
1904  get_status_code() == 204 ||
1905  get_status_code() == 304 ||
1906  _method == HTTPEnum::M_head);
1907 
1908  // Look for key properties in the header fields.
1909  if (get_status_code() == 206) {
1910  string content_range = get_header_value("Content-Range");
1911  if (content_range.empty()) {
1912  downloader_cat.warning()
1913  << _NOTIFY_HTTP_CHANNEL_ID
1914  << "Got 206 response without Content-Range header!\n";
1915  _status_entry._status_code = SC_invalid_http;
1916  _state = S_failure;
1917  return false;
1918 
1919  } else {
1920  if (!parse_content_range(content_range)) {
1921  downloader_cat.warning()
1922  << _NOTIFY_HTTP_CHANNEL_ID
1923  << "Couldn't parse Content-Range: " << content_range << "\n";
1924  _status_entry._status_code = SC_invalid_http;
1925  _state = S_failure;
1926  return false;
1927  }
1928  }
1929 
1930  } else {
1931  _first_byte_delivered = 0;
1932  _last_byte_delivered = 0;
1933  }
1934  if (downloader_cat.is_debug()) {
1935  if (_first_byte_requested != 0 || _last_byte_requested != 0 ||
1936  _first_byte_delivered != 0 || _last_byte_delivered != 0) {
1937  downloader_cat.debug()
1938  << _NOTIFY_HTTP_CHANNEL_ID
1939  << "Requested byte range " << _first_byte_requested
1940  << " to " << _last_byte_delivered
1941  << "; server delivers range " << _first_byte_delivered
1942  << " to " << _last_byte_delivered
1943  << "\n";
1944  }
1945  }
1946 
1947  // Set the _document_spec to reflect what we just retrieved.
1948  _document_spec = DocumentSpec(_request.get_url());
1949  string tag = get_header_value("ETag");
1950  if (!tag.empty()) {
1951  _document_spec.set_tag(HTTPEntityTag(tag));
1952  }
1953  string date = get_header_value("Last-Modified");
1954  if (!date.empty()) {
1955  _document_spec.set_date(HTTPDate(date));
1956  }
1957 
1958  // In case we've got a download in effect, now we know what the
1959  // first byte of the subdocument request will be, so we can open the
1960  // file and position it.
1961  if (_server_response_has_no_body) {
1962  // Never mind on the download.
1963  reset_download_to();
1964  }
1965 
1966  if (!open_download_file()) {
1967  return false;
1968  }
1969 
1970  _got_expected_file_size = false;
1971  _got_file_size = false;
1972  _got_transfer_file_size = false;
1973 
1974  string content_length = get_header_value("Content-Length");
1975  if (!content_length.empty()) {
1976  _file_size = atoi(content_length.c_str());
1977  _got_file_size = true;
1978 
1979  } else if (get_status_code() == 206) {
1980  // Well, we didn't get a content-length from the server, but we
1981  // can infer the number of bytes based on the range we're given.
1982  _file_size = _last_byte_delivered - _first_byte_delivered + 1;
1983  _got_file_size = true;
1984  }
1985  _redirect = get_header_value("Location");
1986 
1987  // The server might have given us just a filename for the redirect.
1988  // In that case, it's relative to the same server. If it's a
1989  // relative path, it's relative to the same directory.
1990  if (_redirect.has_path() && !_redirect.has_authority()) {
1991  URLSpec url = _document_spec.get_url();
1992  Filename path = _redirect.get_path();
1993  if (path.is_local()) {
1994  Filename rel_to = Filename(url.get_path()).get_dirname();
1995  _redirect.set_path(Filename(rel_to, path));
1996  }
1997  _redirect.set_scheme(url.get_scheme());
1998  _redirect.set_authority(url.get_authority());
1999  }
2000 
2001  _state = S_read_header;
2002 
2003  if (_server_response_has_no_body && will_close_connection()) {
2004  // If the server said it will close the connection, we should
2005  // close it too.
2006  close_connection();
2007  }
2008 
2009  // Handle automatic retries and redirects.
2010  int last_status = _last_status_code;
2011  _last_status_code = get_status_code();
2012 
2013  if (get_status_code() == 407 && last_status != 407 && !_proxy.empty()) {
2014  // 407: not authorized to proxy. Try to get the authorization.
2015  string authenticate_request = get_header_value("Proxy-Authenticate");
2016  _proxy_auth =
2017  _client->generate_auth(_proxy, true, authenticate_request);
2018  if (_proxy_auth != (HTTPAuthorization *)NULL) {
2019  _proxy_realm = _proxy_auth->get_realm();
2020  _proxy_username = _client->select_username(_proxy, true, _proxy_realm);
2021  if (!_proxy_username.empty()) {
2022  make_request_text();
2023 
2024  // Roll the state forward to force a new request.
2025  _state = S_begin_body;
2026  return false;
2027  }
2028  }
2029  }
2030 
2031  if (get_status_code() == 401 && last_status != 401) {
2032  // 401: not authorized to remote server. Try to get the authorization.
2033  string authenticate_request = get_header_value("WWW-Authenticate");
2034  _www_auth = _client->generate_auth(_request.get_url(), false, authenticate_request);
2035  if (_www_auth != (HTTPAuthorization *)NULL) {
2036  _www_realm = _www_auth->get_realm();
2037  _www_username = _client->select_username(_request.get_url(), false, _www_realm);
2038  if (!_www_username.empty()) {
2039  make_request_text();
2040 
2041  // Roll the state forward to force a new request.
2042  _state = S_begin_body;
2043  return false;
2044  }
2045  }
2046  }
2047 
2048  if ((get_status_code() == 300 ||
2049  get_status_code() == 301 ||
2050  get_status_code() == 302 ||
2051  get_status_code() == 303 ||
2052  get_status_code() == 307) && !get_redirect().empty()) {
2053  // Redirect. Should we handle it automatically?
2054 
2055  // According to the letter of RFC 2616, 301 and 302 responses to
2056  // POST requests must not be automatically redirected without
2057  // confirmation by the user. In reality, browsers do allow
2058  // automatic redirection of these responses, changing the POST to
2059  // a GET, and we reproduce this behavior here.
2060  if (_method == HTTPEnum::M_post) {
2061  _method = HTTPEnum::M_get;
2062  _body = string();
2063  }
2064 
2065  if (_method == HTTPEnum::M_get || _method == HTTPEnum::M_head) {
2066  // Sure!
2067  URLSpec new_url = get_redirect();
2068  if (find(_redirect_trail.begin(), _redirect_trail.end(),
2069  new_url) != _redirect_trail.end()) {
2070  downloader_cat.warning()
2071  << _NOTIFY_HTTP_CHANNEL_ID
2072  << "cycle detected in redirect to " << new_url << "\n";
2073 
2074  } else {
2075  _redirect_trail.push_back(new_url);
2076 
2077  if (downloader_cat.is_debug()) {
2078  downloader_cat.debug()
2079  << _NOTIFY_HTTP_CHANNEL_ID
2080  << "following redirect to " << new_url << "\n";
2081  }
2082  if (_request.get_url().has_username()) {
2083  new_url.set_username(_request.get_url().get_username());
2084  }
2085  reset_url(_request.get_url(), new_url);
2086  _request.set_url(new_url);
2087  _want_ssl = _request.get_url().is_ssl();
2088  reconsider_proxy();
2089  make_header();
2090  make_request_text();
2091 
2092  // Roll the state forward to force a new request.
2093  _state = S_begin_body;
2094  return false;
2095  }
2096  }
2097  }
2098 
2099  if (_state == S_read_header &&
2100  ((get_status_code() / 100) == 4 || (get_status_code() / 100) == 5) &&
2101  _proxy_serves_document && _proxy_next_index < _proxies.size()) {
2102  // If we were using a proxy (but not tunneling through the proxy)
2103  // and we got some kind of a server error, try the next proxy in
2104  // sequence (if we have one). This handles the case of a working
2105  // proxy that cannot see the host (and so returns 504 or something
2106  // along those lines). Some proxies are so broken they return a
2107  // 404 in this case, so we have to consider that along the same
2108  // lines.
2109  _state = S_try_next_proxy;
2110  return false;
2111  }
2112 
2113  // Otherwise, we're good to go.
2114  return false;
2115 }
2116 
2117 ////////////////////////////////////////////////////////////////////
2118 // Function: HTTPChannel::run_start_direct_file_read
2119 // Access: Private
2120 // Description: This is the first state when reading a file:// URL.
2121 // All it does is skip past the non-existent "header".
2122 ////////////////////////////////////////////////////////////////////
2123 bool HTTPChannel::
2124 run_start_direct_file_read() {
2125  _state = S_read_header;
2126  if (!open_download_file()) {
2127  return false;
2128  }
2129  return false;
2130 }
2131 
2132 ////////////////////////////////////////////////////////////////////
2133 // Function: HTTPChannel::run_read_header
2134 // Access: Private
2135 // Description: In this state we have completely read the header
2136 // lines returned by the server (or proxy) in response
2137 // to our request. This state represents the normal
2138 // stopping point of a call to get_document(), etc.;
2139 // further reads will return the body of the request,
2140 // the requested document.
2141 //
2142 // Normally run_read_header() is not called unless the
2143 // user has elected not to read the returned document
2144 // himself. In fact, the state itself only exists so we
2145 // can make a distinction between S_read_header and
2146 // S_begin_body, where S_read_header is safe to return
2147 // to the user and S_begin_body means we need to start
2148 // skipping the document.
2149 ////////////////////////////////////////////////////////////////////
2150 bool HTTPChannel::
2151 run_read_header() {
2152  _state = S_begin_body;
2153  return false;
2154 }
2155 
2156 ////////////////////////////////////////////////////////////////////
2157 // Function: HTTPChannel::run_begin_body
2158 // Access: Private
2159 // Description: This state begins to skip over the body in
2160 // preparation for making a new request.
2161 ////////////////////////////////////////////////////////////////////
2162 bool HTTPChannel::
2163 run_begin_body() {
2164  if (will_close_connection()) {
2165  // If the socket will close anyway, no point in skipping past the
2166  // previous body; just reset.
2167  if (downloader_cat.is_debug()) {
2168  downloader_cat.debug()
2169  << _NOTIFY_HTTP_CHANNEL_ID
2170  << "resetting to begin body; server would close anyway.\n";
2171  }
2172  reset_to_new();
2173  return false;
2174  }
2175 
2176  if (_server_response_has_no_body) {
2177  // We have already "read" the nonexistent body.
2178  _state = S_read_trailer;
2179 
2180  } else if (get_file_size() > (int)_skip_body_size) {
2181  // If we know the size of the body we are about to skip and it's
2182  // too large, then don't bother skipping it--just drop the
2183  // connection and get a new one.
2184  if (downloader_cat.is_debug()) {
2185  downloader_cat.debug()
2186  << _NOTIFY_HTTP_CHANNEL_ID
2187  << "Dropping connection rather than skipping past "
2188  << get_file_size() << " bytes.\n";
2189  }
2190  reset_to_new();
2191 
2192  } else {
2193  open_read_body();
2194  if (_body_stream == (ISocketStream *)NULL) {
2195  if (downloader_cat.is_debug()) {
2196  downloader_cat.debug()
2197  << _NOTIFY_HTTP_CHANNEL_ID
2198  << "Unable to skip body.\n";
2199  }
2200  reset_to_new();
2201 
2202  } else {
2203  _owns_body_stream = true;
2204  if (_state != S_reading_body) {
2205  reset_body_stream();
2206  }
2207  }
2208  }
2209 
2210  return false;
2211 }
2212 
2213 ////////////////////////////////////////////////////////////////////
2214 // Function: HTTPChannel::run_reading_body
2215 // Access: Private
2216 // Description: In this state we are in the process of reading the
2217 // response's body. We will only come to this function
2218 // if the user did not choose to read the entire body
2219 // himself (by calling open_read_body()).
2220 //
2221 // In this case we should skip past the body to reset
2222 // the connection for making a new request.
2223 ////////////////////////////////////////////////////////////////////
2224 bool HTTPChannel::
2225 run_reading_body() {
2226  if (will_close_connection()) {
2227  // If the socket will close anyway, no point in skipping past the
2228  // previous body; just reset.
2229  if (downloader_cat.is_debug()) {
2230  downloader_cat.debug()
2231  << _NOTIFY_HTTP_CHANNEL_ID
2232  << "resetting to read body; server would close anyway.\n";
2233  }
2234  reset_to_new();
2235  return false;
2236  }
2237 
2238  // Skip the body we've already started.
2239  if (_body_stream == NULL || !_owns_body_stream) {
2240  // Whoops, we're not in skip-body mode. Better reset.
2241  if (downloader_cat.is_debug()) {
2242  downloader_cat.debug()
2243  << _NOTIFY_HTTP_CHANNEL_ID
2244  << "resetting, not in skip-body mode.\n";
2245  }
2246  reset_to_new();
2247  return false;
2248  }
2249 
2250  string line;
2251  getline(*_body_stream, line);
2252  while (!_body_stream->fail() && !_body_stream->eof()) {
2253  if (downloader_cat.is_spam()) {
2254  downloader_cat.spam()
2255  << _NOTIFY_HTTP_CHANNEL_ID
2256  << "skip: " << line << "\n";
2257  }
2258  getline(*_body_stream, line);
2259  }
2260 
2261  if (!_body_stream->is_closed()) {
2262  // There's more to come later.
2263  return true;
2264  }
2265 
2266  reset_body_stream();
2267 
2268  // This should have been set by the call to finished_body(), above.
2269  nassertr(_state != S_reading_body, false);
2270  return false;
2271 }
2272 
2273 ////////////////////////////////////////////////////////////////////
2274 // Function: HTTPChannel::run_read_body
2275 // Access: Private
2276 // Description: In this state we have completely read (or skipped
2277 // over) the body of the response. We should continue
2278 // skipping past the trailer following the body.
2279 //
2280 // Not all bodies come with trailers; in particular, the
2281 // "identity" transfer encoding does not include a
2282 // trailer. It is therefore the responsibility of the
2283 // IdentityStreamBuf or ChunkedStreamBuf to set the
2284 // state appropriately to either S_read_body or
2285 // S_read_trailer following the completion of the body.
2286 ////////////////////////////////////////////////////////////////////
2287 bool HTTPChannel::
2288 run_read_body() {
2289  if (will_close_connection()) {
2290  // If the socket will close anyway, no point in skipping past the
2291  // previous body; just reset.
2292  if (downloader_cat.is_debug()) {
2293  downloader_cat.debug()
2294  << _NOTIFY_HTTP_CHANNEL_ID
2295  << "resetting to read body; server would close anyway.\n";
2296  }
2297  reset_to_new();
2298  return false;
2299  }
2300  // Skip the trailer following the recently-read body.
2301 
2302  string line;
2303  if (!server_getline(line)) {
2304  return true;
2305  }
2306  while (!line.empty()) {
2307  if (!server_getline(line)) {
2308  return true;
2309  }
2310  }
2311 
2312  _state = S_read_trailer;
2313  return false;
2314 }
2315 
2316 ////////////////////////////////////////////////////////////////////
2317 // Function: HTTPChannel::run_read_trailer
2318 // Access: Private
2319 // Description: In this state we have completely read the body and
2320 // the trailer. This state is simply a pass-through
2321 // back to S_ready.
2322 ////////////////////////////////////////////////////////////////////
2323 bool HTTPChannel::
2324 run_read_trailer() {
2325  if (will_close_connection()) {
2326  // If the socket will close anyway, no point in skipping past the
2327  // previous body; just reset.
2328  if (downloader_cat.is_debug()) {
2329  downloader_cat.debug()
2330  << _NOTIFY_HTTP_CHANNEL_ID
2331  << "resetting to read trailer; server would close anyway.\n";
2332  }
2333  reset_to_new();
2334  return false;
2335  }
2336 
2337  _state = S_ready;
2338  return false;
2339 }
2340 
2341 ////////////////////////////////////////////////////////////////////
2342 // Function: HTTPChannel::run_download_to_file
2343 // Access: Private
2344 // Description: After the headers, etc. have been read, this streams
2345 // the download to the named file.
2346 ////////////////////////////////////////////////////////////////////
2347 bool HTTPChannel::
2348 run_download_to_file() {
2349  nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false);
2350 
2351  bool do_throttle = _wanted_nonblocking && _download_throttle;
2352 
2353  static const size_t buffer_size = 4096;
2354  char buffer[buffer_size];
2355 
2356  size_t remaining_this_pass = buffer_size;
2357  if (do_throttle) {
2358  remaining_this_pass = _bytes_per_update;
2359  }
2360 
2361  _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2362  size_t count = _body_stream->gcount();
2363  while (count != 0) {
2364  _download_to_stream->write(buffer, count);
2365  _bytes_downloaded += count;
2366  if (do_throttle) {
2367  nassertr(count <= remaining_this_pass, false);
2368  remaining_this_pass -= count;
2369  if (remaining_this_pass == 0) {
2370  // That's enough for now.
2371  return true;
2372  }
2373  }
2374 
2375  thread_consider_yield();
2376  _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2377  count = _body_stream->gcount();
2378  }
2379 
2380  if (_download_to_stream->fail()) {
2381  downloader_cat.warning()
2382  << _NOTIFY_HTTP_CHANNEL_ID
2383  << "Error writing to " << _download_to_filename << "\n";
2384  _status_entry._status_code = SC_download_write_error;
2385  _state = S_failure;
2386  reset_download_to();
2387  return false;
2388  }
2389 
2390  _download_to_stream->flush();
2391 
2392  if (_body_stream->is_closed()) {
2393  // Done.
2394  reset_body_stream();
2395  close_download_stream();
2396  _started_download = false;
2397  return false;
2398  } else {
2399  // More to come.
2400  return true;
2401  }
2402 }
2403 
2404 ////////////////////////////////////////////////////////////////////
2405 // Function: HTTPChannel::run_download_to_ram
2406 // Access: Private
2407 // Description: After the headers, etc. have been read, this streams
2408 // the download to the specified Ramfile object.
2409 ////////////////////////////////////////////////////////////////////
2410 bool HTTPChannel::
2411 run_download_to_ram() {
2412  nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false);
2413  nassertr(_download_to_ramfile != (Ramfile *)NULL, false);
2414 
2415  bool do_throttle = _wanted_nonblocking && _download_throttle;
2416 
2417  static const size_t buffer_size = 4096;
2418  char buffer[buffer_size];
2419 
2420  size_t remaining_this_pass = buffer_size;
2421  if (do_throttle) {
2422  remaining_this_pass = _bytes_per_update;
2423  }
2424 
2425  _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2426  size_t count = _body_stream->gcount();
2427  while (count != 0) {
2428  _download_to_ramfile->_data += string(buffer, count);
2429  _bytes_downloaded += count;
2430  if (do_throttle) {
2431  nassertr(count <= remaining_this_pass, false);
2432  remaining_this_pass -= count;
2433  if (remaining_this_pass == 0) {
2434  // That's enough for now.
2435  return true;
2436  }
2437  }
2438 
2439  thread_consider_yield();
2440  _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2441  count = _body_stream->gcount();
2442  }
2443 
2444  if (_body_stream->is_closed()) {
2445  // Done.
2446  reset_body_stream();
2447  close_download_stream();
2448  _started_download = false;
2449  return false;
2450  } else {
2451  // More to come.
2452  return true;
2453  }
2454 }
2455 
2456 ////////////////////////////////////////////////////////////////////
2457 // Function: HTTPChannel::run_download_to_stream
2458 // Access: Private
2459 // Description: After the headers, etc. have been read, this streams
2460 // the download to the named file.
2461 ////////////////////////////////////////////////////////////////////
2462 bool HTTPChannel::
2463 run_download_to_stream() {
2464  nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false);
2465 
2466  bool do_throttle = _wanted_nonblocking && _download_throttle;
2467 
2468  static const size_t buffer_size = 4096;
2469  char buffer[buffer_size];
2470 
2471  size_t remaining_this_pass = buffer_size;
2472  if (do_throttle) {
2473  remaining_this_pass = _bytes_per_update;
2474  }
2475 
2476  _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2477  size_t count = _body_stream->gcount();
2478  while (count != 0) {
2479  _download_to_stream->write(buffer, count);
2480  _bytes_downloaded += count;
2481  if (do_throttle) {
2482  nassertr(count <= remaining_this_pass, false);
2483  remaining_this_pass -= count;
2484  if (remaining_this_pass == 0) {
2485  // That's enough for now.
2486  return true;
2487  }
2488  }
2489 
2490  thread_consider_yield();
2491  _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2492  count = _body_stream->gcount();
2493  }
2494 
2495  if (_download_to_stream->fail()) {
2496  downloader_cat.warning()
2497  << _NOTIFY_HTTP_CHANNEL_ID
2498  << "Error writing to stream\n";
2499  _status_entry._status_code = SC_download_write_error;
2500  _state = S_failure;
2501  reset_download_to();
2502  return false;
2503  }
2504 
2505  _download_to_stream->flush();
2506 
2507  if (_body_stream->is_closed()) {
2508  // Done.
2509  reset_body_stream();
2510  close_download_stream();
2511  _started_download = false;
2512  return false;
2513  } else {
2514  // More to come.
2515  return true;
2516  }
2517 }
2518 
2519 
2520 ////////////////////////////////////////////////////////////////////
2521 // Function: HTTPChannel::begin_request
2522 // Access: Private
2523 // Description: Begins a new document request to the server, throwing
2524 // away whatever request was currently pending if
2525 // necessary.
2526 ////////////////////////////////////////////////////////////////////
2527 void HTTPChannel::
2528 begin_request(HTTPEnum::Method method, const DocumentSpec &url,
2529  const string &body, bool nonblocking,
2530  size_t first_byte, size_t last_byte) {
2531 
2532  downloader_cat.info()
2533  << _NOTIFY_HTTP_CHANNEL_ID
2534  << "begin " << method << " " << url << "\n";
2535 
2536  reset_for_new_request();
2537 
2538  _wanted_nonblocking = nonblocking;
2539 #if defined(HAVE_THREADS) && defined(SIMPLE_THREADS)
2540  // In the presence of SIMPLE_THREADS, we always use non-blocking
2541  // I/O. We simulate blocking by yielding the thread.
2542  nonblocking = true;
2543 #endif
2544 
2545  // Get the set of proxies that are appropriate for this URL.
2546  _proxies.clear();
2547  _proxy_next_index = 0;
2548  if (get_allow_proxy()) {
2549  _client->get_proxies_for_url(url.get_url(), _proxies);
2550  }
2551 
2552  // If we still have a live connection to a proxy that is on the
2553  // list, that proxy should be moved immediately to the front of the
2554  // list (to minimize restarting connections unnecessarily).
2555  if (!_bio.is_null() && !_proxies.empty() && !_proxy.empty()) {
2556  Proxies::iterator pi = find(_proxies.begin(), _proxies.end(), _proxy);
2557  if (pi != _proxies.end()) {
2558  _proxies.erase(pi);
2559  _proxies.insert(_proxies.begin(), _proxy);
2560  }
2561  }
2562 
2563  URLSpec new_proxy;
2564  if (_proxy_next_index < _proxies.size()) {
2565  new_proxy = _proxies[_proxy_next_index];
2566  _proxy_next_index++;
2567  }
2568 
2569  // Changing the proxy is grounds for dropping the old connection, if
2570  // any.
2571  if (_proxy != new_proxy) {
2572  _proxy = new_proxy;
2573  _proxy_auth = (HTTPAuthorization *)NULL;
2574  if (downloader_cat.is_debug()) {
2575  downloader_cat.debug()
2576  << _NOTIFY_HTTP_CHANNEL_ID
2577  << "resetting to change proxy to " << _proxy << "\n";
2578  }
2579  reset_to_new();
2580  }
2581 
2582  // Ditto with changing the nonblocking state.
2583  if (_nonblocking != nonblocking) {
2584  _nonblocking = nonblocking;
2585  if (downloader_cat.is_debug()) {
2586  downloader_cat.debug()
2587  << _NOTIFY_HTTP_CHANNEL_ID
2588  << "resetting to change nonblocking state to " << _nonblocking << ".\n";
2589  }
2590  reset_to_new();
2591  }
2592 
2593  reset_url(_request.get_url(), url.get_url());
2594  _request = url;
2595  _document_spec = DocumentSpec();
2596  _method = method;
2597  _body = body;
2598 
2599  // An https-style request means we'll need to establish an SSL
2600  // connection.
2601  _want_ssl = _request.get_url().is_ssl();
2602 
2603  _first_byte_requested = first_byte;
2604  _last_byte_requested = last_byte;
2605  _connect_count = 0;
2606 
2607  reconsider_proxy();
2608 
2609  // Reset from whatever previous request might still be pending.
2610  if (_request.get_url().get_scheme() == "file") {
2611  // A "file" URL just means we're reading a raw file. This only
2612  // supports actual disk files, not the VFS, because we use a
2613  // BIO_new_file() underneath this.
2614  reset_to_new();
2615  _bio = new BioPtr(_request.get_url());
2616  if (_bio->get_bio() != NULL) {
2617  // Successfully opened the file.
2618  _source = new BioStreamPtr(new BioStream(_bio));
2619  _status_entry._status_code = 200;
2620  _state = S_start_direct_file_read;
2621 
2622  // Get the file size.
2623  FILE *fp = NULL;
2624  BIO_get_fp(_bio->get_bio(), &fp);
2625  if (fp != NULL) {
2626  if (fseek(fp, 0, SEEK_END) == 0) {
2627  _file_size = ftell(fp);
2628  _got_file_size = true;
2629  fseek(fp, 0, SEEK_SET);
2630  }
2631  }
2632 
2633  } else {
2634  // Couldn't read the file.
2635  OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
2636  _status_entry._status_code = SC_no_connection;
2637  _state = S_failure;
2638  }
2639 
2640  } else {
2641  // We're reading a normal network URL.
2642  if (_state == S_failure || (_state < S_read_header && _state != S_ready)) {
2643  if (downloader_cat.is_debug()) {
2644  downloader_cat.debug()
2645  << _NOTIFY_HTTP_CHANNEL_ID
2646  << "resetting to clear previous request.\n";
2647  }
2648  reset_to_new();
2649 
2650  } else if (TrueClock::get_global_ptr()->get_short_time() - _last_run_time >= _idle_timeout) {
2651  if (downloader_cat.is_debug()) {
2652  downloader_cat.debug()
2653  << _NOTIFY_HTTP_CHANNEL_ID
2654  << "resetting old connection: "
2655  << TrueClock::get_global_ptr()->get_short_time() - _last_run_time
2656  << " s old.\n";
2657  }
2658  reset_to_new();
2659 
2660  } else if (_state == S_read_header) {
2661  // Roll one step forwards to start skipping past the previous
2662  // body.
2663  _state = S_begin_body;
2664  }
2665  }
2666 
2667  if (_method == HTTPEnum::M_connect) {
2668  _done_state = S_ready;
2669  } else {
2670  _done_state = S_read_header;
2671  }
2672 }
2673 
2674 ////////////////////////////////////////////////////////////////////
2675 // Function: HTTPChannel::reconsider_proxy
2676 // Access: Private
2677 // Description: Reevaluates the flags and strings that are computed
2678 // based on the particular proxy we are attempting to
2679 // connect to. This should be called when we initiate a
2680 // request, and also whenever we change proxies while
2681 // processing a request.
2682 ////////////////////////////////////////////////////////////////////
2683 void HTTPChannel::
2684 reconsider_proxy() {
2685  _proxy_tunnel_now = false;
2686  _proxy_serves_document = false;
2687 
2688  if (!_proxy.empty()) {
2689  // If the user insists we always tunnel through a proxy, or if
2690  // we're opening an SSL connection, or the user has explicitly
2691  // asked for a direct connection of some kind, or if we have a
2692  // SOCKS-style proxy; each of these demands a tunnel through the
2693  // proxy to speak directly to the http server.
2694  _proxy_tunnel_now =
2695  (get_proxy_tunnel() || _want_ssl ||
2696  _method == HTTPEnum::M_connect || _proxy.get_scheme() == "socks");
2697 
2698  // Otherwise (but we still have a proxy), then we ask the proxy to
2699  // hand us the document.
2700  _proxy_serves_document = !_proxy_tunnel_now;
2701  }
2702 
2703  make_header();
2704  make_request_text();
2705 
2706  if (_proxy_tunnel_now) {
2707  // Maybe we need to tunnel through the proxy to connect to the
2708  // server directly.
2709  ostringstream request;
2710  request
2711  << "CONNECT " << _request.get_url().get_server_and_port()
2712  << " " << _client->get_http_version_string() << "\r\n";
2713  if (_client->get_http_version() >= HTTPEnum::HV_11) {
2714  request
2715  << "Host: " << _request.get_url().get_server_and_port() << "\r\n";
2716  }
2717  _proxy_header = request.str();
2718  make_proxy_request_text();
2719 
2720  } else {
2721  _proxy_header = string();
2722  _proxy_request_text = string();
2723  }
2724 }
2725 
2726 
2727 ////////////////////////////////////////////////////////////////////
2728 // Function: HTTPChannel::reset_for_new_request
2729 // Access: Private
2730 // Description: Resets the internal state variables in preparation
2731 // for beginning a new request.
2732 ////////////////////////////////////////////////////////////////////
2733 void HTTPChannel::
2734 reset_for_new_request() {
2735  if (downloader_cat.is_spam()) {
2736  downloader_cat.spam()
2737  << _NOTIFY_HTTP_CHANNEL_ID
2738  << "reset_for_new_request.\n";
2739  }
2740 
2741  reset_download_to();
2742  reset_body_stream();
2743 
2744  _last_status_code = 0;
2745  _status_entry = StatusEntry();
2746 
2747  _response_type = RT_none;
2748  _redirect_trail.clear();
2749  _bytes_downloaded = 0;
2750  _bytes_requested = 0;
2751 }
2752 
2753 ////////////////////////////////////////////////////////////////////
2754 // Function: HTTPChannel::finished_body
2755 // Access: Private
2756 // Description: This is called by the body reading
2757 // classes--ChunkedStreamBuf and IdentityStreamBuf--when
2758 // they have finished reading the body. It advances the
2759 // state appropriately.
2760 //
2761 // has_trailer should be set true if the body type has
2762 // an associated trailer which should be read or
2763 // skipped, or false if there is no trailer.
2764 ////////////////////////////////////////////////////////////////////
2765 void HTTPChannel::
2766 finished_body(bool has_trailer) {
2767  if (will_close_connection() && _download_dest == DD_none) {
2768  if (downloader_cat.is_debug()) {
2769  downloader_cat.debug()
2770  << _NOTIFY_HTTP_CHANNEL_ID
2771  << "resetting to finish body; server would close anyway.\n";
2772  }
2773  reset_to_new();
2774 
2775  } else {
2776  if (has_trailer) {
2777  _state = HTTPChannel::S_read_body;
2778  } else {
2779  _state = HTTPChannel::S_read_trailer;
2780  }
2781  }
2782 }
2783 
2784 ////////////////////////////////////////////////////////////////////
2785 // Function: HTTPChannel::open_download_file
2786 // Access: Private
2787 // Description: If a download has been requested, opens the file on
2788 // disk (or prepares the RamFile or stream) and seeks
2789 // within it to the appropriate _first_byte_delivered
2790 // position, so that downloaded bytes will be written to
2791 // the appropriate point within the file. Returns true
2792 // if the starting position is valid, false otherwise
2793 // (in which case the state is set to S_failure).
2794 ////////////////////////////////////////////////////////////////////
2795 bool HTTPChannel::
2796 open_download_file() {
2797  _subdocument_resumes = (_subdocument_resumes && _first_byte_delivered != 0);
2798 
2799  if (_download_dest == DD_file) {
2801  _download_to_stream = vfs->open_write_file(_download_to_filename, false, !_subdocument_resumes);
2802  if (_download_to_stream == NULL) {
2803  downloader_cat.info()
2804  << _NOTIFY_HTTP_CHANNEL_ID
2805  << "Could not open " << _download_to_filename << " for writing.\n";
2806  _status_entry._status_code = SC_download_open_error;
2807  _state = S_failure;
2808  return false;
2809  }
2810  }
2811 
2812  if (_subdocument_resumes) {
2813  if (_download_dest == DD_file) {
2814  // Windows doesn't complain if you try to seek past the end of
2815  // file--it happily appends enough zero bytes to make the
2816  // difference. Blecch. That means we need to get the file size
2817  // first to check it ourselves.
2818  _download_to_stream->seekp(0, ios::end);
2819  if (_first_byte_delivered > (size_t)_download_to_stream->tellp()) {
2820  downloader_cat.info()
2821  << _NOTIFY_HTTP_CHANNEL_ID
2822  << "Invalid starting position of byte " << _first_byte_delivered
2823  << " within " << _download_to_filename << " (which has "
2824  << _download_to_stream->tellp() << " bytes)\n";
2825  close_download_stream();
2826  _status_entry._status_code = SC_download_invalid_range;
2827  _state = S_failure;
2828  return false;
2829  }
2830 
2831  _download_to_stream->seekp(_first_byte_delivered);
2832 
2833  } else if (_download_dest == DD_ram) {
2834  if (_first_byte_delivered > _download_to_ramfile->_data.length()) {
2835  downloader_cat.info()
2836  << _NOTIFY_HTTP_CHANNEL_ID
2837  << "Invalid starting position of byte " << _first_byte_delivered
2838  << " within Ramfile (which has "
2839  << _download_to_ramfile->_data.length() << " bytes)\n";
2840  close_download_stream();
2841  _status_entry._status_code = SC_download_invalid_range;
2842  _state = S_failure;
2843  return false;
2844  }
2845 
2846  if (_first_byte_delivered == 0) {
2847  _download_to_ramfile->_data = string();
2848  } else {
2849  _download_to_ramfile->_data =
2850  _download_to_ramfile->_data.substr(0, _first_byte_delivered);
2851  }
2852  } else if (_download_dest == DD_stream) {
2853  // Windows doesn't complain if you try to seek past the end of
2854  // file--it happily appends enough zero bytes to make the
2855  // difference. Blecch. That means we need to get the file size
2856  // first to check it ourselves.
2857  _download_to_stream->seekp(0, ios::end);
2858  if (_first_byte_delivered > (size_t)_download_to_stream->tellp()) {
2859  downloader_cat.info()
2860  << _NOTIFY_HTTP_CHANNEL_ID
2861  << "Invalid starting position of byte " << _first_byte_delivered
2862  << " within stream (which has "
2863  << _download_to_stream->tellp() << " bytes)\n";
2864  close_download_stream();
2865  _status_entry._status_code = SC_download_invalid_range;
2866  _state = S_failure;
2867  return false;
2868  }
2869 
2870  _download_to_stream->seekp(_first_byte_delivered);
2871  }
2872 
2873  } else {
2874  // If _subdocument_resumes is false, we should be sure to reset to
2875  // the beginning of the file, regardless of the value of
2876  // _first_byte_delivered.
2877  if (_download_dest == DD_file || _download_dest == DD_stream) {
2878  _download_to_stream->seekp(0);
2879  } else if (_download_dest == DD_ram) {
2880  _download_to_ramfile->_data = string();
2881  }
2882  }
2883 
2884  return true;
2885 }
2886 
2887 
2888 ////////////////////////////////////////////////////////////////////
2889 // Function: HTTPChannel::server_getline
2890 // Access: Private
2891 // Description: Reads a single line from the server's reply. Returns
2892 // true if the line is successfully retrieved, or false
2893 // if a complete line has not yet been received or if
2894 // the connection has been closed.
2895 ////////////////////////////////////////////////////////////////////
2896 bool HTTPChannel::
2897 server_getline(string &str) {
2898  nassertr(!_source.is_null(), false);
2899  int ch = (*_source)->get();
2900  while (!(*_source)->eof() && !(*_source)->fail()) {
2901  switch (ch) {
2902  case '\n':
2903  // end-of-line character, we're done.
2904  str = _working_get;
2905  _working_get = string();
2906  {
2907  // Trim trailing whitespace. We're not required to do this per the
2908  // HTTP spec, but let's be generous.
2909  size_t p = str.length();
2910  while (p > 0 && isspace(str[p - 1])) {
2911  --p;
2912  }
2913  str = str.substr(0, p);
2914  }
2915  if (downloader_cat.is_debug()) {
2916  downloader_cat.debug()
2917  << _NOTIFY_HTTP_CHANNEL_ID
2918  << "recv: " << str << "\n";
2919  }
2920  return true;
2921 
2922  case '\r':
2923  // Ignore CR characters.
2924  break;
2925 
2926  default:
2927  _working_get += (char)ch;
2928  }
2929  ch = (*_source)->get();
2930  }
2931 
2932  check_socket();
2933  return false;
2934 }
2935 
2936 ////////////////////////////////////////////////////////////////////
2937 // Function: HTTPChannel::server_getline_failsafe
2938 // Access: Private
2939 // Description: Reads a line from the server's reply. If the server
2940 // disconnects or times out before sending a reply,
2941 // moves on to the next proxy server (or sets failure
2942 // mode) and returns false; otherwise, returns true.
2943 ////////////////////////////////////////////////////////////////////
2944 bool HTTPChannel::
2945 server_getline_failsafe(string &str) {
2946  if (!server_getline(str)) {
2947  if (_bio.is_null()) {
2948  // Huh, the server hung up on us as soon as we tried to connect.
2949  if (_response_type == RT_hangup) {
2950  // This was our second immediate hangup in a row. Give up.
2951  _status_entry._status_code = SC_lost_connection;
2952  _state = S_try_next_proxy;
2953 
2954  } else {
2955  // Try again, once.
2956  _response_type = RT_hangup;
2957  }
2958 
2959  } else {
2960  double elapsed =
2961  TrueClock::get_global_ptr()->get_short_time() -
2962  _sent_request_time;
2963  if (elapsed > get_http_timeout()) {
2964  // Time to give up.
2965  downloader_cat.info()
2966  << _NOTIFY_HTTP_CHANNEL_ID
2967  << "Timeout waiting for "
2968  << _request.get_url().get_server_and_port()
2969  << " in server_getline_failsafe (" << elapsed
2970  << " seconds elapsed).\n";
2971  _status_entry._status_code = SC_timeout;
2972  _state = S_try_next_proxy;
2973  }
2974  }
2975 
2976  return false;
2977  }
2978  return true;
2979 }
2980 
2981 ////////////////////////////////////////////////////////////////////
2982 // Function: HTTPChannel::server_get
2983 // Access: Private
2984 // Description: Reads a fixed number of bytes from the server's
2985 // reply. Returns true if the indicated number of bytes
2986 // are successfully retrieved, or false if the complete
2987 // set has not yet been received or if the connection
2988 // has been closed.
2989 ////////////////////////////////////////////////////////////////////
2990 bool HTTPChannel::
2991 server_get(string &str, size_t num_bytes) {
2992  nassertr(!_source.is_null(), false);
2993  int ch = (*_source)->get();
2994  while (!(*_source)->eof() && !(*_source)->fail()) {
2995  _working_get += (char)ch;
2996  if (_working_get.length() >= num_bytes) {
2997  str = _working_get;
2998  _working_get = string();
2999  return true;
3000  }
3001 
3002  ch = (*_source)->get();
3003  }
3004 
3005  check_socket();
3006  return false;
3007 }
3008 
3009 ////////////////////////////////////////////////////////////////////
3010 // Function: HTTPChannel::server_get_failsafe
3011 // Access: Private
3012 // Description: Reads a fixed number of bytes from the server. If
3013 // the server disconnects or times out before sending a
3014 // reply, moves on to the next proxy server (or sets
3015 // failure mode) and returns false; otherwise, returns
3016 // true.
3017 ////////////////////////////////////////////////////////////////////
3018 bool HTTPChannel::
3019 server_get_failsafe(string &str, size_t num_bytes) {
3020  if (!server_get(str, num_bytes)) {
3021  if (_bio.is_null()) {
3022  // Huh, the server hung up on us as soon as we tried to connect.
3023  if (_response_type == RT_hangup) {
3024  // This was our second immediate hangup in a row. Give up.
3025  _status_entry._status_code = SC_lost_connection;
3026  _state = S_try_next_proxy;
3027 
3028  } else {
3029  // Try again, once.
3030  _response_type = RT_hangup;
3031  }
3032 
3033  } else {
3034  double elapsed =
3035  TrueClock::get_global_ptr()->get_short_time() -
3036  _sent_request_time;
3037  if (elapsed > get_http_timeout()) {
3038  // Time to give up.
3039  downloader_cat.info()
3040  << _NOTIFY_HTTP_CHANNEL_ID
3041  << "Timeout waiting for "
3042  << _request.get_url().get_server_and_port()
3043  << " in server_get_failsafe (" << elapsed
3044  << " seconds elapsed).\n";
3045  _status_entry._status_code = SC_timeout;
3046  _state = S_try_next_proxy;
3047  }
3048  }
3049 
3050  return false;
3051  }
3052  return true;
3053 }
3054 
3055 ////////////////////////////////////////////////////////////////////
3056 // Function: HTTPChannel::server_send
3057 // Access: Private
3058 // Description: Sends a series of lines to the server. Returns true
3059 // if the buffer is fully sent, or false if some of it
3060 // remains. If this returns false, the function must be
3061 // called again later, passing in the exact same string,
3062 // until the return value is true.
3063 //
3064 // If the secret flag is true, the data is not echoed to
3065 // the log (even in spam mode). This may be desirable
3066 // if the data may contain binary data, or if it may
3067 // contain passwords etc.
3068 ////////////////////////////////////////////////////////////////////
3069 bool HTTPChannel::
3070 server_send(const string &str, bool secret) {
3071  nassertr(str.length() > _sent_so_far, true);
3072 
3073  // Use the underlying BIO to write to the server, instead of the
3074  // BIOStream, which would insist on blocking (and might furthermore
3075  // delay the send due to collect-tcp mode being enabled).
3076  size_t bytes_to_send = str.length() - _sent_so_far;
3077  int write_count =
3078  BIO_write(*_bio, str.data() + _sent_so_far, bytes_to_send);
3079 
3080  if (write_count <= 0) {
3081  if (BIO_should_retry(*_bio)) {
3082  // Temporary failure: the pipe is full. Wait till later.
3083  return false;
3084  }
3085  // Oops, the connection has been closed!
3086  if (downloader_cat.is_debug()) {
3087  downloader_cat.debug()
3088  << _NOTIFY_HTTP_CHANNEL_ID
3089  << "Lost connection to server unexpectedly during write.\n";
3090  }
3091  reset_to_new();
3092  return false;
3093  }
3094 
3095  if (downloader_cat.is_spam()) {
3096  downloader_cat.spam()
3097  << _NOTIFY_HTTP_CHANNEL_ID
3098  << "wrote " << write_count << " bytes to " << _bio << "\n";
3099  }
3100 
3101 #ifndef NDEBUG
3102  if (!secret && downloader_cat.is_debug()) {
3103  show_send(str.substr(0, write_count));
3104  }
3105 #endif
3106 
3107  if (write_count < (int)bytes_to_send) {
3108  _sent_so_far += write_count;
3109  return false;
3110  }
3111 
3112  // Buffer completely sent.
3113  _sent_so_far = 0;
3114  return true;
3115 }
3116 
3117 ////////////////////////////////////////////////////////////////////
3118 // Function: HTTPChannel::parse_http_response
3119 // Access: Private
3120 // Description: Parses the first line sent back from an HTTP server
3121 // or proxy and stores the result in _status_code and
3122 // _http_version, etc. Returns true on success, false
3123 // on invalid response.
3124 ////////////////////////////////////////////////////////////////////
3125 bool HTTPChannel::
3126 parse_http_response(const string &line) {
3127  // The first line back should include the HTTP version and the
3128  // result code.
3129  if (line.length() < 5 || line.substr(0, 5) != string("HTTP/")) {
3130  // Not an HTTP response.
3131  _status_entry._status_code = SC_non_http_response;
3132  if (_response_type == RT_non_http) {
3133  // This was our second non-HTTP response in a row. Give up.
3134  _state = S_try_next_proxy;
3135 
3136  } else {
3137  // Maybe we were just in some bad state. Drop the connection
3138  // and try again, once.
3139  if (downloader_cat.is_debug()) {
3140  downloader_cat.debug()
3141  << _NOTIFY_HTTP_CHANNEL_ID
3142  << "got non-HTTP response, resetting.\n";
3143  }
3144  reset_to_new();
3145  _response_type = RT_non_http;
3146  }
3147  return false;
3148  }
3149 
3150  // Split out the first line into its three components.
3151  size_t p = 5;
3152  while (p < line.length() && !isspace(line[p])) {
3153  p++;
3154  }
3155  _http_version_string = line.substr(0, p);
3156  _http_version = HTTPClient::parse_http_version_string(_http_version_string);
3157 
3158  while (p < line.length() && isspace(line[p])) {
3159  p++;
3160  }
3161  size_t q = p;
3162  while (q < line.length() && !isspace(line[q])) {
3163  q++;
3164  }
3165  string status_code = line.substr(p, q - p);
3166  _status_entry._status_code = atoi(status_code.c_str());
3167 
3168  while (q < line.length() && isspace(line[q])) {
3169  q++;
3170  }
3171  _status_entry._status_string = line.substr(q, line.length() - q);
3172 
3173  return true;
3174 }
3175 
3176 ////////////////////////////////////////////////////////////////////
3177 // Function: HTTPChannel::parse_http_header
3178 // Access: Private
3179 // Description: Reads the series of header lines from the server and
3180 // stores them in _headers. Returns true if there is
3181 // more to read, false when done.
3182 ////////////////////////////////////////////////////////////////////
3183 bool HTTPChannel::
3184 parse_http_header() {
3185  string line;
3186  if (!server_getline(line)) {
3187  return true;
3188  }
3189 
3190  while (!line.empty()) {
3191  if (isspace(line[0])) {
3192  // If the line begins with a space, that continues the previous
3193  // field.
3194  size_t p = 0;
3195  while (p < line.length() && isspace(line[p])) {
3196  p++;
3197  }
3198  _current_field_value += line.substr(p - 1);
3199 
3200  } else {
3201  // If the line does not begin with a space, that defines a new
3202  // field.
3203  if (!_current_field_name.empty()) {
3204  store_header_field(_current_field_name, _current_field_value);
3205  _current_field_value = string();
3206  }
3207 
3208  size_t colon = line.find(':');
3209  if (colon != string::npos) {
3210  _current_field_name = downcase(line.substr(0, colon));
3211  size_t p = colon + 1;
3212  while (p < line.length() && isspace(line[p])) {
3213  p++;
3214  }
3215  _current_field_value = line.substr(p);
3216  }
3217  }
3218 
3219  if (!server_getline(line)) {
3220  return true;
3221  }
3222  }
3223 
3224  // After reading an empty line, we're done with the headers.
3225  if (!_current_field_name.empty()) {
3226  store_header_field(_current_field_name, _current_field_value);
3227  _current_field_value = string();
3228  }
3229 
3230  return false;
3231 }
3232 
3233 ////////////////////////////////////////////////////////////////////
3234 // Function: HTTPChannel::parse_content_range
3235 // Access: Private
3236 // Description: Interprets the "Content-Range" header in the reply,
3237 // and fills in _first_byte_delivered and
3238 // _last_byte_delivered appropriately if the header
3239 // response can be understood.
3240 ////////////////////////////////////////////////////////////////////
3241 bool HTTPChannel::
3242 parse_content_range(const string &content_range) {
3243  // First, get the units indication.
3244  size_t p = 0;
3245  while (p < content_range.length() && !isspace(content_range[p])) {
3246  p++;
3247  }
3248 
3249  string units = content_range.substr(0, p);
3250  while (p < content_range.length() && isspace(content_range[p])) {
3251  p++;
3252  }
3253 
3254  if (units == "bytes") {
3255  const char *c_str = content_range.c_str();
3256  char *endptr;
3257  if (p < content_range.length() && isdigit(content_range[p])) {
3258  long first_byte = strtol(c_str + p, &endptr, 10);
3259  p = endptr - c_str;
3260  if (p < content_range.length() && content_range[p] == '-') {
3261  p++;
3262  if (p < content_range.length() && isdigit(content_range[p])) {
3263  long last_byte = strtol(c_str + p, &endptr, 10);
3264  p = endptr - c_str;
3265 
3266  if (last_byte >= first_byte) {
3267  _first_byte_delivered = first_byte;
3268  _last_byte_delivered = last_byte;
3269  return true;
3270  }
3271  }
3272  }
3273  }
3274  }
3275 
3276  // Invalid or unhandled response.
3277  return false;
3278 }
3279 
3280 
3281 ////////////////////////////////////////////////////////////////////
3282 // Function: HTTPChannel::check_socket
3283 // Access: Private
3284 // Description: Checks whether the connection to the server has been
3285 // closed after a failed read. If it has, issues a
3286 // warning and calls reset_to_new().
3287 ////////////////////////////////////////////////////////////////////
3288 void HTTPChannel::
3289 check_socket() {
3290  nassertv(!_source.is_null());
3291  if ((*_source)->is_closed()) {
3292  if (downloader_cat.is_debug()) {
3293  downloader_cat.debug()
3294  << _NOTIFY_HTTP_CHANNEL_ID
3295  << "Lost connection to server unexpectedly during read.\n";
3296  }
3297  reset_to_new();
3298  }
3299 }
3300 
3301 /*
3302  Certificate verify error codes:
3303 
3304 0 X509_V_OK: ok
3305 
3306  the operation was successful.
3307 
3308 2 X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT: unable to get issuer certificate
3309 
3310  the issuer certificate could not be found: this occurs if the
3311  issuer certificate of an untrusted certificate cannot be found.
3312 
3313 3 X509_V_ERR_UNABLE_TO_GET_CRL unable to get certificate CRL
3314 
3315  the CRL of a certificate could not be found. Unused.
3316 
3317 4 X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE: unable to decrypt
3318 certificate's signature
3319 
3320  the certificate signature could not be decrypted. This means that
3321  the actual signature value could not be determined rather than it
3322  not matching the expected value, this is only meaningful for RSA
3323  keys.
3324 
3325 5 X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE: unable to decrypt CRL's signature
3326 
3327  the CRL signature could not be decrypted: this means that the
3328  actual signature value could not be determined rather than it not
3329  matching the expected value. Unused.
3330 
3331 6 X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY: unable to decode
3332 issuer public key
3333 
3334  the public key in the certificate SubjectPublicKeyInfo could not
3335  be read.
3336 
3337 7 X509_V_ERR_CERT_SIGNATURE_FAILURE: certificate signature failure
3338 
3339  the signature of the certificate is invalid.
3340 
3341 8 X509_V_ERR_CRL_SIGNATURE_FAILURE: CRL signature failure
3342 
3343  the signature of the certificate is invalid. Unused.
3344 
3345 9 X509_V_ERR_CERT_NOT_YET_VALID: certificate is not yet valid
3346 
3347  the certificate is not yet valid: the notBefore date is after the
3348  current time.
3349 
3350 10 X509_V_ERR_CERT_HAS_EXPIRED: certificate has expired
3351 
3352  the certificate has expired: that is the notAfter date is before
3353  the current time.
3354 
3355 11 X509_V_ERR_CRL_NOT_YET_VALID: CRL is not yet valid
3356 
3357  the CRL is not yet valid. Unused.
3358 
3359 12 X509_V_ERR_CRL_HAS_EXPIRED: CRL has expired
3360 
3361  the CRL has expired. Unused.
3362 
3363 13 X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD: format error in
3364 certificate's notBefore field
3365 
3366  the certificate notBefore field contains an invalid time.
3367 
3368 14 X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD: format error in
3369 certificate's notAfter field
3370 
3371  the certificate notAfter field contains an invalid time.
3372 
3373 15 X509_V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD: format error in CRL's
3374 lastUpdate field
3375 
3376  the CRL lastUpdate field contains an invalid time. Unused.
3377 
3378 16 X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD: format error in CRL's
3379 nextUpdate field
3380 
3381  the CRL nextUpdate field contains an invalid time. Unused.
3382 
3383 17 X509_V_ERR_OUT_OF_MEM: out of memory
3384 
3385  an error occurred trying to allocate memory. This should never
3386  happen.
3387 
3388 18 X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT: self signed certificate
3389 
3390  the passed certificate is self signed and the same certificate
3391  cannot be found in the list of trusted certificates.
3392 
3393 19 X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN: self signed certificate in
3394 certificate chain
3395 
3396  the certificate chain could be built up using the untrusted
3397  certificates but the root could not be found locally.
3398 
3399 20 X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY: unable to get local
3400 issuer certificate
3401 
3402  the issuer certificate of a locally looked up certificate could
3403  not be found. This normally means the list of trusted certificates
3404  is not complete.
3405 
3406 21 X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE: unable to verify the
3407 first certificate
3408 
3409  no signatures could be verified because the chain contains only
3410  one certificate and it is not self signed.
3411 
3412 22 X509_V_ERR_CERT_CHAIN_TOO_LONG: certificate chain too long
3413 
3414  the certificate chain length is greater than the supplied maximum
3415  depth. Unused.
3416 
3417 23 X509_V_ERR_CERT_REVOKED: certificate revoked
3418 
3419  the certificate has been revoked. Unused.
3420 
3421 24 X509_V_ERR_INVALID_CA: invalid CA certificate
3422 
3423  a CA certificate is invalid. Either it is not a CA or its
3424  extensions are not consistent with the supplied purpose.
3425 
3426 25 X509_V_ERR_PATH_LENGTH_EXCEEDED: path length constraint exceeded
3427 
3428  the basicConstraints pathlength parameter has been exceeded.
3429 
3430 26 X509_V_ERR_INVALID_PURPOSE: unsupported certificate purpose
3431 
3432  the supplied certificate cannot be used for the specified purpose.
3433 
3434 27 X509_V_ERR_CERT_UNTRUSTED: certificate not trusted
3435 
3436  the root CA is not marked as trusted for the specified purpose.
3437 
3438 28 X509_V_ERR_CERT_REJECTED: certificate rejected
3439 
3440  the root CA is marked to reject the specified purpose.
3441 
3442 29 X509_V_ERR_SUBJECT_ISSUER_MISMATCH: subject issuer mismatch
3443 
3444  the current candidate issuer certificate was rejected because its
3445  subject name did not match the issuer name of the current
3446  certificate. Only displayed when the -issuer_checks option is set.
3447 
3448 30 X509_V_ERR_AKID_SKID_MISMATCH: authority and subject key identifier
3449 mismatch
3450 
3451  the current candidate issuer certificate was rejected because its
3452  subject key identifier was present and did not match the authority
3453  key identifier current certificate. Only displayed when the
3454  -issuer_checks option is set.
3455 
3456 31 X509_V_ERR_AKID_ISSUER_SERIAL_MISMATCH: authority and issuer serial
3457 number mismatch
3458 
3459  the current candidate issuer certificate was rejected because its
3460  issuer name and serial number was present and did not match the
3461  authority key identifier of the current certificate. Only
3462  displayed when the -issuer_checks option is set.
3463 
3464 32 X509_V_ERR_KEYUSAGE_NO_CERTSIGN:key usage does not include
3465 certificate signing
3466 
3467  the current candidate issuer certificate was rejected because its
3468  keyUsage extension does not permit certificate signing.
3469 
3470 50 X509_V_ERR_APPLICATION_VERIFICATION: application verification failure
3471 
3472  an application specific error. Unused.
3473 
3474 */
3475 
3476 ////////////////////////////////////////////////////////////////////
3477 // Function: HTTPChannel::check_preapproved_server_certificate
3478 // Access: Private
3479 // Description: Checks to see if the indicated certificate is on the
3480 // pre-approved list for the current server.
3481 //
3482 // If the full cert itself (including its key) is on the
3483 // pre-approved list, sets both cert_preapproved and
3484 // cert_name_preapproved to true.
3485 //
3486 // If the full cert is not on the pre-approved list, but
3487 // its name matches a name on the pre-approved list,
3488 // sets cert_name_preapproved to true, and
3489 // cert_preapproved to false.
3490 //
3491 // Otherwise, sets both values to false. This doesn't
3492 // mean the cert is necessarily invalid, just that it
3493 // wasn't on the pre-approved list (which is usually
3494 // empty anyway).
3495 ////////////////////////////////////////////////////////////////////
3496 void HTTPChannel::
3497 check_preapproved_server_certificate(X509 *cert, bool &cert_preapproved,
3498  bool &cert_name_preapproved) const {
3499  return _client->check_preapproved_server_certificate(_request.get_url(),
3500  cert, cert_preapproved,
3501  cert_name_preapproved);
3502 }
3503 
3504 ////////////////////////////////////////////////////////////////////
3505 // Function: HTTPChannel::validate_server_name
3506 // Access: Private
3507 // Description: Returns true if the name in the cert matches the
3508 // hostname of the server, false otherwise.
3509 ////////////////////////////////////////////////////////////////////
3510 bool HTTPChannel::
3511 validate_server_name(X509 *cert) {
3512  string hostname = _request.get_url().get_server();
3513 
3514  vector_string cert_names;
3515 
3516  // According to RFC 2818, we should check the DNS name(s) in the
3517  // subjectAltName extension first, if that extension exists.
3518  STACK_OF(GENERAL_NAME) *subject_alt_names =
3519  (STACK_OF(GENERAL_NAME) *)X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL);
3520  if (subject_alt_names != NULL) {
3521  int num_alts = sk_GENERAL_NAME_num(subject_alt_names);
3522  for (int i = 0; i < num_alts; ++i) {
3523  // Get the ith alt name.
3524  const GENERAL_NAME *alt_name =
3525  sk_GENERAL_NAME_value(subject_alt_names, i);
3526 
3527  if (alt_name->type == GEN_DNS) {
3528  char *buffer = NULL;
3529  int len = ASN1_STRING_to_UTF8((unsigned char**)&buffer,
3530  alt_name->d.ia5);
3531  if (len > 0) {
3532  cert_names.push_back(string(buffer, len));
3533  }
3534  if (buffer != NULL) {
3535  OPENSSL_free(buffer);
3536  }
3537  }
3538  }
3539  }
3540 
3541  if (cert_names.empty()) {
3542  // If there were no DNS names, use the common name instead.
3543 
3544  X509_NAME *xname = X509_get_subject_name(cert);
3545  if (xname != NULL) {
3546  string common_name = get_x509_name_component(xname, NID_commonName);
3547  cert_names.push_back(common_name);
3548  }
3549  }
3550 
3551  if (cert_names.empty()) {
3552  downloader_cat.info()
3553  << _NOTIFY_HTTP_CHANNEL_ID
3554  << "Server certificate from " << hostname
3555  << " provides no name.\n";
3556  return false;
3557  }
3558 
3559  if (downloader_cat.is_debug()) {
3560  downloader_cat.debug()
3561  << _NOTIFY_HTTP_CHANNEL_ID
3562  << "Server certificate from " << hostname
3563  << " provides name(s):";
3564  vector_string::const_iterator si;
3565  for (si = cert_names.begin(); si != cert_names.end(); ++si) {
3566  const string &cert_name = (*si);
3567  downloader_cat.debug(false)
3568  << " " << cert_name;
3569  }
3570  downloader_cat.debug(false)
3571  << "\n";
3572  }
3573 
3574  // Now validate the names we found. If any of them matches, the
3575  // cert matches.
3576  vector_string::const_iterator si;
3577  for (si = cert_names.begin(); si != cert_names.end(); ++si) {
3578  const string &cert_name = (*si);
3579 
3580  if (match_cert_name(cert_name, hostname)) {
3581  return true;
3582  }
3583  }
3584 
3585  downloader_cat.info()
3586  << _NOTIFY_HTTP_CHANNEL_ID
3587  << "Server certificate from " << hostname
3588  << " provides wrong name(s):";
3589  for (si = cert_names.begin(); si != cert_names.end(); ++si) {
3590  const string &cert_name = (*si);
3591  downloader_cat.info(false)
3592  << " " << cert_name;
3593  }
3594  downloader_cat.info(false)
3595  << "\n";
3596 
3597  return false;
3598 }
3599 
3600 ////////////////////////////////////////////////////////////////////
3601 // Function: HTTPChannel::match_cert_name
3602 // Access: Private, Static
3603 // Description: Returns true if this particular name from the
3604 // certificate matches the indicated hostname, false
3605 // otherwise.
3606 ////////////////////////////////////////////////////////////////////
3607 bool HTTPChannel::
3608 match_cert_name(const string &cert_name, const string &hostname) {
3609  // We use GlobPattern to match the name. This isn't quite
3610  // consistent with RFC2818, since it also accepts additional
3611  // wildcard characters like "?" and "[]", but I think it's close
3612  // enough.
3613 
3614  GlobPattern pattern(cert_name);
3615  pattern.set_case_sensitive(false);
3616  pattern.set_nomatch_chars(".");
3617  return pattern.matches(hostname);
3618 }
3619 
3620 ////////////////////////////////////////////////////////////////////
3621 // Function: HTTPChannel::get_x509_name_component
3622 // Access: Private, Static
3623 // Description: Returns the indicated component of the X509 name as a
3624 // string, if defined, or empty string if it is not.
3625 ////////////////////////////////////////////////////////////////////
3626 string HTTPChannel::
3627 get_x509_name_component(X509_NAME *name, int nid) {
3628  ASN1_OBJECT *obj = OBJ_nid2obj(nid);
3629 
3630  if (obj == NULL) {
3631  // Unknown nid. See openssl/objects.h.
3632  return string();
3633  }
3634 
3635  int i = X509_NAME_get_index_by_OBJ(name, obj, -1);
3636  if (i < 0) {
3637  return string();
3638  }
3639 
3640  ASN1_STRING *data = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(name, i));
3641  return string((char *)data->data, data->length);
3642 }
3643 
3644 ////////////////////////////////////////////////////////////////////
3645 // Function: HTTPChannel::make_header
3646 // Access: Private
3647 // Description: Formats the appropriate GET or POST (or whatever)
3648 // request to send to the server, based on the current
3649 // _method, _document_spec, _body, and _proxy settings.
3650 ////////////////////////////////////////////////////////////////////
3651 void HTTPChannel::
3652 make_header() {
3653  _proxy_auth = _client->select_auth(_proxy, true, _proxy_realm);
3654  _proxy_username = string();
3655  if (_proxy_auth != (HTTPAuthorization *)NULL) {
3656  _proxy_realm = _proxy_auth->get_realm();
3657  _proxy_username = _client->select_username(_proxy, true, _proxy_realm);
3658  }
3659 
3660  if (_method == HTTPEnum::M_connect) {
3661  // This method doesn't require an HTTP header at all; we'll just
3662  // open a plain connection. (Except when we're using a proxy; but
3663  // in that case, it's the proxy_header we'll need, not the regular
3664  // HTTP header.)
3665  _header = string();
3666  return;
3667  }
3668 
3669  _www_auth = _client->select_auth(_request.get_url(), false, _www_realm);
3670  _www_username = string();
3671  if (_www_auth != (HTTPAuthorization *)NULL) {
3672  _www_realm = _www_auth->get_realm();
3673  _www_username = _client->select_username(_request.get_url(), false, _www_realm);
3674  }
3675 
3676  string request_path;
3677  if (_proxy_serves_document) {
3678  // If we'll be asking the proxy for the document, we need its full
3679  // URL--but we omit the username, which is information just for us.
3680  URLSpec url_no_username = _request.get_url();
3681  url_no_username.set_username(string());
3682  request_path = url_no_username.get_url();
3683 
3684  } else {
3685  // If we'll be asking the server directly for the document, we
3686  // just want its path relative to the server.
3687  request_path = _request.get_url().get_path_and_query();
3688  }
3689 
3690  // HTTP syntax always requires something in the request path. If it
3691  // is empty, put in a star as a placeholder (OPTIONS, for instance,
3692  // uses this).
3693  if (request_path.empty()) {
3694  request_path = "*";
3695  }
3696 
3697  ostringstream stream;
3698 
3699  stream
3700  << _method << " " << request_path << " "
3701  << _client->get_http_version_string() << "\r\n";
3702 
3703  if (_client->get_http_version() >= HTTPEnum::HV_11) {
3704 
3705  stream
3706  << "Host: " << _request.get_url().get_server();
3707  if (!_request.get_url().is_default_port()) {
3708  // It appears that some servers (notably gstatic.com) might
3709  // return a 404 if you include an explicit port number in with
3710  // the Host: header, even if it is the default port. So, don't
3711  // include the port number unless we need to.
3712  stream << ":" << _request.get_url().get_port();
3713  }
3714  stream << "\r\n";
3715  if (!get_persistent_connection()) {
3716  stream
3717  << "Connection: close\r\n";
3718  }
3719  }
3720 
3721  if (_last_byte_requested != 0) {
3722  stream
3723  << "Range: bytes=" << _first_byte_requested << "-"
3724  << _last_byte_requested << "\r\n";
3725 
3726  } else if (_first_byte_requested != 0) {
3727  stream
3728  << "Range: bytes=" << _first_byte_requested << "-\r\n";
3729  }
3730 
3731  switch (_request.get_request_mode()) {
3732  case DocumentSpec::RM_any:
3733  // No particular request; give us any document that matches the
3734  // URL.
3735  if (_first_byte_requested != 0) {
3736  // Unless we're requesting a subrange, in which case if the
3737  // exact document matches, retrieve the subrange indicated;
3738  // otherwise, retrieve the entire document.
3739  if (_request.has_tag()) {
3740  stream
3741  << "If-Range: " << _request.get_tag().get_string() << "\r\n";
3742  } else if (_request.has_date()) {
3743  stream
3744  << "If-Range: " << _request.get_date().get_string() << "\r\n";
3745  }
3746  }
3747  break;
3748 
3749  case DocumentSpec::RM_equal:
3750  // Give us only this particular version of the document, or
3751  // nothing.
3752  if (_request.has_tag()) {
3753  stream
3754  << "If-Match: " << _request.get_tag().get_string() << "\r\n";
3755  }
3756  if (_request.has_date()) {
3757  stream
3758  << "If-Unmodified-Since: " << _request.get_date().get_string()
3759  << "\r\n";
3760  }
3761  break;
3762 
3763  case DocumentSpec::RM_newer:
3764  // Give us anything newer than this document, or nothing.
3765  if (_request.has_tag()) {
3766  stream
3767  << "If-None-Match: " << _request.get_tag().get_string() << "\r\n";
3768  }
3769  if (_request.has_date()) {
3770  stream
3771  << "If-Modified-Since: " << _request.get_date().get_string()
3772  << "\r\n";
3773  }
3774  break;
3775 
3776  case DocumentSpec::RM_equal_or_newer:
3777  // Just don't give us anything older.
3778  if (_request.has_date()) {
3779  // This is a little unreliable: we ask for any document that's
3780  // been modified since one second before our last-modified-date.
3781  // Who knows whether the server will honor this properly.
3782  stream
3783  << "If-Modified-Since: " << (_request.get_date() - 1).get_string()
3784  << "\r\n";
3785  }
3786  break;
3787  }
3788 
3789  switch (_request.get_cache_control()) {
3790  case DocumentSpec::CC_allow_cache:
3791  // Normal, caching behavior.
3792  break;
3793 
3794  case DocumentSpec::CC_revalidate:
3795  // Request the server to revalidate its cache before returning it.
3796  stream
3797  << "Cache-Control: max-age=0\r\n";
3798  break;
3799 
3800  case DocumentSpec::CC_no_cache:
3801  // Request the server to get a fresh copy regardless of its cache.
3802  stream
3803  << "Cache-Control: no-cache\r\n"
3804  << "Pragma: no-cache\r\n";
3805  break;
3806  }
3807 
3808  _client->send_cookies(stream, _request.get_url());
3809 
3810  if (!_body.empty()) {
3811  stream
3812  << "Content-Type: application/x-www-form-urlencoded\r\n"
3813  << "Content-Length: " << _body.length() << "\r\n";
3814  }
3815 
3816  _header = stream.str();
3817 }
3818 
3819 ////////////////////////////////////////////////////////////////////
3820 // Function: HTTPChannel::make_proxy_request_text
3821 // Access: Private
3822 // Description: Builds the _proxy_request_text string. This is a
3823 // special request that will be sent directly to the
3824 // proxy prior to the request tailored for the server.
3825 // Generally this is used to open a tunnelling
3826 // connection for https-over-proxy.
3827 ////////////////////////////////////////////////////////////////////
3828 void HTTPChannel::
3829 make_proxy_request_text() {
3830  _proxy_request_text = _proxy_header;
3831 
3832  if (_proxy_auth != (HTTPAuthorization *)NULL && !_proxy_username.empty()) {
3833  _proxy_request_text += "Proxy-Authorization: ";
3834  _proxy_request_text +=
3835  _proxy_auth->generate(HTTPEnum::M_connect, _request.get_url().get_server_and_port(),
3836  _proxy_username, _body);
3837  _proxy_request_text += "\r\n";
3838  }
3839 
3840  _proxy_request_text += "\r\n";
3841 }
3842 
3843 ////////////////////////////////////////////////////////////////////
3844 // Function: HTTPChannel::make_request_text
3845 // Access: Private
3846 // Description: Builds the _request_text string. This is the
3847 // specific request that will be sent to the server this
3848 // pass, based on the current header and body.
3849 ////////////////////////////////////////////////////////////////////
3850 void HTTPChannel::
3851 make_request_text() {
3852  _request_text = _header;
3853 
3854  if (_proxy_serves_document &&
3855  _proxy_auth != (HTTPAuthorization *)NULL && !_proxy_username.empty()) {
3856  _request_text += "Proxy-Authorization: ";
3857  _request_text +=
3858  _proxy_auth->generate(_method, _request.get_url().get_url(), _proxy_username, _body);
3859  _request_text += "\r\n";
3860  }
3861 
3862  if (_www_auth != (HTTPAuthorization *)NULL && !_www_username.empty()) {
3863  string authorization =
3864  _request_text += "Authorization: ";
3865  _request_text +=
3866  _www_auth->generate(_method, _request.get_url().get_path_and_query(), _www_username, _body);
3867  _request_text += "\r\n";
3868  }
3869 
3870  _request_text += _send_extra_headers;
3871  _request_text += "\r\n";
3872  _request_text += _body;
3873 }
3874 
3875 ////////////////////////////////////////////////////////////////////
3876 // Function: HTTPChannel::reset_url
3877 // Access: Private
3878 // Description: Redirects the next connection to the indicated URL
3879 // (from the previous URL). This resets the socket if
3880 // necessary when we are about to switch servers.
3881 ////////////////////////////////////////////////////////////////////
3882 void HTTPChannel::
3883 reset_url(const URLSpec &old_url, const URLSpec &new_url) {
3884  // If we change between http and https, we have to reset the
3885  // connection regardless of proxy. Otherwise, we have to drop the
3886  // connection if the server or port changes, unless we're
3887  // communicating through a proxy.
3888 
3889  if (new_url.get_scheme() != old_url.get_scheme() ||
3890  (_proxy.empty() && (new_url.get_server() != old_url.get_server() ||
3891  new_url.get_port() != old_url.get_port()))) {
3892  if (downloader_cat.is_debug()) {
3893  downloader_cat.debug()
3894  << _NOTIFY_HTTP_CHANNEL_ID
3895  << "resetting for new server "
3896  << new_url.get_server_and_port() << "\n";
3897  }
3898  reset_to_new();
3899  }
3900 }
3901 
3902 ////////////////////////////////////////////////////////////////////
3903 // Function: HTTPChannel::store_header_field
3904 // Access: Private
3905 // Description: Stores a single name: value pair in the header list,
3906 // or appends the value to the end of the existing
3907 // value, if the header has been repeated.
3908 ////////////////////////////////////////////////////////////////////
3909 void HTTPChannel::
3910 store_header_field(const string &field_name, const string &field_value) {
3911  pair<Headers::iterator, bool> insert_result =
3912  _headers.insert(Headers::value_type(field_name, field_value));
3913 
3914  if (!insert_result.second) {
3915  // It didn't insert; thus, the field already existed. Append the
3916  // new value.
3917  Headers::iterator hi = insert_result.first;
3918  (*hi).second += ", ";
3919  (*hi).second += field_value;
3920  }
3921 
3922  if (field_name == "set-cookie") {
3923  _client->set_cookie(HTTPCookie(field_value, _request.get_url()));
3924  }
3925 }
3926 
3927 #ifndef NDEBUG
3928 ////////////////////////////////////////////////////////////////////
3929 // Function: HTTPChannel::show_send
3930 // Access: Private, Static
3931 // Description: Writes the outgoing message, one line at a time, to
3932 // the debugging log.
3933 ////////////////////////////////////////////////////////////////////
3934 void HTTPChannel::
3935 show_send(const string &message) {
3936  size_t start = 0;
3937  size_t newline = message.find('\n', start);
3938  while (newline != string::npos) {
3939  // Assume every \n is preceded by a \r.
3940  downloader_cat.debug()
3941  << "send: " << message.substr(start, newline - start - 1) << "\n";
3942  start = newline + 1;
3943  newline = message.find('\n', start);
3944  }
3945 
3946  if (start < message.length()) {
3947  downloader_cat.debug()
3948  << "send: " << message.substr(start) << " (no newline)\n";
3949  }
3950 }
3951 #endif // NDEBUG
3952 
3953 ////////////////////////////////////////////////////////////////////
3954 // Function: HTTPChannel::reset_download_to
3955 // Access: Private
3956 // Description: Resets the indication of how the document will be
3957 // downloaded. This must be re-specified after each
3958 // get_document() (or related) call.
3959 ////////////////////////////////////////////////////////////////////
3960 void HTTPChannel::
3961 reset_download_to() {
3962  _started_download = false;
3963  close_download_stream();
3964  _download_dest = DD_none;
3965 }
3966 
3967 ////////////////////////////////////////////////////////////////////
3968 // Function: HTTPChannel::close_download_stream
3969 // Access: Private
3970 // Description: Ensures the file opened for receiving the download
3971 // has been correctly closed.
3972 ////////////////////////////////////////////////////////////////////
3973 void HTTPChannel::
3974 close_download_stream() {
3975  if (_download_to_stream != NULL) {
3976  _download_to_stream->flush();
3977  if (_download_dest == DD_file) {
3978  VirtualFileSystem::close_write_file(_download_to_stream);
3979  }
3980  }
3981  _download_to_ramfile = (Ramfile *)NULL;
3982  _download_to_stream = NULL;
3983 }
3984 
3985 
3986 ////////////////////////////////////////////////////////////////////
3987 // Function: HTTPChannel::reset_to_new
3988 // Access: Private
3989 // Description: Closes the connection and resets the state to S_new.
3990 ////////////////////////////////////////////////////////////////////
3991 void HTTPChannel::
3992 reset_to_new() {
3993  if (downloader_cat.is_spam()) {
3994  downloader_cat.spam()
3995  << _NOTIFY_HTTP_CHANNEL_ID
3996  << "reset_to_new.\n";
3997  }
3998 
3999  close_connection();
4000  _state = S_new;
4001 }
4002 
4003 ////////////////////////////////////////////////////////////////////
4004 // Function: HTTPChannel::reset_body_stream
4005 // Access: Private
4006 // Description: Clears the _body_stream pointer, if it is set.
4007 ////////////////////////////////////////////////////////////////////
4008 void HTTPChannel::
4009 reset_body_stream() {
4010  if (_owns_body_stream) {
4011  if (_body_stream != (ISocketStream *)NULL) {
4012  close_read_body(_body_stream);
4013  nassertv(_body_stream == (ISocketStream *)NULL && !_owns_body_stream);
4014  }
4015  } else {
4016  _body_stream = NULL;
4017  }
4018 }
4019 
4020 
4021 ////////////////////////////////////////////////////////////////////
4022 // Function: HTTPChannel::close_connection
4023 // Access: Private
4024 // Description: Closes the connection but leaves the _state
4025 // unchanged.
4026 ////////////////////////////////////////////////////////////////////
4027 void HTTPChannel::
4028 close_connection() {
4029  reset_body_stream();
4030  _source.clear();
4031  _bio.clear();
4032  _working_get = string();
4033  _sent_so_far = 0;
4034  _read_index++;
4035 }
4036 
4037 ////////////////////////////////////////////////////////////////////
4038 // Function: HTTPChannel::more_useful_status_code
4039 // Access: Private, Static
4040 // Description: Returns true if status code a is a more useful value
4041 // (that is, it represents a more-nearly successfully
4042 // connection attempt, or contains more information)
4043 // than b, or false otherwise.
4044 ////////////////////////////////////////////////////////////////////
4045 bool HTTPChannel::
4046 more_useful_status_code(int a, int b) {
4047  if (a >= 100 && b >= 100) {
4048  // Both represent HTTP responses. Responses from a server (<
4049  // 1000) are better than those from a proxy; we take advantage of
4050  // the fact that we have already added 1000 to proxy responses.
4051  // Except for 407, so let's fix that now.
4052  if (a == 407) {
4053  a += 1000;
4054  }
4055  if (b == 407) {
4056  b += 1000;
4057  }
4058 
4059  // Now just check the series.
4060  int series_a = (a / 100);
4061  int series_b = (b / 100);
4062 
4063  // In general, a lower series is a closer success.
4064  return (series_a < series_b);
4065  }
4066 
4067  if (a < 100 && b < 100) {
4068  // Both represent non-HTTP responses. Here a larger number is
4069  // better.
4070  return (a > b);
4071  }
4072 
4073  if (a < 100) {
4074  // a is a non-HTTP response, while b is an HTTP response. HTTP is
4075  // generally, better, unless we exceeded SC_http_error_watermark.
4076  return (a > SC_http_error_watermark);
4077  }
4078 
4079  // Exactly the opposite case as above.
4080  return (b < SC_http_error_watermark);
4081 }
4082 
4083 
4084 ////////////////////////////////////////////////////////////////////
4085 // Function: HTTPChannel::State output operator
4086 // Description:
4087 ////////////////////////////////////////////////////////////////////
4088 ostream &
4089 operator << (ostream &out, HTTPChannel::State state) {
4090 #ifdef NDEBUG
4091  return out << (int)state;
4092 #else
4093  switch (state) {
4094  case HTTPChannel::S_new:
4095  return out << "new";
4096 
4097  case HTTPChannel::S_try_next_proxy:
4098  return out << "try_next_proxy";
4099 
4100  case HTTPChannel::S_connecting:
4101  return out << "connecting";
4102 
4103  case HTTPChannel::S_connecting_wait:
4104  return out << "connecting_wait";
4105 
4106  case HTTPChannel::S_http_proxy_ready:
4107  return out << "http_proxy_ready";
4108 
4109  case HTTPChannel::S_http_proxy_request_sent:
4110  return out << "http_proxy_request_sent";
4111 
4112  case HTTPChannel::S_http_proxy_reading_header:
4113  return out << "http_proxy_reading_header";
4114 
4115  case HTTPChannel::S_socks_proxy_greet:
4116  return out << "socks_proxy_greet";
4117 
4118  case HTTPChannel::S_socks_proxy_greet_reply:
4119  return out << "socks_proxy_greet_reply";
4120 
4121  case HTTPChannel::S_socks_proxy_connect:
4122  return out << "socks_proxy_connect";
4123 
4124  case HTTPChannel::S_socks_proxy_connect_reply:
4125  return out << "socks_proxy_connect_reply";
4126 
4127  case HTTPChannel::S_setup_ssl:
4128  return out << "setup_ssl";
4129 
4130  case HTTPChannel::S_ssl_handshake:
4131  return out << "ssl_handshake";
4132 
4133  case HTTPChannel::S_ready:
4134  return out << "ready";
4135 
4136  case HTTPChannel::S_request_sent:
4137  return out << "request_sent";
4138 
4139  case HTTPChannel::S_reading_header:
4140  return out << "reading_header";
4141 
4142  case HTTPChannel::S_start_direct_file_read:
4143  return out << "start_direct_file_read";
4144 
4145  case HTTPChannel::S_read_header:
4146  return out << "read_header";
4147 
4148  case HTTPChannel::S_begin_body:
4149  return out << "begin_body";
4150 
4151  case HTTPChannel::S_reading_body:
4152  return out << "reading_body";
4153 
4154  case HTTPChannel::S_read_body:
4155  return out << "read_body";
4156 
4157  case HTTPChannel::S_read_trailer:
4158  return out << "read_trailer";
4159 
4160  case HTTPChannel::S_failure:
4161  return out << "failure";
4162  }
4163 
4164  return out << "invalid state(" << (int)state << ")";
4165 #endif // NDEBUG
4166 }
4167 
4168 #endif // HAVE_OPENSSL
static TrueClock * get_global_ptr()
Returns a pointer to the one TrueClock object in the world.
Definition: trueClock.I:81
A container for a URL, e.g.
Definition: urlSpec.h:29
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS&#39;s file system.
A container for an &quot;entity tag&quot; from an HTTP server.
Definition: httpEntityTag.h:27
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:494
bool is_local() const
Returns true if the filename is local, e.g.
Definition: filename.I:664
This is a convenience class to specialize ConfigVariable as a floating-point type.
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:44
An in-memory buffer specifically designed for downloading files to memory.
Definition: ramfile.h:27
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
A container for an HTTP-legal time/date indication.
Definition: httpDate.h:30
int get_port() const
Returns the port number specified by the URL, or the default port if not specified.
Definition: urlSpec.cxx:84
string get_scheme() const
Returns the scheme specified by the URL, or empty string if no scheme is specified.
Definition: urlSpec.cxx:70
const string & get_url() const
Returns the complete URL specification.
Definition: urlSpec.I:254
string get_server() const
Returns the server name specified by the URL, if any.
Definition: urlSpec.I:198
ostream * open_write_file(const Filename &filename, bool auto_wrap, bool truncate)
Convenience function; returns a newly allocated ostream if the file exists and can be written...
string get_server_and_port() const
Returns a string consisting of the server name, followed by a colon, followed by the port number...
Definition: urlSpec.cxx:137
string get_authority() const
Returns the authority specified by the URL (this includes username, server, and/or port)...
Definition: urlSpec.I:175
string get_path() const
Returns the path specified by the URL, or &quot;/&quot; if no path is specified.
Definition: urlSpec.cxx:153
A descriptor that refers to a particular version of a document.
Definition: documentSpec.h:33
TypeHandle is the identifier used to differentiate C++ class types.
Definition: typeHandle.h:85
static void close_write_file(ostream *stream)
Closes a file opened by a previous call to open_write_file().
void set_username(const string &username)
Replaces the username part of the URL specification.
Definition: urlSpec.cxx:308
This class can be used to test for string matches against standard Unix-shell filename globbing conve...
Definition: globPattern.h:37