Panda3D

httpChannel.cxx

00001 // Filename: httpChannel.cxx
00002 // Created by:  drose (24Sep02)
00003 //
00004 ////////////////////////////////////////////////////////////////////
00005 //
00006 // PANDA 3D SOFTWARE
00007 // Copyright (c) Carnegie Mellon University.  All rights reserved.
00008 //
00009 // All use of this software is subject to the terms of the revised BSD
00010 // license.  You should have received a copy of this license along
00011 // with this source code in a file named "LICENSE."
00012 //
00013 ////////////////////////////////////////////////////////////////////
00014 
00015 #include "httpChannel.h"
00016 #include "httpClient.h"
00017 #include "httpCookie.h"
00018 #include "bioStream.h"
00019 #include "chunkedStream.h"
00020 #include "identityStream.h"
00021 #include "config_downloader.h"
00022 #include "virtualFileMountHTTP.h"
00023 #include "ramfile.h"
00024 #include "globPattern.h"
00025 
00026 #include <stdio.h>
00027 
00028 #ifdef HAVE_OPENSSL
00029 
00030 #if defined(WIN32_VC) || defined(WIN64_VC)
00031   #include <WinSock2.h>
00032   #include <windows.h>  // for select()
00033   #undef X509_NAME
00034 #endif  // WIN32_VC
00035 
00036 TypeHandle HTTPChannel::_type_handle;
00037 
00038 ////////////////////////////////////////////////////////////////////
00039 //     Function: HTTPChannel::Constructor
00040 //       Access: Private
00041 //  Description: 
00042 ////////////////////////////////////////////////////////////////////
00043 HTTPChannel::
00044 HTTPChannel(HTTPClient *client) :
00045   _client(client)
00046 {
00047   ConfigVariableDouble extra_ssl_handshake_time
00048     ("extra-ssl-handshake-time", 0.0,
00049      PRC_DESC("This specifies how much extra time to try to establish"
00050                "the ssl handshake before we bail."));
00051   _extra_ssl_handshake_time = extra_ssl_handshake_time;
00052   _proxy_next_index = 0;
00053   _persistent_connection = false;
00054   _allow_proxy = true;
00055   _proxy_tunnel = http_proxy_tunnel;
00056   _connect_timeout = http_connect_timeout;
00057   _http_timeout = http_timeout;
00058   _skip_body_size = http_skip_body_size;
00059   _idle_timeout = http_idle_timeout;
00060   _blocking_connect = false;
00061   _download_throttle = download_throttle;
00062   _max_bytes_per_second = downloader_byte_rate;
00063   _seconds_per_update = downloader_frequency;
00064   _max_updates_per_second = 1.0f / _seconds_per_update;
00065   _bytes_per_update = int(_max_bytes_per_second * _seconds_per_update);
00066   
00067   // _nonblocking is true if the socket is actually in non-blocking mode.
00068   _nonblocking = false;
00069 
00070   // _wanted_nonblocking is true if the user specifically requested
00071   // one of the non-blocking interfaces.  It is false if the socket is
00072   // only incidentally non-blocking (for instance, because
00073   // SIMPLE_THREADS is on).
00074   _wanted_nonblocking = false;
00075 
00076   _want_ssl = false;
00077   _proxy_serves_document = false;
00078   _proxy_tunnel_now = false;
00079   _first_byte_requested = 0;
00080   _last_byte_requested = 0;
00081   _first_byte_delivered = 0;
00082   _last_byte_delivered = 0;
00083   _read_index = 0;
00084   _expected_file_size = 0;
00085   _file_size = 0;
00086   _transfer_file_size = 0;
00087   _got_expected_file_size = false;
00088   _got_file_size = false;
00089   _got_transfer_file_size = false;
00090   _bytes_downloaded = 0;
00091   _bytes_requested = 0;
00092   _status_entry = StatusEntry();
00093   _response_type = RT_none;
00094   _http_version = _client->get_http_version();
00095   _http_version_string = _client->get_http_version_string();
00096   _state = S_new;
00097   _done_state = S_new;
00098   _started_download = false;
00099   _sent_so_far = 0;
00100   _body_stream = NULL;
00101   _owns_body_stream = false;
00102   _sbio = NULL;
00103   _cipher_list = _client->get_cipher_list();
00104   _last_status_code = 0;
00105   _last_run_time = 0.0f;
00106   _download_to_ramfile = NULL;
00107 }
00108 
00109 ////////////////////////////////////////////////////////////////////
00110 //     Function: HTTPChannel::Destructor
00111 //       Access: Public, Virtual
00112 //  Description: 
00113 ////////////////////////////////////////////////////////////////////
00114 HTTPChannel::
00115 ~HTTPChannel() {
00116   close_connection();
00117   reset_download_to();
00118 }
00119 
00120 ////////////////////////////////////////////////////////////////////
00121 //     Function: HTTPChannel::get_status_string
00122 //       Access: Published
00123 //  Description: Returns the string as returned by the server
00124 //               describing the status code for humans.  This may or
00125 //               may not be meaningful.
00126 ////////////////////////////////////////////////////////////////////
00127 string HTTPChannel::
00128 get_status_string() const {
00129   switch (_status_entry._status_code) {
00130   case SC_incomplete:
00131     return "Connection in progress";
00132 
00133   case SC_internal_error:
00134     return "Internal error";
00135 
00136   case SC_no_connection:
00137     return "No connection";
00138 
00139   case SC_timeout:
00140     return "Timeout on connection";
00141 
00142   case SC_lost_connection:
00143     return "Lost connection";
00144 
00145   case SC_non_http_response:
00146     return "Non-HTTP response";
00147 
00148   case SC_invalid_http:
00149     return "Could not understand HTTP response";
00150 
00151   case SC_socks_invalid_version:
00152     return "Unsupported SOCKS version";
00153 
00154   case SC_socks_no_acceptable_login_method:
00155     return "No acceptable SOCKS login method";
00156 
00157   case SC_socks_refused:
00158     return "SOCKS proxy refused connection";
00159 
00160   case SC_socks_no_connection:
00161     return "SOCKS proxy unable to connect";
00162 
00163   case SC_ssl_internal_failure:
00164     return "SSL internal failure";
00165 
00166   case SC_ssl_no_handshake:
00167     return "No SSL handshake";
00168 
00169   case SC_http_error_watermark:
00170     // This shouldn't be triggered.
00171     return "Internal error";
00172 
00173   case SC_ssl_invalid_server_certificate:
00174     return "SSL invalid server certificate";
00175 
00176   case SC_ssl_unexpected_server:
00177     return "Unexpected SSL server";
00178 
00179   case SC_download_open_error:
00180     return "Error opening file";
00181 
00182   case SC_download_write_error:
00183     return "Error writing to disk";
00184 
00185   case SC_download_invalid_range:
00186     return "Invalid subrange requested";
00187   }
00188 
00189   return _status_entry._status_string;
00190 }
00191 
00192 ////////////////////////////////////////////////////////////////////
00193 //     Function: HTTPChannel::get_header_value
00194 //       Access: Published
00195 //  Description: Returns the HTML header value associated with the
00196 //               indicated key, or empty string if the key was not
00197 //               defined in the message returned by the server.
00198 ////////////////////////////////////////////////////////////////////
00199 string HTTPChannel::
00200 get_header_value(const string &key) const {
00201   Headers::const_iterator hi = _headers.find(downcase(key));
00202   if (hi != _headers.end()) {
00203     return (*hi).second;
00204   }
00205   return string();
00206 }
00207 
00208 ////////////////////////////////////////////////////////////////////
00209 //     Function: HTTPChannel::will_close_connection
00210 //       Access: Published
00211 //  Description: Returns true if the server has indicated it will
00212 //               close the connection after this document has been
00213 //               read, or false if it will remain open (and future
00214 //               documents may be requested on the same connection).
00215 ////////////////////////////////////////////////////////////////////
00216 bool HTTPChannel::
00217 will_close_connection() const {
00218   if (get_http_version() < HTTPEnum::HV_11) {
00219     // pre-HTTP 1.1 always closes.
00220     return true;
00221   }
00222 
00223   string connection = get_header_value("Connection");
00224   if (downcase(connection) == "close") {
00225     // The server says it will close.
00226     return true;
00227   }
00228 
00229   if (connection.empty() && !get_persistent_connection()) {
00230     // The server didn't say, but we asked it to close.
00231     return true;
00232   }
00233 
00234   // Assume the server will keep it open.
00235   return false;
00236 }
00237 
00238 ////////////////////////////////////////////////////////////////////
00239 //     Function: HTTPChannel::get_file_size
00240 //       Access: Published, Virtual
00241 //  Description: Returns the size of the file, if it is known.
00242 //               Returns the value set by set_expected_file_size() if
00243 //               the file size is not known, or 0 if this value was
00244 //               not set.
00245 //
00246 //               If the file is dynamically generated, the size may
00247 //               not be available until a read has started
00248 //               (e.g. open_read_body() has been called); and even
00249 //               then it may increase as more of the file is read due
00250 //               to the nature of HTTP/1.1 requests which can change
00251 //               their minds midstream about how much data they're
00252 //               sending you.
00253 ////////////////////////////////////////////////////////////////////
00254 off_t HTTPChannel::
00255 get_file_size() const {
00256   if (_got_file_size) {
00257     return _file_size;
00258   } else if (_got_transfer_file_size) {
00259     return _transfer_file_size;
00260   } else if (_got_expected_file_size) {
00261     return _expected_file_size;
00262   } else {
00263     return 0;
00264   }
00265 }
00266 
00267 ////////////////////////////////////////////////////////////////////
00268 //     Function: HTTPChannel::write_headers
00269 //       Access: Published
00270 //  Description: Outputs a list of all headers defined by the server
00271 //               to the indicated output stream.
00272 ////////////////////////////////////////////////////////////////////
00273 void HTTPChannel::
00274 write_headers(ostream &out) const {
00275   Headers::const_iterator hi;
00276   for (hi = _headers.begin(); hi != _headers.end(); ++hi) {
00277     out << (*hi).first << ": " << (*hi).second << "\n";
00278   }
00279 }
00280 
00281 ////////////////////////////////////////////////////////////////////
00282 //     Function: HTTPChannel::run
00283 //       Access: Published
00284 //  Description: This must be called from time to time when
00285 //               non-blocking I/O is in use.  It checks for data
00286 //               coming in on the socket and writes data out to the
00287 //               socket when possible, and does whatever processing is
00288 //               required towards completing the current task.
00289 //
00290 //               The return value is true if the task is still pending
00291 //               (and run() will need to be called again in the
00292 //               future), or false if the current task is complete.
00293 ////////////////////////////////////////////////////////////////////
00294 bool HTTPChannel::
00295 run() {
00296   if (_state == _done_state || _state == S_failure) {
00297     clear_extra_headers();
00298     if (!reached_done_state()) {
00299       return false;
00300     }
00301   }
00302 
00303   if (_started_download) {
00304     if (_wanted_nonblocking && _download_throttle) {
00305       double now = TrueClock::get_global_ptr()->get_short_time();
00306       double elapsed = now - _last_run_time;
00307       if (elapsed < _seconds_per_update) {
00308         // Come back later.
00309         thread_yield();
00310         return true;
00311       }
00312       int num_potential_updates = (int)(elapsed / _seconds_per_update);
00313       _last_run_time = now;
00314       _bytes_requested += _bytes_per_update * num_potential_updates;
00315       if (downloader_cat.is_spam()) {
00316         downloader_cat.spam()
00317           << "elapsed = " << elapsed << " num_potential_updates = " 
00318           << num_potential_updates << " bytes_requested = " 
00319           << _bytes_requested << "\n";
00320       }
00321     }
00322 
00323     bool repeat_later = false;
00324     switch (_download_dest) {
00325     case DD_none:
00326       // We're done.
00327       break;
00328 
00329     case DD_file:
00330       repeat_later = run_download_to_file();
00331       break;
00332 
00333     case DD_ram:
00334       repeat_later = run_download_to_ram();
00335       break;
00336 
00337     case DD_stream:
00338       repeat_later = run_download_to_stream();
00339       break;
00340     }
00341     if (repeat_later) {
00342       thread_yield();
00343     }
00344     return repeat_later;
00345   }
00346 
00347   /*
00348   if (downloader_cat.is_spam()) {
00349     downloader_cat.spam()
00350       << "begin run(), _state = " << _state << ", _done_state = "
00351       << _done_state << "\n";
00352   }
00353   */
00354 
00355   if (_state == _done_state) {
00356     return reached_done_state();
00357   }
00358 
00359   bool repeat_later;
00360   do {
00361     // If we're in a state that expects to have a connection already
00362     // (that is, any state other that S_try_next_proxy), then
00363     // reestablish the connection if it has been dropped.
00364     if (_bio.is_null() && _state != S_try_next_proxy) {
00365       if (_connect_count > http_max_connect_count) {
00366         // Too many connection attempts; just give up.  We should
00367         // never trigger this failsafe, since the code in each
00368         // individual case has similar logic to prevent more than two
00369         // consecutive lost connections.
00370         downloader_cat.warning()
00371           << "Too many lost connections, giving up.\n";
00372         _status_entry._status_code = SC_lost_connection;
00373         _state = S_failure;
00374         return false;
00375       }
00376 
00377       // No connection.  Attempt to establish one.
00378       if (_proxy.empty()) {
00379         _bio = new BioPtr(_request.get_url());
00380       } else {
00381         _bio = new BioPtr(_proxy);
00382       }
00383       _source = new BioStreamPtr(new BioStream(_bio));
00384       if (_nonblocking) {
00385         BIO_set_nbio(*_bio, 1);
00386       }
00387 
00388       if (downloader_cat.is_debug()) {
00389         if (_connect_count > 0) {
00390           downloader_cat.debug()
00391             << "Reconnecting to " << _bio->get_server_name() << ":" 
00392             << _bio->get_port() << "\n";
00393         } else {
00394           downloader_cat.debug()
00395             << "Connecting to " << _bio->get_server_name() << ":" 
00396             << _bio->get_port() << "\n";
00397         }
00398       }
00399       
00400       _state = S_connecting;
00401       _started_connecting_time = 
00402         TrueClock::get_global_ptr()->get_short_time();
00403       _connect_count++;
00404     }
00405 
00406     /*
00407     if (downloader_cat.is_spam()) {
00408       downloader_cat.spam()
00409         << "continue run(), _state = " << _state << "\n";
00410     }
00411     */
00412 
00413     switch (_state) {
00414     case S_try_next_proxy:
00415       repeat_later = run_try_next_proxy();
00416       break;
00417 
00418     case S_connecting:
00419       repeat_later = run_connecting();
00420       break;
00421       
00422     case S_connecting_wait:
00423       repeat_later = run_connecting_wait();
00424       break;
00425       
00426     case S_http_proxy_ready:
00427       repeat_later = run_http_proxy_ready();
00428       break;
00429       
00430     case S_http_proxy_request_sent:
00431       repeat_later = run_http_proxy_request_sent();
00432       break;
00433       
00434     case S_http_proxy_reading_header:
00435       repeat_later = run_http_proxy_reading_header();
00436       break;
00437       
00438     case S_socks_proxy_greet:
00439       repeat_later = run_socks_proxy_greet();
00440       break;
00441       
00442     case S_socks_proxy_greet_reply:
00443       repeat_later = run_socks_proxy_greet_reply();
00444       break;
00445       
00446     case S_socks_proxy_connect:
00447       repeat_later = run_socks_proxy_connect();
00448       break;
00449       
00450     case S_socks_proxy_connect_reply:
00451       repeat_later = run_socks_proxy_connect_reply();
00452       break;
00453       
00454     case S_setup_ssl:
00455       repeat_later = run_setup_ssl();
00456       break;
00457       
00458     case S_ssl_handshake:
00459       repeat_later = run_ssl_handshake();
00460       break;
00461       
00462     case S_ready:
00463       repeat_later = run_ready();
00464       break;
00465       
00466     case S_request_sent:
00467       repeat_later = run_request_sent();
00468       break;
00469       
00470     case S_reading_header:
00471       repeat_later = run_reading_header();
00472       break;
00473       
00474     case S_start_direct_file_read:
00475       repeat_later = run_start_direct_file_read();
00476       break;
00477       
00478     case S_read_header:
00479       repeat_later = run_read_header();
00480       break;
00481       
00482     case S_begin_body:
00483       repeat_later = run_begin_body();
00484       break;
00485       
00486     case S_reading_body:
00487       repeat_later = run_reading_body();
00488       break;
00489 
00490     case S_read_body:
00491       repeat_later = run_read_body();
00492       break;
00493 
00494     case S_read_trailer:
00495       repeat_later = run_read_trailer();
00496       break;
00497       
00498     default:
00499       downloader_cat.warning()
00500         << "Unhandled state " << _state << "\n";
00501       return false;
00502     }
00503 
00504     if (_state == _done_state || _state == S_failure) {
00505       clear_extra_headers();
00506       // We've reached our terminal state.
00507       return reached_done_state();
00508     }
00509     thread_consider_yield();
00510   } while (!repeat_later || _bio.is_null());
00511 
00512   /*
00513   if (downloader_cat.is_spam()) {
00514     downloader_cat.spam()
00515       << "later run(), _state = " << _state
00516       << ", _done_state = " << _done_state << "\n";
00517   }
00518   */
00519 
00520   thread_yield();
00521   return true;
00522 }
00523 
00524 ////////////////////////////////////////////////////////////////////
00525 //     Function: HTTPChannel::open_read_body
00526 //       Access: Published
00527 //  Description: Returns a newly-allocated istream suitable for
00528 //               reading the body of the document.  This may only be
00529 //               called immediately after a call to get_document() or
00530 //               post_form(), or after a call to run() has returned
00531 //               false.
00532 //
00533 //               Note that, in nonblocking mode, the returned stream
00534 //               may report an early EOF, even before the actual end
00535 //               of file.  When this happens, you should call
00536 //               stream->is_closed() to determine whether you should
00537 //               attempt to read some more later.
00538 //
00539 //               The user is responsible for passing the returned
00540 //               istream to close_read_body() later.
00541 ////////////////////////////////////////////////////////////////////
00542 ISocketStream *HTTPChannel::
00543 open_read_body() {
00544   reset_body_stream();
00545 
00546   if ((_state != S_read_header && _state != S_begin_body) || _source.is_null()) {
00547     return NULL;
00548   }
00549 
00550   string transfer_coding = downcase(get_header_value("Transfer-Encoding"));
00551 
00552   ISocketStream *result;
00553   if (transfer_coding == "chunked") {
00554     // "chunked" transfer encoding.  This means we will have to decode
00555     // the length of the file as we read it in chunks.  The
00556     // IChunkedStream does this.
00557     _state = S_reading_body;
00558     _read_index++;
00559     result = new IChunkedStream(_source, this);
00560 
00561   } else {
00562     // If the transfer encoding is anything else, assume "identity".
00563     // This is just the literal characters following the header, up
00564     // until _file_size bytes have been read (if content-length was
00565     // specified), or till end of file otherwise.
00566     _state = S_reading_body;
00567     _read_index++;
00568     result = new IIdentityStream(_source, this, _got_file_size, _file_size);
00569   }
00570 
00571   result->_channel = this;
00572   _body_stream = result;
00573   _owns_body_stream = false;
00574 
00575   return result;
00576 }
00577 
00578 ////////////////////////////////////////////////////////////////////
00579 //     Function: HTTPChannel::close_read_body
00580 //       Access: Public
00581 //  Description: Closes a file opened by a previous call to
00582 //               open_read_body().  This really just deletes the
00583 //               istream pointer, but it is recommended to use this
00584 //               interface instead of deleting it explicitly, to help
00585 //               work around compiler issues.
00586 ////////////////////////////////////////////////////////////////////
00587 void HTTPChannel::
00588 close_read_body(istream *stream) const {
00589   if (stream != (istream *)NULL) {
00590     // For some reason--compiler bug in gcc 3.2?--explicitly deleting
00591     // the stream pointer does not call the appropriate global delete
00592     // function; instead apparently calling the system delete
00593     // function.  So we call the delete function by hand instead.
00594 #if !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW)
00595     stream->~istream();
00596     (*global_operator_delete)(stream);
00597 #else
00598     delete stream;
00599 #endif
00600   }
00601 }
00602 
00603 ////////////////////////////////////////////////////////////////////
00604 //     Function: HTTPChannel::download_to_file
00605 //       Access: Published
00606 //  Description: Specifies the name of a file to download the
00607 //               resulting document to.  This should be called
00608 //               immediately after get_document() or
00609 //               begin_get_document() or related functions.
00610 //
00611 //               In the case of the blocking I/O methods like
00612 //               get_document(), this function will download the
00613 //               entire document to the file and return true if it was
00614 //               successfully downloaded, false otherwise.
00615 //
00616 //               In the case of non-blocking I/O methods like
00617 //               begin_get_document(), this function simply indicates an
00618 //               intention to download to the indicated file.  It
00619 //               returns true if the file can be opened for writing,
00620 //               false otherwise, but the contents will not be
00621 //               completely downloaded until run() has returned false.
00622 //               At this time, it is possible that a communications
00623 //               error will have left a partial file, so
00624 //               is_download_complete() may be called to test this.
00625 //
00626 //               If subdocument_resumes is true and the document in
00627 //               question was previously requested as a subdocument
00628 //               (i.e. get_subdocument() with a first_byte value
00629 //               greater than zero), this will automatically seek to
00630 //               the appropriate byte within the file for writing the
00631 //               output.  In this case, the file must already exist
00632 //               and must have at least first_byte bytes in it.  If
00633 //               subdocument_resumes is false, a subdocument will
00634 //               always be downloaded beginning at the first byte of
00635 //               the file.
00636 ////////////////////////////////////////////////////////////////////
00637 bool HTTPChannel::
00638 download_to_file(const Filename &filename, bool subdocument_resumes) {
00639   reset_download_to();
00640   _download_to_filename = filename;
00641   _download_to_filename.set_binary();
00642   _download_to_file.close();
00643   _download_to_file.clear();
00644   _subdocument_resumes = subdocument_resumes;
00645 
00646   _download_dest = DD_file;
00647 
00648   if (_wanted_nonblocking && _state != S_read_header) {
00649     // In nonblocking mode, we can't start the download yet; that will
00650     // be done later as run() is called.
00651     return true;
00652   }
00653 
00654   // In normal, blocking mode, go ahead and do the download.
00655   if (!open_download_file()) {
00656     reset_download_to();
00657     return false;
00658   }
00659 
00660   while (run()) {
00661   }
00662   return is_download_complete() && is_valid();
00663 }
00664 
00665 ////////////////////////////////////////////////////////////////////
00666 //     Function: HTTPChannel::download_to_ram
00667 //       Access: Published
00668 //  Description: Specifies a Ramfile object to download the
00669 //               resulting document to.  This should be called
00670 //               immediately after get_document() or
00671 //               begin_get_document() or related functions.
00672 //
00673 //               In the case of the blocking I/O methods like
00674 //               get_document(), this function will download the
00675 //               entire document to the Ramfile and return true if it
00676 //               was successfully downloaded, false otherwise.
00677 //
00678 //               In the case of non-blocking I/O methods like
00679 //               begin_get_document(), this function simply indicates an
00680 //               intention to download to the indicated Ramfile.  It
00681 //               returns true if the file can be opened for writing,
00682 //               false otherwise, but the contents will not be
00683 //               completely downloaded until run() has returned false.
00684 //               At this time, it is possible that a communications
00685 //               error will have left a partial file, so
00686 //               is_download_complete() may be called to test this.
00687 //
00688 //               If subdocument_resumes is true and the document in
00689 //               question was previously requested as a subdocument
00690 //               (i.e. get_subdocument() with a first_byte value
00691 //               greater than zero), this will automatically seek to
00692 //               the appropriate byte within the Ramfile for writing
00693 //               the output.  In this case, the Ramfile must already
00694 //               have at least first_byte bytes in it.
00695 ////////////////////////////////////////////////////////////////////
00696 bool HTTPChannel::
00697 download_to_ram(Ramfile *ramfile, bool subdocument_resumes) {
00698   nassertr(ramfile != (Ramfile *)NULL, false);
00699   reset_download_to();
00700   ramfile->_pos = 0;
00701   _download_to_ramfile = ramfile;
00702   _download_dest = DD_ram;
00703   _subdocument_resumes = (subdocument_resumes && _first_byte_delivered != 0);
00704 
00705   if (_wanted_nonblocking && _state != S_read_header) {
00706     // In nonblocking mode, we can't start the download yet; that will
00707     // be done later as run() is called.
00708     return true;
00709   }
00710 
00711   // In normal, blocking mode, go ahead and do the download.
00712   if (!open_download_file()) {
00713     reset_download_to();
00714     return false;
00715   }
00716 
00717   while (run()) {
00718   }
00719   return is_download_complete() && is_valid();
00720 }
00721 
00722 ////////////////////////////////////////////////////////////////////
00723 //     Function: HTTPChannel::download_to_stream
00724 //       Access: Published
00725 //  Description: Specifies the name of an ostream to download the
00726 //               resulting document to.  This should be called
00727 //               immediately after get_document() or
00728 //               begin_get_document() or related functions.
00729 //
00730 //               In the case of the blocking I/O methods like
00731 //               get_document(), this function will download the
00732 //               entire document to the file and return true if it was
00733 //               successfully downloaded, false otherwise.
00734 //
00735 //               In the case of non-blocking I/O methods like
00736 //               begin_get_document(), this function simply indicates an
00737 //               intention to download to the indicated file.  It
00738 //               returns true if the file can be opened for writing,
00739 //               false otherwise, but the contents will not be
00740 //               completely downloaded until run() has returned false.
00741 //               At this time, it is possible that a communications
00742 //               error will have left a partial file, so
00743 //               is_download_complete() may be called to test this.
00744 //
00745 //               If subdocument_resumes is true and the document in
00746 //               question was previously requested as a subdocument
00747 //               (i.e. get_subdocument() with a first_byte value
00748 //               greater than zero), this will automatically seek to
00749 //               the appropriate byte within the file for writing the
00750 //               output.  In this case, the file must already exist
00751 //               and must have at least first_byte bytes in it.  If
00752 //               subdocument_resumes is false, a subdocument will
00753 //               always be downloaded beginning at the first byte of
00754 //               the file.
00755 ////////////////////////////////////////////////////////////////////
00756 bool HTTPChannel::
00757 download_to_stream(ostream *strm, bool subdocument_resumes) {
00758   reset_download_to();
00759   _download_to_stream = strm;
00760   _download_to_stream->clear();
00761   _subdocument_resumes = subdocument_resumes;
00762 
00763   _download_dest = DD_stream;
00764 
00765   if (_wanted_nonblocking && _state != S_read_header) {
00766     // In nonblocking mode, we can't start the download yet; that will
00767     // be done later as run() is called.
00768     return true;
00769   }
00770 
00771   // In normal, blocking mode, go ahead and do the download.
00772   if (!open_download_file()) {
00773     reset_download_to();
00774     return false;
00775   }
00776 
00777   while (run()) {
00778   }
00779   return is_download_complete() && is_valid();
00780 }
00781 
00782 ////////////////////////////////////////////////////////////////////
00783 //     Function: HTTPChannel::get_connection
00784 //       Access: Published
00785 //  Description: Returns the connection that was established via a
00786 //               previous call to connect_to() or begin_connect_to(),
00787 //               or NULL if the connection attempt failed or if those
00788 //               methods have not recently been called.
00789 //
00790 //               This stream has been allocated from the free store.
00791 //               It is the user's responsibility to delete this
00792 //               pointer when finished with it.
00793 ////////////////////////////////////////////////////////////////////
00794 SocketStream *HTTPChannel::
00795 get_connection() {
00796   if (!is_connection_ready()) {
00797     return NULL;
00798   }
00799 
00800   BioStream *stream = _source->get_stream();
00801   _source->set_stream(NULL);
00802 
00803   // We're now passing ownership of the connection to the caller.
00804   if (downloader_cat.is_debug()) {
00805     downloader_cat.debug()
00806       << "passing ownership of connection to caller.\n";
00807   }
00808   reset_to_new();
00809 
00810   return stream;
00811 }
00812 
00813 ////////////////////////////////////////////////////////////////////
00814 //     Function: HTTPChannel::downcase
00815 //       Access: Public, Static
00816 //  Description: Returns the input string with all uppercase letters
00817 //               converted to lowercase.
00818 ////////////////////////////////////////////////////////////////////
00819 string HTTPChannel::
00820 downcase(const string &s) {
00821   string result;
00822   result.reserve(s.size());
00823   string::const_iterator p;
00824   for (p = s.begin(); p != s.end(); ++p) {
00825     result += tolower(*p);
00826   }
00827   return result;
00828 }
00829 
00830 ////////////////////////////////////////////////////////////////////
00831 //     Function: HTTPChannel::body_stream_destructs
00832 //       Access: Public
00833 //  Description: Called by ISocketStream destructor when _body_stream
00834 //               is destructing.
00835 ////////////////////////////////////////////////////////////////////
00836 void HTTPChannel::
00837 body_stream_destructs(ISocketStream *stream) {
00838   if (stream == _body_stream) {
00839     if (_state == S_reading_body) {
00840       switch (_body_stream->get_read_state()) {
00841       case ISocketStream::RS_complete:
00842         finished_body(false);
00843         break;
00844         
00845       case ISocketStream::RS_error:
00846         _state = HTTPChannel::S_failure;
00847         _status_entry._status_code = HTTPChannel::SC_lost_connection;
00848         break;
00849     
00850       default:
00851         break;
00852       }
00853     }
00854     _body_stream = NULL;
00855     _owns_body_stream = false;
00856   }
00857 }
00858 
00859 
00860 ////////////////////////////////////////////////////////////////////
00861 //     Function: HTTPChannel::reached_done_state
00862 //       Access: Private
00863 //  Description: Called by run() after it reaches the done state, this
00864 //               simply checks to see if a download was requested, and
00865 //               begins the download if it has been.
00866 ////////////////////////////////////////////////////////////////////
00867 bool HTTPChannel::
00868 reached_done_state() {
00869   /*
00870   if (downloader_cat.is_spam()) {
00871     downloader_cat.spam()
00872       << "terminating run(), _state = " << _state
00873       << ", _done_state = " << _done_state << "\n";
00874   }
00875   */
00876 
00877   if (_state == S_failure) {
00878     // We had to give up.  Each proxy we tried, in sequence, failed.
00879     // But maybe the last attempt didn't give us the most informative
00880     // response; go back and find the best one.
00881     if (!_status_list.empty()) {
00882       _status_list.push_back(_status_entry);
00883       if (downloader_cat.is_debug()) {
00884         downloader_cat.debug()
00885           << "Reexamining failure responses.\n";
00886       }
00887       size_t best_i = 0;
00888       if (downloader_cat.is_debug()) {
00889         downloader_cat.debug()
00890           << "  " << 0 << ". " << _status_list[0]._status_code << " "
00891           << _status_list[0]._status_string << "\n";
00892       }
00893       for (size_t i = 1; i < _status_list.size(); i++) {
00894         if (downloader_cat.is_debug()) {
00895           downloader_cat.debug()
00896             << "  " << i << ". " << _status_list[i]._status_code << " "
00897             << _status_list[i]._status_string << "\n";
00898         }
00899         if (more_useful_status_code(_status_list[i]._status_code, 
00900                                     _status_list[best_i]._status_code)) {
00901           best_i = i;
00902         }
00903       }
00904       if (downloader_cat.is_debug()) {
00905         downloader_cat.debug()
00906           << "chose index " << best_i << ", above.\n";
00907       }
00908       _status_entry = _status_list[best_i];
00909       _status_list.clear();
00910     }
00911 
00912     return false;
00913   }
00914 
00915   // We don't need the list of previous failures any more--we've connected.
00916   _status_list.clear();
00917 
00918   if (_download_dest == DD_none) {
00919     // All done.
00920     return false;
00921     
00922   } else {
00923     // Oops, we have to download the body now.
00924     open_read_body();
00925     if (_body_stream == (ISocketStream *)NULL) {
00926       if (downloader_cat.is_debug()) {
00927         downloader_cat.debug()
00928           << "Unable to download body: " << _request.get_url() << "\n";
00929       }
00930       return false;
00931 
00932     } else {
00933       _owns_body_stream = true;
00934       if (_state != S_reading_body) {
00935         reset_body_stream();
00936       }
00937       _started_download = true;
00938 
00939       _done_state = S_read_trailer;
00940       _last_run_time = TrueClock::get_global_ptr()->get_short_time();
00941       return true;
00942     }
00943   }
00944 }
00945   
00946 ////////////////////////////////////////////////////////////////////
00947 //     Function: HTTPChannel::run_try_next_proxy
00948 //       Access: Private
00949 //  Description: This state is reached when a previous connection
00950 //               attempt fails.  If we have multiple proxies in line
00951 //               to try, it sets us up for the next proxy and tries to
00952 //               connect again; otherwise, it sets the state to
00953 //               S_failure.
00954 ////////////////////////////////////////////////////////////////////
00955 bool HTTPChannel::
00956 run_try_next_proxy() {
00957   if (_proxy_next_index < _proxies.size()) {
00958     // Record the previous proxy's status entry, so we can come back
00959     // to it later if we get nonsense from the remaining proxies.
00960     _status_list.push_back(_status_entry);
00961     _status_entry = StatusEntry();
00962 
00963     // Now try the next proxy in sequence.
00964     _proxy = _proxies[_proxy_next_index];
00965     _proxy_auth = (HTTPAuthorization *)NULL;
00966     _proxy_next_index++;
00967     close_connection();
00968     reconsider_proxy();
00969     _state = S_connecting;
00970 
00971     return false;
00972   }
00973 
00974   // No more proxies to try, or we're not using a proxy.
00975   _state = S_failure;
00976   return false;
00977 }
00978   
00979 ////////////////////////////////////////////////////////////////////
00980 //     Function: HTTPChannel::run_connecting
00981 //       Access: Private
00982 //  Description: In this state, we have not yet established a
00983 //               network connection to the server (or proxy).
00984 ////////////////////////////////////////////////////////////////////
00985 bool HTTPChannel::
00986 run_connecting() {
00987   _status_entry = StatusEntry();
00988 
00989   if (BIO_do_connect(*_bio) <= 0) {
00990     if (BIO_should_retry(*_bio)) {
00991       _state = S_connecting_wait;
00992       return false;
00993     }
00994     downloader_cat.info()
00995       << "Could not connect to " << _bio->get_server_name() << ":" 
00996       << _bio->get_port() << "\n";
00997     OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
00998     _status_entry._status_code = SC_no_connection;
00999     _state = S_try_next_proxy;
01000     return false;
01001   }
01002 
01003   if (downloader_cat.is_debug()) {
01004     downloader_cat.debug()
01005       << "Connected to " << _bio->get_server_name() << ":" 
01006       << _bio->get_port() << "\n";
01007   }
01008 
01009   if (_proxy_tunnel_now) {
01010     if (_proxy.get_scheme() == "socks") {
01011       _state = S_socks_proxy_greet;
01012     } else {
01013       _state = S_http_proxy_ready;
01014     }
01015 
01016   } else {
01017     if (_want_ssl) {
01018       _state = S_setup_ssl;
01019     } else {
01020       _state = S_ready;
01021     }
01022   }
01023   return false;
01024 }
01025 
01026 ////////////////////////////////////////////////////////////////////
01027 //     Function: HTTPChannel::run_connecting_wait
01028 //       Access: Private
01029 //  Description: Here we have begun to establish a nonblocking
01030 //               connection, but we got a come-back-later message, so
01031 //               we are waiting for the socket to finish connecting.
01032 ////////////////////////////////////////////////////////////////////
01033 bool HTTPChannel::
01034 run_connecting_wait() {
01035   int fd = -1;
01036   BIO_get_fd(*_bio, &fd);
01037   if (fd < 0) {
01038     downloader_cat.warning()
01039       << "nonblocking socket BIO has no file descriptor.\n";
01040     // This shouldn't be possible.
01041     _status_entry._status_code = SC_internal_error;
01042     _state = S_try_next_proxy;
01043     return false;
01044   }
01045 
01046   if (downloader_cat.is_spam()) {
01047     downloader_cat.spam()
01048       << "waiting to connect to " << _request.get_url().get_server_and_port() << ".\n";
01049   }
01050   fd_set wset;
01051   FD_ZERO(&wset);
01052   FD_SET(fd, &wset);
01053   struct timeval tv;
01054   if (get_blocking_connect()) {
01055     // Since we'll be blocking on this connect, fill in the timeout
01056     // into the structure.
01057     tv.tv_sec = (int)_connect_timeout;
01058     tv.tv_usec = (int)((_connect_timeout - tv.tv_sec) * 1000000.0);
01059   } else {
01060     // We won't block on this connect, so select() for 0 time.
01061     tv.tv_sec = 0;
01062     tv.tv_usec = 0;
01063   }
01064   int errcode = select(fd + 1, NULL, &wset, NULL, &tv);
01065   if (errcode < 0) {
01066     downloader_cat.warning()
01067       << "Error in select.\n";
01068     // This shouldn't be possible.
01069     _status_entry._status_code = SC_internal_error;
01070     _state = S_try_next_proxy;
01071     return false;
01072   }
01073   
01074   if (errcode == 0) {
01075     // Nothing's happened so far; come back later.
01076     if (get_blocking_connect() ||
01077         (TrueClock::get_global_ptr()->get_short_time() - 
01078          _started_connecting_time > get_connect_timeout())) {
01079       // Time to give up.
01080       downloader_cat.info()
01081         << "Timeout connecting to " 
01082         << _request.get_url().get_server_and_port() 
01083         << " for " << _request.get_url()
01084         << ".\n";
01085       _status_entry._status_code = SC_timeout;
01086       _state = S_try_next_proxy;
01087       return false;
01088     }
01089     return true;
01090   }
01091   
01092   // The socket is now ready for writing.
01093   _state = S_connecting;
01094   return false;
01095 }
01096 
01097 
01098 ////////////////////////////////////////////////////////////////////
01099 //     Function: HTTPChannel::run_http_proxy_ready
01100 //       Access: Private
01101 //  Description: This state is reached only after first establishing a
01102 //               connection to the proxy, if a proxy is in use and we
01103 //               are tunneling through it via a CONNECT command.
01104 ////////////////////////////////////////////////////////////////////
01105 bool HTTPChannel::
01106 run_http_proxy_ready() {
01107   // If there's a request to be sent to the proxy, send it now.
01108   nassertr(!_proxy_request_text.empty(), false);
01109   if (!server_send(_proxy_request_text, false)) {
01110     return true;
01111   }
01112     
01113   // All done sending request.
01114   _state = S_http_proxy_request_sent;
01115   _sent_request_time = TrueClock::get_global_ptr()->get_short_time();
01116   return false;
01117 }
01118 
01119 ////////////////////////////////////////////////////////////////////
01120 //     Function: HTTPChannel::run_http_proxy_request_sent
01121 //       Access: Private
01122 //  Description: This state is reached only after we have sent a
01123 //               special message to the proxy and we are waiting for
01124 //               the proxy's response.  It is not used in the normal
01125 //               http-over-proxy case, which does not require a
01126 //               special message to the proxy.
01127 ////////////////////////////////////////////////////////////////////
01128 bool HTTPChannel::
01129 run_http_proxy_request_sent() {
01130   // Wait for the first line to come back from the server.
01131   string line;
01132   if (!server_getline_failsafe(line)) {
01133     return true;
01134   }
01135 
01136   // Skip unexpected blank lines.  We're just being generous here.
01137   while (line.empty()) {
01138     if (!server_getline_failsafe(line)) {
01139       return true;
01140     }
01141   }
01142 
01143   if (!parse_http_response(line)) {
01144     return false;
01145   }
01146 
01147   _state = S_http_proxy_reading_header;
01148   _current_field_name = string();
01149   _current_field_value = string();
01150   _headers.clear();
01151   _got_file_size = false;
01152   _got_transfer_file_size = false;
01153   return false;
01154 }
01155 
01156 ////////////////////////////////////////////////////////////////////
01157 //     Function: HTTPChannel::run_http_proxy_reading_header
01158 //       Access: Private
01159 //  Description: In this state we are reading the header lines from
01160 //               the proxy's response to our special message.
01161 ////////////////////////////////////////////////////////////////////
01162 bool HTTPChannel::
01163 run_http_proxy_reading_header() {
01164   if (parse_http_header()) {
01165     return true;
01166   }
01167 
01168   _redirect = get_header_value("Location");
01169   // We can take the proxy's word for it that this is the actual URL
01170   // for the redirect.
01171 
01172   _server_response_has_no_body = 
01173     (get_status_code() / 100 == 1 ||
01174      get_status_code() == 204 ||
01175      get_status_code() == 304);
01176 
01177   int last_status = _last_status_code;
01178   _last_status_code = get_status_code();
01179 
01180   if (get_status_code() == 407 && last_status != 407 && !_proxy.empty()) {
01181     // 407: not authorized to proxy.  Try to get the authorization.
01182     string authenticate_request = get_header_value("Proxy-Authenticate");
01183     _proxy_auth = _client->generate_auth(_proxy, true, authenticate_request);
01184     if (_proxy_auth != (HTTPAuthorization *)NULL) {
01185       _proxy_realm = _proxy_auth->get_realm();
01186       _proxy_username = _client->select_username(_proxy, true, _proxy_realm);
01187       if (!_proxy_username.empty()) {
01188         make_proxy_request_text();
01189 
01190         // Roll the state forward to force a new request.
01191         _state = S_begin_body;
01192         return false;
01193       }
01194     }
01195   }
01196 
01197   if (!is_valid()) {
01198     // Proxy wouldn't open connection.
01199 
01200     // Change some of the status codes a proxy might return to
01201     // differentiate them from similar status codes the destination
01202     // server might have returned.
01203     if (get_status_code() != 407) {
01204       _status_entry._status_code += 1000;
01205     }
01206 
01207     _state = S_try_next_proxy;
01208     return false;
01209   }
01210 
01211   // Now we have a tunnel opened through the proxy.
01212   make_request_text();
01213 
01214   if (_want_ssl) {
01215     _state = S_setup_ssl;
01216   } else {
01217     _state = S_ready;
01218   }
01219 
01220   return false;
01221 }
01222 
01223 ////////////////////////////////////////////////////////////////////
01224 //     Function: HTTPChannel::run_socks_proxy_greet
01225 //       Access: Private
01226 //  Description: This state is reached only after first establishing a
01227 //               connection to a SOCKS proxy, with which we now have
01228 //               to negotiate a connection.
01229 ////////////////////////////////////////////////////////////////////
01230 bool HTTPChannel::
01231 run_socks_proxy_greet() {
01232   static const char socks_greeting[] = {
01233     0x05, // Socks version 5
01234     0x01, // Number of supported login methods
01235     0x00, // Login method 0: no authentication
01236     /*
01237     0x01, // Login method 1: GSSAPI
01238     0x02  // Login method 2: username/password
01239     */
01240   };
01241   static const int socks_greeting_len = sizeof(socks_greeting);
01242   if (!server_send(string(socks_greeting, socks_greeting_len), true)) {
01243     return true;
01244   }
01245   _sent_request_time = TrueClock::get_global_ptr()->get_short_time();
01246     
01247   // All done sending request.
01248   _state = S_socks_proxy_greet_reply;
01249   return false;
01250 }
01251 
01252 ////////////////////////////////////////////////////////////////////
01253 //     Function: HTTPChannel::run_socks_proxy_greet_reply
01254 //       Access: Private
01255 //  Description: We are waiting for the SOCKS proxy to respond to our
01256 //               greeting.
01257 ////////////////////////////////////////////////////////////////////
01258 bool HTTPChannel::
01259 run_socks_proxy_greet_reply() {
01260   string reply;
01261 
01262   // Get the two-byte reply from the SOCKS server.
01263   if (!server_get_failsafe(reply, 2)) {
01264     return true;
01265   }
01266 
01267   if (reply[0] != 0x05) {
01268     // We only speak Socks5.
01269     downloader_cat.info()
01270       << "Rejecting Socks version " << (int)reply[0] << "\n";
01271     _status_entry._status_code = SC_socks_invalid_version;
01272     _state = S_try_next_proxy;
01273     return false;
01274   }
01275 
01276   if (reply[1] == (char)0xff) {
01277     downloader_cat.info()
01278       << "Socks server does not accept our available login methods.\n";
01279     _status_entry._status_code = SC_socks_no_acceptable_login_method;
01280     _state = S_try_next_proxy;
01281     return false;
01282   }
01283 
01284   if (reply[1] == 0x00) {
01285     // No login method required.  Proceed directly to the connect
01286     // message.
01287     _state = S_socks_proxy_connect;
01288     return false;
01289   }
01290 
01291   // The server accepted a login method we didn't offer!
01292   downloader_cat.info()
01293     << "Socks server accepted unrequested login method "
01294     << (int)reply[1] << "\n";
01295   _status_entry._status_code = SC_socks_no_acceptable_login_method;
01296   _state = S_try_next_proxy;
01297   return false;
01298 }
01299 
01300 ////////////////////////////////////////////////////////////////////
01301 //     Function: HTTPChannel::run_socks_proxy_connect
01302 //       Access: Private
01303 //  Description: The SOCKS proxy has accepted us, and now we may issue
01304 //               the connect request.
01305 ////////////////////////////////////////////////////////////////////
01306 bool HTTPChannel::
01307 run_socks_proxy_connect() {
01308   static const char socks_connect[] = {
01309     0x05, // Socks version 5
01310     0x01, // Command 1: connect
01311     0x00, // reserved
01312     0x03, // DNS name
01313   };
01314   static const int socks_connect_len = sizeof(socks_connect);
01315 
01316   string hostname = _request.get_url().get_server();
01317   int port = _request.get_url().get_port();
01318 
01319   if (downloader_cat.is_debug()) {
01320     downloader_cat.debug()
01321       << "Requesting SOCKS5 connection to " 
01322       << _request.get_url().get_server_and_port() << "\n";
01323   }
01324 
01325   string connect = 
01326     string(socks_connect, socks_connect_len) +
01327     string(1, (char)hostname.length()) +
01328     hostname +
01329     string(1, (char)((port >> 8) & 0xff)) +
01330     string(1, (char)(port & 0xff));
01331 
01332   if (!server_send(connect, true)) {
01333     return true;
01334   }
01335   _sent_request_time = TrueClock::get_global_ptr()->get_short_time();
01336     
01337   _state = S_socks_proxy_connect_reply;
01338   return false;
01339 }
01340 
01341 ////////////////////////////////////////////////////////////////////
01342 //     Function: HTTPChannel::run_socks_proxy_connect_reply
01343 //       Access: Private
01344 //  Description: We are waiting for the SOCKS proxy to honor our
01345 //               connect request.
01346 ////////////////////////////////////////////////////////////////////
01347 bool HTTPChannel::
01348 run_socks_proxy_connect_reply() {
01349   string reply;
01350 
01351   // Get the first two bytes of the connect reply.
01352   if (!server_get_failsafe(reply, 2)) {
01353     return true;
01354   }
01355 
01356   if (reply[0] != 0x05) {
01357     // We only speak Socks5.
01358     downloader_cat.info()
01359       << "Rejecting Socks version " << (int)reply[0] << "\n";
01360     close_connection();  // connection is now bad.
01361     _status_entry._status_code = SC_socks_invalid_version;
01362     _state = S_try_next_proxy;
01363     return false;
01364   }
01365 
01366   if (reply[1] != 0x00) {
01367     downloader_cat.info()
01368       << "Connection refused, SOCKS code " << (int)reply[1] << "\n";
01369     /*
01370       Socks error codes (from RFC1928):
01371              o  X'00' succeeded
01372              o  X'01' general SOCKS server failure
01373              o  X'02' connection not allowed by ruleset
01374              o  X'03' Network unreachable
01375              o  X'04' Host unreachable
01376              o  X'05' Connection refused
01377              o  X'06' TTL expired
01378              o  X'07' Command not supported
01379              o  X'08' Address type not supported
01380              o  X'09' to X'FF' unassigned
01381     */
01382 
01383     switch (reply[1]) {
01384     case 0x03:
01385     case 0x04:
01386     case 0x05:
01387     case 0x06:
01388       // These generally mean the same thing: the SOCKS proxy tried,
01389       // but couldn't reach the host.
01390       _status_entry._status_code = SC_socks_no_connection;
01391       break;
01392 
01393     default:
01394       _status_entry._status_code = SC_socks_refused;
01395     }
01396     
01397     close_connection();  // connection is now bad.
01398     _state = S_try_next_proxy;
01399     return false;
01400   }
01401 
01402   // Now put those bytes back, and get five bytes of the reply.
01403   _working_get = reply;
01404   if (!server_get_failsafe(reply, 5)) {
01405     return true;
01406   }
01407 
01408   // Figure out how many bytes total we will expect for the reply.
01409   int total_bytes = 6;
01410 
01411   switch (reply[3]) {
01412   case 0x01:  // IPv4
01413     total_bytes += 4;
01414     break;
01415 
01416   case 0x03:  // DNS
01417     total_bytes += (unsigned int)reply[4];
01418     break;
01419 
01420   default:
01421     downloader_cat.info()
01422       << "Unsupported SOCKS address type: " << (int)reply[3] << "\n";
01423     _status_entry._status_code = SC_socks_invalid_version;
01424     _state = S_try_next_proxy;
01425     return false;
01426   }
01427 
01428   // Now put back the bytes we've read so far, and get the rest of
01429   // them.
01430   _working_get = reply;
01431   if (!server_get_failsafe(reply, total_bytes)) {
01432     return true;
01433   }
01434 
01435   if (downloader_cat.is_debug()) {
01436     // Finally, we can decode the whole thing.
01437     string connect_host;
01438 
01439     switch (reply[3]) {
01440     case 0x01:  // IPv4
01441       {
01442         ostringstream strm;
01443         strm << (unsigned int)(unsigned char)reply[4] << "." 
01444              << (unsigned int)(unsigned char)reply[5] << "."
01445              << (unsigned int)(unsigned char)reply[6] << "." 
01446              << (unsigned int)(unsigned char)reply[7];
01447         connect_host = strm.str();
01448       }
01449       break;
01450       
01451     case 0x03:  // DNS
01452       connect_host = string(&reply[5], (unsigned int)reply[4]);
01453       break;
01454     }
01455     
01456     int connect_port =
01457       (((unsigned int)(unsigned char)reply[total_bytes - 2]) << 8) |
01458       ((unsigned int)(unsigned char)reply[total_bytes - 1]);
01459     
01460     downloader_cat.debug()
01461       << _proxy << " directed us to " << connect_host << ":"
01462       << connect_port << "\n";
01463   }
01464 
01465   if (_want_ssl) {
01466     _state = S_setup_ssl;
01467   } else {
01468     _state = S_ready;
01469   }
01470 
01471   return false;
01472 }
01473 
01474 ////////////////////////////////////////////////////////////////////
01475 //     Function: HTTPChannel::run_setup_ssl
01476 //       Access: Private
01477 //  Description: This state begins elevating our existing, unsecure
01478 //               connection to a secure, SSL connection.
01479 ////////////////////////////////////////////////////////////////////
01480 bool HTTPChannel::
01481 run_setup_ssl() {
01482   _sbio = BIO_new_ssl(_client->get_ssl_ctx(), true);
01483   BIO_push(_sbio, *_bio);
01484 
01485   SSL *ssl = NULL;
01486   BIO_get_ssl(_sbio, &ssl);
01487   nassertr(ssl != (SSL *)NULL, false);
01488 
01489   // We only take one word at a time from the _cipher_list.  If that
01490   // connection fails, then we take the next word.
01491   string cipher_list = _cipher_list;
01492   if (!cipher_list.empty()) {
01493     size_t space = cipher_list.find(" ");
01494     if (space != string::npos) {
01495       cipher_list = cipher_list.substr(0, space);
01496     }
01497   }
01498 
01499   if (downloader_cat.is_debug()) {
01500     downloader_cat.debug()
01501       << "Setting ssl-cipher-list '" << cipher_list << "'\n";
01502   }
01503   int result = SSL_set_cipher_list(ssl, cipher_list.c_str());
01504   if (result == 0) {
01505     downloader_cat.error()
01506       << "Invalid cipher list: '" << cipher_list << "'\n";
01507     OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
01508     _status_entry._status_code = SC_ssl_internal_failure;
01509     _state = S_failure;
01510     return false;
01511   }
01512 
01513   // It would be nice to use something like SSL_set_client_cert_cb()
01514   // here to set a callback to provide the certificate should it be
01515   // requested, or even to potentially provide any of a number of
01516   // certificates according to the server's CA presented, but that
01517   // interface as provided by OpenSSL is broken since there's no way
01518   // to pass additional data to the callback function (and hence no
01519   // way to tie it back to the HTTPChannel object, other than by
01520   // building a messy mapping of SSL pointers back to HTTPChannel
01521   // pointers).
01522   if (_client->load_client_certificate()) {
01523     SSL_use_certificate(ssl, _client->_client_certificate_pub);
01524     SSL_use_PrivateKey(ssl, _client->_client_certificate_priv);
01525     if (!SSL_check_private_key(ssl)) {
01526       downloader_cat.warning()
01527         << "Client private key does not match public key!\n";
01528     }
01529   }
01530 
01531   if (downloader_cat.is_spam()) {
01532     downloader_cat.spam()
01533       << "SSL Ciphers available:\n";
01534     const char *name;
01535     int pri = 0;
01536     name = SSL_get_cipher_list(ssl, pri);
01537     while (name != NULL) {
01538       downloader_cat.spam()
01539         << "  " << pri + 1 << ". " << name << "\n";
01540       pri++;
01541       name = SSL_get_cipher_list(ssl, pri);
01542     }
01543   }
01544 
01545   if (downloader_cat.is_debug()) {
01546     downloader_cat.debug()
01547       << "performing SSL handshake\n";
01548   }
01549   _state = S_ssl_handshake;
01550 
01551   // We start the connect timer over again when we reach the SSL
01552   // handshake.
01553   _started_connecting_time = 
01554     TrueClock::get_global_ptr()->get_short_time();
01555 
01556   return false;
01557 }
01558 
01559 ////////////////////////////////////////////////////////////////////
01560 //     Function: HTTPChannel::run_ssl_handshake
01561 //       Access: Private
01562 //  Description: This state performs the SSL handshake with the
01563 //               server, and also verifies the server's identity when
01564 //               the handshake has successfully completed.
01565 ////////////////////////////////////////////////////////////////////
01566 bool HTTPChannel::
01567 run_ssl_handshake() {
01568   if (BIO_do_handshake(_sbio) <= 0) {
01569     if (BIO_should_retry(_sbio)) {
01570       double elapsed =
01571         TrueClock::get_global_ptr()->get_short_time() -
01572         _started_connecting_time;
01573       if (elapsed <= get_connect_timeout() + _extra_ssl_handshake_time) {
01574         // Keep trying.
01575         return true;
01576       }
01577       // Time to give up on the handshake.
01578     }
01579 
01580     downloader_cat.info()
01581       << "Could not establish SSL handshake with " 
01582       << _request.get_url().get_server_and_port() << "\n";
01583     OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
01584 
01585     // It seems to be an error to free sbio at this point; perhaps
01586     // it's already been freed?
01587 
01588     if (!_cipher_list.empty()) {
01589       // If we've got another cipher to try, do so.
01590       size_t space = _cipher_list.find(" ");
01591       if (space != string::npos) {
01592         while (space < _cipher_list.length() && _cipher_list[space] == ' ') {
01593           ++space;
01594         }
01595         _cipher_list = _cipher_list.substr(space);
01596         if (!_cipher_list.empty()) {
01597           close_connection();
01598           reconsider_proxy();
01599           _state = S_connecting;
01600           return false;
01601         }
01602       }
01603     }
01604 
01605     // All done trying ciphers; they all failed.
01606     _cipher_list = _client->get_cipher_list();
01607     _status_entry._status_code = SC_ssl_no_handshake;
01608     _state = S_failure;
01609     return false;
01610   }
01611 
01612   SSL *ssl = NULL;
01613   BIO_get_ssl(_sbio, &ssl);
01614   nassertr(ssl != (SSL *)NULL, false);
01615 
01616   if (!_nonblocking) {
01617     SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
01618   }
01619 
01620   const SSL_CIPHER *cipher = SSL_get_current_cipher(ssl);
01621   if (cipher == (const SSL_CIPHER *)NULL) {
01622     downloader_cat.warning()
01623       << "No current cipher on SSL connection.\n";
01624   } else {
01625     if (downloader_cat.is_debug()) {
01626       downloader_cat.debug()
01627         << "Using cipher " << SSL_CIPHER_get_name((SSL_CIPHER *) cipher) << "\n";
01628     }
01629   }
01630 
01631   // Now that we've made an SSL handshake, we can use the SSL bio to
01632   // do all of our communication henceforth.
01633   _bio->set_bio(_sbio);
01634   _sbio = NULL;
01635 
01636   X509 *cert = SSL_get_peer_certificate(ssl);
01637   if (cert == (X509 *)NULL) {
01638     downloader_cat.info()
01639       << "No certificate was presented by server.\n";
01640 
01641     // This shouldn't be possible, per the SSL specs.
01642     _status_entry._status_code = SC_ssl_invalid_server_certificate;
01643     _state = S_failure;
01644     return false;
01645   }
01646   
01647   X509_NAME *subject = X509_get_subject_name(cert);
01648   if (downloader_cat.is_debug()) {
01649     string org_name = get_x509_name_component(subject, NID_organizationName);
01650     string org_unit_name = get_x509_name_component(subject, NID_organizationalUnitName);
01651     string common_name = get_x509_name_component(subject, NID_commonName);
01652     
01653     downloader_cat.debug()
01654       << "Server is " << common_name << " from " << org_unit_name
01655       << " / " << org_name << "\n";
01656 
01657     if (downloader_cat.is_spam()) {
01658       downloader_cat.spam()
01659         << "Received certificate from server:\n" << flush;
01660       X509_print_fp(stderr, cert);
01661       fflush(stderr);
01662     }
01663   }
01664 
01665   bool cert_preapproved = false;
01666   bool cert_name_preapproved = false;
01667   check_preapproved_server_certificate(cert, cert_preapproved, cert_name_preapproved);
01668 
01669   // Now verify the server certificate is valid.
01670   long verify_result = SSL_get_verify_result(ssl);
01671   bool cert_valid = true;
01672 
01673   if (verify_result == X509_V_ERR_CERT_HAS_EXPIRED) {
01674     downloader_cat.info()
01675       << "Expired certificate from " << _request.get_url().get_server_and_port() << "\n";
01676     if (_client->get_verify_ssl() == HTTPClient::VS_normal && !cert_preapproved) { 
01677       cert_valid = false;
01678     }
01679 
01680   } else if (verify_result == X509_V_ERR_CERT_NOT_YET_VALID) {
01681     downloader_cat.info()
01682       << "Premature certificate from " << _request.get_url().get_server_and_port() << "\n";
01683     if (_client->get_verify_ssl() == HTTPClient::VS_normal && !cert_preapproved) {
01684       cert_valid = false;
01685     }
01686 
01687   } else if (verify_result == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT ||
01688              verify_result == X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN) {
01689     downloader_cat.info()
01690       << "Self-signed certificate from " << _request.get_url().get_server_and_port() << "\n";
01691     if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_preapproved) {
01692       cert_valid = false;
01693     }
01694 
01695   } else if (verify_result != X509_V_OK) {
01696     downloader_cat.info()
01697       << "Unable to verify identity of " << _request.get_url().get_server_and_port()
01698       << ", verify error code " << verify_result << "\n";
01699     if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_preapproved) {
01700       cert_valid = false;
01701     }
01702   }
01703   
01704   if (!cert_valid) {
01705     _status_entry._status_code = SC_ssl_invalid_server_certificate;
01706     _state = S_failure;
01707     return false;
01708   }
01709   
01710   if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_name_preapproved) {
01711     // Check that the server is someone we expected to be talking
01712     // to.
01713     if (!validate_server_name(cert)) {
01714       _status_entry._status_code = SC_ssl_unexpected_server;
01715       _state = S_failure;
01716       return false;
01717     }
01718   }
01719   
01720   X509_free(cert);
01721 
01722   _state = S_ready;
01723   return false;
01724 }
01725 
01726 ////////////////////////////////////////////////////////////////////
01727 //     Function: HTTPChannel::run_ready
01728 //       Access: Private
01729 //  Description: This is the main "ready" state.  In this state, we
01730 //               have established a (possibly secure) connection to
01731 //               the server (or proxy), and the server (or proxy) is
01732 //               idle and waiting for us to send a request.
01733 //
01734 //               If persistent_connection is true, we will generally
01735 //               come back to this state after finishing each request
01736 //               on a given connection.
01737 ////////////////////////////////////////////////////////////////////
01738 bool HTTPChannel::
01739 run_ready() {
01740   // If there's a request to be sent upstream, send it now.
01741   if (!_request_text.empty()) {
01742    if (!server_send(_request_text, false)) {
01743       return true;
01744     }
01745   }
01746     
01747   // All done sending request.
01748   _state = S_request_sent;
01749   _sent_request_time = TrueClock::get_global_ptr()->get_short_time();
01750   return false;
01751 }
01752 
01753 ////////////////////////////////////////////////////////////////////
01754 //     Function: HTTPChannel::run_request_sent
01755 //       Access: Private
01756 //  Description: In this state we have sent our request to the server
01757 //               (or proxy) and we are waiting for a response.
01758 ////////////////////////////////////////////////////////////////////
01759 bool HTTPChannel::
01760 run_request_sent() {
01761   // Wait for the first line to come back from the server.
01762   string line;
01763   if (!server_getline_failsafe(line)) {
01764     return true;
01765   }
01766 
01767   // Skip unexpected blank lines.  We're just being generous here.
01768   while (line.empty()) {
01769     if (!server_getline_failsafe(line)) {
01770       return true;
01771     }
01772   }
01773 
01774   if (!parse_http_response(line)) {
01775     // Not an HTTP response.  _state is already set appropriately.
01776     return false;
01777   }
01778 
01779   _state = S_reading_header;
01780   _current_field_name = string();
01781   _current_field_value = string();
01782   _headers.clear();
01783   _got_file_size = false;
01784   _got_transfer_file_size = false;
01785   return false;
01786 }
01787 
01788 ////////////////////////////////////////////////////////////////////
01789 //     Function: HTTPChannel::run_reading_header
01790 //       Access: Private
01791 //  Description: In this state we have received the first response to
01792 //               our request from the server (or proxy) and we are
01793 //               reading the set of header lines preceding the
01794 //               requested document.
01795 ////////////////////////////////////////////////////////////////////
01796 bool HTTPChannel::
01797 run_reading_header() {
01798   if (parse_http_header()) {
01799     if (_bio.is_null()) {
01800       downloader_cat.info()
01801         << "Connection lost while reading HTTP response.\n";
01802       if (_response_type == RT_http_hangup) {
01803         // This was our second hangup in a row.  Give up.
01804         _status_entry._status_code = SC_lost_connection;
01805         _state = S_try_next_proxy;
01806         
01807       } else {
01808         // Try again, once.
01809         _response_type = RT_http_hangup;
01810       }
01811 
01812     } else {
01813       double elapsed =
01814         TrueClock::get_global_ptr()->get_short_time() -
01815         _sent_request_time;
01816       if (elapsed > get_http_timeout()) {
01817         // Time to give up.
01818         downloader_cat.info()
01819           << "Timeout waiting for "
01820           << _request.get_url().get_server_and_port() 
01821           << " in run_reading_header (" << elapsed 
01822           << " seconds elapsed).\n";
01823         _status_entry._status_code = SC_timeout;
01824         _state = S_try_next_proxy;
01825       }
01826     }
01827     return true;
01828   }
01829   _response_type = RT_http_complete;
01830 
01831   // Ok, we've established an HTTP connection to the server.  Our
01832   // extra send headers have done their job; clear them for next time.
01833   clear_extra_headers();
01834 
01835   _server_response_has_no_body = 
01836     (get_status_code() / 100 == 1 ||
01837      get_status_code() == 204 ||
01838      get_status_code() == 304 || 
01839      _method == HTTPEnum::M_head);
01840 
01841   // Look for key properties in the header fields.
01842   if (get_status_code() == 206) {
01843     string content_range = get_header_value("Content-Range");
01844     if (content_range.empty()) {
01845       downloader_cat.warning()
01846         << "Got 206 response without Content-Range header!\n";
01847       _status_entry._status_code = SC_invalid_http;
01848       _state = S_failure;
01849       return false;
01850 
01851     } else {
01852       if (!parse_content_range(content_range)) {
01853         downloader_cat.warning()
01854           << "Couldn't parse Content-Range: " << content_range << "\n";
01855         _status_entry._status_code = SC_invalid_http;
01856         _state = S_failure;
01857         return false;
01858       }
01859     }
01860 
01861   } else {
01862     _first_byte_delivered = 0;
01863     _last_byte_delivered = 0;
01864   }
01865   if (downloader_cat.is_debug()) {
01866     if (_first_byte_requested != 0 || _last_byte_requested != 0 ||
01867         _first_byte_delivered != 0 || _last_byte_delivered != 0) {
01868       downloader_cat.debug()
01869         << "Requested byte range " << _first_byte_requested
01870         << " to " << _last_byte_delivered
01871         << "; server delivers range " << _first_byte_delivered
01872         << " to " << _last_byte_delivered
01873         << "\n";
01874     }
01875   }
01876 
01877   // Set the _document_spec to reflect what we just retrieved.
01878   _document_spec = DocumentSpec(_request.get_url());
01879   string tag = get_header_value("ETag");
01880   if (!tag.empty()) {
01881     _document_spec.set_tag(HTTPEntityTag(tag));
01882   }
01883   string date = get_header_value("Last-Modified");
01884   if (!date.empty()) {
01885     _document_spec.set_date(HTTPDate(date));
01886   }
01887 
01888   // In case we've got a download in effect, now we know what the
01889   // first byte of the subdocument request will be, so we can open the
01890   // file and position it.
01891   if (_server_response_has_no_body) {
01892     // Never mind on the download.
01893     reset_download_to();
01894   }
01895 
01896   if (!open_download_file()) {
01897     return false;
01898   }
01899 
01900   _got_expected_file_size = false;
01901   _got_file_size = false;
01902   _got_transfer_file_size = false;
01903   
01904   string content_length = get_header_value("Content-Length");
01905   if (!content_length.empty()) {
01906     _file_size = atoi(content_length.c_str());
01907     _got_file_size = true;
01908 
01909   } else if (get_status_code() == 206) {
01910     // Well, we didn't get a content-length from the server, but we
01911     // can infer the number of bytes based on the range we're given.
01912     _file_size = _last_byte_delivered - _first_byte_delivered + 1;
01913     _got_file_size = true;
01914   }
01915   _redirect = get_header_value("Location");
01916 
01917   // The server might have given us just a filename for the redirect.
01918   // In that case, it's relative to the same server.  If it's a
01919   // relative path, it's relative to the same directory.
01920   if (_redirect.has_path() && !_redirect.has_authority()) {
01921     URLSpec url = _document_spec.get_url();
01922     Filename path = _redirect.get_path();
01923     if (path.is_local()) {
01924       Filename rel_to = Filename(url.get_path()).get_dirname();
01925       _redirect.set_path(Filename(rel_to, path));
01926     }
01927     _redirect.set_scheme(url.get_scheme());
01928     _redirect.set_authority(url.get_authority());
01929   }
01930 
01931   _state = S_read_header;
01932 
01933   if (_server_response_has_no_body && will_close_connection()) {
01934     // If the server said it will close the connection, we should
01935     // close it too.
01936     close_connection();
01937   }
01938 
01939   // Handle automatic retries and redirects.
01940   int last_status = _last_status_code;
01941   _last_status_code = get_status_code();
01942 
01943   if (get_status_code() == 407 && last_status != 407 && !_proxy.empty()) {
01944     // 407: not authorized to proxy.  Try to get the authorization.
01945     string authenticate_request = get_header_value("Proxy-Authenticate");
01946     _proxy_auth = 
01947       _client->generate_auth(_proxy, true, authenticate_request);
01948     if (_proxy_auth != (HTTPAuthorization *)NULL) {
01949       _proxy_realm = _proxy_auth->get_realm();
01950       _proxy_username = _client->select_username(_proxy, true, _proxy_realm);
01951       if (!_proxy_username.empty()) {
01952         make_request_text();
01953 
01954         // Roll the state forward to force a new request.
01955         _state = S_begin_body;
01956         return false;
01957       }
01958     }
01959   }
01960 
01961   if (get_status_code() == 401 && last_status != 401) {
01962     // 401: not authorized to remote server.  Try to get the authorization.
01963     string authenticate_request = get_header_value("WWW-Authenticate");
01964     _www_auth = _client->generate_auth(_request.get_url(), false, authenticate_request);
01965     if (_www_auth != (HTTPAuthorization *)NULL) {
01966       _www_realm = _www_auth->get_realm();
01967       _www_username = _client->select_username(_request.get_url(), false, _www_realm);
01968       if (!_www_username.empty()) {
01969         make_request_text();
01970       
01971         // Roll the state forward to force a new request.
01972         _state = S_begin_body;
01973         return false;
01974       }
01975     }
01976   }
01977 
01978   if ((get_status_code() == 300 || 
01979        get_status_code() == 301 ||
01980        get_status_code() == 302 ||
01981        get_status_code() == 303 ||
01982        get_status_code() == 307) && !get_redirect().empty()) {
01983     // Redirect.  Should we handle it automatically?
01984 
01985     // According to the letter of RFC 2616, 301 and 302 responses to
01986     // POST requests must not be automatically redirected without
01987     // confirmation by the user.  In reality, browsers do allow
01988     // automatic redirection of these responses, changing the POST to
01989     // a GET, and we reproduce this behavior here.
01990     if (_method == HTTPEnum::M_post) {
01991       _method = HTTPEnum::M_get;
01992       _body = string();
01993     }
01994 
01995     if (_method == HTTPEnum::M_get || _method == HTTPEnum::M_head) {
01996       // Sure!
01997       URLSpec new_url = get_redirect();
01998       if (find(_redirect_trail.begin(), _redirect_trail.end(),
01999                new_url) != _redirect_trail.end()) {
02000         downloader_cat.warning()
02001           << "cycle detected in redirect to " << new_url << "\n";
02002         
02003       } else {
02004         _redirect_trail.push_back(new_url);
02005 
02006         if (downloader_cat.is_debug()) {
02007           downloader_cat.debug()
02008             << "following redirect to " << new_url << "\n";
02009         }
02010         if (_request.get_url().has_username()) {
02011           new_url.set_username(_request.get_url().get_username());
02012         }
02013         reset_url(_request.get_url(), new_url);
02014         _request.set_url(new_url);
02015         _want_ssl = _request.get_url().is_ssl();
02016         reconsider_proxy();
02017         make_header();
02018         make_request_text();
02019 
02020         // Roll the state forward to force a new request.
02021         _state = S_begin_body;
02022         return false;
02023       }
02024     }
02025   }
02026 
02027   if (_state == S_read_header && 
02028       ((get_status_code() / 100) == 4 || (get_status_code() / 100) == 5) &&
02029       _proxy_serves_document && _proxy_next_index < _proxies.size()) {
02030     // If we were using a proxy (but not tunneling through the proxy)
02031     // and we got some kind of a server error, try the next proxy in
02032     // sequence (if we have one).  This handles the case of a working
02033     // proxy that cannot see the host (and so returns 504 or something
02034     // along those lines).  Some proxies are so broken they return a
02035     // 404 in this case, so we have to consider that along the same
02036     // lines.
02037     _state = S_try_next_proxy;
02038     return false;
02039   }
02040 
02041   // Otherwise, we're good to go.
02042   return false;
02043 }
02044 
02045 ////////////////////////////////////////////////////////////////////
02046 //     Function: HTTPChannel::run_start_direct_file_read
02047 //       Access: Private
02048 //  Description: This is the first state when reading a file:// URL.
02049 //               All it does is skip past the non-existent "header".
02050 ////////////////////////////////////////////////////////////////////
02051 bool HTTPChannel::
02052 run_start_direct_file_read() {
02053   _state = S_read_header;
02054   if (!open_download_file()) {
02055     return false;
02056   }
02057   return false;
02058 }
02059 
02060 ////////////////////////////////////////////////////////////////////
02061 //     Function: HTTPChannel::run_read_header
02062 //       Access: Private
02063 //  Description: In this state we have completely read the header
02064 //               lines returned by the server (or proxy) in response
02065 //               to our request.  This state represents the normal
02066 //               stopping point of a call to get_document(), etc.;
02067 //               further reads will return the body of the request,
02068 //               the requested document.
02069 //
02070 //               Normally run_read_header() is not called unless the
02071 //               user has elected not to read the returned document
02072 //               himself.  In fact, the state itself only exists so we
02073 //               can make a distinction between S_read_header and
02074 //               S_begin_body, where S_read_header is safe to return
02075 //               to the user and S_begin_body means we need to start
02076 //               skipping the document.
02077 ////////////////////////////////////////////////////////////////////
02078 bool HTTPChannel::
02079 run_read_header() {
02080   _state = S_begin_body;
02081   return false;
02082 }
02083 
02084 ////////////////////////////////////////////////////////////////////
02085 //     Function: HTTPChannel::run_begin_body
02086 //       Access: Private
02087 //  Description: This state begins to skip over the body in
02088 //               preparation for making a new request.
02089 ////////////////////////////////////////////////////////////////////
02090 bool HTTPChannel::
02091 run_begin_body() {
02092   if (will_close_connection()) {
02093     // If the socket will close anyway, no point in skipping past the
02094     // previous body; just reset.
02095     if (downloader_cat.is_debug()) {
02096       downloader_cat.debug()
02097         << "resetting to begin body; server would close anyway.\n";
02098     }
02099     reset_to_new();
02100     return false;
02101   }
02102 
02103   if (_server_response_has_no_body) {
02104     // We have already "read" the nonexistent body.
02105     _state = S_read_trailer;
02106 
02107   } else if (get_file_size() > (int)_skip_body_size) {
02108     // If we know the size of the body we are about to skip and it's
02109     // too large, then don't bother skipping it--just drop the
02110     // connection and get a new one.
02111     if (downloader_cat.is_debug()) {
02112       downloader_cat.debug()
02113         << "Dropping connection rather than skipping past " 
02114         << get_file_size() << " bytes.\n";
02115     }
02116     reset_to_new();
02117 
02118   } else {
02119     open_read_body();
02120     if (_body_stream == (ISocketStream *)NULL) {
02121       if (downloader_cat.is_debug()) {
02122         downloader_cat.debug()
02123           << "Unable to skip body.\n";
02124       }
02125       reset_to_new();
02126       
02127     } else {
02128       _owns_body_stream = true;
02129       if (_state != S_reading_body) {
02130         reset_body_stream();
02131       }
02132     }
02133   }
02134 
02135   return false;
02136 }
02137 
02138 ////////////////////////////////////////////////////////////////////
02139 //     Function: HTTPChannel::run_reading_body
02140 //       Access: Private
02141 //  Description: In this state we are in the process of reading the
02142 //               response's body.  We will only come to this function
02143 //               if the user did not choose to read the entire body
02144 //               himself (by calling open_read_body()).
02145 //
02146 //               In this case we should skip past the body to reset
02147 //               the connection for making a new request.
02148 ////////////////////////////////////////////////////////////////////
02149 bool HTTPChannel::
02150 run_reading_body() {
02151   if (will_close_connection()) {
02152     // If the socket will close anyway, no point in skipping past the
02153     // previous body; just reset.
02154     if (downloader_cat.is_debug()) {
02155       downloader_cat.debug()
02156         << "resetting to read body; server would close anyway.\n";
02157     }
02158     reset_to_new();
02159     return false;
02160   }
02161 
02162   // Skip the body we've already started.
02163   if (_body_stream == NULL || !_owns_body_stream) {
02164     // Whoops, we're not in skip-body mode.  Better reset.
02165     if (downloader_cat.is_debug()) {
02166       downloader_cat.debug()
02167         << "resetting, not in skip-body mode.\n";
02168     }
02169     reset_to_new();
02170     return false;
02171   }
02172 
02173   string line;
02174   getline(*_body_stream, line);
02175   while (!_body_stream->fail() && !_body_stream->eof()) {
02176     if (downloader_cat.is_spam()) {
02177       downloader_cat.spam() << "skip: " << line << "\n";
02178     }
02179     getline(*_body_stream, line);
02180   }
02181 
02182   if (!_body_stream->is_closed()) {
02183     // There's more to come later.
02184     return true;
02185   }
02186 
02187   reset_body_stream();
02188 
02189   // This should have been set by the call to finished_body(), above.
02190   nassertr(_state != S_reading_body, false);
02191   return false;
02192 }
02193 
02194 ////////////////////////////////////////////////////////////////////
02195 //     Function: HTTPChannel::run_read_body
02196 //       Access: Private
02197 //  Description: In this state we have completely read (or skipped
02198 //               over) the body of the response.  We should continue
02199 //               skipping past the trailer following the body.
02200 //
02201 //               Not all bodies come with trailers; in particular, the
02202 //               "identity" transfer encoding does not include a
02203 //               trailer.  It is therefore the responsibility of the
02204 //               IdentityStreamBuf or ChunkedStreamBuf to set the
02205 //               state appropriately to either S_read_body or
02206 //               S_read_trailer following the completion of the body.
02207 ////////////////////////////////////////////////////////////////////
02208 bool HTTPChannel::
02209 run_read_body() {
02210   if (will_close_connection()) {
02211     // If the socket will close anyway, no point in skipping past the
02212     // previous body; just reset.
02213     if (downloader_cat.is_debug()) {
02214       downloader_cat.debug()
02215         << "resetting to read body; server would close anyway.\n";
02216     }
02217     reset_to_new();
02218     return false;
02219   }
02220   // Skip the trailer following the recently-read body.
02221 
02222   string line;
02223   if (!server_getline(line)) {
02224     return true;
02225   }
02226   while (!line.empty()) {
02227     if (!server_getline(line)) {
02228       return true;
02229     }
02230   }
02231 
02232   _state = S_read_trailer;
02233   return false;
02234 }
02235 
02236 ////////////////////////////////////////////////////////////////////
02237 //     Function: HTTPChannel::run_read_trailer
02238 //       Access: Private
02239 //  Description: In this state we have completely read the body and
02240 //               the trailer.  This state is simply a pass-through
02241 //               back to S_ready.
02242 ////////////////////////////////////////////////////////////////////
02243 bool HTTPChannel::
02244 run_read_trailer() {
02245   if (will_close_connection()) {
02246     // If the socket will close anyway, no point in skipping past the
02247     // previous body; just reset.
02248     if (downloader_cat.is_debug()) {
02249       downloader_cat.debug()
02250         << "resetting to read trailer; server would close anyway.\n";
02251     }
02252     reset_to_new();
02253     return false;
02254   }
02255 
02256   _state = S_ready;
02257   return false;
02258 }
02259 
02260 ////////////////////////////////////////////////////////////////////
02261 //     Function: HTTPChannel::run_download_to_file
02262 //       Access: Private
02263 //  Description: After the headers, etc. have been read, this streams
02264 //               the download to the named file.
02265 ////////////////////////////////////////////////////////////////////
02266 bool HTTPChannel::
02267 run_download_to_file() {
02268   nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false);
02269 
02270   bool do_throttle = _wanted_nonblocking && _download_throttle;
02271 
02272   static const size_t buffer_size = 4096;
02273   char buffer[buffer_size];
02274 
02275   size_t remaining_this_pass = buffer_size;
02276   if (do_throttle) {
02277     remaining_this_pass = _bytes_per_update;
02278   }
02279 
02280   _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
02281   size_t count = _body_stream->gcount();
02282   while (count != 0) {
02283     _download_to_file.write(buffer, count);
02284     _bytes_downloaded += count;
02285     if (do_throttle) {
02286       nassertr(count <= remaining_this_pass, false);
02287       remaining_this_pass -= count;
02288       if (remaining_this_pass == 0) {
02289         // That's enough for now.
02290         return true;
02291       }
02292     }
02293 
02294     thread_consider_yield();
02295     _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
02296     count = _body_stream->gcount();
02297   }
02298 
02299   if (_download_to_file.fail()) {
02300     downloader_cat.warning()
02301       << "Error writing to " << _download_to_filename << "\n";
02302     _status_entry._status_code = SC_download_write_error;
02303     _state = S_failure;
02304     reset_download_to();
02305     return false;
02306   }
02307 
02308   _download_to_file.flush();
02309 
02310   if (_body_stream->is_closed()) {
02311     // Done.
02312     reset_body_stream();
02313     _download_to_file.close();
02314     _started_download = false;
02315     return false;
02316   } else {
02317     // More to come.
02318     return true;
02319   }
02320 }
02321 
02322 ////////////////////////////////////////////////////////////////////
02323 //     Function: HTTPChannel::run_download_to_ram
02324 //       Access: Private
02325 //  Description: After the headers, etc. have been read, this streams
02326 //               the download to the specified Ramfile object.
02327 ////////////////////////////////////////////////////////////////////
02328 bool HTTPChannel::
02329 run_download_to_ram() {
02330   nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false);
02331   nassertr(_download_to_ramfile != (Ramfile *)NULL, false);
02332 
02333   bool do_throttle = _wanted_nonblocking && _download_throttle;
02334 
02335   static const size_t buffer_size = 4096;
02336   char buffer[buffer_size];
02337 
02338   size_t remaining_this_pass = buffer_size;
02339   if (do_throttle) {
02340     remaining_this_pass = _bytes_per_update;
02341   }
02342 
02343   _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
02344   size_t count = _body_stream->gcount();
02345   while (count != 0) {
02346     _download_to_ramfile->_data += string(buffer, count);
02347     _bytes_downloaded += count;
02348     if (do_throttle) {
02349       nassertr(count <= remaining_this_pass, false);
02350       remaining_this_pass -= count;
02351       if (remaining_this_pass == 0) {
02352         // That's enough for now.
02353         return true;
02354       }
02355     }
02356 
02357     thread_consider_yield();
02358     _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
02359     count = _body_stream->gcount();
02360   }
02361 
02362   if (_body_stream->is_closed()) {
02363     // Done.
02364     reset_body_stream();
02365     _started_download = false;
02366     return false;
02367   } else {
02368     // More to come.
02369     return true;
02370   }
02371 }
02372 
02373 ////////////////////////////////////////////////////////////////////
02374 //     Function: HTTPChannel::run_download_to_stream
02375 //       Access: Private
02376 //  Description: After the headers, etc. have been read, this streams
02377 //               the download to the named file.
02378 ////////////////////////////////////////////////////////////////////
02379 bool HTTPChannel::
02380 run_download_to_stream() {
02381   nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false);
02382 
02383   bool do_throttle = _wanted_nonblocking && _download_throttle;
02384 
02385   static const size_t buffer_size = 4096;
02386   char buffer[buffer_size];
02387 
02388   size_t remaining_this_pass = buffer_size;
02389   if (do_throttle) {
02390     remaining_this_pass = _bytes_per_update;
02391   }
02392 
02393   _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
02394   size_t count = _body_stream->gcount();
02395   while (count != 0) {
02396     _download_to_stream->write(buffer, count);
02397     _bytes_downloaded += count;
02398     if (do_throttle) {
02399       nassertr(count <= remaining_this_pass, false);
02400       remaining_this_pass -= count;
02401       if (remaining_this_pass == 0) {
02402         // That's enough for now.
02403         return true;
02404       }
02405     }
02406 
02407     thread_consider_yield();
02408     _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
02409     count = _body_stream->gcount();
02410   }
02411 
02412   if (_download_to_stream->fail()) {
02413     downloader_cat.warning()
02414       << "Error writing to stream\n";
02415     _status_entry._status_code = SC_download_write_error;
02416     _state = S_failure;
02417     reset_download_to();
02418     return false;
02419   }
02420 
02421   _download_to_stream->flush();
02422 
02423   if (_body_stream->is_closed()) {
02424     // Done.
02425     reset_body_stream();
02426     _download_to_stream = NULL;
02427     _started_download = false;
02428     return false;
02429   } else {
02430     // More to come.
02431     return true;
02432   }
02433 }
02434 
02435 
02436 ////////////////////////////////////////////////////////////////////
02437 //     Function: HTTPChannel::begin_request
02438 //       Access: Private
02439 //  Description: Begins a new document request to the server, throwing
02440 //               away whatever request was currently pending if
02441 //               necessary.
02442 ////////////////////////////////////////////////////////////////////
02443 void HTTPChannel::
02444 begin_request(HTTPEnum::Method method, const DocumentSpec &url,
02445               const string &body, bool nonblocking, 
02446               size_t first_byte, size_t last_byte) {
02447   
02448   downloader_cat.info()
02449     << method << " " << url << "\n";
02450                 
02451   reset_for_new_request();
02452 
02453   _wanted_nonblocking = nonblocking;
02454 #if defined(HAVE_THREADS) && defined(SIMPLE_THREADS)
02455   // In the presence of SIMPLE_THREADS, we always use non-blocking
02456   // I/O.  We simulate blocking by yielding the thread.
02457   nonblocking = true;
02458 #endif
02459 
02460   // Get the set of proxies that are appropriate for this URL.
02461   _proxies.clear();
02462   _proxy_next_index = 0;
02463   if (get_allow_proxy()) {
02464     _client->get_proxies_for_url(url.get_url(), _proxies);
02465   }
02466 
02467   // If we still have a live connection to a proxy that is on the
02468   // list, that proxy should be moved immediately to the front of the
02469   // list (to minimize restarting connections unnecessarily).
02470   if (!_bio.is_null() && !_proxies.empty() && !_proxy.empty()) {
02471     Proxies::iterator pi = find(_proxies.begin(), _proxies.end(), _proxy);
02472     if (pi != _proxies.end()) {
02473       _proxies.erase(pi);
02474       _proxies.insert(_proxies.begin(), _proxy);
02475     }
02476   }
02477 
02478   URLSpec new_proxy;
02479   if (_proxy_next_index < _proxies.size()) {
02480     new_proxy = _proxies[_proxy_next_index];
02481     _proxy_next_index++;
02482   }
02483 
02484   // Changing the proxy is grounds for dropping the old connection, if
02485   // any.
02486   if (_proxy != new_proxy) {
02487     _proxy = new_proxy;
02488     _proxy_auth = (HTTPAuthorization *)NULL;
02489     if (downloader_cat.is_debug()) {
02490       downloader_cat.debug()
02491         << "resetting to change proxy to " << _proxy << "\n";
02492     }
02493     reset_to_new();
02494   }
02495 
02496   // Ditto with changing the nonblocking state.
02497   if (_nonblocking != nonblocking) {
02498     _nonblocking = nonblocking;
02499     if (downloader_cat.is_debug()) {
02500       downloader_cat.debug()
02501         << "resetting to change nonblocking state to " << _nonblocking << ".\n";
02502     }
02503     reset_to_new();
02504   }
02505 
02506   reset_url(_request.get_url(), url.get_url());
02507   _request = url;
02508   _document_spec = DocumentSpec();
02509   _method = method;
02510   _body = body;
02511 
02512   // An https-style request means we'll need to establish an SSL
02513   // connection.
02514   _want_ssl = _request.get_url().is_ssl();
02515 
02516   _first_byte_requested = first_byte;
02517   _last_byte_requested = last_byte;
02518   _connect_count = 0;
02519 
02520   reconsider_proxy();
02521 
02522   // Reset from whatever previous request might still be pending.
02523   if (_request.get_url().get_scheme() == "file") {
02524     // A "file" URL just means we're reading a raw file.  This only
02525     // supports actual disk files, not the VFS, because we use a
02526     // BIO_new_file() underneath this.
02527     reset_to_new();
02528     _bio = new BioPtr(_request.get_url());
02529     if (_bio->get_bio() != NULL) {
02530       // Successfully opened the file.
02531       _source = new BioStreamPtr(new BioStream(_bio));
02532       _status_entry._status_code = 200;
02533       _state = S_start_direct_file_read;
02534 
02535       // Get the file size.
02536       FILE *fp = NULL;
02537       BIO_get_fp(_bio->get_bio(), &fp);
02538       if (fp != NULL) {
02539         if (fseek(fp, 0, SEEK_END) == 0) {
02540           _file_size = ftell(fp);
02541           _got_file_size = true;
02542           fseek(fp, 0, SEEK_SET);
02543         }
02544       }
02545 
02546     } else {
02547       // Couldn't read the file.
02548       OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
02549       _status_entry._status_code = SC_no_connection;
02550       _state = S_failure;
02551     }
02552 
02553   } else {
02554     // We're reading a normal network URL.
02555     if (_state == S_failure || (_state < S_read_header && _state != S_ready)) {
02556       if (downloader_cat.is_debug()) {
02557         downloader_cat.debug()
02558           << "resetting to clear previous request.\n";
02559       }
02560       reset_to_new();
02561       
02562     } else if (TrueClock::get_global_ptr()->get_short_time() - _last_run_time >= _idle_timeout) {
02563       if (downloader_cat.is_debug()) {
02564         downloader_cat.debug()
02565           << "resetting old connection: " 
02566           << TrueClock::get_global_ptr()->get_short_time() - _last_run_time
02567           << " s old.\n";
02568       }
02569       reset_to_new();
02570       
02571     } else if (_state == S_read_header) {
02572       // Roll one step forwards to start skipping past the previous
02573       // body.
02574       _state = S_begin_body;
02575     }
02576   }
02577 
02578   if (_method == HTTPEnum::M_connect) {
02579     _done_state = S_ready;
02580   } else {
02581     _done_state = S_read_header;
02582   }
02583 }
02584 
02585 ////////////////////////////////////////////////////////////////////
02586 //     Function: HTTPChannel::reconsider_proxy
02587 //       Access: Private
02588 //  Description: Reevaluates the flags and strings that are computed
02589 //               based on the particular proxy we are attempting to
02590 //               connect to.  This should be called when we initiate a
02591 //               request, and also whenever we change proxies while
02592 //               processing a request.
02593 ////////////////////////////////////////////////////////////////////
02594 void HTTPChannel::
02595 reconsider_proxy() {
02596   _proxy_tunnel_now = false;
02597   _proxy_serves_document = false;
02598   
02599   if (!_proxy.empty()) {
02600     // If the user insists we always tunnel through a proxy, or if
02601     // we're opening an SSL connection, or the user has explicitly
02602     // asked for a direct connection of some kind, or if we have a
02603     // SOCKS-style proxy; each of these demands a tunnel through the
02604     // proxy to speak directly to the http server.
02605     _proxy_tunnel_now =
02606       (get_proxy_tunnel() || _want_ssl ||
02607        _method == HTTPEnum::M_connect || _proxy.get_scheme() == "socks");
02608 
02609     // Otherwise (but we still have a proxy), then we ask the proxy to
02610     // hand us the document.
02611     _proxy_serves_document = !_proxy_tunnel_now;
02612   }
02613 
02614   make_header();
02615   make_request_text();
02616 
02617   if (_proxy_tunnel_now) {
02618     // Maybe we need to tunnel through the proxy to connect to the
02619     // server directly.
02620     ostringstream request;
02621     request 
02622       << "CONNECT " << _request.get_url().get_server_and_port()
02623       << " " << _client->get_http_version_string() << "\r\n";
02624     if (_client->get_http_version() >= HTTPEnum::HV_11) {
02625       request 
02626         << "Host: " << _request.get_url().get_server_and_port() << "\r\n";
02627     }
02628     _proxy_header = request.str();
02629     make_proxy_request_text();
02630 
02631   } else {
02632     _proxy_header = string();
02633     _proxy_request_text = string();
02634   }
02635 }
02636 
02637 
02638 ////////////////////////////////////////////////////////////////////
02639 //     Function: HTTPChannel::reset_for_new_request
02640 //       Access: Private
02641 //  Description: Resets the internal state variables in preparation
02642 //               for beginning a new request.
02643 ////////////////////////////////////////////////////////////////////
02644 void HTTPChannel::
02645 reset_for_new_request() {
02646   reset_download_to();
02647   reset_body_stream();
02648 
02649   _last_status_code = 0;
02650   _status_entry = StatusEntry();
02651 
02652   _response_type = RT_none;
02653   _redirect_trail.clear();
02654   _bytes_downloaded = 0;
02655   _bytes_requested = 0;
02656 }
02657 
02658 ////////////////////////////////////////////////////////////////////
02659 //     Function: HTTPChannel::finished_body
02660 //       Access: Private
02661 //  Description: This is called by the body reading
02662 //               classes--ChunkedStreamBuf and IdentityStreamBuf--when
02663 //               they have finished reading the body.  It advances the
02664 //               state appropriately.
02665 //
02666 //               has_trailer should be set true if the body type has
02667 //               an associated trailer which should be read or
02668 //               skipped, or false if there is no trailer.
02669 ////////////////////////////////////////////////////////////////////
02670 void HTTPChannel::
02671 finished_body(bool has_trailer) {
02672   if (will_close_connection() && _download_dest == DD_none) {
02673     if (downloader_cat.is_debug()) {
02674       downloader_cat.debug()
02675         << "resetting to finish body; server would close anyway.\n";
02676     }
02677     reset_to_new();
02678 
02679   } else {
02680     if (has_trailer) {
02681       _state = HTTPChannel::S_read_body;
02682     } else {
02683       _state = HTTPChannel::S_read_trailer;
02684     }
02685   }
02686 }
02687 
02688 ////////////////////////////////////////////////////////////////////
02689 //     Function: HTTPChannel::open_download_file
02690 //       Access: Private
02691 //  Description: If a download has been requested, opens the file on
02692 //               disk (or prepares the RamFile or stream) and seeks
02693 //               within it to the appropriate _first_byte_delivered
02694 //               position, so that downloaded bytes will be written to
02695 //               the appropriate point within the file.  Returns true
02696 //               if the starting position is valid, false otherwise
02697 //               (in which case the state is set to S_failure).
02698 ////////////////////////////////////////////////////////////////////
02699 bool HTTPChannel::
02700 open_download_file() {
02701   _subdocument_resumes = (_subdocument_resumes && _first_byte_delivered != 0);
02702 
02703   if (_download_dest == DD_file) {
02704     if (!_download_to_filename.open_write(_download_to_file, !_subdocument_resumes)) {
02705       downloader_cat.info()
02706         << "Could not open " << _download_to_filename << " for writing.\n";
02707       _status_entry._status_code = SC_download_open_error;
02708       _state = S_failure;
02709       return false;
02710     }
02711   }
02712 
02713   if (_subdocument_resumes) {
02714     if (_download_dest == DD_file) {
02715       // Windows doesn't complain if you try to seek past the end of
02716       // file--it happily appends enough zero bytes to make the
02717       // difference.  Blecch.  That means we need to get the file size
02718       // first to check it ourselves.
02719       _download_to_file.seekp(0, ios::end);
02720       if (_first_byte_delivered > (size_t)_download_to_file.tellp()) {
02721         downloader_cat.info()
02722           << "Invalid starting position of byte " << _first_byte_delivered
02723           << " within " << _download_to_filename << " (which has " 
02724           << _download_to_file.tellp() << " bytes)\n";
02725         _download_to_file.close();
02726         _status_entry._status_code = SC_download_invalid_range;
02727         _state = S_failure;
02728         return false;
02729       }
02730       
02731       _download_to_file.seekp(_first_byte_delivered);
02732       
02733     } else if (_download_dest == DD_ram) {
02734       if (_first_byte_delivered > _download_to_ramfile->_data.length()) {
02735         downloader_cat.info()
02736           << "Invalid starting position of byte " << _first_byte_delivered 
02737           << " within Ramfile (which has " 
02738           << _download_to_ramfile->_data.length() << " bytes)\n";
02739         _status_entry._status_code = SC_download_invalid_range;
02740         _state = S_failure;
02741         return false;
02742       }
02743 
02744       if (_first_byte_delivered == 0) {
02745         _download_to_ramfile->_data = string();
02746       } else {
02747         _download_to_ramfile->_data = 
02748           _download_to_ramfile->_data.substr(0, _first_byte_delivered);
02749       }
02750     } else if (_download_dest == DD_stream) {
02751       // Windows doesn't complain if you try to seek past the end of
02752       // file--it happily appends enough zero bytes to make the
02753       // difference.  Blecch.  That means we need to get the file size
02754       // first to check it ourselves.
02755       _download_to_stream->seekp(0, ios::end);
02756       if (_first_byte_delivered > (size_t)_download_to_stream->tellp()) {
02757         downloader_cat.info()
02758           << "Invalid starting position of byte " << _first_byte_delivered
02759           << " within stream (which has " 
02760           << _download_to_stream->tellp() << " bytes)\n";
02761         _download_to_stream = NULL;
02762         _status_entry._status_code = SC_download_invalid_range;
02763         _state = S_failure;
02764         return false;
02765       }
02766       
02767       _download_to_stream->seekp(_first_byte_delivered);
02768     }
02769 
02770   } else {
02771     // If _subdocument_resumes is false, we should be sure to reset to
02772     // the beginning of the file, regardless of the value of
02773     // _first_byte_delivered.
02774     if (_download_dest == DD_file) {
02775       _download_to_file.seekp(0);
02776     } else if (_download_dest == DD_ram) {
02777       _download_to_ramfile->_data = string();
02778     } else if (_download_dest == DD_stream) {
02779       _download_to_stream->seekp(0);
02780     }
02781   }
02782 
02783   return true;
02784 }
02785 
02786 
02787 ////////////////////////////////////////////////////////////////////
02788 //     Function: HTTPChannel::server_getline
02789 //       Access: Private
02790 //  Description: Reads a single line from the server's reply.  Returns
02791 //               true if the line is successfully retrieved, or false
02792 //               if a complete line has not yet been received or if
02793 //               the connection has been closed.
02794 ////////////////////////////////////////////////////////////////////
02795 bool HTTPChannel::
02796 server_getline(string &str) {
02797   nassertr(!_source.is_null(), false);
02798   int ch = (*_source)->get();
02799   while (!(*_source)->eof() && !(*_source)->fail()) {
02800     switch (ch) {
02801     case '\n':
02802       // end-of-line character, we're done.
02803       str = _working_get;
02804       _working_get = string();
02805       {
02806         // Trim trailing whitespace.  We're not required to do this per the
02807         // HTTP spec, but let's be generous.
02808         size_t p = str.length();
02809         while (p > 0 && isspace(str[p - 1])) {
02810           --p;
02811         }
02812         str = str.substr(0, p);
02813       }
02814       if (downloader_cat.is_debug()) {
02815         downloader_cat.debug() << "recv: " << str << "\n";
02816       }
02817       return true;
02818 
02819     case '\r':
02820       // Ignore CR characters.
02821       break;
02822 
02823     default:
02824       _working_get += (char)ch;
02825     }
02826     ch = (*_source)->get();
02827   }
02828 
02829   check_socket();
02830   return false;
02831 }
02832 
02833 ////////////////////////////////////////////////////////////////////
02834 //     Function: HTTPChannel::server_getline_failsafe
02835 //       Access: Private
02836 //  Description: Reads a line from the server's reply.  If the server
02837 //               disconnects or times out before sending a reply,
02838 //               moves on to the next proxy server (or sets failure
02839 //               mode) and returns false; otherwise, returns true.
02840 ////////////////////////////////////////////////////////////////////
02841 bool HTTPChannel::
02842 server_getline_failsafe(string &str) {
02843   if (!server_getline(str)) {
02844     if (_bio.is_null()) {
02845       // Huh, the server hung up on us as soon as we tried to connect.
02846       if (_response_type == RT_hangup) {
02847         // This was our second immediate hangup in a row.  Give up.
02848         _status_entry._status_code = SC_lost_connection;
02849         _state = S_try_next_proxy;
02850         
02851       } else {
02852         // Try again, once.
02853         _response_type = RT_hangup;
02854       }
02855 
02856     } else {
02857       double elapsed =
02858         TrueClock::get_global_ptr()->get_short_time() -
02859         _sent_request_time;
02860       if (elapsed > get_http_timeout()) {
02861         // Time to give up.
02862         downloader_cat.info()
02863           << "Timeout waiting for "
02864           << _request.get_url().get_server_and_port() 
02865           << " in server_getline_failsafe (" << elapsed 
02866           << " seconds elapsed).\n";
02867         _status_entry._status_code = SC_timeout;
02868         _state = S_try_next_proxy;
02869       }
02870     }
02871     
02872     return false;
02873   }
02874   return true;
02875 }
02876 
02877 ////////////////////////////////////////////////////////////////////
02878 //     Function: HTTPChannel::server_get
02879 //       Access: Private
02880 //  Description: Reads a fixed number of bytes from the server's
02881 //               reply.  Returns true if the indicated number of bytes
02882 //               are successfully retrieved, or false if the complete
02883 //               set has not yet been received or if the connection
02884 //               has been closed.
02885 ////////////////////////////////////////////////////////////////////
02886 bool HTTPChannel::
02887 server_get(string &str, size_t num_bytes) {
02888   nassertr(!_source.is_null(), false);
02889   int ch = (*_source)->get();
02890   while (!(*_source)->eof() && !(*_source)->fail()) {
02891     _working_get += (char)ch;
02892     if (_working_get.length() >= num_bytes) {
02893       str = _working_get;
02894       _working_get = string();
02895       return true;
02896     }
02897 
02898     ch = (*_source)->get();
02899   }
02900 
02901   check_socket();
02902   return false;
02903 }
02904 
02905 ////////////////////////////////////////////////////////////////////
02906 //     Function: HTTPChannel::server_get_failsafe
02907 //       Access: Private
02908 //  Description: Reads a fixed number of bytes from the server.  If
02909 //               the server disconnects or times out before sending a
02910 //               reply, moves on to the next proxy server (or sets
02911 //               failure mode) and returns false; otherwise, returns
02912 //               true.
02913 ////////////////////////////////////////////////////////////////////
02914 bool HTTPChannel::
02915 server_get_failsafe(string &str, size_t num_bytes) {
02916   if (!server_get(str, num_bytes)) {
02917     if (_bio.is_null()) {
02918       // Huh, the server hung up on us as soon as we tried to connect.
02919       if (_response_type == RT_hangup) {
02920         // This was our second immediate hangup in a row.  Give up.
02921         _status_entry._status_code = SC_lost_connection;
02922         _state = S_try_next_proxy;
02923         
02924       } else {
02925         // Try again, once.
02926         _response_type = RT_hangup;
02927       }
02928       
02929     } else {
02930       double elapsed =
02931         TrueClock::get_global_ptr()->get_short_time() -
02932         _sent_request_time;
02933       if (elapsed > get_http_timeout()) {
02934         // Time to give up.
02935         downloader_cat.info()
02936           << "Timeout waiting for "
02937           << _request.get_url().get_server_and_port() 
02938           << " in server_get_failsafe (" << elapsed 
02939           << " seconds elapsed).\n";
02940         _status_entry._status_code = SC_timeout;
02941         _state = S_try_next_proxy;
02942       }
02943     }
02944     
02945     return false;
02946   }
02947   return true;
02948 }
02949 
02950 ////////////////////////////////////////////////////////////////////
02951 //     Function: HTTPChannel::server_send
02952 //       Access: Private
02953 //  Description: Sends a series of lines to the server.  Returns true
02954 //               if the buffer is fully sent, or false if some of it
02955 //               remains.  If this returns false, the function must be
02956 //               called again later, passing in the exact same string,
02957 //               until the return value is true.
02958 //
02959 //               If the secret flag is true, the data is not echoed to
02960 //               the log (even in spam mode).  This may be desirable
02961 //               if the data may contain binary data, or if it may
02962 //               contain passwords etc.
02963 ////////////////////////////////////////////////////////////////////
02964 bool HTTPChannel::
02965 server_send(const string &str, bool secret) {
02966   nassertr(str.length() > _sent_so_far, true);
02967 
02968   // Use the underlying BIO to write to the server, instead of the
02969   // BIOStream, which would insist on blocking (and might furthermore
02970   // delay the send due to collect-tcp mode being enabled).
02971   size_t bytes_to_send = str.length() - _sent_so_far;
02972   int write_count =
02973     BIO_write(*_bio, str.data() + _sent_so_far, bytes_to_send);
02974     
02975   if (write_count <= 0) {
02976     if (BIO_should_retry(*_bio)) {
02977       // Temporary failure: the pipe is full.  Wait till later.
02978       return false;
02979     }
02980     // Oops, the connection has been closed!
02981     if (downloader_cat.is_debug()) {
02982       downloader_cat.debug()
02983         << "Lost connection to server unexpectedly during write.\n";
02984     }
02985     reset_to_new();
02986     return false;
02987   }
02988 
02989   if (downloader_cat.is_spam()) {
02990     downloader_cat.spam()
02991       << "wrote " << write_count << " bytes to " << _bio << "\n";
02992   }
02993   
02994 #ifndef NDEBUG
02995   if (!secret && downloader_cat.is_debug()) {
02996     show_send(str.substr(0, write_count));
02997   }
02998 #endif
02999   
03000   if (write_count < (int)bytes_to_send) {
03001     _sent_so_far += write_count;
03002     return false;
03003   }
03004 
03005   // Buffer completely sent.
03006   _sent_so_far = 0;
03007   return true;
03008 }
03009 
03010 ////////////////////////////////////////////////////////////////////
03011 //     Function: HTTPChannel::parse_http_response
03012 //       Access: Private
03013 //  Description: Parses the first line sent back from an HTTP server
03014 //               or proxy and stores the result in _status_code and
03015 //               _http_version, etc.  Returns true on success, false
03016 //               on invalid response.
03017 ////////////////////////////////////////////////////////////////////
03018 bool HTTPChannel::
03019 parse_http_response(const string &line) {
03020   // The first line back should include the HTTP version and the
03021   // result code.
03022   if (line.length() < 5 || line.substr(0, 5) != string("HTTP/")) {
03023     // Not an HTTP response.
03024     _status_entry._status_code = SC_non_http_response;
03025     if (_response_type == RT_non_http) {
03026       // This was our second non-HTTP response in a row.  Give up.
03027       _state = S_try_next_proxy;
03028 
03029     } else {
03030       // Maybe we were just in some bad state.  Drop the connection
03031       // and try again, once.
03032       if (downloader_cat.is_debug()) {
03033         downloader_cat.debug()
03034           << "got non-HTTP response, resetting.\n";
03035       }
03036       reset_to_new();
03037       _response_type = RT_non_http;
03038     }
03039     return false;
03040   }
03041 
03042   // Split out the first line into its three components.
03043   size_t p = 5;
03044   while (p < line.length() && !isspace(line[p])) {
03045     p++;
03046   }
03047   _http_version_string = line.substr(0, p);
03048   _http_version = HTTPClient::parse_http_version_string(_http_version_string);
03049 
03050   while (p < line.length() && isspace(line[p])) {
03051     p++;
03052   }
03053   size_t q = p;
03054   while (q < line.length() && !isspace(line[q])) {
03055     q++;
03056   }
03057   string status_code = line.substr(p, q - p);
03058   _status_entry._status_code = atoi(status_code.c_str());
03059 
03060   while (q < line.length() && isspace(line[q])) {
03061     q++;
03062   }
03063   _status_entry._status_string = line.substr(q, line.length() - q);
03064 
03065   return true;
03066 }
03067 
03068 ////////////////////////////////////////////////////////////////////
03069 //     Function: HTTPChannel::parse_http_header
03070 //       Access: Private
03071 //  Description: Reads the series of header lines from the server and
03072 //               stores them in _headers.  Returns true if there is
03073 //               more to read, false when done.
03074 ////////////////////////////////////////////////////////////////////
03075 bool HTTPChannel::
03076 parse_http_header() {
03077   string line;
03078   if (!server_getline(line)) {
03079     return true;
03080   }
03081 
03082   while (!line.empty()) {
03083     if (isspace(line[0])) {
03084       // If the line begins with a space, that continues the previous
03085       // field.
03086       size_t p = 0;
03087       while (p < line.length() && isspace(line[p])) {
03088         p++;
03089       }
03090       _current_field_value += line.substr(p - 1);
03091 
03092     } else {
03093       // If the line does not begin with a space, that defines a new
03094       // field.
03095       if (!_current_field_name.empty()) {
03096         store_header_field(_current_field_name, _current_field_value);
03097         _current_field_value = string();
03098       }
03099 
03100       size_t colon = line.find(':');
03101       if (colon != string::npos) {
03102         _current_field_name = downcase(line.substr(0, colon));
03103         size_t p = colon + 1;
03104         while (p < line.length() && isspace(line[p])) {
03105           p++;
03106         }
03107         _current_field_value = line.substr(p);
03108       }
03109     }
03110 
03111     if (!server_getline(line)) {
03112       return true;
03113     }
03114   }
03115 
03116   // After reading an empty line, we're done with the headers.
03117   if (!_current_field_name.empty()) {
03118     store_header_field(_current_field_name, _current_field_value);
03119     _current_field_value = string();
03120   }
03121 
03122   return false;
03123 }
03124 
03125 ////////////////////////////////////////////////////////////////////
03126 //     Function: HTTPChannel::parse_content_range
03127 //       Access: Private
03128 //  Description: Interprets the "Content-Range" header in the reply,
03129 //               and fills in _first_byte_delivered and
03130 //               _last_byte_delivered appropriately if the header
03131 //               response can be understood.
03132 ////////////////////////////////////////////////////////////////////
03133 bool HTTPChannel::
03134 parse_content_range(const string &content_range) {
03135   // First, get the units indication.
03136   size_t p = 0;
03137   while (p < content_range.length() && !isspace(content_range[p])) {
03138     p++;
03139   }
03140 
03141   string units = content_range.substr(0, p);
03142   while (p < content_range.length() && isspace(content_range[p])) {
03143     p++;
03144   }
03145 
03146   if (units == "bytes") {
03147     const char *c_str = content_range.c_str();
03148     char *endptr;
03149     if (p < content_range.length() && isdigit(content_range[p])) {
03150       long first_byte = strtol(c_str + p, &endptr, 10);
03151       p = endptr - c_str;
03152       if (p < content_range.length() && content_range[p] == '-') {
03153         p++;
03154         if (p < content_range.length() && isdigit(content_range[p])) {
03155           long last_byte = strtol(c_str + p, &endptr, 10);
03156           p = endptr - c_str;
03157           
03158           if (last_byte >= first_byte) {
03159             _first_byte_delivered = first_byte;
03160             _last_byte_delivered = last_byte;
03161             return true;
03162           }
03163         }
03164       }
03165     }
03166   }
03167     
03168   // Invalid or unhandled response.
03169   return false;
03170 }
03171 
03172 
03173 ////////////////////////////////////////////////////////////////////
03174 //     Function: HTTPChannel::check_socket
03175 //       Access: Private
03176 //  Description: Checks whether the connection to the server has been
03177 //               closed after a failed read.  If it has, issues a
03178 //               warning and calls reset_to_new().
03179 ////////////////////////////////////////////////////////////////////
03180 void HTTPChannel::
03181 check_socket() {
03182   nassertv(!_source.is_null());
03183   if ((*_source)->is_closed()) {
03184     if (downloader_cat.is_debug()) {
03185       downloader_cat.debug()
03186         << "Lost connection to server unexpectedly during read.\n";
03187     }
03188     reset_to_new();
03189   }
03190 }
03191 
03192 /*
03193    Certificate verify error codes:
03194 
03195 0 X509_V_OK: ok
03196 
03197     the operation was successful.
03198 
03199 2 X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT: unable to get issuer certificate
03200 
03201     the issuer certificate could not be found: this occurs if the
03202     issuer certificate of an untrusted certificate cannot be found.
03203 
03204 3 X509_V_ERR_UNABLE_TO_GET_CRL unable to get certificate CRL
03205 
03206     the CRL of a certificate could not be found. Unused.
03207 
03208 4 X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE: unable to decrypt
03209 certificate's signature
03210 
03211     the certificate signature could not be decrypted. This means that
03212     the actual signature value could not be determined rather than it
03213     not matching the expected value, this is only meaningful for RSA
03214     keys.
03215 
03216 5 X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE: unable to decrypt CRL's signature
03217 
03218     the CRL signature could not be decrypted: this means that the
03219     actual signature value could not be determined rather than it not
03220     matching the expected value. Unused.
03221 
03222 6 X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY: unable to decode
03223 issuer public key
03224 
03225     the public key in the certificate SubjectPublicKeyInfo could not
03226     be read.
03227 
03228 7 X509_V_ERR_CERT_SIGNATURE_FAILURE: certificate signature failure
03229 
03230     the signature of the certificate is invalid.
03231 
03232 8 X509_V_ERR_CRL_SIGNATURE_FAILURE: CRL signature failure
03233 
03234     the signature of the certificate is invalid. Unused.
03235 
03236 9 X509_V_ERR_CERT_NOT_YET_VALID: certificate is not yet valid
03237 
03238     the certificate is not yet valid: the notBefore date is after the
03239     current time.
03240 
03241 10 X509_V_ERR_CERT_HAS_EXPIRED: certificate has expired
03242 
03243     the certificate has expired: that is the notAfter date is before
03244     the current time.
03245 
03246 11 X509_V_ERR_CRL_NOT_YET_VALID: CRL is not yet valid
03247 
03248     the CRL is not yet valid. Unused.
03249 
03250 12 X509_V_ERR_CRL_HAS_EXPIRED: CRL has expired
03251 
03252     the CRL has expired. Unused.
03253 
03254 13 X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD: format error in
03255 certificate's notBefore field
03256 
03257     the certificate notBefore field contains an invalid time.
03258 
03259 14 X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD: format error in
03260 certificate's notAfter field
03261 
03262     the certificate notAfter field contains an invalid time.
03263 
03264 15 X509_V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD: format error in CRL's
03265 lastUpdate field
03266 
03267     the CRL lastUpdate field contains an invalid time. Unused.
03268 
03269 16 X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD: format error in CRL's
03270 nextUpdate field
03271 
03272     the CRL nextUpdate field contains an invalid time. Unused.
03273 
03274 17 X509_V_ERR_OUT_OF_MEM: out of memory
03275 
03276     an error occurred trying to allocate memory. This should never
03277     happen.
03278 
03279 18 X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT: self signed certificate
03280 
03281     the passed certificate is self signed and the same certificate
03282     cannot be found in the list of trusted certificates.
03283 
03284 19 X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN: self signed certificate in
03285 certificate chain
03286 
03287     the certificate chain could be built up using the untrusted
03288     certificates but the root could not be found locally.
03289 
03290 20 X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY: unable to get local
03291 issuer certificate
03292 
03293     the issuer certificate of a locally looked up certificate could
03294     not be found. This normally means the list of trusted certificates
03295     is not complete.
03296 
03297 21 X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE: unable to verify the
03298 first certificate
03299 
03300     no signatures could be verified because the chain contains only
03301     one certificate and it is not self signed.
03302 
03303 22 X509_V_ERR_CERT_CHAIN_TOO_LONG: certificate chain too long
03304 
03305     the certificate chain length is greater than the supplied maximum
03306     depth. Unused.
03307 
03308 23 X509_V_ERR_CERT_REVOKED: certificate revoked
03309 
03310     the certificate has been revoked. Unused.
03311 
03312 24 X509_V_ERR_INVALID_CA: invalid CA certificate
03313 
03314     a CA certificate is invalid. Either it is not a CA or its
03315     extensions are not consistent with the supplied purpose.
03316 
03317 25 X509_V_ERR_PATH_LENGTH_EXCEEDED: path length constraint exceeded
03318 
03319     the basicConstraints pathlength parameter has been exceeded.
03320 
03321 26 X509_V_ERR_INVALID_PURPOSE: unsupported certificate purpose
03322 
03323     the supplied certificate cannot be used for the specified purpose.
03324 
03325 27 X509_V_ERR_CERT_UNTRUSTED: certificate not trusted
03326 
03327     the root CA is not marked as trusted for the specified purpose.
03328 
03329 28 X509_V_ERR_CERT_REJECTED: certificate rejected
03330 
03331     the root CA is marked to reject the specified purpose.
03332 
03333 29 X509_V_ERR_SUBJECT_ISSUER_MISMATCH: subject issuer mismatch
03334 
03335     the current candidate issuer certificate was rejected because its
03336     subject name did not match the issuer name of the current
03337     certificate. Only displayed when the -issuer_checks option is set.
03338 
03339 30 X509_V_ERR_AKID_SKID_MISMATCH: authority and subject key identifier
03340 mismatch
03341 
03342     the current candidate issuer certificate was rejected because its
03343     subject key identifier was present and did not match the authority
03344     key identifier current certificate. Only displayed when the
03345     -issuer_checks option is set.
03346 
03347 31 X509_V_ERR_AKID_ISSUER_SERIAL_MISMATCH: authority and issuer serial
03348 number mismatch
03349 
03350     the current candidate issuer certificate was rejected because its
03351     issuer name and serial number was present and did not match the
03352     authority key identifier of the current certificate. Only
03353     displayed when the -issuer_checks option is set.
03354 
03355 32 X509_V_ERR_KEYUSAGE_NO_CERTSIGN:key usage does not include
03356 certificate signing
03357 
03358     the current candidate issuer certificate was rejected because its
03359     keyUsage extension does not permit certificate signing.
03360 
03361 50 X509_V_ERR_APPLICATION_VERIFICATION: application verification failure
03362 
03363     an application specific error. Unused.
03364 
03365 */
03366 
03367 ////////////////////////////////////////////////////////////////////
03368 //     Function: HTTPChannel::check_preapproved_server_certificate
03369 //       Access: Private
03370 //  Description: Checks to see if the indicated certificate is on the
03371 //               pre-approved list for the current server.
03372 //
03373 //               If the full cert itself (including its key) is on the
03374 //               pre-approved list, sets both cert_preapproved and
03375 //               cert_name_preapproved to true.
03376 //
03377 //               If the full cert is not on the pre-approved list, but
03378 //               its name matches a name on the pre-approved list,
03379 //               sets cert_name_preapproved to true, and
03380 //               cert_preapproved to false.
03381 //
03382 //               Otherwise, sets both values to false.  This doesn't
03383 //               mean the cert is necessarily invalid, just that it
03384 //               wasn't on the pre-approved list (which is usually
03385 //               empty anyway).
03386 ////////////////////////////////////////////////////////////////////
03387 void HTTPChannel::
03388 check_preapproved_server_certificate(X509 *cert, bool &cert_preapproved, 
03389                                      bool &cert_name_preapproved) const {
03390   return _client->check_preapproved_server_certificate(_request.get_url(),
03391                                                        cert, cert_preapproved,
03392                                                        cert_name_preapproved);
03393 }
03394 
03395 ////////////////////////////////////////////////////////////////////
03396 //     Function: HTTPChannel::validate_server_name
03397 //       Access: Private
03398 //  Description: Returns true if the name in the cert matches the
03399 //               hostname of the server, false otherwise.
03400 ////////////////////////////////////////////////////////////////////
03401 bool HTTPChannel::
03402 validate_server_name(X509 *cert) {
03403   string hostname = _request.get_url().get_server();
03404 
03405   vector_string cert_names;
03406 
03407   // According to RFC 2818, we should check the DNS name(s) in the
03408   // subjectAltName extension first, if that extension exists.
03409   STACK_OF(GENERAL_NAME) *subject_alt_names =
03410     (STACK_OF(GENERAL_NAME) *)X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL);
03411   if (subject_alt_names != NULL) {
03412     int num_alts = sk_GENERAL_NAME_num(subject_alt_names);
03413     for (int i = 0; i < num_alts; ++i) {
03414       // Get the ith alt name.
03415       const GENERAL_NAME *alt_name = 
03416         sk_GENERAL_NAME_value(subject_alt_names, i);
03417       
03418       if (alt_name->type == GEN_DNS) {
03419         char *buffer = NULL;
03420         int len = ASN1_STRING_to_UTF8((unsigned char**)&buffer,
03421                                       alt_name->d.ia5);
03422         if (len > 0) {
03423           cert_names.push_back(string(buffer, len));
03424         }
03425         if (buffer != NULL) {
03426           OPENSSL_free(buffer);
03427         }
03428       }
03429     }
03430   }
03431 
03432   if (cert_names.empty()) {
03433     // If there were no DNS names, use the common name instead.
03434   
03435     X509_NAME *xname = X509_get_subject_name(cert);
03436     if (xname != NULL) {
03437       string common_name = get_x509_name_component(xname, NID_commonName);
03438       cert_names.push_back(common_name);
03439     }
03440   }
03441 
03442   if (cert_names.empty()) {
03443     downloader_cat.info()
03444       << "Server certificate from " << hostname
03445       << " provides no name.\n";
03446     return false;
03447   }
03448 
03449   if (downloader_cat.is_debug()) {
03450     downloader_cat.debug()
03451       << "Server certificate from " << hostname
03452       << " provides name(s):";
03453     vector_string::const_iterator si;
03454     for (si = cert_names.begin(); si != cert_names.end(); ++si) {
03455       const string &cert_name = (*si);
03456       downloader_cat.debug(false)
03457         << " " << cert_name;
03458     }
03459     downloader_cat.debug(false)
03460       << "\n";
03461   }
03462 
03463   // Now validate the names we found.  If any of them matches, the
03464   // cert matches.
03465   vector_string::const_iterator si;
03466   for (si = cert_names.begin(); si != cert_names.end(); ++si) {
03467     const string &cert_name = (*si);
03468 
03469     if (match_cert_name(cert_name, hostname)) {
03470       return true;
03471     }
03472   }
03473 
03474   downloader_cat.info()
03475     << "Server certificate from " << hostname
03476     << " provides wrong name(s):";
03477   for (si = cert_names.begin(); si != cert_names.end(); ++si) {
03478     const string &cert_name = (*si);
03479     downloader_cat.info(false)
03480       << " " << cert_name;
03481   }
03482   downloader_cat.info(false)
03483     << "\n";
03484 
03485   return false;
03486 }
03487 
03488 ////////////////////////////////////////////////////////////////////
03489 //     Function: HTTPChannel::match_cert_name
03490 //       Access: Private, Static
03491 //  Description: Returns true if this particular name from the
03492 //               certificate matches the indicated hostname, false
03493 //               otherwise.
03494 ////////////////////////////////////////////////////////////////////
03495 bool HTTPChannel::
03496 match_cert_name(const string &cert_name, const string &hostname) {
03497   // We use GlobPattern to match the name.  This isn't quite
03498   // consistent with RFC2818, since it also accepts additional
03499   // wildcard characters like "?" and "[]", but I think it's close
03500   // enough.
03501 
03502   GlobPattern pattern(cert_name);
03503   pattern.set_case_sensitive(false);
03504   pattern.set_nomatch_chars(".");
03505   return pattern.matches(hostname);
03506 }
03507     
03508 ////////////////////////////////////////////////////////////////////
03509 //     Function: HTTPChannel::get_x509_name_component
03510 //       Access: Private, Static
03511 //  Description: Returns the indicated component of the X509 name as a
03512 //               string, if defined, or empty string if it is not.
03513 ////////////////////////////////////////////////////////////////////
03514 string HTTPChannel::
03515 get_x509_name_component(X509_NAME *name, int nid) {
03516   ASN1_OBJECT *obj = OBJ_nid2obj(nid);
03517 
03518   if (obj == NULL) {
03519     // Unknown nid.  See openssl/objects.h.
03520     return string();
03521   }
03522 
03523   int i = X509_NAME_get_index_by_OBJ(name, obj, -1);
03524   if (i < 0) {
03525     return string();
03526   }
03527 
03528   ASN1_STRING *data = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(name, i));
03529   return string((char *)data->data, data->length);  
03530 }
03531 
03532 ////////////////////////////////////////////////////////////////////
03533 //     Function: HTTPChannel::make_header
03534 //       Access: Private
03535 //  Description: Formats the appropriate GET or POST (or whatever)
03536 //               request to send to the server, based on the current
03537 //               _method, _document_spec, _body, and _proxy settings.
03538 ////////////////////////////////////////////////////////////////////
03539 void HTTPChannel::
03540 make_header() {
03541   _proxy_auth = _client->select_auth(_proxy, true, _proxy_realm);
03542   _proxy_username = string();
03543   if (_proxy_auth != (HTTPAuthorization *)NULL) {
03544     _proxy_realm = _proxy_auth->get_realm();
03545     _proxy_username = _client->select_username(_proxy, true, _proxy_realm);
03546   }
03547 
03548   if (_method == HTTPEnum::M_connect) {
03549     // This method doesn't require an HTTP header at all; we'll just
03550     // open a plain connection.  (Except when we're using a proxy; but
03551     // in that case, it's the proxy_header we'll need, not the regular
03552     // HTTP header.)
03553     _header = string();
03554     return;
03555   }
03556 
03557   _www_auth = _client->select_auth(_request.get_url(), false, _www_realm);
03558   _www_username = string();
03559   if (_www_auth != (HTTPAuthorization *)NULL) {
03560     _www_realm = _www_auth->get_realm();
03561     _www_username = _client->select_username(_request.get_url(), false, _www_realm);
03562   }
03563 
03564   string request_path;
03565   if (_proxy_serves_document) {
03566     // If we'll be asking the proxy for the document, we need its full
03567     // URL--but we omit the username, which is information just for us.
03568     URLSpec url_no_username = _request.get_url();
03569     url_no_username.set_username(string());
03570     request_path = url_no_username.get_url();
03571 
03572   } else {
03573     // If we'll be asking the server directly for the document, we
03574     // just want its path relative to the server.
03575     request_path = _request.get_url().get_path_and_query();
03576   }
03577 
03578   // HTTP syntax always requires something in the request path.  If it
03579   // is empty, put in a star as a placeholder (OPTIONS, for instance,
03580   // uses this).
03581   if (request_path.empty()) {
03582     request_path = "*";
03583   }
03584 
03585   ostringstream stream;
03586 
03587   stream 
03588     << _method << " " << request_path << " " 
03589     << _client->get_http_version_string() << "\r\n";
03590 
03591   if (_client->get_http_version() >= HTTPEnum::HV_11) {
03592     
03593     stream 
03594       << "Host: " << _request.get_url().get_server();
03595     if (!_request.get_url().is_default_port()) {
03596       // It appears that some servers (notably gstatic.com) might
03597       // return a 404 if you include an explicit port number in with
03598       // the Host: header, even if it is the default port.  So, don't
03599       // include the port number unless we need to.
03600       stream << ":" << _request.get_url().get_port();
03601     }
03602     stream << "\r\n";
03603     if (!get_persistent_connection()) {
03604       stream
03605         << "Connection: close\r\n";
03606     }
03607   }
03608 
03609   if (_last_byte_requested != 0) {
03610     stream 
03611       << "Range: bytes=" << _first_byte_requested << "-" 
03612       << _last_byte_requested << "\r\n";
03613 
03614   } else if (_first_byte_requested != 0) {
03615     stream 
03616       << "Range: bytes=" << _first_byte_requested << "-\r\n";
03617   }
03618 
03619   switch (_request.get_request_mode()) {
03620   case DocumentSpec::RM_any:
03621     // No particular request; give us any document that matches the
03622     // URL.  
03623     if (_first_byte_requested != 0) {
03624       // Unless we're requesting a subrange, in which case if the
03625       // exact document matches, retrieve the subrange indicated;
03626       // otherwise, retrieve the entire document.
03627       if (_request.has_tag()) {
03628         stream
03629           << "If-Range: " << _request.get_tag().get_string() << "\r\n";
03630       } else if (_request.has_date()) {
03631         stream
03632           << "If-Range: " << _request.get_date().get_string() << "\r\n";
03633       }
03634     }
03635     break;
03636 
03637   case DocumentSpec::RM_equal:
03638     // Give us only this particular version of the document, or
03639     // nothing.
03640     if (_request.has_tag()) {
03641       stream
03642         << "If-Match: " << _request.get_tag().get_string() << "\r\n";
03643     }
03644     if (_request.has_date()) {
03645       stream
03646         << "If-Unmodified-Since: " << _request.get_date().get_string()
03647         << "\r\n";
03648     }
03649     break;
03650 
03651   case DocumentSpec::RM_newer:
03652     // Give us anything newer than this document, or nothing.
03653     if (_request.has_tag()) {
03654       stream
03655         << "If-None-Match: " << _request.get_tag().get_string() << "\r\n";
03656     }
03657     if (_request.has_date()) {
03658       stream
03659         << "If-Modified-Since: " << _request.get_date().get_string()
03660         << "\r\n";
03661     }
03662     break;
03663 
03664   case DocumentSpec::RM_equal_or_newer:
03665     // Just don't give us anything older.
03666     if (_request.has_date()) {
03667       // This is a little unreliable: we ask for any document that's
03668       // been modified since one second before our last-modified-date.
03669       // Who knows whether the server will honor this properly.
03670       stream
03671         << "If-Modified-Since: " << (_request.get_date() - 1).get_string()
03672         << "\r\n";
03673     }
03674     break;
03675   }
03676 
03677   switch (_request.get_cache_control()) {
03678   case DocumentSpec::CC_allow_cache:
03679     // Normal, caching behavior.
03680     break;
03681 
03682   case DocumentSpec::CC_revalidate:
03683     // Request the server to revalidate its cache before returning it.
03684     stream
03685       << "Cache-Control: max-age=0\r\n";
03686     break;
03687 
03688   case DocumentSpec::CC_no_cache:
03689     // Request the server to get a fresh copy regardless of its cache.
03690     stream
03691       << "Cache-Control: no-cache\r\n"
03692       << "Pragma: no-cache\r\n";
03693     break;
03694   }
03695 
03696   _client->send_cookies(stream, _request.get_url());
03697 
03698   if (!_body.empty()) {
03699     stream
03700       << "Content-Type: application/x-www-form-urlencoded\r\n"
03701       << "Content-Length: " << _body.length() << "\r\n";
03702   }
03703 
03704   _header = stream.str();
03705 }
03706 
03707 ////////////////////////////////////////////////////////////////////
03708 //     Function: HTTPChannel::make_proxy_request_text
03709 //       Access: Private
03710 //  Description: Builds the _proxy_request_text string.  This is a
03711 //               special request that will be sent directly to the
03712 //               proxy prior to the request tailored for the server.
03713 //               Generally this is used to open a tunnelling
03714 //               connection for https-over-proxy.
03715 ////////////////////////////////////////////////////////////////////
03716 void HTTPChannel::
03717 make_proxy_request_text() {
03718   _proxy_request_text = _proxy_header;
03719 
03720   if (_proxy_auth != (HTTPAuthorization *)NULL && !_proxy_username.empty()) {
03721     _proxy_request_text += "Proxy-Authorization: ";
03722     _proxy_request_text += 
03723       _proxy_auth->generate(HTTPEnum::M_connect, _request.get_url().get_server_and_port(),
03724                             _proxy_username, _body);
03725     _proxy_request_text += "\r\n";
03726   }
03727     
03728   _proxy_request_text += "\r\n";
03729 }
03730 
03731 ////////////////////////////////////////////////////////////////////
03732 //     Function: HTTPChannel::make_request_text
03733 //       Access: Private
03734 //  Description: Builds the _request_text string.  This is the
03735 //               specific request that will be sent to the server this
03736 //               pass, based on the current header and body.
03737 ////////////////////////////////////////////////////////////////////
03738 void HTTPChannel::
03739 make_request_text() {
03740   _request_text = _header;
03741 
03742   if (_proxy_serves_document &&
03743       _proxy_auth != (HTTPAuthorization *)NULL && !_proxy_username.empty()) {
03744     _request_text += "Proxy-Authorization: ";
03745     _request_text += 
03746       _proxy_auth->generate(_method, _request.get_url().get_url(), _proxy_username, _body);
03747     _request_text += "\r\n";
03748   }
03749 
03750   if (_www_auth != (HTTPAuthorization *)NULL && !_www_username.empty()) {
03751     string authorization = 
03752     _request_text += "Authorization: ";
03753     _request_text +=
03754       _www_auth->generate(_method, _request.get_url().get_path_and_query(), _www_username, _body);
03755     _request_text += "\r\n";
03756   }
03757 
03758   _request_text += _send_extra_headers;
03759   _request_text += "\r\n";
03760   _request_text += _body;
03761 }
03762   
03763 ////////////////////////////////////////////////////////////////////
03764 //     Function: HTTPChannel::reset_url
03765 //       Access: Private
03766 //  Description: Redirects the next connection to the indicated URL
03767 //               (from the previous URL).  This resets the socket if
03768 //               necessary when we are about to switch servers.
03769 ////////////////////////////////////////////////////////////////////
03770 void HTTPChannel::
03771 reset_url(const URLSpec &old_url, const URLSpec &new_url) {
03772   // If we change between http and https, we have to reset the
03773   // connection regardless of proxy.  Otherwise, we have to drop the
03774   // connection if the server or port changes, unless we're
03775   // communicating through a proxy.
03776 
03777   if (new_url.get_scheme() != old_url.get_scheme() ||
03778       (_proxy.empty() && (new_url.get_server() != old_url.get_server() || 
03779                           new_url.get_port() != old_url.get_port()))) {
03780     if (downloader_cat.is_debug()) {
03781       downloader_cat.debug()
03782         << "resetting for new server " 
03783         << new_url.get_server_and_port() << "\n";
03784     }
03785     reset_to_new();
03786   }
03787 }
03788 
03789 ////////////////////////////////////////////////////////////////////
03790 //     Function: HTTPChannel::store_header_field
03791 //       Access: Private
03792 //  Description: Stores a single name: value pair in the header list,
03793 //               or appends the value to the end of the existing
03794 //               value, if the header has been repeated.
03795 ////////////////////////////////////////////////////////////////////
03796 void HTTPChannel::
03797 store_header_field(const string &field_name, const string &field_value) {
03798   pair<Headers::iterator, bool> insert_result =
03799     _headers.insert(Headers::value_type(field_name, field_value));
03800 
03801   if (!insert_result.second) {
03802     // It didn't insert; thus, the field already existed.  Append the
03803     // new value.
03804     Headers::iterator hi = insert_result.first;
03805     (*hi).second += ", ";
03806     (*hi).second += field_value;
03807   }
03808 
03809   if (field_name == "set-cookie") {
03810     _client->set_cookie(HTTPCookie(field_value, _request.get_url()));
03811   }
03812 }
03813 
03814 #ifndef NDEBUG
03815 ////////////////////////////////////////////////////////////////////
03816 //     Function: HTTPChannel::show_send
03817 //       Access: Private, Static
03818 //  Description: Writes the outgoing message, one line at a time, to
03819 //               the debugging log.
03820 ////////////////////////////////////////////////////////////////////
03821 void HTTPChannel::
03822 show_send(const string &message) {
03823   size_t start = 0;
03824   size_t newline = message.find('\n', start);
03825   while (newline != string::npos) {
03826     // Assume every \n is preceded by a \r.
03827     downloader_cat.debug()
03828       << "send: " << message.substr(start, newline - start - 1) << "\n";
03829     start = newline + 1;
03830     newline = message.find('\n', start);
03831   }
03832 
03833   if (start < message.length()) {
03834     downloader_cat.debug()
03835       << "send: " << message.substr(start) << " (no newline)\n";
03836   }
03837 }
03838 #endif   // NDEBUG
03839 
03840 ////////////////////////////////////////////////////////////////////
03841 //     Function: HTTPChannel::reset_download_to
03842 //       Access: Private
03843 //  Description: Resets the indication of how the document will be
03844 //               downloaded.  This must be re-specified after each
03845 //               get_document() (or related) call.
03846 ////////////////////////////////////////////////////////////////////
03847 void HTTPChannel::
03848 reset_download_to() {
03849   _started_download = false;
03850   _download_to_file.close();
03851   _download_to_ramfile = (Ramfile *)NULL;
03852   _download_to_stream = NULL;
03853   _download_dest = DD_none;
03854 }
03855 
03856 ////////////////////////////////////////////////////////////////////
03857 //     Function: HTTPChannel::reset_to_new
03858 //       Access: Private
03859 //  Description: Closes the connection and resets the state to S_new.
03860 ////////////////////////////////////////////////////////////////////
03861 void HTTPChannel::
03862 reset_to_new() {
03863   close_connection();
03864   _state = S_new;
03865 }
03866 
03867 ////////////////////////////////////////////////////////////////////
03868 //     Function: HTTPChannel::reset_body_stream
03869 //       Access: Private
03870 //  Description: Clears the _body_stream pointer, if it is set.
03871 ////////////////////////////////////////////////////////////////////
03872 void HTTPChannel::
03873 reset_body_stream() {
03874   if (_owns_body_stream) {
03875     if (_body_stream != (ISocketStream *)NULL) {
03876       close_read_body(_body_stream);
03877       nassertv(_body_stream == (ISocketStream *)NULL && !_owns_body_stream);
03878     }
03879   } else {
03880     _body_stream = NULL;
03881   }
03882 }
03883 
03884 
03885 ////////////////////////////////////////////////////////////////////
03886 //     Function: HTTPChannel::close_connection
03887 //       Access: Private
03888 //  Description: Closes the connection but leaves the _state
03889 //               unchanged.
03890 ////////////////////////////////////////////////////////////////////
03891 void HTTPChannel::
03892 close_connection() {
03893   reset_body_stream();
03894   _source.clear();
03895   _bio.clear();
03896   _working_get = string();
03897   _sent_so_far = 0;
03898   _read_index++;
03899 }
03900 
03901 ////////////////////////////////////////////////////////////////////
03902 //     Function: HTTPChannel::more_useful_status_code
03903 //       Access: Private, Static
03904 //  Description: Returns true if status code a is a more useful value
03905 //               (that is, it represents a more-nearly successfully
03906 //               connection attempt, or contains more information)
03907 //               than b, or false otherwise.
03908 ////////////////////////////////////////////////////////////////////
03909 bool HTTPChannel::
03910 more_useful_status_code(int a, int b) {
03911   if (a >= 100 && b >= 100) {
03912     // Both represent HTTP responses.  Responses from a server (<
03913     // 1000) are better than those from a proxy; we take advantage of
03914     // the fact that we have already added 1000 to proxy responses.
03915     // Except for 407, so let's fix that now.
03916     if (a == 407) { 
03917       a += 1000;
03918     }
03919     if (b == 407) { 
03920       b += 1000;
03921     }
03922 
03923     // Now just check the series.
03924     int series_a = (a / 100);
03925     int series_b = (b / 100);
03926 
03927     // In general, a lower series is a closer success.
03928     return (series_a < series_b);
03929   }
03930 
03931   if (a < 100 && b < 100) {
03932     // Both represent non-HTTP responses.  Here a larger number is
03933     // better.
03934     return (a > b);
03935   }
03936 
03937   if (a < 100) {
03938     // a is a non-HTTP response, while b is an HTTP response.  HTTP is
03939     // generally, better, unless we exceeded SC_http_error_watermark.
03940     return (a > SC_http_error_watermark);
03941   }
03942 
03943   // Exactly the opposite case as above.
03944   return (b < SC_http_error_watermark);
03945 }
03946 
03947 
03948 ////////////////////////////////////////////////////////////////////
03949 //     Function: HTTPChannel::State output operator
03950 //  Description: 
03951 ////////////////////////////////////////////////////////////////////
03952 ostream &
03953 operator << (ostream &out, HTTPChannel::State state) {
03954 #ifdef NDEBUG
03955   return out << (int)state;
03956 #else
03957   switch (state) {
03958   case HTTPChannel::S_new:
03959     return out << "new";
03960 
03961   case HTTPChannel::S_try_next_proxy:
03962     return out << "try_next_proxy";
03963 
03964   case HTTPChannel::S_connecting:
03965     return out << "connecting";
03966 
03967   case HTTPChannel::S_connecting_wait:
03968     return out << "connecting_wait";
03969 
03970   case HTTPChannel::S_http_proxy_ready:
03971     return out << "http_proxy_ready";
03972 
03973   case HTTPChannel::S_http_proxy_request_sent:
03974     return out << "http_proxy_request_sent";
03975 
03976   case HTTPChannel::S_http_proxy_reading_header:
03977     return out << "http_proxy_reading_header";
03978 
03979   case HTTPChannel::S_socks_proxy_greet:
03980     return out << "socks_proxy_greet";
03981 
03982   case HTTPChannel::S_socks_proxy_greet_reply:
03983     return out << "socks_proxy_greet_reply";
03984 
03985   case HTTPChannel::S_socks_proxy_connect:
03986     return out << "socks_proxy_connect";
03987 
03988   case HTTPChannel::S_socks_proxy_connect_reply:
03989     return out << "socks_proxy_connect_reply";
03990 
03991   case HTTPChannel::S_setup_ssl:
03992     return out << "setup_ssl";
03993 
03994   case HTTPChannel::S_ssl_handshake:
03995     return out << "ssl_handshake";
03996 
03997   case HTTPChannel::S_ready:
03998     return out << "ready";
03999 
04000   case HTTPChannel::S_request_sent:
04001     return out << "request_sent";
04002 
04003   case HTTPChannel::S_reading_header:
04004     return out << "reading_header";
04005 
04006   case HTTPChannel::S_start_direct_file_read:
04007     return out << "start_direct_file_read";
04008 
04009   case HTTPChannel::S_read_header:
04010     return out << "read_header";
04011 
04012   case HTTPChannel::S_begin_body:
04013     return out << "begin_body";
04014 
04015   case HTTPChannel::S_reading_body:
04016     return out << "reading_body";
04017 
04018   case HTTPChannel::S_read_body:
04019     return out << "read_body";
04020 
04021   case HTTPChannel::S_read_trailer:
04022     return out << "read_trailer";
04023 
04024   case HTTPChannel::S_failure:
04025     return out << "failure";
04026   }
04027 
04028   return out << "invalid state(" << (int)state << ")";
04029 #endif  // NDEBUG
04030 }
04031 
04032 #endif  // HAVE_OPENSSL
 All Classes Functions Variables Enumerations