Panda3D
|
00001 // Filename: httpChannel.cxx 00002 // Created by: drose (24Sep02) 00003 // 00004 //////////////////////////////////////////////////////////////////// 00005 // 00006 // PANDA 3D SOFTWARE 00007 // Copyright (c) Carnegie Mellon University. All rights reserved. 00008 // 00009 // All use of this software is subject to the terms of the revised BSD 00010 // license. You should have received a copy of this license along 00011 // with this source code in a file named "LICENSE." 00012 // 00013 //////////////////////////////////////////////////////////////////// 00014 00015 #include "httpChannel.h" 00016 #include "httpClient.h" 00017 #include "httpCookie.h" 00018 #include "bioStream.h" 00019 #include "chunkedStream.h" 00020 #include "identityStream.h" 00021 #include "config_downloader.h" 00022 #include "virtualFileMountHTTP.h" 00023 #include "ramfile.h" 00024 #include "globPattern.h" 00025 00026 #include <stdio.h> 00027 00028 #ifdef HAVE_OPENSSL 00029 00030 #if defined(WIN32_VC) || defined(WIN64_VC) 00031 #include <WinSock2.h> 00032 #include <windows.h> // for select() 00033 #undef X509_NAME 00034 #endif // WIN32_VC 00035 00036 TypeHandle HTTPChannel::_type_handle; 00037 00038 //////////////////////////////////////////////////////////////////// 00039 // Function: HTTPChannel::Constructor 00040 // Access: Private 00041 // Description: 00042 //////////////////////////////////////////////////////////////////// 00043 HTTPChannel:: 00044 HTTPChannel(HTTPClient *client) : 00045 _client(client) 00046 { 00047 ConfigVariableDouble extra_ssl_handshake_time 00048 ("extra-ssl-handshake-time", 0.0, 00049 PRC_DESC("This specifies how much extra time to try to establish" 00050 "the ssl handshake before we bail.")); 00051 _extra_ssl_handshake_time = extra_ssl_handshake_time; 00052 _proxy_next_index = 0; 00053 _persistent_connection = false; 00054 _allow_proxy = true; 00055 _proxy_tunnel = http_proxy_tunnel; 00056 _connect_timeout = http_connect_timeout; 00057 _http_timeout = http_timeout; 00058 _skip_body_size = http_skip_body_size; 00059 _idle_timeout = http_idle_timeout; 00060 _blocking_connect = false; 00061 _download_throttle = download_throttle; 00062 _max_bytes_per_second = downloader_byte_rate; 00063 _seconds_per_update = downloader_frequency; 00064 _max_updates_per_second = 1.0f / _seconds_per_update; 00065 _bytes_per_update = int(_max_bytes_per_second * _seconds_per_update); 00066 00067 // _nonblocking is true if the socket is actually in non-blocking mode. 00068 _nonblocking = false; 00069 00070 // _wanted_nonblocking is true if the user specifically requested 00071 // one of the non-blocking interfaces. It is false if the socket is 00072 // only incidentally non-blocking (for instance, because 00073 // SIMPLE_THREADS is on). 00074 _wanted_nonblocking = false; 00075 00076 _want_ssl = false; 00077 _proxy_serves_document = false; 00078 _proxy_tunnel_now = false; 00079 _first_byte_requested = 0; 00080 _last_byte_requested = 0; 00081 _first_byte_delivered = 0; 00082 _last_byte_delivered = 0; 00083 _read_index = 0; 00084 _expected_file_size = 0; 00085 _file_size = 0; 00086 _transfer_file_size = 0; 00087 _got_expected_file_size = false; 00088 _got_file_size = false; 00089 _got_transfer_file_size = false; 00090 _bytes_downloaded = 0; 00091 _bytes_requested = 0; 00092 _status_entry = StatusEntry(); 00093 _response_type = RT_none; 00094 _http_version = _client->get_http_version(); 00095 _http_version_string = _client->get_http_version_string(); 00096 _state = S_new; 00097 _done_state = S_new; 00098 _started_download = false; 00099 _sent_so_far = 0; 00100 _body_stream = NULL; 00101 _owns_body_stream = false; 00102 _sbio = NULL; 00103 _cipher_list = _client->get_cipher_list(); 00104 _last_status_code = 0; 00105 _last_run_time = 0.0f; 00106 _download_to_ramfile = NULL; 00107 } 00108 00109 //////////////////////////////////////////////////////////////////// 00110 // Function: HTTPChannel::Destructor 00111 // Access: Public, Virtual 00112 // Description: 00113 //////////////////////////////////////////////////////////////////// 00114 HTTPChannel:: 00115 ~HTTPChannel() { 00116 close_connection(); 00117 reset_download_to(); 00118 } 00119 00120 //////////////////////////////////////////////////////////////////// 00121 // Function: HTTPChannel::get_status_string 00122 // Access: Published 00123 // Description: Returns the string as returned by the server 00124 // describing the status code for humans. This may or 00125 // may not be meaningful. 00126 //////////////////////////////////////////////////////////////////// 00127 string HTTPChannel:: 00128 get_status_string() const { 00129 switch (_status_entry._status_code) { 00130 case SC_incomplete: 00131 return "Connection in progress"; 00132 00133 case SC_internal_error: 00134 return "Internal error"; 00135 00136 case SC_no_connection: 00137 return "No connection"; 00138 00139 case SC_timeout: 00140 return "Timeout on connection"; 00141 00142 case SC_lost_connection: 00143 return "Lost connection"; 00144 00145 case SC_non_http_response: 00146 return "Non-HTTP response"; 00147 00148 case SC_invalid_http: 00149 return "Could not understand HTTP response"; 00150 00151 case SC_socks_invalid_version: 00152 return "Unsupported SOCKS version"; 00153 00154 case SC_socks_no_acceptable_login_method: 00155 return "No acceptable SOCKS login method"; 00156 00157 case SC_socks_refused: 00158 return "SOCKS proxy refused connection"; 00159 00160 case SC_socks_no_connection: 00161 return "SOCKS proxy unable to connect"; 00162 00163 case SC_ssl_internal_failure: 00164 return "SSL internal failure"; 00165 00166 case SC_ssl_no_handshake: 00167 return "No SSL handshake"; 00168 00169 case SC_http_error_watermark: 00170 // This shouldn't be triggered. 00171 return "Internal error"; 00172 00173 case SC_ssl_invalid_server_certificate: 00174 return "SSL invalid server certificate"; 00175 00176 case SC_ssl_unexpected_server: 00177 return "Unexpected SSL server"; 00178 00179 case SC_download_open_error: 00180 return "Error opening file"; 00181 00182 case SC_download_write_error: 00183 return "Error writing to disk"; 00184 00185 case SC_download_invalid_range: 00186 return "Invalid subrange requested"; 00187 } 00188 00189 return _status_entry._status_string; 00190 } 00191 00192 //////////////////////////////////////////////////////////////////// 00193 // Function: HTTPChannel::get_header_value 00194 // Access: Published 00195 // Description: Returns the HTML header value associated with the 00196 // indicated key, or empty string if the key was not 00197 // defined in the message returned by the server. 00198 //////////////////////////////////////////////////////////////////// 00199 string HTTPChannel:: 00200 get_header_value(const string &key) const { 00201 Headers::const_iterator hi = _headers.find(downcase(key)); 00202 if (hi != _headers.end()) { 00203 return (*hi).second; 00204 } 00205 return string(); 00206 } 00207 00208 //////////////////////////////////////////////////////////////////// 00209 // Function: HTTPChannel::will_close_connection 00210 // Access: Published 00211 // Description: Returns true if the server has indicated it will 00212 // close the connection after this document has been 00213 // read, or false if it will remain open (and future 00214 // documents may be requested on the same connection). 00215 //////////////////////////////////////////////////////////////////// 00216 bool HTTPChannel:: 00217 will_close_connection() const { 00218 if (get_http_version() < HTTPEnum::HV_11) { 00219 // pre-HTTP 1.1 always closes. 00220 return true; 00221 } 00222 00223 string connection = get_header_value("Connection"); 00224 if (downcase(connection) == "close") { 00225 // The server says it will close. 00226 return true; 00227 } 00228 00229 if (connection.empty() && !get_persistent_connection()) { 00230 // The server didn't say, but we asked it to close. 00231 return true; 00232 } 00233 00234 // Assume the server will keep it open. 00235 return false; 00236 } 00237 00238 //////////////////////////////////////////////////////////////////// 00239 // Function: HTTPChannel::get_file_size 00240 // Access: Published, Virtual 00241 // Description: Returns the size of the file, if it is known. 00242 // Returns the value set by set_expected_file_size() if 00243 // the file size is not known, or 0 if this value was 00244 // not set. 00245 // 00246 // If the file is dynamically generated, the size may 00247 // not be available until a read has started 00248 // (e.g. open_read_body() has been called); and even 00249 // then it may increase as more of the file is read due 00250 // to the nature of HTTP/1.1 requests which can change 00251 // their minds midstream about how much data they're 00252 // sending you. 00253 //////////////////////////////////////////////////////////////////// 00254 off_t HTTPChannel:: 00255 get_file_size() const { 00256 if (_got_file_size) { 00257 return _file_size; 00258 } else if (_got_transfer_file_size) { 00259 return _transfer_file_size; 00260 } else if (_got_expected_file_size) { 00261 return _expected_file_size; 00262 } else { 00263 return 0; 00264 } 00265 } 00266 00267 //////////////////////////////////////////////////////////////////// 00268 // Function: HTTPChannel::write_headers 00269 // Access: Published 00270 // Description: Outputs a list of all headers defined by the server 00271 // to the indicated output stream. 00272 //////////////////////////////////////////////////////////////////// 00273 void HTTPChannel:: 00274 write_headers(ostream &out) const { 00275 Headers::const_iterator hi; 00276 for (hi = _headers.begin(); hi != _headers.end(); ++hi) { 00277 out << (*hi).first << ": " << (*hi).second << "\n"; 00278 } 00279 } 00280 00281 //////////////////////////////////////////////////////////////////// 00282 // Function: HTTPChannel::run 00283 // Access: Published 00284 // Description: This must be called from time to time when 00285 // non-blocking I/O is in use. It checks for data 00286 // coming in on the socket and writes data out to the 00287 // socket when possible, and does whatever processing is 00288 // required towards completing the current task. 00289 // 00290 // The return value is true if the task is still pending 00291 // (and run() will need to be called again in the 00292 // future), or false if the current task is complete. 00293 //////////////////////////////////////////////////////////////////// 00294 bool HTTPChannel:: 00295 run() { 00296 if (_state == _done_state || _state == S_failure) { 00297 clear_extra_headers(); 00298 if (!reached_done_state()) { 00299 return false; 00300 } 00301 } 00302 00303 if (_started_download) { 00304 if (_wanted_nonblocking && _download_throttle) { 00305 double now = TrueClock::get_global_ptr()->get_short_time(); 00306 double elapsed = now - _last_run_time; 00307 if (elapsed < _seconds_per_update) { 00308 // Come back later. 00309 thread_yield(); 00310 return true; 00311 } 00312 int num_potential_updates = (int)(elapsed / _seconds_per_update); 00313 _last_run_time = now; 00314 _bytes_requested += _bytes_per_update * num_potential_updates; 00315 if (downloader_cat.is_spam()) { 00316 downloader_cat.spam() 00317 << "elapsed = " << elapsed << " num_potential_updates = " 00318 << num_potential_updates << " bytes_requested = " 00319 << _bytes_requested << "\n"; 00320 } 00321 } 00322 00323 bool repeat_later = false; 00324 switch (_download_dest) { 00325 case DD_none: 00326 // We're done. 00327 break; 00328 00329 case DD_file: 00330 repeat_later = run_download_to_file(); 00331 break; 00332 00333 case DD_ram: 00334 repeat_later = run_download_to_ram(); 00335 break; 00336 00337 case DD_stream: 00338 repeat_later = run_download_to_stream(); 00339 break; 00340 } 00341 if (repeat_later) { 00342 thread_yield(); 00343 } 00344 return repeat_later; 00345 } 00346 00347 /* 00348 if (downloader_cat.is_spam()) { 00349 downloader_cat.spam() 00350 << "begin run(), _state = " << _state << ", _done_state = " 00351 << _done_state << "\n"; 00352 } 00353 */ 00354 00355 if (_state == _done_state) { 00356 return reached_done_state(); 00357 } 00358 00359 bool repeat_later; 00360 do { 00361 // If we're in a state that expects to have a connection already 00362 // (that is, any state other that S_try_next_proxy), then 00363 // reestablish the connection if it has been dropped. 00364 if (_bio.is_null() && _state != S_try_next_proxy) { 00365 if (_connect_count > http_max_connect_count) { 00366 // Too many connection attempts; just give up. We should 00367 // never trigger this failsafe, since the code in each 00368 // individual case has similar logic to prevent more than two 00369 // consecutive lost connections. 00370 downloader_cat.warning() 00371 << "Too many lost connections, giving up.\n"; 00372 _status_entry._status_code = SC_lost_connection; 00373 _state = S_failure; 00374 return false; 00375 } 00376 00377 // No connection. Attempt to establish one. 00378 if (_proxy.empty()) { 00379 _bio = new BioPtr(_request.get_url()); 00380 } else { 00381 _bio = new BioPtr(_proxy); 00382 } 00383 _source = new BioStreamPtr(new BioStream(_bio)); 00384 if (_nonblocking) { 00385 BIO_set_nbio(*_bio, 1); 00386 } 00387 00388 if (downloader_cat.is_debug()) { 00389 if (_connect_count > 0) { 00390 downloader_cat.debug() 00391 << "Reconnecting to " << _bio->get_server_name() << ":" 00392 << _bio->get_port() << "\n"; 00393 } else { 00394 downloader_cat.debug() 00395 << "Connecting to " << _bio->get_server_name() << ":" 00396 << _bio->get_port() << "\n"; 00397 } 00398 } 00399 00400 _state = S_connecting; 00401 _started_connecting_time = 00402 TrueClock::get_global_ptr()->get_short_time(); 00403 _connect_count++; 00404 } 00405 00406 /* 00407 if (downloader_cat.is_spam()) { 00408 downloader_cat.spam() 00409 << "continue run(), _state = " << _state << "\n"; 00410 } 00411 */ 00412 00413 switch (_state) { 00414 case S_try_next_proxy: 00415 repeat_later = run_try_next_proxy(); 00416 break; 00417 00418 case S_connecting: 00419 repeat_later = run_connecting(); 00420 break; 00421 00422 case S_connecting_wait: 00423 repeat_later = run_connecting_wait(); 00424 break; 00425 00426 case S_http_proxy_ready: 00427 repeat_later = run_http_proxy_ready(); 00428 break; 00429 00430 case S_http_proxy_request_sent: 00431 repeat_later = run_http_proxy_request_sent(); 00432 break; 00433 00434 case S_http_proxy_reading_header: 00435 repeat_later = run_http_proxy_reading_header(); 00436 break; 00437 00438 case S_socks_proxy_greet: 00439 repeat_later = run_socks_proxy_greet(); 00440 break; 00441 00442 case S_socks_proxy_greet_reply: 00443 repeat_later = run_socks_proxy_greet_reply(); 00444 break; 00445 00446 case S_socks_proxy_connect: 00447 repeat_later = run_socks_proxy_connect(); 00448 break; 00449 00450 case S_socks_proxy_connect_reply: 00451 repeat_later = run_socks_proxy_connect_reply(); 00452 break; 00453 00454 case S_setup_ssl: 00455 repeat_later = run_setup_ssl(); 00456 break; 00457 00458 case S_ssl_handshake: 00459 repeat_later = run_ssl_handshake(); 00460 break; 00461 00462 case S_ready: 00463 repeat_later = run_ready(); 00464 break; 00465 00466 case S_request_sent: 00467 repeat_later = run_request_sent(); 00468 break; 00469 00470 case S_reading_header: 00471 repeat_later = run_reading_header(); 00472 break; 00473 00474 case S_start_direct_file_read: 00475 repeat_later = run_start_direct_file_read(); 00476 break; 00477 00478 case S_read_header: 00479 repeat_later = run_read_header(); 00480 break; 00481 00482 case S_begin_body: 00483 repeat_later = run_begin_body(); 00484 break; 00485 00486 case S_reading_body: 00487 repeat_later = run_reading_body(); 00488 break; 00489 00490 case S_read_body: 00491 repeat_later = run_read_body(); 00492 break; 00493 00494 case S_read_trailer: 00495 repeat_later = run_read_trailer(); 00496 break; 00497 00498 default: 00499 downloader_cat.warning() 00500 << "Unhandled state " << _state << "\n"; 00501 return false; 00502 } 00503 00504 if (_state == _done_state || _state == S_failure) { 00505 clear_extra_headers(); 00506 // We've reached our terminal state. 00507 return reached_done_state(); 00508 } 00509 thread_consider_yield(); 00510 } while (!repeat_later || _bio.is_null()); 00511 00512 /* 00513 if (downloader_cat.is_spam()) { 00514 downloader_cat.spam() 00515 << "later run(), _state = " << _state 00516 << ", _done_state = " << _done_state << "\n"; 00517 } 00518 */ 00519 00520 thread_yield(); 00521 return true; 00522 } 00523 00524 //////////////////////////////////////////////////////////////////// 00525 // Function: HTTPChannel::open_read_body 00526 // Access: Published 00527 // Description: Returns a newly-allocated istream suitable for 00528 // reading the body of the document. This may only be 00529 // called immediately after a call to get_document() or 00530 // post_form(), or after a call to run() has returned 00531 // false. 00532 // 00533 // Note that, in nonblocking mode, the returned stream 00534 // may report an early EOF, even before the actual end 00535 // of file. When this happens, you should call 00536 // stream->is_closed() to determine whether you should 00537 // attempt to read some more later. 00538 // 00539 // The user is responsible for passing the returned 00540 // istream to close_read_body() later. 00541 //////////////////////////////////////////////////////////////////// 00542 ISocketStream *HTTPChannel:: 00543 open_read_body() { 00544 reset_body_stream(); 00545 00546 if ((_state != S_read_header && _state != S_begin_body) || _source.is_null()) { 00547 return NULL; 00548 } 00549 00550 string transfer_coding = downcase(get_header_value("Transfer-Encoding")); 00551 00552 ISocketStream *result; 00553 if (transfer_coding == "chunked") { 00554 // "chunked" transfer encoding. This means we will have to decode 00555 // the length of the file as we read it in chunks. The 00556 // IChunkedStream does this. 00557 _state = S_reading_body; 00558 _read_index++; 00559 result = new IChunkedStream(_source, this); 00560 00561 } else { 00562 // If the transfer encoding is anything else, assume "identity". 00563 // This is just the literal characters following the header, up 00564 // until _file_size bytes have been read (if content-length was 00565 // specified), or till end of file otherwise. 00566 _state = S_reading_body; 00567 _read_index++; 00568 result = new IIdentityStream(_source, this, _got_file_size, _file_size); 00569 } 00570 00571 result->_channel = this; 00572 _body_stream = result; 00573 _owns_body_stream = false; 00574 00575 return result; 00576 } 00577 00578 //////////////////////////////////////////////////////////////////// 00579 // Function: HTTPChannel::close_read_body 00580 // Access: Public 00581 // Description: Closes a file opened by a previous call to 00582 // open_read_body(). This really just deletes the 00583 // istream pointer, but it is recommended to use this 00584 // interface instead of deleting it explicitly, to help 00585 // work around compiler issues. 00586 //////////////////////////////////////////////////////////////////// 00587 void HTTPChannel:: 00588 close_read_body(istream *stream) const { 00589 if (stream != (istream *)NULL) { 00590 // For some reason--compiler bug in gcc 3.2?--explicitly deleting 00591 // the stream pointer does not call the appropriate global delete 00592 // function; instead apparently calling the system delete 00593 // function. So we call the delete function by hand instead. 00594 #if !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW) 00595 stream->~istream(); 00596 (*global_operator_delete)(stream); 00597 #else 00598 delete stream; 00599 #endif 00600 } 00601 } 00602 00603 //////////////////////////////////////////////////////////////////// 00604 // Function: HTTPChannel::download_to_file 00605 // Access: Published 00606 // Description: Specifies the name of a file to download the 00607 // resulting document to. This should be called 00608 // immediately after get_document() or 00609 // begin_get_document() or related functions. 00610 // 00611 // In the case of the blocking I/O methods like 00612 // get_document(), this function will download the 00613 // entire document to the file and return true if it was 00614 // successfully downloaded, false otherwise. 00615 // 00616 // In the case of non-blocking I/O methods like 00617 // begin_get_document(), this function simply indicates an 00618 // intention to download to the indicated file. It 00619 // returns true if the file can be opened for writing, 00620 // false otherwise, but the contents will not be 00621 // completely downloaded until run() has returned false. 00622 // At this time, it is possible that a communications 00623 // error will have left a partial file, so 00624 // is_download_complete() may be called to test this. 00625 // 00626 // If subdocument_resumes is true and the document in 00627 // question was previously requested as a subdocument 00628 // (i.e. get_subdocument() with a first_byte value 00629 // greater than zero), this will automatically seek to 00630 // the appropriate byte within the file for writing the 00631 // output. In this case, the file must already exist 00632 // and must have at least first_byte bytes in it. If 00633 // subdocument_resumes is false, a subdocument will 00634 // always be downloaded beginning at the first byte of 00635 // the file. 00636 //////////////////////////////////////////////////////////////////// 00637 bool HTTPChannel:: 00638 download_to_file(const Filename &filename, bool subdocument_resumes) { 00639 reset_download_to(); 00640 _download_to_filename = filename; 00641 _download_to_filename.set_binary(); 00642 _download_to_file.close(); 00643 _download_to_file.clear(); 00644 _subdocument_resumes = subdocument_resumes; 00645 00646 _download_dest = DD_file; 00647 00648 if (_wanted_nonblocking && _state != S_read_header) { 00649 // In nonblocking mode, we can't start the download yet; that will 00650 // be done later as run() is called. 00651 return true; 00652 } 00653 00654 // In normal, blocking mode, go ahead and do the download. 00655 if (!open_download_file()) { 00656 reset_download_to(); 00657 return false; 00658 } 00659 00660 while (run()) { 00661 } 00662 return is_download_complete() && is_valid(); 00663 } 00664 00665 //////////////////////////////////////////////////////////////////// 00666 // Function: HTTPChannel::download_to_ram 00667 // Access: Published 00668 // Description: Specifies a Ramfile object to download the 00669 // resulting document to. This should be called 00670 // immediately after get_document() or 00671 // begin_get_document() or related functions. 00672 // 00673 // In the case of the blocking I/O methods like 00674 // get_document(), this function will download the 00675 // entire document to the Ramfile and return true if it 00676 // was successfully downloaded, false otherwise. 00677 // 00678 // In the case of non-blocking I/O methods like 00679 // begin_get_document(), this function simply indicates an 00680 // intention to download to the indicated Ramfile. It 00681 // returns true if the file can be opened for writing, 00682 // false otherwise, but the contents will not be 00683 // completely downloaded until run() has returned false. 00684 // At this time, it is possible that a communications 00685 // error will have left a partial file, so 00686 // is_download_complete() may be called to test this. 00687 // 00688 // If subdocument_resumes is true and the document in 00689 // question was previously requested as a subdocument 00690 // (i.e. get_subdocument() with a first_byte value 00691 // greater than zero), this will automatically seek to 00692 // the appropriate byte within the Ramfile for writing 00693 // the output. In this case, the Ramfile must already 00694 // have at least first_byte bytes in it. 00695 //////////////////////////////////////////////////////////////////// 00696 bool HTTPChannel:: 00697 download_to_ram(Ramfile *ramfile, bool subdocument_resumes) { 00698 nassertr(ramfile != (Ramfile *)NULL, false); 00699 reset_download_to(); 00700 ramfile->_pos = 0; 00701 _download_to_ramfile = ramfile; 00702 _download_dest = DD_ram; 00703 _subdocument_resumes = (subdocument_resumes && _first_byte_delivered != 0); 00704 00705 if (_wanted_nonblocking && _state != S_read_header) { 00706 // In nonblocking mode, we can't start the download yet; that will 00707 // be done later as run() is called. 00708 return true; 00709 } 00710 00711 // In normal, blocking mode, go ahead and do the download. 00712 if (!open_download_file()) { 00713 reset_download_to(); 00714 return false; 00715 } 00716 00717 while (run()) { 00718 } 00719 return is_download_complete() && is_valid(); 00720 } 00721 00722 //////////////////////////////////////////////////////////////////// 00723 // Function: HTTPChannel::download_to_stream 00724 // Access: Published 00725 // Description: Specifies the name of an ostream to download the 00726 // resulting document to. This should be called 00727 // immediately after get_document() or 00728 // begin_get_document() or related functions. 00729 // 00730 // In the case of the blocking I/O methods like 00731 // get_document(), this function will download the 00732 // entire document to the file and return true if it was 00733 // successfully downloaded, false otherwise. 00734 // 00735 // In the case of non-blocking I/O methods like 00736 // begin_get_document(), this function simply indicates an 00737 // intention to download to the indicated file. It 00738 // returns true if the file can be opened for writing, 00739 // false otherwise, but the contents will not be 00740 // completely downloaded until run() has returned false. 00741 // At this time, it is possible that a communications 00742 // error will have left a partial file, so 00743 // is_download_complete() may be called to test this. 00744 // 00745 // If subdocument_resumes is true and the document in 00746 // question was previously requested as a subdocument 00747 // (i.e. get_subdocument() with a first_byte value 00748 // greater than zero), this will automatically seek to 00749 // the appropriate byte within the file for writing the 00750 // output. In this case, the file must already exist 00751 // and must have at least first_byte bytes in it. If 00752 // subdocument_resumes is false, a subdocument will 00753 // always be downloaded beginning at the first byte of 00754 // the file. 00755 //////////////////////////////////////////////////////////////////// 00756 bool HTTPChannel:: 00757 download_to_stream(ostream *strm, bool subdocument_resumes) { 00758 reset_download_to(); 00759 _download_to_stream = strm; 00760 _download_to_stream->clear(); 00761 _subdocument_resumes = subdocument_resumes; 00762 00763 _download_dest = DD_stream; 00764 00765 if (_wanted_nonblocking && _state != S_read_header) { 00766 // In nonblocking mode, we can't start the download yet; that will 00767 // be done later as run() is called. 00768 return true; 00769 } 00770 00771 // In normal, blocking mode, go ahead and do the download. 00772 if (!open_download_file()) { 00773 reset_download_to(); 00774 return false; 00775 } 00776 00777 while (run()) { 00778 } 00779 return is_download_complete() && is_valid(); 00780 } 00781 00782 //////////////////////////////////////////////////////////////////// 00783 // Function: HTTPChannel::get_connection 00784 // Access: Published 00785 // Description: Returns the connection that was established via a 00786 // previous call to connect_to() or begin_connect_to(), 00787 // or NULL if the connection attempt failed or if those 00788 // methods have not recently been called. 00789 // 00790 // This stream has been allocated from the free store. 00791 // It is the user's responsibility to delete this 00792 // pointer when finished with it. 00793 //////////////////////////////////////////////////////////////////// 00794 SocketStream *HTTPChannel:: 00795 get_connection() { 00796 if (!is_connection_ready()) { 00797 return NULL; 00798 } 00799 00800 BioStream *stream = _source->get_stream(); 00801 _source->set_stream(NULL); 00802 00803 // We're now passing ownership of the connection to the caller. 00804 if (downloader_cat.is_debug()) { 00805 downloader_cat.debug() 00806 << "passing ownership of connection to caller.\n"; 00807 } 00808 reset_to_new(); 00809 00810 return stream; 00811 } 00812 00813 //////////////////////////////////////////////////////////////////// 00814 // Function: HTTPChannel::downcase 00815 // Access: Public, Static 00816 // Description: Returns the input string with all uppercase letters 00817 // converted to lowercase. 00818 //////////////////////////////////////////////////////////////////// 00819 string HTTPChannel:: 00820 downcase(const string &s) { 00821 string result; 00822 result.reserve(s.size()); 00823 string::const_iterator p; 00824 for (p = s.begin(); p != s.end(); ++p) { 00825 result += tolower(*p); 00826 } 00827 return result; 00828 } 00829 00830 //////////////////////////////////////////////////////////////////// 00831 // Function: HTTPChannel::body_stream_destructs 00832 // Access: Public 00833 // Description: Called by ISocketStream destructor when _body_stream 00834 // is destructing. 00835 //////////////////////////////////////////////////////////////////// 00836 void HTTPChannel:: 00837 body_stream_destructs(ISocketStream *stream) { 00838 if (stream == _body_stream) { 00839 if (_state == S_reading_body) { 00840 switch (_body_stream->get_read_state()) { 00841 case ISocketStream::RS_complete: 00842 finished_body(false); 00843 break; 00844 00845 case ISocketStream::RS_error: 00846 _state = HTTPChannel::S_failure; 00847 _status_entry._status_code = HTTPChannel::SC_lost_connection; 00848 break; 00849 00850 default: 00851 break; 00852 } 00853 } 00854 _body_stream = NULL; 00855 _owns_body_stream = false; 00856 } 00857 } 00858 00859 00860 //////////////////////////////////////////////////////////////////// 00861 // Function: HTTPChannel::reached_done_state 00862 // Access: Private 00863 // Description: Called by run() after it reaches the done state, this 00864 // simply checks to see if a download was requested, and 00865 // begins the download if it has been. 00866 //////////////////////////////////////////////////////////////////// 00867 bool HTTPChannel:: 00868 reached_done_state() { 00869 /* 00870 if (downloader_cat.is_spam()) { 00871 downloader_cat.spam() 00872 << "terminating run(), _state = " << _state 00873 << ", _done_state = " << _done_state << "\n"; 00874 } 00875 */ 00876 00877 if (_state == S_failure) { 00878 // We had to give up. Each proxy we tried, in sequence, failed. 00879 // But maybe the last attempt didn't give us the most informative 00880 // response; go back and find the best one. 00881 if (!_status_list.empty()) { 00882 _status_list.push_back(_status_entry); 00883 if (downloader_cat.is_debug()) { 00884 downloader_cat.debug() 00885 << "Reexamining failure responses.\n"; 00886 } 00887 size_t best_i = 0; 00888 if (downloader_cat.is_debug()) { 00889 downloader_cat.debug() 00890 << " " << 0 << ". " << _status_list[0]._status_code << " " 00891 << _status_list[0]._status_string << "\n"; 00892 } 00893 for (size_t i = 1; i < _status_list.size(); i++) { 00894 if (downloader_cat.is_debug()) { 00895 downloader_cat.debug() 00896 << " " << i << ". " << _status_list[i]._status_code << " " 00897 << _status_list[i]._status_string << "\n"; 00898 } 00899 if (more_useful_status_code(_status_list[i]._status_code, 00900 _status_list[best_i]._status_code)) { 00901 best_i = i; 00902 } 00903 } 00904 if (downloader_cat.is_debug()) { 00905 downloader_cat.debug() 00906 << "chose index " << best_i << ", above.\n"; 00907 } 00908 _status_entry = _status_list[best_i]; 00909 _status_list.clear(); 00910 } 00911 00912 return false; 00913 } 00914 00915 // We don't need the list of previous failures any more--we've connected. 00916 _status_list.clear(); 00917 00918 if (_download_dest == DD_none) { 00919 // All done. 00920 return false; 00921 00922 } else { 00923 // Oops, we have to download the body now. 00924 open_read_body(); 00925 if (_body_stream == (ISocketStream *)NULL) { 00926 if (downloader_cat.is_debug()) { 00927 downloader_cat.debug() 00928 << "Unable to download body: " << _request.get_url() << "\n"; 00929 } 00930 return false; 00931 00932 } else { 00933 _owns_body_stream = true; 00934 if (_state != S_reading_body) { 00935 reset_body_stream(); 00936 } 00937 _started_download = true; 00938 00939 _done_state = S_read_trailer; 00940 _last_run_time = TrueClock::get_global_ptr()->get_short_time(); 00941 return true; 00942 } 00943 } 00944 } 00945 00946 //////////////////////////////////////////////////////////////////// 00947 // Function: HTTPChannel::run_try_next_proxy 00948 // Access: Private 00949 // Description: This state is reached when a previous connection 00950 // attempt fails. If we have multiple proxies in line 00951 // to try, it sets us up for the next proxy and tries to 00952 // connect again; otherwise, it sets the state to 00953 // S_failure. 00954 //////////////////////////////////////////////////////////////////// 00955 bool HTTPChannel:: 00956 run_try_next_proxy() { 00957 if (_proxy_next_index < _proxies.size()) { 00958 // Record the previous proxy's status entry, so we can come back 00959 // to it later if we get nonsense from the remaining proxies. 00960 _status_list.push_back(_status_entry); 00961 _status_entry = StatusEntry(); 00962 00963 // Now try the next proxy in sequence. 00964 _proxy = _proxies[_proxy_next_index]; 00965 _proxy_auth = (HTTPAuthorization *)NULL; 00966 _proxy_next_index++; 00967 close_connection(); 00968 reconsider_proxy(); 00969 _state = S_connecting; 00970 00971 return false; 00972 } 00973 00974 // No more proxies to try, or we're not using a proxy. 00975 _state = S_failure; 00976 return false; 00977 } 00978 00979 //////////////////////////////////////////////////////////////////// 00980 // Function: HTTPChannel::run_connecting 00981 // Access: Private 00982 // Description: In this state, we have not yet established a 00983 // network connection to the server (or proxy). 00984 //////////////////////////////////////////////////////////////////// 00985 bool HTTPChannel:: 00986 run_connecting() { 00987 _status_entry = StatusEntry(); 00988 00989 if (BIO_do_connect(*_bio) <= 0) { 00990 if (BIO_should_retry(*_bio)) { 00991 _state = S_connecting_wait; 00992 return false; 00993 } 00994 downloader_cat.info() 00995 << "Could not connect to " << _bio->get_server_name() << ":" 00996 << _bio->get_port() << "\n"; 00997 OpenSSLWrapper::get_global_ptr()->notify_ssl_errors(); 00998 _status_entry._status_code = SC_no_connection; 00999 _state = S_try_next_proxy; 01000 return false; 01001 } 01002 01003 if (downloader_cat.is_debug()) { 01004 downloader_cat.debug() 01005 << "Connected to " << _bio->get_server_name() << ":" 01006 << _bio->get_port() << "\n"; 01007 } 01008 01009 if (_proxy_tunnel_now) { 01010 if (_proxy.get_scheme() == "socks") { 01011 _state = S_socks_proxy_greet; 01012 } else { 01013 _state = S_http_proxy_ready; 01014 } 01015 01016 } else { 01017 if (_want_ssl) { 01018 _state = S_setup_ssl; 01019 } else { 01020 _state = S_ready; 01021 } 01022 } 01023 return false; 01024 } 01025 01026 //////////////////////////////////////////////////////////////////// 01027 // Function: HTTPChannel::run_connecting_wait 01028 // Access: Private 01029 // Description: Here we have begun to establish a nonblocking 01030 // connection, but we got a come-back-later message, so 01031 // we are waiting for the socket to finish connecting. 01032 //////////////////////////////////////////////////////////////////// 01033 bool HTTPChannel:: 01034 run_connecting_wait() { 01035 int fd = -1; 01036 BIO_get_fd(*_bio, &fd); 01037 if (fd < 0) { 01038 downloader_cat.warning() 01039 << "nonblocking socket BIO has no file descriptor.\n"; 01040 // This shouldn't be possible. 01041 _status_entry._status_code = SC_internal_error; 01042 _state = S_try_next_proxy; 01043 return false; 01044 } 01045 01046 if (downloader_cat.is_spam()) { 01047 downloader_cat.spam() 01048 << "waiting to connect to " << _request.get_url().get_server_and_port() << ".\n"; 01049 } 01050 fd_set wset; 01051 FD_ZERO(&wset); 01052 FD_SET(fd, &wset); 01053 struct timeval tv; 01054 if (get_blocking_connect()) { 01055 // Since we'll be blocking on this connect, fill in the timeout 01056 // into the structure. 01057 tv.tv_sec = (int)_connect_timeout; 01058 tv.tv_usec = (int)((_connect_timeout - tv.tv_sec) * 1000000.0); 01059 } else { 01060 // We won't block on this connect, so select() for 0 time. 01061 tv.tv_sec = 0; 01062 tv.tv_usec = 0; 01063 } 01064 int errcode = select(fd + 1, NULL, &wset, NULL, &tv); 01065 if (errcode < 0) { 01066 downloader_cat.warning() 01067 << "Error in select.\n"; 01068 // This shouldn't be possible. 01069 _status_entry._status_code = SC_internal_error; 01070 _state = S_try_next_proxy; 01071 return false; 01072 } 01073 01074 if (errcode == 0) { 01075 // Nothing's happened so far; come back later. 01076 if (get_blocking_connect() || 01077 (TrueClock::get_global_ptr()->get_short_time() - 01078 _started_connecting_time > get_connect_timeout())) { 01079 // Time to give up. 01080 downloader_cat.info() 01081 << "Timeout connecting to " 01082 << _request.get_url().get_server_and_port() 01083 << " for " << _request.get_url() 01084 << ".\n"; 01085 _status_entry._status_code = SC_timeout; 01086 _state = S_try_next_proxy; 01087 return false; 01088 } 01089 return true; 01090 } 01091 01092 // The socket is now ready for writing. 01093 _state = S_connecting; 01094 return false; 01095 } 01096 01097 01098 //////////////////////////////////////////////////////////////////// 01099 // Function: HTTPChannel::run_http_proxy_ready 01100 // Access: Private 01101 // Description: This state is reached only after first establishing a 01102 // connection to the proxy, if a proxy is in use and we 01103 // are tunneling through it via a CONNECT command. 01104 //////////////////////////////////////////////////////////////////// 01105 bool HTTPChannel:: 01106 run_http_proxy_ready() { 01107 // If there's a request to be sent to the proxy, send it now. 01108 nassertr(!_proxy_request_text.empty(), false); 01109 if (!server_send(_proxy_request_text, false)) { 01110 return true; 01111 } 01112 01113 // All done sending request. 01114 _state = S_http_proxy_request_sent; 01115 _sent_request_time = TrueClock::get_global_ptr()->get_short_time(); 01116 return false; 01117 } 01118 01119 //////////////////////////////////////////////////////////////////// 01120 // Function: HTTPChannel::run_http_proxy_request_sent 01121 // Access: Private 01122 // Description: This state is reached only after we have sent a 01123 // special message to the proxy and we are waiting for 01124 // the proxy's response. It is not used in the normal 01125 // http-over-proxy case, which does not require a 01126 // special message to the proxy. 01127 //////////////////////////////////////////////////////////////////// 01128 bool HTTPChannel:: 01129 run_http_proxy_request_sent() { 01130 // Wait for the first line to come back from the server. 01131 string line; 01132 if (!server_getline_failsafe(line)) { 01133 return true; 01134 } 01135 01136 // Skip unexpected blank lines. We're just being generous here. 01137 while (line.empty()) { 01138 if (!server_getline_failsafe(line)) { 01139 return true; 01140 } 01141 } 01142 01143 if (!parse_http_response(line)) { 01144 return false; 01145 } 01146 01147 _state = S_http_proxy_reading_header; 01148 _current_field_name = string(); 01149 _current_field_value = string(); 01150 _headers.clear(); 01151 _got_file_size = false; 01152 _got_transfer_file_size = false; 01153 return false; 01154 } 01155 01156 //////////////////////////////////////////////////////////////////// 01157 // Function: HTTPChannel::run_http_proxy_reading_header 01158 // Access: Private 01159 // Description: In this state we are reading the header lines from 01160 // the proxy's response to our special message. 01161 //////////////////////////////////////////////////////////////////// 01162 bool HTTPChannel:: 01163 run_http_proxy_reading_header() { 01164 if (parse_http_header()) { 01165 return true; 01166 } 01167 01168 _redirect = get_header_value("Location"); 01169 // We can take the proxy's word for it that this is the actual URL 01170 // for the redirect. 01171 01172 _server_response_has_no_body = 01173 (get_status_code() / 100 == 1 || 01174 get_status_code() == 204 || 01175 get_status_code() == 304); 01176 01177 int last_status = _last_status_code; 01178 _last_status_code = get_status_code(); 01179 01180 if (get_status_code() == 407 && last_status != 407 && !_proxy.empty()) { 01181 // 407: not authorized to proxy. Try to get the authorization. 01182 string authenticate_request = get_header_value("Proxy-Authenticate"); 01183 _proxy_auth = _client->generate_auth(_proxy, true, authenticate_request); 01184 if (_proxy_auth != (HTTPAuthorization *)NULL) { 01185 _proxy_realm = _proxy_auth->get_realm(); 01186 _proxy_username = _client->select_username(_proxy, true, _proxy_realm); 01187 if (!_proxy_username.empty()) { 01188 make_proxy_request_text(); 01189 01190 // Roll the state forward to force a new request. 01191 _state = S_begin_body; 01192 return false; 01193 } 01194 } 01195 } 01196 01197 if (!is_valid()) { 01198 // Proxy wouldn't open connection. 01199 01200 // Change some of the status codes a proxy might return to 01201 // differentiate them from similar status codes the destination 01202 // server might have returned. 01203 if (get_status_code() != 407) { 01204 _status_entry._status_code += 1000; 01205 } 01206 01207 _state = S_try_next_proxy; 01208 return false; 01209 } 01210 01211 // Now we have a tunnel opened through the proxy. 01212 make_request_text(); 01213 01214 if (_want_ssl) { 01215 _state = S_setup_ssl; 01216 } else { 01217 _state = S_ready; 01218 } 01219 01220 return false; 01221 } 01222 01223 //////////////////////////////////////////////////////////////////// 01224 // Function: HTTPChannel::run_socks_proxy_greet 01225 // Access: Private 01226 // Description: This state is reached only after first establishing a 01227 // connection to a SOCKS proxy, with which we now have 01228 // to negotiate a connection. 01229 //////////////////////////////////////////////////////////////////// 01230 bool HTTPChannel:: 01231 run_socks_proxy_greet() { 01232 static const char socks_greeting[] = { 01233 0x05, // Socks version 5 01234 0x01, // Number of supported login methods 01235 0x00, // Login method 0: no authentication 01236 /* 01237 0x01, // Login method 1: GSSAPI 01238 0x02 // Login method 2: username/password 01239 */ 01240 }; 01241 static const int socks_greeting_len = sizeof(socks_greeting); 01242 if (!server_send(string(socks_greeting, socks_greeting_len), true)) { 01243 return true; 01244 } 01245 _sent_request_time = TrueClock::get_global_ptr()->get_short_time(); 01246 01247 // All done sending request. 01248 _state = S_socks_proxy_greet_reply; 01249 return false; 01250 } 01251 01252 //////////////////////////////////////////////////////////////////// 01253 // Function: HTTPChannel::run_socks_proxy_greet_reply 01254 // Access: Private 01255 // Description: We are waiting for the SOCKS proxy to respond to our 01256 // greeting. 01257 //////////////////////////////////////////////////////////////////// 01258 bool HTTPChannel:: 01259 run_socks_proxy_greet_reply() { 01260 string reply; 01261 01262 // Get the two-byte reply from the SOCKS server. 01263 if (!server_get_failsafe(reply, 2)) { 01264 return true; 01265 } 01266 01267 if (reply[0] != 0x05) { 01268 // We only speak Socks5. 01269 downloader_cat.info() 01270 << "Rejecting Socks version " << (int)reply[0] << "\n"; 01271 _status_entry._status_code = SC_socks_invalid_version; 01272 _state = S_try_next_proxy; 01273 return false; 01274 } 01275 01276 if (reply[1] == (char)0xff) { 01277 downloader_cat.info() 01278 << "Socks server does not accept our available login methods.\n"; 01279 _status_entry._status_code = SC_socks_no_acceptable_login_method; 01280 _state = S_try_next_proxy; 01281 return false; 01282 } 01283 01284 if (reply[1] == 0x00) { 01285 // No login method required. Proceed directly to the connect 01286 // message. 01287 _state = S_socks_proxy_connect; 01288 return false; 01289 } 01290 01291 // The server accepted a login method we didn't offer! 01292 downloader_cat.info() 01293 << "Socks server accepted unrequested login method " 01294 << (int)reply[1] << "\n"; 01295 _status_entry._status_code = SC_socks_no_acceptable_login_method; 01296 _state = S_try_next_proxy; 01297 return false; 01298 } 01299 01300 //////////////////////////////////////////////////////////////////// 01301 // Function: HTTPChannel::run_socks_proxy_connect 01302 // Access: Private 01303 // Description: The SOCKS proxy has accepted us, and now we may issue 01304 // the connect request. 01305 //////////////////////////////////////////////////////////////////// 01306 bool HTTPChannel:: 01307 run_socks_proxy_connect() { 01308 static const char socks_connect[] = { 01309 0x05, // Socks version 5 01310 0x01, // Command 1: connect 01311 0x00, // reserved 01312 0x03, // DNS name 01313 }; 01314 static const int socks_connect_len = sizeof(socks_connect); 01315 01316 string hostname = _request.get_url().get_server(); 01317 int port = _request.get_url().get_port(); 01318 01319 if (downloader_cat.is_debug()) { 01320 downloader_cat.debug() 01321 << "Requesting SOCKS5 connection to " 01322 << _request.get_url().get_server_and_port() << "\n"; 01323 } 01324 01325 string connect = 01326 string(socks_connect, socks_connect_len) + 01327 string(1, (char)hostname.length()) + 01328 hostname + 01329 string(1, (char)((port >> 8) & 0xff)) + 01330 string(1, (char)(port & 0xff)); 01331 01332 if (!server_send(connect, true)) { 01333 return true; 01334 } 01335 _sent_request_time = TrueClock::get_global_ptr()->get_short_time(); 01336 01337 _state = S_socks_proxy_connect_reply; 01338 return false; 01339 } 01340 01341 //////////////////////////////////////////////////////////////////// 01342 // Function: HTTPChannel::run_socks_proxy_connect_reply 01343 // Access: Private 01344 // Description: We are waiting for the SOCKS proxy to honor our 01345 // connect request. 01346 //////////////////////////////////////////////////////////////////// 01347 bool HTTPChannel:: 01348 run_socks_proxy_connect_reply() { 01349 string reply; 01350 01351 // Get the first two bytes of the connect reply. 01352 if (!server_get_failsafe(reply, 2)) { 01353 return true; 01354 } 01355 01356 if (reply[0] != 0x05) { 01357 // We only speak Socks5. 01358 downloader_cat.info() 01359 << "Rejecting Socks version " << (int)reply[0] << "\n"; 01360 close_connection(); // connection is now bad. 01361 _status_entry._status_code = SC_socks_invalid_version; 01362 _state = S_try_next_proxy; 01363 return false; 01364 } 01365 01366 if (reply[1] != 0x00) { 01367 downloader_cat.info() 01368 << "Connection refused, SOCKS code " << (int)reply[1] << "\n"; 01369 /* 01370 Socks error codes (from RFC1928): 01371 o X'00' succeeded 01372 o X'01' general SOCKS server failure 01373 o X'02' connection not allowed by ruleset 01374 o X'03' Network unreachable 01375 o X'04' Host unreachable 01376 o X'05' Connection refused 01377 o X'06' TTL expired 01378 o X'07' Command not supported 01379 o X'08' Address type not supported 01380 o X'09' to X'FF' unassigned 01381 */ 01382 01383 switch (reply[1]) { 01384 case 0x03: 01385 case 0x04: 01386 case 0x05: 01387 case 0x06: 01388 // These generally mean the same thing: the SOCKS proxy tried, 01389 // but couldn't reach the host. 01390 _status_entry._status_code = SC_socks_no_connection; 01391 break; 01392 01393 default: 01394 _status_entry._status_code = SC_socks_refused; 01395 } 01396 01397 close_connection(); // connection is now bad. 01398 _state = S_try_next_proxy; 01399 return false; 01400 } 01401 01402 // Now put those bytes back, and get five bytes of the reply. 01403 _working_get = reply; 01404 if (!server_get_failsafe(reply, 5)) { 01405 return true; 01406 } 01407 01408 // Figure out how many bytes total we will expect for the reply. 01409 int total_bytes = 6; 01410 01411 switch (reply[3]) { 01412 case 0x01: // IPv4 01413 total_bytes += 4; 01414 break; 01415 01416 case 0x03: // DNS 01417 total_bytes += (unsigned int)reply[4]; 01418 break; 01419 01420 default: 01421 downloader_cat.info() 01422 << "Unsupported SOCKS address type: " << (int)reply[3] << "\n"; 01423 _status_entry._status_code = SC_socks_invalid_version; 01424 _state = S_try_next_proxy; 01425 return false; 01426 } 01427 01428 // Now put back the bytes we've read so far, and get the rest of 01429 // them. 01430 _working_get = reply; 01431 if (!server_get_failsafe(reply, total_bytes)) { 01432 return true; 01433 } 01434 01435 if (downloader_cat.is_debug()) { 01436 // Finally, we can decode the whole thing. 01437 string connect_host; 01438 01439 switch (reply[3]) { 01440 case 0x01: // IPv4 01441 { 01442 ostringstream strm; 01443 strm << (unsigned int)(unsigned char)reply[4] << "." 01444 << (unsigned int)(unsigned char)reply[5] << "." 01445 << (unsigned int)(unsigned char)reply[6] << "." 01446 << (unsigned int)(unsigned char)reply[7]; 01447 connect_host = strm.str(); 01448 } 01449 break; 01450 01451 case 0x03: // DNS 01452 connect_host = string(&reply[5], (unsigned int)reply[4]); 01453 break; 01454 } 01455 01456 int connect_port = 01457 (((unsigned int)(unsigned char)reply[total_bytes - 2]) << 8) | 01458 ((unsigned int)(unsigned char)reply[total_bytes - 1]); 01459 01460 downloader_cat.debug() 01461 << _proxy << " directed us to " << connect_host << ":" 01462 << connect_port << "\n"; 01463 } 01464 01465 if (_want_ssl) { 01466 _state = S_setup_ssl; 01467 } else { 01468 _state = S_ready; 01469 } 01470 01471 return false; 01472 } 01473 01474 //////////////////////////////////////////////////////////////////// 01475 // Function: HTTPChannel::run_setup_ssl 01476 // Access: Private 01477 // Description: This state begins elevating our existing, unsecure 01478 // connection to a secure, SSL connection. 01479 //////////////////////////////////////////////////////////////////// 01480 bool HTTPChannel:: 01481 run_setup_ssl() { 01482 _sbio = BIO_new_ssl(_client->get_ssl_ctx(), true); 01483 BIO_push(_sbio, *_bio); 01484 01485 SSL *ssl = NULL; 01486 BIO_get_ssl(_sbio, &ssl); 01487 nassertr(ssl != (SSL *)NULL, false); 01488 01489 // We only take one word at a time from the _cipher_list. If that 01490 // connection fails, then we take the next word. 01491 string cipher_list = _cipher_list; 01492 if (!cipher_list.empty()) { 01493 size_t space = cipher_list.find(" "); 01494 if (space != string::npos) { 01495 cipher_list = cipher_list.substr(0, space); 01496 } 01497 } 01498 01499 if (downloader_cat.is_debug()) { 01500 downloader_cat.debug() 01501 << "Setting ssl-cipher-list '" << cipher_list << "'\n"; 01502 } 01503 int result = SSL_set_cipher_list(ssl, cipher_list.c_str()); 01504 if (result == 0) { 01505 downloader_cat.error() 01506 << "Invalid cipher list: '" << cipher_list << "'\n"; 01507 OpenSSLWrapper::get_global_ptr()->notify_ssl_errors(); 01508 _status_entry._status_code = SC_ssl_internal_failure; 01509 _state = S_failure; 01510 return false; 01511 } 01512 01513 // It would be nice to use something like SSL_set_client_cert_cb() 01514 // here to set a callback to provide the certificate should it be 01515 // requested, or even to potentially provide any of a number of 01516 // certificates according to the server's CA presented, but that 01517 // interface as provided by OpenSSL is broken since there's no way 01518 // to pass additional data to the callback function (and hence no 01519 // way to tie it back to the HTTPChannel object, other than by 01520 // building a messy mapping of SSL pointers back to HTTPChannel 01521 // pointers). 01522 if (_client->load_client_certificate()) { 01523 SSL_use_certificate(ssl, _client->_client_certificate_pub); 01524 SSL_use_PrivateKey(ssl, _client->_client_certificate_priv); 01525 if (!SSL_check_private_key(ssl)) { 01526 downloader_cat.warning() 01527 << "Client private key does not match public key!\n"; 01528 } 01529 } 01530 01531 if (downloader_cat.is_spam()) { 01532 downloader_cat.spam() 01533 << "SSL Ciphers available:\n"; 01534 const char *name; 01535 int pri = 0; 01536 name = SSL_get_cipher_list(ssl, pri); 01537 while (name != NULL) { 01538 downloader_cat.spam() 01539 << " " << pri + 1 << ". " << name << "\n"; 01540 pri++; 01541 name = SSL_get_cipher_list(ssl, pri); 01542 } 01543 } 01544 01545 if (downloader_cat.is_debug()) { 01546 downloader_cat.debug() 01547 << "performing SSL handshake\n"; 01548 } 01549 _state = S_ssl_handshake; 01550 01551 // We start the connect timer over again when we reach the SSL 01552 // handshake. 01553 _started_connecting_time = 01554 TrueClock::get_global_ptr()->get_short_time(); 01555 01556 return false; 01557 } 01558 01559 //////////////////////////////////////////////////////////////////// 01560 // Function: HTTPChannel::run_ssl_handshake 01561 // Access: Private 01562 // Description: This state performs the SSL handshake with the 01563 // server, and also verifies the server's identity when 01564 // the handshake has successfully completed. 01565 //////////////////////////////////////////////////////////////////// 01566 bool HTTPChannel:: 01567 run_ssl_handshake() { 01568 if (BIO_do_handshake(_sbio) <= 0) { 01569 if (BIO_should_retry(_sbio)) { 01570 double elapsed = 01571 TrueClock::get_global_ptr()->get_short_time() - 01572 _started_connecting_time; 01573 if (elapsed <= get_connect_timeout() + _extra_ssl_handshake_time) { 01574 // Keep trying. 01575 return true; 01576 } 01577 // Time to give up on the handshake. 01578 } 01579 01580 downloader_cat.info() 01581 << "Could not establish SSL handshake with " 01582 << _request.get_url().get_server_and_port() << "\n"; 01583 OpenSSLWrapper::get_global_ptr()->notify_ssl_errors(); 01584 01585 // It seems to be an error to free sbio at this point; perhaps 01586 // it's already been freed? 01587 01588 if (!_cipher_list.empty()) { 01589 // If we've got another cipher to try, do so. 01590 size_t space = _cipher_list.find(" "); 01591 if (space != string::npos) { 01592 while (space < _cipher_list.length() && _cipher_list[space] == ' ') { 01593 ++space; 01594 } 01595 _cipher_list = _cipher_list.substr(space); 01596 if (!_cipher_list.empty()) { 01597 close_connection(); 01598 reconsider_proxy(); 01599 _state = S_connecting; 01600 return false; 01601 } 01602 } 01603 } 01604 01605 // All done trying ciphers; they all failed. 01606 _cipher_list = _client->get_cipher_list(); 01607 _status_entry._status_code = SC_ssl_no_handshake; 01608 _state = S_failure; 01609 return false; 01610 } 01611 01612 SSL *ssl = NULL; 01613 BIO_get_ssl(_sbio, &ssl); 01614 nassertr(ssl != (SSL *)NULL, false); 01615 01616 if (!_nonblocking) { 01617 SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY); 01618 } 01619 01620 const SSL_CIPHER *cipher = SSL_get_current_cipher(ssl); 01621 if (cipher == (const SSL_CIPHER *)NULL) { 01622 downloader_cat.warning() 01623 << "No current cipher on SSL connection.\n"; 01624 } else { 01625 if (downloader_cat.is_debug()) { 01626 downloader_cat.debug() 01627 << "Using cipher " << SSL_CIPHER_get_name((SSL_CIPHER *) cipher) << "\n"; 01628 } 01629 } 01630 01631 // Now that we've made an SSL handshake, we can use the SSL bio to 01632 // do all of our communication henceforth. 01633 _bio->set_bio(_sbio); 01634 _sbio = NULL; 01635 01636 X509 *cert = SSL_get_peer_certificate(ssl); 01637 if (cert == (X509 *)NULL) { 01638 downloader_cat.info() 01639 << "No certificate was presented by server.\n"; 01640 01641 // This shouldn't be possible, per the SSL specs. 01642 _status_entry._status_code = SC_ssl_invalid_server_certificate; 01643 _state = S_failure; 01644 return false; 01645 } 01646 01647 X509_NAME *subject = X509_get_subject_name(cert); 01648 if (downloader_cat.is_debug()) { 01649 string org_name = get_x509_name_component(subject, NID_organizationName); 01650 string org_unit_name = get_x509_name_component(subject, NID_organizationalUnitName); 01651 string common_name = get_x509_name_component(subject, NID_commonName); 01652 01653 downloader_cat.debug() 01654 << "Server is " << common_name << " from " << org_unit_name 01655 << " / " << org_name << "\n"; 01656 01657 if (downloader_cat.is_spam()) { 01658 downloader_cat.spam() 01659 << "Received certificate from server:\n" << flush; 01660 X509_print_fp(stderr, cert); 01661 fflush(stderr); 01662 } 01663 } 01664 01665 bool cert_preapproved = false; 01666 bool cert_name_preapproved = false; 01667 check_preapproved_server_certificate(cert, cert_preapproved, cert_name_preapproved); 01668 01669 // Now verify the server certificate is valid. 01670 long verify_result = SSL_get_verify_result(ssl); 01671 bool cert_valid = true; 01672 01673 if (verify_result == X509_V_ERR_CERT_HAS_EXPIRED) { 01674 downloader_cat.info() 01675 << "Expired certificate from " << _request.get_url().get_server_and_port() << "\n"; 01676 if (_client->get_verify_ssl() == HTTPClient::VS_normal && !cert_preapproved) { 01677 cert_valid = false; 01678 } 01679 01680 } else if (verify_result == X509_V_ERR_CERT_NOT_YET_VALID) { 01681 downloader_cat.info() 01682 << "Premature certificate from " << _request.get_url().get_server_and_port() << "\n"; 01683 if (_client->get_verify_ssl() == HTTPClient::VS_normal && !cert_preapproved) { 01684 cert_valid = false; 01685 } 01686 01687 } else if (verify_result == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT || 01688 verify_result == X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN) { 01689 downloader_cat.info() 01690 << "Self-signed certificate from " << _request.get_url().get_server_and_port() << "\n"; 01691 if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_preapproved) { 01692 cert_valid = false; 01693 } 01694 01695 } else if (verify_result != X509_V_OK) { 01696 downloader_cat.info() 01697 << "Unable to verify identity of " << _request.get_url().get_server_and_port() 01698 << ", verify error code " << verify_result << "\n"; 01699 if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_preapproved) { 01700 cert_valid = false; 01701 } 01702 } 01703 01704 if (!cert_valid) { 01705 _status_entry._status_code = SC_ssl_invalid_server_certificate; 01706 _state = S_failure; 01707 return false; 01708 } 01709 01710 if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_name_preapproved) { 01711 // Check that the server is someone we expected to be talking 01712 // to. 01713 if (!validate_server_name(cert)) { 01714 _status_entry._status_code = SC_ssl_unexpected_server; 01715 _state = S_failure; 01716 return false; 01717 } 01718 } 01719 01720 X509_free(cert); 01721 01722 _state = S_ready; 01723 return false; 01724 } 01725 01726 //////////////////////////////////////////////////////////////////// 01727 // Function: HTTPChannel::run_ready 01728 // Access: Private 01729 // Description: This is the main "ready" state. In this state, we 01730 // have established a (possibly secure) connection to 01731 // the server (or proxy), and the server (or proxy) is 01732 // idle and waiting for us to send a request. 01733 // 01734 // If persistent_connection is true, we will generally 01735 // come back to this state after finishing each request 01736 // on a given connection. 01737 //////////////////////////////////////////////////////////////////// 01738 bool HTTPChannel:: 01739 run_ready() { 01740 // If there's a request to be sent upstream, send it now. 01741 if (!_request_text.empty()) { 01742 if (!server_send(_request_text, false)) { 01743 return true; 01744 } 01745 } 01746 01747 // All done sending request. 01748 _state = S_request_sent; 01749 _sent_request_time = TrueClock::get_global_ptr()->get_short_time(); 01750 return false; 01751 } 01752 01753 //////////////////////////////////////////////////////////////////// 01754 // Function: HTTPChannel::run_request_sent 01755 // Access: Private 01756 // Description: In this state we have sent our request to the server 01757 // (or proxy) and we are waiting for a response. 01758 //////////////////////////////////////////////////////////////////// 01759 bool HTTPChannel:: 01760 run_request_sent() { 01761 // Wait for the first line to come back from the server. 01762 string line; 01763 if (!server_getline_failsafe(line)) { 01764 return true; 01765 } 01766 01767 // Skip unexpected blank lines. We're just being generous here. 01768 while (line.empty()) { 01769 if (!server_getline_failsafe(line)) { 01770 return true; 01771 } 01772 } 01773 01774 if (!parse_http_response(line)) { 01775 // Not an HTTP response. _state is already set appropriately. 01776 return false; 01777 } 01778 01779 _state = S_reading_header; 01780 _current_field_name = string(); 01781 _current_field_value = string(); 01782 _headers.clear(); 01783 _got_file_size = false; 01784 _got_transfer_file_size = false; 01785 return false; 01786 } 01787 01788 //////////////////////////////////////////////////////////////////// 01789 // Function: HTTPChannel::run_reading_header 01790 // Access: Private 01791 // Description: In this state we have received the first response to 01792 // our request from the server (or proxy) and we are 01793 // reading the set of header lines preceding the 01794 // requested document. 01795 //////////////////////////////////////////////////////////////////// 01796 bool HTTPChannel:: 01797 run_reading_header() { 01798 if (parse_http_header()) { 01799 if (_bio.is_null()) { 01800 downloader_cat.info() 01801 << "Connection lost while reading HTTP response.\n"; 01802 if (_response_type == RT_http_hangup) { 01803 // This was our second hangup in a row. Give up. 01804 _status_entry._status_code = SC_lost_connection; 01805 _state = S_try_next_proxy; 01806 01807 } else { 01808 // Try again, once. 01809 _response_type = RT_http_hangup; 01810 } 01811 01812 } else { 01813 double elapsed = 01814 TrueClock::get_global_ptr()->get_short_time() - 01815 _sent_request_time; 01816 if (elapsed > get_http_timeout()) { 01817 // Time to give up. 01818 downloader_cat.info() 01819 << "Timeout waiting for " 01820 << _request.get_url().get_server_and_port() 01821 << " in run_reading_header (" << elapsed 01822 << " seconds elapsed).\n"; 01823 _status_entry._status_code = SC_timeout; 01824 _state = S_try_next_proxy; 01825 } 01826 } 01827 return true; 01828 } 01829 _response_type = RT_http_complete; 01830 01831 // Ok, we've established an HTTP connection to the server. Our 01832 // extra send headers have done their job; clear them for next time. 01833 clear_extra_headers(); 01834 01835 _server_response_has_no_body = 01836 (get_status_code() / 100 == 1 || 01837 get_status_code() == 204 || 01838 get_status_code() == 304 || 01839 _method == HTTPEnum::M_head); 01840 01841 // Look for key properties in the header fields. 01842 if (get_status_code() == 206) { 01843 string content_range = get_header_value("Content-Range"); 01844 if (content_range.empty()) { 01845 downloader_cat.warning() 01846 << "Got 206 response without Content-Range header!\n"; 01847 _status_entry._status_code = SC_invalid_http; 01848 _state = S_failure; 01849 return false; 01850 01851 } else { 01852 if (!parse_content_range(content_range)) { 01853 downloader_cat.warning() 01854 << "Couldn't parse Content-Range: " << content_range << "\n"; 01855 _status_entry._status_code = SC_invalid_http; 01856 _state = S_failure; 01857 return false; 01858 } 01859 } 01860 01861 } else { 01862 _first_byte_delivered = 0; 01863 _last_byte_delivered = 0; 01864 } 01865 if (downloader_cat.is_debug()) { 01866 if (_first_byte_requested != 0 || _last_byte_requested != 0 || 01867 _first_byte_delivered != 0 || _last_byte_delivered != 0) { 01868 downloader_cat.debug() 01869 << "Requested byte range " << _first_byte_requested 01870 << " to " << _last_byte_delivered 01871 << "; server delivers range " << _first_byte_delivered 01872 << " to " << _last_byte_delivered 01873 << "\n"; 01874 } 01875 } 01876 01877 // Set the _document_spec to reflect what we just retrieved. 01878 _document_spec = DocumentSpec(_request.get_url()); 01879 string tag = get_header_value("ETag"); 01880 if (!tag.empty()) { 01881 _document_spec.set_tag(HTTPEntityTag(tag)); 01882 } 01883 string date = get_header_value("Last-Modified"); 01884 if (!date.empty()) { 01885 _document_spec.set_date(HTTPDate(date)); 01886 } 01887 01888 // In case we've got a download in effect, now we know what the 01889 // first byte of the subdocument request will be, so we can open the 01890 // file and position it. 01891 if (_server_response_has_no_body) { 01892 // Never mind on the download. 01893 reset_download_to(); 01894 } 01895 01896 if (!open_download_file()) { 01897 return false; 01898 } 01899 01900 _got_expected_file_size = false; 01901 _got_file_size = false; 01902 _got_transfer_file_size = false; 01903 01904 string content_length = get_header_value("Content-Length"); 01905 if (!content_length.empty()) { 01906 _file_size = atoi(content_length.c_str()); 01907 _got_file_size = true; 01908 01909 } else if (get_status_code() == 206) { 01910 // Well, we didn't get a content-length from the server, but we 01911 // can infer the number of bytes based on the range we're given. 01912 _file_size = _last_byte_delivered - _first_byte_delivered + 1; 01913 _got_file_size = true; 01914 } 01915 _redirect = get_header_value("Location"); 01916 01917 // The server might have given us just a filename for the redirect. 01918 // In that case, it's relative to the same server. If it's a 01919 // relative path, it's relative to the same directory. 01920 if (_redirect.has_path() && !_redirect.has_authority()) { 01921 URLSpec url = _document_spec.get_url(); 01922 Filename path = _redirect.get_path(); 01923 if (path.is_local()) { 01924 Filename rel_to = Filename(url.get_path()).get_dirname(); 01925 _redirect.set_path(Filename(rel_to, path)); 01926 } 01927 _redirect.set_scheme(url.get_scheme()); 01928 _redirect.set_authority(url.get_authority()); 01929 } 01930 01931 _state = S_read_header; 01932 01933 if (_server_response_has_no_body && will_close_connection()) { 01934 // If the server said it will close the connection, we should 01935 // close it too. 01936 close_connection(); 01937 } 01938 01939 // Handle automatic retries and redirects. 01940 int last_status = _last_status_code; 01941 _last_status_code = get_status_code(); 01942 01943 if (get_status_code() == 407 && last_status != 407 && !_proxy.empty()) { 01944 // 407: not authorized to proxy. Try to get the authorization. 01945 string authenticate_request = get_header_value("Proxy-Authenticate"); 01946 _proxy_auth = 01947 _client->generate_auth(_proxy, true, authenticate_request); 01948 if (_proxy_auth != (HTTPAuthorization *)NULL) { 01949 _proxy_realm = _proxy_auth->get_realm(); 01950 _proxy_username = _client->select_username(_proxy, true, _proxy_realm); 01951 if (!_proxy_username.empty()) { 01952 make_request_text(); 01953 01954 // Roll the state forward to force a new request. 01955 _state = S_begin_body; 01956 return false; 01957 } 01958 } 01959 } 01960 01961 if (get_status_code() == 401 && last_status != 401) { 01962 // 401: not authorized to remote server. Try to get the authorization. 01963 string authenticate_request = get_header_value("WWW-Authenticate"); 01964 _www_auth = _client->generate_auth(_request.get_url(), false, authenticate_request); 01965 if (_www_auth != (HTTPAuthorization *)NULL) { 01966 _www_realm = _www_auth->get_realm(); 01967 _www_username = _client->select_username(_request.get_url(), false, _www_realm); 01968 if (!_www_username.empty()) { 01969 make_request_text(); 01970 01971 // Roll the state forward to force a new request. 01972 _state = S_begin_body; 01973 return false; 01974 } 01975 } 01976 } 01977 01978 if ((get_status_code() == 300 || 01979 get_status_code() == 301 || 01980 get_status_code() == 302 || 01981 get_status_code() == 303 || 01982 get_status_code() == 307) && !get_redirect().empty()) { 01983 // Redirect. Should we handle it automatically? 01984 01985 // According to the letter of RFC 2616, 301 and 302 responses to 01986 // POST requests must not be automatically redirected without 01987 // confirmation by the user. In reality, browsers do allow 01988 // automatic redirection of these responses, changing the POST to 01989 // a GET, and we reproduce this behavior here. 01990 if (_method == HTTPEnum::M_post) { 01991 _method = HTTPEnum::M_get; 01992 _body = string(); 01993 } 01994 01995 if (_method == HTTPEnum::M_get || _method == HTTPEnum::M_head) { 01996 // Sure! 01997 URLSpec new_url = get_redirect(); 01998 if (find(_redirect_trail.begin(), _redirect_trail.end(), 01999 new_url) != _redirect_trail.end()) { 02000 downloader_cat.warning() 02001 << "cycle detected in redirect to " << new_url << "\n"; 02002 02003 } else { 02004 _redirect_trail.push_back(new_url); 02005 02006 if (downloader_cat.is_debug()) { 02007 downloader_cat.debug() 02008 << "following redirect to " << new_url << "\n"; 02009 } 02010 if (_request.get_url().has_username()) { 02011 new_url.set_username(_request.get_url().get_username()); 02012 } 02013 reset_url(_request.get_url(), new_url); 02014 _request.set_url(new_url); 02015 _want_ssl = _request.get_url().is_ssl(); 02016 reconsider_proxy(); 02017 make_header(); 02018 make_request_text(); 02019 02020 // Roll the state forward to force a new request. 02021 _state = S_begin_body; 02022 return false; 02023 } 02024 } 02025 } 02026 02027 if (_state == S_read_header && 02028 ((get_status_code() / 100) == 4 || (get_status_code() / 100) == 5) && 02029 _proxy_serves_document && _proxy_next_index < _proxies.size()) { 02030 // If we were using a proxy (but not tunneling through the proxy) 02031 // and we got some kind of a server error, try the next proxy in 02032 // sequence (if we have one). This handles the case of a working 02033 // proxy that cannot see the host (and so returns 504 or something 02034 // along those lines). Some proxies are so broken they return a 02035 // 404 in this case, so we have to consider that along the same 02036 // lines. 02037 _state = S_try_next_proxy; 02038 return false; 02039 } 02040 02041 // Otherwise, we're good to go. 02042 return false; 02043 } 02044 02045 //////////////////////////////////////////////////////////////////// 02046 // Function: HTTPChannel::run_start_direct_file_read 02047 // Access: Private 02048 // Description: This is the first state when reading a file:// URL. 02049 // All it does is skip past the non-existent "header". 02050 //////////////////////////////////////////////////////////////////// 02051 bool HTTPChannel:: 02052 run_start_direct_file_read() { 02053 _state = S_read_header; 02054 if (!open_download_file()) { 02055 return false; 02056 } 02057 return false; 02058 } 02059 02060 //////////////////////////////////////////////////////////////////// 02061 // Function: HTTPChannel::run_read_header 02062 // Access: Private 02063 // Description: In this state we have completely read the header 02064 // lines returned by the server (or proxy) in response 02065 // to our request. This state represents the normal 02066 // stopping point of a call to get_document(), etc.; 02067 // further reads will return the body of the request, 02068 // the requested document. 02069 // 02070 // Normally run_read_header() is not called unless the 02071 // user has elected not to read the returned document 02072 // himself. In fact, the state itself only exists so we 02073 // can make a distinction between S_read_header and 02074 // S_begin_body, where S_read_header is safe to return 02075 // to the user and S_begin_body means we need to start 02076 // skipping the document. 02077 //////////////////////////////////////////////////////////////////// 02078 bool HTTPChannel:: 02079 run_read_header() { 02080 _state = S_begin_body; 02081 return false; 02082 } 02083 02084 //////////////////////////////////////////////////////////////////// 02085 // Function: HTTPChannel::run_begin_body 02086 // Access: Private 02087 // Description: This state begins to skip over the body in 02088 // preparation for making a new request. 02089 //////////////////////////////////////////////////////////////////// 02090 bool HTTPChannel:: 02091 run_begin_body() { 02092 if (will_close_connection()) { 02093 // If the socket will close anyway, no point in skipping past the 02094 // previous body; just reset. 02095 if (downloader_cat.is_debug()) { 02096 downloader_cat.debug() 02097 << "resetting to begin body; server would close anyway.\n"; 02098 } 02099 reset_to_new(); 02100 return false; 02101 } 02102 02103 if (_server_response_has_no_body) { 02104 // We have already "read" the nonexistent body. 02105 _state = S_read_trailer; 02106 02107 } else if (get_file_size() > (int)_skip_body_size) { 02108 // If we know the size of the body we are about to skip and it's 02109 // too large, then don't bother skipping it--just drop the 02110 // connection and get a new one. 02111 if (downloader_cat.is_debug()) { 02112 downloader_cat.debug() 02113 << "Dropping connection rather than skipping past " 02114 << get_file_size() << " bytes.\n"; 02115 } 02116 reset_to_new(); 02117 02118 } else { 02119 open_read_body(); 02120 if (_body_stream == (ISocketStream *)NULL) { 02121 if (downloader_cat.is_debug()) { 02122 downloader_cat.debug() 02123 << "Unable to skip body.\n"; 02124 } 02125 reset_to_new(); 02126 02127 } else { 02128 _owns_body_stream = true; 02129 if (_state != S_reading_body) { 02130 reset_body_stream(); 02131 } 02132 } 02133 } 02134 02135 return false; 02136 } 02137 02138 //////////////////////////////////////////////////////////////////// 02139 // Function: HTTPChannel::run_reading_body 02140 // Access: Private 02141 // Description: In this state we are in the process of reading the 02142 // response's body. We will only come to this function 02143 // if the user did not choose to read the entire body 02144 // himself (by calling open_read_body()). 02145 // 02146 // In this case we should skip past the body to reset 02147 // the connection for making a new request. 02148 //////////////////////////////////////////////////////////////////// 02149 bool HTTPChannel:: 02150 run_reading_body() { 02151 if (will_close_connection()) { 02152 // If the socket will close anyway, no point in skipping past the 02153 // previous body; just reset. 02154 if (downloader_cat.is_debug()) { 02155 downloader_cat.debug() 02156 << "resetting to read body; server would close anyway.\n"; 02157 } 02158 reset_to_new(); 02159 return false; 02160 } 02161 02162 // Skip the body we've already started. 02163 if (_body_stream == NULL || !_owns_body_stream) { 02164 // Whoops, we're not in skip-body mode. Better reset. 02165 if (downloader_cat.is_debug()) { 02166 downloader_cat.debug() 02167 << "resetting, not in skip-body mode.\n"; 02168 } 02169 reset_to_new(); 02170 return false; 02171 } 02172 02173 string line; 02174 getline(*_body_stream, line); 02175 while (!_body_stream->fail() && !_body_stream->eof()) { 02176 if (downloader_cat.is_spam()) { 02177 downloader_cat.spam() << "skip: " << line << "\n"; 02178 } 02179 getline(*_body_stream, line); 02180 } 02181 02182 if (!_body_stream->is_closed()) { 02183 // There's more to come later. 02184 return true; 02185 } 02186 02187 reset_body_stream(); 02188 02189 // This should have been set by the call to finished_body(), above. 02190 nassertr(_state != S_reading_body, false); 02191 return false; 02192 } 02193 02194 //////////////////////////////////////////////////////////////////// 02195 // Function: HTTPChannel::run_read_body 02196 // Access: Private 02197 // Description: In this state we have completely read (or skipped 02198 // over) the body of the response. We should continue 02199 // skipping past the trailer following the body. 02200 // 02201 // Not all bodies come with trailers; in particular, the 02202 // "identity" transfer encoding does not include a 02203 // trailer. It is therefore the responsibility of the 02204 // IdentityStreamBuf or ChunkedStreamBuf to set the 02205 // state appropriately to either S_read_body or 02206 // S_read_trailer following the completion of the body. 02207 //////////////////////////////////////////////////////////////////// 02208 bool HTTPChannel:: 02209 run_read_body() { 02210 if (will_close_connection()) { 02211 // If the socket will close anyway, no point in skipping past the 02212 // previous body; just reset. 02213 if (downloader_cat.is_debug()) { 02214 downloader_cat.debug() 02215 << "resetting to read body; server would close anyway.\n"; 02216 } 02217 reset_to_new(); 02218 return false; 02219 } 02220 // Skip the trailer following the recently-read body. 02221 02222 string line; 02223 if (!server_getline(line)) { 02224 return true; 02225 } 02226 while (!line.empty()) { 02227 if (!server_getline(line)) { 02228 return true; 02229 } 02230 } 02231 02232 _state = S_read_trailer; 02233 return false; 02234 } 02235 02236 //////////////////////////////////////////////////////////////////// 02237 // Function: HTTPChannel::run_read_trailer 02238 // Access: Private 02239 // Description: In this state we have completely read the body and 02240 // the trailer. This state is simply a pass-through 02241 // back to S_ready. 02242 //////////////////////////////////////////////////////////////////// 02243 bool HTTPChannel:: 02244 run_read_trailer() { 02245 if (will_close_connection()) { 02246 // If the socket will close anyway, no point in skipping past the 02247 // previous body; just reset. 02248 if (downloader_cat.is_debug()) { 02249 downloader_cat.debug() 02250 << "resetting to read trailer; server would close anyway.\n"; 02251 } 02252 reset_to_new(); 02253 return false; 02254 } 02255 02256 _state = S_ready; 02257 return false; 02258 } 02259 02260 //////////////////////////////////////////////////////////////////// 02261 // Function: HTTPChannel::run_download_to_file 02262 // Access: Private 02263 // Description: After the headers, etc. have been read, this streams 02264 // the download to the named file. 02265 //////////////////////////////////////////////////////////////////// 02266 bool HTTPChannel:: 02267 run_download_to_file() { 02268 nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false); 02269 02270 bool do_throttle = _wanted_nonblocking && _download_throttle; 02271 02272 static const size_t buffer_size = 4096; 02273 char buffer[buffer_size]; 02274 02275 size_t remaining_this_pass = buffer_size; 02276 if (do_throttle) { 02277 remaining_this_pass = _bytes_per_update; 02278 } 02279 02280 _body_stream->read(buffer, min(buffer_size, remaining_this_pass)); 02281 size_t count = _body_stream->gcount(); 02282 while (count != 0) { 02283 _download_to_file.write(buffer, count); 02284 _bytes_downloaded += count; 02285 if (do_throttle) { 02286 nassertr(count <= remaining_this_pass, false); 02287 remaining_this_pass -= count; 02288 if (remaining_this_pass == 0) { 02289 // That's enough for now. 02290 return true; 02291 } 02292 } 02293 02294 thread_consider_yield(); 02295 _body_stream->read(buffer, min(buffer_size, remaining_this_pass)); 02296 count = _body_stream->gcount(); 02297 } 02298 02299 if (_download_to_file.fail()) { 02300 downloader_cat.warning() 02301 << "Error writing to " << _download_to_filename << "\n"; 02302 _status_entry._status_code = SC_download_write_error; 02303 _state = S_failure; 02304 reset_download_to(); 02305 return false; 02306 } 02307 02308 _download_to_file.flush(); 02309 02310 if (_body_stream->is_closed()) { 02311 // Done. 02312 reset_body_stream(); 02313 _download_to_file.close(); 02314 _started_download = false; 02315 return false; 02316 } else { 02317 // More to come. 02318 return true; 02319 } 02320 } 02321 02322 //////////////////////////////////////////////////////////////////// 02323 // Function: HTTPChannel::run_download_to_ram 02324 // Access: Private 02325 // Description: After the headers, etc. have been read, this streams 02326 // the download to the specified Ramfile object. 02327 //////////////////////////////////////////////////////////////////// 02328 bool HTTPChannel:: 02329 run_download_to_ram() { 02330 nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false); 02331 nassertr(_download_to_ramfile != (Ramfile *)NULL, false); 02332 02333 bool do_throttle = _wanted_nonblocking && _download_throttle; 02334 02335 static const size_t buffer_size = 4096; 02336 char buffer[buffer_size]; 02337 02338 size_t remaining_this_pass = buffer_size; 02339 if (do_throttle) { 02340 remaining_this_pass = _bytes_per_update; 02341 } 02342 02343 _body_stream->read(buffer, min(buffer_size, remaining_this_pass)); 02344 size_t count = _body_stream->gcount(); 02345 while (count != 0) { 02346 _download_to_ramfile->_data += string(buffer, count); 02347 _bytes_downloaded += count; 02348 if (do_throttle) { 02349 nassertr(count <= remaining_this_pass, false); 02350 remaining_this_pass -= count; 02351 if (remaining_this_pass == 0) { 02352 // That's enough for now. 02353 return true; 02354 } 02355 } 02356 02357 thread_consider_yield(); 02358 _body_stream->read(buffer, min(buffer_size, remaining_this_pass)); 02359 count = _body_stream->gcount(); 02360 } 02361 02362 if (_body_stream->is_closed()) { 02363 // Done. 02364 reset_body_stream(); 02365 _started_download = false; 02366 return false; 02367 } else { 02368 // More to come. 02369 return true; 02370 } 02371 } 02372 02373 //////////////////////////////////////////////////////////////////// 02374 // Function: HTTPChannel::run_download_to_stream 02375 // Access: Private 02376 // Description: After the headers, etc. have been read, this streams 02377 // the download to the named file. 02378 //////////////////////////////////////////////////////////////////// 02379 bool HTTPChannel:: 02380 run_download_to_stream() { 02381 nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false); 02382 02383 bool do_throttle = _wanted_nonblocking && _download_throttle; 02384 02385 static const size_t buffer_size = 4096; 02386 char buffer[buffer_size]; 02387 02388 size_t remaining_this_pass = buffer_size; 02389 if (do_throttle) { 02390 remaining_this_pass = _bytes_per_update; 02391 } 02392 02393 _body_stream->read(buffer, min(buffer_size, remaining_this_pass)); 02394 size_t count = _body_stream->gcount(); 02395 while (count != 0) { 02396 _download_to_stream->write(buffer, count); 02397 _bytes_downloaded += count; 02398 if (do_throttle) { 02399 nassertr(count <= remaining_this_pass, false); 02400 remaining_this_pass -= count; 02401 if (remaining_this_pass == 0) { 02402 // That's enough for now. 02403 return true; 02404 } 02405 } 02406 02407 thread_consider_yield(); 02408 _body_stream->read(buffer, min(buffer_size, remaining_this_pass)); 02409 count = _body_stream->gcount(); 02410 } 02411 02412 if (_download_to_stream->fail()) { 02413 downloader_cat.warning() 02414 << "Error writing to stream\n"; 02415 _status_entry._status_code = SC_download_write_error; 02416 _state = S_failure; 02417 reset_download_to(); 02418 return false; 02419 } 02420 02421 _download_to_stream->flush(); 02422 02423 if (_body_stream->is_closed()) { 02424 // Done. 02425 reset_body_stream(); 02426 _download_to_stream = NULL; 02427 _started_download = false; 02428 return false; 02429 } else { 02430 // More to come. 02431 return true; 02432 } 02433 } 02434 02435 02436 //////////////////////////////////////////////////////////////////// 02437 // Function: HTTPChannel::begin_request 02438 // Access: Private 02439 // Description: Begins a new document request to the server, throwing 02440 // away whatever request was currently pending if 02441 // necessary. 02442 //////////////////////////////////////////////////////////////////// 02443 void HTTPChannel:: 02444 begin_request(HTTPEnum::Method method, const DocumentSpec &url, 02445 const string &body, bool nonblocking, 02446 size_t first_byte, size_t last_byte) { 02447 02448 downloader_cat.info() 02449 << method << " " << url << "\n"; 02450 02451 reset_for_new_request(); 02452 02453 _wanted_nonblocking = nonblocking; 02454 #if defined(HAVE_THREADS) && defined(SIMPLE_THREADS) 02455 // In the presence of SIMPLE_THREADS, we always use non-blocking 02456 // I/O. We simulate blocking by yielding the thread. 02457 nonblocking = true; 02458 #endif 02459 02460 // Get the set of proxies that are appropriate for this URL. 02461 _proxies.clear(); 02462 _proxy_next_index = 0; 02463 if (get_allow_proxy()) { 02464 _client->get_proxies_for_url(url.get_url(), _proxies); 02465 } 02466 02467 // If we still have a live connection to a proxy that is on the 02468 // list, that proxy should be moved immediately to the front of the 02469 // list (to minimize restarting connections unnecessarily). 02470 if (!_bio.is_null() && !_proxies.empty() && !_proxy.empty()) { 02471 Proxies::iterator pi = find(_proxies.begin(), _proxies.end(), _proxy); 02472 if (pi != _proxies.end()) { 02473 _proxies.erase(pi); 02474 _proxies.insert(_proxies.begin(), _proxy); 02475 } 02476 } 02477 02478 URLSpec new_proxy; 02479 if (_proxy_next_index < _proxies.size()) { 02480 new_proxy = _proxies[_proxy_next_index]; 02481 _proxy_next_index++; 02482 } 02483 02484 // Changing the proxy is grounds for dropping the old connection, if 02485 // any. 02486 if (_proxy != new_proxy) { 02487 _proxy = new_proxy; 02488 _proxy_auth = (HTTPAuthorization *)NULL; 02489 if (downloader_cat.is_debug()) { 02490 downloader_cat.debug() 02491 << "resetting to change proxy to " << _proxy << "\n"; 02492 } 02493 reset_to_new(); 02494 } 02495 02496 // Ditto with changing the nonblocking state. 02497 if (_nonblocking != nonblocking) { 02498 _nonblocking = nonblocking; 02499 if (downloader_cat.is_debug()) { 02500 downloader_cat.debug() 02501 << "resetting to change nonblocking state to " << _nonblocking << ".\n"; 02502 } 02503 reset_to_new(); 02504 } 02505 02506 reset_url(_request.get_url(), url.get_url()); 02507 _request = url; 02508 _document_spec = DocumentSpec(); 02509 _method = method; 02510 _body = body; 02511 02512 // An https-style request means we'll need to establish an SSL 02513 // connection. 02514 _want_ssl = _request.get_url().is_ssl(); 02515 02516 _first_byte_requested = first_byte; 02517 _last_byte_requested = last_byte; 02518 _connect_count = 0; 02519 02520 reconsider_proxy(); 02521 02522 // Reset from whatever previous request might still be pending. 02523 if (_request.get_url().get_scheme() == "file") { 02524 // A "file" URL just means we're reading a raw file. This only 02525 // supports actual disk files, not the VFS, because we use a 02526 // BIO_new_file() underneath this. 02527 reset_to_new(); 02528 _bio = new BioPtr(_request.get_url()); 02529 if (_bio->get_bio() != NULL) { 02530 // Successfully opened the file. 02531 _source = new BioStreamPtr(new BioStream(_bio)); 02532 _status_entry._status_code = 200; 02533 _state = S_start_direct_file_read; 02534 02535 // Get the file size. 02536 FILE *fp = NULL; 02537 BIO_get_fp(_bio->get_bio(), &fp); 02538 if (fp != NULL) { 02539 if (fseek(fp, 0, SEEK_END) == 0) { 02540 _file_size = ftell(fp); 02541 _got_file_size = true; 02542 fseek(fp, 0, SEEK_SET); 02543 } 02544 } 02545 02546 } else { 02547 // Couldn't read the file. 02548 OpenSSLWrapper::get_global_ptr()->notify_ssl_errors(); 02549 _status_entry._status_code = SC_no_connection; 02550 _state = S_failure; 02551 } 02552 02553 } else { 02554 // We're reading a normal network URL. 02555 if (_state == S_failure || (_state < S_read_header && _state != S_ready)) { 02556 if (downloader_cat.is_debug()) { 02557 downloader_cat.debug() 02558 << "resetting to clear previous request.\n"; 02559 } 02560 reset_to_new(); 02561 02562 } else if (TrueClock::get_global_ptr()->get_short_time() - _last_run_time >= _idle_timeout) { 02563 if (downloader_cat.is_debug()) { 02564 downloader_cat.debug() 02565 << "resetting old connection: " 02566 << TrueClock::get_global_ptr()->get_short_time() - _last_run_time 02567 << " s old.\n"; 02568 } 02569 reset_to_new(); 02570 02571 } else if (_state == S_read_header) { 02572 // Roll one step forwards to start skipping past the previous 02573 // body. 02574 _state = S_begin_body; 02575 } 02576 } 02577 02578 if (_method == HTTPEnum::M_connect) { 02579 _done_state = S_ready; 02580 } else { 02581 _done_state = S_read_header; 02582 } 02583 } 02584 02585 //////////////////////////////////////////////////////////////////// 02586 // Function: HTTPChannel::reconsider_proxy 02587 // Access: Private 02588 // Description: Reevaluates the flags and strings that are computed 02589 // based on the particular proxy we are attempting to 02590 // connect to. This should be called when we initiate a 02591 // request, and also whenever we change proxies while 02592 // processing a request. 02593 //////////////////////////////////////////////////////////////////// 02594 void HTTPChannel:: 02595 reconsider_proxy() { 02596 _proxy_tunnel_now = false; 02597 _proxy_serves_document = false; 02598 02599 if (!_proxy.empty()) { 02600 // If the user insists we always tunnel through a proxy, or if 02601 // we're opening an SSL connection, or the user has explicitly 02602 // asked for a direct connection of some kind, or if we have a 02603 // SOCKS-style proxy; each of these demands a tunnel through the 02604 // proxy to speak directly to the http server. 02605 _proxy_tunnel_now = 02606 (get_proxy_tunnel() || _want_ssl || 02607 _method == HTTPEnum::M_connect || _proxy.get_scheme() == "socks"); 02608 02609 // Otherwise (but we still have a proxy), then we ask the proxy to 02610 // hand us the document. 02611 _proxy_serves_document = !_proxy_tunnel_now; 02612 } 02613 02614 make_header(); 02615 make_request_text(); 02616 02617 if (_proxy_tunnel_now) { 02618 // Maybe we need to tunnel through the proxy to connect to the 02619 // server directly. 02620 ostringstream request; 02621 request 02622 << "CONNECT " << _request.get_url().get_server_and_port() 02623 << " " << _client->get_http_version_string() << "\r\n"; 02624 if (_client->get_http_version() >= HTTPEnum::HV_11) { 02625 request 02626 << "Host: " << _request.get_url().get_server_and_port() << "\r\n"; 02627 } 02628 _proxy_header = request.str(); 02629 make_proxy_request_text(); 02630 02631 } else { 02632 _proxy_header = string(); 02633 _proxy_request_text = string(); 02634 } 02635 } 02636 02637 02638 //////////////////////////////////////////////////////////////////// 02639 // Function: HTTPChannel::reset_for_new_request 02640 // Access: Private 02641 // Description: Resets the internal state variables in preparation 02642 // for beginning a new request. 02643 //////////////////////////////////////////////////////////////////// 02644 void HTTPChannel:: 02645 reset_for_new_request() { 02646 reset_download_to(); 02647 reset_body_stream(); 02648 02649 _last_status_code = 0; 02650 _status_entry = StatusEntry(); 02651 02652 _response_type = RT_none; 02653 _redirect_trail.clear(); 02654 _bytes_downloaded = 0; 02655 _bytes_requested = 0; 02656 } 02657 02658 //////////////////////////////////////////////////////////////////// 02659 // Function: HTTPChannel::finished_body 02660 // Access: Private 02661 // Description: This is called by the body reading 02662 // classes--ChunkedStreamBuf and IdentityStreamBuf--when 02663 // they have finished reading the body. It advances the 02664 // state appropriately. 02665 // 02666 // has_trailer should be set true if the body type has 02667 // an associated trailer which should be read or 02668 // skipped, or false if there is no trailer. 02669 //////////////////////////////////////////////////////////////////// 02670 void HTTPChannel:: 02671 finished_body(bool has_trailer) { 02672 if (will_close_connection() && _download_dest == DD_none) { 02673 if (downloader_cat.is_debug()) { 02674 downloader_cat.debug() 02675 << "resetting to finish body; server would close anyway.\n"; 02676 } 02677 reset_to_new(); 02678 02679 } else { 02680 if (has_trailer) { 02681 _state = HTTPChannel::S_read_body; 02682 } else { 02683 _state = HTTPChannel::S_read_trailer; 02684 } 02685 } 02686 } 02687 02688 //////////////////////////////////////////////////////////////////// 02689 // Function: HTTPChannel::open_download_file 02690 // Access: Private 02691 // Description: If a download has been requested, opens the file on 02692 // disk (or prepares the RamFile or stream) and seeks 02693 // within it to the appropriate _first_byte_delivered 02694 // position, so that downloaded bytes will be written to 02695 // the appropriate point within the file. Returns true 02696 // if the starting position is valid, false otherwise 02697 // (in which case the state is set to S_failure). 02698 //////////////////////////////////////////////////////////////////// 02699 bool HTTPChannel:: 02700 open_download_file() { 02701 _subdocument_resumes = (_subdocument_resumes && _first_byte_delivered != 0); 02702 02703 if (_download_dest == DD_file) { 02704 if (!_download_to_filename.open_write(_download_to_file, !_subdocument_resumes)) { 02705 downloader_cat.info() 02706 << "Could not open " << _download_to_filename << " for writing.\n"; 02707 _status_entry._status_code = SC_download_open_error; 02708 _state = S_failure; 02709 return false; 02710 } 02711 } 02712 02713 if (_subdocument_resumes) { 02714 if (_download_dest == DD_file) { 02715 // Windows doesn't complain if you try to seek past the end of 02716 // file--it happily appends enough zero bytes to make the 02717 // difference. Blecch. That means we need to get the file size 02718 // first to check it ourselves. 02719 _download_to_file.seekp(0, ios::end); 02720 if (_first_byte_delivered > (size_t)_download_to_file.tellp()) { 02721 downloader_cat.info() 02722 << "Invalid starting position of byte " << _first_byte_delivered 02723 << " within " << _download_to_filename << " (which has " 02724 << _download_to_file.tellp() << " bytes)\n"; 02725 _download_to_file.close(); 02726 _status_entry._status_code = SC_download_invalid_range; 02727 _state = S_failure; 02728 return false; 02729 } 02730 02731 _download_to_file.seekp(_first_byte_delivered); 02732 02733 } else if (_download_dest == DD_ram) { 02734 if (_first_byte_delivered > _download_to_ramfile->_data.length()) { 02735 downloader_cat.info() 02736 << "Invalid starting position of byte " << _first_byte_delivered 02737 << " within Ramfile (which has " 02738 << _download_to_ramfile->_data.length() << " bytes)\n"; 02739 _status_entry._status_code = SC_download_invalid_range; 02740 _state = S_failure; 02741 return false; 02742 } 02743 02744 if (_first_byte_delivered == 0) { 02745 _download_to_ramfile->_data = string(); 02746 } else { 02747 _download_to_ramfile->_data = 02748 _download_to_ramfile->_data.substr(0, _first_byte_delivered); 02749 } 02750 } else if (_download_dest == DD_stream) { 02751 // Windows doesn't complain if you try to seek past the end of 02752 // file--it happily appends enough zero bytes to make the 02753 // difference. Blecch. That means we need to get the file size 02754 // first to check it ourselves. 02755 _download_to_stream->seekp(0, ios::end); 02756 if (_first_byte_delivered > (size_t)_download_to_stream->tellp()) { 02757 downloader_cat.info() 02758 << "Invalid starting position of byte " << _first_byte_delivered 02759 << " within stream (which has " 02760 << _download_to_stream->tellp() << " bytes)\n"; 02761 _download_to_stream = NULL; 02762 _status_entry._status_code = SC_download_invalid_range; 02763 _state = S_failure; 02764 return false; 02765 } 02766 02767 _download_to_stream->seekp(_first_byte_delivered); 02768 } 02769 02770 } else { 02771 // If _subdocument_resumes is false, we should be sure to reset to 02772 // the beginning of the file, regardless of the value of 02773 // _first_byte_delivered. 02774 if (_download_dest == DD_file) { 02775 _download_to_file.seekp(0); 02776 } else if (_download_dest == DD_ram) { 02777 _download_to_ramfile->_data = string(); 02778 } else if (_download_dest == DD_stream) { 02779 _download_to_stream->seekp(0); 02780 } 02781 } 02782 02783 return true; 02784 } 02785 02786 02787 //////////////////////////////////////////////////////////////////// 02788 // Function: HTTPChannel::server_getline 02789 // Access: Private 02790 // Description: Reads a single line from the server's reply. Returns 02791 // true if the line is successfully retrieved, or false 02792 // if a complete line has not yet been received or if 02793 // the connection has been closed. 02794 //////////////////////////////////////////////////////////////////// 02795 bool HTTPChannel:: 02796 server_getline(string &str) { 02797 nassertr(!_source.is_null(), false); 02798 int ch = (*_source)->get(); 02799 while (!(*_source)->eof() && !(*_source)->fail()) { 02800 switch (ch) { 02801 case '\n': 02802 // end-of-line character, we're done. 02803 str = _working_get; 02804 _working_get = string(); 02805 { 02806 // Trim trailing whitespace. We're not required to do this per the 02807 // HTTP spec, but let's be generous. 02808 size_t p = str.length(); 02809 while (p > 0 && isspace(str[p - 1])) { 02810 --p; 02811 } 02812 str = str.substr(0, p); 02813 } 02814 if (downloader_cat.is_debug()) { 02815 downloader_cat.debug() << "recv: " << str << "\n"; 02816 } 02817 return true; 02818 02819 case '\r': 02820 // Ignore CR characters. 02821 break; 02822 02823 default: 02824 _working_get += (char)ch; 02825 } 02826 ch = (*_source)->get(); 02827 } 02828 02829 check_socket(); 02830 return false; 02831 } 02832 02833 //////////////////////////////////////////////////////////////////// 02834 // Function: HTTPChannel::server_getline_failsafe 02835 // Access: Private 02836 // Description: Reads a line from the server's reply. If the server 02837 // disconnects or times out before sending a reply, 02838 // moves on to the next proxy server (or sets failure 02839 // mode) and returns false; otherwise, returns true. 02840 //////////////////////////////////////////////////////////////////// 02841 bool HTTPChannel:: 02842 server_getline_failsafe(string &str) { 02843 if (!server_getline(str)) { 02844 if (_bio.is_null()) { 02845 // Huh, the server hung up on us as soon as we tried to connect. 02846 if (_response_type == RT_hangup) { 02847 // This was our second immediate hangup in a row. Give up. 02848 _status_entry._status_code = SC_lost_connection; 02849 _state = S_try_next_proxy; 02850 02851 } else { 02852 // Try again, once. 02853 _response_type = RT_hangup; 02854 } 02855 02856 } else { 02857 double elapsed = 02858 TrueClock::get_global_ptr()->get_short_time() - 02859 _sent_request_time; 02860 if (elapsed > get_http_timeout()) { 02861 // Time to give up. 02862 downloader_cat.info() 02863 << "Timeout waiting for " 02864 << _request.get_url().get_server_and_port() 02865 << " in server_getline_failsafe (" << elapsed 02866 << " seconds elapsed).\n"; 02867 _status_entry._status_code = SC_timeout; 02868 _state = S_try_next_proxy; 02869 } 02870 } 02871 02872 return false; 02873 } 02874 return true; 02875 } 02876 02877 //////////////////////////////////////////////////////////////////// 02878 // Function: HTTPChannel::server_get 02879 // Access: Private 02880 // Description: Reads a fixed number of bytes from the server's 02881 // reply. Returns true if the indicated number of bytes 02882 // are successfully retrieved, or false if the complete 02883 // set has not yet been received or if the connection 02884 // has been closed. 02885 //////////////////////////////////////////////////////////////////// 02886 bool HTTPChannel:: 02887 server_get(string &str, size_t num_bytes) { 02888 nassertr(!_source.is_null(), false); 02889 int ch = (*_source)->get(); 02890 while (!(*_source)->eof() && !(*_source)->fail()) { 02891 _working_get += (char)ch; 02892 if (_working_get.length() >= num_bytes) { 02893 str = _working_get; 02894 _working_get = string(); 02895 return true; 02896 } 02897 02898 ch = (*_source)->get(); 02899 } 02900 02901 check_socket(); 02902 return false; 02903 } 02904 02905 //////////////////////////////////////////////////////////////////// 02906 // Function: HTTPChannel::server_get_failsafe 02907 // Access: Private 02908 // Description: Reads a fixed number of bytes from the server. If 02909 // the server disconnects or times out before sending a 02910 // reply, moves on to the next proxy server (or sets 02911 // failure mode) and returns false; otherwise, returns 02912 // true. 02913 //////////////////////////////////////////////////////////////////// 02914 bool HTTPChannel:: 02915 server_get_failsafe(string &str, size_t num_bytes) { 02916 if (!server_get(str, num_bytes)) { 02917 if (_bio.is_null()) { 02918 // Huh, the server hung up on us as soon as we tried to connect. 02919 if (_response_type == RT_hangup) { 02920 // This was our second immediate hangup in a row. Give up. 02921 _status_entry._status_code = SC_lost_connection; 02922 _state = S_try_next_proxy; 02923 02924 } else { 02925 // Try again, once. 02926 _response_type = RT_hangup; 02927 } 02928 02929 } else { 02930 double elapsed = 02931 TrueClock::get_global_ptr()->get_short_time() - 02932 _sent_request_time; 02933 if (elapsed > get_http_timeout()) { 02934 // Time to give up. 02935 downloader_cat.info() 02936 << "Timeout waiting for " 02937 << _request.get_url().get_server_and_port() 02938 << " in server_get_failsafe (" << elapsed 02939 << " seconds elapsed).\n"; 02940 _status_entry._status_code = SC_timeout; 02941 _state = S_try_next_proxy; 02942 } 02943 } 02944 02945 return false; 02946 } 02947 return true; 02948 } 02949 02950 //////////////////////////////////////////////////////////////////// 02951 // Function: HTTPChannel::server_send 02952 // Access: Private 02953 // Description: Sends a series of lines to the server. Returns true 02954 // if the buffer is fully sent, or false if some of it 02955 // remains. If this returns false, the function must be 02956 // called again later, passing in the exact same string, 02957 // until the return value is true. 02958 // 02959 // If the secret flag is true, the data is not echoed to 02960 // the log (even in spam mode). This may be desirable 02961 // if the data may contain binary data, or if it may 02962 // contain passwords etc. 02963 //////////////////////////////////////////////////////////////////// 02964 bool HTTPChannel:: 02965 server_send(const string &str, bool secret) { 02966 nassertr(str.length() > _sent_so_far, true); 02967 02968 // Use the underlying BIO to write to the server, instead of the 02969 // BIOStream, which would insist on blocking (and might furthermore 02970 // delay the send due to collect-tcp mode being enabled). 02971 size_t bytes_to_send = str.length() - _sent_so_far; 02972 int write_count = 02973 BIO_write(*_bio, str.data() + _sent_so_far, bytes_to_send); 02974 02975 if (write_count <= 0) { 02976 if (BIO_should_retry(*_bio)) { 02977 // Temporary failure: the pipe is full. Wait till later. 02978 return false; 02979 } 02980 // Oops, the connection has been closed! 02981 if (downloader_cat.is_debug()) { 02982 downloader_cat.debug() 02983 << "Lost connection to server unexpectedly during write.\n"; 02984 } 02985 reset_to_new(); 02986 return false; 02987 } 02988 02989 if (downloader_cat.is_spam()) { 02990 downloader_cat.spam() 02991 << "wrote " << write_count << " bytes to " << _bio << "\n"; 02992 } 02993 02994 #ifndef NDEBUG 02995 if (!secret && downloader_cat.is_debug()) { 02996 show_send(str.substr(0, write_count)); 02997 } 02998 #endif 02999 03000 if (write_count < (int)bytes_to_send) { 03001 _sent_so_far += write_count; 03002 return false; 03003 } 03004 03005 // Buffer completely sent. 03006 _sent_so_far = 0; 03007 return true; 03008 } 03009 03010 //////////////////////////////////////////////////////////////////// 03011 // Function: HTTPChannel::parse_http_response 03012 // Access: Private 03013 // Description: Parses the first line sent back from an HTTP server 03014 // or proxy and stores the result in _status_code and 03015 // _http_version, etc. Returns true on success, false 03016 // on invalid response. 03017 //////////////////////////////////////////////////////////////////// 03018 bool HTTPChannel:: 03019 parse_http_response(const string &line) { 03020 // The first line back should include the HTTP version and the 03021 // result code. 03022 if (line.length() < 5 || line.substr(0, 5) != string("HTTP/")) { 03023 // Not an HTTP response. 03024 _status_entry._status_code = SC_non_http_response; 03025 if (_response_type == RT_non_http) { 03026 // This was our second non-HTTP response in a row. Give up. 03027 _state = S_try_next_proxy; 03028 03029 } else { 03030 // Maybe we were just in some bad state. Drop the connection 03031 // and try again, once. 03032 if (downloader_cat.is_debug()) { 03033 downloader_cat.debug() 03034 << "got non-HTTP response, resetting.\n"; 03035 } 03036 reset_to_new(); 03037 _response_type = RT_non_http; 03038 } 03039 return false; 03040 } 03041 03042 // Split out the first line into its three components. 03043 size_t p = 5; 03044 while (p < line.length() && !isspace(line[p])) { 03045 p++; 03046 } 03047 _http_version_string = line.substr(0, p); 03048 _http_version = HTTPClient::parse_http_version_string(_http_version_string); 03049 03050 while (p < line.length() && isspace(line[p])) { 03051 p++; 03052 } 03053 size_t q = p; 03054 while (q < line.length() && !isspace(line[q])) { 03055 q++; 03056 } 03057 string status_code = line.substr(p, q - p); 03058 _status_entry._status_code = atoi(status_code.c_str()); 03059 03060 while (q < line.length() && isspace(line[q])) { 03061 q++; 03062 } 03063 _status_entry._status_string = line.substr(q, line.length() - q); 03064 03065 return true; 03066 } 03067 03068 //////////////////////////////////////////////////////////////////// 03069 // Function: HTTPChannel::parse_http_header 03070 // Access: Private 03071 // Description: Reads the series of header lines from the server and 03072 // stores them in _headers. Returns true if there is 03073 // more to read, false when done. 03074 //////////////////////////////////////////////////////////////////// 03075 bool HTTPChannel:: 03076 parse_http_header() { 03077 string line; 03078 if (!server_getline(line)) { 03079 return true; 03080 } 03081 03082 while (!line.empty()) { 03083 if (isspace(line[0])) { 03084 // If the line begins with a space, that continues the previous 03085 // field. 03086 size_t p = 0; 03087 while (p < line.length() && isspace(line[p])) { 03088 p++; 03089 } 03090 _current_field_value += line.substr(p - 1); 03091 03092 } else { 03093 // If the line does not begin with a space, that defines a new 03094 // field. 03095 if (!_current_field_name.empty()) { 03096 store_header_field(_current_field_name, _current_field_value); 03097 _current_field_value = string(); 03098 } 03099 03100 size_t colon = line.find(':'); 03101 if (colon != string::npos) { 03102 _current_field_name = downcase(line.substr(0, colon)); 03103 size_t p = colon + 1; 03104 while (p < line.length() && isspace(line[p])) { 03105 p++; 03106 } 03107 _current_field_value = line.substr(p); 03108 } 03109 } 03110 03111 if (!server_getline(line)) { 03112 return true; 03113 } 03114 } 03115 03116 // After reading an empty line, we're done with the headers. 03117 if (!_current_field_name.empty()) { 03118 store_header_field(_current_field_name, _current_field_value); 03119 _current_field_value = string(); 03120 } 03121 03122 return false; 03123 } 03124 03125 //////////////////////////////////////////////////////////////////// 03126 // Function: HTTPChannel::parse_content_range 03127 // Access: Private 03128 // Description: Interprets the "Content-Range" header in the reply, 03129 // and fills in _first_byte_delivered and 03130 // _last_byte_delivered appropriately if the header 03131 // response can be understood. 03132 //////////////////////////////////////////////////////////////////// 03133 bool HTTPChannel:: 03134 parse_content_range(const string &content_range) { 03135 // First, get the units indication. 03136 size_t p = 0; 03137 while (p < content_range.length() && !isspace(content_range[p])) { 03138 p++; 03139 } 03140 03141 string units = content_range.substr(0, p); 03142 while (p < content_range.length() && isspace(content_range[p])) { 03143 p++; 03144 } 03145 03146 if (units == "bytes") { 03147 const char *c_str = content_range.c_str(); 03148 char *endptr; 03149 if (p < content_range.length() && isdigit(content_range[p])) { 03150 long first_byte = strtol(c_str + p, &endptr, 10); 03151 p = endptr - c_str; 03152 if (p < content_range.length() && content_range[p] == '-') { 03153 p++; 03154 if (p < content_range.length() && isdigit(content_range[p])) { 03155 long last_byte = strtol(c_str + p, &endptr, 10); 03156 p = endptr - c_str; 03157 03158 if (last_byte >= first_byte) { 03159 _first_byte_delivered = first_byte; 03160 _last_byte_delivered = last_byte; 03161 return true; 03162 } 03163 } 03164 } 03165 } 03166 } 03167 03168 // Invalid or unhandled response. 03169 return false; 03170 } 03171 03172 03173 //////////////////////////////////////////////////////////////////// 03174 // Function: HTTPChannel::check_socket 03175 // Access: Private 03176 // Description: Checks whether the connection to the server has been 03177 // closed after a failed read. If it has, issues a 03178 // warning and calls reset_to_new(). 03179 //////////////////////////////////////////////////////////////////// 03180 void HTTPChannel:: 03181 check_socket() { 03182 nassertv(!_source.is_null()); 03183 if ((*_source)->is_closed()) { 03184 if (downloader_cat.is_debug()) { 03185 downloader_cat.debug() 03186 << "Lost connection to server unexpectedly during read.\n"; 03187 } 03188 reset_to_new(); 03189 } 03190 } 03191 03192 /* 03193 Certificate verify error codes: 03194 03195 0 X509_V_OK: ok 03196 03197 the operation was successful. 03198 03199 2 X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT: unable to get issuer certificate 03200 03201 the issuer certificate could not be found: this occurs if the 03202 issuer certificate of an untrusted certificate cannot be found. 03203 03204 3 X509_V_ERR_UNABLE_TO_GET_CRL unable to get certificate CRL 03205 03206 the CRL of a certificate could not be found. Unused. 03207 03208 4 X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE: unable to decrypt 03209 certificate's signature 03210 03211 the certificate signature could not be decrypted. This means that 03212 the actual signature value could not be determined rather than it 03213 not matching the expected value, this is only meaningful for RSA 03214 keys. 03215 03216 5 X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE: unable to decrypt CRL's signature 03217 03218 the CRL signature could not be decrypted: this means that the 03219 actual signature value could not be determined rather than it not 03220 matching the expected value. Unused. 03221 03222 6 X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY: unable to decode 03223 issuer public key 03224 03225 the public key in the certificate SubjectPublicKeyInfo could not 03226 be read. 03227 03228 7 X509_V_ERR_CERT_SIGNATURE_FAILURE: certificate signature failure 03229 03230 the signature of the certificate is invalid. 03231 03232 8 X509_V_ERR_CRL_SIGNATURE_FAILURE: CRL signature failure 03233 03234 the signature of the certificate is invalid. Unused. 03235 03236 9 X509_V_ERR_CERT_NOT_YET_VALID: certificate is not yet valid 03237 03238 the certificate is not yet valid: the notBefore date is after the 03239 current time. 03240 03241 10 X509_V_ERR_CERT_HAS_EXPIRED: certificate has expired 03242 03243 the certificate has expired: that is the notAfter date is before 03244 the current time. 03245 03246 11 X509_V_ERR_CRL_NOT_YET_VALID: CRL is not yet valid 03247 03248 the CRL is not yet valid. Unused. 03249 03250 12 X509_V_ERR_CRL_HAS_EXPIRED: CRL has expired 03251 03252 the CRL has expired. Unused. 03253 03254 13 X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD: format error in 03255 certificate's notBefore field 03256 03257 the certificate notBefore field contains an invalid time. 03258 03259 14 X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD: format error in 03260 certificate's notAfter field 03261 03262 the certificate notAfter field contains an invalid time. 03263 03264 15 X509_V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD: format error in CRL's 03265 lastUpdate field 03266 03267 the CRL lastUpdate field contains an invalid time. Unused. 03268 03269 16 X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD: format error in CRL's 03270 nextUpdate field 03271 03272 the CRL nextUpdate field contains an invalid time. Unused. 03273 03274 17 X509_V_ERR_OUT_OF_MEM: out of memory 03275 03276 an error occurred trying to allocate memory. This should never 03277 happen. 03278 03279 18 X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT: self signed certificate 03280 03281 the passed certificate is self signed and the same certificate 03282 cannot be found in the list of trusted certificates. 03283 03284 19 X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN: self signed certificate in 03285 certificate chain 03286 03287 the certificate chain could be built up using the untrusted 03288 certificates but the root could not be found locally. 03289 03290 20 X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY: unable to get local 03291 issuer certificate 03292 03293 the issuer certificate of a locally looked up certificate could 03294 not be found. This normally means the list of trusted certificates 03295 is not complete. 03296 03297 21 X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE: unable to verify the 03298 first certificate 03299 03300 no signatures could be verified because the chain contains only 03301 one certificate and it is not self signed. 03302 03303 22 X509_V_ERR_CERT_CHAIN_TOO_LONG: certificate chain too long 03304 03305 the certificate chain length is greater than the supplied maximum 03306 depth. Unused. 03307 03308 23 X509_V_ERR_CERT_REVOKED: certificate revoked 03309 03310 the certificate has been revoked. Unused. 03311 03312 24 X509_V_ERR_INVALID_CA: invalid CA certificate 03313 03314 a CA certificate is invalid. Either it is not a CA or its 03315 extensions are not consistent with the supplied purpose. 03316 03317 25 X509_V_ERR_PATH_LENGTH_EXCEEDED: path length constraint exceeded 03318 03319 the basicConstraints pathlength parameter has been exceeded. 03320 03321 26 X509_V_ERR_INVALID_PURPOSE: unsupported certificate purpose 03322 03323 the supplied certificate cannot be used for the specified purpose. 03324 03325 27 X509_V_ERR_CERT_UNTRUSTED: certificate not trusted 03326 03327 the root CA is not marked as trusted for the specified purpose. 03328 03329 28 X509_V_ERR_CERT_REJECTED: certificate rejected 03330 03331 the root CA is marked to reject the specified purpose. 03332 03333 29 X509_V_ERR_SUBJECT_ISSUER_MISMATCH: subject issuer mismatch 03334 03335 the current candidate issuer certificate was rejected because its 03336 subject name did not match the issuer name of the current 03337 certificate. Only displayed when the -issuer_checks option is set. 03338 03339 30 X509_V_ERR_AKID_SKID_MISMATCH: authority and subject key identifier 03340 mismatch 03341 03342 the current candidate issuer certificate was rejected because its 03343 subject key identifier was present and did not match the authority 03344 key identifier current certificate. Only displayed when the 03345 -issuer_checks option is set. 03346 03347 31 X509_V_ERR_AKID_ISSUER_SERIAL_MISMATCH: authority and issuer serial 03348 number mismatch 03349 03350 the current candidate issuer certificate was rejected because its 03351 issuer name and serial number was present and did not match the 03352 authority key identifier of the current certificate. Only 03353 displayed when the -issuer_checks option is set. 03354 03355 32 X509_V_ERR_KEYUSAGE_NO_CERTSIGN:key usage does not include 03356 certificate signing 03357 03358 the current candidate issuer certificate was rejected because its 03359 keyUsage extension does not permit certificate signing. 03360 03361 50 X509_V_ERR_APPLICATION_VERIFICATION: application verification failure 03362 03363 an application specific error. Unused. 03364 03365 */ 03366 03367 //////////////////////////////////////////////////////////////////// 03368 // Function: HTTPChannel::check_preapproved_server_certificate 03369 // Access: Private 03370 // Description: Checks to see if the indicated certificate is on the 03371 // pre-approved list for the current server. 03372 // 03373 // If the full cert itself (including its key) is on the 03374 // pre-approved list, sets both cert_preapproved and 03375 // cert_name_preapproved to true. 03376 // 03377 // If the full cert is not on the pre-approved list, but 03378 // its name matches a name on the pre-approved list, 03379 // sets cert_name_preapproved to true, and 03380 // cert_preapproved to false. 03381 // 03382 // Otherwise, sets both values to false. This doesn't 03383 // mean the cert is necessarily invalid, just that it 03384 // wasn't on the pre-approved list (which is usually 03385 // empty anyway). 03386 //////////////////////////////////////////////////////////////////// 03387 void HTTPChannel:: 03388 check_preapproved_server_certificate(X509 *cert, bool &cert_preapproved, 03389 bool &cert_name_preapproved) const { 03390 return _client->check_preapproved_server_certificate(_request.get_url(), 03391 cert, cert_preapproved, 03392 cert_name_preapproved); 03393 } 03394 03395 //////////////////////////////////////////////////////////////////// 03396 // Function: HTTPChannel::validate_server_name 03397 // Access: Private 03398 // Description: Returns true if the name in the cert matches the 03399 // hostname of the server, false otherwise. 03400 //////////////////////////////////////////////////////////////////// 03401 bool HTTPChannel:: 03402 validate_server_name(X509 *cert) { 03403 string hostname = _request.get_url().get_server(); 03404 03405 vector_string cert_names; 03406 03407 // According to RFC 2818, we should check the DNS name(s) in the 03408 // subjectAltName extension first, if that extension exists. 03409 STACK_OF(GENERAL_NAME) *subject_alt_names = 03410 (STACK_OF(GENERAL_NAME) *)X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL); 03411 if (subject_alt_names != NULL) { 03412 int num_alts = sk_GENERAL_NAME_num(subject_alt_names); 03413 for (int i = 0; i < num_alts; ++i) { 03414 // Get the ith alt name. 03415 const GENERAL_NAME *alt_name = 03416 sk_GENERAL_NAME_value(subject_alt_names, i); 03417 03418 if (alt_name->type == GEN_DNS) { 03419 char *buffer = NULL; 03420 int len = ASN1_STRING_to_UTF8((unsigned char**)&buffer, 03421 alt_name->d.ia5); 03422 if (len > 0) { 03423 cert_names.push_back(string(buffer, len)); 03424 } 03425 if (buffer != NULL) { 03426 OPENSSL_free(buffer); 03427 } 03428 } 03429 } 03430 } 03431 03432 if (cert_names.empty()) { 03433 // If there were no DNS names, use the common name instead. 03434 03435 X509_NAME *xname = X509_get_subject_name(cert); 03436 if (xname != NULL) { 03437 string common_name = get_x509_name_component(xname, NID_commonName); 03438 cert_names.push_back(common_name); 03439 } 03440 } 03441 03442 if (cert_names.empty()) { 03443 downloader_cat.info() 03444 << "Server certificate from " << hostname 03445 << " provides no name.\n"; 03446 return false; 03447 } 03448 03449 if (downloader_cat.is_debug()) { 03450 downloader_cat.debug() 03451 << "Server certificate from " << hostname 03452 << " provides name(s):"; 03453 vector_string::const_iterator si; 03454 for (si = cert_names.begin(); si != cert_names.end(); ++si) { 03455 const string &cert_name = (*si); 03456 downloader_cat.debug(false) 03457 << " " << cert_name; 03458 } 03459 downloader_cat.debug(false) 03460 << "\n"; 03461 } 03462 03463 // Now validate the names we found. If any of them matches, the 03464 // cert matches. 03465 vector_string::const_iterator si; 03466 for (si = cert_names.begin(); si != cert_names.end(); ++si) { 03467 const string &cert_name = (*si); 03468 03469 if (match_cert_name(cert_name, hostname)) { 03470 return true; 03471 } 03472 } 03473 03474 downloader_cat.info() 03475 << "Server certificate from " << hostname 03476 << " provides wrong name(s):"; 03477 for (si = cert_names.begin(); si != cert_names.end(); ++si) { 03478 const string &cert_name = (*si); 03479 downloader_cat.info(false) 03480 << " " << cert_name; 03481 } 03482 downloader_cat.info(false) 03483 << "\n"; 03484 03485 return false; 03486 } 03487 03488 //////////////////////////////////////////////////////////////////// 03489 // Function: HTTPChannel::match_cert_name 03490 // Access: Private, Static 03491 // Description: Returns true if this particular name from the 03492 // certificate matches the indicated hostname, false 03493 // otherwise. 03494 //////////////////////////////////////////////////////////////////// 03495 bool HTTPChannel:: 03496 match_cert_name(const string &cert_name, const string &hostname) { 03497 // We use GlobPattern to match the name. This isn't quite 03498 // consistent with RFC2818, since it also accepts additional 03499 // wildcard characters like "?" and "[]", but I think it's close 03500 // enough. 03501 03502 GlobPattern pattern(cert_name); 03503 pattern.set_case_sensitive(false); 03504 pattern.set_nomatch_chars("."); 03505 return pattern.matches(hostname); 03506 } 03507 03508 //////////////////////////////////////////////////////////////////// 03509 // Function: HTTPChannel::get_x509_name_component 03510 // Access: Private, Static 03511 // Description: Returns the indicated component of the X509 name as a 03512 // string, if defined, or empty string if it is not. 03513 //////////////////////////////////////////////////////////////////// 03514 string HTTPChannel:: 03515 get_x509_name_component(X509_NAME *name, int nid) { 03516 ASN1_OBJECT *obj = OBJ_nid2obj(nid); 03517 03518 if (obj == NULL) { 03519 // Unknown nid. See openssl/objects.h. 03520 return string(); 03521 } 03522 03523 int i = X509_NAME_get_index_by_OBJ(name, obj, -1); 03524 if (i < 0) { 03525 return string(); 03526 } 03527 03528 ASN1_STRING *data = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(name, i)); 03529 return string((char *)data->data, data->length); 03530 } 03531 03532 //////////////////////////////////////////////////////////////////// 03533 // Function: HTTPChannel::make_header 03534 // Access: Private 03535 // Description: Formats the appropriate GET or POST (or whatever) 03536 // request to send to the server, based on the current 03537 // _method, _document_spec, _body, and _proxy settings. 03538 //////////////////////////////////////////////////////////////////// 03539 void HTTPChannel:: 03540 make_header() { 03541 _proxy_auth = _client->select_auth(_proxy, true, _proxy_realm); 03542 _proxy_username = string(); 03543 if (_proxy_auth != (HTTPAuthorization *)NULL) { 03544 _proxy_realm = _proxy_auth->get_realm(); 03545 _proxy_username = _client->select_username(_proxy, true, _proxy_realm); 03546 } 03547 03548 if (_method == HTTPEnum::M_connect) { 03549 // This method doesn't require an HTTP header at all; we'll just 03550 // open a plain connection. (Except when we're using a proxy; but 03551 // in that case, it's the proxy_header we'll need, not the regular 03552 // HTTP header.) 03553 _header = string(); 03554 return; 03555 } 03556 03557 _www_auth = _client->select_auth(_request.get_url(), false, _www_realm); 03558 _www_username = string(); 03559 if (_www_auth != (HTTPAuthorization *)NULL) { 03560 _www_realm = _www_auth->get_realm(); 03561 _www_username = _client->select_username(_request.get_url(), false, _www_realm); 03562 } 03563 03564 string request_path; 03565 if (_proxy_serves_document) { 03566 // If we'll be asking the proxy for the document, we need its full 03567 // URL--but we omit the username, which is information just for us. 03568 URLSpec url_no_username = _request.get_url(); 03569 url_no_username.set_username(string()); 03570 request_path = url_no_username.get_url(); 03571 03572 } else { 03573 // If we'll be asking the server directly for the document, we 03574 // just want its path relative to the server. 03575 request_path = _request.get_url().get_path_and_query(); 03576 } 03577 03578 // HTTP syntax always requires something in the request path. If it 03579 // is empty, put in a star as a placeholder (OPTIONS, for instance, 03580 // uses this). 03581 if (request_path.empty()) { 03582 request_path = "*"; 03583 } 03584 03585 ostringstream stream; 03586 03587 stream 03588 << _method << " " << request_path << " " 03589 << _client->get_http_version_string() << "\r\n"; 03590 03591 if (_client->get_http_version() >= HTTPEnum::HV_11) { 03592 03593 stream 03594 << "Host: " << _request.get_url().get_server(); 03595 if (!_request.get_url().is_default_port()) { 03596 // It appears that some servers (notably gstatic.com) might 03597 // return a 404 if you include an explicit port number in with 03598 // the Host: header, even if it is the default port. So, don't 03599 // include the port number unless we need to. 03600 stream << ":" << _request.get_url().get_port(); 03601 } 03602 stream << "\r\n"; 03603 if (!get_persistent_connection()) { 03604 stream 03605 << "Connection: close\r\n"; 03606 } 03607 } 03608 03609 if (_last_byte_requested != 0) { 03610 stream 03611 << "Range: bytes=" << _first_byte_requested << "-" 03612 << _last_byte_requested << "\r\n"; 03613 03614 } else if (_first_byte_requested != 0) { 03615 stream 03616 << "Range: bytes=" << _first_byte_requested << "-\r\n"; 03617 } 03618 03619 switch (_request.get_request_mode()) { 03620 case DocumentSpec::RM_any: 03621 // No particular request; give us any document that matches the 03622 // URL. 03623 if (_first_byte_requested != 0) { 03624 // Unless we're requesting a subrange, in which case if the 03625 // exact document matches, retrieve the subrange indicated; 03626 // otherwise, retrieve the entire document. 03627 if (_request.has_tag()) { 03628 stream 03629 << "If-Range: " << _request.get_tag().get_string() << "\r\n"; 03630 } else if (_request.has_date()) { 03631 stream 03632 << "If-Range: " << _request.get_date().get_string() << "\r\n"; 03633 } 03634 } 03635 break; 03636 03637 case DocumentSpec::RM_equal: 03638 // Give us only this particular version of the document, or 03639 // nothing. 03640 if (_request.has_tag()) { 03641 stream 03642 << "If-Match: " << _request.get_tag().get_string() << "\r\n"; 03643 } 03644 if (_request.has_date()) { 03645 stream 03646 << "If-Unmodified-Since: " << _request.get_date().get_string() 03647 << "\r\n"; 03648 } 03649 break; 03650 03651 case DocumentSpec::RM_newer: 03652 // Give us anything newer than this document, or nothing. 03653 if (_request.has_tag()) { 03654 stream 03655 << "If-None-Match: " << _request.get_tag().get_string() << "\r\n"; 03656 } 03657 if (_request.has_date()) { 03658 stream 03659 << "If-Modified-Since: " << _request.get_date().get_string() 03660 << "\r\n"; 03661 } 03662 break; 03663 03664 case DocumentSpec::RM_equal_or_newer: 03665 // Just don't give us anything older. 03666 if (_request.has_date()) { 03667 // This is a little unreliable: we ask for any document that's 03668 // been modified since one second before our last-modified-date. 03669 // Who knows whether the server will honor this properly. 03670 stream 03671 << "If-Modified-Since: " << (_request.get_date() - 1).get_string() 03672 << "\r\n"; 03673 } 03674 break; 03675 } 03676 03677 switch (_request.get_cache_control()) { 03678 case DocumentSpec::CC_allow_cache: 03679 // Normal, caching behavior. 03680 break; 03681 03682 case DocumentSpec::CC_revalidate: 03683 // Request the server to revalidate its cache before returning it. 03684 stream 03685 << "Cache-Control: max-age=0\r\n"; 03686 break; 03687 03688 case DocumentSpec::CC_no_cache: 03689 // Request the server to get a fresh copy regardless of its cache. 03690 stream 03691 << "Cache-Control: no-cache\r\n" 03692 << "Pragma: no-cache\r\n"; 03693 break; 03694 } 03695 03696 _client->send_cookies(stream, _request.get_url()); 03697 03698 if (!_body.empty()) { 03699 stream 03700 << "Content-Type: application/x-www-form-urlencoded\r\n" 03701 << "Content-Length: " << _body.length() << "\r\n"; 03702 } 03703 03704 _header = stream.str(); 03705 } 03706 03707 //////////////////////////////////////////////////////////////////// 03708 // Function: HTTPChannel::make_proxy_request_text 03709 // Access: Private 03710 // Description: Builds the _proxy_request_text string. This is a 03711 // special request that will be sent directly to the 03712 // proxy prior to the request tailored for the server. 03713 // Generally this is used to open a tunnelling 03714 // connection for https-over-proxy. 03715 //////////////////////////////////////////////////////////////////// 03716 void HTTPChannel:: 03717 make_proxy_request_text() { 03718 _proxy_request_text = _proxy_header; 03719 03720 if (_proxy_auth != (HTTPAuthorization *)NULL && !_proxy_username.empty()) { 03721 _proxy_request_text += "Proxy-Authorization: "; 03722 _proxy_request_text += 03723 _proxy_auth->generate(HTTPEnum::M_connect, _request.get_url().get_server_and_port(), 03724 _proxy_username, _body); 03725 _proxy_request_text += "\r\n"; 03726 } 03727 03728 _proxy_request_text += "\r\n"; 03729 } 03730 03731 //////////////////////////////////////////////////////////////////// 03732 // Function: HTTPChannel::make_request_text 03733 // Access: Private 03734 // Description: Builds the _request_text string. This is the 03735 // specific request that will be sent to the server this 03736 // pass, based on the current header and body. 03737 //////////////////////////////////////////////////////////////////// 03738 void HTTPChannel:: 03739 make_request_text() { 03740 _request_text = _header; 03741 03742 if (_proxy_serves_document && 03743 _proxy_auth != (HTTPAuthorization *)NULL && !_proxy_username.empty()) { 03744 _request_text += "Proxy-Authorization: "; 03745 _request_text += 03746 _proxy_auth->generate(_method, _request.get_url().get_url(), _proxy_username, _body); 03747 _request_text += "\r\n"; 03748 } 03749 03750 if (_www_auth != (HTTPAuthorization *)NULL && !_www_username.empty()) { 03751 string authorization = 03752 _request_text += "Authorization: "; 03753 _request_text += 03754 _www_auth->generate(_method, _request.get_url().get_path_and_query(), _www_username, _body); 03755 _request_text += "\r\n"; 03756 } 03757 03758 _request_text += _send_extra_headers; 03759 _request_text += "\r\n"; 03760 _request_text += _body; 03761 } 03762 03763 //////////////////////////////////////////////////////////////////// 03764 // Function: HTTPChannel::reset_url 03765 // Access: Private 03766 // Description: Redirects the next connection to the indicated URL 03767 // (from the previous URL). This resets the socket if 03768 // necessary when we are about to switch servers. 03769 //////////////////////////////////////////////////////////////////// 03770 void HTTPChannel:: 03771 reset_url(const URLSpec &old_url, const URLSpec &new_url) { 03772 // If we change between http and https, we have to reset the 03773 // connection regardless of proxy. Otherwise, we have to drop the 03774 // connection if the server or port changes, unless we're 03775 // communicating through a proxy. 03776 03777 if (new_url.get_scheme() != old_url.get_scheme() || 03778 (_proxy.empty() && (new_url.get_server() != old_url.get_server() || 03779 new_url.get_port() != old_url.get_port()))) { 03780 if (downloader_cat.is_debug()) { 03781 downloader_cat.debug() 03782 << "resetting for new server " 03783 << new_url.get_server_and_port() << "\n"; 03784 } 03785 reset_to_new(); 03786 } 03787 } 03788 03789 //////////////////////////////////////////////////////////////////// 03790 // Function: HTTPChannel::store_header_field 03791 // Access: Private 03792 // Description: Stores a single name: value pair in the header list, 03793 // or appends the value to the end of the existing 03794 // value, if the header has been repeated. 03795 //////////////////////////////////////////////////////////////////// 03796 void HTTPChannel:: 03797 store_header_field(const string &field_name, const string &field_value) { 03798 pair<Headers::iterator, bool> insert_result = 03799 _headers.insert(Headers::value_type(field_name, field_value)); 03800 03801 if (!insert_result.second) { 03802 // It didn't insert; thus, the field already existed. Append the 03803 // new value. 03804 Headers::iterator hi = insert_result.first; 03805 (*hi).second += ", "; 03806 (*hi).second += field_value; 03807 } 03808 03809 if (field_name == "set-cookie") { 03810 _client->set_cookie(HTTPCookie(field_value, _request.get_url())); 03811 } 03812 } 03813 03814 #ifndef NDEBUG 03815 //////////////////////////////////////////////////////////////////// 03816 // Function: HTTPChannel::show_send 03817 // Access: Private, Static 03818 // Description: Writes the outgoing message, one line at a time, to 03819 // the debugging log. 03820 //////////////////////////////////////////////////////////////////// 03821 void HTTPChannel:: 03822 show_send(const string &message) { 03823 size_t start = 0; 03824 size_t newline = message.find('\n', start); 03825 while (newline != string::npos) { 03826 // Assume every \n is preceded by a \r. 03827 downloader_cat.debug() 03828 << "send: " << message.substr(start, newline - start - 1) << "\n"; 03829 start = newline + 1; 03830 newline = message.find('\n', start); 03831 } 03832 03833 if (start < message.length()) { 03834 downloader_cat.debug() 03835 << "send: " << message.substr(start) << " (no newline)\n"; 03836 } 03837 } 03838 #endif // NDEBUG 03839 03840 //////////////////////////////////////////////////////////////////// 03841 // Function: HTTPChannel::reset_download_to 03842 // Access: Private 03843 // Description: Resets the indication of how the document will be 03844 // downloaded. This must be re-specified after each 03845 // get_document() (or related) call. 03846 //////////////////////////////////////////////////////////////////// 03847 void HTTPChannel:: 03848 reset_download_to() { 03849 _started_download = false; 03850 _download_to_file.close(); 03851 _download_to_ramfile = (Ramfile *)NULL; 03852 _download_to_stream = NULL; 03853 _download_dest = DD_none; 03854 } 03855 03856 //////////////////////////////////////////////////////////////////// 03857 // Function: HTTPChannel::reset_to_new 03858 // Access: Private 03859 // Description: Closes the connection and resets the state to S_new. 03860 //////////////////////////////////////////////////////////////////// 03861 void HTTPChannel:: 03862 reset_to_new() { 03863 close_connection(); 03864 _state = S_new; 03865 } 03866 03867 //////////////////////////////////////////////////////////////////// 03868 // Function: HTTPChannel::reset_body_stream 03869 // Access: Private 03870 // Description: Clears the _body_stream pointer, if it is set. 03871 //////////////////////////////////////////////////////////////////// 03872 void HTTPChannel:: 03873 reset_body_stream() { 03874 if (_owns_body_stream) { 03875 if (_body_stream != (ISocketStream *)NULL) { 03876 close_read_body(_body_stream); 03877 nassertv(_body_stream == (ISocketStream *)NULL && !_owns_body_stream); 03878 } 03879 } else { 03880 _body_stream = NULL; 03881 } 03882 } 03883 03884 03885 //////////////////////////////////////////////////////////////////// 03886 // Function: HTTPChannel::close_connection 03887 // Access: Private 03888 // Description: Closes the connection but leaves the _state 03889 // unchanged. 03890 //////////////////////////////////////////////////////////////////// 03891 void HTTPChannel:: 03892 close_connection() { 03893 reset_body_stream(); 03894 _source.clear(); 03895 _bio.clear(); 03896 _working_get = string(); 03897 _sent_so_far = 0; 03898 _read_index++; 03899 } 03900 03901 //////////////////////////////////////////////////////////////////// 03902 // Function: HTTPChannel::more_useful_status_code 03903 // Access: Private, Static 03904 // Description: Returns true if status code a is a more useful value 03905 // (that is, it represents a more-nearly successfully 03906 // connection attempt, or contains more information) 03907 // than b, or false otherwise. 03908 //////////////////////////////////////////////////////////////////// 03909 bool HTTPChannel:: 03910 more_useful_status_code(int a, int b) { 03911 if (a >= 100 && b >= 100) { 03912 // Both represent HTTP responses. Responses from a server (< 03913 // 1000) are better than those from a proxy; we take advantage of 03914 // the fact that we have already added 1000 to proxy responses. 03915 // Except for 407, so let's fix that now. 03916 if (a == 407) { 03917 a += 1000; 03918 } 03919 if (b == 407) { 03920 b += 1000; 03921 } 03922 03923 // Now just check the series. 03924 int series_a = (a / 100); 03925 int series_b = (b / 100); 03926 03927 // In general, a lower series is a closer success. 03928 return (series_a < series_b); 03929 } 03930 03931 if (a < 100 && b < 100) { 03932 // Both represent non-HTTP responses. Here a larger number is 03933 // better. 03934 return (a > b); 03935 } 03936 03937 if (a < 100) { 03938 // a is a non-HTTP response, while b is an HTTP response. HTTP is 03939 // generally, better, unless we exceeded SC_http_error_watermark. 03940 return (a > SC_http_error_watermark); 03941 } 03942 03943 // Exactly the opposite case as above. 03944 return (b < SC_http_error_watermark); 03945 } 03946 03947 03948 //////////////////////////////////////////////////////////////////// 03949 // Function: HTTPChannel::State output operator 03950 // Description: 03951 //////////////////////////////////////////////////////////////////// 03952 ostream & 03953 operator << (ostream &out, HTTPChannel::State state) { 03954 #ifdef NDEBUG 03955 return out << (int)state; 03956 #else 03957 switch (state) { 03958 case HTTPChannel::S_new: 03959 return out << "new"; 03960 03961 case HTTPChannel::S_try_next_proxy: 03962 return out << "try_next_proxy"; 03963 03964 case HTTPChannel::S_connecting: 03965 return out << "connecting"; 03966 03967 case HTTPChannel::S_connecting_wait: 03968 return out << "connecting_wait"; 03969 03970 case HTTPChannel::S_http_proxy_ready: 03971 return out << "http_proxy_ready"; 03972 03973 case HTTPChannel::S_http_proxy_request_sent: 03974 return out << "http_proxy_request_sent"; 03975 03976 case HTTPChannel::S_http_proxy_reading_header: 03977 return out << "http_proxy_reading_header"; 03978 03979 case HTTPChannel::S_socks_proxy_greet: 03980 return out << "socks_proxy_greet"; 03981 03982 case HTTPChannel::S_socks_proxy_greet_reply: 03983 return out << "socks_proxy_greet_reply"; 03984 03985 case HTTPChannel::S_socks_proxy_connect: 03986 return out << "socks_proxy_connect"; 03987 03988 case HTTPChannel::S_socks_proxy_connect_reply: 03989 return out << "socks_proxy_connect_reply"; 03990 03991 case HTTPChannel::S_setup_ssl: 03992 return out << "setup_ssl"; 03993 03994 case HTTPChannel::S_ssl_handshake: 03995 return out << "ssl_handshake"; 03996 03997 case HTTPChannel::S_ready: 03998 return out << "ready"; 03999 04000 case HTTPChannel::S_request_sent: 04001 return out << "request_sent"; 04002 04003 case HTTPChannel::S_reading_header: 04004 return out << "reading_header"; 04005 04006 case HTTPChannel::S_start_direct_file_read: 04007 return out << "start_direct_file_read"; 04008 04009 case HTTPChannel::S_read_header: 04010 return out << "read_header"; 04011 04012 case HTTPChannel::S_begin_body: 04013 return out << "begin_body"; 04014 04015 case HTTPChannel::S_reading_body: 04016 return out << "reading_body"; 04017 04018 case HTTPChannel::S_read_body: 04019 return out << "read_body"; 04020 04021 case HTTPChannel::S_read_trailer: 04022 return out << "read_trailer"; 04023 04024 case HTTPChannel::S_failure: 04025 return out << "failure"; 04026 } 04027 04028 return out << "invalid state(" << (int)state << ")"; 04029 #endif // NDEBUG 04030 } 04031 04032 #endif // HAVE_OPENSSL