15 #include "httpChannel.h" 16 #include "httpClient.h" 17 #include "httpCookie.h" 18 #include "bioStream.h" 19 #include "chunkedStream.h" 20 #include "identityStream.h" 21 #include "config_downloader.h" 22 #include "virtualFileSystem.h" 23 #include "virtualFileMountHTTP.h" 25 #include "globPattern.h" 31 #if defined(WIN32_VC) || defined(WIN64_VC) 39 #define _NOTIFY_HTTP_CHANNEL_ID "[" << this << "] " 47 HTTPChannel(HTTPClient *client) :
50 if (downloader_cat.is_debug()) {
51 downloader_cat.debug()
52 << _NOTIFY_HTTP_CHANNEL_ID
57 (
"extra-ssl-handshake-time", 0.0,
58 PRC_DESC(
"This specifies how much extra time to try to establish" 59 "the ssl handshake before we bail."));
60 _extra_ssl_handshake_time = extra_ssl_handshake_time;
61 _proxy_next_index = 0;
62 _persistent_connection =
false;
64 _proxy_tunnel = http_proxy_tunnel;
65 _connect_timeout = http_connect_timeout;
66 _http_timeout = http_timeout;
67 _skip_body_size = http_skip_body_size;
68 _idle_timeout = http_idle_timeout;
69 _blocking_connect =
false;
70 _download_throttle = download_throttle;
71 _max_bytes_per_second = downloader_byte_rate;
72 _seconds_per_update = downloader_frequency;
73 _max_updates_per_second = 1.0f / _seconds_per_update;
74 _bytes_per_update = int(_max_bytes_per_second * _seconds_per_update);
83 _wanted_nonblocking =
false;
86 _proxy_serves_document =
false;
87 _proxy_tunnel_now =
false;
88 _first_byte_requested = 0;
89 _last_byte_requested = 0;
90 _first_byte_delivered = 0;
91 _last_byte_delivered = 0;
93 _expected_file_size = 0;
95 _transfer_file_size = 0;
96 _got_expected_file_size =
false;
97 _got_file_size =
false;
98 _got_transfer_file_size =
false;
99 _bytes_downloaded = 0;
100 _bytes_requested = 0;
101 _status_entry = StatusEntry();
102 _response_type = RT_none;
103 _http_version = _client->get_http_version();
104 _http_version_string = _client->get_http_version_string();
107 _started_download =
false;
110 _owns_body_stream =
false;
112 _cipher_list = _client->get_cipher_list();
113 _last_status_code = 0;
114 _last_run_time = 0.0f;
115 _download_to_ramfile = NULL;
116 _download_to_stream = NULL;
126 if (downloader_cat.is_debug()) {
127 downloader_cat.debug()
128 << _NOTIFY_HTTP_CHANNEL_ID
144 get_status_string()
const {
145 switch (_status_entry._status_code) {
147 return "Connection in progress";
149 case SC_internal_error:
150 return "Internal error";
152 case SC_no_connection:
153 return "No connection";
156 return "Timeout on connection";
158 case SC_lost_connection:
159 return "Lost connection";
161 case SC_non_http_response:
162 return "Non-HTTP response";
164 case SC_invalid_http:
165 return "Could not understand HTTP response";
167 case SC_socks_invalid_version:
168 return "Unsupported SOCKS version";
170 case SC_socks_no_acceptable_login_method:
171 return "No acceptable SOCKS login method";
173 case SC_socks_refused:
174 return "SOCKS proxy refused connection";
176 case SC_socks_no_connection:
177 return "SOCKS proxy unable to connect";
179 case SC_ssl_internal_failure:
180 return "SSL internal failure";
182 case SC_ssl_no_handshake:
183 return "No SSL handshake";
185 case SC_http_error_watermark:
187 return "Internal error";
189 case SC_ssl_invalid_server_certificate:
190 return "SSL invalid server certificate";
192 case SC_ssl_unexpected_server:
193 return "Unexpected SSL server";
195 case SC_download_open_error:
196 return "Error opening file";
198 case SC_download_write_error:
199 return "Error writing to disk";
201 case SC_download_invalid_range:
202 return "Invalid subrange requested";
205 return _status_entry._status_string;
216 get_header_value(
const string &key)
const {
217 Headers::const_iterator hi = _headers.find(downcase(key));
218 if (hi != _headers.end()) {
233 will_close_connection()
const {
234 if (get_http_version() < HTTPEnum::HV_11) {
239 string connection = get_header_value(
"Connection");
240 if (downcase(connection) ==
"close") {
245 if (connection.empty() && !get_persistent_connection()) {
270 streamsize HTTPChannel::
271 get_file_size()
const {
272 if (_got_file_size) {
274 }
else if (_got_transfer_file_size) {
275 return _transfer_file_size;
276 }
else if (_got_expected_file_size) {
277 return _expected_file_size;
290 write_headers(ostream &out)
const {
291 Headers::const_iterator hi;
292 for (hi = _headers.begin(); hi != _headers.end(); ++hi) {
293 out << (*hi).first <<
": " << (*hi).second <<
"\n";
312 if (downloader_cat.is_spam()) {
313 downloader_cat.spam()
314 << _NOTIFY_HTTP_CHANNEL_ID
318 if (_state == _done_state || _state == S_failure) {
319 clear_extra_headers();
320 if (!reached_done_state()) {
325 if (_started_download) {
326 if (_wanted_nonblocking && _download_throttle) {
328 double elapsed = now - _last_run_time;
329 if (elapsed < _seconds_per_update) {
334 int num_potential_updates = (int)(elapsed / _seconds_per_update);
335 _last_run_time = now;
336 _bytes_requested += _bytes_per_update * num_potential_updates;
337 if (downloader_cat.is_spam()) {
338 downloader_cat.spam()
339 << _NOTIFY_HTTP_CHANNEL_ID
340 <<
"elapsed = " << elapsed <<
" num_potential_updates = " 341 << num_potential_updates <<
" bytes_requested = " 342 << _bytes_requested <<
"\n";
346 bool repeat_later =
false;
347 switch (_download_dest) {
353 repeat_later = run_download_to_file();
357 repeat_later = run_download_to_ram();
361 repeat_later = run_download_to_stream();
379 if (_state == _done_state) {
380 return reached_done_state();
388 if (_bio.is_null() && _state != S_try_next_proxy) {
389 if (_connect_count > http_max_connect_count) {
394 downloader_cat.warning()
395 << _NOTIFY_HTTP_CHANNEL_ID
396 <<
"Too many lost connections, giving up.\n";
397 _status_entry._status_code = SC_lost_connection;
403 if (_proxy.empty()) {
404 _bio =
new BioPtr(_request.get_url());
406 _bio =
new BioPtr(_proxy);
408 _source =
new BioStreamPtr(
new BioStream(_bio));
410 BIO_set_nbio(*_bio, 1);
413 if (downloader_cat.is_debug()) {
414 if (_connect_count > 0) {
415 downloader_cat.debug()
416 << _NOTIFY_HTTP_CHANNEL_ID
417 <<
"Reconnecting to " << _bio->get_server_name() <<
":" 418 << _bio->get_port() <<
"\n";
420 downloader_cat.debug()
421 << _NOTIFY_HTTP_CHANNEL_ID
422 <<
"Connecting to " << _bio->get_server_name() <<
":" 423 << _bio->get_port() <<
"\n";
427 _state = S_connecting;
428 _started_connecting_time =
442 case S_try_next_proxy:
443 repeat_later = run_try_next_proxy();
447 repeat_later = run_connecting();
450 case S_connecting_wait:
451 repeat_later = run_connecting_wait();
454 case S_http_proxy_ready:
455 repeat_later = run_http_proxy_ready();
458 case S_http_proxy_request_sent:
459 repeat_later = run_http_proxy_request_sent();
462 case S_http_proxy_reading_header:
463 repeat_later = run_http_proxy_reading_header();
466 case S_socks_proxy_greet:
467 repeat_later = run_socks_proxy_greet();
470 case S_socks_proxy_greet_reply:
471 repeat_later = run_socks_proxy_greet_reply();
474 case S_socks_proxy_connect:
475 repeat_later = run_socks_proxy_connect();
478 case S_socks_proxy_connect_reply:
479 repeat_later = run_socks_proxy_connect_reply();
483 repeat_later = run_setup_ssl();
486 case S_ssl_handshake:
487 repeat_later = run_ssl_handshake();
491 repeat_later = run_ready();
495 repeat_later = run_request_sent();
498 case S_reading_header:
499 repeat_later = run_reading_header();
502 case S_start_direct_file_read:
503 repeat_later = run_start_direct_file_read();
507 repeat_later = run_read_header();
511 repeat_later = run_begin_body();
515 repeat_later = run_reading_body();
519 repeat_later = run_read_body();
523 repeat_later = run_read_trailer();
527 downloader_cat.warning()
528 << _NOTIFY_HTTP_CHANNEL_ID
529 <<
"Unhandled state " << _state <<
"\n";
533 if (_state == _done_state || _state == S_failure) {
534 clear_extra_headers();
536 return reached_done_state();
538 thread_consider_yield();
539 }
while (!repeat_later || _bio.is_null());
572 ISocketStream *HTTPChannel::
576 if ((_state != S_read_header && _state != S_begin_body) || _source.is_null()) {
580 string transfer_coding = downcase(get_header_value(
"Transfer-Encoding"));
582 ISocketStream *result;
583 if (transfer_coding ==
"chunked") {
587 _state = S_reading_body;
589 result =
new IChunkedStream(_source,
this);
596 _state = S_reading_body;
598 result =
new IIdentityStream(_source,
this, _got_file_size, _file_size);
601 result->_channel =
this;
602 _body_stream = result;
603 _owns_body_stream =
false;
618 close_read_body(istream *stream)
const {
619 if (stream != (istream *)NULL) {
624 #if !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW) 626 (*global_operator_delete)(stream);
668 download_to_file(
const Filename &filename,
bool subdocument_resumes) {
670 _download_to_filename = filename;
672 _subdocument_resumes = subdocument_resumes;
674 _download_dest = DD_file;
676 if (_wanted_nonblocking && _state != S_read_header) {
683 if (!open_download_file()) {
690 return is_download_complete() && is_valid();
725 download_to_ram(
Ramfile *ramfile,
bool subdocument_resumes) {
726 nassertr(ramfile != (
Ramfile *)NULL,
false);
729 _download_to_ramfile = ramfile;
730 _download_dest = DD_ram;
731 _subdocument_resumes = (subdocument_resumes && _first_byte_delivered != 0);
733 if (_wanted_nonblocking && _state != S_read_header) {
740 if (!open_download_file()) {
747 return is_download_complete() && is_valid();
785 download_to_stream(ostream *strm,
bool subdocument_resumes) {
787 _download_to_stream = strm;
788 _download_to_stream->clear();
789 _subdocument_resumes = subdocument_resumes;
791 _download_dest = DD_stream;
793 if (_wanted_nonblocking && _state != S_read_header) {
800 if (!open_download_file()) {
807 return is_download_complete() && is_valid();
822 SocketStream *HTTPChannel::
824 if (!is_connection_ready()) {
828 BioStream *stream = _source->get_stream();
829 _source->set_stream(NULL);
832 if (downloader_cat.is_debug()) {
833 downloader_cat.debug()
834 << _NOTIFY_HTTP_CHANNEL_ID
835 <<
"passing ownership of connection to caller.\n";
849 downcase(
const string &s) {
851 result.reserve(s.size());
852 string::const_iterator p;
853 for (p = s.begin(); p != s.end(); ++p) {
854 result += tolower(*p);
866 body_stream_destructs(ISocketStream *stream) {
867 if (stream == _body_stream) {
868 if (_state == S_reading_body) {
869 switch (_body_stream->get_read_state()) {
870 case ISocketStream::RS_complete:
871 finished_body(
false);
874 case ISocketStream::RS_error:
875 _state = HTTPChannel::S_failure;
876 _status_entry._status_code = HTTPChannel::SC_lost_connection;
884 _owns_body_stream =
false;
897 reached_done_state() {
907 if (_state == S_failure) {
911 if (!_status_list.empty()) {
912 _status_list.push_back(_status_entry);
913 if (downloader_cat.is_debug()) {
914 downloader_cat.debug()
915 << _NOTIFY_HTTP_CHANNEL_ID
916 <<
"Reexamining failure responses.\n";
919 if (downloader_cat.is_debug()) {
920 downloader_cat.debug()
921 << _NOTIFY_HTTP_CHANNEL_ID
922 <<
" " << 0 <<
". " << _status_list[0]._status_code <<
" " 923 << _status_list[0]._status_string <<
"\n";
925 for (
size_t i = 1; i < _status_list.size(); i++) {
926 if (downloader_cat.is_debug()) {
927 downloader_cat.debug()
928 << _NOTIFY_HTTP_CHANNEL_ID
929 <<
" " << i <<
". " << _status_list[i]._status_code <<
" " 930 << _status_list[i]._status_string <<
"\n";
932 if (more_useful_status_code(_status_list[i]._status_code,
933 _status_list[best_i]._status_code)) {
937 if (downloader_cat.is_debug()) {
938 downloader_cat.debug()
939 << _NOTIFY_HTTP_CHANNEL_ID
940 <<
"chose index " << best_i <<
", above.\n";
942 _status_entry = _status_list[best_i];
943 _status_list.clear();
950 _status_list.clear();
952 if (_download_dest == DD_none) {
959 if (_body_stream == (ISocketStream *)NULL) {
960 if (downloader_cat.is_debug()) {
961 downloader_cat.debug()
962 << _NOTIFY_HTTP_CHANNEL_ID
963 <<
"Unable to download body: " << _request.get_url() <<
"\n";
968 _owns_body_stream =
true;
969 if (_state != S_reading_body) {
972 _started_download =
true;
974 _done_state = S_read_trailer;
991 run_try_next_proxy() {
992 if (_proxy_next_index < _proxies.size()) {
995 _status_list.push_back(_status_entry);
996 _status_entry = StatusEntry();
999 _proxy = _proxies[_proxy_next_index];
1000 _proxy_auth = (HTTPAuthorization *)NULL;
1001 _proxy_next_index++;
1004 _state = S_connecting;
1022 _status_entry = StatusEntry();
1024 if (BIO_do_connect(*_bio) <= 0) {
1025 if (BIO_should_retry(*_bio)) {
1026 _state = S_connecting_wait;
1029 downloader_cat.info()
1030 << _NOTIFY_HTTP_CHANNEL_ID
1031 <<
"Could not connect to " << _bio->get_server_name() <<
":" 1032 << _bio->get_port() <<
"\n";
1033 OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
1034 _status_entry._status_code = SC_no_connection;
1035 _state = S_try_next_proxy;
1039 if (downloader_cat.is_debug()) {
1040 downloader_cat.debug()
1041 << _NOTIFY_HTTP_CHANNEL_ID
1042 <<
"Connected to " << _bio->get_server_name() <<
":" 1043 << _bio->get_port() <<
"\n";
1046 if (_proxy_tunnel_now) {
1047 if (_proxy.get_scheme() ==
"socks") {
1048 _state = S_socks_proxy_greet;
1050 _state = S_http_proxy_ready;
1055 _state = S_setup_ssl;
1071 run_connecting_wait() {
1073 BIO_get_fd(*_bio, &fd);
1075 downloader_cat.warning()
1076 << _NOTIFY_HTTP_CHANNEL_ID
1077 <<
"nonblocking socket BIO has no file descriptor.\n";
1079 _status_entry._status_code = SC_internal_error;
1080 _state = S_try_next_proxy;
1084 if (downloader_cat.is_spam()) {
1085 downloader_cat.spam()
1086 << _NOTIFY_HTTP_CHANNEL_ID
1087 <<
"waiting to connect to " << _request.get_url().get_server_and_port() <<
".\n";
1093 if (get_blocking_connect()) {
1096 tv.tv_sec = (int)_connect_timeout;
1097 tv.tv_usec = (int)((_connect_timeout - tv.tv_sec) * 1000000.0);
1103 int errcode = select(fd + 1, NULL, &wset, NULL, &tv);
1105 downloader_cat.warning()
1106 << _NOTIFY_HTTP_CHANNEL_ID
1107 <<
"Error in select.\n";
1109 _status_entry._status_code = SC_internal_error;
1110 _state = S_try_next_proxy;
1116 if (get_blocking_connect() ||
1118 _started_connecting_time > get_connect_timeout())) {
1120 downloader_cat.info()
1121 << _NOTIFY_HTTP_CHANNEL_ID
1122 <<
"Timeout connecting to " 1123 << _request.get_url().get_server_and_port()
1124 <<
" for " << _request.get_url()
1126 _status_entry._status_code = SC_timeout;
1127 _state = S_try_next_proxy;
1134 _state = S_connecting;
1147 run_http_proxy_ready() {
1149 nassertr(!_proxy_request_text.empty(),
false);
1150 if (!server_send(_proxy_request_text,
false)) {
1155 _state = S_http_proxy_request_sent;
1170 run_http_proxy_request_sent() {
1173 if (!server_getline_failsafe(line)) {
1178 while (line.empty()) {
1179 if (!server_getline_failsafe(line)) {
1184 if (!parse_http_response(line)) {
1188 _state = S_http_proxy_reading_header;
1189 _current_field_name = string();
1190 _current_field_value = string();
1192 _got_file_size =
false;
1193 _got_transfer_file_size =
false;
1204 run_http_proxy_reading_header() {
1205 if (parse_http_header()) {
1209 _redirect = get_header_value(
"Location");
1213 _server_response_has_no_body =
1214 (get_status_code() / 100 == 1 ||
1215 get_status_code() == 204 ||
1216 get_status_code() == 304);
1218 int last_status = _last_status_code;
1219 _last_status_code = get_status_code();
1221 if (get_status_code() == 407 && last_status != 407 && !_proxy.empty()) {
1223 string authenticate_request = get_header_value(
"Proxy-Authenticate");
1224 _proxy_auth = _client->generate_auth(_proxy,
true, authenticate_request);
1225 if (_proxy_auth != (HTTPAuthorization *)NULL) {
1226 _proxy_realm = _proxy_auth->get_realm();
1227 _proxy_username = _client->select_username(_proxy,
true, _proxy_realm);
1228 if (!_proxy_username.empty()) {
1229 make_proxy_request_text();
1232 _state = S_begin_body;
1244 if (get_status_code() != 407) {
1245 _status_entry._status_code += 1000;
1248 _state = S_try_next_proxy;
1253 make_request_text();
1256 _state = S_setup_ssl;
1272 run_socks_proxy_greet() {
1273 static const char socks_greeting[] = {
1282 static const int socks_greeting_len =
sizeof(socks_greeting);
1283 if (!server_send(
string(socks_greeting, socks_greeting_len),
true)) {
1289 _state = S_socks_proxy_greet_reply;
1300 run_socks_proxy_greet_reply() {
1304 if (!server_get_failsafe(reply, 2)) {
1308 if (reply[0] != 0x05) {
1310 downloader_cat.info()
1311 << _NOTIFY_HTTP_CHANNEL_ID
1312 <<
"Rejecting Socks version " << (int)reply[0] <<
"\n";
1313 _status_entry._status_code = SC_socks_invalid_version;
1314 _state = S_try_next_proxy;
1318 if (reply[1] == (
char)0xff) {
1319 downloader_cat.info()
1320 << _NOTIFY_HTTP_CHANNEL_ID
1321 <<
"Socks server does not accept our available login methods.\n";
1322 _status_entry._status_code = SC_socks_no_acceptable_login_method;
1323 _state = S_try_next_proxy;
1327 if (reply[1] == 0x00) {
1330 _state = S_socks_proxy_connect;
1335 downloader_cat.info()
1336 << _NOTIFY_HTTP_CHANNEL_ID
1337 <<
"Socks server accepted unrequested login method " 1338 << (int)reply[1] <<
"\n";
1339 _status_entry._status_code = SC_socks_no_acceptable_login_method;
1340 _state = S_try_next_proxy;
1351 run_socks_proxy_connect() {
1352 static const char socks_connect[] = {
1358 static const int socks_connect_len =
sizeof(socks_connect);
1360 string hostname = _request.get_url().get_server();
1361 int port = _request.get_url().get_port();
1363 if (downloader_cat.is_debug()) {
1364 downloader_cat.debug()
1365 << _NOTIFY_HTTP_CHANNEL_ID
1366 <<
"Requesting SOCKS5 connection to " 1367 << _request.get_url().get_server_and_port() <<
"\n";
1371 string(socks_connect, socks_connect_len) +
1372 string(1, (
char)hostname.length()) +
1374 string(1, (
char)((port >> 8) & 0xff)) +
1375 string(1, (
char)(port & 0xff));
1377 if (!server_send(connect,
true)) {
1382 _state = S_socks_proxy_connect_reply;
1393 run_socks_proxy_connect_reply() {
1397 if (!server_get_failsafe(reply, 2)) {
1401 if (reply[0] != 0x05) {
1403 downloader_cat.info()
1404 << _NOTIFY_HTTP_CHANNEL_ID
1405 <<
"Rejecting Socks version " << (int)reply[0] <<
"\n";
1407 _status_entry._status_code = SC_socks_invalid_version;
1408 _state = S_try_next_proxy;
1412 if (reply[1] != 0x00) {
1413 downloader_cat.info()
1414 << _NOTIFY_HTTP_CHANNEL_ID
1415 <<
"Connection refused, SOCKS code " << (int)reply[1] <<
"\n";
1437 _status_entry._status_code = SC_socks_no_connection;
1441 _status_entry._status_code = SC_socks_refused;
1445 _state = S_try_next_proxy;
1450 _working_get = reply;
1451 if (!server_get_failsafe(reply, 5)) {
1456 int total_bytes = 6;
1464 total_bytes += (
unsigned int)reply[4];
1468 downloader_cat.info()
1469 << _NOTIFY_HTTP_CHANNEL_ID
1470 <<
"Unsupported SOCKS address type: " << (int)reply[3] <<
"\n";
1471 _status_entry._status_code = SC_socks_invalid_version;
1472 _state = S_try_next_proxy;
1478 _working_get = reply;
1479 if (!server_get_failsafe(reply, total_bytes)) {
1483 if (downloader_cat.is_debug()) {
1485 string connect_host;
1491 strm << (
unsigned int)(
unsigned char)reply[4] <<
"." 1492 << (
unsigned int)(
unsigned char)reply[5] <<
"." 1493 << (
unsigned int)(
unsigned char)reply[6] <<
"." 1494 << (
unsigned int)(
unsigned char)reply[7];
1495 connect_host = strm.str();
1500 connect_host = string(&reply[5], (
unsigned int)reply[4]);
1505 (((
unsigned int)(
unsigned char)reply[total_bytes - 2]) << 8) |
1506 ((
unsigned int)(
unsigned char)reply[total_bytes - 1]);
1508 downloader_cat.debug()
1509 << _NOTIFY_HTTP_CHANNEL_ID
1510 << _proxy <<
" directed us to " << connect_host <<
":" 1511 << connect_port <<
"\n";
1515 _state = S_setup_ssl;
1531 _sbio = BIO_new_ssl(_client->get_ssl_ctx(),
true);
1532 BIO_push(_sbio, *_bio);
1535 BIO_get_ssl(_sbio, &ssl);
1536 nassertr(ssl != (SSL *)NULL,
false);
1540 string cipher_list = _cipher_list;
1541 if (!cipher_list.empty()) {
1542 size_t space = cipher_list.find(
" ");
1543 if (space != string::npos) {
1544 cipher_list = cipher_list.substr(0, space);
1548 if (downloader_cat.is_debug()) {
1549 downloader_cat.debug()
1550 << _NOTIFY_HTTP_CHANNEL_ID
1551 <<
"Setting ssl-cipher-list '" << cipher_list <<
"'\n";
1553 int result = SSL_set_cipher_list(ssl, cipher_list.c_str());
1555 downloader_cat.error()
1556 << _NOTIFY_HTTP_CHANNEL_ID
1557 <<
"Invalid cipher list: '" << cipher_list <<
"'\n";
1558 OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
1559 _status_entry._status_code = SC_ssl_internal_failure;
1573 if (_client->load_client_certificate()) {
1574 SSL_use_certificate(ssl, _client->_client_certificate_pub);
1575 SSL_use_PrivateKey(ssl, _client->_client_certificate_priv);
1576 if (!SSL_check_private_key(ssl)) {
1577 downloader_cat.warning()
1578 << _NOTIFY_HTTP_CHANNEL_ID
1579 <<
"Client private key does not match public key!\n";
1583 if (downloader_cat.is_spam()) {
1584 downloader_cat.spam()
1585 << _NOTIFY_HTTP_CHANNEL_ID
1586 <<
"SSL Ciphers available:\n";
1589 name = SSL_get_cipher_list(ssl, pri);
1590 while (name != NULL) {
1591 downloader_cat.spam()
1592 << _NOTIFY_HTTP_CHANNEL_ID
1593 <<
" " << pri + 1 <<
". " << name <<
"\n";
1595 name = SSL_get_cipher_list(ssl, pri);
1599 if (downloader_cat.is_debug()) {
1600 downloader_cat.debug()
1601 << _NOTIFY_HTTP_CHANNEL_ID
1602 <<
"performing SSL handshake\n";
1604 _state = S_ssl_handshake;
1608 _started_connecting_time =
1622 run_ssl_handshake() {
1623 if (BIO_do_handshake(_sbio) <= 0) {
1624 if (BIO_should_retry(_sbio)) {
1627 _started_connecting_time;
1628 if (elapsed <= get_connect_timeout() + _extra_ssl_handshake_time) {
1635 downloader_cat.info()
1636 << _NOTIFY_HTTP_CHANNEL_ID
1637 <<
"Could not establish SSL handshake with " 1638 << _request.get_url().get_server_and_port() <<
"\n";
1639 OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
1644 if (!_cipher_list.empty()) {
1646 size_t space = _cipher_list.find(
" ");
1647 if (space != string::npos) {
1648 while (space < _cipher_list.length() && _cipher_list[space] ==
' ') {
1651 _cipher_list = _cipher_list.substr(space);
1652 if (!_cipher_list.empty()) {
1655 _state = S_connecting;
1662 _cipher_list = _client->get_cipher_list();
1663 _status_entry._status_code = SC_ssl_no_handshake;
1669 BIO_get_ssl(_sbio, &ssl);
1670 nassertr(ssl != (SSL *)NULL,
false);
1672 if (!_nonblocking) {
1673 SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
1676 const SSL_CIPHER *cipher = SSL_get_current_cipher(ssl);
1677 if (cipher == (
const SSL_CIPHER *)NULL) {
1678 downloader_cat.warning()
1679 << _NOTIFY_HTTP_CHANNEL_ID
1680 <<
"No current cipher on SSL connection.\n";
1682 if (downloader_cat.is_debug()) {
1683 downloader_cat.debug()
1684 << _NOTIFY_HTTP_CHANNEL_ID
1685 <<
"Using cipher " << SSL_CIPHER_get_name((SSL_CIPHER *) cipher) <<
"\n";
1691 _bio->set_bio(_sbio);
1694 X509 *cert = SSL_get_peer_certificate(ssl);
1695 if (cert == (X509 *)NULL) {
1696 downloader_cat.info()
1697 << _NOTIFY_HTTP_CHANNEL_ID
1698 <<
"No certificate was presented by server.\n";
1701 _status_entry._status_code = SC_ssl_invalid_server_certificate;
1706 X509_NAME *subject = X509_get_subject_name(cert);
1707 if (downloader_cat.is_debug()) {
1708 string org_name = get_x509_name_component(subject, NID_organizationName);
1709 string org_unit_name = get_x509_name_component(subject, NID_organizationalUnitName);
1710 string common_name = get_x509_name_component(subject, NID_commonName);
1712 downloader_cat.debug()
1713 << _NOTIFY_HTTP_CHANNEL_ID
1714 <<
"Server is " << common_name <<
" from " << org_unit_name
1715 <<
" / " << org_name <<
"\n";
1717 if (downloader_cat.is_spam()) {
1718 downloader_cat.spam()
1719 << _NOTIFY_HTTP_CHANNEL_ID
1720 <<
"Received certificate from server:\n" << flush;
1721 X509_print_fp(stderr, cert);
1726 bool cert_preapproved =
false;
1727 bool cert_name_preapproved =
false;
1728 check_preapproved_server_certificate(cert, cert_preapproved, cert_name_preapproved);
1731 long verify_result = SSL_get_verify_result(ssl);
1732 bool cert_valid =
true;
1734 if (verify_result == X509_V_ERR_CERT_HAS_EXPIRED) {
1735 downloader_cat.info()
1736 << _NOTIFY_HTTP_CHANNEL_ID
1737 <<
"Expired certificate from " << _request.get_url().get_server_and_port() <<
"\n";
1738 if (_client->get_verify_ssl() == HTTPClient::VS_normal && !cert_preapproved) {
1742 }
else if (verify_result == X509_V_ERR_CERT_NOT_YET_VALID) {
1743 downloader_cat.info()
1744 << _NOTIFY_HTTP_CHANNEL_ID
1745 <<
"Premature certificate from " << _request.get_url().get_server_and_port() <<
"\n";
1746 if (_client->get_verify_ssl() == HTTPClient::VS_normal && !cert_preapproved) {
1750 }
else if (verify_result == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT ||
1751 verify_result == X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN) {
1752 downloader_cat.info()
1753 << _NOTIFY_HTTP_CHANNEL_ID
1754 <<
"Self-signed certificate from " << _request.get_url().get_server_and_port() <<
"\n";
1755 if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_preapproved) {
1759 }
else if (verify_result != X509_V_OK) {
1760 downloader_cat.info()
1761 << _NOTIFY_HTTP_CHANNEL_ID
1762 <<
"Unable to verify identity of " << _request.get_url().get_server_and_port()
1763 <<
", verify error code " << verify_result <<
"\n";
1764 if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_preapproved) {
1770 _status_entry._status_code = SC_ssl_invalid_server_certificate;
1775 if (_client->get_verify_ssl() != HTTPClient::VS_no_verify && !cert_name_preapproved) {
1778 if (!validate_server_name(cert)) {
1779 _status_entry._status_code = SC_ssl_unexpected_server;
1806 if (!_request_text.empty()) {
1807 if (!server_send(_request_text,
false)) {
1813 _state = S_request_sent;
1825 run_request_sent() {
1828 if (!server_getline_failsafe(line)) {
1833 while (line.empty()) {
1834 if (!server_getline_failsafe(line)) {
1839 if (!parse_http_response(line)) {
1844 _state = S_reading_header;
1845 _current_field_name = string();
1846 _current_field_value = string();
1848 _got_file_size =
false;
1849 _got_transfer_file_size =
false;
1862 run_reading_header() {
1863 if (parse_http_header()) {
1864 if (_bio.is_null()) {
1865 downloader_cat.info()
1866 << _NOTIFY_HTTP_CHANNEL_ID
1867 <<
"Connection lost while reading HTTP response.\n";
1868 if (_response_type == RT_http_hangup) {
1870 _status_entry._status_code = SC_lost_connection;
1871 _state = S_try_next_proxy;
1875 _response_type = RT_http_hangup;
1882 if (elapsed > get_http_timeout()) {
1884 downloader_cat.info()
1885 << _NOTIFY_HTTP_CHANNEL_ID
1886 <<
"Timeout waiting for " 1887 << _request.get_url().get_server_and_port()
1888 <<
" in run_reading_header (" << elapsed
1889 <<
" seconds elapsed).\n";
1890 _status_entry._status_code = SC_timeout;
1891 _state = S_try_next_proxy;
1896 _response_type = RT_http_complete;
1900 clear_extra_headers();
1902 _server_response_has_no_body =
1903 (get_status_code() / 100 == 1 ||
1904 get_status_code() == 204 ||
1905 get_status_code() == 304 ||
1906 _method == HTTPEnum::M_head);
1909 if (get_status_code() == 206) {
1910 string content_range = get_header_value(
"Content-Range");
1911 if (content_range.empty()) {
1912 downloader_cat.warning()
1913 << _NOTIFY_HTTP_CHANNEL_ID
1914 <<
"Got 206 response without Content-Range header!\n";
1915 _status_entry._status_code = SC_invalid_http;
1920 if (!parse_content_range(content_range)) {
1921 downloader_cat.warning()
1922 << _NOTIFY_HTTP_CHANNEL_ID
1923 <<
"Couldn't parse Content-Range: " << content_range <<
"\n";
1924 _status_entry._status_code = SC_invalid_http;
1931 _first_byte_delivered = 0;
1932 _last_byte_delivered = 0;
1934 if (downloader_cat.is_debug()) {
1935 if (_first_byte_requested != 0 || _last_byte_requested != 0 ||
1936 _first_byte_delivered != 0 || _last_byte_delivered != 0) {
1937 downloader_cat.debug()
1938 << _NOTIFY_HTTP_CHANNEL_ID
1939 <<
"Requested byte range " << _first_byte_requested
1940 <<
" to " << _last_byte_delivered
1941 <<
"; server delivers range " << _first_byte_delivered
1942 <<
" to " << _last_byte_delivered
1949 string tag = get_header_value(
"ETag");
1953 string date = get_header_value(
"Last-Modified");
1954 if (!date.empty()) {
1955 _document_spec.set_date(
HTTPDate(date));
1961 if (_server_response_has_no_body) {
1963 reset_download_to();
1966 if (!open_download_file()) {
1970 _got_expected_file_size =
false;
1971 _got_file_size =
false;
1972 _got_transfer_file_size =
false;
1974 string content_length = get_header_value(
"Content-Length");
1975 if (!content_length.empty()) {
1976 _file_size = atoi(content_length.c_str());
1977 _got_file_size =
true;
1979 }
else if (get_status_code() == 206) {
1982 _file_size = _last_byte_delivered - _first_byte_delivered + 1;
1983 _got_file_size =
true;
1985 _redirect = get_header_value(
"Location");
1990 if (_redirect.has_path() && !_redirect.has_authority()) {
1992 Filename path = _redirect.get_path();
1995 _redirect.set_path(
Filename(rel_to, path));
2001 _state = S_read_header;
2003 if (_server_response_has_no_body && will_close_connection()) {
2010 int last_status = _last_status_code;
2011 _last_status_code = get_status_code();
2013 if (get_status_code() == 407 && last_status != 407 && !_proxy.empty()) {
2015 string authenticate_request = get_header_value(
"Proxy-Authenticate");
2017 _client->generate_auth(_proxy,
true, authenticate_request);
2018 if (_proxy_auth != (HTTPAuthorization *)NULL) {
2019 _proxy_realm = _proxy_auth->get_realm();
2020 _proxy_username = _client->select_username(_proxy,
true, _proxy_realm);
2021 if (!_proxy_username.empty()) {
2022 make_request_text();
2025 _state = S_begin_body;
2031 if (get_status_code() == 401 && last_status != 401) {
2033 string authenticate_request = get_header_value(
"WWW-Authenticate");
2034 _www_auth = _client->generate_auth(_request.get_url(),
false, authenticate_request);
2035 if (_www_auth != (HTTPAuthorization *)NULL) {
2036 _www_realm = _www_auth->get_realm();
2037 _www_username = _client->select_username(_request.get_url(),
false, _www_realm);
2038 if (!_www_username.empty()) {
2039 make_request_text();
2042 _state = S_begin_body;
2048 if ((get_status_code() == 300 ||
2049 get_status_code() == 301 ||
2050 get_status_code() == 302 ||
2051 get_status_code() == 303 ||
2052 get_status_code() == 307) && !get_redirect().empty()) {
2060 if (_method == HTTPEnum::M_post) {
2061 _method = HTTPEnum::M_get;
2065 if (_method == HTTPEnum::M_get || _method == HTTPEnum::M_head) {
2067 URLSpec new_url = get_redirect();
2068 if (find(_redirect_trail.begin(), _redirect_trail.end(),
2069 new_url) != _redirect_trail.end()) {
2070 downloader_cat.warning()
2071 << _NOTIFY_HTTP_CHANNEL_ID
2072 <<
"cycle detected in redirect to " << new_url <<
"\n";
2075 _redirect_trail.push_back(new_url);
2077 if (downloader_cat.is_debug()) {
2078 downloader_cat.debug()
2079 << _NOTIFY_HTTP_CHANNEL_ID
2080 <<
"following redirect to " << new_url <<
"\n";
2082 if (_request.get_url().has_username()) {
2083 new_url.
set_username(_request.get_url().get_username());
2085 reset_url(_request.get_url(), new_url);
2086 _request.set_url(new_url);
2087 _want_ssl = _request.get_url().is_ssl();
2090 make_request_text();
2093 _state = S_begin_body;
2099 if (_state == S_read_header &&
2100 ((get_status_code() / 100) == 4 || (get_status_code() / 100) == 5) &&
2101 _proxy_serves_document && _proxy_next_index < _proxies.size()) {
2109 _state = S_try_next_proxy;
2124 run_start_direct_file_read() {
2125 _state = S_read_header;
2126 if (!open_download_file()) {
2152 _state = S_begin_body;
2164 if (will_close_connection()) {
2167 if (downloader_cat.is_debug()) {
2168 downloader_cat.debug()
2169 << _NOTIFY_HTTP_CHANNEL_ID
2170 <<
"resetting to begin body; server would close anyway.\n";
2176 if (_server_response_has_no_body) {
2178 _state = S_read_trailer;
2180 }
else if (get_file_size() > (
int)_skip_body_size) {
2184 if (downloader_cat.is_debug()) {
2185 downloader_cat.debug()
2186 << _NOTIFY_HTTP_CHANNEL_ID
2187 <<
"Dropping connection rather than skipping past " 2188 << get_file_size() <<
" bytes.\n";
2194 if (_body_stream == (ISocketStream *)NULL) {
2195 if (downloader_cat.is_debug()) {
2196 downloader_cat.debug()
2197 << _NOTIFY_HTTP_CHANNEL_ID
2198 <<
"Unable to skip body.\n";
2203 _owns_body_stream =
true;
2204 if (_state != S_reading_body) {
2205 reset_body_stream();
2225 run_reading_body() {
2226 if (will_close_connection()) {
2229 if (downloader_cat.is_debug()) {
2230 downloader_cat.debug()
2231 << _NOTIFY_HTTP_CHANNEL_ID
2232 <<
"resetting to read body; server would close anyway.\n";
2239 if (_body_stream == NULL || !_owns_body_stream) {
2241 if (downloader_cat.is_debug()) {
2242 downloader_cat.debug()
2243 << _NOTIFY_HTTP_CHANNEL_ID
2244 <<
"resetting, not in skip-body mode.\n";
2251 getline(*_body_stream, line);
2252 while (!_body_stream->fail() && !_body_stream->eof()) {
2253 if (downloader_cat.is_spam()) {
2254 downloader_cat.spam()
2255 << _NOTIFY_HTTP_CHANNEL_ID
2256 <<
"skip: " << line <<
"\n";
2258 getline(*_body_stream, line);
2261 if (!_body_stream->is_closed()) {
2266 reset_body_stream();
2269 nassertr(_state != S_reading_body,
false);
2289 if (will_close_connection()) {
2292 if (downloader_cat.is_debug()) {
2293 downloader_cat.debug()
2294 << _NOTIFY_HTTP_CHANNEL_ID
2295 <<
"resetting to read body; server would close anyway.\n";
2303 if (!server_getline(line)) {
2306 while (!line.empty()) {
2307 if (!server_getline(line)) {
2312 _state = S_read_trailer;
2324 run_read_trailer() {
2325 if (will_close_connection()) {
2328 if (downloader_cat.is_debug()) {
2329 downloader_cat.debug()
2330 << _NOTIFY_HTTP_CHANNEL_ID
2331 <<
"resetting to read trailer; server would close anyway.\n";
2348 run_download_to_file() {
2349 nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream,
false);
2351 bool do_throttle = _wanted_nonblocking && _download_throttle;
2353 static const size_t buffer_size = 4096;
2354 char buffer[buffer_size];
2356 size_t remaining_this_pass = buffer_size;
2358 remaining_this_pass = _bytes_per_update;
2361 _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2362 size_t count = _body_stream->gcount();
2363 while (count != 0) {
2364 _download_to_stream->write(buffer, count);
2365 _bytes_downloaded += count;
2367 nassertr(count <= remaining_this_pass,
false);
2368 remaining_this_pass -= count;
2369 if (remaining_this_pass == 0) {
2375 thread_consider_yield();
2376 _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2377 count = _body_stream->gcount();
2380 if (_download_to_stream->fail()) {
2381 downloader_cat.warning()
2382 << _NOTIFY_HTTP_CHANNEL_ID
2383 <<
"Error writing to " << _download_to_filename <<
"\n";
2384 _status_entry._status_code = SC_download_write_error;
2386 reset_download_to();
2390 _download_to_stream->flush();
2392 if (_body_stream->is_closed()) {
2394 reset_body_stream();
2395 close_download_stream();
2396 _started_download =
false;
2411 run_download_to_ram() {
2412 nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream,
false);
2413 nassertr(_download_to_ramfile != (
Ramfile *)NULL,
false);
2415 bool do_throttle = _wanted_nonblocking && _download_throttle;
2417 static const size_t buffer_size = 4096;
2418 char buffer[buffer_size];
2420 size_t remaining_this_pass = buffer_size;
2422 remaining_this_pass = _bytes_per_update;
2425 _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2426 size_t count = _body_stream->gcount();
2427 while (count != 0) {
2428 _download_to_ramfile->_data += string(buffer, count);
2429 _bytes_downloaded += count;
2431 nassertr(count <= remaining_this_pass,
false);
2432 remaining_this_pass -= count;
2433 if (remaining_this_pass == 0) {
2439 thread_consider_yield();
2440 _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2441 count = _body_stream->gcount();
2444 if (_body_stream->is_closed()) {
2446 reset_body_stream();
2447 close_download_stream();
2448 _started_download =
false;
2463 run_download_to_stream() {
2464 nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream,
false);
2466 bool do_throttle = _wanted_nonblocking && _download_throttle;
2468 static const size_t buffer_size = 4096;
2469 char buffer[buffer_size];
2471 size_t remaining_this_pass = buffer_size;
2473 remaining_this_pass = _bytes_per_update;
2476 _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2477 size_t count = _body_stream->gcount();
2478 while (count != 0) {
2479 _download_to_stream->write(buffer, count);
2480 _bytes_downloaded += count;
2482 nassertr(count <= remaining_this_pass,
false);
2483 remaining_this_pass -= count;
2484 if (remaining_this_pass == 0) {
2490 thread_consider_yield();
2491 _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
2492 count = _body_stream->gcount();
2495 if (_download_to_stream->fail()) {
2496 downloader_cat.warning()
2497 << _NOTIFY_HTTP_CHANNEL_ID
2498 <<
"Error writing to stream\n";
2499 _status_entry._status_code = SC_download_write_error;
2501 reset_download_to();
2505 _download_to_stream->flush();
2507 if (_body_stream->is_closed()) {
2509 reset_body_stream();
2510 close_download_stream();
2511 _started_download =
false;
2528 begin_request(HTTPEnum::Method method,
const DocumentSpec &url,
2529 const string &body,
bool nonblocking,
2530 size_t first_byte,
size_t last_byte) {
2532 downloader_cat.info()
2533 << _NOTIFY_HTTP_CHANNEL_ID
2534 <<
"begin " << method <<
" " << url <<
"\n";
2536 reset_for_new_request();
2538 _wanted_nonblocking = nonblocking;
2539 #if defined(HAVE_THREADS) && defined(SIMPLE_THREADS) 2547 _proxy_next_index = 0;
2548 if (get_allow_proxy()) {
2549 _client->get_proxies_for_url(url.get_url(), _proxies);
2555 if (!_bio.is_null() && !_proxies.empty() && !_proxy.empty()) {
2556 Proxies::iterator pi = find(_proxies.begin(), _proxies.end(), _proxy);
2557 if (pi != _proxies.end()) {
2559 _proxies.insert(_proxies.begin(), _proxy);
2564 if (_proxy_next_index < _proxies.size()) {
2565 new_proxy = _proxies[_proxy_next_index];
2566 _proxy_next_index++;
2571 if (_proxy != new_proxy) {
2573 _proxy_auth = (HTTPAuthorization *)NULL;
2574 if (downloader_cat.is_debug()) {
2575 downloader_cat.debug()
2576 << _NOTIFY_HTTP_CHANNEL_ID
2577 <<
"resetting to change proxy to " << _proxy <<
"\n";
2583 if (_nonblocking != nonblocking) {
2584 _nonblocking = nonblocking;
2585 if (downloader_cat.is_debug()) {
2586 downloader_cat.debug()
2587 << _NOTIFY_HTTP_CHANNEL_ID
2588 <<
"resetting to change nonblocking state to " << _nonblocking <<
".\n";
2593 reset_url(_request.get_url(), url.get_url());
2601 _want_ssl = _request.get_url().is_ssl();
2603 _first_byte_requested = first_byte;
2604 _last_byte_requested = last_byte;
2610 if (_request.get_url().get_scheme() ==
"file") {
2615 _bio =
new BioPtr(_request.get_url());
2616 if (_bio->get_bio() != NULL) {
2618 _source =
new BioStreamPtr(
new BioStream(_bio));
2619 _status_entry._status_code = 200;
2620 _state = S_start_direct_file_read;
2624 BIO_get_fp(_bio->get_bio(), &fp);
2626 if (fseek(fp, 0, SEEK_END) == 0) {
2627 _file_size = ftell(fp);
2628 _got_file_size =
true;
2629 fseek(fp, 0, SEEK_SET);
2635 OpenSSLWrapper::get_global_ptr()->notify_ssl_errors();
2636 _status_entry._status_code = SC_no_connection;
2642 if (_state == S_failure || (_state < S_read_header && _state != S_ready)) {
2643 if (downloader_cat.is_debug()) {
2644 downloader_cat.debug()
2645 << _NOTIFY_HTTP_CHANNEL_ID
2646 <<
"resetting to clear previous request.\n";
2651 if (downloader_cat.is_debug()) {
2652 downloader_cat.debug()
2653 << _NOTIFY_HTTP_CHANNEL_ID
2654 <<
"resetting old connection: " 2660 }
else if (_state == S_read_header) {
2663 _state = S_begin_body;
2667 if (_method == HTTPEnum::M_connect) {
2668 _done_state = S_ready;
2670 _done_state = S_read_header;
2684 reconsider_proxy() {
2685 _proxy_tunnel_now =
false;
2686 _proxy_serves_document =
false;
2688 if (!_proxy.empty()) {
2695 (get_proxy_tunnel() || _want_ssl ||
2696 _method == HTTPEnum::M_connect || _proxy.get_scheme() ==
"socks");
2700 _proxy_serves_document = !_proxy_tunnel_now;
2704 make_request_text();
2706 if (_proxy_tunnel_now) {
2711 <<
"CONNECT " << _request.get_url().get_server_and_port()
2712 <<
" " << _client->get_http_version_string() <<
"\r\n";
2713 if (_client->get_http_version() >= HTTPEnum::HV_11) {
2715 <<
"Host: " << _request.get_url().get_server_and_port() <<
"\r\n";
2717 _proxy_header = request.str();
2718 make_proxy_request_text();
2721 _proxy_header = string();
2722 _proxy_request_text = string();
2734 reset_for_new_request() {
2735 if (downloader_cat.is_spam()) {
2736 downloader_cat.spam()
2737 << _NOTIFY_HTTP_CHANNEL_ID
2738 <<
"reset_for_new_request.\n";
2741 reset_download_to();
2742 reset_body_stream();
2744 _last_status_code = 0;
2745 _status_entry = StatusEntry();
2747 _response_type = RT_none;
2748 _redirect_trail.clear();
2749 _bytes_downloaded = 0;
2750 _bytes_requested = 0;
2766 finished_body(
bool has_trailer) {
2767 if (will_close_connection() && _download_dest == DD_none) {
2768 if (downloader_cat.is_debug()) {
2769 downloader_cat.debug()
2770 << _NOTIFY_HTTP_CHANNEL_ID
2771 <<
"resetting to finish body; server would close anyway.\n";
2777 _state = HTTPChannel::S_read_body;
2779 _state = HTTPChannel::S_read_trailer;
2796 open_download_file() {
2797 _subdocument_resumes = (_subdocument_resumes && _first_byte_delivered != 0);
2799 if (_download_dest == DD_file) {
2801 _download_to_stream = vfs->
open_write_file(_download_to_filename,
false, !_subdocument_resumes);
2802 if (_download_to_stream == NULL) {
2803 downloader_cat.info()
2804 << _NOTIFY_HTTP_CHANNEL_ID
2805 <<
"Could not open " << _download_to_filename <<
" for writing.\n";
2806 _status_entry._status_code = SC_download_open_error;
2812 if (_subdocument_resumes) {
2813 if (_download_dest == DD_file) {
2818 _download_to_stream->seekp(0, ios::end);
2819 if (_first_byte_delivered > (
size_t)_download_to_stream->tellp()) {
2820 downloader_cat.info()
2821 << _NOTIFY_HTTP_CHANNEL_ID
2822 <<
"Invalid starting position of byte " << _first_byte_delivered
2823 <<
" within " << _download_to_filename <<
" (which has " 2824 << _download_to_stream->tellp() <<
" bytes)\n";
2825 close_download_stream();
2826 _status_entry._status_code = SC_download_invalid_range;
2831 _download_to_stream->seekp(_first_byte_delivered);
2833 }
else if (_download_dest == DD_ram) {
2834 if (_first_byte_delivered > _download_to_ramfile->_data.length()) {
2835 downloader_cat.info()
2836 << _NOTIFY_HTTP_CHANNEL_ID
2837 <<
"Invalid starting position of byte " << _first_byte_delivered
2838 <<
" within Ramfile (which has " 2839 << _download_to_ramfile->_data.length() <<
" bytes)\n";
2840 close_download_stream();
2841 _status_entry._status_code = SC_download_invalid_range;
2846 if (_first_byte_delivered == 0) {
2847 _download_to_ramfile->_data = string();
2849 _download_to_ramfile->_data =
2850 _download_to_ramfile->_data.substr(0, _first_byte_delivered);
2852 }
else if (_download_dest == DD_stream) {
2857 _download_to_stream->seekp(0, ios::end);
2858 if (_first_byte_delivered > (
size_t)_download_to_stream->tellp()) {
2859 downloader_cat.info()
2860 << _NOTIFY_HTTP_CHANNEL_ID
2861 <<
"Invalid starting position of byte " << _first_byte_delivered
2862 <<
" within stream (which has " 2863 << _download_to_stream->tellp() <<
" bytes)\n";
2864 close_download_stream();
2865 _status_entry._status_code = SC_download_invalid_range;
2870 _download_to_stream->seekp(_first_byte_delivered);
2877 if (_download_dest == DD_file || _download_dest == DD_stream) {
2878 _download_to_stream->seekp(0);
2879 }
else if (_download_dest == DD_ram) {
2880 _download_to_ramfile->_data = string();
2897 server_getline(
string &str) {
2898 nassertr(!_source.is_null(),
false);
2899 int ch = (*_source)->get();
2900 while (!(*_source)->eof() && !(*_source)->fail()) {
2905 _working_get = string();
2909 size_t p = str.length();
2910 while (p > 0 && isspace(str[p - 1])) {
2913 str = str.substr(0, p);
2915 if (downloader_cat.is_debug()) {
2916 downloader_cat.debug()
2917 << _NOTIFY_HTTP_CHANNEL_ID
2918 <<
"recv: " << str <<
"\n";
2927 _working_get += (char)ch;
2929 ch = (*_source)->get();
2945 server_getline_failsafe(
string &str) {
2946 if (!server_getline(str)) {
2947 if (_bio.is_null()) {
2949 if (_response_type == RT_hangup) {
2951 _status_entry._status_code = SC_lost_connection;
2952 _state = S_try_next_proxy;
2956 _response_type = RT_hangup;
2963 if (elapsed > get_http_timeout()) {
2965 downloader_cat.info()
2966 << _NOTIFY_HTTP_CHANNEL_ID
2967 <<
"Timeout waiting for " 2968 << _request.get_url().get_server_and_port()
2969 <<
" in server_getline_failsafe (" << elapsed
2970 <<
" seconds elapsed).\n";
2971 _status_entry._status_code = SC_timeout;
2972 _state = S_try_next_proxy;
2991 server_get(
string &str,
size_t num_bytes) {
2992 nassertr(!_source.is_null(),
false);
2993 int ch = (*_source)->get();
2994 while (!(*_source)->eof() && !(*_source)->fail()) {
2995 _working_get += (char)ch;
2996 if (_working_get.length() >= num_bytes) {
2998 _working_get = string();
3002 ch = (*_source)->get();
3019 server_get_failsafe(
string &str,
size_t num_bytes) {
3020 if (!server_get(str, num_bytes)) {
3021 if (_bio.is_null()) {
3023 if (_response_type == RT_hangup) {
3025 _status_entry._status_code = SC_lost_connection;
3026 _state = S_try_next_proxy;
3030 _response_type = RT_hangup;
3037 if (elapsed > get_http_timeout()) {
3039 downloader_cat.info()
3040 << _NOTIFY_HTTP_CHANNEL_ID
3041 <<
"Timeout waiting for " 3042 << _request.get_url().get_server_and_port()
3043 <<
" in server_get_failsafe (" << elapsed
3044 <<
" seconds elapsed).\n";
3045 _status_entry._status_code = SC_timeout;
3046 _state = S_try_next_proxy;
3070 server_send(
const string &str,
bool secret) {
3071 nassertr(str.length() > _sent_so_far,
true);
3076 size_t bytes_to_send = str.length() - _sent_so_far;
3078 BIO_write(*_bio, str.data() + _sent_so_far, bytes_to_send);
3080 if (write_count <= 0) {
3081 if (BIO_should_retry(*_bio)) {
3086 if (downloader_cat.is_debug()) {
3087 downloader_cat.debug()
3088 << _NOTIFY_HTTP_CHANNEL_ID
3089 <<
"Lost connection to server unexpectedly during write.\n";
3095 if (downloader_cat.is_spam()) {
3096 downloader_cat.spam()
3097 << _NOTIFY_HTTP_CHANNEL_ID
3098 <<
"wrote " << write_count <<
" bytes to " << _bio <<
"\n";
3102 if (!secret && downloader_cat.is_debug()) {
3103 show_send(str.substr(0, write_count));
3107 if (write_count < (
int)bytes_to_send) {
3108 _sent_so_far += write_count;
3126 parse_http_response(
const string &line) {
3129 if (line.length() < 5 || line.substr(0, 5) != string(
"HTTP/")) {
3131 _status_entry._status_code = SC_non_http_response;
3132 if (_response_type == RT_non_http) {
3134 _state = S_try_next_proxy;
3139 if (downloader_cat.is_debug()) {
3140 downloader_cat.debug()
3141 << _NOTIFY_HTTP_CHANNEL_ID
3142 <<
"got non-HTTP response, resetting.\n";
3145 _response_type = RT_non_http;
3152 while (p < line.length() && !isspace(line[p])) {
3155 _http_version_string = line.substr(0, p);
3156 _http_version = HTTPClient::parse_http_version_string(_http_version_string);
3158 while (p < line.length() && isspace(line[p])) {
3162 while (q < line.length() && !isspace(line[q])) {
3165 string status_code = line.substr(p, q - p);
3166 _status_entry._status_code = atoi(status_code.c_str());
3168 while (q < line.length() && isspace(line[q])) {
3171 _status_entry._status_string = line.substr(q, line.length() - q);
3184 parse_http_header() {
3186 if (!server_getline(line)) {
3190 while (!line.empty()) {
3191 if (isspace(line[0])) {
3195 while (p < line.length() && isspace(line[p])) {
3198 _current_field_value += line.substr(p - 1);
3203 if (!_current_field_name.empty()) {
3204 store_header_field(_current_field_name, _current_field_value);
3205 _current_field_value = string();
3208 size_t colon = line.find(
':');
3209 if (colon != string::npos) {
3210 _current_field_name = downcase(line.substr(0, colon));
3211 size_t p = colon + 1;
3212 while (p < line.length() && isspace(line[p])) {
3215 _current_field_value = line.substr(p);
3219 if (!server_getline(line)) {
3225 if (!_current_field_name.empty()) {
3226 store_header_field(_current_field_name, _current_field_value);
3227 _current_field_value = string();
3242 parse_content_range(
const string &content_range) {
3245 while (p < content_range.length() && !isspace(content_range[p])) {
3249 string units = content_range.substr(0, p);
3250 while (p < content_range.length() && isspace(content_range[p])) {
3254 if (units ==
"bytes") {
3255 const char *c_str = content_range.c_str();
3257 if (p < content_range.length() && isdigit(content_range[p])) {
3258 long first_byte = strtol(c_str + p, &endptr, 10);
3260 if (p < content_range.length() && content_range[p] ==
'-') {
3262 if (p < content_range.length() && isdigit(content_range[p])) {
3263 long last_byte = strtol(c_str + p, &endptr, 10);
3266 if (last_byte >= first_byte) {
3267 _first_byte_delivered = first_byte;
3268 _last_byte_delivered = last_byte;
3290 nassertv(!_source.is_null());
3291 if ((*_source)->is_closed()) {
3292 if (downloader_cat.is_debug()) {
3293 downloader_cat.debug()
3294 << _NOTIFY_HTTP_CHANNEL_ID
3295 <<
"Lost connection to server unexpectedly during read.\n";
3497 check_preapproved_server_certificate(X509 *cert,
bool &cert_preapproved,
3498 bool &cert_name_preapproved)
const {
3499 return _client->check_preapproved_server_certificate(_request.get_url(),
3500 cert, cert_preapproved,
3501 cert_name_preapproved);
3511 validate_server_name(X509 *cert) {
3512 string hostname = _request.get_url().get_server();
3514 vector_string cert_names;
3518 STACK_OF(GENERAL_NAME) *subject_alt_names =
3519 (STACK_OF(GENERAL_NAME) *)X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL);
3520 if (subject_alt_names != NULL) {
3521 int num_alts = sk_GENERAL_NAME_num(subject_alt_names);
3522 for (
int i = 0; i < num_alts; ++i) {
3524 const GENERAL_NAME *alt_name =
3525 sk_GENERAL_NAME_value(subject_alt_names, i);
3527 if (alt_name->type == GEN_DNS) {
3528 char *buffer = NULL;
3529 int len = ASN1_STRING_to_UTF8((
unsigned char**)&buffer,
3532 cert_names.push_back(
string(buffer, len));
3534 if (buffer != NULL) {
3535 OPENSSL_free(buffer);
3541 if (cert_names.empty()) {
3544 X509_NAME *xname = X509_get_subject_name(cert);
3545 if (xname != NULL) {
3546 string common_name = get_x509_name_component(xname, NID_commonName);
3547 cert_names.push_back(common_name);
3551 if (cert_names.empty()) {
3552 downloader_cat.info()
3553 << _NOTIFY_HTTP_CHANNEL_ID
3554 <<
"Server certificate from " << hostname
3555 <<
" provides no name.\n";
3559 if (downloader_cat.is_debug()) {
3560 downloader_cat.debug()
3561 << _NOTIFY_HTTP_CHANNEL_ID
3562 <<
"Server certificate from " << hostname
3563 <<
" provides name(s):";
3564 vector_string::const_iterator si;
3565 for (si = cert_names.begin(); si != cert_names.end(); ++si) {
3566 const string &cert_name = (*si);
3567 downloader_cat.debug(
false)
3568 <<
" " << cert_name;
3570 downloader_cat.debug(
false)
3576 vector_string::const_iterator si;
3577 for (si = cert_names.begin(); si != cert_names.end(); ++si) {
3578 const string &cert_name = (*si);
3580 if (match_cert_name(cert_name, hostname)) {
3585 downloader_cat.info()
3586 << _NOTIFY_HTTP_CHANNEL_ID
3587 <<
"Server certificate from " << hostname
3588 <<
" provides wrong name(s):";
3589 for (si = cert_names.begin(); si != cert_names.end(); ++si) {
3590 const string &cert_name = (*si);
3591 downloader_cat.info(
false)
3592 <<
" " << cert_name;
3594 downloader_cat.info(
false)
3608 match_cert_name(
const string &cert_name,
const string &hostname) {
3615 pattern.set_case_sensitive(
false);
3616 pattern.set_nomatch_chars(
".");
3617 return pattern.matches(hostname);
3626 string HTTPChannel::
3627 get_x509_name_component(X509_NAME *name,
int nid) {
3628 ASN1_OBJECT *obj = OBJ_nid2obj(nid);
3635 int i = X509_NAME_get_index_by_OBJ(name, obj, -1);
3640 ASN1_STRING *data = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(name, i));
3641 return string((
char *)data->data, data->length);
3653 _proxy_auth = _client->select_auth(_proxy,
true, _proxy_realm);
3654 _proxy_username = string();
3655 if (_proxy_auth != (HTTPAuthorization *)NULL) {
3656 _proxy_realm = _proxy_auth->get_realm();
3657 _proxy_username = _client->select_username(_proxy,
true, _proxy_realm);
3660 if (_method == HTTPEnum::M_connect) {
3669 _www_auth = _client->select_auth(_request.get_url(),
false, _www_realm);
3670 _www_username = string();
3671 if (_www_auth != (HTTPAuthorization *)NULL) {
3672 _www_realm = _www_auth->get_realm();
3673 _www_username = _client->select_username(_request.get_url(),
false, _www_realm);
3676 string request_path;
3677 if (_proxy_serves_document) {
3682 request_path = url_no_username.
get_url();
3687 request_path = _request.get_url().get_path_and_query();
3693 if (request_path.empty()) {
3700 << _method <<
" " << request_path <<
" " 3701 << _client->get_http_version_string() <<
"\r\n";
3703 if (_client->get_http_version() >= HTTPEnum::HV_11) {
3706 <<
"Host: " << _request.get_url().get_server();
3707 if (!_request.get_url().is_default_port()) {
3712 stream <<
":" << _request.get_url().get_port();
3715 if (!get_persistent_connection()) {
3717 <<
"Connection: close\r\n";
3721 if (_last_byte_requested != 0) {
3723 <<
"Range: bytes=" << _first_byte_requested <<
"-" 3724 << _last_byte_requested <<
"\r\n";
3726 }
else if (_first_byte_requested != 0) {
3728 <<
"Range: bytes=" << _first_byte_requested <<
"-\r\n";
3731 switch (_request.get_request_mode()) {
3732 case DocumentSpec::RM_any:
3735 if (_first_byte_requested != 0) {
3739 if (_request.has_tag()) {
3741 <<
"If-Range: " << _request.get_tag().get_string() <<
"\r\n";
3742 }
else if (_request.has_date()) {
3744 <<
"If-Range: " << _request.get_date().get_string() <<
"\r\n";
3749 case DocumentSpec::RM_equal:
3752 if (_request.has_tag()) {
3754 <<
"If-Match: " << _request.get_tag().get_string() <<
"\r\n";
3756 if (_request.has_date()) {
3758 <<
"If-Unmodified-Since: " << _request.get_date().get_string()
3763 case DocumentSpec::RM_newer:
3765 if (_request.has_tag()) {
3767 <<
"If-None-Match: " << _request.get_tag().get_string() <<
"\r\n";
3769 if (_request.has_date()) {
3771 <<
"If-Modified-Since: " << _request.get_date().get_string()
3776 case DocumentSpec::RM_equal_or_newer:
3778 if (_request.has_date()) {
3783 <<
"If-Modified-Since: " << (_request.get_date() - 1).get_string()
3789 switch (_request.get_cache_control()) {
3790 case DocumentSpec::CC_allow_cache:
3794 case DocumentSpec::CC_revalidate:
3797 <<
"Cache-Control: max-age=0\r\n";
3800 case DocumentSpec::CC_no_cache:
3803 <<
"Cache-Control: no-cache\r\n" 3804 <<
"Pragma: no-cache\r\n";
3808 _client->send_cookies(stream, _request.get_url());
3810 if (!_body.empty()) {
3812 <<
"Content-Type: application/x-www-form-urlencoded\r\n" 3813 <<
"Content-Length: " << _body.length() <<
"\r\n";
3816 _header = stream.str();
3829 make_proxy_request_text() {
3830 _proxy_request_text = _proxy_header;
3832 if (_proxy_auth != (HTTPAuthorization *)NULL && !_proxy_username.empty()) {
3833 _proxy_request_text +=
"Proxy-Authorization: ";
3834 _proxy_request_text +=
3835 _proxy_auth->generate(HTTPEnum::M_connect, _request.get_url().get_server_and_port(),
3836 _proxy_username, _body);
3837 _proxy_request_text +=
"\r\n";
3840 _proxy_request_text +=
"\r\n";
3851 make_request_text() {
3852 _request_text = _header;
3854 if (_proxy_serves_document &&
3855 _proxy_auth != (HTTPAuthorization *)NULL && !_proxy_username.empty()) {
3856 _request_text +=
"Proxy-Authorization: ";
3858 _proxy_auth->generate(_method, _request.get_url().get_url(), _proxy_username, _body);
3859 _request_text +=
"\r\n";
3862 if (_www_auth != (HTTPAuthorization *)NULL && !_www_username.empty()) {
3863 string authorization =
3864 _request_text +=
"Authorization: ";
3866 _www_auth->generate(_method, _request.get_url().get_path_and_query(), _www_username, _body);
3867 _request_text +=
"\r\n";
3870 _request_text += _send_extra_headers;
3871 _request_text +=
"\r\n";
3872 _request_text += _body;
3892 if (downloader_cat.is_debug()) {
3893 downloader_cat.debug()
3894 << _NOTIFY_HTTP_CHANNEL_ID
3895 <<
"resetting for new server " 3910 store_header_field(
const string &field_name,
const string &field_value) {
3911 pair<Headers::iterator, bool> insert_result =
3912 _headers.insert(Headers::value_type(field_name, field_value));
3914 if (!insert_result.second) {
3917 Headers::iterator hi = insert_result.first;
3918 (*hi).second +=
", ";
3919 (*hi).second += field_value;
3922 if (field_name ==
"set-cookie") {
3923 _client->set_cookie(HTTPCookie(field_value, _request.get_url()));
3935 show_send(
const string &message) {
3937 size_t newline = message.find(
'\n', start);
3938 while (newline != string::npos) {
3940 downloader_cat.debug()
3941 <<
"send: " << message.substr(start, newline - start - 1) <<
"\n";
3942 start = newline + 1;
3943 newline = message.find(
'\n', start);
3946 if (start < message.length()) {
3947 downloader_cat.debug()
3948 <<
"send: " << message.substr(start) <<
" (no newline)\n";
3961 reset_download_to() {
3962 _started_download =
false;
3963 close_download_stream();
3964 _download_dest = DD_none;
3974 close_download_stream() {
3975 if (_download_to_stream != NULL) {
3976 _download_to_stream->flush();
3977 if (_download_dest == DD_file) {
3981 _download_to_ramfile = (
Ramfile *)NULL;
3982 _download_to_stream = NULL;
3993 if (downloader_cat.is_spam()) {
3994 downloader_cat.spam()
3995 << _NOTIFY_HTTP_CHANNEL_ID
3996 <<
"reset_to_new.\n";
4009 reset_body_stream() {
4010 if (_owns_body_stream) {
4011 if (_body_stream != (ISocketStream *)NULL) {
4012 close_read_body(_body_stream);
4013 nassertv(_body_stream == (ISocketStream *)NULL && !_owns_body_stream);
4016 _body_stream = NULL;
4028 close_connection() {
4029 reset_body_stream();
4032 _working_get = string();
4046 more_useful_status_code(
int a,
int b) {
4047 if (a >= 100 && b >= 100) {
4060 int series_a = (a / 100);
4061 int series_b = (b / 100);
4064 return (series_a < series_b);
4067 if (a < 100 && b < 100) {
4076 return (a > SC_http_error_watermark);
4080 return (b < SC_http_error_watermark);
4089 operator << (ostream &out, HTTPChannel::State state) {
4091 return out << (int)state;
4094 case HTTPChannel::S_new:
4095 return out <<
"new";
4097 case HTTPChannel::S_try_next_proxy:
4098 return out <<
"try_next_proxy";
4100 case HTTPChannel::S_connecting:
4101 return out <<
"connecting";
4103 case HTTPChannel::S_connecting_wait:
4104 return out <<
"connecting_wait";
4106 case HTTPChannel::S_http_proxy_ready:
4107 return out <<
"http_proxy_ready";
4109 case HTTPChannel::S_http_proxy_request_sent:
4110 return out <<
"http_proxy_request_sent";
4112 case HTTPChannel::S_http_proxy_reading_header:
4113 return out <<
"http_proxy_reading_header";
4115 case HTTPChannel::S_socks_proxy_greet:
4116 return out <<
"socks_proxy_greet";
4118 case HTTPChannel::S_socks_proxy_greet_reply:
4119 return out <<
"socks_proxy_greet_reply";
4121 case HTTPChannel::S_socks_proxy_connect:
4122 return out <<
"socks_proxy_connect";
4124 case HTTPChannel::S_socks_proxy_connect_reply:
4125 return out <<
"socks_proxy_connect_reply";
4127 case HTTPChannel::S_setup_ssl:
4128 return out <<
"setup_ssl";
4130 case HTTPChannel::S_ssl_handshake:
4131 return out <<
"ssl_handshake";
4133 case HTTPChannel::S_ready:
4134 return out <<
"ready";
4136 case HTTPChannel::S_request_sent:
4137 return out <<
"request_sent";
4139 case HTTPChannel::S_reading_header:
4140 return out <<
"reading_header";
4142 case HTTPChannel::S_start_direct_file_read:
4143 return out <<
"start_direct_file_read";
4145 case HTTPChannel::S_read_header:
4146 return out <<
"read_header";
4148 case HTTPChannel::S_begin_body:
4149 return out <<
"begin_body";
4151 case HTTPChannel::S_reading_body:
4152 return out <<
"reading_body";
4154 case HTTPChannel::S_read_body:
4155 return out <<
"read_body";
4157 case HTTPChannel::S_read_trailer:
4158 return out <<
"read_trailer";
4160 case HTTPChannel::S_failure:
4161 return out <<
"failure";
4164 return out <<
"invalid state(" << (int)state <<
")";
4168 #endif // HAVE_OPENSSL static TrueClock * get_global_ptr()
Returns a pointer to the one TrueClock object in the world.
string get_server_and_port() const
Returns a string consisting of the server name, followed by a colon, followed by the port number...
A container for a URL, e.g.
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS's file system.
A container for an "entity tag" from an HTTP server.
string get_server() const
Returns the server name specified by the URL, if any.
void set_binary()
Indicates that the filename represents a binary file.
int get_port() const
Returns the port number specified by the URL, or the default port if not specified.
This is a convenience class to specialize ConfigVariable as a floating-point type.
The name of a file, such as a texture file or an Egg file.
string get_path() const
Returns the path specified by the URL, or "/" if no path is specified.
An in-memory buffer specifically designed for downloading files to memory.
string get_authority() const
Returns the authority specified by the URL (this includes username, server, and/or port)...
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
A container for an HTTP-legal time/date indication.
bool is_local() const
Returns true if the filename is local, e.g.
ostream * open_write_file(const Filename &filename, bool auto_wrap, bool truncate)
Convenience function; returns a newly allocated ostream if the file exists and can be written...
const string & get_url() const
Returns the complete URL specification.
string get_scheme() const
Returns the scheme specified by the URL, or empty string if no scheme is specified.
A descriptor that refers to a particular version of a document.
TypeHandle is the identifier used to differentiate C++ class types.
static void close_write_file(ostream *stream)
Closes a file opened by a previous call to open_write_file().
void set_username(const string &username)
Replaces the username part of the URL specification.
This class can be used to test for string matches against standard Unix-shell filename globbing conve...