37 using std::ostringstream;
40 using std::streamsize;
41 using std::stringstream;
45 const char Multifile::_header[] =
"pmf\0\n\r";
46 const size_t Multifile::_header_size = 6;
51 const int Multifile::_current_major_ver = 1;
53 const int Multifile::_current_minor_ver = 1;
62 const char Multifile::_encrypt_header[] =
"crypty";
63 const size_t Multifile::_encrypt_header_size = 6;
107 _read_filew(_read_file),
108 _read_write_filew(_read_write_file)
111 (
"multifile-encryption-iteration-count", 0,
112 PRC_DESC(
"This is a special value of encryption-iteration-count used to encrypt "
113 "subfiles within a multifile. It has a default value of 0 (just one "
114 "application), on the assumption that the files from a multifile must "
115 "be loaded quickly, without paying the cost of an expensive hash on "
116 "each subfile in order to decrypt it."));
121 _owns_stream =
false;
125 _needs_repack =
false;
127 _timestamp_dirty =
false;
128 _record_timestamp =
true;
130 _new_scale_factor = 1;
131 _encryption_flag =
false;
132 _encryption_iteration_count = multifile_encryption_iteration_count;
138 EncryptStreamBuf tbuf;
139 _encryption_algorithm = tbuf.get_algorithm();
140 _encryption_key_length = tbuf.get_key_length();
161 open_read(
const Filename &multifile_name,
const streampos &offset) {
168 if (vfile ==
nullptr) {
171 istream *multifile_stream = vfile->open_read_file(
false);
172 if (multifile_stream ==
nullptr) {
176 _timestamp = vfile->get_timestamp();
177 _timestamp_dirty =
true;
180 _multifile_name = multifile_name;
195 const streampos &offset) {
197 _timestamp = time(
nullptr);
198 _timestamp_dirty =
true;
199 _read = multifile_stream;
200 _owns_stream = owns_pointer;
222 _timestamp = time(
nullptr);
223 _timestamp_dirty =
true;
224 _write = &_write_file;
225 _multifile_name = multifile_name;
238 open_write(ostream *multifile_stream,
bool owns_pointer) {
240 _timestamp = time(
nullptr);
241 _timestamp_dirty =
true;
242 _write = multifile_stream;
243 _owns_stream = owns_pointer;
244 _write->seekp(0, ios::beg);
262 bool exists = fname.
exists();
269 _timestamp = time(
nullptr);
271 _timestamp_dirty =
true;
272 _read = &_read_write_filew;
273 _write = &_read_write_file;
274 _multifile_name = multifile_name;
295 _timestamp = time(
nullptr);
296 _timestamp_dirty =
true;
303 _write = multifile_stream;
305 _write->seekp(0, ios::beg);
308 multifile_stream->seekg(0, ios::end);
309 if (multifile_stream->tellg() == (streampos)0) {
325 if (_new_scale_factor != _scale_factor) {
335 if (_read !=
nullptr) {
337 if (!_read->
unref()) {
340 }
else if (_write !=
nullptr) {
348 _owns_stream =
false;
351 _needs_repack =
false;
353 _timestamp_dirty =
false;
355 _new_scale_factor = 1;
356 _encryption_flag =
false;
362 _read_write_file.close();
388 nassertv(scale_factor != (
size_t)0);
390 if (_next_index == (streampos)0) {
392 _scale_factor = scale_factor;
400 _new_scale_factor = scale_factor;
418 int compression_level) {
422 if (multifile_always_binary) {
431 string name = standardize_subfile_name(subfile_name);
433 Subfile *subfile =
new Subfile;
434 subfile->_name = name;
435 subfile->_source_filename = fname;
437 subfile->_flags |= SF_text;
440 add_new_subfile(subfile, compression_level);
443 _timestamp = time(
nullptr);
444 _timestamp_dirty =
true;
465 add_subfile(
const string &subfile_name, istream *subfile_data,
466 int compression_level) {
469 string name = standardize_subfile_name(subfile_name);
471 Subfile *subfile =
new Subfile;
472 subfile->_name = name;
473 subfile->_source = subfile_data;
474 add_new_subfile(subfile, compression_level);
492 int compression_level) {
496 if (multifile_always_binary) {
505 string name = standardize_subfile_name(subfile_name);
518 Subfile *subfile =
new Subfile;
519 subfile->_name = name;
520 subfile->_source_filename = fname;
522 subfile->_flags |= SF_text;
525 add_new_subfile(subfile, compression_level);
528 _timestamp = time(
nullptr);
529 _timestamp_dirty =
true;
539 Multifile::CertRecord::
540 CertRecord(X509 *cert) :
548 Multifile::CertRecord::
549 CertRecord(
const Multifile::CertRecord ©) :
550 _cert(X509_dup(copy._cert))
557 Multifile::CertRecord::
565 void Multifile::CertRecord::
566 operator = (
const Multifile::CertRecord &other) {
568 _cert = X509_dup(other._cert);
570 #endif // HAVE_OPENSSL
600 const Filename &pkey,
const string &password) {
603 if (chain.empty() && pkey.empty()) {
606 return add_signature(certificate, password);
609 CertChain cert_chain;
613 string certificate_data;
614 if (!vfs->read_file(certificate, certificate_data,
true)) {
616 <<
"Could not read " << certificate <<
".\n";
621 BIO *certificate_mbio = BIO_new_mem_buf((
void *)certificate_data.data(), certificate_data.size());
622 X509 *x509 = PEM_read_bio_X509(certificate_mbio,
nullptr,
nullptr, (
void *)
"");
623 BIO_free(certificate_mbio);
624 if (x509 ==
nullptr) {
626 <<
"Could not read certificate in " << certificate <<
".\n";
632 cert_chain.push_back(CertRecord(x509));
635 if (!chain.empty()) {
637 if (!vfs->read_file(chain, chain_data,
true)) {
639 <<
"Could not read " << chain <<
".\n";
643 BIO *chain_mbio = BIO_new_mem_buf((
void *)chain_data.data(), chain_data.size());
644 X509 *c = PEM_read_bio_X509(chain_mbio,
nullptr,
nullptr, (
void *)
"");
645 while (c !=
nullptr) {
646 cert_chain.push_back(c);
647 c = PEM_read_bio_X509(chain_mbio,
nullptr,
nullptr, (
void *)
"");
649 BIO_free(chain_mbio);
651 if (cert_chain.size() == 1) {
653 <<
"Could not read certificate chain in " << chain <<
".\n";
661 if (!vfs->read_file(pkey, pkey_data,
true)) {
663 <<
"Could not read " << pkey <<
".\n";
667 BIO *pkey_mbio = BIO_new_mem_buf((
void *)pkey_data.data(), pkey_data.size());
668 EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio,
nullptr,
nullptr,
669 (
void *)password.c_str());
671 if (evp_pkey ==
nullptr) {
673 <<
"Could not read private key in " << pkey <<
".\n";
677 bool result = add_signature(cert_chain, evp_pkey);
679 EVP_PKEY_free(evp_pkey);
683 #endif // HAVE_OPENSSL
700 add_signature(
const Filename &composite,
const string &password) {
704 string composite_data;
705 if (!vfs->read_file(composite, composite_data,
true)) {
707 <<
"Could not read " << composite <<
".\n";
712 BIO *pkey_mbio = BIO_new_mem_buf((
void *)composite_data.data(), composite_data.size());
713 EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio,
nullptr,
nullptr,
714 (
void *)password.c_str());
716 if (evp_pkey ==
nullptr) {
718 <<
"Could not read private key in " << composite <<
".\n";
723 CertChain cert_chain;
725 BIO *chain_mbio = BIO_new_mem_buf((
void *)composite_data.data(), composite_data.size());
726 X509 *c = PEM_read_bio_X509(chain_mbio,
nullptr,
nullptr, (
void *)
"");
727 while (c !=
nullptr) {
728 cert_chain.push_back(c);
729 c = PEM_read_bio_X509(chain_mbio,
nullptr,
nullptr, (
void *)
"");
731 BIO_free(chain_mbio);
733 if (cert_chain.empty()) {
735 <<
"Could not read certificates in " << composite <<
".\n";
742 bool found_match =
false;
743 for (i = 0; i < cert_chain.size(); ++i) {
744 X509 *c = cert_chain[i]._cert;
745 if (X509_check_private_key(c, evp_pkey)) {
749 cert_chain.insert(cert_chain.begin(), cert_chain[i]);
750 cert_chain.erase(cert_chain.begin() + i + 1);
758 <<
"No certificates in " << composite <<
" match key.\n";
762 bool result = add_signature(cert_chain, evp_pkey);
764 EVP_PKEY_free(evp_pkey);
768 #endif // HAVE_OPENSSL
793 add_signature(
const Multifile::CertChain &cert_chain, EVP_PKEY *pkey) {
804 if (cert_chain.empty()) {
806 <<
"No certificate given.\n";
810 if (pkey ==
nullptr) {
812 <<
"No private key given.\n";
816 if (!X509_check_private_key(cert_chain[0]._cert, pkey)) {
818 <<
"Private key does not match certificate.\n";
823 stringstream der_stream;
825 der_writer.add_uint32((uint32_t)cert_chain.size());
827 CertChain::const_iterator ci;
828 for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) {
829 X509 *cert = (*ci)._cert;
831 int der_len = i2d_X509(cert,
nullptr);
832 unsigned char *der_buf =
new unsigned char[der_len];
833 unsigned char *p = der_buf;
835 der_writer.append_data(der_buf, der_len);
841 Subfile *subfile =
new Subfile;
842 subfile->_pkey = pkey;
843 subfile->_flags |= SF_signature;
844 subfile->_source = &der_stream;
848 nassertr(_new_subfiles.empty(),
false);
849 _new_subfiles.push_back(subfile);
850 bool result =
flush();
856 #endif // HAVE_OPENSSL
871 get_num_signatures()
const {
873 return _signatures.size();
875 #endif // HAVE_OPENSSL
882 const Multifile::CertChain &Multifile::
883 get_signature(
int n)
const {
885 static CertChain error_chain;
886 nassertr(n >= 0 && n < (
int)_signatures.size(), error_chain);
887 return _signatures[n];
889 #endif // HAVE_OPENSSL
900 get_signature_subject_name(
int n)
const {
901 const CertChain &cert_chain = get_signature(n);
902 nassertr(!cert_chain.empty(),
string());
904 X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert);
905 if (xname !=
nullptr) {
908 BIO *mbio = BIO_new(BIO_s_mem());
909 X509_NAME_print_ex(mbio, xname, 0, XN_FLAG_RFC2253);
912 long pp_size = BIO_get_mem_data(mbio, &pp);
913 string name(pp, pp_size);
920 #endif // HAVE_OPENSSL
932 get_signature_friendly_name(
int n)
const {
933 const CertChain &cert_chain = get_signature(n);
934 nassertr(!cert_chain.empty(),
string());
936 static const int nid_choices[] = {
937 NID_pkcs9_emailAddress,
938 NID_subject_alt_name,
944 for (
int ni = 0; nid_choices[ni] != -1; ++ni) {
945 int nid = nid_choices[ni];
948 X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert);
949 if (xname !=
nullptr) {
950 int pos = X509_NAME_get_index_by_NID(xname, nid, -1);
954 X509_NAME_ENTRY *xentry = X509_NAME_get_entry(xname, pos);
955 if (xentry !=
nullptr) {
956 ASN1_STRING *data = X509_NAME_ENTRY_get_data(xentry);
957 if (data !=
nullptr) {
961 BIO *mbio = BIO_new(BIO_s_mem());
962 ASN1_STRING_print_ex(mbio, data, ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB);
965 long pp_size = BIO_get_mem_data(mbio, &pp);
966 string name(pp, pp_size);
977 #endif // HAVE_OPENSSL
990 get_signature_public_key(
int n)
const {
991 const CertChain &cert_chain = get_signature(n);
992 nassertr(!cert_chain.empty(),
string());
994 EVP_PKEY *pkey = X509_get_pubkey(cert_chain[0]._cert);
995 if (pkey !=
nullptr) {
996 int key_len = i2d_PublicKey(pkey,
nullptr);
997 unsigned char *key_buf =
new unsigned char[key_len];
998 unsigned char *p = key_buf;
999 i2d_PublicKey(pkey, &p);
1001 for (
int i = 0; i < key_len; ++i) {
1002 result += tohex(key_buf[i] >> 4);
1003 result += tohex(key_buf[i]);
1011 #endif // HAVE_OPENSSL
1019 print_signature_certificate(
int n, ostream &out)
const {
1020 const CertChain &cert_chain = get_signature(n);
1021 nassertv(!cert_chain.empty());
1023 BIO *mbio = BIO_new(BIO_s_mem());
1024 X509_print(mbio, cert_chain[0]._cert);
1027 long pp_size = BIO_get_mem_data(mbio, &pp);
1028 out.write(pp, pp_size);
1031 #endif // HAVE_OPENSSL
1039 write_signature_certificate(
int n, ostream &out)
const {
1040 const CertChain &cert_chain = get_signature(n);
1041 nassertv(!cert_chain.empty());
1043 BIO *mbio = BIO_new(BIO_s_mem());
1045 CertChain::const_iterator ci;
1046 for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) {
1047 X509 *c = (*ci)._cert;
1048 X509_print(mbio, c);
1049 PEM_write_bio_X509(mbio, c);
1053 long pp_size = BIO_get_mem_data(mbio, &pp);
1054 out.write(pp, pp_size);
1057 #endif // HAVE_OPENSSL
1067 validate_signature_certificate(
int n)
const {
1068 int verify_result = -1;
1070 const CertChain &chain = get_signature(n);
1071 nassertr(!chain.empty(),
false);
1073 OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
1077 X509 *x509 = chain[0]._cert;
1078 STACK_OF(X509) *stack =
nullptr;
1079 if (chain.size() > 1) {
1080 stack = sk_X509_new(
nullptr);
1081 for (
size_t n = 1; n < chain.size(); ++n) {
1082 sk_X509_push(stack, chain[n]._cert);
1087 X509_STORE_CTX *ctx = X509_STORE_CTX_new();
1088 X509_STORE_CTX_init(ctx, sslw->get_x509_store(), x509, stack);
1089 X509_STORE_CTX_set_cert(ctx, x509);
1091 if (X509_verify_cert(ctx)) {
1094 verify_result = X509_STORE_CTX_get_error(ctx);
1097 if (express_cat.is_debug()) {
1099 << get_signature_subject_name(n) <<
": validate " << verify_result
1103 sk_X509_free(stack);
1104 X509_STORE_CTX_cleanup(ctx);
1105 X509_STORE_CTX_free(ctx);
1107 return verify_result;
1109 #endif // HAVE_OPENSSL
1132 bool new_file = (_next_index == (streampos)0);
1136 if (!write_header()) {
1141 if (_file_minor_ver != _current_minor_ver) {
1148 nassertr(_write !=
nullptr,
false);
1151 PendingSubfiles::iterator pi;
1152 for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
1153 Subfile *subfile = (*pi);
1154 subfile->rewrite_index_flags(*_write);
1157 _removed_subfiles.clear();
1159 bool wrote_ok =
true;
1161 if (!_new_subfiles.empty() || new_file) {
1165 if (_last_index != (streampos)0) {
1166 _write->seekp(0, ios::end);
1167 if (_write->fail()) {
1169 <<
"Unable to seek Multifile " << _multifile_name <<
".\n";
1172 _next_index = _write->tellp();
1173 _next_index = pad_to_streampos(_next_index);
1177 _write->seekp(_last_index);
1179 writer.
add_uint32(streampos_to_word(_next_index));
1182 _write->seekp(_next_index);
1183 nassertr(_next_index == _write->tellp(),
false);
1187 for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
1188 Subfile *subfile = (*pi);
1189 _last_index = _next_index;
1190 _next_index = subfile->write_index(*_write, _next_index,
this);
1191 nassertr(_next_index == _write->tellp(),
false);
1192 _next_index = pad_to_streampos(_next_index);
1193 nassertr(_next_index == _write->tellp(),
false);
1200 nassertr(_next_index == _write->tellp(),
false);
1201 _next_index = pad_to_streampos(_next_index);
1204 for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
1205 Subfile *subfile = (*pi);
1207 if (_read !=
nullptr) {
1209 _next_index = subfile->write_data(*_write, _read->
get_istream(),
1214 _next_index = subfile->write_data(*_write,
nullptr, _next_index,
this);
1217 nassertr(_next_index == _write->tellp(),
false);
1218 _next_index = pad_to_streampos(_next_index);
1219 if (subfile->is_data_invalid()) {
1223 if (!subfile->is_cert_special()) {
1224 _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos());
1226 nassertr(_next_index == _write->tellp(),
false);
1233 for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
1234 Subfile *subfile = (*pi);
1235 subfile->rewrite_index_data_start(*_write,
this);
1238 _new_subfiles.clear();
1242 if (_timestamp_dirty) {
1243 nassertr(!_write->fail(),
false);
1244 static const size_t timestamp_pos = _header_prefix.size() + _header_size + 2 + 2 + 4;
1245 _write->seekp(timestamp_pos);
1246 nassertr(!_write->fail(),
false);
1249 if (_record_timestamp) {
1254 _timestamp_dirty =
false;
1258 if (!wrote_ok || _write->fail()) {
1260 <<
"Unable to update Multifile " << _multifile_name <<
".\n";
1282 if (_next_index == (streampos)0) {
1285 _needs_repack =
false;
1290 nassertr(!_multifile_name.empty(),
false);
1294 if (dirname.empty()) {
1302 <<
"Unable to open temporary file " << temp_filename <<
"\n";
1308 PendingSubfiles::iterator pi;
1309 for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
1310 Subfile *subfile = (*pi);
1313 _removed_subfiles.clear();
1314 _new_subfiles.clear();
1315 std::copy(_subfiles.
begin(), _subfiles.
end(), std::back_inserter(_new_subfiles));
1318 _last_data_byte = 0;
1319 _scale_factor = _new_scale_factor;
1331 Filename orig_name = _multifile_name;
1335 if (!temp_filename.
rename_to(orig_name)) {
1337 <<
"Unable to rename temporary file " << temp_filename <<
" to "
1338 << orig_name <<
".\n";
1344 <<
"Unable to read newly repacked " << _multifile_name
1359 return _subfiles.
size();
1369 find_subfile._name = standardize_subfile_name(subfile_name);
1370 Subfiles::const_iterator fi;
1372 if (fi == _subfiles.
end()) {
1376 return (fi - _subfiles.
begin());
1386 string prefix = subfile_name;
1387 if (!prefix.empty()) {
1392 Subfiles::const_iterator fi;
1394 if (fi == _subfiles.
end()) {
1401 Subfile *subfile = (*fi);
1402 return (subfile->_name.length() > prefix.length() &&
1403 subfile->_name.substr(0, prefix.length()) == prefix);
1417 scan_directory(vector_string &contents,
const string &subfile_name)
const {
1418 string prefix = subfile_name;
1419 if (!prefix.empty()) {
1424 Subfiles::const_iterator fi;
1427 string previous =
"";
1428 while (fi != _subfiles.
end()) {
1429 Subfile *subfile = (*fi);
1430 if (!(subfile->_name.length() > prefix.length() &&
1431 subfile->_name.substr(0, prefix.length()) == prefix)) {
1437 size_t slash = subfile->_name.find(
'/', prefix.length());
1438 string basename = subfile->_name.substr(prefix.length(), slash - prefix.length());
1439 if (basename != previous) {
1440 contents.push_back(basename);
1441 previous = basename;
1461 nassertv(index >= 0 && index < (
int)_subfiles.
size());
1462 Subfile *subfile = _subfiles[index];
1463 subfile->_flags |= SF_deleted;
1464 _removed_subfiles.push_back(subfile);
1465 _subfiles.erase(_subfiles.
begin() + index);
1467 _timestamp = time(
nullptr);
1468 _timestamp_dirty =
true;
1470 _needs_repack =
true;
1479 static string empty_string;
1480 nassertr(index >= 0 && index < (
int)_subfiles.
size(), empty_string);
1482 return _subfiles[index]->_name;
1492 nassertr(index >= 0 && index < (
int)_subfiles.
size(), 0);
1493 return _subfiles[index]->_uncompressed_length;
1504 nassertr(index >= 0 && index < (
int)_subfiles.
size(), 0);
1508 return _subfiles[index]->_timestamp;
1518 nassertr(index >= 0 && index < (
int)_subfiles.
size(),
false);
1519 return (_subfiles[index]->_flags & SF_compressed) != 0;
1528 nassertr(index >= 0 && index < (
int)_subfiles.
size(),
false);
1529 return (_subfiles[index]->_flags & SF_encrypted) != 0;
1541 nassertr(index >= 0 && index < (
int)_subfiles.
size(),
false);
1542 return (_subfiles[index]->_flags & SF_text) != 0;
1556 return normalize_streampos(_next_index + (streampos)4);
1568 nassertr(index >= 0 && index < (
int)_subfiles.
size(), 0);
1569 return _subfiles[index]->_data_start;
1581 nassertr(index >= 0 && index < (
int)_subfiles.
size(), 0);
1582 return _subfiles[index]->_data_length;
1605 nassertr(index >= 0 && index < (
int)_subfiles.
size(),
nullptr);
1606 Subfile *subfile = _subfiles[index];
1608 if (subfile->_source !=
nullptr ||
1609 !subfile->_source_filename.empty()) {
1615 nassertr(subfile == _subfiles[index],
nullptr);
1629 if (stream !=
nullptr) {
1634 #if !defined(WIN32_VC) && !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW)
1636 (*global_operator_delete)(stream);
1649 nassertr(index >= 0 && index < (
int)_subfiles.
size(),
false);
1652 if (multifile_always_binary) {
1659 if ((_subfiles[index]->_flags & SF_text) != 0) {
1669 <<
"Unable to write to file " << filename <<
"\n";
1682 nassertr(index >= 0 && index < (
int)_subfiles.
size(),
false);
1685 if (in ==
nullptr) {
1689 static const size_t buffer_size = 4096;
1690 char buffer[buffer_size];
1692 in->read(buffer, buffer_size);
1693 size_t count = in->gcount();
1694 while (count != 0) {
1695 out.write(buffer, count);
1696 in->read(buffer, buffer_size);
1697 count = in->gcount();
1700 bool failed = (in->fail() && !in->eof());
1702 nassertr(!failed,
false);
1704 return (!out.fail());
1720 nassertr(index >= 0 && index < (
int)_subfiles.
size(),
false);
1722 if (!filename.
exists()) {
1724 <<
"File is missing: " << filename <<
"\n";
1731 if ((_subfiles[index]->_flags & SF_text) != 0) {
1732 if (express_cat.is_debug()) {
1734 <<
"File is not binary: " << filename <<
"\n";
1741 if ((_subfiles[index]->_flags & SF_text) == 0) {
1742 if (express_cat.is_debug()) {
1744 <<
"File is not text: " << filename <<
"\n";
1752 if ((_subfiles[index]->_flags & SF_text) != 0) {
1760 if (in1 ==
nullptr) {
1768 <<
"Cannot read " << filename <<
"\n";
1774 in2.seekg(0, ios::end);
1775 streampos file_size = in2.tellg();
1786 int byte1 = in1->get();
1787 int byte2 = in2.get();
1788 while (!in1->fail() && !in2.fail()) {
1789 if (byte1 != byte2) {
1797 bool failed = (in1->fail() && !in1->eof()) || (in2.fail() && !in2.eof());
1800 nassertr(!failed,
false);
1809 output(ostream &out)
const {
1818 ls(ostream &out)
const {
1820 for (
int i = 0; i < num_subfiles; i++) {
1822 out << subfile_name <<
"\n";
1841 string new_header_prefix = header_prefix;
1843 if (!new_header_prefix.empty()) {
1845 if (new_header_prefix[0] !=
'#') {
1846 new_header_prefix = string(
"#") + new_header_prefix;
1850 if (new_header_prefix[new_header_prefix.size() - 1] !=
'\n') {
1851 new_header_prefix += string(
"\n");
1855 size_t newline = new_header_prefix.find(
'\n');
1856 while (newline < new_header_prefix.size() - 1) {
1857 if (new_header_prefix[newline + 1] !=
'#') {
1858 new_header_prefix = new_header_prefix.substr(0, newline + 1) + string(
"#") + new_header_prefix.substr(newline + 1);
1860 newline = new_header_prefix.find(
'#', newline);
1864 if (_header_prefix != new_header_prefix) {
1865 _header_prefix = new_header_prefix;
1866 _needs_repack =
true;
1887 result.append((
const char *)&pv[0], pv.size());
1899 nassertr(index >= 0 && index < (
int)_subfiles.
size(),
false);
1904 nassertr(index >= 0 && index < (
int)_subfiles.
size(),
false);
1905 Subfile *subfile = _subfiles[index];
1907 if (subfile->_source !=
nullptr ||
1908 !subfile->_source_filename.empty()) {
1914 nassertr(subfile == _subfiles[index],
false);
1917 result.reserve(subfile->_uncompressed_length);
1919 bool success =
true;
1920 if (subfile->_flags & (SF_encrypted | SF_compressed)) {
1924 if (in ==
nullptr) {
1934 static const size_t buffer_size = 4096;
1935 char buffer[buffer_size];
1937 streamsize pos = _offset + subfile->_data_start;
1938 size_t max_bytes = subfile->_data_length;
1939 streamsize count = 0;
1942 streamsize num_bytes = (streamsize)min(buffer_size, max_bytes);
1943 _read->
seek_read(pos, buffer, num_bytes, count, eof);
1944 while (count != 0) {
1945 thread_consider_yield();
1946 nassertr(count <= (streamsize)max_bytes,
false);
1947 result.insert(result.end(), buffer, buffer + (
size_t)count);
1948 max_bytes -= (size_t)count;
1951 num_bytes = (streamsize)min(buffer_size, max_bytes);
1952 _read->
seek_read(pos, buffer, num_bytes, count, eof);
1959 ostringstream message;
1962 nassert_raise(message.str());
1974 streampos Multifile::
1975 pad_to_streampos(streampos fpos) {
1976 nassertr(_write !=
nullptr, fpos);
1977 nassertr(_write->tellp() == fpos, fpos);
1978 streampos new_fpos = normalize_streampos(fpos);
1979 while (fpos < new_fpos) {
1983 nassertr(_write->tellp() == fpos, fpos);
1991 add_new_subfile(Subfile *subfile,
int compression_level) {
1992 if (compression_level != 0) {
1994 express_cat.warning()
1995 <<
"zlib not compiled in; cannot generated compressed multifiles.\n";
1996 compression_level = 0;
1998 subfile->_flags |= SF_compressed;
1999 subfile->_compression_level = compression_level;
2004 if (_encryption_flag) {
2005 subfile->_flags |= SF_encrypted;
2007 #endif // HAVE_OPENSSL
2009 if (_next_index != (streampos)0) {
2012 _needs_repack =
true;
2015 std::pair<Subfiles::iterator, bool> insert_result = _subfiles.insert(subfile);
2016 if (!insert_result.second) {
2019 Subfile *old_subfile = (*insert_result.first);
2020 old_subfile->_flags |= SF_deleted;
2024 PendingSubfiles::iterator ni = find(_new_subfiles.begin(), _new_subfiles.end(), old_subfile);
2025 if (ni != _new_subfiles.end()) {
2026 _new_subfiles.erase(ni);
2031 _removed_subfiles.push_back(old_subfile);
2034 (*insert_result.first) = subfile;
2037 _new_subfiles.push_back(subfile);
2047 nassertr(subfile->_source ==
nullptr &&
2048 subfile->_source_filename.empty(),
nullptr);
2052 nassertr(subfile->_data_start != (streampos)0,
nullptr);
2054 new ISubStream(_read, _offset + subfile->_data_start,
2055 _offset + subfile->_data_start + (streampos)subfile->_data_length);
2057 if ((subfile->_flags & SF_encrypted) != 0) {
2058 #ifndef HAVE_OPENSSL
2060 <<
"OpenSSL not compiled in; cannot read encrypted multifiles.\n";
2063 #else // HAVE_OPENSSL
2066 IDecryptStream *wrapper =
2067 new IDecryptStream(stream,
true, _encryption_password);
2071 char this_header[_encrypt_header_size];
2072 stream->read(this_header, _encrypt_header_size);
2073 if (stream->fail() || stream->gcount() != (
unsigned)_encrypt_header_size ||
2074 memcmp(this_header, _encrypt_header, _encrypt_header_size) != 0) {
2076 <<
"Unable to decrypt subfile " << subfile->_name <<
".\n";
2080 #endif // HAVE_OPENSSL
2083 if ((subfile->_flags & SF_compressed) != 0) {
2086 <<
"zlib not compiled in; cannot read compressed multifiles.\n";
2092 IDecompressStream *wrapper =
new IDecompressStream(stream,
true);
2097 if (stream->fail()) {
2110 standardize_subfile_name(
const string &subfile_name)
const {
2113 if (name.empty() || name ==
"/") {
2118 if (name[0] ==
'/') {
2120 }
else if (name.length() > 2 && name[0] ==
'.' && name[1] ==
'/') {
2133 PendingSubfiles::iterator pi;
2134 for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
2135 Subfile *subfile = (*pi);
2136 subfile->rewrite_index_flags(*_write);
2139 _removed_subfiles.clear();
2143 _new_subfiles.clear();
2146 for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) {
2147 Subfile *subfile = (*pi);
2150 _cert_special.clear();
2152 _signatures.clear();
2153 #endif // HAVE_OPENSSL
2155 Subfiles::iterator fi;
2156 for (fi = _subfiles.
begin(); fi != _subfiles.
end(); ++fi) {
2157 Subfile *subfile = (*fi);
2169 nassertr(_read !=
nullptr,
false);
2175 char this_header[_header_size];
2176 read->seekg(_offset);
2183 _header_prefix = string();
2184 int ch = read->get();
2187 while (ch != EOF && ch ==
'#') {
2189 while (ch != EOF && ch !=
'\n') {
2190 _header_prefix += ch;
2194 while (ch != EOF && (isspace(ch) || ch ==
'\r')) {
2195 _header_prefix += ch;
2202 this_header[0] = ch;
2203 read->read(this_header + 1, _header_size - 1);
2204 if (read->fail() || read->gcount() != (
unsigned)(_header_size - 1)) {
2206 <<
"Unable to read Multifile header " << _multifile_name <<
".\n";
2212 if (memcmp(this_header, _header, _header_size) != 0) {
2214 << _multifile_name <<
" is not a Multifile.\n";
2222 _file_major_ver = reader.get_int16();
2223 _file_minor_ver = reader.get_int16();
2224 _scale_factor = reader.get_uint32();
2225 _new_scale_factor = _scale_factor;
2227 if (read->eof() || read->fail()) {
2229 << _multifile_name <<
" header is truncated.\n";
2235 if (_file_major_ver != _current_major_ver ||
2236 (_file_major_ver == _current_major_ver &&
2237 _file_minor_ver > _current_minor_ver)) {
2239 << _multifile_name <<
" has version " << _file_major_ver <<
"."
2240 << _file_minor_ver <<
", expecting version "
2241 << _current_major_ver <<
"." << _current_minor_ver <<
".\n";
2247 _record_timestamp =
true;
2248 if (_file_minor_ver >= 1) {
2249 time_t read_timestamp = reader.get_uint32();
2250 if (read_timestamp == 0) {
2253 _record_timestamp =
false;
2255 _timestamp = read_timestamp;
2257 _timestamp_dirty =
false;
2261 streampos curr_pos = read->tellg() - _offset;
2262 _next_index = normalize_streampos(curr_pos);
2263 if (_next_index > curr_pos) {
2264 read->ignore(_next_index - curr_pos);
2267 _last_data_byte = 0;
2268 streampos index_forward;
2269 streamoff bytes_skipped = 0;
2270 bool read_cert_special =
false;
2272 Subfile *subfile =
new Subfile;
2273 index_forward = subfile->read_index(*read, _next_index,
this);
2274 while (index_forward != (streampos)0) {
2275 _last_index = _next_index;
2276 if (subfile->is_deleted()) {
2278 _needs_repack =
true;
2280 }
else if (subfile->is_cert_special()) {
2282 _cert_special.push_back(subfile);
2283 read_cert_special =
true;
2287 if (!subfile->is_cert_special()) {
2288 if (bytes_skipped != 0) {
2291 _needs_repack =
true;
2293 if (read_cert_special) {
2296 _needs_repack =
true;
2298 _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos());
2300 streampos curr_pos = read->tellg() - _offset;
2301 bytes_skipped = index_forward - normalize_streampos(curr_pos);
2302 _next_index = index_forward;
2303 if (_next_index > curr_pos) {
2304 read->ignore(_next_index - curr_pos);
2306 subfile =
new Subfile;
2307 index_forward = subfile->read_index(*read, _next_index,
this);
2309 if (subfile->is_index_invalid()) {
2311 <<
"Error reading index for " << _multifile_name <<
".\n";
2319 for (
size_t si = 1; si < _subfiles.
size() && !_needs_repack; ++si) {
2320 if (*_subfiles[si] < *_subfiles[si - 1]) {
2321 _needs_repack =
true;
2325 if (_needs_repack) {
2327 size_t before_size = _subfiles.
size();
2329 size_t after_size = _subfiles.
size();
2333 nassertr(before_size == after_size,
true);
2346 _file_major_ver = _current_major_ver;
2347 _file_minor_ver = _current_minor_ver;
2349 nassertr(_write !=
nullptr,
false);
2350 nassertr(_write->tellp() == (streampos)0,
false);
2351 _write->write(_header_prefix.data(), _header_prefix.size());
2352 _write->write(_header, _header_size);
2354 writer.add_int16(_current_major_ver);
2355 writer.add_int16(_current_minor_ver);
2356 writer.add_uint32(_scale_factor);
2358 if (_record_timestamp) {
2359 writer.add_uint32(_timestamp);
2361 writer.add_uint32(0);
2362 _timestamp_dirty =
false;
2365 _next_index = _write->tellp();
2366 _next_index = pad_to_streampos(_next_index);
2369 if (_write->fail()) {
2371 <<
"Unable to write header for " << _multifile_name <<
".\n";
2388 check_signatures() {
2390 PendingSubfiles::iterator pi;
2392 for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) {
2393 Subfile *subfile = (*pi);
2394 nassertv((subfile->_flags & SF_signature) != 0);
2398 nassertv(stream !=
nullptr);
2400 size_t sig_size = reader.get_uint32();
2401 vector_uchar sig_data = reader.extract_bytes(sig_size);
2403 size_t num_certs = reader.get_uint32();
2406 vector_uchar buffer;
2414 EVP_PKEY *pkey =
nullptr;
2415 if (!buffer.empty()) {
2416 #if OPENSSL_VERSION_NUMBER >= 0x00908000L
2418 const unsigned char *bp, *bp_end;
2421 unsigned char *bp, *bp_end;
2423 bp = (
unsigned char *)&buffer[0];
2424 bp_end = bp + buffer.size();
2425 X509 *x509 = d2i_X509(
nullptr, &bp, bp_end - bp);
2426 while (num_certs > 0 && x509 !=
nullptr) {
2427 chain.push_back(CertRecord(x509));
2429 x509 = d2i_X509(
nullptr, &bp, bp_end - bp);
2431 if (num_certs != 0 || x509 !=
nullptr) {
2432 express_cat.warning()
2433 <<
"Extra data in signature record.\n";
2437 if (!chain.empty()) {
2438 pkey = X509_get_pubkey(chain[0]._cert);
2441 if (pkey !=
nullptr) {
2442 EVP_MD_CTX *md_ctx = EVP_MD_CTX_create();
2443 EVP_VerifyInit(md_ctx, EVP_sha1());
2445 nassertv(_read !=
nullptr);
2451 read->seekg(_offset);
2452 streampos bytes_remaining = _last_data_byte;
2453 static const size_t buffer_size = 4096;
2454 char buffer[buffer_size];
2455 read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2456 size_t count = read->gcount();
2457 while (count != 0) {
2458 nassertv(count <= buffer_size);
2459 EVP_VerifyUpdate(md_ctx, buffer, count);
2460 bytes_remaining -= count;
2461 read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2462 count = read->gcount();
2464 nassertv(bytes_remaining == (streampos)0);
2469 EVP_VerifyFinal(md_ctx, sig_data.data(), sig_data.size(), pkey);
2470 if (verify_result == 1) {
2472 _signatures.push_back(chain);
2475 _needs_repack =
true;
2479 #endif // HAVE_OPENSSL
2481 _cert_special.clear();
2490 streampos Multifile::Subfile::
2491 read_index(istream &read, streampos fpos,
Multifile *multifile) {
2492 nassertr(read.tellg() - multifile->_offset == fpos, fpos);
2498 streampos next_index = multifile->word_to_streampos(reader.get_uint32());
2500 _flags |= SF_index_invalid;
2504 if (next_index == (streampos)0) {
2510 _index_start = fpos;
2513 _data_start = multifile->word_to_streampos(reader.get_uint32());
2514 _data_length = reader.get_uint32();
2515 _flags = reader.get_uint16();
2516 if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
2517 _uncompressed_length = reader.get_uint32();
2519 _uncompressed_length = _data_length;
2521 if (multifile->_file_minor_ver < 1) {
2524 _timestamp = reader.get_uint32();
2525 if (_timestamp == 0) {
2530 size_t name_length = reader.get_uint16();
2532 _flags |= SF_index_invalid;
2537 char *name_buffer = (
char *)PANDA_MALLOC_ARRAY(name_length);
2538 nassertr(name_buffer !=
nullptr, next_index);
2539 for (
size_t ni = 0; ni < name_length; ni++) {
2540 name_buffer[ni] = read.get() ^ 0xff;
2542 _name = string(name_buffer, name_length);
2543 PANDA_FREE_ARRAY(name_buffer);
2546 _flags |= SF_index_invalid;
2550 _index_length = read.tellg() - fpos - multifile->_offset;
2562 streampos Multifile::Subfile::
2563 write_index(ostream &write, streampos fpos,
Multifile *multifile) {
2564 nassertr(write.tellp() - multifile->_offset == fpos, fpos);
2566 _index_start = fpos;
2572 dg.
add_uint32(multifile->streampos_to_word(_data_start));
2575 if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
2585 string::iterator ni;
2586 for (ni = _name.begin(); ni != _name.end(); ++ni) {
2590 size_t this_index_size = 4 + dg.
get_length();
2593 streampos next_index = fpos + (streampos)this_index_size;
2596 idg.
add_uint32(multifile->streampos_to_word(next_index));
2598 write.write((
const char *)idg.get_data(), idg.get_length());
2601 _index_length = write.tellp() - fpos - multifile->_offset;
2619 streampos Multifile::Subfile::
2620 write_data(ostream &write, istream *read, streampos fpos,
2622 nassertr(write.tellp() - multifile->_offset == fpos, fpos);
2624 istream *source = _source;
2625 pifstream source_file;
2626 if (source ==
nullptr && !_source_filename.empty()) {
2628 if (!_source_filename.open_read(source_file)) {
2631 <<
"Unable to read " << _source_filename <<
".\n";
2632 _flags |= SF_data_invalid;
2634 _uncompressed_length = 0;
2636 source = &source_file;
2640 if (source ==
nullptr) {
2643 if (read ==
nullptr) {
2646 <<
"No source for subfile " << _name <<
".\n";
2647 _flags |= SF_data_invalid;
2650 read->seekg(_data_start + multifile->_offset);
2651 for (
size_t p = 0; p < _data_length; p++) {
2652 int byte = read->get();
2653 if (read->eof() || read->fail()) {
2656 <<
"Unexpected EOF for subfile " << _name <<
".\n";
2657 _flags |= SF_data_invalid;
2665 ostream *putter = &write;
2666 bool delete_putter =
false;
2668 #ifndef HAVE_OPENSSL
2671 nassertr((_flags & SF_encrypted) == 0, fpos);
2673 #else // HAVE_OPENSSL
2674 if ((_flags & SF_encrypted) != 0) {
2676 OEncryptStream *encrypt =
new OEncryptStream;
2677 encrypt->set_iteration_count(multifile->_encryption_iteration_count);
2678 encrypt->open(putter, delete_putter, multifile->_encryption_password);
2681 delete_putter =
true;
2685 putter->write(_encrypt_header, _encrypt_header_size);
2687 #endif // HAVE_OPENSSL
2692 nassertr((_flags & SF_compressed) == 0, fpos);
2694 if ((_flags & SF_compressed) != 0) {
2696 putter =
new OCompressStream(putter, delete_putter, _compression_level);
2697 delete_putter =
true;
2701 streampos write_start = fpos;
2702 _uncompressed_length = 0;
2704 #ifndef HAVE_OPENSSL
2706 nassertr((_flags & SF_signature) == 0, fpos);
2708 #else // HAVE_OPENSSL
2709 if ((_flags & SF_signature) != 0) {
2716 nassertr(read !=
nullptr, fpos);
2719 nassertr(_pkey !=
nullptr, fpos);
2721 EVP_MD_CTX *md_ctx = EVP_MD_CTX_create();
2722 EVP_SignInit(md_ctx, EVP_sha1());
2726 nassertr(multifile->_last_data_byte < fpos, fpos);
2727 read->seekg(multifile->_offset);
2728 streampos bytes_remaining = multifile->_last_data_byte;
2729 static const size_t buffer_size = 4096;
2730 char buffer[buffer_size];
2731 read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2732 size_t count = read->gcount();
2733 while (count != 0) {
2734 nassertr(count <= buffer_size, fpos);
2735 EVP_SignUpdate(md_ctx, buffer, count);
2736 bytes_remaining -= count;
2737 read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2738 count = read->gcount();
2740 nassertr(bytes_remaining == (streampos)0, fpos);
2743 unsigned int max_size = EVP_PKEY_size(_pkey);
2744 unsigned char *sig_data =
new unsigned char[max_size];
2745 unsigned int sig_size;
2746 if (!EVP_SignFinal(md_ctx, sig_data, &sig_size, _pkey)) {
2747 OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
2748 sslw->notify_ssl_errors();
2750 nassertr(sig_size <= max_size, fpos);
2753 writer.add_uint32(sig_size);
2754 putter->write((
char *)sig_data, sig_size);
2755 _uncompressed_length += 4 + sig_size;
2759 EVP_MD_CTX_destroy(md_ctx);
2761 #endif // HAVE_OPENSSL
2764 static const size_t buffer_size = 4096;
2765 char buffer[buffer_size];
2767 source->read(buffer, buffer_size);
2768 size_t count = source->gcount();
2769 while (count != 0) {
2770 _uncompressed_length += count;
2771 putter->write(buffer, count);
2772 source->read(buffer, buffer_size);
2773 count = source->gcount();
2776 if (delete_putter) {
2780 streampos write_end = write.tellp() - multifile->_offset;
2781 _data_length = (size_t)(write_end - write_start);
2790 if (!_source_filename.empty()) {
2791 _timestamp = _source_filename.get_timestamp();
2793 if (_timestamp == 0) {
2794 _timestamp = time(
nullptr);
2799 source_file.close();
2801 return fpos + (streampos)_data_length;
2808 void Multifile::Subfile::
2809 rewrite_index_data_start(ostream &write,
Multifile *multifile) {
2810 nassertv(_index_start != (streampos)0);
2812 static const size_t data_start_offset = 4;
2813 size_t data_start_pos = _index_start + (streampos)data_start_offset;
2814 write.seekp(data_start_pos + multifile->_offset);
2815 nassertv(!write.fail());
2818 writer.add_uint32(multifile->streampos_to_word(_data_start));
2819 writer.add_uint32(_data_length);
2820 writer.add_uint16(_flags);
2821 if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
2822 writer.add_uint32(_uncompressed_length);
2824 if (multifile->_record_timestamp) {
2825 writer.add_uint32(_timestamp);
2827 writer.add_uint32(0);
2835 void Multifile::Subfile::
2836 rewrite_index_flags(ostream &write) {
2839 if (_index_start != (streampos)0) {
2840 static const size_t flags_offset = 4 + 4 + 4;
2841 size_t flags_pos = _index_start + (streampos)flags_offset;
2842 write.seekp(flags_pos);
2843 nassertv(!write.fail());
2846 writer.add_uint16(_flags);