Panda3D
|
00001 // Filename: multifile.cxx 00002 // Created by: mike (09Jan97) 00003 // 00004 //////////////////////////////////////////////////////////////////// 00005 // 00006 // PANDA 3D SOFTWARE 00007 // Copyright (c) Carnegie Mellon University. All rights reserved. 00008 // 00009 // All use of this software is subject to the terms of the revised BSD 00010 // license. You should have received a copy of this license along 00011 // with this source code in a file named "LICENSE." 00012 // 00013 //////////////////////////////////////////////////////////////////// 00014 00015 #include "multifile.h" 00016 00017 #include "config_express.h" 00018 #include "streamWriter.h" 00019 #include "streamReader.h" 00020 #include "datagram.h" 00021 #include "zStream.h" 00022 #include "encryptStream.h" 00023 #include "virtualFileSystem.h" 00024 #include "virtualFile.h" 00025 00026 #include <algorithm> 00027 #include <iterator> 00028 #include <time.h> 00029 00030 // This sequence of bytes begins each Multifile to identify it as a 00031 // Multifile. 00032 const char Multifile::_header[] = "pmf\0\n\r"; 00033 const size_t Multifile::_header_size = 6; 00034 00035 // These numbers identify the version of the Multifile. Generally, a 00036 // change in the major version is intolerable; while a Multifile with 00037 // an older minor version may still be read. 00038 const int Multifile::_current_major_ver = 1; 00039 00040 const int Multifile::_current_minor_ver = 1; 00041 // Bumped to version 1.1 on 6/8/06 to add timestamps. 00042 00043 // To confirm that the supplied password matches, we write the 00044 // Mutifile magic header at the beginning of the encrypted stream. 00045 // I suppose this does compromise the encryption security a tiny 00046 // bit by making it easy for crackers to validate that a 00047 // particular password guess matches or doesn't match, but the 00048 // encryption algorithm doesn't depend on this being difficult 00049 // anyway. 00050 const char Multifile::_encrypt_header[] = "crypty"; 00051 const size_t Multifile::_encrypt_header_size = 6; 00052 00053 00054 00055 // 00056 // A Multifile consists of the following elements: 00057 // 00058 // (1) A header. This is always the first n bytes of the Multifile, 00059 // and contains a magic number to identify the file, as well as 00060 // version numbers and any file-specific parameters. 00061 // 00062 // char[6] The string Multifile::_header, a magic number. 00063 // int16 The file's major version number 00064 // int16 The file's minor version number 00065 // uint32 Scale factor. This scales all address references within 00066 // the file. Normally 1, this may be set larger to 00067 // support Multifiles larger than 4GB. 00068 // uint32 An overall modification timestamp for the entire multifile. 00069 00070 // 00071 // (2) Zero or more index entries, one for each subfile within the 00072 // Multifile. These entries are of variable length. The first one of 00073 // these immediately follows the header, and the first word of each 00074 // index entry contains the address of the next index entry. A zero 00075 // "next" address marks the end of the chain. These may appear at any 00076 // point within the Multifile; they do not necessarily appear in 00077 // sequential order at the beginning of the file (although they will 00078 // after the file has been "packed"). 00079 // 00080 // uint32 The address of the next entry. 0 to mark the end. 00081 // uint32 The address of this subfile's data record. 00082 // uint32 The length in bytes of this subfile's data record. 00083 // uint16 The Subfile::_flags member. 00084 // [uint32] The original, uncompressed and unencrypted length of the 00085 // subfile, if it is compressed or encrypted. This field 00086 // is only present if one or both of the SF_compressed 00087 // or SF_encrypted bits are set in _flags. 00088 // uint32 A modification timestamp for the subfile. 00089 // uint16 The length in bytes of the subfile's name. 00090 // char[n] The subfile's name. 00091 // 00092 // (3) Zero or more data entries, one for each subfile. These may 00093 // appear at any point within the Multifile; they do not necessarily 00094 // follow each index entry, nor are they necessarily all grouped 00095 // together at the end (although they will be all grouped together at 00096 // the end after the file has been "packed"). These are just blocks 00097 // of literal data. 00098 // 00099 00100 //////////////////////////////////////////////////////////////////// 00101 // Function: Multifile::Constructor 00102 // Access: Published 00103 // Description: 00104 //////////////////////////////////////////////////////////////////// 00105 Multifile:: 00106 Multifile() : 00107 _read_filew(_read_file), 00108 _read_write_filew(_read_write_file) 00109 { 00110 ConfigVariableInt multifile_encryption_iteration_count 00111 ("multifile-encryption-iteration-count", 0, 00112 PRC_DESC("This is a special value of encryption-iteration-count used to encrypt " 00113 "subfiles within a multifile. It has a default value of 0 (just one " 00114 "application), on the assumption that the files from a multifile must " 00115 "be loaded quickly, without paying the cost of an expensive hash on " 00116 "each subfile in order to decrypt it.")); 00117 00118 _read = (IStreamWrapper *)NULL; 00119 _write = (ostream *)NULL; 00120 _offset = 0; 00121 _owns_stream = false; 00122 _next_index = 0; 00123 _last_index = 0; 00124 _last_data_byte = 0; 00125 _needs_repack = false; 00126 _timestamp = 0; 00127 _timestamp_dirty = false; 00128 _record_timestamp = true; 00129 _scale_factor = 1; 00130 _new_scale_factor = 1; 00131 _encryption_flag = false; 00132 _encryption_iteration_count = multifile_encryption_iteration_count; 00133 _file_major_ver = 0; 00134 _file_minor_ver = 0; 00135 00136 #ifdef HAVE_OPENSSL 00137 // Get these values from the config file via an EncryptStreamBuf. 00138 EncryptStreamBuf tbuf; 00139 _encryption_algorithm = tbuf.get_algorithm(); 00140 _encryption_key_length = tbuf.get_key_length(); 00141 #endif 00142 } 00143 00144 //////////////////////////////////////////////////////////////////// 00145 // Function: Multifile::Destructor 00146 // Access: Published 00147 // Description: 00148 //////////////////////////////////////////////////////////////////// 00149 Multifile:: 00150 ~Multifile() { 00151 close(); 00152 } 00153 00154 //////////////////////////////////////////////////////////////////// 00155 // Function: Multifile::Copy Constructor 00156 // Access: Private 00157 // Description: Don't try to copy Multifiles. 00158 //////////////////////////////////////////////////////////////////// 00159 Multifile:: 00160 Multifile(const Multifile ©) : 00161 _read_filew(_read_file), 00162 _read_write_filew(_read_write_file) 00163 { 00164 nassertv(false); 00165 } 00166 00167 //////////////////////////////////////////////////////////////////// 00168 // Function: Multifile::Copy Assignment Operator 00169 // Access: Private 00170 // Description: Don't try to copy Multifiles. 00171 //////////////////////////////////////////////////////////////////// 00172 void Multifile:: 00173 operator = (const Multifile ©) { 00174 nassertv(false); 00175 } 00176 00177 //////////////////////////////////////////////////////////////////// 00178 // Function: Multifile::open_read 00179 // Access: Published 00180 // Description: Opens the named Multifile on disk for reading. The 00181 // Multifile index is read in, and the list of subfiles 00182 // becomes available; individual subfiles may then be 00183 // extracted or read, but the list of subfiles may not 00184 // be modified. 00185 // 00186 // Also see the version of open_read() which accepts an 00187 // istream. Returns true on success, false on failure. 00188 //////////////////////////////////////////////////////////////////// 00189 bool Multifile:: 00190 open_read(const Filename &multifile_name, const streampos &offset) { 00191 close(); 00192 Filename fname = multifile_name; 00193 fname.set_binary(); 00194 00195 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00196 PT(VirtualFile) vfile = vfs->get_file(fname); 00197 if (vfile == NULL) { 00198 return false; 00199 } 00200 istream *multifile_stream = vfile->open_read_file(false); 00201 if (multifile_stream == NULL) { 00202 return false; 00203 } 00204 00205 _timestamp = vfile->get_timestamp(); 00206 _timestamp_dirty = true; 00207 _read = new IStreamWrapper(multifile_stream, true); 00208 _owns_stream = true; 00209 _multifile_name = multifile_name; 00210 _offset = offset; 00211 return read_index(); 00212 } 00213 00214 //////////////////////////////////////////////////////////////////// 00215 // Function: Multifile::open_read 00216 // Access: Public 00217 // Description: Opens an anonymous Multifile for reading using an 00218 // istream. There must be seek functionality via 00219 // seekg() and tellg() on the istream. 00220 // 00221 // If owns_pointer is true, then the Multifile assumes 00222 // ownership of the stream pointer and will delete it 00223 // when the multifile is closed, including if this 00224 // function returns false. 00225 //////////////////////////////////////////////////////////////////// 00226 bool Multifile:: 00227 open_read(IStreamWrapper *multifile_stream, bool owns_pointer, 00228 const streampos &offset) { 00229 close(); 00230 _timestamp = time(NULL); 00231 _timestamp_dirty = true; 00232 _read = multifile_stream; 00233 _owns_stream = owns_pointer; 00234 _offset = offset; 00235 return read_index(); 00236 } 00237 00238 //////////////////////////////////////////////////////////////////// 00239 // Function: Multifile::open_write 00240 // Access: Published 00241 // Description: Opens the named Multifile on disk for writing. If 00242 // there already exists a file by that name, it is 00243 // truncated. The Multifile is then prepared for 00244 // accepting a brand new set of subfiles, which will be 00245 // written to the indicated filename. Individual 00246 // subfiles may not be extracted or read. 00247 // 00248 // Also see the version of open_write() which accepts an 00249 // ostream. Returns true on success, false on failure. 00250 //////////////////////////////////////////////////////////////////// 00251 bool Multifile:: 00252 open_write(const Filename &multifile_name) { 00253 close(); 00254 Filename fname = multifile_name; 00255 fname.set_binary(); 00256 if (!fname.open_write(_write_file, true)) { 00257 return false; 00258 } 00259 _timestamp = time(NULL); 00260 _timestamp_dirty = true; 00261 _write = &_write_file; 00262 _multifile_name = multifile_name; 00263 return true; 00264 } 00265 00266 //////////////////////////////////////////////////////////////////// 00267 // Function: Multifile::open_write 00268 // Access: Public 00269 // Description: Opens an anonymous Multifile for writing using an 00270 // ostream. There must be seek functionality via 00271 // seekp() and tellp() on the pstream. 00272 // 00273 // If owns_pointer is true, then the Multifile assumes 00274 // ownership of the stream pointer and will delete it 00275 // when the multifile is closed, including if this 00276 // function returns false. 00277 //////////////////////////////////////////////////////////////////// 00278 bool Multifile:: 00279 open_write(ostream *multifile_stream, bool owns_pointer) { 00280 close(); 00281 _timestamp = time(NULL); 00282 _timestamp_dirty = true; 00283 _write = multifile_stream; 00284 _owns_stream = owns_pointer; 00285 _write->seekp(0, ios::beg); 00286 return true; 00287 } 00288 00289 //////////////////////////////////////////////////////////////////// 00290 // Function: Multifile::open_read_write 00291 // Access: Published 00292 // Description: Opens the named Multifile on disk for reading and 00293 // writing. If there already exists a file by that 00294 // name, its index is read. Subfiles may be added or 00295 // removed, and the resulting changes will be written to 00296 // the named file. 00297 // 00298 // Also see the version of open_read_write() which 00299 // accepts an iostream. Returns true on success, false 00300 // on failure. 00301 //////////////////////////////////////////////////////////////////// 00302 bool Multifile:: 00303 open_read_write(const Filename &multifile_name) { 00304 close(); 00305 Filename fname = multifile_name; 00306 fname.set_binary(); 00307 bool exists = fname.exists(); 00308 if (!fname.open_read_write(_read_write_file)) { 00309 return false; 00310 } 00311 if (exists) { 00312 _timestamp = fname.get_timestamp(); 00313 } else { 00314 _timestamp = time(NULL); 00315 } 00316 _timestamp_dirty = true; 00317 _read = &_read_write_filew; 00318 _write = &_read_write_file; 00319 _multifile_name = multifile_name; 00320 00321 if (exists) { 00322 return read_index(); 00323 } else { 00324 return true; 00325 } 00326 } 00327 00328 //////////////////////////////////////////////////////////////////// 00329 // Function: Multifile::open_read_write 00330 // Access: Public 00331 // Description: Opens an anonymous Multifile for reading and writing 00332 // using an iostream. There must be seek functionality 00333 // via seekg()/seekp() and tellg()/tellp() on the 00334 // iostream. 00335 // 00336 // If owns_pointer is true, then the Multifile assumes 00337 // ownership of the stream pointer and will delete it 00338 // when the multifile is closed, including if this 00339 // function returns false. 00340 //////////////////////////////////////////////////////////////////// 00341 bool Multifile:: 00342 open_read_write(iostream *multifile_stream, bool owns_pointer) { 00343 close(); 00344 _timestamp = time(NULL); 00345 _timestamp_dirty = true; 00346 00347 // We don't support locking when opening a file in read-write mode, 00348 // because we don't bother with locking on write. But we need to 00349 // have an IStreamWrapper to assign to the _read member, so we 00350 // create one on-the-fly here. 00351 _read = new StreamWrapper(multifile_stream, owns_pointer); 00352 _write = multifile_stream; 00353 _owns_stream = true; // Because we own the StreamWrapper, above. 00354 _write->seekp(0, ios::beg); 00355 00356 // Check whether the read stream is empty. 00357 multifile_stream->seekg(0, ios::end); 00358 if (multifile_stream->tellg() == (streampos)0) { 00359 // The read stream is empty, which is always valid. 00360 return true; 00361 } 00362 00363 // The read stream is not empty, so we'd better have a valid 00364 // Multifile. 00365 return read_index(); 00366 } 00367 00368 //////////////////////////////////////////////////////////////////// 00369 // Function: Multifile::close 00370 // Access: Published 00371 // Description: Closes the Multifile if it is open. All changes are 00372 // flushed to disk, and the file becomes invalid for 00373 // further operations until the next call to open(). 00374 //////////////////////////////////////////////////////////////////// 00375 void Multifile:: 00376 close() { 00377 if (_new_scale_factor != _scale_factor) { 00378 // If we have changed the scale factor recently, we need to force 00379 // a repack. 00380 repack(); 00381 } else { 00382 flush(); 00383 } 00384 00385 if (_owns_stream) { 00386 // We prefer to delete the IStreamWrapper over the ostream, if 00387 // possible. 00388 if (_read != (IStreamWrapper *)NULL) { 00389 delete _read; 00390 } else if (_write != (ostream *)NULL) { 00391 delete _write; 00392 } 00393 } 00394 00395 _read = (IStreamWrapper *)NULL; 00396 _write = (ostream *)NULL; 00397 _offset = 0; 00398 _owns_stream = false; 00399 _next_index = 0; 00400 _last_index = 0; 00401 _needs_repack = false; 00402 _timestamp = 0; 00403 _timestamp_dirty = false; 00404 _scale_factor = 1; 00405 _new_scale_factor = 1; 00406 _encryption_flag = false; 00407 _file_major_ver = 0; 00408 _file_minor_ver = 0; 00409 00410 _read_file.close(); 00411 _write_file.close(); 00412 _read_write_file.close(); 00413 _multifile_name = Filename(); 00414 00415 clear_subfiles(); 00416 } 00417 00418 //////////////////////////////////////////////////////////////////// 00419 // Function: Multifile::set_scale_factor 00420 // Access: Published 00421 // Description: Changes the internal scale factor for this Multifile. 00422 // 00423 // This is normally 1, but it may be set to any 00424 // arbitrary value (greater than zero) to support 00425 // Multifile archives that exceed 4GB, if necessary. 00426 // (Individual subfiles may still not exceed 4GB.) 00427 // 00428 // All addresses within the file are rounded up to the 00429 // next multiple of _scale_factor, and zeros are written 00430 // to the file to fill the resulting gaps. Then the 00431 // address is divided by _scale_factor and written out 00432 // as a 32-bit integer. Thus, setting a scale factor of 00433 // 2 supports up to 8GB files, 3 supports 12GB files, 00434 // etc. 00435 // 00436 // Calling this function on an already-existing 00437 // Multifile will have no immediate effect until a 00438 // future call to repack() or close() (or until the 00439 // Multifile is destructed). 00440 //////////////////////////////////////////////////////////////////// 00441 void Multifile:: 00442 set_scale_factor(size_t scale_factor) { 00443 nassertv(is_write_valid()); 00444 nassertv(scale_factor != (size_t)0); 00445 00446 if (_next_index == (streampos)0) { 00447 // If it's a brand new Multifile, we can go ahead and set it 00448 // immediately. 00449 _scale_factor = scale_factor; 00450 } else { 00451 // Otherwise, we'd better have read access so we can repack it 00452 // later. 00453 nassertv(is_read_valid()); 00454 } 00455 00456 // Setting the _new_scale_factor different from the _scale_factor 00457 // will force a repack operation on close. 00458 _new_scale_factor = scale_factor; 00459 } 00460 00461 //////////////////////////////////////////////////////////////////// 00462 // Function: Multifile::add_subfile 00463 // Access: Published 00464 // Description: Adds a file on disk as a subfile to the Multifile. 00465 // The file named by filename will be read and added to 00466 // the Multifile at the next call to flush(). If there 00467 // already exists a subfile with the indicated name, it 00468 // is replaced without examining its contents (but see 00469 // also update_subfile). 00470 // 00471 // Filename::set_binary() or set_text() must have been 00472 // called previously to specify the nature of the source 00473 // file. If set_text() was called, the text flag will 00474 // be set on the subfile. 00475 // 00476 // Returns the subfile name on success (it might have 00477 // been modified slightly), or empty string on failure. 00478 //////////////////////////////////////////////////////////////////// 00479 string Multifile:: 00480 add_subfile(const string &subfile_name, const Filename &filename, 00481 int compression_level) { 00482 nassertr(is_write_valid(), string()); 00483 00484 Filename fname = filename; 00485 if (multifile_always_binary) { 00486 fname.set_binary(); 00487 } 00488 00489 nassertr(fname.is_binary_or_text(), string()); 00490 00491 if (!fname.exists()) { 00492 return string(); 00493 } 00494 string name = standardize_subfile_name(subfile_name); 00495 if (!name.empty()) { 00496 Subfile *subfile = new Subfile; 00497 subfile->_name = name; 00498 subfile->_source_filename = fname; 00499 if (fname.is_text()) { 00500 subfile->_flags |= SF_text; 00501 } 00502 00503 add_new_subfile(subfile, compression_level); 00504 } 00505 00506 _timestamp = time(NULL); 00507 _timestamp_dirty = true; 00508 00509 return name; 00510 } 00511 00512 //////////////////////////////////////////////////////////////////// 00513 // Function: Multifile::add_subfile 00514 // Access: Public 00515 // Description: Adds a file from a stream as a subfile to the Multifile. 00516 // The indicated istream will be read and its contents 00517 // added to the Multifile at the next call to flush(). 00518 // The file will be added as a binary subfile. 00519 // 00520 // Note that the istream must remain untouched and 00521 // unused by any other code until flush() is called. At 00522 // that time, the Multifile will read the entire 00523 // contents of the istream from the current file 00524 // position to the end of the file. Subsequently, the 00525 // Multifile will *not* close or delete the istream. It 00526 // is the caller's responsibility to ensure that the 00527 // istream pointer does not destruct during the lifetime 00528 // of the Multifile. 00529 // 00530 // Returns the subfile name on success (it might have 00531 // been modified slightly), or empty string on failure. 00532 //////////////////////////////////////////////////////////////////// 00533 string Multifile:: 00534 add_subfile(const string &subfile_name, istream *subfile_data, 00535 int compression_level) { 00536 nassertr(is_write_valid(), string()); 00537 00538 string name = standardize_subfile_name(subfile_name); 00539 if (!name.empty()) { 00540 Subfile *subfile = new Subfile; 00541 subfile->_name = name; 00542 subfile->_source = subfile_data; 00543 add_new_subfile(subfile, compression_level); 00544 } 00545 00546 return name; 00547 } 00548 00549 //////////////////////////////////////////////////////////////////// 00550 // Function: Multifile::update_subfile 00551 // Access: Published 00552 // Description: Adds a file on disk to the subfile. If a subfile 00553 // already exists with the same name, its contents are 00554 // compared byte-for-byte to the disk file, and it is 00555 // replaced only if it is different; otherwise, the 00556 // multifile is left unchanged. 00557 // 00558 // Filename::set_binary() or set_text() must have been 00559 // called previously to specify the nature of the source 00560 // file. If set_text() was called, the text flag will 00561 // be set on the subfile. 00562 //////////////////////////////////////////////////////////////////// 00563 string Multifile:: 00564 update_subfile(const string &subfile_name, const Filename &filename, 00565 int compression_level) { 00566 nassertr(is_write_valid(), string()); 00567 00568 Filename fname = filename; 00569 if (multifile_always_binary) { 00570 fname.set_binary(); 00571 } 00572 00573 nassertr(fname.is_binary_or_text(), string()); 00574 00575 if (!fname.exists()) { 00576 return string(); 00577 } 00578 string name = standardize_subfile_name(subfile_name); 00579 if (!name.empty()) { 00580 int index = find_subfile(name); 00581 if (index >= 0) { 00582 // The subfile already exists; compare it to the source file. 00583 if (compare_subfile(index, fname)) { 00584 // The files are identical; do nothing. 00585 return name; 00586 } 00587 } 00588 00589 // The subfile does not already exist or it is different from the 00590 // source file. Add the new source file. 00591 Subfile *subfile = new Subfile; 00592 subfile->_name = name; 00593 subfile->_source_filename = fname; 00594 if (fname.is_text()) { 00595 subfile->_flags |= SF_text; 00596 } 00597 00598 add_new_subfile(subfile, compression_level); 00599 } 00600 00601 _timestamp = time(NULL); 00602 _timestamp_dirty = true; 00603 00604 return name; 00605 } 00606 00607 #ifdef HAVE_OPENSSL 00608 //////////////////////////////////////////////////////////////////// 00609 // Function: Multifile::add_signature 00610 // Access: Published 00611 // Description: Adds a new signature to the Multifile. This 00612 // signature associates the indicated certificate with 00613 // the current contents of the Multifile. When the 00614 // Multifile is read later, the signature will still be 00615 // present only if the Multifile is unchanged; any 00616 // subsequent changes to the Multifile will 00617 // automatically invalidate and remove the signature. 00618 // 00619 // The chain filename may be empty if the certificate 00620 // does not require an authenticating certificate chain 00621 // (e.g. because it is self-signed). 00622 // 00623 // The specified private key must match the certificate, 00624 // and the Multifile must be open in read-write mode. 00625 // The private key is only used for generating the 00626 // signature; it is not written to the Multifile and 00627 // cannot be retrieved from the Multifile later. 00628 // (However, the certificate *can* be retrieved from the 00629 // Multifile later, to identify the entity that created 00630 // the signature.) 00631 // 00632 // This implicitly causes a repack() operation if one is 00633 // needed. Returns true on success, false on failure. 00634 // 00635 // This flavor of add_signature() reads the certificate 00636 // and private key from a PEM-formatted file, for 00637 // instance as generated by the openssl command. If the 00638 // private key file is password-encrypted, the third 00639 // parameter will be used as the password to decrypt it. 00640 //////////////////////////////////////////////////////////////////// 00641 bool Multifile:: 00642 add_signature(const Filename &certificate, const Filename &chain, 00643 const Filename &pkey, const string &password) { 00644 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00645 00646 if (chain.empty() && pkey.empty()) { 00647 // If the second two filenames are empty, assume we're going for 00648 // the composite mode, where everything's stuffed into the first 00649 // file. 00650 return add_signature(certificate, password); 00651 } 00652 00653 CertChain cert_chain; 00654 00655 // Read the certificate file from VFS. First, read the complete 00656 // file into memory. 00657 string certificate_data; 00658 if (!vfs->read_file(certificate, certificate_data, true)) { 00659 express_cat.info() 00660 << "Could not read " << certificate << ".\n"; 00661 return false; 00662 } 00663 00664 // Create an in-memory BIO to read the "file" from the buffer. 00665 BIO *certificate_mbio = BIO_new_mem_buf((void *)certificate_data.data(), certificate_data.size()); 00666 X509 *x509 = PEM_read_bio_X509(certificate_mbio, NULL, NULL, (void *)""); 00667 BIO_free(certificate_mbio); 00668 if (x509 == NULL) { 00669 express_cat.info() 00670 << "Could not read certificate in " << certificate << ".\n"; 00671 return false; 00672 } 00673 00674 // Store the first X509--the actual certificate--as the first record 00675 // in our CertChain object. 00676 cert_chain.push_back(CertRecord(x509)); 00677 00678 // Read the rest of the certificates in the chain file. 00679 if (!chain.empty()) { 00680 string chain_data; 00681 if (!vfs->read_file(chain, chain_data, true)) { 00682 express_cat.info() 00683 << "Could not read " << chain << ".\n"; 00684 return false; 00685 } 00686 00687 BIO *chain_mbio = BIO_new_mem_buf((void *)chain_data.data(), chain_data.size()); 00688 X509 *c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)""); 00689 while (c != NULL) { 00690 cert_chain.push_back(c); 00691 c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)""); 00692 } 00693 BIO_free(chain_mbio); 00694 00695 if (cert_chain.size() == 1) { 00696 express_cat.info() 00697 << "Could not read certificate chain in " << chain << ".\n"; 00698 return false; 00699 } 00700 } 00701 00702 // Now do the same thing with the private key. This one may be 00703 // password-encrypted on disk. 00704 string pkey_data; 00705 if (!vfs->read_file(pkey, pkey_data, true)) { 00706 express_cat.info() 00707 << "Could not read " << pkey << ".\n"; 00708 return false; 00709 } 00710 00711 BIO *pkey_mbio = BIO_new_mem_buf((void *)pkey_data.data(), pkey_data.size()); 00712 EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, NULL, NULL, 00713 (void *)password.c_str()); 00714 BIO_free(pkey_mbio); 00715 if (evp_pkey == NULL) { 00716 express_cat.info() 00717 << "Could not read private key in " << pkey << ".\n"; 00718 return false; 00719 } 00720 00721 bool result = add_signature(cert_chain, evp_pkey); 00722 00723 EVP_PKEY_free(evp_pkey); 00724 00725 return result; 00726 } 00727 #endif // HAVE_OPENSSL 00728 00729 #ifdef HAVE_OPENSSL 00730 //////////////////////////////////////////////////////////////////// 00731 // Function: Multifile::add_signature 00732 // Access: Published 00733 // Description: Adds a new signature to the Multifile. This 00734 // signature associates the indicated certificate with 00735 // the current contents of the Multifile. When the 00736 // Multifile is read later, the signature will still be 00737 // present only if the Multifile is unchanged; any 00738 // subsequent changes to the Multifile will 00739 // automatically invalidate and remove the signature. 00740 // 00741 // This flavor of add_signature() reads the certificate, 00742 // private key, and certificate chain from the same 00743 // PEM-formatted file. It takes the first private key 00744 // found as the intended key, and then uses the first 00745 // certificate found that matches that key as the 00746 // signing certificate. Any other certificates in the 00747 // file are taken to be part of the chain. 00748 //////////////////////////////////////////////////////////////////// 00749 bool Multifile:: 00750 add_signature(const Filename &composite, const string &password) { 00751 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00752 00753 // First, read the complete file into memory. 00754 string composite_data; 00755 if (!vfs->read_file(composite, composite_data, true)) { 00756 express_cat.info() 00757 << "Could not read " << composite << ".\n"; 00758 return false; 00759 } 00760 00761 // Get the private key. 00762 BIO *pkey_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size()); 00763 EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, NULL, NULL, 00764 (void *)password.c_str()); 00765 BIO_free(pkey_mbio); 00766 if (evp_pkey == NULL) { 00767 express_cat.info() 00768 << "Could not read private key in " << composite << ".\n"; 00769 return false; 00770 } 00771 00772 // Now read all of the certificates. 00773 CertChain cert_chain; 00774 00775 BIO *chain_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size()); 00776 X509 *c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)""); 00777 while (c != NULL) { 00778 cert_chain.push_back(c); 00779 c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)""); 00780 } 00781 BIO_free(chain_mbio); 00782 00783 if (cert_chain.empty()) { 00784 express_cat.info() 00785 << "Could not read certificates in " << composite << ".\n"; 00786 return false; 00787 } 00788 00789 // Now find the certificate that matches the signature, and move it 00790 // to the front of the chain. 00791 size_t i; 00792 bool found_match = false; 00793 for (i = 0; i < cert_chain.size(); ++i) { 00794 X509 *c = cert_chain[i]._cert; 00795 if (X509_check_private_key(c, evp_pkey)) { 00796 found_match = true; 00797 if (i != 0) { 00798 // Move this entry to the beginning. 00799 cert_chain.insert(cert_chain.begin(), cert_chain[i]); 00800 cert_chain.erase(cert_chain.begin() + i + 1); 00801 } 00802 break; 00803 } 00804 } 00805 00806 if (!found_match) { 00807 express_cat.info() 00808 << "No certificates in " << composite << " match key.\n"; 00809 return false; 00810 } 00811 00812 bool result = add_signature(cert_chain, evp_pkey); 00813 00814 EVP_PKEY_free(evp_pkey); 00815 00816 return result; 00817 } 00818 #endif // HAVE_OPENSSL 00819 00820 #ifdef HAVE_OPENSSL 00821 //////////////////////////////////////////////////////////////////// 00822 // Function: Multifile::add_signature 00823 // Access: Published 00824 // Description: Adds a new signature to the Multifile. This 00825 // signature associates the indicated certificate with 00826 // the current contents of the Multifile. When the 00827 // Multifile is read later, the signature will still be 00828 // present only if the Multifile is unchanged; any 00829 // subsequent changes to the Multifile will 00830 // automatically invalidate and remove the signature. 00831 // 00832 // If chain is non-NULL, it represents the certificate 00833 // chain that validates the certificate. 00834 // 00835 // The specified private key must match the certificate, 00836 // and the Multifile must be open in read-write mode. 00837 // The private key is only used for generating the 00838 // signature; it is not written to the Multifile and 00839 // cannot be retrieved from the Multifile later. 00840 // (However, the certificate *can* be retrieved from the 00841 // Multifile later, to identify the entity that created 00842 // the signature.) 00843 // 00844 // This implicitly causes a repack() operation if one is 00845 // needed. Returns true on success, false on failure. 00846 //////////////////////////////////////////////////////////////////// 00847 bool Multifile:: 00848 add_signature(X509 *certificate, STACK_OF(X509) *chain, EVP_PKEY *pkey) { 00849 // Convert the certificate and chain into our own CertChain 00850 // structure. 00851 CertChain cert_chain; 00852 cert_chain.push_back(CertRecord(certificate)); 00853 if (chain != NULL) { 00854 int num = sk_X509_num(chain); 00855 for (int i = 0; i < num; ++i) { 00856 cert_chain.push_back(CertRecord((X509 *)sk_X509_value(chain, i))); 00857 } 00858 } 00859 00860 return add_signature(cert_chain, pkey); 00861 } 00862 #endif // HAVE_OPENSSL 00863 00864 #ifdef HAVE_OPENSSL 00865 //////////////////////////////////////////////////////////////////// 00866 // Function: Multifile::add_signature 00867 // Access: Published 00868 // Description: Adds a new signature to the Multifile. This 00869 // signature associates the indicated certificate with 00870 // the current contents of the Multifile. When the 00871 // Multifile is read later, the signature will still be 00872 // present only if the Multifile is unchanged; any 00873 // subsequent changes to the Multifile will 00874 // automatically invalidate and remove the signature. 00875 // 00876 // The signature certificate is the first certificate on 00877 // the CertChain object. Any remaining certificates are 00878 // support certificates to authenticate the first one. 00879 // 00880 // The specified private key must match the certificate, 00881 // and the Multifile must be open in read-write mode. 00882 // The private key is only used for generating the 00883 // signature; it is not written to the Multifile and 00884 // cannot be retrieved from the Multifile later. 00885 // (However, the certificate *can* be retrieved from the 00886 // Multifile later, to identify the entity that created 00887 // the signature.) 00888 // 00889 // This implicitly causes a repack() operation if one is 00890 // needed. Returns true on success, false on failure. 00891 //////////////////////////////////////////////////////////////////// 00892 bool Multifile:: 00893 add_signature(const Multifile::CertChain &cert_chain, EVP_PKEY *pkey) { 00894 if (_needs_repack) { 00895 if (!repack()) { 00896 return false; 00897 } 00898 } else { 00899 if (!flush()) { 00900 return false; 00901 } 00902 } 00903 00904 if (cert_chain.empty()) { 00905 express_cat.info() 00906 << "No certificate given.\n"; 00907 return false; 00908 } 00909 00910 if (pkey == NULL) { 00911 express_cat.info() 00912 << "No private key given.\n"; 00913 return false; 00914 } 00915 00916 if (!X509_check_private_key(cert_chain[0]._cert, pkey)) { 00917 express_cat.info() 00918 << "Private key does not match certificate.\n"; 00919 return false; 00920 } 00921 00922 // Now encode that list of certs to a stream in DER form. 00923 stringstream der_stream; 00924 StreamWriter der_writer(der_stream); 00925 der_writer.add_uint32(cert_chain.size()); 00926 00927 CertChain::const_iterator ci; 00928 for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) { 00929 X509 *cert = (*ci)._cert; 00930 00931 int der_len = i2d_X509(cert, NULL); 00932 unsigned char *der_buf = new unsigned char[der_len]; 00933 unsigned char *p = der_buf; 00934 i2d_X509(cert, &p); 00935 der_writer.append_data(der_buf, der_len); 00936 delete[] der_buf; 00937 } 00938 00939 // Create a temporary Subfile for writing out the signature. 00940 der_stream.seekg(0); 00941 Subfile *subfile = new Subfile; 00942 subfile->_pkey = pkey; 00943 subfile->_flags |= SF_signature; 00944 subfile->_source = &der_stream; 00945 00946 // Write the new Subfile at the end. The cert_special subfiles 00947 // always go at the end, because they're not the part of the file 00948 // that's signed. 00949 nassertr(_new_subfiles.empty(), false); 00950 _new_subfiles.push_back(subfile); 00951 bool result = flush(); 00952 00953 delete subfile; 00954 00955 return result; 00956 } 00957 #endif // HAVE_OPENSSL 00958 00959 #ifdef HAVE_OPENSSL 00960 //////////////////////////////////////////////////////////////////// 00961 // Function: Multifile::get_num_signatures 00962 // Access: Published 00963 // Description: Returns the number of matching signatures found on 00964 // the Multifile. These signatures may be iterated via 00965 // get_signature() and related methods. 00966 // 00967 // A signature on this list is guaranteed to match the 00968 // Multifile contents, proving that the Multifile has 00969 // been unmodified since the signature was applied. 00970 // However, this does not guarantee that the certificate 00971 // itself is actually from who it says it is from; only 00972 // that it matches the Multifile contents. See 00973 // validate_signature_certificate() to authenticate a 00974 // particular certificate. 00975 //////////////////////////////////////////////////////////////////// 00976 int Multifile:: 00977 get_num_signatures() const { 00978 ((Multifile *)this)->check_signatures(); 00979 return _signatures.size(); 00980 } 00981 #endif // HAVE_OPENSSL 00982 00983 #ifdef HAVE_OPENSSL 00984 //////////////////////////////////////////////////////////////////// 00985 // Function: Multifile::get_signature 00986 // Access: Published 00987 // Description: Returns the nth signature found on the Multifile. 00988 // See the comments in get_num_signatures(). 00989 //////////////////////////////////////////////////////////////////// 00990 const Multifile::CertChain &Multifile:: 00991 get_signature(int n) const { 00992 ((Multifile *)this)->check_signatures(); 00993 static CertChain error_chain; 00994 nassertr(n >= 0 && n < (int)_signatures.size(), error_chain); 00995 return _signatures[n]; 00996 } 00997 #endif // HAVE_OPENSSL 00998 00999 #ifdef HAVE_OPENSSL 01000 //////////////////////////////////////////////////////////////////// 01001 // Function: Multifile::get_signature_subject_name 01002 // Access: Published 01003 // Description: Returns the "subject name" for the nth signature found 01004 // on the Multifile. This is a string formatted 01005 // according to RFC2253 that should more-or-less 01006 // identify a particular certificate; when paired with 01007 // the public key (see get_signature_public_key()), it 01008 // can uniquely identify a certificate. See the 01009 // comments in get_num_signatures(). 01010 //////////////////////////////////////////////////////////////////// 01011 string Multifile:: 01012 get_signature_subject_name(int n) const { 01013 const CertChain &cert_chain = get_signature(n); 01014 nassertr(!cert_chain.empty(), string()); 01015 01016 X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert); 01017 if (xname != NULL) { 01018 // We use "print" to dump the output to a memory BIO. Is 01019 // there an easier way to extract the X509_NAME text? Curse 01020 // these incomplete docs. 01021 BIO *mbio = BIO_new(BIO_s_mem()); 01022 X509_NAME_print_ex(mbio, xname, 0, XN_FLAG_RFC2253); 01023 01024 char *pp; 01025 long pp_size = BIO_get_mem_data(mbio, &pp); 01026 string name(pp, pp_size); 01027 BIO_free(mbio); 01028 return name; 01029 } 01030 01031 return string(); 01032 } 01033 #endif // HAVE_OPENSSL 01034 01035 #ifdef HAVE_OPENSSL 01036 //////////////////////////////////////////////////////////////////// 01037 // Function: Multifile::get_signature_friendly_name 01038 // Access: Published 01039 // Description: Returns a "friendly name" for the nth signature found 01040 // on the Multifile. This attempts to extract out the 01041 // most meaningful part of the subject name. It returns 01042 // the emailAddress, if it is defined; otherwise, it 01043 // returns the commonName. 01044 01045 // See the comments in get_num_signatures(). 01046 //////////////////////////////////////////////////////////////////// 01047 string Multifile:: 01048 get_signature_friendly_name(int n) const { 01049 const CertChain &cert_chain = get_signature(n); 01050 nassertr(!cert_chain.empty(), string()); 01051 01052 static const int nid_choices[] = { 01053 NID_pkcs9_emailAddress, 01054 NID_subject_alt_name, 01055 NID_commonName, 01056 -1, 01057 }; 01058 01059 // Choose the first NID that exists on the cert. 01060 for (int ni = 0; nid_choices[ni] != -1; ++ni) { 01061 int nid = nid_choices[ni]; 01062 01063 // A complex OpenSSL interface to extract out the name in utf-8. 01064 X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert); 01065 if (xname != NULL) { 01066 int pos = X509_NAME_get_index_by_NID(xname, nid, -1); 01067 if (pos != -1) { 01068 // We just get the first common name. I guess it's possible to 01069 // have more than one; not sure what that means in this context. 01070 X509_NAME_ENTRY *xentry = X509_NAME_get_entry(xname, pos); 01071 if (xentry != NULL) { 01072 ASN1_STRING *data = X509_NAME_ENTRY_get_data(xentry); 01073 if (data != NULL) { 01074 // We use "print" to dump the output to a memory BIO. Is 01075 // there an easier way to decode the ASN1_STRING? Curse 01076 // these incomplete docs. 01077 BIO *mbio = BIO_new(BIO_s_mem()); 01078 ASN1_STRING_print_ex(mbio, data, ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB); 01079 01080 char *pp; 01081 long pp_size = BIO_get_mem_data(mbio, &pp); 01082 string name(pp, pp_size); 01083 BIO_free(mbio); 01084 return name; 01085 } 01086 } 01087 } 01088 } 01089 } 01090 01091 return string(); 01092 } 01093 #endif // HAVE_OPENSSL 01094 01095 #ifdef HAVE_OPENSSL 01096 //////////////////////////////////////////////////////////////////// 01097 // Function: Multifile::get_signature_public_key 01098 // Access: Published 01099 // Description: Returns the public key used for the nth signature 01100 // found on the Multifile. This is encoded in DER form 01101 // and returned as a string of hex digits. 01102 // 01103 // This can be used, in conjunction with the subject 01104 // name (see get_signature_subject_name()), to uniquely 01105 // identify a particular certificate and its subsequent 01106 // reissues. See the comments in get_num_signatures(). 01107 //////////////////////////////////////////////////////////////////// 01108 string Multifile:: 01109 get_signature_public_key(int n) const { 01110 const CertChain &cert_chain = get_signature(n); 01111 nassertr(!cert_chain.empty(), string()); 01112 01113 EVP_PKEY *pkey = X509_get_pubkey(cert_chain[0]._cert); 01114 if (pkey != NULL) { 01115 int key_len = i2d_PublicKey(pkey, NULL); 01116 unsigned char *key_buf = new unsigned char[key_len]; 01117 unsigned char *p = key_buf; 01118 i2d_PublicKey(pkey, &p); 01119 string result; 01120 for (int i = 0; i < key_len; ++i) { 01121 result += tohex(key_buf[i] >> 4); 01122 result += tohex(key_buf[i]); 01123 } 01124 delete[] key_buf; 01125 return result; 01126 } 01127 01128 return string(); 01129 } 01130 #endif // HAVE_OPENSSL 01131 01132 #ifdef HAVE_OPENSSL 01133 //////////////////////////////////////////////////////////////////// 01134 // Function: Multifile::print_signature_certificate 01135 // Access: Published 01136 // Description: Writes the certificate for the nth signature, in 01137 // user-readable verbose form, to the indicated stream. 01138 // See the comments in get_num_signatures(). 01139 //////////////////////////////////////////////////////////////////// 01140 void Multifile:: 01141 print_signature_certificate(int n, ostream &out) const { 01142 const CertChain &cert_chain = get_signature(n); 01143 nassertv(!cert_chain.empty()); 01144 01145 BIO *mbio = BIO_new(BIO_s_mem()); 01146 X509_print(mbio, cert_chain[0]._cert); 01147 01148 char *pp; 01149 long pp_size = BIO_get_mem_data(mbio, &pp); 01150 out.write(pp, pp_size); 01151 BIO_free(mbio); 01152 } 01153 #endif // HAVE_OPENSSL 01154 01155 #ifdef HAVE_OPENSSL 01156 //////////////////////////////////////////////////////////////////// 01157 // Function: Multifile::write_signature_certificate 01158 // Access: Published 01159 // Description: Writes the certificate for the nth signature, in 01160 // PEM form, to the indicated stream. See the comments 01161 // in get_num_signatures(). 01162 //////////////////////////////////////////////////////////////////// 01163 void Multifile:: 01164 write_signature_certificate(int n, ostream &out) const { 01165 const CertChain &cert_chain = get_signature(n); 01166 nassertv(!cert_chain.empty()); 01167 01168 BIO *mbio = BIO_new(BIO_s_mem()); 01169 01170 CertChain::const_iterator ci; 01171 for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) { 01172 X509 *c = (*ci)._cert; 01173 X509_print(mbio, c); 01174 PEM_write_bio_X509(mbio, c); 01175 } 01176 01177 char *pp; 01178 long pp_size = BIO_get_mem_data(mbio, &pp); 01179 out.write(pp, pp_size); 01180 BIO_free(mbio); 01181 } 01182 #endif // HAVE_OPENSSL 01183 01184 #ifdef HAVE_OPENSSL 01185 //////////////////////////////////////////////////////////////////// 01186 // Function: Multifile::validate_signature_certificate 01187 // Access: Published 01188 // Description: Checks that the certificate used for the nth 01189 // signature is a valid, authorized certificate with 01190 // some known certificate authority. Returns 0 if it 01191 // is valid, -1 if there is some error, or the 01192 // corresponding OpenSSL error code if it is invalid, 01193 // out-of-date, or self-signed. 01194 //////////////////////////////////////////////////////////////////// 01195 int Multifile:: 01196 validate_signature_certificate(int n) const { 01197 int verify_result = -1; 01198 01199 const CertChain &chain = get_signature(n); 01200 nassertr(!chain.empty(), false); 01201 01202 OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr(); 01203 01204 // Copy our CertChain structure into an X509 pointer and 01205 // accompanying STACK_OF(X509) pointer. 01206 X509 *x509 = chain[0]._cert; 01207 STACK_OF(X509) *stack = NULL; 01208 if (chain.size() > 1) { 01209 stack = sk_X509_new(NULL); 01210 for (size_t n = 1; n < chain.size(); ++n) { 01211 sk_X509_push(stack, chain[n]._cert); 01212 } 01213 } 01214 01215 // Create the X509_STORE_CTX for verifying the cert and chain. 01216 X509_STORE_CTX *ctx = X509_STORE_CTX_new(); 01217 X509_STORE_CTX_init(ctx, sslw->get_x509_store(), x509, stack); 01218 X509_STORE_CTX_set_cert(ctx, x509); 01219 01220 if (X509_verify_cert(ctx)) { 01221 verify_result = 0; 01222 } else { 01223 verify_result = X509_STORE_CTX_get_error(ctx); 01224 } 01225 01226 if (express_cat.is_debug()) { 01227 express_cat.debug() 01228 << get_signature_subject_name(n) << ": validate " << verify_result 01229 << "\n"; 01230 } 01231 01232 sk_X509_free(stack); 01233 X509_STORE_CTX_cleanup(ctx); 01234 X509_STORE_CTX_free(ctx); 01235 01236 return verify_result; 01237 } 01238 #endif // HAVE_OPENSSL 01239 01240 //////////////////////////////////////////////////////////////////// 01241 // Function: Multifile::flush 01242 // Access: Published 01243 // Description: Writes all contents of the Multifile to disk. Until 01244 // flush() is called, add_subfile() and remove_subfile() 01245 // do not actually do anything to disk. At this point, 01246 // all of the recently-added subfiles are read and their 01247 // contents are added to the end of the Multifile, and 01248 // the recently-removed subfiles are marked gone from 01249 // the Multifile. 01250 // 01251 // This may result in a suboptimal index. To guarantee 01252 // that the index is written at the beginning of the 01253 // file, call repack() instead of flush(). 01254 // 01255 // It is not necessary to call flush() explicitly unless 01256 // you are concerned about reading the recently-added 01257 // subfiles immediately. 01258 // 01259 // Returns true on success, false on failure. 01260 //////////////////////////////////////////////////////////////////// 01261 bool Multifile:: 01262 flush() { 01263 if (!is_write_valid()) { 01264 return false; 01265 } 01266 01267 bool new_file = (_next_index == (streampos)0); 01268 if (new_file) { 01269 // If we don't have an index yet, we don't have a header. Write 01270 // the header. 01271 if (!write_header()) { 01272 return false; 01273 } 01274 01275 } else { 01276 if (_file_minor_ver != _current_minor_ver) { 01277 // If we *do* have an index already, but this is an old version 01278 // multifile, we have to completely rewrite it anyway. 01279 return repack(); 01280 } 01281 } 01282 01283 nassertr(_write != (ostream *)NULL, false); 01284 01285 // First, mark out all of the removed subfiles. 01286 PendingSubfiles::iterator pi; 01287 for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) { 01288 Subfile *subfile = (*pi); 01289 subfile->rewrite_index_flags(*_write); 01290 delete subfile; 01291 } 01292 _removed_subfiles.clear(); 01293 01294 bool wrote_ok = true; 01295 01296 if (!_new_subfiles.empty() || new_file) { 01297 // Add a few more files to the end. We always add subfiles at the 01298 // end of the multifile, so go there first. 01299 sort(_new_subfiles.begin(), _new_subfiles.end(), IndirectLess<Subfile>()); 01300 if (_last_index != (streampos)0) { 01301 _write->seekp(0, ios::end); 01302 if (_write->fail()) { 01303 express_cat.info() 01304 << "Unable to seek Multifile " << _multifile_name << ".\n"; 01305 return false; 01306 } 01307 _next_index = _write->tellp(); 01308 _next_index = pad_to_streampos(_next_index); 01309 01310 // And update the forward link from the last_index to point to 01311 // this new index location. 01312 _write->seekp(_last_index); 01313 StreamWriter writer(_write, false); 01314 writer.add_uint32(streampos_to_word(_next_index)); 01315 } 01316 01317 _write->seekp(_next_index); 01318 nassertr(_next_index == _write->tellp(), false); 01319 01320 // Ok, here we are at the end of the file. Write out the 01321 // recently-added subfiles here. First, count up the index size. 01322 for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) { 01323 Subfile *subfile = (*pi); 01324 _last_index = _next_index; 01325 _next_index = subfile->write_index(*_write, _next_index, this); 01326 nassertr(_next_index == _write->tellp(), false); 01327 _next_index = pad_to_streampos(_next_index); 01328 nassertr(_next_index == _write->tellp(), false); 01329 } 01330 01331 // Now we're at the end of the index. Write a 0 here to mark the 01332 // end. 01333 StreamWriter writer(_write, false); 01334 writer.add_uint32(0); 01335 _next_index += 4; 01336 nassertr(_next_index == _write->tellp(), false); 01337 _next_index = pad_to_streampos(_next_index); 01338 01339 // All right, now write out each subfile's data. 01340 for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) { 01341 Subfile *subfile = (*pi); 01342 01343 if (_read != (IStreamWrapper *)NULL) { 01344 _read->acquire(); 01345 _next_index = subfile->write_data(*_write, _read->get_istream(), 01346 _next_index, this); 01347 _read->release(); 01348 01349 } else { 01350 _next_index = subfile->write_data(*_write, NULL, _next_index, this); 01351 } 01352 01353 nassertr(_next_index == _write->tellp(), false); 01354 _next_index = pad_to_streampos(_next_index); 01355 if (subfile->is_data_invalid()) { 01356 wrote_ok = false; 01357 } 01358 01359 if (!subfile->is_cert_special()) { 01360 _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos()); 01361 } 01362 nassertr(_next_index == _write->tellp(), false); 01363 } 01364 01365 // Now go back and fill in the proper addresses for the data start. 01366 // We didn't do it in the first pass, because we don't really want 01367 // to keep all those file handles open, and so we didn't have to 01368 // determine each file's length ahead of time. 01369 for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) { 01370 Subfile *subfile = (*pi); 01371 subfile->rewrite_index_data_start(*_write, this); 01372 } 01373 01374 _new_subfiles.clear(); 01375 } 01376 01377 // Also update the overall timestamp. 01378 if (_timestamp_dirty) { 01379 nassertr(!_write->fail(), false); 01380 static const size_t timestamp_pos = _header_prefix.size() + _header_size + 2 + 2 + 4; 01381 _write->seekp(timestamp_pos); 01382 nassertr(!_write->fail(), false); 01383 01384 StreamWriter writer(*_write); 01385 if (_record_timestamp) { 01386 writer.add_uint32(_timestamp); 01387 } else { 01388 writer.add_uint32(0); 01389 } 01390 _timestamp_dirty = false; 01391 } 01392 01393 _write->flush(); 01394 if (!wrote_ok || _write->fail()) { 01395 express_cat.info() 01396 << "Unable to update Multifile " << _multifile_name << ".\n"; 01397 close(); 01398 return false; 01399 } 01400 01401 return true; 01402 } 01403 01404 //////////////////////////////////////////////////////////////////// 01405 // Function: Multifile::repack 01406 // Access: Published 01407 // Description: Forces a complete rewrite of the Multifile and all of 01408 // its contents, so that its index will appear at the 01409 // beginning of the file with all of the subfiles listed 01410 // in alphabetical order. This is considered optimal 01411 // for reading, and is the standard configuration; but 01412 // it is not essential to do this. 01413 // 01414 // It is only valid to call this if the Multifile was 01415 // opened using open_read_write() and an explicit 01416 // filename, rather than an iostream. Also, we must 01417 // have write permission to the directory containing the 01418 // Multifile. 01419 // 01420 // Returns true on success, false on failure. 01421 //////////////////////////////////////////////////////////////////// 01422 bool Multifile:: 01423 repack() { 01424 if (_next_index == (streampos)0) { 01425 // If the Multifile hasn't yet been written, this is really just a 01426 // flush operation. 01427 _needs_repack = false; 01428 return flush(); 01429 } 01430 01431 nassertr(is_write_valid() && is_read_valid(), false); 01432 nassertr(!_multifile_name.empty(), false); 01433 01434 // First, we open a temporary filename to copy the Multifile to. 01435 Filename dirname = _multifile_name.get_dirname(); 01436 if (dirname.empty()) { 01437 dirname = "."; 01438 } 01439 Filename temp_filename = Filename::temporary(dirname, "mftemp"); 01440 temp_filename.set_binary(); 01441 pofstream temp; 01442 if (!temp_filename.open_write(temp)) { 01443 express_cat.info() 01444 << "Unable to open temporary file " << temp_filename << "\n"; 01445 return false; 01446 } 01447 01448 // Now we scrub our internal structures so it looks like we're a 01449 // brand new Multifile. 01450 PendingSubfiles::iterator pi; 01451 for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) { 01452 Subfile *subfile = (*pi); 01453 delete subfile; 01454 } 01455 _removed_subfiles.clear(); 01456 _new_subfiles.clear(); 01457 copy(_subfiles.begin(), _subfiles.end(), back_inserter(_new_subfiles)); 01458 _next_index = 0; 01459 _last_index = 0; 01460 _last_data_byte = 0; 01461 _scale_factor = _new_scale_factor; 01462 01463 // And we write our contents to our new temporary file. 01464 _write = &temp; 01465 if (!flush()) { 01466 temp.close(); 01467 temp_filename.unlink(); 01468 return false; 01469 } 01470 01471 // Now close everything, and move the temporary file back over our 01472 // original file. 01473 Filename orig_name = _multifile_name; 01474 temp.close(); 01475 close(); 01476 orig_name.unlink(); 01477 if (!temp_filename.rename_to(orig_name)) { 01478 express_cat.info() 01479 << "Unable to rename temporary file " << temp_filename << " to " 01480 << orig_name << ".\n"; 01481 return false; 01482 } 01483 01484 if (!open_read_write(orig_name)) { 01485 express_cat.info() 01486 << "Unable to read newly repacked " << _multifile_name 01487 << ".\n"; 01488 return false; 01489 } 01490 01491 return true; 01492 } 01493 01494 //////////////////////////////////////////////////////////////////// 01495 // Function: Multifile::get_num_subfiles 01496 // Access: Published 01497 // Description: Returns the number of subfiles within the Multifile. 01498 // The subfiles may be accessed in alphabetical order by 01499 // iterating through [0 .. get_num_subfiles()). 01500 //////////////////////////////////////////////////////////////////// 01501 int Multifile:: 01502 get_num_subfiles() const { 01503 return _subfiles.size(); 01504 } 01505 01506 //////////////////////////////////////////////////////////////////// 01507 // Function: Multifile::find_subfile 01508 // Access: Published 01509 // Description: Returns the index of the subfile with the indicated 01510 // name, or -1 if the named subfile is not within the 01511 // Multifile. 01512 //////////////////////////////////////////////////////////////////// 01513 int Multifile:: 01514 find_subfile(const string &subfile_name) const { 01515 Subfile find_subfile; 01516 find_subfile._name = standardize_subfile_name(subfile_name); 01517 Subfiles::const_iterator fi; 01518 fi = _subfiles.find(&find_subfile); 01519 if (fi == _subfiles.end()) { 01520 // Not present. 01521 return -1; 01522 } 01523 return (fi - _subfiles.begin()); 01524 } 01525 01526 //////////////////////////////////////////////////////////////////// 01527 // Function: Multifile::has_directory 01528 // Access: Published 01529 // Description: Returns true if the indicated subfile name is the 01530 // directory prefix to one or more files within the 01531 // Multifile. That is, the Multifile contains at least 01532 // one file named "subfile_name/...". 01533 //////////////////////////////////////////////////////////////////// 01534 bool Multifile:: 01535 has_directory(const string &subfile_name) const { 01536 string prefix = subfile_name; 01537 if (!prefix.empty()) { 01538 prefix += '/'; 01539 } 01540 Subfile find_subfile; 01541 find_subfile._name = prefix; 01542 Subfiles::const_iterator fi; 01543 fi = _subfiles.upper_bound(&find_subfile); 01544 if (fi == _subfiles.end()) { 01545 // Not present. 01546 return false; 01547 } 01548 01549 // At least one subfile exists whose name sorts after prefix. If it 01550 // contains prefix as the initial substring, then we have a match. 01551 Subfile *subfile = (*fi); 01552 return (subfile->_name.length() > prefix.length() && 01553 subfile->_name.substr(0, prefix.length()) == prefix); 01554 } 01555 01556 //////////////////////////////////////////////////////////////////// 01557 // Function: Multifile::scan_directory 01558 // Access: Published 01559 // Description: Considers subfile_name to be the name of a 01560 // subdirectory within the Multifile, but not a file 01561 // itself; fills the given vector up with the sorted list 01562 // of subdirectories or files within the named 01563 // directory. 01564 // 01565 // Note that directories do not exist explicitly within 01566 // a Multifile; this just checks for the existence of 01567 // files with the given initial prefix. 01568 // 01569 // Returns true if successful, false otherwise. 01570 //////////////////////////////////////////////////////////////////// 01571 bool Multifile:: 01572 scan_directory(vector_string &contents, const string &subfile_name) const { 01573 string prefix = subfile_name; 01574 if (!prefix.empty()) { 01575 prefix += '/'; 01576 } 01577 Subfile find_subfile; 01578 find_subfile._name = prefix; 01579 Subfiles::const_iterator fi; 01580 fi = _subfiles.upper_bound(&find_subfile); 01581 01582 string previous = ""; 01583 while (fi != _subfiles.end()) { 01584 Subfile *subfile = (*fi); 01585 if (!(subfile->_name.length() > prefix.length() && 01586 subfile->_name.substr(0, prefix.length()) == prefix)) { 01587 // We've reached the end of the list of subfiles beneath the 01588 // indicated directory prefix. 01589 return true; 01590 } 01591 01592 size_t slash = subfile->_name.find('/', prefix.length()); 01593 string basename = subfile->_name.substr(prefix.length(), slash - prefix.length()); 01594 if (basename != previous) { 01595 contents.push_back(basename); 01596 previous = basename; 01597 } 01598 ++fi; 01599 } 01600 01601 return true; 01602 } 01603 01604 //////////////////////////////////////////////////////////////////// 01605 // Function: Multifile::remove_subfile 01606 // Access: Published 01607 // Description: Removes the nth subfile from the Multifile. This 01608 // will cause all subsequent index numbers to decrease 01609 // by one. The file will not actually be removed from 01610 // the disk until the next call to flush(). 01611 // 01612 // Note that this does not actually remove the data from 01613 // the indicated subfile; it simply removes it from the 01614 // index. The Multifile will not be reduced in size 01615 // after this operation, until the next call to 01616 // repack(). 01617 //////////////////////////////////////////////////////////////////// 01618 void Multifile:: 01619 remove_subfile(int index) { 01620 nassertv(is_write_valid()); 01621 nassertv(index >= 0 && index < (int)_subfiles.size()); 01622 Subfile *subfile = _subfiles[index]; 01623 subfile->_flags |= SF_deleted; 01624 _removed_subfiles.push_back(subfile); 01625 _subfiles.erase(_subfiles.begin() + index); 01626 01627 _timestamp = time(NULL); 01628 _timestamp_dirty = true; 01629 01630 _needs_repack = true; 01631 } 01632 01633 //////////////////////////////////////////////////////////////////// 01634 // Function: Multifile::get_subfile_name 01635 // Access: Published 01636 // Description: Returns the name of the nth subfile. 01637 //////////////////////////////////////////////////////////////////// 01638 const string &Multifile:: 01639 get_subfile_name(int index) const { 01640 #ifndef NDEBUG 01641 static string empty_string; 01642 nassertr(index >= 0 && index < (int)_subfiles.size(), empty_string); 01643 #endif 01644 return _subfiles[index]->_name; 01645 } 01646 01647 //////////////////////////////////////////////////////////////////// 01648 // Function: Multifile::get_subfile_length 01649 // Access: Published 01650 // Description: Returns the uncompressed data length of the nth 01651 // subfile. This might return 0 if the subfile has 01652 // recently been added and flush() has not yet been 01653 // called. 01654 //////////////////////////////////////////////////////////////////// 01655 size_t Multifile:: 01656 get_subfile_length(int index) const { 01657 nassertr(index >= 0 && index < (int)_subfiles.size(), 0); 01658 return _subfiles[index]->_uncompressed_length; 01659 } 01660 01661 //////////////////////////////////////////////////////////////////// 01662 // Function: Multifile::get_subfile_timestamp 01663 // Access: Published 01664 // Description: Returns the modification time of the nth 01665 // subfile. If this is called on an older .mf file, 01666 // which did not store individual timestamps in the file 01667 // (or if get_record_timestamp() is false), this will 01668 // return the modification time of the overall 01669 // multifile. 01670 //////////////////////////////////////////////////////////////////// 01671 time_t Multifile:: 01672 get_subfile_timestamp(int index) const { 01673 nassertr(index >= 0 && index < (int)_subfiles.size(), 0); 01674 if (!get_record_timestamp()) { 01675 return get_timestamp(); 01676 } else { 01677 return _subfiles[index]->_timestamp; 01678 } 01679 } 01680 01681 //////////////////////////////////////////////////////////////////// 01682 // Function: Multifile::is_subfile_compressed 01683 // Access: Published 01684 // Description: Returns true if the indicated subfile has been 01685 // compressed when stored within the archive, false 01686 // otherwise. 01687 //////////////////////////////////////////////////////////////////// 01688 bool Multifile:: 01689 is_subfile_compressed(int index) const { 01690 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01691 return (_subfiles[index]->_flags & SF_compressed) != 0; 01692 } 01693 01694 //////////////////////////////////////////////////////////////////// 01695 // Function: Multifile::is_subfile_encrypted 01696 // Access: Published 01697 // Description: Returns true if the indicated subfile has been 01698 // encrypted when stored within the archive, false 01699 // otherwise. 01700 //////////////////////////////////////////////////////////////////// 01701 bool Multifile:: 01702 is_subfile_encrypted(int index) const { 01703 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01704 return (_subfiles[index]->_flags & SF_encrypted) != 0; 01705 } 01706 01707 //////////////////////////////////////////////////////////////////// 01708 // Function: Multifile::is_subfile_text 01709 // Access: Published 01710 // Description: Returns true if the indicated subfile represents text 01711 // data, or false if it represents binary data. If the 01712 // file is text data, it may have been processed by 01713 // end-of-line conversion when it was added. (But the 01714 // actual bits in the multifile will represent the 01715 // standard Unix end-of-line convention, e.g. \n instead 01716 // of \r\n.) 01717 //////////////////////////////////////////////////////////////////// 01718 bool Multifile:: 01719 is_subfile_text(int index) const { 01720 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01721 return (_subfiles[index]->_flags & SF_text) != 0; 01722 } 01723 01724 //////////////////////////////////////////////////////////////////// 01725 // Function: Multifile::get_index_end 01726 // Access: Published 01727 // Description: Returns the first byte that is guaranteed to follow 01728 // any index byte already written to disk in the 01729 // Multifile. 01730 // 01731 // This number is largely meaningless in many cases, but 01732 // if needs_repack() is false, and the file is flushed, 01733 // this will indicate the number of bytes in the header 01734 // + index. Everything at this byte position and later 01735 // will be actual data. 01736 //////////////////////////////////////////////////////////////////// 01737 streampos Multifile:: 01738 get_index_end() const { 01739 return normalize_streampos(_next_index + (streampos)4); 01740 } 01741 01742 //////////////////////////////////////////////////////////////////// 01743 // Function: Multifile::get_subfile_internal_start 01744 // Access: Published 01745 // Description: Returns the starting byte position within the 01746 // Multifile at which the indicated subfile begins. 01747 // This may be used, with get_subfile_internal_length(), 01748 // for low-level access to the subfile, but usually it 01749 // is better to use open_read_subfile() instead (which 01750 // automatically decrypts and/or uncompresses the 01751 // subfile data). 01752 //////////////////////////////////////////////////////////////////// 01753 streampos Multifile:: 01754 get_subfile_internal_start(int index) const { 01755 nassertr(index >= 0 && index < (int)_subfiles.size(), 0); 01756 return _subfiles[index]->_data_start; 01757 } 01758 01759 //////////////////////////////////////////////////////////////////// 01760 // Function: Multifile::get_subfile_internal_length 01761 // Access: Published 01762 // Description: Returns the number of bytes the indicated subfile 01763 // consumes within the archive. For compressed 01764 // subfiles, this will generally be smaller than 01765 // get_subfile_length(); for encrypted (but 01766 // noncompressed) subfiles, it may be slightly 01767 // different, for noncompressed and nonencrypted 01768 // subfiles, it will be equal. 01769 //////////////////////////////////////////////////////////////////// 01770 size_t Multifile:: 01771 get_subfile_internal_length(int index) const { 01772 nassertr(index >= 0 && index < (int)_subfiles.size(), 0); 01773 return _subfiles[index]->_data_length; 01774 } 01775 01776 //////////////////////////////////////////////////////////////////// 01777 // Function: Multifile::open_read_subfile 01778 // Access: Published 01779 // Description: Returns an istream that may be used to read the 01780 // indicated subfile. You may seek() within this 01781 // istream to your heart's content; even though it will 01782 // be a reference to the already-opened pfstream of the 01783 // Multifile itself, byte 0 appears to be the beginning 01784 // of the subfile and EOF appears to be the end of the 01785 // subfile. 01786 // 01787 // The returned istream will have been allocated via 01788 // new; you should pass the pointer to 01789 // close_read_subfile() when you are finished with it to 01790 // delete it and release its resources. 01791 // 01792 // Any future calls to repack() or close() (or the 01793 // Multifile destructor) will invalidate all currently 01794 // open subfile pointers. 01795 // 01796 // The return value will be NULL if the stream cannot be 01797 // opened for some reason. 01798 //////////////////////////////////////////////////////////////////// 01799 istream *Multifile:: 01800 open_read_subfile(int index) { 01801 nassertr(is_read_valid(), NULL); 01802 nassertr(index >= 0 && index < (int)_subfiles.size(), NULL); 01803 Subfile *subfile = _subfiles[index]; 01804 01805 if (subfile->_source != (istream *)NULL || 01806 !subfile->_source_filename.empty()) { 01807 // The subfile has not yet been copied into the physical 01808 // Multifile. Force a flush operation to incorporate it. 01809 flush(); 01810 01811 // That shouldn't change the subfile index or delete the subfile 01812 // pointer. 01813 nassertr(subfile == _subfiles[index], NULL); 01814 } 01815 01816 return open_read_subfile(subfile); 01817 } 01818 01819 //////////////////////////////////////////////////////////////////// 01820 // Function: Multifile::close_read_subfile 01821 // Access: Published, Static 01822 // Description: Closes a file opened by a previous call to 01823 // open_read_subfile(). This really just deletes the 01824 // istream pointer, but it is recommended to use this 01825 // interface instead of deleting it explicitly, to help 01826 // work around compiler issues. 01827 //////////////////////////////////////////////////////////////////// 01828 void Multifile:: 01829 close_read_subfile(istream *stream) { 01830 if (stream != (istream *)NULL) { 01831 // For some reason--compiler bug in gcc 3.2?--explicitly deleting 01832 // the stream pointer does not call the appropriate global delete 01833 // function; instead apparently calling the system delete 01834 // function. So we call the delete function by hand instead. 01835 #if !defined(WIN32_VC) && !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW) 01836 stream->~istream(); 01837 (*global_operator_delete)(stream); 01838 #else 01839 delete stream; 01840 #endif 01841 } 01842 } 01843 01844 //////////////////////////////////////////////////////////////////// 01845 // Function: Multifile::extract_subfile 01846 // Access: Published 01847 // Description: Extracts the nth subfile into a file with the given 01848 // name. 01849 //////////////////////////////////////////////////////////////////// 01850 bool Multifile:: 01851 extract_subfile(int index, const Filename &filename) { 01852 nassertr(is_read_valid(), false); 01853 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01854 01855 Filename fname = filename; 01856 if (multifile_always_binary) { 01857 fname.set_binary(); 01858 } 01859 01860 if (!fname.is_binary_or_text()) { 01861 // If we haven't specified binary or text, infer it from the type 01862 // of the subfile. 01863 if ((_subfiles[index]->_flags & SF_text) != 0) { 01864 fname.set_text(); 01865 } else { 01866 fname.set_binary(); 01867 } 01868 } 01869 fname.make_dir(); 01870 pofstream out; 01871 if (!fname.open_write(out, true)) { 01872 express_cat.info() 01873 << "Unable to write to file " << filename << "\n"; 01874 return false; 01875 } 01876 01877 return extract_subfile_to(index, out); 01878 } 01879 01880 //////////////////////////////////////////////////////////////////// 01881 // Function: Multifile::extract_subfile_to 01882 // Access: Public 01883 // Description: Extracts the nth subfile to the indicated ostream. 01884 //////////////////////////////////////////////////////////////////// 01885 bool Multifile:: 01886 extract_subfile_to(int index, ostream &out) { 01887 nassertr(is_read_valid(), false); 01888 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01889 01890 istream *in = open_read_subfile(index); 01891 if (in == (istream *)NULL) { 01892 return false; 01893 } 01894 01895 static const size_t buffer_size = 4096; 01896 char buffer[buffer_size]; 01897 01898 in->read(buffer, buffer_size); 01899 size_t count = in->gcount(); 01900 while (count != 0) { 01901 out.write(buffer, count); 01902 in->read(buffer, buffer_size); 01903 count = in->gcount(); 01904 } 01905 01906 bool failed = (in->fail() && !in->eof()); 01907 close_read_subfile(in); 01908 nassertr(!failed, false); 01909 01910 return (!out.fail()); 01911 } 01912 01913 //////////////////////////////////////////////////////////////////// 01914 // Function: Multifile::compare_subfile 01915 // Access: Published 01916 // Description: Performs a byte-for-byte comparison of the indicated 01917 // file on disk with the nth subfile. Returns true if 01918 // the files are equivalent, or false if they are 01919 // different (or the file is missing). 01920 // 01921 // If Filename::set_binary() or set_text() has already 01922 // been called, it specifies the nature of the source 01923 // file. If this is different from the text flag of the 01924 // subfile, the comparison will always return false. 01925 // If this has not been specified, it will be set from 01926 // the text flag of the subfile. 01927 //////////////////////////////////////////////////////////////////// 01928 bool Multifile:: 01929 compare_subfile(int index, const Filename &filename) { 01930 nassertr(is_read_valid(), false); 01931 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01932 01933 if (!filename.exists()) { 01934 express_cat.info() 01935 << "File is missing: " << filename << "\n"; 01936 return false; 01937 } 01938 01939 Filename fname = filename; 01940 if (fname.is_binary()) { 01941 // If we've specified a binary file, it had better be a binary 01942 // subfile. 01943 if ((_subfiles[index]->_flags & SF_text) != 0) { 01944 if (express_cat.is_debug()) { 01945 express_cat.debug() 01946 << "File is not binary: " << filename << "\n"; 01947 } 01948 return false; 01949 } 01950 01951 } else if (fname.is_text()) { 01952 // If we've specified a text file, it had better be a text 01953 // subfile. 01954 if ((_subfiles[index]->_flags & SF_text) == 0) { 01955 if (express_cat.is_debug()) { 01956 express_cat.debug() 01957 << "File is not text: " << filename << "\n"; 01958 } 01959 return false; 01960 } 01961 01962 } else { 01963 // If we haven't specified binary or text, infer it from the type 01964 // of the subfile. 01965 if ((_subfiles[index]->_flags & SF_text) != 0) { 01966 fname.set_text(); 01967 } else { 01968 fname.set_binary(); 01969 } 01970 } 01971 01972 istream *in1 = open_read_subfile(index); 01973 if (in1 == (istream *)NULL) { 01974 return false; 01975 } 01976 01977 pifstream in2; 01978 01979 if (!fname.open_read(in2)) { 01980 express_cat.info() 01981 << "Cannot read " << filename << "\n"; 01982 return false; 01983 } 01984 01985 if (fname.is_binary()) { 01986 // Check the file size. 01987 in2.seekg(0, ios::end); 01988 streampos file_size = in2.tellg(); 01989 01990 if (file_size != (streampos)get_subfile_length(index)) { 01991 // The files have different sizes. 01992 close_read_subfile(in1); 01993 return false; 01994 } 01995 } 01996 01997 // Check the file data, byte-for-byte. 01998 in2.seekg(0); 01999 int byte1 = in1->get(); 02000 int byte2 = in2.get(); 02001 while (!in1->fail() && !in1->eof() && 02002 !in2.fail() && !in2.eof()) { 02003 if (byte1 != byte2) { 02004 close_read_subfile(in1); 02005 return false; 02006 } 02007 byte1 = in1->get(); 02008 byte2 = in2.get(); 02009 } 02010 02011 bool failed = (in1->fail() && !in1->eof()) || (in2.fail() && !in2.eof()); 02012 close_read_subfile(in1); 02013 02014 nassertr(!failed, false); 02015 02016 return true; 02017 } 02018 02019 //////////////////////////////////////////////////////////////////// 02020 // Function: Multifile::output 02021 // Access: Published 02022 // Description: 02023 //////////////////////////////////////////////////////////////////// 02024 void Multifile:: 02025 output(ostream &out) const { 02026 out << "Multifile " << _multifile_name << ", " << get_num_subfiles() 02027 << " subfiles.\n"; 02028 } 02029 02030 //////////////////////////////////////////////////////////////////// 02031 // Function: Multifile::ls 02032 // Access: Published 02033 // Description: Shows a list of all subfiles within the Multifile. 02034 //////////////////////////////////////////////////////////////////// 02035 void Multifile:: 02036 ls(ostream &out) const { 02037 int num_subfiles = get_num_subfiles(); 02038 for (int i = 0; i < num_subfiles; i++) { 02039 string subfile_name = get_subfile_name(i); 02040 out << subfile_name << "\n"; 02041 } 02042 } 02043 02044 //////////////////////////////////////////////////////////////////// 02045 // Function: Multifile::set_header_prefix 02046 // Access: Published 02047 // Description: Sets the string which is written to the Multifile 02048 // before the Multifile header. This string must begin 02049 // with a hash mark and end with a newline character; 02050 // and if it includes embedded newline characters, each 02051 // one must be followed by a hash mark. If these 02052 // conditions are not initially true, the string will be 02053 // modified as necessary to make it so. 02054 // 02055 // This is primarily useful as a simple hack to allow 02056 // p3d applications to be run directly from the command 02057 // line on Unix-like systems. 02058 // 02059 // The return value is true if successful, or false on 02060 // failure (for instance, because the header prefix 02061 // violates the above rules). 02062 //////////////////////////////////////////////////////////////////// 02063 void Multifile:: 02064 set_header_prefix(const string &header_prefix) { 02065 string new_header_prefix = header_prefix; 02066 02067 if (!new_header_prefix.empty()) { 02068 // It must begin with a hash mark. 02069 if (new_header_prefix[0] != '#') { 02070 new_header_prefix = string("#") + new_header_prefix; 02071 } 02072 02073 // It must end with a newline. 02074 if (new_header_prefix[new_header_prefix.size() - 1] != '\n') { 02075 new_header_prefix += string("\n"); 02076 } 02077 02078 // Embedded newlines must be followed by a hash mark. 02079 size_t newline = new_header_prefix.find('\n'); 02080 while (newline < new_header_prefix.size() - 1) { 02081 if (new_header_prefix[newline + 1] != '#') { 02082 new_header_prefix = new_header_prefix.substr(0, newline + 1) + string("#") + new_header_prefix.substr(newline + 1); 02083 } 02084 newline = new_header_prefix.find('#', newline); 02085 } 02086 } 02087 02088 if (_header_prefix != new_header_prefix) { 02089 _header_prefix = new_header_prefix; 02090 _needs_repack = true; 02091 } 02092 } 02093 02094 02095 //////////////////////////////////////////////////////////////////// 02096 // Function: Multifile::read_subfile 02097 // Access: Public 02098 // Description: Fills a string with the entire contents of 02099 // the indicated subfile. 02100 //////////////////////////////////////////////////////////////////// 02101 bool Multifile:: 02102 read_subfile(int index, string &result) { 02103 result = string(); 02104 02105 // We use a temporary pvector, because dynamic accumulation of a 02106 // pvector seems to be many times faster than that of a string, at 02107 // least on the Windows implementation of STL. 02108 pvector<unsigned char> pv; 02109 if (!read_subfile(index, pv)) { 02110 return false; 02111 } 02112 02113 if (!pv.empty()) { 02114 result.append((const char *)&pv[0], pv.size()); 02115 } 02116 02117 return true; 02118 } 02119 02120 //////////////////////////////////////////////////////////////////// 02121 // Function: Multifile::read_subfile 02122 // Access: Public 02123 // Description: Fills a pvector with the entire contents of 02124 // the indicated subfile. 02125 //////////////////////////////////////////////////////////////////// 02126 bool Multifile:: 02127 read_subfile(int index, pvector<unsigned char> &result) { 02128 nassertr(is_read_valid(), false); 02129 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 02130 result.clear(); 02131 02132 // Now look up the particular Subfile we are reading. 02133 nassertr(is_read_valid(), NULL); 02134 nassertr(index >= 0 && index < (int)_subfiles.size(), NULL); 02135 Subfile *subfile = _subfiles[index]; 02136 02137 if (subfile->_source != (istream *)NULL || 02138 !subfile->_source_filename.empty()) { 02139 // The subfile has not yet been copied into the physical 02140 // Multifile. Force a flush operation to incorporate it. 02141 flush(); 02142 02143 // That shouldn't change the subfile index or delete the subfile 02144 // pointer. 02145 nassertr(subfile == _subfiles[index], NULL); 02146 } 02147 02148 result.reserve(subfile->_uncompressed_length); 02149 02150 bool success = true; 02151 if (subfile->_flags & (SF_encrypted | SF_compressed)) { 02152 // If the subfile is encrypted or compressed, we can't read it 02153 // directly. Fall back to the generic implementation. 02154 istream *in = open_read_subfile(index); 02155 if (in == (istream *)NULL) { 02156 return false; 02157 } 02158 02159 success = VirtualFile::simple_read_file(in, result); 02160 close_read_subfile(in); 02161 02162 } else { 02163 // But if the subfile is just a plain file, we can just read the 02164 // data directly from the Multifile, without paying the cost of an 02165 // ISubStream. 02166 static const size_t buffer_size = 4096; 02167 char buffer[buffer_size]; 02168 02169 streamsize pos = _offset + subfile->_data_start; 02170 size_t max_bytes = subfile->_data_length; 02171 streamsize count = 0; 02172 bool eof = true; 02173 02174 streamsize num_bytes = (streamsize)min(buffer_size, max_bytes); 02175 _read->seek_read(pos, buffer, num_bytes, count, eof); 02176 while (count != 0) { 02177 thread_consider_yield(); 02178 nassertr(count <= (streamsize)max_bytes, false); 02179 result.insert(result.end(), buffer, buffer + (size_t)count); 02180 max_bytes -= (size_t)count; 02181 pos += count; 02182 02183 num_bytes = (streamsize)min(buffer_size, max_bytes); 02184 _read->seek_read(pos, buffer, num_bytes, count, eof); 02185 } 02186 02187 success = !eof; 02188 } 02189 02190 if (!success) { 02191 ostringstream message; 02192 message << "I/O error reading from " << get_multifile_name() << " at " 02193 << get_subfile_name(index); 02194 nassert_raise(message.str()); 02195 return false; 02196 } 02197 02198 return true; 02199 } 02200 02201 //////////////////////////////////////////////////////////////////// 02202 // Function: Multifile::pad_to_streampos 02203 // Access: Private 02204 // Description: Assumes the _write pointer is at the indicated fpos, 02205 // rounds the fpos up to the next legitimate address 02206 // (using normalize_streampos()), and writes enough 02207 // zeroes to the stream to fill the gap. Returns the 02208 // new fpos. 02209 //////////////////////////////////////////////////////////////////// 02210 streampos Multifile:: 02211 pad_to_streampos(streampos fpos) { 02212 nassertr(_write != (ostream *)NULL, fpos); 02213 nassertr(_write->tellp() == fpos, fpos); 02214 streampos new_fpos = normalize_streampos(fpos); 02215 while (fpos < new_fpos) { 02216 _write->put(0); 02217 fpos += 1; // VC++ doesn't define streampos++ (!) 02218 } 02219 nassertr(_write->tellp() == fpos, fpos); 02220 return fpos; 02221 } 02222 02223 //////////////////////////////////////////////////////////////////// 02224 // Function: Multifile::add_new_subfile 02225 // Access: Private 02226 // Description: Adds a newly-allocated Subfile pointer to the 02227 // Multifile. 02228 //////////////////////////////////////////////////////////////////// 02229 void Multifile:: 02230 add_new_subfile(Subfile *subfile, int compression_level) { 02231 if (compression_level != 0) { 02232 #ifndef HAVE_ZLIB 02233 express_cat.warning() 02234 << "zlib not compiled in; cannot generated compressed multifiles.\n"; 02235 compression_level = 0; 02236 #else // HAVE_ZLIB 02237 subfile->_flags |= SF_compressed; 02238 subfile->_compression_level = compression_level; 02239 #endif // HAVE_ZLIB 02240 } 02241 02242 #ifdef HAVE_OPENSSL 02243 if (_encryption_flag) { 02244 subfile->_flags |= SF_encrypted; 02245 } 02246 #endif // HAVE_OPENSSL 02247 02248 if (_next_index != (streampos)0) { 02249 // If we're adding a Subfile to an already-existing Multifile, we 02250 // will eventually need to repack the file. 02251 _needs_repack = true; 02252 } 02253 02254 pair<Subfiles::iterator, bool> insert_result = _subfiles.insert(subfile); 02255 if (!insert_result.second) { 02256 // Hmm, unable to insert. There must already be a subfile by that 02257 // name. Remove the old one. 02258 Subfile *old_subfile = (*insert_result.first); 02259 old_subfile->_flags |= SF_deleted; 02260 02261 // Maybe it was just added to the _new_subfiles list. In this case, remove it from that list. 02262 PendingSubfiles::iterator ni = find(_new_subfiles.begin(), _new_subfiles.end(), old_subfile); 02263 if (ni != _new_subfiles.end()) { 02264 _new_subfiles.erase(ni); 02265 02266 } else { 02267 // Otherwise, add it to the _removed_subfiles list, so we can remove the old one. 02268 _removed_subfiles.push_back(old_subfile); 02269 } 02270 02271 (*insert_result.first) = subfile; 02272 } 02273 02274 _new_subfiles.push_back(subfile); 02275 } 02276 02277 //////////////////////////////////////////////////////////////////// 02278 // Function: Multifile::open_read_subfile 02279 // Access: Private 02280 // Description: This variant of open_read_subfile() is used 02281 // internally only, and accepts a pointer to the 02282 // internal Subfile object, which is assumed to be valid 02283 // and written to the multifile. 02284 //////////////////////////////////////////////////////////////////// 02285 istream *Multifile:: 02286 open_read_subfile(Subfile *subfile) { 02287 nassertr(subfile->_source == (istream *)NULL && 02288 subfile->_source_filename.empty(), NULL); 02289 02290 // Return an ISubStream object that references into the open 02291 // Multifile istream. 02292 nassertr(subfile->_data_start != (streampos)0, NULL); 02293 istream *stream = 02294 new ISubStream(_read, _offset + subfile->_data_start, 02295 _offset + subfile->_data_start + (streampos)subfile->_data_length); 02296 02297 if ((subfile->_flags & SF_encrypted) != 0) { 02298 #ifndef HAVE_OPENSSL 02299 express_cat.error() 02300 << "OpenSSL not compiled in; cannot read encrypted multifiles.\n"; 02301 delete stream; 02302 return NULL; 02303 #else // HAVE_OPENSSL 02304 // The subfile is encrypted. So actually, return an 02305 // IDecryptStream that wraps around the ISubStream. 02306 IDecryptStream *wrapper = 02307 new IDecryptStream(stream, true, _encryption_password); 02308 stream = wrapper; 02309 02310 // Validate the password by confirming that the encryption header 02311 // matches. 02312 char this_header[_encrypt_header_size]; 02313 stream->read(this_header, _encrypt_header_size); 02314 if (stream->fail() || stream->gcount() != (unsigned)_encrypt_header_size || 02315 memcmp(this_header, _encrypt_header, _encrypt_header_size) != 0) { 02316 express_cat.error() 02317 << "Unable to decrypt subfile " << subfile->_name << ".\n"; 02318 delete stream; 02319 return NULL; 02320 } 02321 #endif // HAVE_OPENSSL 02322 } 02323 02324 if ((subfile->_flags & SF_compressed) != 0) { 02325 #ifndef HAVE_ZLIB 02326 express_cat.error() 02327 << "zlib not compiled in; cannot read compressed multifiles.\n"; 02328 delete stream; 02329 return NULL; 02330 #else // HAVE_ZLIB 02331 // Oops, the subfile is compressed. So actually, return an 02332 // IDecompressStream that wraps around the ISubStream. 02333 IDecompressStream *wrapper = new IDecompressStream(stream, true); 02334 stream = wrapper; 02335 #endif // HAVE_ZLIB 02336 } 02337 02338 if (stream->fail()) { 02339 // Hmm, some inexplicable problem. 02340 delete stream; 02341 return NULL; 02342 } 02343 02344 return stream; 02345 } 02346 02347 //////////////////////////////////////////////////////////////////// 02348 // Function: Multifile::standardize_subfile_name 02349 // Access: Private 02350 // Description: Returns the standard form of the subfile name. 02351 //////////////////////////////////////////////////////////////////// 02352 string Multifile:: 02353 standardize_subfile_name(const string &subfile_name) const { 02354 Filename name = subfile_name; 02355 name.standardize(); 02356 if (name.empty() || name == "/") { 02357 // Invalid empty name. 02358 return string(); 02359 } 02360 02361 if (name[0] == '/') { 02362 return name.get_fullpath().substr(1); 02363 } else if (name.length() > 2 && name[0] == '.' && name[1] == '/') { 02364 return name.get_fullpath().substr(2); 02365 } else { 02366 return name.get_fullpath(); 02367 } 02368 } 02369 02370 //////////////////////////////////////////////////////////////////// 02371 // Function: Multifile::clear_subfiles 02372 // Access: Private 02373 // Description: Removes the set of subfiles from the tables and frees 02374 // their associated memory. 02375 //////////////////////////////////////////////////////////////////// 02376 void Multifile:: 02377 clear_subfiles() { 02378 PendingSubfiles::iterator pi; 02379 for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) { 02380 Subfile *subfile = (*pi); 02381 subfile->rewrite_index_flags(*_write); 02382 delete subfile; 02383 } 02384 _removed_subfiles.clear(); 02385 02386 // We don't have to delete the ones in _new_subfiles, because these 02387 // also appear in _subfiles. 02388 _new_subfiles.clear(); 02389 02390 #ifdef HAVE_OPENSSL 02391 for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) { 02392 Subfile *subfile = (*pi); 02393 delete subfile; 02394 } 02395 _cert_special.clear(); 02396 02397 _signatures.clear(); 02398 #endif // HAVE_OPENSSL 02399 02400 Subfiles::iterator fi; 02401 for (fi = _subfiles.begin(); fi != _subfiles.end(); ++fi) { 02402 Subfile *subfile = (*fi); 02403 delete subfile; 02404 } 02405 _subfiles.clear(); 02406 } 02407 02408 //////////////////////////////////////////////////////////////////// 02409 // Function: Multifile::read_index 02410 // Access: Private 02411 // Description: Reads the Multifile header and index. Returns true 02412 // if successful, false if the Multifile is not valid. 02413 //////////////////////////////////////////////////////////////////// 02414 bool Multifile:: 02415 read_index() { 02416 nassertr(_read != (IStreamWrapper *)NULL, false); 02417 02418 // We acquire the IStreamWrapper lock for the duration of this 02419 // method. 02420 _read->acquire(); 02421 istream *read = _read->get_istream(); 02422 02423 char this_header[_header_size]; 02424 read->seekg(_offset); 02425 02426 // Here's a special case: if the multifile begins with a hash 02427 // character, then we continue reading and discarding lines of ASCII 02428 // text, until we come across a nonempty line that does not begin 02429 // with a hash character. This allows a P3D application (which is a 02430 // multifile) to be run directly on the command line on Unix-based 02431 // systems. 02432 _header_prefix = string(); 02433 int ch = read->get(); 02434 02435 if (ch == '#') { 02436 while (ch != EOF && ch == '#') { 02437 // Skip to the end of the line. 02438 while (ch != EOF && ch != '\n') { 02439 _header_prefix += ch; 02440 ch = read->get(); 02441 } 02442 // Skip to the first non-whitespace character of the line. 02443 while (ch != EOF && (isspace(ch) || ch == '\r')) { 02444 _header_prefix += ch; 02445 ch = read->get(); 02446 } 02447 } 02448 } 02449 02450 // Now read the actual Multifile header. 02451 this_header[0] = ch; 02452 read->read(this_header + 1, _header_size - 1); 02453 if (read->fail() || read->gcount() != (unsigned)(_header_size - 1)) { 02454 express_cat.info() 02455 << "Unable to read Multifile header " << _multifile_name << ".\n"; 02456 _read->release(); 02457 close(); 02458 return false; 02459 } 02460 02461 if (memcmp(this_header, _header, _header_size) != 0) { 02462 express_cat.info() 02463 << _multifile_name << " is not a Multifile.\n"; 02464 _read->release(); 02465 close(); 02466 return false; 02467 } 02468 02469 // Now get the version numbers out. 02470 StreamReader reader(read, false); 02471 _file_major_ver = reader.get_int16(); 02472 _file_minor_ver = reader.get_int16(); 02473 _scale_factor = reader.get_uint32(); 02474 _new_scale_factor = _scale_factor; 02475 02476 if (read->eof() || read->fail()) { 02477 express_cat.info() 02478 << _multifile_name << " header is truncated.\n"; 02479 _read->release(); 02480 close(); 02481 return false; 02482 } 02483 02484 if (_file_major_ver != _current_major_ver || 02485 (_file_major_ver == _current_major_ver && 02486 _file_minor_ver > _current_minor_ver)) { 02487 express_cat.info() 02488 << _multifile_name << " has version " << _file_major_ver << "." 02489 << _file_minor_ver << ", expecting version " 02490 << _current_major_ver << "." << _current_minor_ver << ".\n"; 02491 _read->release(); 02492 close(); 02493 return false; 02494 } 02495 02496 _record_timestamp = true; 02497 if (_file_minor_ver >= 1) { 02498 time_t read_timestamp = reader.get_uint32(); 02499 if (read_timestamp == 0) { 02500 // If we read a 0 timestamp from the file, that implies that we 02501 // don't want to record a timestamp in this particular file. 02502 _record_timestamp = false; 02503 } else { 02504 _timestamp = read_timestamp; 02505 } 02506 _timestamp_dirty = false; 02507 } 02508 02509 // Now read the index out. 02510 _next_index = read->tellg() - _offset; 02511 _next_index = normalize_streampos(_next_index); 02512 read->seekg(_next_index + _offset); 02513 _last_index = 0; 02514 _last_data_byte = 0; 02515 streampos index_forward; 02516 streamoff bytes_skipped = 0; 02517 bool read_cert_special = false; 02518 02519 Subfile *subfile = new Subfile; 02520 index_forward = subfile->read_index(*read, _next_index, this); 02521 while (index_forward != (streampos)0) { 02522 _last_index = _next_index; 02523 if (subfile->is_deleted()) { 02524 // Ignore deleted Subfiles in the index. 02525 _needs_repack = true; 02526 delete subfile; 02527 } else if (subfile->is_cert_special()) { 02528 // Certificate chains and signature files get stored in a 02529 // special list. 02530 _cert_special.push_back(subfile); 02531 read_cert_special = true; 02532 } else { 02533 _subfiles.push_back(subfile); 02534 } 02535 if (!subfile->is_cert_special()) { 02536 if (bytes_skipped != 0) { 02537 // If the index entries don't follow exactly sequentially 02538 // (except for the cert special files), the file ought to be 02539 // repacked. 02540 _needs_repack = true; 02541 } 02542 if (read_cert_special) { 02543 // If we read a normal subfile following a cert_special entry, 02544 // the file ought to be repacked (certificates have to go at 02545 // the end). 02546 _needs_repack = true; 02547 } 02548 _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos()); 02549 } 02550 streampos curr_pos = normalize_streampos(read->tellg() - _offset); 02551 bytes_skipped = index_forward - curr_pos; 02552 read->seekg(index_forward + _offset); 02553 _next_index = index_forward; 02554 subfile = new Subfile; 02555 index_forward = subfile->read_index(*read, _next_index, this); 02556 } 02557 if (subfile->is_index_invalid()) { 02558 express_cat.info() 02559 << "Error reading index for " << _multifile_name << ".\n"; 02560 _read->release(); 02561 close(); 02562 delete subfile; 02563 return false; 02564 } 02565 02566 // Check if the list is already sorted. If it is not, we need a 02567 // repack. 02568 for (size_t si = 1; si < _subfiles.size() && !_needs_repack; ++si) { 02569 if (*_subfiles[si] < *_subfiles[si - 1]) { 02570 _needs_repack = true; 02571 } 02572 } 02573 02574 if (_needs_repack) { 02575 // At least sort them now. 02576 size_t before_size = _subfiles.size(); 02577 _subfiles.sort(); 02578 size_t after_size = _subfiles.size(); 02579 02580 // If these don't match, the same filename appeared twice in the 02581 // index, which shouldn't be possible. 02582 nassertr(before_size == after_size, true); 02583 } 02584 02585 delete subfile; 02586 _read->release(); 02587 return true; 02588 } 02589 02590 //////////////////////////////////////////////////////////////////// 02591 // Function: Multifile::write_header 02592 // Access: Private 02593 // Description: Writes just the header part of the Multifile, not the 02594 // index. 02595 //////////////////////////////////////////////////////////////////// 02596 bool Multifile:: 02597 write_header() { 02598 _file_major_ver = _current_major_ver; 02599 _file_minor_ver = _current_minor_ver; 02600 02601 nassertr(_write != (ostream *)NULL, false); 02602 nassertr(_write->tellp() == (streampos)0, false); 02603 _write->write(_header_prefix.data(), _header_prefix.size()); 02604 _write->write(_header, _header_size); 02605 StreamWriter writer(_write, false); 02606 writer.add_int16(_current_major_ver); 02607 writer.add_int16(_current_minor_ver); 02608 writer.add_uint32(_scale_factor); 02609 02610 if (_record_timestamp) { 02611 writer.add_uint32(_timestamp); 02612 } else { 02613 writer.add_uint32(0); 02614 _timestamp_dirty = false; 02615 } 02616 02617 _next_index = _write->tellp(); 02618 _next_index = pad_to_streampos(_next_index); 02619 _last_index = 0; 02620 02621 if (_write->fail()) { 02622 express_cat.info() 02623 << "Unable to write header for " << _multifile_name << ".\n"; 02624 close(); 02625 return false; 02626 } 02627 02628 return true; 02629 } 02630 02631 //////////////////////////////////////////////////////////////////// 02632 // Function: Multifile::check_signatures 02633 // Access: Private 02634 // Description: Walks through the list of _cert_special entries in 02635 // the Multifile, moving any valid signatures found to 02636 // _signatures. After this call, _cert_special will be 02637 // empty. 02638 // 02639 // This does not check the validity of the certificates 02640 // themselves. It only checks that they correctly sign 02641 // the Multifile contents. 02642 //////////////////////////////////////////////////////////////////// 02643 void Multifile:: 02644 check_signatures() { 02645 #ifdef HAVE_OPENSSL 02646 PendingSubfiles::iterator pi; 02647 02648 for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) { 02649 Subfile *subfile = (*pi); 02650 nassertv((subfile->_flags & SF_signature) != 0); 02651 02652 // Extract the signature data and certificate separately. 02653 istream *stream = open_read_subfile(subfile); 02654 nassertv(stream != NULL); 02655 StreamReader reader(*stream); 02656 size_t sig_size = reader.get_uint32(); 02657 string sig_string = reader.extract_bytes(sig_size); 02658 02659 size_t num_certs = reader.get_uint32(); 02660 02661 // Read the remaining buffer of certificate data. 02662 pvector<unsigned char> buffer; 02663 bool success = VirtualFile::simple_read_file(stream, buffer); 02664 nassertv(success); 02665 close_read_subfile(stream); 02666 02667 // Now convert each of the certificates to an X509 object, and 02668 // store it in our CertChain. 02669 CertChain chain; 02670 EVP_PKEY *pkey = NULL; 02671 if (!buffer.empty()) { 02672 #if OPENSSL_VERSION_NUMBER >= 0x00908000L 02673 // Beginning in 0.9.8, d2i_X509() accepted a const unsigned char **. 02674 const unsigned char *bp, *bp_end; 02675 #else 02676 // Prior to 0.9.8, d2i_X509() accepted an unsigned char **. 02677 unsigned char *bp, *bp_end; 02678 #endif 02679 bp = (unsigned char *)&buffer[0]; 02680 bp_end = bp + buffer.size(); 02681 X509 *x509 = d2i_X509(NULL, &bp, bp_end - bp); 02682 while (num_certs > 0 && x509 != NULL) { 02683 chain.push_back(CertRecord(x509)); 02684 --num_certs; 02685 x509 = d2i_X509(NULL, &bp, bp_end - bp); 02686 } 02687 if (num_certs != 0 || x509 != NULL) { 02688 express_cat.warning() 02689 << "Extra data in signature record.\n"; 02690 } 02691 } 02692 02693 if (!chain.empty()) { 02694 pkey = X509_get_pubkey(chain[0]._cert); 02695 } 02696 02697 if (pkey != NULL) { 02698 EVP_MD_CTX *md_ctx; 02699 #ifdef SSL_097 02700 md_ctx = EVP_MD_CTX_create(); 02701 #else 02702 md_ctx = new EVP_MD_CTX; 02703 #endif 02704 EVP_VerifyInit(md_ctx, EVP_sha1()); 02705 02706 nassertv(_read != NULL); 02707 _read->acquire(); 02708 istream *read = _read->get_istream(); 02709 02710 // Read and hash the multifile contents, but only up till 02711 // _last_data_byte. 02712 read->seekg(_offset); 02713 streampos bytes_remaining = _last_data_byte; 02714 static const size_t buffer_size = 4096; 02715 char buffer[buffer_size]; 02716 read->read(buffer, min((streampos)buffer_size, bytes_remaining)); 02717 size_t count = read->gcount(); 02718 while (count != 0) { 02719 nassertv(count <= buffer_size); 02720 EVP_VerifyUpdate(md_ctx, buffer, count); 02721 bytes_remaining -= count; 02722 read->read(buffer, min((streampos)buffer_size, bytes_remaining)); 02723 count = read->gcount(); 02724 } 02725 nassertv(bytes_remaining == (streampos)0); 02726 _read->release(); 02727 02728 // Now check that the signature matches the hash. 02729 int verify_result = 02730 EVP_VerifyFinal(md_ctx, 02731 (unsigned char *)sig_string.data(), 02732 sig_string.size(), pkey); 02733 if (verify_result == 1) { 02734 // The signature matches; save the certificate and its chain. 02735 _signatures.push_back(chain); 02736 } else { 02737 // Bad match. 02738 _needs_repack = true; 02739 } 02740 } 02741 } 02742 #endif // HAVE_OPENSSL 02743 02744 _cert_special.clear(); 02745 } 02746 02747 //////////////////////////////////////////////////////////////////// 02748 // Function: Multifile::Subfile::read_index 02749 // Access: Public 02750 // Description: Reads the index record for the Subfile from the 02751 // indicated istream. Assumes the istream has already 02752 // been positioned to the indicated stream position, 02753 // fpos, the start of the index record. Returns the 02754 // position within the file of the next index record. 02755 //////////////////////////////////////////////////////////////////// 02756 streampos Multifile::Subfile:: 02757 read_index(istream &read, streampos fpos, Multifile *multifile) { 02758 nassertr(read.tellg() - multifile->_offset == fpos, fpos); 02759 02760 // First, get the next stream position. We do this separately, 02761 // because if it is zero, we don't get anything else. 02762 StreamReader reader(read); 02763 02764 streampos next_index = multifile->word_to_streampos(reader.get_uint32()); 02765 if (read.eof() || read.fail()) { 02766 _flags |= SF_index_invalid; 02767 return 0; 02768 } 02769 02770 if (next_index == (streampos)0) { 02771 return 0; 02772 } 02773 02774 // Now get the rest of the index. 02775 02776 _index_start = fpos; 02777 _index_length = 0; 02778 02779 _data_start = multifile->word_to_streampos(reader.get_uint32()); 02780 _data_length = reader.get_uint32(); 02781 _flags = reader.get_uint16(); 02782 if ((_flags & (SF_compressed | SF_encrypted)) != 0) { 02783 _uncompressed_length = reader.get_uint32(); 02784 } else { 02785 _uncompressed_length = _data_length; 02786 } 02787 if (multifile->_file_minor_ver < 1) { 02788 _timestamp = multifile->get_timestamp(); 02789 } else { 02790 _timestamp = reader.get_uint32(); 02791 if (_timestamp == 0) { 02792 _timestamp = multifile->get_timestamp(); 02793 } 02794 } 02795 02796 size_t name_length = reader.get_uint16(); 02797 if (read.eof() || read.fail()) { 02798 _flags |= SF_index_invalid; 02799 return 0; 02800 } 02801 02802 // And finally, get the rest of the name. 02803 char *name_buffer = (char *)PANDA_MALLOC_ARRAY(name_length); 02804 nassertr(name_buffer != (char *)NULL, next_index); 02805 for (size_t ni = 0; ni < name_length; ni++) { 02806 name_buffer[ni] = read.get() ^ 0xff; 02807 } 02808 _name = string(name_buffer, name_length); 02809 PANDA_FREE_ARRAY(name_buffer); 02810 02811 if (read.eof() || read.fail()) { 02812 _flags |= SF_index_invalid; 02813 return 0; 02814 } 02815 02816 _index_length = read.tellg() - fpos - multifile->_offset; 02817 return next_index; 02818 } 02819 02820 //////////////////////////////////////////////////////////////////// 02821 // Function: Multifile::Subfile::write_index 02822 // Access: Public 02823 // Description: Writes the index record for the Subfile to the 02824 // indicated ostream. Assumes the istream has already 02825 // been positioned to the indicated stream position, 02826 // fpos, the start of the index record, and that this is 02827 // the effective end of the file. Returns the position 02828 // within the file of the next index record. 02829 // 02830 // The _index_start member is updated by this operation. 02831 //////////////////////////////////////////////////////////////////// 02832 streampos Multifile::Subfile:: 02833 write_index(ostream &write, streampos fpos, Multifile *multifile) { 02834 nassertr(write.tellp() - multifile->_offset == fpos, fpos); 02835 02836 _index_start = fpos; 02837 _index_length = 0; 02838 02839 // This will be the contents of this particular index record. We 02840 // build it up first since it will be variable length. 02841 Datagram dg; 02842 dg.add_uint32(multifile->streampos_to_word(_data_start)); 02843 dg.add_uint32(_data_length); 02844 dg.add_uint16(_flags); 02845 if ((_flags & (SF_compressed | SF_encrypted)) != 0) { 02846 dg.add_uint32(_uncompressed_length); 02847 } 02848 dg.add_uint32(_timestamp); 02849 dg.add_uint16(_name.length()); 02850 02851 // For no real good reason, we'll invert all the bits in the name. 02852 // The only reason we do this is to make it inconvenient for a 02853 // casual browser of the Multifile to discover the names of the 02854 // files stored within it. Naturally, this isn't real obfuscation 02855 // or security. 02856 string::iterator ni; 02857 for (ni = _name.begin(); ni != _name.end(); ++ni) { 02858 dg.add_int8((*ni) ^ 0xff); 02859 } 02860 02861 size_t this_index_size = 4 + dg.get_length(); 02862 02863 // Plus, we will write out the next index address first. 02864 streampos next_index = fpos + (streampos)this_index_size; 02865 02866 Datagram idg; 02867 idg.add_uint32(multifile->streampos_to_word(next_index)); 02868 02869 write.write((const char *)idg.get_data(), idg.get_length()); 02870 write.write((const char *)dg.get_data(), dg.get_length()); 02871 02872 _index_length = write.tellp() - fpos - multifile->_offset; 02873 return next_index; 02874 } 02875 02876 //////////////////////////////////////////////////////////////////// 02877 // Function: Multifile::Subfile::write_data 02878 // Access: Public 02879 // Description: Writes the data record for the Subfile to the 02880 // indicated ostream: the actual contents of the 02881 // Subfile. Assumes the istream has already been 02882 // positioned to the indicated stream position, fpos, 02883 // the start of the data record, and that this is the 02884 // effective end of the file. Returns the position 02885 // within the file of the next data record. 02886 // 02887 // The _data_start, _data_length, and 02888 // _uncompressed_length members are updated by this 02889 // operation. 02890 // 02891 // If the "read" pointer is non-NULL, it is the readable 02892 // istream of a Multifile in which the Subfile might 02893 // already be packed. This is used for reading the 02894 // contents of the Subfile during a repack() operation. 02895 //////////////////////////////////////////////////////////////////// 02896 streampos Multifile::Subfile:: 02897 write_data(ostream &write, istream *read, streampos fpos, 02898 Multifile *multifile) { 02899 nassertr(write.tellp() - multifile->_offset == fpos, fpos); 02900 02901 istream *source = _source; 02902 pifstream source_file; 02903 if (source == (istream *)NULL && !_source_filename.empty()) { 02904 // If we have a filename, open it up and read that. 02905 if (!_source_filename.open_read(source_file)) { 02906 // Unable to open the source file. 02907 express_cat.info() 02908 << "Unable to read " << _source_filename << ".\n"; 02909 _flags |= SF_data_invalid; 02910 _data_length = 0; 02911 _uncompressed_length = 0; 02912 } else { 02913 source = &source_file; 02914 } 02915 } 02916 02917 if (source == (istream *)NULL) { 02918 // We don't have any source data. Perhaps we're reading from an 02919 // already-packed Subfile (e.g. during repack()). 02920 if (read == (istream *)NULL) { 02921 // No, we're just screwed. 02922 express_cat.info() 02923 << "No source for subfile " << _name << ".\n"; 02924 _flags |= SF_data_invalid; 02925 } else { 02926 // Read the data from the original Multifile. 02927 read->seekg(_data_start + multifile->_offset); 02928 for (size_t p = 0; p < _data_length; p++) { 02929 int byte = read->get(); 02930 if (read->eof() || read->fail()) { 02931 // Unexpected EOF or other failure on the source file. 02932 express_cat.info() 02933 << "Unexpected EOF for subfile " << _name << ".\n"; 02934 _flags |= SF_data_invalid; 02935 break; 02936 } 02937 write.put(byte); 02938 } 02939 } 02940 } else { 02941 // We do have source data. Copy it in, and also measure its 02942 // length. 02943 ostream *putter = &write; 02944 bool delete_putter = false; 02945 02946 #ifndef HAVE_OPENSSL 02947 // Without OpenSSL, we can't support encryption. The flag had 02948 // better not be set. 02949 nassertr((_flags & SF_encrypted) == 0, fpos); 02950 02951 #else // HAVE_OPENSSL 02952 if ((_flags & SF_encrypted) != 0) { 02953 // Write it encrypted. 02954 OEncryptStream *encrypt = new OEncryptStream; 02955 encrypt->set_iteration_count(multifile->_encryption_iteration_count); 02956 encrypt->open(putter, delete_putter, multifile->_encryption_password); 02957 02958 putter = encrypt; 02959 delete_putter = true; 02960 02961 // Also write the encrypt_header to the beginning of the 02962 // encrypted stream, so we can validate the password on 02963 // decryption. 02964 putter->write(_encrypt_header, _encrypt_header_size); 02965 } 02966 #endif // HAVE_OPENSSL 02967 02968 #ifndef HAVE_ZLIB 02969 // Without ZLIB, we can't support compression. The flag had 02970 // better not be set. 02971 nassertr((_flags & SF_compressed) == 0, fpos); 02972 #else // HAVE_ZLIB 02973 if ((_flags & SF_compressed) != 0) { 02974 // Write it compressed. 02975 putter = new OCompressStream(putter, delete_putter, _compression_level); 02976 delete_putter = true; 02977 } 02978 #endif // HAVE_ZLIB 02979 02980 streampos write_start = fpos; 02981 _uncompressed_length = 0; 02982 02983 #ifndef HAVE_OPENSSL 02984 // We also need OpenSSL for signatures. 02985 nassertr((_flags & SF_signature) == 0, fpos); 02986 02987 #else // HAVE_OPENSSL 02988 if ((_flags & SF_signature) != 0) { 02989 // If it's a special signature record, precede the record data 02990 // (the certificate itself) with the signature data generated 02991 // against the multifile contents. 02992 02993 // In order to generate a signature, we need to have a valid 02994 // read pointer. 02995 nassertr(read != NULL, fpos); 02996 02997 // And we also need to have a private key. 02998 nassertr(_pkey != NULL, fpos); 02999 03000 EVP_MD_CTX *md_ctx; 03001 #ifdef SSL_097 03002 md_ctx = EVP_MD_CTX_create(); 03003 #else 03004 md_ctx = new EVP_MD_CTX; 03005 #endif 03006 EVP_SignInit(md_ctx, EVP_sha1()); 03007 03008 // Read and hash the multifile contents, but only up till 03009 // _last_data_byte. 03010 nassertr(multifile->_last_data_byte < fpos, fpos); 03011 read->seekg(multifile->_offset); 03012 streampos bytes_remaining = multifile->_last_data_byte; 03013 static const size_t buffer_size = 4096; 03014 char buffer[buffer_size]; 03015 read->read(buffer, min((streampos)buffer_size, bytes_remaining)); 03016 size_t count = read->gcount(); 03017 while (count != 0) { 03018 nassertr(count <= buffer_size, fpos); 03019 EVP_SignUpdate(md_ctx, buffer, count); 03020 bytes_remaining -= count; 03021 read->read(buffer, min((streampos)buffer_size, bytes_remaining)); 03022 count = read->gcount(); 03023 } 03024 nassertr(bytes_remaining == (streampos)0, fpos); 03025 03026 // Now generate and write out the signature. 03027 unsigned int max_size = EVP_PKEY_size(_pkey); 03028 unsigned char *sig_data = new unsigned char[max_size]; 03029 unsigned int sig_size; 03030 if (!EVP_SignFinal(md_ctx, sig_data, &sig_size, _pkey)) { 03031 OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr(); 03032 sslw->notify_ssl_errors(); 03033 } 03034 nassertr(sig_size <= max_size, fpos); 03035 03036 StreamWriter writer(*putter); 03037 writer.add_uint32(sig_size); 03038 putter->write((char *)sig_data, sig_size); 03039 _uncompressed_length += 4 + sig_size; 03040 03041 delete[] sig_data; 03042 03043 #ifdef SSL_097 03044 EVP_MD_CTX_destroy(md_ctx); 03045 #else 03046 delete md_ctx; 03047 #endif 03048 } 03049 #endif // HAVE_OPENSSL 03050 03051 // Finally, we can write out the data itself. 03052 static const size_t buffer_size = 4096; 03053 char buffer[buffer_size]; 03054 03055 source->read(buffer, buffer_size); 03056 size_t count = source->gcount(); 03057 while (count != 0) { 03058 _uncompressed_length += count; 03059 putter->write(buffer, count); 03060 source->read(buffer, buffer_size); 03061 count = source->gcount(); 03062 } 03063 03064 if (delete_putter) { 03065 delete putter; 03066 } 03067 03068 streampos write_end = write.tellp() - multifile->_offset; 03069 _data_length = (size_t)(write_end - write_start); 03070 } 03071 03072 // We can't set _data_start until down here, after we have read the 03073 // Subfile. (In case we are running during repack()). 03074 _data_start = fpos; 03075 03076 // Get the modification timestamp for this subfile. This is read 03077 // from the source file, if we have a filename; otherwise, it's the 03078 // current time. 03079 if (!_source_filename.empty()) { 03080 _timestamp = _source_filename.get_timestamp(); 03081 } 03082 if (_timestamp == 0) { 03083 _timestamp = time(NULL); 03084 } 03085 03086 _source = (istream *)NULL; 03087 _source_filename = Filename(); 03088 source_file.close(); 03089 03090 return fpos + (streampos)_data_length; 03091 } 03092 03093 //////////////////////////////////////////////////////////////////// 03094 // Function: Multifile::Subfile::rewrite_index_data_start 03095 // Access: Public 03096 // Description: Seeks within the indicate pfstream back to the index 03097 // record and rewrites just the _data_start and 03098 // _data_length part of the index record. 03099 //////////////////////////////////////////////////////////////////// 03100 void Multifile::Subfile:: 03101 rewrite_index_data_start(ostream &write, Multifile *multifile) { 03102 nassertv(_index_start != (streampos)0); 03103 03104 static const size_t data_start_offset = 4; 03105 size_t data_start_pos = _index_start + (streampos)data_start_offset; 03106 write.seekp(data_start_pos + multifile->_offset); 03107 nassertv(!write.fail()); 03108 03109 StreamWriter writer(write); 03110 writer.add_uint32(multifile->streampos_to_word(_data_start)); 03111 writer.add_uint32(_data_length); 03112 writer.add_uint16(_flags); 03113 if ((_flags & (SF_compressed | SF_encrypted)) != 0) { 03114 writer.add_uint32(_uncompressed_length); 03115 } 03116 if (multifile->_record_timestamp) { 03117 writer.add_uint32(_timestamp); 03118 } else { 03119 writer.add_uint32(0); 03120 } 03121 } 03122 03123 //////////////////////////////////////////////////////////////////// 03124 // Function: Multifile::Subfile::rewrite_index_flags 03125 // Access: Public 03126 // Description: Seeks within the indicated ostream back to the index 03127 // record and rewrites just the _flags part of the 03128 // index record. 03129 //////////////////////////////////////////////////////////////////// 03130 void Multifile::Subfile:: 03131 rewrite_index_flags(ostream &write) { 03132 // If the subfile has never even been recorded to disk, we don't 03133 // need to do anything at all in this function. 03134 if (_index_start != (streampos)0) { 03135 static const size_t flags_offset = 4 + 4 + 4; 03136 size_t flags_pos = _index_start + (streampos)flags_offset; 03137 write.seekp(flags_pos); 03138 nassertv(!write.fail()); 03139 03140 StreamWriter writer(write); 03141 writer.add_uint16(_flags); 03142 } 03143 }