Panda3D
|
00001 // Filename: multifile.cxx 00002 // Created by: mike (09Jan97) 00003 // 00004 //////////////////////////////////////////////////////////////////// 00005 // 00006 // PANDA 3D SOFTWARE 00007 // Copyright (c) Carnegie Mellon University. All rights reserved. 00008 // 00009 // All use of this software is subject to the terms of the revised BSD 00010 // license. You should have received a copy of this license along 00011 // with this source code in a file named "LICENSE." 00012 // 00013 //////////////////////////////////////////////////////////////////// 00014 00015 #include "multifile.h" 00016 00017 #include "config_express.h" 00018 #include "streamWriter.h" 00019 #include "streamReader.h" 00020 #include "datagram.h" 00021 #include "zStream.h" 00022 #include "encryptStream.h" 00023 #include "virtualFileSystem.h" 00024 #include "virtualFile.h" 00025 00026 #include <algorithm> 00027 #include <time.h> 00028 00029 // This sequence of bytes begins each Multifile to identify it as a 00030 // Multifile. 00031 const char Multifile::_header[] = "pmf\0\n\r"; 00032 const size_t Multifile::_header_size = 6; 00033 00034 // These numbers identify the version of the Multifile. Generally, a 00035 // change in the major version is intolerable; while a Multifile with 00036 // an older minor version may still be read. 00037 const int Multifile::_current_major_ver = 1; 00038 00039 const int Multifile::_current_minor_ver = 1; 00040 // Bumped to version 1.1 on 6/8/06 to add timestamps. 00041 00042 // To confirm that the supplied password matches, we write the 00043 // Mutifile magic header at the beginning of the encrypted stream. 00044 // I suppose this does compromise the encryption security a tiny 00045 // bit by making it easy for crackers to validate that a 00046 // particular password guess matches or doesn't match, but the 00047 // encryption algorithm doesn't depend on this being difficult 00048 // anyway. 00049 const char Multifile::_encrypt_header[] = "crypty"; 00050 const size_t Multifile::_encrypt_header_size = 6; 00051 00052 00053 00054 // 00055 // A Multifile consists of the following elements: 00056 // 00057 // (1) A header. This is always the first n bytes of the Multifile, 00058 // and contains a magic number to identify the file, as well as 00059 // version numbers and any file-specific parameters. 00060 // 00061 // char[6] The string Multifile::_header, a magic number. 00062 // int16 The file's major version number 00063 // int16 The file's minor version number 00064 // uint32 Scale factor. This scales all address references within 00065 // the file. Normally 1, this may be set larger to 00066 // support Multifiles larger than 4GB. 00067 // uint32 An overall modification timestamp for the entire multifile. 00068 00069 // 00070 // (2) Zero or more index entries, one for each subfile within the 00071 // Multifile. These entries are of variable length. The first one of 00072 // these immediately follows the header, and the first word of each 00073 // index entry contains the address of the next index entry. A zero 00074 // "next" address marks the end of the chain. These may appear at any 00075 // point within the Multifile; they do not necessarily appear in 00076 // sequential order at the beginning of the file (although they will 00077 // after the file has been "packed"). 00078 // 00079 // uint32 The address of the next entry. 0 to mark the end. 00080 // uint32 The address of this subfile's data record. 00081 // uint32 The length in bytes of this subfile's data record. 00082 // uint16 The Subfile::_flags member. 00083 // [uint32] The original, uncompressed and unencrypted length of the 00084 // subfile, if it is compressed or encrypted. This field 00085 // is only present if one or both of the SF_compressed 00086 // or SF_encrypted bits are set in _flags. 00087 // uint32 A modification timestamp for the subfile. 00088 // uint16 The length in bytes of the subfile's name. 00089 // char[n] The subfile's name. 00090 // 00091 // (3) Zero or more data entries, one for each subfile. These may 00092 // appear at any point within the Multifile; they do not necessarily 00093 // follow each index entry, nor are they necessarily all grouped 00094 // together at the end (although they will be all grouped together at 00095 // the end after the file has been "packed"). These are just blocks 00096 // of literal data. 00097 // 00098 00099 //////////////////////////////////////////////////////////////////// 00100 // Function: Multifile::Constructor 00101 // Access: Published 00102 // Description: 00103 //////////////////////////////////////////////////////////////////// 00104 Multifile:: 00105 Multifile() : 00106 _read_filew(_read_file), 00107 _read_write_filew(_read_write_file) 00108 { 00109 ConfigVariableInt multifile_encryption_iteration_count 00110 ("multifile-encryption-iteration-count", 0, 00111 PRC_DESC("This is a special value of encryption-iteration-count used to encrypt " 00112 "subfiles within a multifile. It has a default value of 0 (just one " 00113 "application), on the assumption that the files from a multifile must " 00114 "be loaded quickly, without paying the cost of an expensive hash on " 00115 "each subfile in order to decrypt it.")); 00116 00117 _read = (IStreamWrapper *)NULL; 00118 _write = (ostream *)NULL; 00119 _offset = 0; 00120 _owns_stream = false; 00121 _next_index = 0; 00122 _last_index = 0; 00123 _last_data_byte = 0; 00124 _needs_repack = false; 00125 _timestamp = 0; 00126 _timestamp_dirty = false; 00127 _record_timestamp = true; 00128 _scale_factor = 1; 00129 _new_scale_factor = 1; 00130 _encryption_flag = false; 00131 _encryption_iteration_count = multifile_encryption_iteration_count; 00132 _file_major_ver = 0; 00133 _file_minor_ver = 0; 00134 00135 #ifdef HAVE_OPENSSL 00136 // Get these values from the config file via an EncryptStreamBuf. 00137 EncryptStreamBuf tbuf; 00138 _encryption_algorithm = tbuf.get_algorithm(); 00139 _encryption_key_length = tbuf.get_key_length(); 00140 #endif 00141 } 00142 00143 //////////////////////////////////////////////////////////////////// 00144 // Function: Multifile::Destructor 00145 // Access: Published 00146 // Description: 00147 //////////////////////////////////////////////////////////////////// 00148 Multifile:: 00149 ~Multifile() { 00150 close(); 00151 } 00152 00153 //////////////////////////////////////////////////////////////////// 00154 // Function: Multifile::Copy Constructor 00155 // Access: Private 00156 // Description: Don't try to copy Multifiles. 00157 //////////////////////////////////////////////////////////////////// 00158 Multifile:: 00159 Multifile(const Multifile ©) : 00160 _read_filew(_read_file), 00161 _read_write_filew(_read_write_file) 00162 { 00163 nassertv(false); 00164 } 00165 00166 //////////////////////////////////////////////////////////////////// 00167 // Function: Multifile::Copy Assignment Operator 00168 // Access: Private 00169 // Description: Don't try to copy Multifiles. 00170 //////////////////////////////////////////////////////////////////// 00171 void Multifile:: 00172 operator = (const Multifile ©) { 00173 nassertv(false); 00174 } 00175 00176 //////////////////////////////////////////////////////////////////// 00177 // Function: Multifile::open_read 00178 // Access: Published 00179 // Description: Opens the named Multifile on disk for reading. The 00180 // Multifile index is read in, and the list of subfiles 00181 // becomes available; individual subfiles may then be 00182 // extracted or read, but the list of subfiles may not 00183 // be modified. 00184 // 00185 // Also see the version of open_read() which accepts an 00186 // istream. Returns true on success, false on failure. 00187 //////////////////////////////////////////////////////////////////// 00188 bool Multifile:: 00189 open_read(const Filename &multifile_name, const streampos &offset) { 00190 close(); 00191 Filename fname = multifile_name; 00192 fname.set_binary(); 00193 00194 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00195 PT(VirtualFile) vfile = vfs->get_file(fname); 00196 if (vfile == NULL) { 00197 return false; 00198 } 00199 istream *multifile_stream = vfile->open_read_file(false); 00200 if (multifile_stream == NULL) { 00201 return false; 00202 } 00203 00204 _timestamp = vfile->get_timestamp(); 00205 _timestamp_dirty = true; 00206 _read = new IStreamWrapper(multifile_stream, true); 00207 _owns_stream = true; 00208 _multifile_name = multifile_name; 00209 _offset = offset; 00210 return read_index(); 00211 } 00212 00213 //////////////////////////////////////////////////////////////////// 00214 // Function: Multifile::open_read 00215 // Access: Public 00216 // Description: Opens an anonymous Multifile for reading using an 00217 // istream. There must be seek functionality via 00218 // seekg() and tellg() on the istream. 00219 // 00220 // If owns_pointer is true, then the Multifile assumes 00221 // ownership of the stream pointer and will delete it 00222 // when the multifile is closed, including if this 00223 // function returns false. 00224 //////////////////////////////////////////////////////////////////// 00225 bool Multifile:: 00226 open_read(IStreamWrapper *multifile_stream, bool owns_pointer, 00227 const streampos &offset) { 00228 close(); 00229 _timestamp = time(NULL); 00230 _timestamp_dirty = true; 00231 _read = multifile_stream; 00232 _owns_stream = owns_pointer; 00233 _offset = offset; 00234 return read_index(); 00235 } 00236 00237 //////////////////////////////////////////////////////////////////// 00238 // Function: Multifile::open_write 00239 // Access: Published 00240 // Description: Opens the named Multifile on disk for writing. If 00241 // there already exists a file by that name, it is 00242 // truncated. The Multifile is then prepared for 00243 // accepting a brand new set of subfiles, which will be 00244 // written to the indicated filename. Individual 00245 // subfiles may not be extracted or read. 00246 // 00247 // Also see the version of open_write() which accepts an 00248 // ostream. Returns true on success, false on failure. 00249 //////////////////////////////////////////////////////////////////// 00250 bool Multifile:: 00251 open_write(const Filename &multifile_name) { 00252 close(); 00253 Filename fname = multifile_name; 00254 fname.set_binary(); 00255 if (!fname.open_write(_write_file, true)) { 00256 return false; 00257 } 00258 _timestamp = time(NULL); 00259 _timestamp_dirty = true; 00260 _write = &_write_file; 00261 _multifile_name = multifile_name; 00262 return true; 00263 } 00264 00265 //////////////////////////////////////////////////////////////////// 00266 // Function: Multifile::open_write 00267 // Access: Public 00268 // Description: Opens an anonymous Multifile for writing using an 00269 // ostream. There must be seek functionality via 00270 // seekp() and tellp() on the pstream. 00271 // 00272 // If owns_pointer is true, then the Multifile assumes 00273 // ownership of the stream pointer and will delete it 00274 // when the multifile is closed, including if this 00275 // function returns false. 00276 //////////////////////////////////////////////////////////////////// 00277 bool Multifile:: 00278 open_write(ostream *multifile_stream, bool owns_pointer) { 00279 close(); 00280 _timestamp = time(NULL); 00281 _timestamp_dirty = true; 00282 _write = multifile_stream; 00283 _owns_stream = owns_pointer; 00284 _write->seekp(0, ios::beg); 00285 return true; 00286 } 00287 00288 //////////////////////////////////////////////////////////////////// 00289 // Function: Multifile::open_read_write 00290 // Access: Published 00291 // Description: Opens the named Multifile on disk for reading and 00292 // writing. If there already exists a file by that 00293 // name, its index is read. Subfiles may be added or 00294 // removed, and the resulting changes will be written to 00295 // the named file. 00296 // 00297 // Also see the version of open_read_write() which 00298 // accepts an iostream. Returns true on success, false 00299 // on failure. 00300 //////////////////////////////////////////////////////////////////// 00301 bool Multifile:: 00302 open_read_write(const Filename &multifile_name) { 00303 close(); 00304 Filename fname = multifile_name; 00305 fname.set_binary(); 00306 bool exists = fname.exists(); 00307 if (!fname.open_read_write(_read_write_file)) { 00308 return false; 00309 } 00310 if (exists) { 00311 _timestamp = fname.get_timestamp(); 00312 } else { 00313 _timestamp = time(NULL); 00314 } 00315 _timestamp_dirty = true; 00316 _read = &_read_write_filew; 00317 _write = &_read_write_file; 00318 _multifile_name = multifile_name; 00319 00320 if (exists) { 00321 return read_index(); 00322 } else { 00323 return true; 00324 } 00325 } 00326 00327 //////////////////////////////////////////////////////////////////// 00328 // Function: Multifile::open_read_write 00329 // Access: Public 00330 // Description: Opens an anonymous Multifile for reading and writing 00331 // using an iostream. There must be seek functionality 00332 // via seekg()/seekp() and tellg()/tellp() on the 00333 // iostream. 00334 // 00335 // If owns_pointer is true, then the Multifile assumes 00336 // ownership of the stream pointer and will delete it 00337 // when the multifile is closed, including if this 00338 // function returns false. 00339 //////////////////////////////////////////////////////////////////// 00340 bool Multifile:: 00341 open_read_write(iostream *multifile_stream, bool owns_pointer) { 00342 close(); 00343 _timestamp = time(NULL); 00344 _timestamp_dirty = true; 00345 00346 // We don't support locking when opening a file in read-write mode, 00347 // because we don't bother with locking on write. But we need to 00348 // have an IStreamWrapper to assign to the _read member, so we 00349 // create one on-the-fly here. 00350 _read = new StreamWrapper(multifile_stream, owns_pointer); 00351 _write = multifile_stream; 00352 _owns_stream = true; // Because we own the StreamWrapper, above. 00353 _write->seekp(0, ios::beg); 00354 00355 // Check whether the read stream is empty. 00356 multifile_stream->seekg(0, ios::end); 00357 if (multifile_stream->tellg() == (streampos)0) { 00358 // The read stream is empty, which is always valid. 00359 return true; 00360 } 00361 00362 // The read stream is not empty, so we'd better have a valid 00363 // Multifile. 00364 return read_index(); 00365 } 00366 00367 //////////////////////////////////////////////////////////////////// 00368 // Function: Multifile::close 00369 // Access: Published 00370 // Description: Closes the Multifile if it is open. All changes are 00371 // flushed to disk, and the file becomes invalid for 00372 // further operations until the next call to open(). 00373 //////////////////////////////////////////////////////////////////// 00374 void Multifile:: 00375 close() { 00376 if (_new_scale_factor != _scale_factor) { 00377 // If we have changed the scale factor recently, we need to force 00378 // a repack. 00379 repack(); 00380 } else { 00381 flush(); 00382 } 00383 00384 if (_owns_stream) { 00385 // We prefer to delete the IStreamWrapper over the ostream, if 00386 // possible. 00387 if (_read != (IStreamWrapper *)NULL) { 00388 delete _read; 00389 } else if (_write != (ostream *)NULL) { 00390 delete _write; 00391 } 00392 } 00393 00394 _read = (IStreamWrapper *)NULL; 00395 _write = (ostream *)NULL; 00396 _offset = 0; 00397 _owns_stream = false; 00398 _next_index = 0; 00399 _last_index = 0; 00400 _needs_repack = false; 00401 _timestamp = 0; 00402 _timestamp_dirty = false; 00403 _scale_factor = 1; 00404 _new_scale_factor = 1; 00405 _encryption_flag = false; 00406 _file_major_ver = 0; 00407 _file_minor_ver = 0; 00408 00409 _read_file.close(); 00410 _write_file.close(); 00411 _read_write_file.close(); 00412 _multifile_name = Filename(); 00413 00414 clear_subfiles(); 00415 } 00416 00417 //////////////////////////////////////////////////////////////////// 00418 // Function: Multifile::set_scale_factor 00419 // Access: Published 00420 // Description: Changes the internal scale factor for this Multifile. 00421 // 00422 // This is normally 1, but it may be set to any 00423 // arbitrary value (greater than zero) to support 00424 // Multifile archives that exceed 4GB, if necessary. 00425 // (Individual subfiles may still not exceed 4GB.) 00426 // 00427 // All addresses within the file are rounded up to the 00428 // next multiple of _scale_factor, and zeros are written 00429 // to the file to fill the resulting gaps. Then the 00430 // address is divided by _scale_factor and written out 00431 // as a 32-bit integer. Thus, setting a scale factor of 00432 // 2 supports up to 8GB files, 3 supports 12GB files, 00433 // etc. 00434 // 00435 // Calling this function on an already-existing 00436 // Multifile will have no immediate effect until a 00437 // future call to repack() or close() (or until the 00438 // Multifile is destructed). 00439 //////////////////////////////////////////////////////////////////// 00440 void Multifile:: 00441 set_scale_factor(size_t scale_factor) { 00442 nassertv(is_write_valid()); 00443 nassertv(scale_factor != (size_t)0); 00444 00445 if (_next_index == (streampos)0) { 00446 // If it's a brand new Multifile, we can go ahead and set it 00447 // immediately. 00448 _scale_factor = scale_factor; 00449 } else { 00450 // Otherwise, we'd better have read access so we can repack it 00451 // later. 00452 nassertv(is_read_valid()); 00453 } 00454 00455 // Setting the _new_scale_factor different from the _scale_factor 00456 // will force a repack operation on close. 00457 _new_scale_factor = scale_factor; 00458 } 00459 00460 //////////////////////////////////////////////////////////////////// 00461 // Function: Multifile::add_subfile 00462 // Access: Published 00463 // Description: Adds a file on disk as a subfile to the Multifile. 00464 // The file named by filename will be read and added to 00465 // the Multifile at the next call to flush(). If there 00466 // already exists a subfile with the indicated name, it 00467 // is replaced without examining its contents (but see 00468 // also update_subfile). 00469 // 00470 // Returns the subfile name on success (it might have 00471 // been modified slightly), or empty string on failure. 00472 //////////////////////////////////////////////////////////////////// 00473 string Multifile:: 00474 add_subfile(const string &subfile_name, const Filename &filename, 00475 int compression_level) { 00476 nassertr(is_write_valid(), string()); 00477 00478 if (!filename.exists()) { 00479 return string(); 00480 } 00481 string name = standardize_subfile_name(subfile_name); 00482 if (!name.empty()) { 00483 Subfile *subfile = new Subfile; 00484 subfile->_name = name; 00485 subfile->_source_filename = filename; 00486 subfile->_source_filename.set_binary(); 00487 00488 add_new_subfile(subfile, compression_level); 00489 } 00490 00491 _timestamp = time(NULL); 00492 _timestamp_dirty = true; 00493 00494 return name; 00495 } 00496 00497 //////////////////////////////////////////////////////////////////// 00498 // Function: Multifile::add_subfile 00499 // Access: Public 00500 // Description: Adds a file from a stream as a subfile to the Multifile. 00501 // The indicated istream will be read and its contents 00502 // added to the Multifile at the next call to flush(). 00503 // 00504 // Note that the istream must remain untouched and 00505 // unused by any other code until flush() is called. At 00506 // that time, the Multifile will read the entire 00507 // contents of the istream from the current file 00508 // position to the end of the file. Subsequently, the 00509 // Multifile will *not* close or delete the istream. It 00510 // is the caller's responsibility to ensure that the 00511 // istream pointer does not destruct during the lifetime 00512 // of the Multifile. 00513 // 00514 // Returns the subfile name on success (it might have 00515 // been modified slightly), or empty string on failure. 00516 //////////////////////////////////////////////////////////////////// 00517 string Multifile:: 00518 add_subfile(const string &subfile_name, istream *subfile_data, 00519 int compression_level) { 00520 nassertr(is_write_valid(), string()); 00521 00522 string name = standardize_subfile_name(subfile_name); 00523 if (!name.empty()) { 00524 Subfile *subfile = new Subfile; 00525 subfile->_name = name; 00526 subfile->_source = subfile_data; 00527 add_new_subfile(subfile, compression_level); 00528 } 00529 00530 return name; 00531 } 00532 00533 //////////////////////////////////////////////////////////////////// 00534 // Function: Multifile::update_subfile 00535 // Access: Published 00536 // Description: Adds a file on disk to the subfile. If a subfile 00537 // already exists with the same name, its contents are 00538 // compared byte-for-byte to the disk file, and it is 00539 // replaced only if it is different; otherwise, the 00540 // multifile is left unchanged. 00541 //////////////////////////////////////////////////////////////////// 00542 string Multifile:: 00543 update_subfile(const string &subfile_name, const Filename &filename, 00544 int compression_level) { 00545 nassertr(is_write_valid(), string()); 00546 00547 if (!filename.exists()) { 00548 return string(); 00549 } 00550 string name = standardize_subfile_name(subfile_name); 00551 if (!name.empty()) { 00552 int index = find_subfile(name); 00553 if (index >= 0) { 00554 // The subfile already exists; compare it to the source file. 00555 if (compare_subfile(index, filename)) { 00556 // The files are identical; do nothing. 00557 return name; 00558 } 00559 } 00560 00561 // The subfile does not already exist or it is different from the 00562 // source file. Add the new source file. 00563 Subfile *subfile = new Subfile; 00564 subfile->_name = name; 00565 subfile->_source_filename = filename; 00566 subfile->_source_filename.set_binary(); 00567 00568 add_new_subfile(subfile, compression_level); 00569 } 00570 00571 _timestamp = time(NULL); 00572 _timestamp_dirty = true; 00573 00574 return name; 00575 } 00576 00577 #ifdef HAVE_OPENSSL 00578 //////////////////////////////////////////////////////////////////// 00579 // Function: Multifile::add_signature 00580 // Access: Published 00581 // Description: Adds a new signature to the Multifile. This 00582 // signature associates the indicated certificate with 00583 // the current contents of the Multifile. When the 00584 // Multifile is read later, the signature will still be 00585 // present only if the Multifile is unchanged; any 00586 // subsequent changes to the Multifile will 00587 // automatically invalidate and remove the signature. 00588 // 00589 // The chain filename may be empty if the certificate 00590 // does not require an authenticating certificate chain 00591 // (e.g. because it is self-signed). 00592 // 00593 // The specified private key must match the certificate, 00594 // and the Multifile must be open in read-write mode. 00595 // The private key is only used for generating the 00596 // signature; it is not written to the Multifile and 00597 // cannot be retrieved from the Multifile later. 00598 // (However, the certificate *can* be retrieved from the 00599 // Multifile later, to identify the entity that created 00600 // the signature.) 00601 // 00602 // This implicitly causes a repack() operation if one is 00603 // needed. Returns true on success, false on failure. 00604 // 00605 // This flavor of add_signature() reads the certificate 00606 // and private key from a PEM-formatted file, for 00607 // instance as generated by the openssl command. If the 00608 // private key file is password-encrypted, the third 00609 // parameter will be used as the password to decrypt it. 00610 //////////////////////////////////////////////////////////////////// 00611 bool Multifile:: 00612 add_signature(const Filename &certificate, const Filename &chain, 00613 const Filename &pkey, const string &password) { 00614 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00615 00616 if (chain.empty() && pkey.empty()) { 00617 // If the second two filenames are empty, assume we're going for 00618 // the composite mode, where everything's stuffed into the first 00619 // file. 00620 return add_signature(certificate, password); 00621 } 00622 00623 CertChain cert_chain; 00624 00625 // Read the certificate file from VFS. First, read the complete 00626 // file into memory. 00627 string certificate_data; 00628 if (!vfs->read_file(certificate, certificate_data, true)) { 00629 express_cat.info() 00630 << "Could not read " << certificate << ".\n"; 00631 return false; 00632 } 00633 00634 // Create an in-memory BIO to read the "file" from the buffer. 00635 BIO *certificate_mbio = BIO_new_mem_buf((void *)certificate_data.data(), certificate_data.size()); 00636 X509 *x509 = PEM_read_bio_X509(certificate_mbio, NULL, NULL, (void *)""); 00637 BIO_free(certificate_mbio); 00638 if (x509 == NULL) { 00639 express_cat.info() 00640 << "Could not read certificate in " << certificate << ".\n"; 00641 return false; 00642 } 00643 00644 // Store the first X509--the actual certificate--as the first record 00645 // in our CertChain object. 00646 cert_chain.push_back(CertRecord(x509)); 00647 00648 // Read the rest of the certificates in the chain file. 00649 if (!chain.empty()) { 00650 string chain_data; 00651 if (!vfs->read_file(chain, chain_data, true)) { 00652 express_cat.info() 00653 << "Could not read " << chain << ".\n"; 00654 return false; 00655 } 00656 00657 BIO *chain_mbio = BIO_new_mem_buf((void *)chain_data.data(), chain_data.size()); 00658 X509 *c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)""); 00659 while (c != NULL) { 00660 cert_chain.push_back(c); 00661 c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)""); 00662 } 00663 BIO_free(chain_mbio); 00664 00665 if (cert_chain.size() == 1) { 00666 express_cat.info() 00667 << "Could not read certificate chain in " << chain << ".\n"; 00668 return false; 00669 } 00670 } 00671 00672 // Now do the same thing with the private key. This one may be 00673 // password-encrypted on disk. 00674 string pkey_data; 00675 if (!vfs->read_file(pkey, pkey_data, true)) { 00676 express_cat.info() 00677 << "Could not read " << pkey << ".\n"; 00678 return false; 00679 } 00680 00681 BIO *pkey_mbio = BIO_new_mem_buf((void *)pkey_data.data(), pkey_data.size()); 00682 EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, NULL, NULL, 00683 (void *)password.c_str()); 00684 BIO_free(pkey_mbio); 00685 if (evp_pkey == NULL) { 00686 express_cat.info() 00687 << "Could not read private key in " << pkey << ".\n"; 00688 return false; 00689 } 00690 00691 bool result = add_signature(cert_chain, evp_pkey); 00692 00693 EVP_PKEY_free(evp_pkey); 00694 00695 return result; 00696 } 00697 #endif // HAVE_OPENSSL 00698 00699 #ifdef HAVE_OPENSSL 00700 //////////////////////////////////////////////////////////////////// 00701 // Function: Multifile::add_signature 00702 // Access: Published 00703 // Description: Adds a new signature to the Multifile. This 00704 // signature associates the indicated certificate with 00705 // the current contents of the Multifile. When the 00706 // Multifile is read later, the signature will still be 00707 // present only if the Multifile is unchanged; any 00708 // subsequent changes to the Multifile will 00709 // automatically invalidate and remove the signature. 00710 // 00711 // This flavor of add_signature() reads the certificate, 00712 // private key, and certificate chain from the same 00713 // PEM-formatted file. It takes the first private key 00714 // found as the intended key, and then uses the first 00715 // certificate found that matches that key as the 00716 // signing certificate. Any other certificates in the 00717 // file are taken to be part of the chain. 00718 //////////////////////////////////////////////////////////////////// 00719 bool Multifile:: 00720 add_signature(const Filename &composite, const string &password) { 00721 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00722 00723 // First, read the complete file into memory. 00724 string composite_data; 00725 if (!vfs->read_file(composite, composite_data, true)) { 00726 express_cat.info() 00727 << "Could not read " << composite << ".\n"; 00728 return false; 00729 } 00730 00731 // Get the private key. 00732 BIO *pkey_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size()); 00733 EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, NULL, NULL, 00734 (void *)password.c_str()); 00735 BIO_free(pkey_mbio); 00736 if (evp_pkey == NULL) { 00737 express_cat.info() 00738 << "Could not read private key in " << composite << ".\n"; 00739 return false; 00740 } 00741 00742 // Now read all of the certificates. 00743 CertChain cert_chain; 00744 00745 BIO *chain_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size()); 00746 X509 *c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)""); 00747 while (c != NULL) { 00748 cert_chain.push_back(c); 00749 c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)""); 00750 } 00751 BIO_free(chain_mbio); 00752 00753 if (cert_chain.empty()) { 00754 express_cat.info() 00755 << "Could not read certificates in " << composite << ".\n"; 00756 return false; 00757 } 00758 00759 // Now find the certificate that matches the signature, and move it 00760 // to the front of the chain. 00761 size_t i; 00762 bool found_match = false; 00763 for (i = 0; i < cert_chain.size(); ++i) { 00764 X509 *c = cert_chain[i]._cert; 00765 if (X509_check_private_key(c, evp_pkey)) { 00766 found_match = true; 00767 if (i != 0) { 00768 // Move this entry to the beginning. 00769 cert_chain.insert(cert_chain.begin(), cert_chain[i]); 00770 cert_chain.erase(cert_chain.begin() + i + 1); 00771 } 00772 break; 00773 } 00774 } 00775 00776 if (!found_match) { 00777 express_cat.info() 00778 << "No certificates in " << composite << " match key.\n"; 00779 return false; 00780 } 00781 00782 bool result = add_signature(cert_chain, evp_pkey); 00783 00784 EVP_PKEY_free(evp_pkey); 00785 00786 return result; 00787 } 00788 #endif // HAVE_OPENSSL 00789 00790 #ifdef HAVE_OPENSSL 00791 //////////////////////////////////////////////////////////////////// 00792 // Function: Multifile::add_signature 00793 // Access: Published 00794 // Description: Adds a new signature to the Multifile. This 00795 // signature associates the indicated certificate with 00796 // the current contents of the Multifile. When the 00797 // Multifile is read later, the signature will still be 00798 // present only if the Multifile is unchanged; any 00799 // subsequent changes to the Multifile will 00800 // automatically invalidate and remove the signature. 00801 // 00802 // If chain is non-NULL, it represents the certificate 00803 // chain that validates the certificate. 00804 // 00805 // The specified private key must match the certificate, 00806 // and the Multifile must be open in read-write mode. 00807 // The private key is only used for generating the 00808 // signature; it is not written to the Multifile and 00809 // cannot be retrieved from the Multifile later. 00810 // (However, the certificate *can* be retrieved from the 00811 // Multifile later, to identify the entity that created 00812 // the signature.) 00813 // 00814 // This implicitly causes a repack() operation if one is 00815 // needed. Returns true on success, false on failure. 00816 //////////////////////////////////////////////////////////////////// 00817 bool Multifile:: 00818 add_signature(X509 *certificate, STACK_OF(X509) *chain, EVP_PKEY *pkey) { 00819 // Convert the certificate and chain into our own CertChain 00820 // structure. 00821 CertChain cert_chain; 00822 cert_chain.push_back(CertRecord(certificate)); 00823 if (chain != NULL) { 00824 int num = sk_X509_num(chain); 00825 for (int i = 0; i < num; ++i) { 00826 cert_chain.push_back(CertRecord((X509 *)sk_X509_value(chain, i))); 00827 } 00828 } 00829 00830 return add_signature(cert_chain, pkey); 00831 } 00832 #endif // HAVE_OPENSSL 00833 00834 #ifdef HAVE_OPENSSL 00835 //////////////////////////////////////////////////////////////////// 00836 // Function: Multifile::add_signature 00837 // Access: Published 00838 // Description: Adds a new signature to the Multifile. This 00839 // signature associates the indicated certificate with 00840 // the current contents of the Multifile. When the 00841 // Multifile is read later, the signature will still be 00842 // present only if the Multifile is unchanged; any 00843 // subsequent changes to the Multifile will 00844 // automatically invalidate and remove the signature. 00845 // 00846 // The signature certificate is the first certificate on 00847 // the CertChain object. Any remaining certificates are 00848 // support certificates to authenticate the first one. 00849 // 00850 // The specified private key must match the certificate, 00851 // and the Multifile must be open in read-write mode. 00852 // The private key is only used for generating the 00853 // signature; it is not written to the Multifile and 00854 // cannot be retrieved from the Multifile later. 00855 // (However, the certificate *can* be retrieved from the 00856 // Multifile later, to identify the entity that created 00857 // the signature.) 00858 // 00859 // This implicitly causes a repack() operation if one is 00860 // needed. Returns true on success, false on failure. 00861 //////////////////////////////////////////////////////////////////// 00862 bool Multifile:: 00863 add_signature(const Multifile::CertChain &cert_chain, EVP_PKEY *pkey) { 00864 if (_needs_repack) { 00865 if (!repack()) { 00866 return false; 00867 } 00868 } else { 00869 if (!flush()) { 00870 return false; 00871 } 00872 } 00873 00874 if (cert_chain.empty()) { 00875 express_cat.info() 00876 << "No certificate given.\n"; 00877 return false; 00878 } 00879 00880 if (pkey == NULL) { 00881 express_cat.info() 00882 << "No private key given.\n"; 00883 return false; 00884 } 00885 00886 if (!X509_check_private_key(cert_chain[0]._cert, pkey)) { 00887 express_cat.info() 00888 << "Private key does not match certificate.\n"; 00889 return false; 00890 } 00891 00892 // Now encode that list of certs to a stream in DER form. 00893 stringstream der_stream; 00894 StreamWriter der_writer(der_stream); 00895 der_writer.add_uint32(cert_chain.size()); 00896 00897 CertChain::const_iterator ci; 00898 for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) { 00899 X509 *cert = (*ci)._cert; 00900 00901 int der_len = i2d_X509(cert, NULL); 00902 unsigned char *der_buf = new unsigned char[der_len]; 00903 unsigned char *p = der_buf; 00904 i2d_X509(cert, &p); 00905 der_writer.append_data(der_buf, der_len); 00906 delete[] der_buf; 00907 } 00908 00909 // Create a temporary Subfile for writing out the signature. 00910 der_stream.seekg(0); 00911 Subfile *subfile = new Subfile; 00912 subfile->_pkey = pkey; 00913 subfile->_flags |= SF_signature; 00914 subfile->_source = &der_stream; 00915 00916 // Write the new Subfile at the end. The cert_special subfiles 00917 // always go at the end, because they're not the part of the file 00918 // that's signed. 00919 nassertr(_new_subfiles.empty(), false); 00920 _new_subfiles.push_back(subfile); 00921 bool result = flush(); 00922 00923 delete subfile; 00924 00925 return result; 00926 } 00927 #endif // HAVE_OPENSSL 00928 00929 #ifdef HAVE_OPENSSL 00930 //////////////////////////////////////////////////////////////////// 00931 // Function: Multifile::get_num_signatures 00932 // Access: Published 00933 // Description: Returns the number of matching signatures found on 00934 // the Multifile. These signatures may be iterated via 00935 // get_signature() and related methods. 00936 // 00937 // A signature on this list is guaranteed to match the 00938 // Multifile contents, proving that the Multifile has 00939 // been unmodified since the signature was applied. 00940 // However, this does not guarantee that the certificate 00941 // itself is actually from who it says it is from; only 00942 // that it matches the Multifile contents. See 00943 // validate_signature_certificate() to authenticate a 00944 // particular certificate. 00945 //////////////////////////////////////////////////////////////////// 00946 int Multifile:: 00947 get_num_signatures() const { 00948 ((Multifile *)this)->check_signatures(); 00949 return _signatures.size(); 00950 } 00951 #endif // HAVE_OPENSSL 00952 00953 #ifdef HAVE_OPENSSL 00954 //////////////////////////////////////////////////////////////////// 00955 // Function: Multifile::get_signature 00956 // Access: Published 00957 // Description: Returns the nth signature found on the Multifile. 00958 // See the comments in get_num_signatures(). 00959 //////////////////////////////////////////////////////////////////// 00960 const Multifile::CertChain &Multifile:: 00961 get_signature(int n) const { 00962 static CertChain error_chain; 00963 nassertr(n >= 0 && n < (int)_signatures.size(), error_chain); 00964 return _signatures[n]; 00965 } 00966 #endif // HAVE_OPENSSL 00967 00968 #ifdef HAVE_OPENSSL 00969 //////////////////////////////////////////////////////////////////// 00970 // Function: Multifile::get_signature_subject_name 00971 // Access: Published 00972 // Description: Returns the "subject name" for the nth signature found 00973 // on the Multifile. This is a string formatted 00974 // according to RFC2253 that should more-or-less 00975 // identify a particular certificate; when paired with 00976 // the public key (see get_signature_public_key()), it 00977 // can uniquely identify a certificate. See the 00978 // comments in get_num_signatures(). 00979 //////////////////////////////////////////////////////////////////// 00980 string Multifile:: 00981 get_signature_subject_name(int n) const { 00982 const CertChain &cert_chain = get_signature(n); 00983 nassertr(!cert_chain.empty(), string()); 00984 00985 X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert); 00986 if (xname != NULL) { 00987 // We use "print" to dump the output to a memory BIO. Is 00988 // there an easier way to extract the X509_NAME text? Curse 00989 // these incomplete docs. 00990 BIO *mbio = BIO_new(BIO_s_mem()); 00991 X509_NAME_print_ex(mbio, xname, 0, XN_FLAG_RFC2253); 00992 00993 char *pp; 00994 long pp_size = BIO_get_mem_data(mbio, &pp); 00995 string name(pp, pp_size); 00996 BIO_free(mbio); 00997 return name; 00998 } 00999 01000 return string(); 01001 } 01002 #endif // HAVE_OPENSSL 01003 01004 #ifdef HAVE_OPENSSL 01005 //////////////////////////////////////////////////////////////////// 01006 // Function: Multifile::get_signature_friendly_name 01007 // Access: Published 01008 // Description: Returns a "friendly name" for the nth signature found 01009 // on the Multifile. This attempts to extract out the 01010 // most meaningful part of the subject name. It returns 01011 // the emailAddress, if it is defined; otherwise, it 01012 // returns the commonName. 01013 01014 // See the comments in get_num_signatures(). 01015 //////////////////////////////////////////////////////////////////// 01016 string Multifile:: 01017 get_signature_friendly_name(int n) const { 01018 const CertChain &cert_chain = get_signature(n); 01019 nassertr(!cert_chain.empty(), string()); 01020 01021 static const int nid_choices[] = { 01022 NID_pkcs9_emailAddress, 01023 NID_commonName, 01024 -1, 01025 }; 01026 01027 // Choose the first NID that exists on the cert. 01028 for (int ni = 0; nid_choices[ni] != -1; ++ni) { 01029 int nid = nid_choices[ni]; 01030 01031 // A complex OpenSSL interface to extract out the name in utf-8. 01032 X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert); 01033 if (xname != NULL) { 01034 int pos = X509_NAME_get_index_by_NID(xname, nid, -1); 01035 if (pos != -1) { 01036 // We just get the first common name. I guess it's possible to 01037 // have more than one; not sure what that means in this context. 01038 X509_NAME_ENTRY *xentry = X509_NAME_get_entry(xname, pos); 01039 if (xentry != NULL) { 01040 ASN1_STRING *data = X509_NAME_ENTRY_get_data(xentry); 01041 if (data != NULL) { 01042 // We use "print" to dump the output to a memory BIO. Is 01043 // there an easier way to decode the ASN1_STRING? Curse 01044 // these incomplete docs. 01045 BIO *mbio = BIO_new(BIO_s_mem()); 01046 ASN1_STRING_print_ex(mbio, data, ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB); 01047 01048 char *pp; 01049 long pp_size = BIO_get_mem_data(mbio, &pp); 01050 string name(pp, pp_size); 01051 BIO_free(mbio); 01052 return name; 01053 } 01054 } 01055 } 01056 } 01057 } 01058 01059 return string(); 01060 } 01061 #endif // HAVE_OPENSSL 01062 01063 #ifdef HAVE_OPENSSL 01064 //////////////////////////////////////////////////////////////////// 01065 // Function: Multifile::get_signature_public_key 01066 // Access: Published 01067 // Description: Returns the public key used for the nth signature 01068 // found on the Multifile. This is encoded in DER form 01069 // and returned as a string of hex digits. 01070 // 01071 // This can be used, in conjunction with the subject 01072 // name (see get_signature_subject_name()), to uniquely 01073 // identify a particular certificate and its subsequent 01074 // reissues. See the comments in get_num_signatures(). 01075 //////////////////////////////////////////////////////////////////// 01076 string Multifile:: 01077 get_signature_public_key(int n) const { 01078 const CertChain &cert_chain = get_signature(n); 01079 nassertr(!cert_chain.empty(), string()); 01080 01081 EVP_PKEY *pkey = X509_get_pubkey(cert_chain[0]._cert); 01082 if (pkey != NULL) { 01083 int key_len = i2d_PublicKey(pkey, NULL); 01084 unsigned char *key_buf = new unsigned char[key_len]; 01085 unsigned char *p = key_buf; 01086 i2d_PublicKey(pkey, &p); 01087 string result; 01088 for (int i = 0; i < key_len; ++i) { 01089 result += tohex(key_buf[i] >> 4); 01090 result += tohex(key_buf[i]); 01091 } 01092 delete[] key_buf; 01093 return result; 01094 } 01095 01096 return string(); 01097 } 01098 #endif // HAVE_OPENSSL 01099 01100 #ifdef HAVE_OPENSSL 01101 //////////////////////////////////////////////////////////////////// 01102 // Function: Multifile::write_signature_certificate 01103 // Access: Published 01104 // Description: Writes the certificate for the nth signature, in 01105 // verbose form, to the indicated stream. See the 01106 // comments in get_num_signatures(). 01107 //////////////////////////////////////////////////////////////////// 01108 void Multifile:: 01109 write_signature_certificate(int n, ostream &out) const { 01110 const CertChain &cert_chain = get_signature(n); 01111 nassertv(!cert_chain.empty()); 01112 01113 BIO *mbio = BIO_new(BIO_s_mem()); 01114 X509_print(mbio, cert_chain[0]._cert); 01115 01116 char *pp; 01117 long pp_size = BIO_get_mem_data(mbio, &pp); 01118 out.write(pp, pp_size); 01119 BIO_free(mbio); 01120 } 01121 #endif // HAVE_OPENSSL 01122 01123 #ifdef HAVE_OPENSSL 01124 //////////////////////////////////////////////////////////////////// 01125 // Function: Multifile::validate_signature_certificate 01126 // Access: Published 01127 // Description: Checks that the certificate used for the nth 01128 // signature is a valid, authorized certificate with 01129 // some known certificate authority. Returns 0 if it 01130 // is valid, -1 if there is some error, or the 01131 // corresponding OpenSSL error code if it is invalid, 01132 // out-of-date, or self-signed. 01133 //////////////////////////////////////////////////////////////////// 01134 int Multifile:: 01135 validate_signature_certificate(int n) const { 01136 int verify_result = -1; 01137 01138 const CertChain &chain = get_signature(n); 01139 nassertr(!chain.empty(), false); 01140 01141 OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr(); 01142 01143 // Copy our CertChain structure into an X509 pointer and 01144 // accompanying STACK_OF(X509) pointer. 01145 X509 *x509 = chain[0]._cert; 01146 STACK_OF(X509) *stack = NULL; 01147 if (chain.size() > 1) { 01148 stack = sk_X509_new(NULL); 01149 for (size_t n = 1; n < chain.size(); ++n) { 01150 sk_X509_push(stack, chain[n]._cert); 01151 } 01152 } 01153 01154 // Create the X509_STORE_CTX for verifying the cert and chain. 01155 X509_STORE_CTX *ctx = X509_STORE_CTX_new(); 01156 X509_STORE_CTX_init(ctx, sslw->get_x509_store(), x509, stack); 01157 X509_STORE_CTX_set_cert(ctx, x509); 01158 01159 if (X509_verify_cert(ctx)) { 01160 verify_result = 0; 01161 } else { 01162 verify_result = X509_STORE_CTX_get_error(ctx); 01163 } 01164 01165 if (express_cat.is_debug()) { 01166 express_cat.debug() 01167 << get_signature_subject_name(n) << ": validate " << verify_result 01168 << "\n"; 01169 } 01170 01171 sk_X509_free(stack); 01172 X509_STORE_CTX_cleanup(ctx); 01173 X509_STORE_CTX_free(ctx); 01174 01175 return verify_result; 01176 } 01177 #endif // HAVE_OPENSSL 01178 01179 //////////////////////////////////////////////////////////////////// 01180 // Function: Multifile::flush 01181 // Access: Published 01182 // Description: Writes all contents of the Multifile to disk. Until 01183 // flush() is called, add_subfile() and remove_subfile() 01184 // do not actually do anything to disk. At this point, 01185 // all of the recently-added subfiles are read and their 01186 // contents are added to the end of the Multifile, and 01187 // the recently-removed subfiles are marked gone from 01188 // the Multifile. 01189 // 01190 // This may result in a suboptimal index. To guarantee 01191 // that the index is written at the beginning of the 01192 // file, call repack() instead of flush(). 01193 // 01194 // It is not necessary to call flush() explicitly unless 01195 // you are concerned about reading the recently-added 01196 // subfiles immediately. 01197 // 01198 // Returns true on success, false on failure. 01199 //////////////////////////////////////////////////////////////////// 01200 bool Multifile:: 01201 flush() { 01202 if (!is_write_valid()) { 01203 return false; 01204 } 01205 01206 bool new_file = (_next_index == (streampos)0); 01207 if (new_file) { 01208 // If we don't have an index yet, we don't have a header. Write 01209 // the header. 01210 if (!write_header()) { 01211 return false; 01212 } 01213 01214 } else { 01215 if (_file_minor_ver != _current_minor_ver) { 01216 // If we *do* have an index already, but this is an old version 01217 // multifile, we have to completely rewrite it anyway. 01218 return repack(); 01219 } 01220 } 01221 01222 nassertr(_write != (ostream *)NULL, false); 01223 01224 // First, mark out all of the removed subfiles. 01225 PendingSubfiles::iterator pi; 01226 for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) { 01227 Subfile *subfile = (*pi); 01228 subfile->rewrite_index_flags(*_write); 01229 delete subfile; 01230 } 01231 _removed_subfiles.clear(); 01232 01233 bool wrote_ok = true; 01234 01235 if (!_new_subfiles.empty() || new_file) { 01236 // Add a few more files to the end. We always add subfiles at the 01237 // end of the multifile, so go there first. 01238 sort(_new_subfiles.begin(), _new_subfiles.end(), IndirectLess<Subfile>()); 01239 if (_last_index != (streampos)0) { 01240 _write->seekp(0, ios::end); 01241 if (_write->fail()) { 01242 express_cat.info() 01243 << "Unable to seek Multifile " << _multifile_name << ".\n"; 01244 return false; 01245 } 01246 _next_index = _write->tellp(); 01247 _next_index = pad_to_streampos(_next_index); 01248 01249 // And update the forward link from the last_index to point to 01250 // this new index location. 01251 _write->seekp(_last_index); 01252 StreamWriter writer(_write, false); 01253 writer.add_uint32(streampos_to_word(_next_index)); 01254 } 01255 01256 _write->seekp(_next_index); 01257 nassertr(_next_index == _write->tellp(), false); 01258 01259 // Ok, here we are at the end of the file. Write out the 01260 // recently-added subfiles here. First, count up the index size. 01261 for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) { 01262 Subfile *subfile = (*pi); 01263 _last_index = _next_index; 01264 _next_index = subfile->write_index(*_write, _next_index, this); 01265 nassertr(_next_index == _write->tellp(), false); 01266 _next_index = pad_to_streampos(_next_index); 01267 nassertr(_next_index == _write->tellp(), false); 01268 } 01269 01270 // Now we're at the end of the index. Write a 0 here to mark the 01271 // end. 01272 StreamWriter writer(_write, false); 01273 writer.add_uint32(0); 01274 _next_index += 4; 01275 nassertr(_next_index == _write->tellp(), false); 01276 _next_index = pad_to_streampos(_next_index); 01277 01278 // All right, now write out each subfile's data. 01279 for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) { 01280 Subfile *subfile = (*pi); 01281 01282 if (_read != (IStreamWrapper *)NULL) { 01283 _read->acquire(); 01284 _next_index = subfile->write_data(*_write, _read->get_istream(), 01285 _next_index, this); 01286 _read->release(); 01287 01288 } else { 01289 _next_index = subfile->write_data(*_write, NULL, _next_index, this); 01290 } 01291 01292 nassertr(_next_index == _write->tellp(), false); 01293 _next_index = pad_to_streampos(_next_index); 01294 if (subfile->is_data_invalid()) { 01295 wrote_ok = false; 01296 } 01297 01298 if (!subfile->is_cert_special()) { 01299 _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos()); 01300 } 01301 nassertr(_next_index == _write->tellp(), false); 01302 } 01303 01304 // Now go back and fill in the proper addresses for the data start. 01305 // We didn't do it in the first pass, because we don't really want 01306 // to keep all those file handles open, and so we didn't have to 01307 // determine each file's length ahead of time. 01308 for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) { 01309 Subfile *subfile = (*pi); 01310 subfile->rewrite_index_data_start(*_write, this); 01311 } 01312 01313 _new_subfiles.clear(); 01314 } 01315 01316 // Also update the overall timestamp. 01317 if (_timestamp_dirty) { 01318 nassertr(!_write->fail(), false); 01319 static const size_t timestamp_pos = _header_prefix.size() + _header_size + 2 + 2 + 4; 01320 _write->seekp(timestamp_pos); 01321 nassertr(!_write->fail(), false); 01322 01323 StreamWriter writer(*_write); 01324 if (_record_timestamp) { 01325 writer.add_uint32(_timestamp); 01326 } else { 01327 writer.add_uint32(0); 01328 } 01329 _timestamp_dirty = false; 01330 } 01331 01332 _write->flush(); 01333 if (!wrote_ok || _write->fail()) { 01334 express_cat.info() 01335 << "Unable to update Multifile " << _multifile_name << ".\n"; 01336 close(); 01337 return false; 01338 } 01339 01340 return true; 01341 } 01342 01343 //////////////////////////////////////////////////////////////////// 01344 // Function: Multifile::repack 01345 // Access: Published 01346 // Description: Forces a complete rewrite of the Multifile and all of 01347 // its contents, so that its index will appear at the 01348 // beginning of the file with all of the subfiles listed 01349 // in alphabetical order. This is considered optimal 01350 // for reading, and is the standard configuration; but 01351 // it is not essential to do this. 01352 // 01353 // It is only valid to call this if the Multifile was 01354 // opened using open_read_write() and an explicit 01355 // filename, rather than an iostream. Also, we must 01356 // have write permission to the directory containing the 01357 // Multifile. 01358 // 01359 // Returns true on success, false on failure. 01360 //////////////////////////////////////////////////////////////////// 01361 bool Multifile:: 01362 repack() { 01363 if (_next_index == (streampos)0) { 01364 // If the Multifile hasn't yet been written, this is really just a 01365 // flush operation. 01366 _needs_repack = false; 01367 return flush(); 01368 } 01369 01370 nassertr(is_write_valid() && is_read_valid(), false); 01371 nassertr(!_multifile_name.empty(), false); 01372 01373 // First, we open a temporary filename to copy the Multifile to. 01374 Filename dirname = _multifile_name.get_dirname(); 01375 if (dirname.empty()) { 01376 dirname = "."; 01377 } 01378 Filename temp_filename = Filename::temporary(dirname, "mftemp"); 01379 temp_filename.set_binary(); 01380 pofstream temp; 01381 if (!temp_filename.open_write(temp)) { 01382 express_cat.info() 01383 << "Unable to open temporary file " << temp_filename << "\n"; 01384 return false; 01385 } 01386 01387 // Now we scrub our internal structures so it looks like we're a 01388 // brand new Multifile. 01389 PendingSubfiles::iterator pi; 01390 for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) { 01391 Subfile *subfile = (*pi); 01392 delete subfile; 01393 } 01394 _removed_subfiles.clear(); 01395 _new_subfiles.clear(); 01396 copy(_subfiles.begin(), _subfiles.end(), back_inserter(_new_subfiles)); 01397 _next_index = 0; 01398 _last_index = 0; 01399 _last_data_byte = 0; 01400 _scale_factor = _new_scale_factor; 01401 01402 // And we write our contents to our new temporary file. 01403 _write = &temp; 01404 if (!flush()) { 01405 temp.close(); 01406 temp_filename.unlink(); 01407 return false; 01408 } 01409 01410 // Now close everything, and move the temporary file back over our 01411 // original file. 01412 Filename orig_name = _multifile_name; 01413 temp.close(); 01414 close(); 01415 orig_name.unlink(); 01416 if (!temp_filename.rename_to(orig_name)) { 01417 express_cat.info() 01418 << "Unable to rename temporary file " << temp_filename << " to " 01419 << orig_name << ".\n"; 01420 return false; 01421 } 01422 01423 if (!open_read_write(orig_name)) { 01424 express_cat.info() 01425 << "Unable to read newly repacked " << _multifile_name 01426 << ".\n"; 01427 return false; 01428 } 01429 01430 return true; 01431 } 01432 01433 //////////////////////////////////////////////////////////////////// 01434 // Function: Multifile::get_num_subfiles 01435 // Access: Published 01436 // Description: Returns the number of subfiles within the Multifile. 01437 // The subfiles may be accessed in alphabetical order by 01438 // iterating through [0 .. get_num_subfiles()). 01439 //////////////////////////////////////////////////////////////////// 01440 int Multifile:: 01441 get_num_subfiles() const { 01442 return _subfiles.size(); 01443 } 01444 01445 //////////////////////////////////////////////////////////////////// 01446 // Function: Multifile::find_subfile 01447 // Access: Published 01448 // Description: Returns the index of the subfile with the indicated 01449 // name, or -1 if the named subfile is not within the 01450 // Multifile. 01451 //////////////////////////////////////////////////////////////////// 01452 int Multifile:: 01453 find_subfile(const string &subfile_name) const { 01454 Subfile find_subfile; 01455 find_subfile._name = standardize_subfile_name(subfile_name); 01456 Subfiles::const_iterator fi; 01457 fi = _subfiles.find(&find_subfile); 01458 if (fi == _subfiles.end()) { 01459 // Not present. 01460 return -1; 01461 } 01462 return (fi - _subfiles.begin()); 01463 } 01464 01465 //////////////////////////////////////////////////////////////////// 01466 // Function: Multifile::has_directory 01467 // Access: Published 01468 // Description: Returns true if the indicated subfile name is the 01469 // directory prefix to one or more files within the 01470 // Multifile. That is, the Multifile contains at least 01471 // one file named "subfile_name/...". 01472 //////////////////////////////////////////////////////////////////// 01473 bool Multifile:: 01474 has_directory(const string &subfile_name) const { 01475 string prefix = subfile_name; 01476 if (!prefix.empty()) { 01477 prefix += '/'; 01478 } 01479 Subfile find_subfile; 01480 find_subfile._name = prefix; 01481 Subfiles::const_iterator fi; 01482 fi = _subfiles.upper_bound(&find_subfile); 01483 if (fi == _subfiles.end()) { 01484 // Not present. 01485 return false; 01486 } 01487 01488 // At least one subfile exists whose name sorts after prefix. If it 01489 // contains prefix as the initial substring, then we have a match. 01490 Subfile *subfile = (*fi); 01491 return (subfile->_name.length() > prefix.length() && 01492 subfile->_name.substr(0, prefix.length()) == prefix); 01493 } 01494 01495 //////////////////////////////////////////////////////////////////// 01496 // Function: Multifile::scan_directory 01497 // Access: Published 01498 // Description: Considers subfile_name to be the name of a 01499 // subdirectory within the Multifile, but not a file 01500 // itself; fills the given vector up with the sorted list 01501 // of subdirectories or files within the named 01502 // directory. 01503 // 01504 // Note that directories do not exist explicitly within 01505 // a Multifile; this just checks for the existence of 01506 // files with the given initial prefix. 01507 // 01508 // Returns true if successful, false otherwise. 01509 //////////////////////////////////////////////////////////////////// 01510 bool Multifile:: 01511 scan_directory(vector_string &contents, const string &subfile_name) const { 01512 string prefix = subfile_name; 01513 if (!prefix.empty()) { 01514 prefix += '/'; 01515 } 01516 Subfile find_subfile; 01517 find_subfile._name = prefix; 01518 Subfiles::const_iterator fi; 01519 fi = _subfiles.upper_bound(&find_subfile); 01520 01521 string previous = ""; 01522 while (fi != _subfiles.end()) { 01523 Subfile *subfile = (*fi); 01524 if (!(subfile->_name.length() > prefix.length() && 01525 subfile->_name.substr(0, prefix.length()) == prefix)) { 01526 // We've reached the end of the list of subfiles beneath the 01527 // indicated directory prefix. 01528 return true; 01529 } 01530 01531 size_t slash = subfile->_name.find('/', prefix.length()); 01532 string basename = subfile->_name.substr(prefix.length(), slash - prefix.length()); 01533 if (basename != previous) { 01534 contents.push_back(basename); 01535 previous = basename; 01536 } 01537 ++fi; 01538 } 01539 01540 return true; 01541 } 01542 01543 //////////////////////////////////////////////////////////////////// 01544 // Function: Multifile::remove_subfile 01545 // Access: Published 01546 // Description: Removes the nth subfile from the Multifile. This 01547 // will cause all subsequent index numbers to decrease 01548 // by one. The file will not actually be removed from 01549 // the disk until the next call to flush(). 01550 // 01551 // Note that this does not actually remove the data from 01552 // the indicated subfile; it simply removes it from the 01553 // index. The Multifile will not be reduced in size 01554 // after this operation, until the next call to 01555 // repack(). 01556 //////////////////////////////////////////////////////////////////// 01557 void Multifile:: 01558 remove_subfile(int index) { 01559 nassertv(is_write_valid()); 01560 nassertv(index >= 0 && index < (int)_subfiles.size()); 01561 Subfile *subfile = _subfiles[index]; 01562 subfile->_flags |= SF_deleted; 01563 _removed_subfiles.push_back(subfile); 01564 _subfiles.erase(_subfiles.begin() + index); 01565 01566 _timestamp = time(NULL); 01567 _timestamp_dirty = true; 01568 01569 _needs_repack = true; 01570 } 01571 01572 //////////////////////////////////////////////////////////////////// 01573 // Function: Multifile::get_subfile_name 01574 // Access: Published 01575 // Description: Returns the name of the nth subfile. 01576 //////////////////////////////////////////////////////////////////// 01577 const string &Multifile:: 01578 get_subfile_name(int index) const { 01579 #ifndef NDEBUG 01580 static string empty_string; 01581 nassertr(index >= 0 && index < (int)_subfiles.size(), empty_string); 01582 #endif 01583 return _subfiles[index]->_name; 01584 } 01585 01586 //////////////////////////////////////////////////////////////////// 01587 // Function: Multifile::get_subfile_length 01588 // Access: Published 01589 // Description: Returns the uncompressed data length of the nth 01590 // subfile. This might return 0 if the subfile has 01591 // recently been added and flush() has not yet been 01592 // called. 01593 //////////////////////////////////////////////////////////////////// 01594 size_t Multifile:: 01595 get_subfile_length(int index) const { 01596 nassertr(index >= 0 && index < (int)_subfiles.size(), 0); 01597 return _subfiles[index]->_uncompressed_length; 01598 } 01599 01600 //////////////////////////////////////////////////////////////////// 01601 // Function: Multifile::get_subfile_timestamp 01602 // Access: Published 01603 // Description: Returns the modification time of the nth 01604 // subfile. If this is called on an older .mf file, 01605 // which did not store individual timestamps in the file 01606 // (or if get_record_timestamp() is false), this will 01607 // return the modification time of the overall 01608 // multifile. 01609 //////////////////////////////////////////////////////////////////// 01610 time_t Multifile:: 01611 get_subfile_timestamp(int index) const { 01612 nassertr(index >= 0 && index < (int)_subfiles.size(), 0); 01613 if (!get_record_timestamp()) { 01614 return get_timestamp(); 01615 } else { 01616 return _subfiles[index]->_timestamp; 01617 } 01618 } 01619 01620 //////////////////////////////////////////////////////////////////// 01621 // Function: Multifile::is_subfile_compressed 01622 // Access: Published 01623 // Description: Returns true if the indicated subfile has been 01624 // compressed when stored within the archive, false 01625 // otherwise. 01626 //////////////////////////////////////////////////////////////////// 01627 bool Multifile:: 01628 is_subfile_compressed(int index) const { 01629 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01630 return (_subfiles[index]->_flags & SF_compressed) != 0; 01631 } 01632 01633 //////////////////////////////////////////////////////////////////// 01634 // Function: Multifile::is_subfile_encrypted 01635 // Access: Published 01636 // Description: Returns true if the indicated subfile has been 01637 // encrypted when stored within the archive, false 01638 // otherwise. 01639 //////////////////////////////////////////////////////////////////// 01640 bool Multifile:: 01641 is_subfile_encrypted(int index) const { 01642 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01643 return (_subfiles[index]->_flags & SF_encrypted) != 0; 01644 } 01645 01646 //////////////////////////////////////////////////////////////////// 01647 // Function: Multifile::get_index_end 01648 // Access: Published 01649 // Description: Returns the first byte that is guaranteed to follow 01650 // any index byte already written to disk in the 01651 // Multifile. 01652 // 01653 // This number is largely meaningless in many cases, but 01654 // if needs_repack() is false, and the file is flushed, 01655 // this will indicate the number of bytes in the header 01656 // + index. Everything at this byte position and later 01657 // will be actual data. 01658 //////////////////////////////////////////////////////////////////// 01659 streampos Multifile:: 01660 get_index_end() const { 01661 return normalize_streampos(_next_index + (streampos)4); 01662 } 01663 01664 //////////////////////////////////////////////////////////////////// 01665 // Function: Multifile::get_subfile_internal_start 01666 // Access: Published 01667 // Description: Returns the starting byte position within the 01668 // Multifile at which the indicated subfile begins. 01669 // This may be used, with get_subfile_internal_length(), 01670 // for low-level access to the subfile, but usually it 01671 // is better to use open_read_subfile() instead (which 01672 // automatically decrypts and/or uncompresses the 01673 // subfile data). 01674 //////////////////////////////////////////////////////////////////// 01675 streampos Multifile:: 01676 get_subfile_internal_start(int index) const { 01677 nassertr(index >= 0 && index < (int)_subfiles.size(), 0); 01678 return _subfiles[index]->_data_start; 01679 } 01680 01681 //////////////////////////////////////////////////////////////////// 01682 // Function: Multifile::get_subfile_internal_length 01683 // Access: Published 01684 // Description: Returns the number of bytes the indicated subfile 01685 // consumes within the archive. For compressed 01686 // subfiles, this will generally be smaller than 01687 // get_subfile_length(); for encrypted (but 01688 // noncompressed) subfiles, it may be slightly 01689 // different, for noncompressed and nonencrypted 01690 // subfiles, it will be equal. 01691 //////////////////////////////////////////////////////////////////// 01692 size_t Multifile:: 01693 get_subfile_internal_length(int index) const { 01694 nassertr(index >= 0 && index < (int)_subfiles.size(), 0); 01695 return _subfiles[index]->_data_length; 01696 } 01697 01698 //////////////////////////////////////////////////////////////////// 01699 // Function: Multifile::open_read_subfile 01700 // Access: Published 01701 // Description: Returns an istream that may be used to read the 01702 // indicated subfile. You may seek() within this 01703 // istream to your heart's content; even though it will 01704 // be a reference to the already-opened pfstream of the 01705 // Multifile itself, byte 0 appears to be the beginning 01706 // of the subfile and EOF appears to be the end of the 01707 // subfile. 01708 // 01709 // The returned istream will have been allocated via 01710 // new; you should pass the pointer to 01711 // close_read_subfile() when you are finished with it to 01712 // delete it and release its resources. 01713 // 01714 // Any future calls to repack() or close() (or the 01715 // Multifile destructor) will invalidate all currently 01716 // open subfile pointers. 01717 // 01718 // The return value will be NULL if the stream cannot be 01719 // opened for some reason. 01720 //////////////////////////////////////////////////////////////////// 01721 istream *Multifile:: 01722 open_read_subfile(int index) { 01723 nassertr(is_read_valid(), NULL); 01724 nassertr(index >= 0 && index < (int)_subfiles.size(), NULL); 01725 Subfile *subfile = _subfiles[index]; 01726 01727 if (subfile->_source != (istream *)NULL || 01728 !subfile->_source_filename.empty()) { 01729 // The subfile has not yet been copied into the physical 01730 // Multifile. Force a flush operation to incorporate it. 01731 flush(); 01732 01733 // That shouldn't change the subfile index or delete the subfile 01734 // pointer. 01735 nassertr(subfile == _subfiles[index], NULL); 01736 } 01737 01738 return open_read_subfile(subfile); 01739 } 01740 01741 //////////////////////////////////////////////////////////////////// 01742 // Function: Multifile::close_read_subfile 01743 // Access: Published, Static 01744 // Description: Closes a file opened by a previous call to 01745 // open_read_subfile(). This really just deletes the 01746 // istream pointer, but it is recommended to use this 01747 // interface instead of deleting it explicitly, to help 01748 // work around compiler issues. 01749 //////////////////////////////////////////////////////////////////// 01750 void Multifile:: 01751 close_read_subfile(istream *stream) { 01752 if (stream != (istream *)NULL) { 01753 // For some reason--compiler bug in gcc 3.2?--explicitly deleting 01754 // the stream pointer does not call the appropriate global delete 01755 // function; instead apparently calling the system delete 01756 // function. So we call the delete function by hand instead. 01757 #if !defined(WIN32_VC) && !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW) 01758 stream->~istream(); 01759 (*global_operator_delete)(stream); 01760 #else 01761 delete stream; 01762 #endif 01763 } 01764 } 01765 01766 //////////////////////////////////////////////////////////////////// 01767 // Function: Multifile::extract_subfile 01768 // Access: Published 01769 // Description: Extracts the nth subfile into a file with the given 01770 // name. 01771 //////////////////////////////////////////////////////////////////// 01772 bool Multifile:: 01773 extract_subfile(int index, const Filename &filename) { 01774 nassertr(is_read_valid(), false); 01775 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01776 01777 Filename fname = filename; 01778 fname.set_binary(); 01779 fname.make_dir(); 01780 pofstream out; 01781 if (!fname.open_write(out, true)) { 01782 express_cat.info() 01783 << "Unable to write to file " << filename << "\n"; 01784 return false; 01785 } 01786 01787 return extract_subfile_to(index, out); 01788 } 01789 01790 //////////////////////////////////////////////////////////////////// 01791 // Function: Multifile::extract_subfile_to 01792 // Access: Public 01793 // Description: Extracts the nth subfile to the indicated ostream. 01794 //////////////////////////////////////////////////////////////////// 01795 bool Multifile:: 01796 extract_subfile_to(int index, ostream &out) { 01797 nassertr(is_read_valid(), false); 01798 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01799 01800 istream *in = open_read_subfile(index); 01801 if (in == (istream *)NULL) { 01802 return false; 01803 } 01804 01805 static const size_t buffer_size = 4096; 01806 char buffer[buffer_size]; 01807 01808 in->read(buffer, buffer_size); 01809 size_t count = in->gcount(); 01810 while (count != 0) { 01811 out.write(buffer, count); 01812 in->read(buffer, buffer_size); 01813 count = in->gcount(); 01814 } 01815 01816 bool failed = (in->fail() && !in->eof()); 01817 close_read_subfile(in); 01818 nassertr(!failed, false); 01819 01820 return (!out.fail()); 01821 } 01822 01823 //////////////////////////////////////////////////////////////////// 01824 // Function: Multifile::compare_subfile 01825 // Access: Published 01826 // Description: Performs a byte-for-byte comparison of the indicated 01827 // file on disk with the nth subfile. Returns true if 01828 // the files are equivalent, or false if they are 01829 // different (or the file is missing). 01830 //////////////////////////////////////////////////////////////////// 01831 bool Multifile:: 01832 compare_subfile(int index, const Filename &filename) { 01833 nassertr(is_read_valid(), false); 01834 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01835 01836 if (!filename.exists()) { 01837 express_cat.info() 01838 << "File is missing: " << filename << "\n"; 01839 return false; 01840 } 01841 01842 istream *in1 = open_read_subfile(index); 01843 if (in1 == (istream *)NULL) { 01844 return false; 01845 } 01846 01847 pifstream in2; 01848 Filename bin_filename = Filename::binary_filename(filename); 01849 if (!bin_filename.open_read(in2)) { 01850 express_cat.info() 01851 << "Cannot read " << filename << "\n"; 01852 return false; 01853 } 01854 01855 // Check the file size. 01856 in2.seekg(0, ios::end); 01857 streampos file_size = in2.tellg(); 01858 01859 if (file_size != (streampos)get_subfile_length(index)) { 01860 // The files have different sizes. 01861 close_read_subfile(in1); 01862 return false; 01863 } 01864 01865 // Check the file data, byte-for-byte. 01866 in2.seekg(0); 01867 int byte1 = in1->get(); 01868 int byte2 = in2.get(); 01869 while (!in1->fail() && !in1->eof() && 01870 !in2.fail() && !in2.eof()) { 01871 if (byte1 != byte2) { 01872 close_read_subfile(in1); 01873 return false; 01874 } 01875 byte1 = in1->get(); 01876 byte2 = in2.get(); 01877 } 01878 01879 bool failed = (in1->fail() && !in1->eof()) || (in2.fail() && !in2.eof()); 01880 close_read_subfile(in1); 01881 01882 nassertr(!failed, false); 01883 01884 return true; 01885 } 01886 01887 //////////////////////////////////////////////////////////////////// 01888 // Function: Multifile::output 01889 // Access: Published 01890 // Description: 01891 //////////////////////////////////////////////////////////////////// 01892 void Multifile:: 01893 output(ostream &out) const { 01894 out << "Multifile " << _multifile_name << ", " << get_num_subfiles() 01895 << " subfiles.\n"; 01896 } 01897 01898 //////////////////////////////////////////////////////////////////// 01899 // Function: Multifile::ls 01900 // Access: Published 01901 // Description: Shows a list of all subfiles within the Multifile. 01902 //////////////////////////////////////////////////////////////////// 01903 void Multifile:: 01904 ls(ostream &out) const { 01905 int num_subfiles = get_num_subfiles(); 01906 for (int i = 0; i < num_subfiles; i++) { 01907 string subfile_name = get_subfile_name(i); 01908 out << subfile_name << "\n"; 01909 } 01910 } 01911 01912 //////////////////////////////////////////////////////////////////// 01913 // Function: Multifile::set_header_prefix 01914 // Access: Published 01915 // Description: Sets the string which is written to the Multifile 01916 // before the Multifile header. This string must begin 01917 // with a hash mark and end with a newline character; 01918 // and if it includes embedded newline characters, each 01919 // one must be followed by a hash mark. If these 01920 // conditions are not initially true, the string will be 01921 // modified as necessary to make it so. 01922 // 01923 // This is primarily useful as a simple hack to allow 01924 // p3d applications to be run directly from the command 01925 // line on Unix-like systems. 01926 // 01927 // The return value is true if successful, or false on 01928 // failure (for instance, because the header prefix 01929 // violates the above rules). 01930 //////////////////////////////////////////////////////////////////// 01931 void Multifile:: 01932 set_header_prefix(const string &header_prefix) { 01933 string new_header_prefix = header_prefix; 01934 01935 if (!new_header_prefix.empty()) { 01936 // It must begin with a hash mark. 01937 if (new_header_prefix[0] != '#') { 01938 new_header_prefix = string("#") + new_header_prefix; 01939 } 01940 01941 // It must end with a newline. 01942 if (new_header_prefix[new_header_prefix.size() - 1] != '\n') { 01943 new_header_prefix += string("\n"); 01944 } 01945 01946 // Embedded newlines must be followed by a hash mark. 01947 size_t newline = new_header_prefix.find('\n'); 01948 while (newline < new_header_prefix.size() - 1) { 01949 if (new_header_prefix[newline + 1] != '#') { 01950 new_header_prefix = new_header_prefix.substr(0, newline + 1) + string("#") + new_header_prefix.substr(newline + 1); 01951 } 01952 newline = new_header_prefix.find('#', newline); 01953 } 01954 } 01955 01956 if (_header_prefix != new_header_prefix) { 01957 _header_prefix = new_header_prefix; 01958 _needs_repack = true; 01959 } 01960 } 01961 01962 01963 //////////////////////////////////////////////////////////////////// 01964 // Function: Multifile::read_subfile 01965 // Access: Public 01966 // Description: Fills a string with the entire contents of 01967 // the indicated subfile. 01968 //////////////////////////////////////////////////////////////////// 01969 bool Multifile:: 01970 read_subfile(int index, string &result) { 01971 result = string(); 01972 01973 // We use a temporary pvector, because dynamic accumulation of a 01974 // pvector seems to be many times faster than that of a string, at 01975 // least on the Windows implementation of STL. 01976 pvector<unsigned char> pv; 01977 if (!read_subfile(index, pv)) { 01978 return false; 01979 } 01980 01981 if (!pv.empty()) { 01982 result.append((const char *)&pv[0], pv.size()); 01983 } 01984 01985 return true; 01986 } 01987 01988 //////////////////////////////////////////////////////////////////// 01989 // Function: Multifile::read_subfile 01990 // Access: Public 01991 // Description: Fills a pvector with the entire contents of 01992 // the indicated subfile. 01993 //////////////////////////////////////////////////////////////////// 01994 bool Multifile:: 01995 read_subfile(int index, pvector<unsigned char> &result) { 01996 nassertr(is_read_valid(), false); 01997 nassertr(index >= 0 && index < (int)_subfiles.size(), false); 01998 result.clear(); 01999 02000 // Now look up the particular Subfile we are reading. 02001 nassertr(is_read_valid(), NULL); 02002 nassertr(index >= 0 && index < (int)_subfiles.size(), NULL); 02003 Subfile *subfile = _subfiles[index]; 02004 02005 if (subfile->_source != (istream *)NULL || 02006 !subfile->_source_filename.empty()) { 02007 // The subfile has not yet been copied into the physical 02008 // Multifile. Force a flush operation to incorporate it. 02009 flush(); 02010 02011 // That shouldn't change the subfile index or delete the subfile 02012 // pointer. 02013 nassertr(subfile == _subfiles[index], NULL); 02014 } 02015 02016 result.reserve(subfile->_uncompressed_length); 02017 02018 bool success = true; 02019 if (subfile->_flags & (SF_encrypted | SF_compressed)) { 02020 // If the subfile is encrypted or compressed, we can't read it 02021 // directly. Fall back to the generic implementation. 02022 istream *in = open_read_subfile(index); 02023 if (in == (istream *)NULL) { 02024 return false; 02025 } 02026 02027 success = VirtualFile::simple_read_file(in, result); 02028 close_read_subfile(in); 02029 02030 } else { 02031 // But if the subfile is just a plain file, we can just read the 02032 // data directly from the Multifile, without paying the cost of an 02033 // ISubStream. 02034 static const size_t buffer_size = 4096; 02035 char buffer[buffer_size]; 02036 02037 streamsize pos = _offset + subfile->_data_start; 02038 size_t max_bytes = subfile->_data_length; 02039 streamsize count = 0; 02040 bool eof = true; 02041 02042 streamsize num_bytes = (streamsize)min(buffer_size, max_bytes); 02043 _read->seek_read(pos, buffer, num_bytes, count, eof); 02044 while (count != 0) { 02045 thread_consider_yield(); 02046 nassertr(count <= max_bytes, false); 02047 result.insert(result.end(), buffer, buffer + (size_t)count); 02048 max_bytes -= (size_t)count; 02049 pos += count; 02050 02051 num_bytes = (streamsize)min(buffer_size, max_bytes); 02052 _read->seek_read(pos, buffer, num_bytes, count, eof); 02053 } 02054 02055 success = !eof; 02056 } 02057 02058 if (!success) { 02059 ostringstream message; 02060 message << "I/O error reading from " << get_multifile_name() << " at " 02061 << get_subfile_name(index); 02062 nassert_raise(message.str()); 02063 return false; 02064 } 02065 02066 return true; 02067 } 02068 02069 //////////////////////////////////////////////////////////////////// 02070 // Function: Multifile::pad_to_streampos 02071 // Access: Private 02072 // Description: Assumes the _write pointer is at the indicated fpos, 02073 // rounds the fpos up to the next legitimate address 02074 // (using normalize_streampos()), and writes enough 02075 // zeroes to the stream to fill the gap. Returns the 02076 // new fpos. 02077 //////////////////////////////////////////////////////////////////// 02078 streampos Multifile:: 02079 pad_to_streampos(streampos fpos) { 02080 nassertr(_write != (ostream *)NULL, fpos); 02081 nassertr(_write->tellp() == fpos, fpos); 02082 streampos new_fpos = normalize_streampos(fpos); 02083 while (fpos < new_fpos) { 02084 _write->put(0); 02085 fpos += 1; // VC++ doesn't define streampos++ (!) 02086 } 02087 nassertr(_write->tellp() == fpos, fpos); 02088 return fpos; 02089 } 02090 02091 //////////////////////////////////////////////////////////////////// 02092 // Function: Multifile::add_new_subfile 02093 // Access: Private 02094 // Description: Adds a newly-allocated Subfile pointer to the 02095 // Multifile. 02096 //////////////////////////////////////////////////////////////////// 02097 void Multifile:: 02098 add_new_subfile(Subfile *subfile, int compression_level) { 02099 if (compression_level != 0) { 02100 #ifndef HAVE_ZLIB 02101 express_cat.warning() 02102 << "zlib not compiled in; cannot generated compressed multifiles.\n"; 02103 compression_level = 0; 02104 #else // HAVE_ZLIB 02105 subfile->_flags |= SF_compressed; 02106 subfile->_compression_level = compression_level; 02107 #endif // HAVE_ZLIB 02108 } 02109 02110 #ifdef HAVE_OPENSSL 02111 if (_encryption_flag) { 02112 subfile->_flags |= SF_encrypted; 02113 } 02114 #endif // HAVE_OPENSSL 02115 02116 if (_next_index != (streampos)0) { 02117 // If we're adding a Subfile to an already-existing Multifile, we 02118 // will eventually need to repack the file. 02119 _needs_repack = true; 02120 } 02121 02122 pair<Subfiles::iterator, bool> insert_result = _subfiles.insert(subfile); 02123 if (!insert_result.second) { 02124 // Hmm, unable to insert. There must already be a subfile by that 02125 // name. Remove the old one. 02126 Subfile *old_subfile = (*insert_result.first); 02127 old_subfile->_flags |= SF_deleted; 02128 02129 // Maybe it was just added to the _new_subfiles list. In this case, remove it from that list. 02130 PendingSubfiles::iterator ni = find(_new_subfiles.begin(), _new_subfiles.end(), old_subfile); 02131 if (ni != _new_subfiles.end()) { 02132 _new_subfiles.erase(ni); 02133 02134 } else { 02135 // Otherwise, add it to the _removed_subfiles list, so we can remove the old one. 02136 _removed_subfiles.push_back(old_subfile); 02137 } 02138 02139 (*insert_result.first) = subfile; 02140 } 02141 02142 _new_subfiles.push_back(subfile); 02143 } 02144 02145 //////////////////////////////////////////////////////////////////// 02146 // Function: Multifile::open_read_subfile 02147 // Access: Private 02148 // Description: This variant of open_read_subfile() is used 02149 // internally only, and accepts a pointer to the 02150 // internal Subfile object, which is assumed to be valid 02151 // and written to the multifile. 02152 //////////////////////////////////////////////////////////////////// 02153 istream *Multifile:: 02154 open_read_subfile(Subfile *subfile) { 02155 nassertr(subfile->_source == (istream *)NULL && 02156 subfile->_source_filename.empty(), NULL); 02157 02158 // Return an ISubStream object that references into the open 02159 // Multifile istream. 02160 nassertr(subfile->_data_start != (streampos)0, NULL); 02161 istream *stream = 02162 new ISubStream(_read, _offset + subfile->_data_start, 02163 _offset + subfile->_data_start + (streampos)subfile->_data_length); 02164 02165 if ((subfile->_flags & SF_encrypted) != 0) { 02166 #ifndef HAVE_OPENSSL 02167 express_cat.error() 02168 << "OpenSSL not compiled in; cannot read encrypted multifiles.\n"; 02169 delete stream; 02170 return NULL; 02171 #else // HAVE_OPENSSL 02172 // The subfile is encrypted. So actually, return an 02173 // IDecryptStream that wraps around the ISubStream. 02174 IDecryptStream *wrapper = 02175 new IDecryptStream(stream, true, _encryption_password); 02176 stream = wrapper; 02177 02178 // Validate the password by confirming that the encryption header 02179 // matches. 02180 char this_header[_encrypt_header_size]; 02181 stream->read(this_header, _encrypt_header_size); 02182 if (stream->fail() || stream->gcount() != (unsigned)_encrypt_header_size || 02183 memcmp(this_header, _encrypt_header, _encrypt_header_size) != 0) { 02184 express_cat.error() 02185 << "Unable to decrypt subfile " << subfile->_name << ".\n"; 02186 delete stream; 02187 return NULL; 02188 } 02189 #endif // HAVE_OPENSSL 02190 } 02191 02192 if ((subfile->_flags & SF_compressed) != 0) { 02193 #ifndef HAVE_ZLIB 02194 express_cat.error() 02195 << "zlib not compiled in; cannot read compressed multifiles.\n"; 02196 delete stream; 02197 return NULL; 02198 #else // HAVE_ZLIB 02199 // Oops, the subfile is compressed. So actually, return an 02200 // IDecompressStream that wraps around the ISubStream. 02201 IDecompressStream *wrapper = new IDecompressStream(stream, true); 02202 stream = wrapper; 02203 #endif // HAVE_ZLIB 02204 } 02205 02206 if (stream->fail()) { 02207 // Hmm, some inexplicable problem. 02208 delete stream; 02209 return NULL; 02210 } 02211 02212 return stream; 02213 } 02214 02215 //////////////////////////////////////////////////////////////////// 02216 // Function: Multifile::standardize_subfile_name 02217 // Access: Private 02218 // Description: Returns the standard form of the subfile name. 02219 //////////////////////////////////////////////////////////////////// 02220 string Multifile:: 02221 standardize_subfile_name(const string &subfile_name) const { 02222 Filename name = subfile_name; 02223 name.standardize(); 02224 if (name.empty() || name == "/") { 02225 // Invalid empty name. 02226 return string(); 02227 } 02228 02229 if (name[0] == '/') { 02230 return name.get_fullpath().substr(1); 02231 } else if (name.length() > 2 && name[0] == '.' && name[1] == '/') { 02232 return name.get_fullpath().substr(2); 02233 } else { 02234 return name.get_fullpath(); 02235 } 02236 } 02237 02238 //////////////////////////////////////////////////////////////////// 02239 // Function: Multifile::clear_subfiles 02240 // Access: Private 02241 // Description: Removes the set of subfiles from the tables and frees 02242 // their associated memory. 02243 //////////////////////////////////////////////////////////////////// 02244 void Multifile:: 02245 clear_subfiles() { 02246 PendingSubfiles::iterator pi; 02247 for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) { 02248 Subfile *subfile = (*pi); 02249 subfile->rewrite_index_flags(*_write); 02250 delete subfile; 02251 } 02252 _removed_subfiles.clear(); 02253 02254 // We don't have to delete the ones in _new_subfiles, because these 02255 // also appear in _subfiles. 02256 _new_subfiles.clear(); 02257 02258 #ifdef HAVE_OPENSSL 02259 for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) { 02260 Subfile *subfile = (*pi); 02261 delete subfile; 02262 } 02263 _cert_special.clear(); 02264 02265 _signatures.clear(); 02266 #endif // HAVE_OPENSSL 02267 02268 Subfiles::iterator fi; 02269 for (fi = _subfiles.begin(); fi != _subfiles.end(); ++fi) { 02270 Subfile *subfile = (*fi); 02271 delete subfile; 02272 } 02273 _subfiles.clear(); 02274 } 02275 02276 //////////////////////////////////////////////////////////////////// 02277 // Function: Multifile::read_index 02278 // Access: Private 02279 // Description: Reads the Multifile header and index. Returns true 02280 // if successful, false if the Multifile is not valid. 02281 //////////////////////////////////////////////////////////////////// 02282 bool Multifile:: 02283 read_index() { 02284 nassertr(_read != (IStreamWrapper *)NULL, false); 02285 02286 // We acquire the IStreamWrapper lock for the duration of this 02287 // method. 02288 _read->acquire(); 02289 istream *read = _read->get_istream(); 02290 02291 char this_header[_header_size]; 02292 read->seekg(_offset); 02293 02294 // Here's a special case: if the multifile begins with a hash 02295 // character, then we continue reading and discarding lines of ASCII 02296 // text, until we come across a nonempty line that does not begin 02297 // with a hash character. This allows a P3D application (which is a 02298 // multifile) to be run directly on the command line on Unix-based 02299 // systems. 02300 _header_prefix = string(); 02301 int ch = read->get(); 02302 02303 if (ch == '#') { 02304 while (ch != EOF && ch == '#') { 02305 // Skip to the end of the line. 02306 while (ch != EOF && ch != '\n') { 02307 _header_prefix += ch; 02308 ch = read->get(); 02309 } 02310 // Skip to the first non-whitespace character of the line. 02311 while (ch != EOF && (isspace(ch) || ch == '\r')) { 02312 _header_prefix += ch; 02313 ch = read->get(); 02314 } 02315 } 02316 } 02317 02318 // Now read the actual Multifile header. 02319 this_header[0] = ch; 02320 read->read(this_header + 1, _header_size - 1); 02321 if (read->fail() || read->gcount() != (unsigned)(_header_size - 1)) { 02322 express_cat.info() 02323 << "Unable to read Multifile header " << _multifile_name << ".\n"; 02324 _read->release(); 02325 close(); 02326 return false; 02327 } 02328 02329 if (memcmp(this_header, _header, _header_size) != 0) { 02330 express_cat.info() 02331 << _multifile_name << " is not a Multifile.\n"; 02332 _read->release(); 02333 close(); 02334 return false; 02335 } 02336 02337 // Now get the version numbers out. 02338 StreamReader reader(read, false); 02339 _file_major_ver = reader.get_int16(); 02340 _file_minor_ver = reader.get_int16(); 02341 _scale_factor = reader.get_uint32(); 02342 _new_scale_factor = _scale_factor; 02343 02344 if (read->eof() || read->fail()) { 02345 express_cat.info() 02346 << _multifile_name << " header is truncated.\n"; 02347 _read->release(); 02348 close(); 02349 return false; 02350 } 02351 02352 if (_file_major_ver != _current_major_ver || 02353 (_file_major_ver == _current_major_ver && 02354 _file_minor_ver > _current_minor_ver)) { 02355 express_cat.info() 02356 << _multifile_name << " has version " << _file_major_ver << "." 02357 << _file_minor_ver << ", expecting version " 02358 << _current_major_ver << "." << _current_minor_ver << ".\n"; 02359 _read->release(); 02360 close(); 02361 return false; 02362 } 02363 02364 _record_timestamp = true; 02365 if (_file_minor_ver >= 1) { 02366 time_t read_timestamp = reader.get_uint32(); 02367 if (read_timestamp == 0) { 02368 // If we read a 0 timestamp from the file, that implies that we 02369 // don't want to record a timestamp in this particular file. 02370 _record_timestamp = false; 02371 } else { 02372 _timestamp = read_timestamp; 02373 } 02374 _timestamp_dirty = false; 02375 } 02376 02377 // Now read the index out. 02378 _next_index = read->tellg() - _offset; 02379 _next_index = normalize_streampos(_next_index); 02380 read->seekg(_next_index + _offset); 02381 _last_index = 0; 02382 _last_data_byte = 0; 02383 streampos index_forward; 02384 streamoff bytes_skipped = 0; 02385 bool read_cert_special = false; 02386 02387 Subfile *subfile = new Subfile; 02388 index_forward = subfile->read_index(*read, _next_index, this); 02389 while (index_forward != (streampos)0) { 02390 _last_index = _next_index; 02391 if (subfile->is_deleted()) { 02392 // Ignore deleted Subfiles in the index. 02393 _needs_repack = true; 02394 delete subfile; 02395 } else if (subfile->is_cert_special()) { 02396 // Certificate chains and signature files get stored in a 02397 // special list. 02398 _cert_special.push_back(subfile); 02399 read_cert_special = true; 02400 } else { 02401 _subfiles.push_back(subfile); 02402 } 02403 if (!subfile->is_cert_special()) { 02404 if (bytes_skipped != 0) { 02405 // If the index entries don't follow exactly sequentially 02406 // (except for the cert special files), the file ought to be 02407 // repacked. 02408 _needs_repack = true; 02409 } 02410 if (read_cert_special) { 02411 // If we read a normal subfile following a cert_special entry, 02412 // the file ought to be repacked (certificates have to go at 02413 // the end). 02414 _needs_repack = true; 02415 } 02416 _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos()); 02417 } 02418 streampos curr_pos = normalize_streampos(read->tellg() - _offset); 02419 bytes_skipped = index_forward - curr_pos; 02420 read->seekg(index_forward + _offset); 02421 _next_index = index_forward; 02422 subfile = new Subfile; 02423 index_forward = subfile->read_index(*read, _next_index, this); 02424 } 02425 if (subfile->is_index_invalid()) { 02426 express_cat.info() 02427 << "Error reading index for " << _multifile_name << ".\n"; 02428 _read->release(); 02429 close(); 02430 delete subfile; 02431 return false; 02432 } 02433 02434 // Check if the list is already sorted. If it is not, we need a 02435 // repack. 02436 for (size_t si = 1; si < _subfiles.size() && !_needs_repack; ++si) { 02437 if (*_subfiles[si] < *_subfiles[si - 1]) { 02438 _needs_repack = true; 02439 } 02440 } 02441 02442 if (_needs_repack) { 02443 // At least sort them now. 02444 size_t before_size = _subfiles.size(); 02445 _subfiles.sort(); 02446 size_t after_size = _subfiles.size(); 02447 02448 // If these don't match, the same filename appeared twice in the 02449 // index, which shouldn't be possible. 02450 nassertr(before_size == after_size, true); 02451 } 02452 02453 delete subfile; 02454 _read->release(); 02455 return true; 02456 } 02457 02458 //////////////////////////////////////////////////////////////////// 02459 // Function: Multifile::write_header 02460 // Access: Private 02461 // Description: Writes just the header part of the Multifile, not the 02462 // index. 02463 //////////////////////////////////////////////////////////////////// 02464 bool Multifile:: 02465 write_header() { 02466 _file_major_ver = _current_major_ver; 02467 _file_minor_ver = _current_minor_ver; 02468 02469 nassertr(_write != (ostream *)NULL, false); 02470 nassertr(_write->tellp() == (streampos)0, false); 02471 _write->write(_header_prefix.data(), _header_prefix.size()); 02472 _write->write(_header, _header_size); 02473 StreamWriter writer(_write, false); 02474 writer.add_int16(_current_major_ver); 02475 writer.add_int16(_current_minor_ver); 02476 writer.add_uint32(_scale_factor); 02477 02478 if (_record_timestamp) { 02479 writer.add_uint32(_timestamp); 02480 } else { 02481 writer.add_uint32(0); 02482 _timestamp_dirty = false; 02483 } 02484 02485 _next_index = _write->tellp(); 02486 _next_index = pad_to_streampos(_next_index); 02487 _last_index = 0; 02488 02489 if (_write->fail()) { 02490 express_cat.info() 02491 << "Unable to write header for " << _multifile_name << ".\n"; 02492 close(); 02493 return false; 02494 } 02495 02496 return true; 02497 } 02498 02499 //////////////////////////////////////////////////////////////////// 02500 // Function: Multifile::check_signatures 02501 // Access: Private 02502 // Description: Walks through the list of _cert_special entries in 02503 // the Multifile, moving any valid signatures found to 02504 // _signatures. After this call, _cert_special will be 02505 // empty. 02506 // 02507 // This does not check the validity of the certificates 02508 // themselves. It only checks that they correctly sign 02509 // the Multifile contents. 02510 //////////////////////////////////////////////////////////////////// 02511 void Multifile:: 02512 check_signatures() { 02513 #ifdef HAVE_OPENSSL 02514 PendingSubfiles::iterator pi; 02515 02516 for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) { 02517 Subfile *subfile = (*pi); 02518 nassertv((subfile->_flags & SF_signature) != 0); 02519 02520 // Extract the signature data and certificate separately. 02521 istream *stream = open_read_subfile(subfile); 02522 nassertv(stream != NULL); 02523 StreamReader reader(*stream); 02524 size_t sig_size = reader.get_uint32(); 02525 string sig_string = reader.extract_bytes(sig_size); 02526 02527 size_t num_certs = reader.get_uint32(); 02528 02529 // Read the remaining buffer of certificate data. 02530 pvector<unsigned char> buffer; 02531 bool success = VirtualFile::simple_read_file(stream, buffer); 02532 nassertv(success); 02533 close_read_subfile(stream); 02534 02535 // Now convert each of the certificates to an X509 object, and 02536 // store it in our CertChain. 02537 CertChain chain; 02538 EVP_PKEY *pkey = NULL; 02539 if (!buffer.empty()) { 02540 #if OPENSSL_VERSION_NUMBER >= 0x00908000L 02541 // Beginning in 0.9.8, d2i_X509() accepted a const unsigned char **. 02542 const unsigned char *bp, *bp_end; 02543 #else 02544 // Prior to 0.9.8, d2i_X509() accepted an unsigned char **. 02545 unsigned char *bp, *bp_end; 02546 #endif 02547 bp = (unsigned char *)&buffer[0]; 02548 bp_end = bp + buffer.size(); 02549 X509 *x509 = d2i_X509(NULL, &bp, bp_end - bp); 02550 while (num_certs > 0 && x509 != NULL) { 02551 chain.push_back(CertRecord(x509)); 02552 --num_certs; 02553 x509 = d2i_X509(NULL, &bp, bp_end - bp); 02554 } 02555 if (num_certs != 0 || x509 != NULL) { 02556 express_cat.warning() 02557 << "Extra data in signature record.\n"; 02558 } 02559 } 02560 02561 if (!chain.empty()) { 02562 pkey = X509_get_pubkey(chain[0]._cert); 02563 } 02564 02565 if (pkey != NULL) { 02566 EVP_MD_CTX *md_ctx; 02567 #ifdef SSL_097 02568 md_ctx = EVP_MD_CTX_create(); 02569 #else 02570 md_ctx = new EVP_MD_CTX; 02571 #endif 02572 EVP_VerifyInit(md_ctx, EVP_sha1()); 02573 02574 nassertv(_read != NULL); 02575 _read->acquire(); 02576 istream *read = _read->get_istream(); 02577 02578 // Read and hash the multifile contents, but only up till 02579 // _last_data_byte. 02580 read->seekg(_offset); 02581 streampos bytes_remaining = _last_data_byte; 02582 static const size_t buffer_size = 4096; 02583 char buffer[buffer_size]; 02584 read->read(buffer, min((streampos)buffer_size, bytes_remaining)); 02585 size_t count = read->gcount(); 02586 while (count != 0) { 02587 nassertv(count <= buffer_size); 02588 EVP_VerifyUpdate(md_ctx, buffer, count); 02589 bytes_remaining -= count; 02590 read->read(buffer, min((streampos)buffer_size, bytes_remaining)); 02591 count = read->gcount(); 02592 } 02593 nassertv(bytes_remaining == (streampos)0); 02594 _read->release(); 02595 02596 // Now check that the signature matches the hash. 02597 int verify_result = 02598 EVP_VerifyFinal(md_ctx, 02599 (unsigned char *)sig_string.data(), 02600 sig_string.size(), pkey); 02601 if (verify_result == 1) { 02602 // The signature matches; save the certificate and its chain. 02603 _signatures.push_back(chain); 02604 } else { 02605 // Bad match. 02606 _needs_repack = true; 02607 } 02608 } 02609 } 02610 #endif // HAVE_OPENSSL 02611 02612 _cert_special.clear(); 02613 } 02614 02615 //////////////////////////////////////////////////////////////////// 02616 // Function: Multifile::Subfile::read_index 02617 // Access: Public 02618 // Description: Reads the index record for the Subfile from the 02619 // indicated istream. Assumes the istream has already 02620 // been positioned to the indicated stream position, 02621 // fpos, the start of the index record. Returns the 02622 // position within the file of the next index record. 02623 //////////////////////////////////////////////////////////////////// 02624 streampos Multifile::Subfile:: 02625 read_index(istream &read, streampos fpos, Multifile *multifile) { 02626 nassertr(read.tellg() - multifile->_offset == fpos, fpos); 02627 02628 // First, get the next stream position. We do this separately, 02629 // because if it is zero, we don't get anything else. 02630 StreamReader reader(read); 02631 02632 streampos next_index = multifile->word_to_streampos(reader.get_uint32()); 02633 if (read.eof() || read.fail()) { 02634 _flags |= SF_index_invalid; 02635 return 0; 02636 } 02637 02638 if (next_index == (streampos)0) { 02639 return 0; 02640 } 02641 02642 // Now get the rest of the index. 02643 02644 _index_start = fpos; 02645 _index_length = 0; 02646 02647 _data_start = multifile->word_to_streampos(reader.get_uint32()); 02648 _data_length = reader.get_uint32(); 02649 _flags = reader.get_uint16(); 02650 if ((_flags & (SF_compressed | SF_encrypted)) != 0) { 02651 _uncompressed_length = reader.get_uint32(); 02652 } else { 02653 _uncompressed_length = _data_length; 02654 } 02655 if (multifile->_file_minor_ver < 1) { 02656 _timestamp = multifile->get_timestamp(); 02657 } else { 02658 _timestamp = reader.get_uint32(); 02659 if (_timestamp == 0) { 02660 _timestamp = multifile->get_timestamp(); 02661 } 02662 } 02663 02664 size_t name_length = reader.get_uint16(); 02665 if (read.eof() || read.fail()) { 02666 _flags |= SF_index_invalid; 02667 return 0; 02668 } 02669 02670 // And finally, get the rest of the name. 02671 char *name_buffer = (char *)PANDA_MALLOC_ARRAY(name_length); 02672 nassertr(name_buffer != (char *)NULL, next_index); 02673 for (size_t ni = 0; ni < name_length; ni++) { 02674 name_buffer[ni] = read.get() ^ 0xff; 02675 } 02676 _name = string(name_buffer, name_length); 02677 PANDA_FREE_ARRAY(name_buffer); 02678 02679 if (read.eof() || read.fail()) { 02680 _flags |= SF_index_invalid; 02681 return 0; 02682 } 02683 02684 _index_length = read.tellg() - fpos - multifile->_offset; 02685 return next_index; 02686 } 02687 02688 //////////////////////////////////////////////////////////////////// 02689 // Function: Multifile::Subfile::write_index 02690 // Access: Public 02691 // Description: Writes the index record for the Subfile to the 02692 // indicated ostream. Assumes the istream has already 02693 // been positioned to the indicated stream position, 02694 // fpos, the start of the index record, and that this is 02695 // the effective end of the file. Returns the position 02696 // within the file of the next index record. 02697 // 02698 // The _index_start member is updated by this operation. 02699 //////////////////////////////////////////////////////////////////// 02700 streampos Multifile::Subfile:: 02701 write_index(ostream &write, streampos fpos, Multifile *multifile) { 02702 nassertr(write.tellp() - multifile->_offset == fpos, fpos); 02703 02704 _index_start = fpos; 02705 _index_length = 0; 02706 02707 // This will be the contents of this particular index record. We 02708 // build it up first since it will be variable length. 02709 Datagram dg; 02710 dg.add_uint32(multifile->streampos_to_word(_data_start)); 02711 dg.add_uint32(_data_length); 02712 dg.add_uint16(_flags); 02713 if ((_flags & (SF_compressed | SF_encrypted)) != 0) { 02714 dg.add_uint32(_uncompressed_length); 02715 } 02716 dg.add_uint32(_timestamp); 02717 dg.add_uint16(_name.length()); 02718 02719 // For no real good reason, we'll invert all the bits in the name. 02720 // The only reason we do this is to make it inconvenient for a 02721 // casual browser of the Multifile to discover the names of the 02722 // files stored within it. Naturally, this isn't real obfuscation 02723 // or security. 02724 string::iterator ni; 02725 for (ni = _name.begin(); ni != _name.end(); ++ni) { 02726 dg.add_int8((*ni) ^ 0xff); 02727 } 02728 02729 size_t this_index_size = 4 + dg.get_length(); 02730 02731 // Plus, we will write out the next index address first. 02732 streampos next_index = fpos + (streampos)this_index_size; 02733 02734 Datagram idg; 02735 idg.add_uint32(multifile->streampos_to_word(next_index)); 02736 02737 write.write((const char *)idg.get_data(), idg.get_length()); 02738 write.write((const char *)dg.get_data(), dg.get_length()); 02739 02740 _index_length = write.tellp() - fpos - multifile->_offset; 02741 return next_index; 02742 } 02743 02744 //////////////////////////////////////////////////////////////////// 02745 // Function: Multifile::Subfile::write_data 02746 // Access: Public 02747 // Description: Writes the data record for the Subfile to the 02748 // indicated ostream: the actual contents of the 02749 // Subfile. Assumes the istream has already been 02750 // positioned to the indicated stream position, fpos, 02751 // the start of the data record, and that this is the 02752 // effective end of the file. Returns the position 02753 // within the file of the next data record. 02754 // 02755 // The _data_start, _data_length, and 02756 // _uncompressed_length members are updated by this 02757 // operation. 02758 // 02759 // If the "read" pointer is non-NULL, it is the readable 02760 // istream of a Multifile in which the Subfile might 02761 // already be packed. This is used for reading the 02762 // contents of the Subfile during a repack() operation. 02763 //////////////////////////////////////////////////////////////////// 02764 streampos Multifile::Subfile:: 02765 write_data(ostream &write, istream *read, streampos fpos, 02766 Multifile *multifile) { 02767 nassertr(write.tellp() - multifile->_offset == fpos, fpos); 02768 02769 istream *source = _source; 02770 pifstream source_file; 02771 if (source == (istream *)NULL && !_source_filename.empty()) { 02772 // If we have a filename, open it up and read that. 02773 if (!_source_filename.open_read(source_file)) { 02774 // Unable to open the source file. 02775 express_cat.info() 02776 << "Unable to read " << _source_filename << ".\n"; 02777 _flags |= SF_data_invalid; 02778 _data_length = 0; 02779 _uncompressed_length = 0; 02780 } else { 02781 source = &source_file; 02782 } 02783 } 02784 02785 if (source == (istream *)NULL) { 02786 // We don't have any source data. Perhaps we're reading from an 02787 // already-packed Subfile (e.g. during repack()). 02788 if (read == (istream *)NULL) { 02789 // No, we're just screwed. 02790 express_cat.info() 02791 << "No source for subfile " << _name << ".\n"; 02792 _flags |= SF_data_invalid; 02793 } else { 02794 // Read the data from the original Multifile. 02795 read->seekg(_data_start + multifile->_offset); 02796 for (size_t p = 0; p < _data_length; p++) { 02797 int byte = read->get(); 02798 if (read->eof() || read->fail()) { 02799 // Unexpected EOF or other failure on the source file. 02800 express_cat.info() 02801 << "Unexpected EOF for subfile " << _name << ".\n"; 02802 _flags |= SF_data_invalid; 02803 break; 02804 } 02805 write.put(byte); 02806 } 02807 } 02808 } else { 02809 // We do have source data. Copy it in, and also measure its 02810 // length. 02811 ostream *putter = &write; 02812 bool delete_putter = false; 02813 02814 #ifndef HAVE_OPENSSL 02815 // Without OpenSSL, we can't support encryption. The flag had 02816 // better not be set. 02817 nassertr((_flags & SF_encrypted) == 0, fpos); 02818 02819 #else // HAVE_OPENSSL 02820 if ((_flags & SF_encrypted) != 0) { 02821 // Write it encrypted. 02822 OEncryptStream *encrypt = new OEncryptStream; 02823 encrypt->set_iteration_count(multifile->_encryption_iteration_count); 02824 encrypt->open(putter, delete_putter, multifile->_encryption_password); 02825 02826 putter = encrypt; 02827 delete_putter = true; 02828 02829 // Also write the encrypt_header to the beginning of the 02830 // encrypted stream, so we can validate the password on 02831 // decryption. 02832 putter->write(_encrypt_header, _encrypt_header_size); 02833 } 02834 #endif // HAVE_OPENSSL 02835 02836 #ifndef HAVE_ZLIB 02837 // Without ZLIB, we can't support compression. The flag had 02838 // better not be set. 02839 nassertr((_flags & SF_compressed) == 0, fpos); 02840 #else // HAVE_ZLIB 02841 if ((_flags & SF_compressed) != 0) { 02842 // Write it compressed. 02843 putter = new OCompressStream(putter, delete_putter, _compression_level); 02844 delete_putter = true; 02845 } 02846 #endif // HAVE_ZLIB 02847 02848 streampos write_start = fpos; 02849 _uncompressed_length = 0; 02850 02851 #ifndef HAVE_OPENSSL 02852 // We also need OpenSSL for signatures. 02853 nassertr((_flags & SF_signature) == 0, fpos); 02854 02855 #else // HAVE_OPENSSL 02856 if ((_flags & SF_signature) != 0) { 02857 // If it's a special signature record, precede the record data 02858 // (the certificate itself) with the signature data generated 02859 // against the multifile contents. 02860 02861 // In order to generate a signature, we need to have a valid 02862 // read pointer. 02863 nassertr(read != NULL, fpos); 02864 02865 // And we also need to have a private key. 02866 nassertr(_pkey != NULL, fpos); 02867 02868 EVP_MD_CTX *md_ctx; 02869 #ifdef SSL_097 02870 md_ctx = EVP_MD_CTX_create(); 02871 #else 02872 md_ctx = new EVP_MD_CTX; 02873 #endif 02874 EVP_SignInit(md_ctx, EVP_sha1()); 02875 02876 // Read and hash the multifile contents, but only up till 02877 // _last_data_byte. 02878 nassertr(multifile->_last_data_byte < fpos, fpos); 02879 read->seekg(multifile->_offset); 02880 streampos bytes_remaining = multifile->_last_data_byte; 02881 static const size_t buffer_size = 4096; 02882 char buffer[buffer_size]; 02883 read->read(buffer, min((streampos)buffer_size, bytes_remaining)); 02884 size_t count = read->gcount(); 02885 while (count != 0) { 02886 nassertr(count <= buffer_size, fpos); 02887 EVP_SignUpdate(md_ctx, buffer, count); 02888 bytes_remaining -= count; 02889 read->read(buffer, min((streampos)buffer_size, bytes_remaining)); 02890 count = read->gcount(); 02891 } 02892 nassertr(bytes_remaining == (streampos)0, fpos); 02893 02894 // Now generate and write out the signature. 02895 unsigned int max_size = EVP_PKEY_size(_pkey); 02896 unsigned char *sig_data = new unsigned char[max_size]; 02897 unsigned int sig_size; 02898 if (!EVP_SignFinal(md_ctx, sig_data, &sig_size, _pkey)) { 02899 OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr(); 02900 sslw->notify_ssl_errors(); 02901 } 02902 nassertr(sig_size <= max_size, fpos); 02903 02904 StreamWriter writer(*putter); 02905 writer.add_uint32(sig_size); 02906 putter->write((char *)sig_data, sig_size); 02907 _uncompressed_length += 4 + sig_size; 02908 02909 delete[] sig_data; 02910 02911 #ifdef SSL_097 02912 EVP_MD_CTX_destroy(md_ctx); 02913 #else 02914 delete md_ctx; 02915 #endif 02916 } 02917 #endif // HAVE_OPENSSL 02918 02919 // Finally, we can write out the data itself. 02920 static const size_t buffer_size = 4096; 02921 char buffer[buffer_size]; 02922 02923 source->read(buffer, buffer_size); 02924 size_t count = source->gcount(); 02925 while (count != 0) { 02926 _uncompressed_length += count; 02927 putter->write(buffer, count); 02928 source->read(buffer, buffer_size); 02929 count = source->gcount(); 02930 } 02931 02932 if (delete_putter) { 02933 delete putter; 02934 } 02935 02936 streampos write_end = write.tellp() - multifile->_offset; 02937 _data_length = (size_t)(write_end - write_start); 02938 } 02939 02940 // We can't set _data_start until down here, after we have read the 02941 // Subfile. (In case we are running during repack()). 02942 _data_start = fpos; 02943 02944 // Get the modification timestamp for this subfile. This is read 02945 // from the source file, if we have a filename; otherwise, it's the 02946 // current time. 02947 if (!_source_filename.empty()) { 02948 _timestamp = _source_filename.get_timestamp(); 02949 } 02950 if (_timestamp == 0) { 02951 _timestamp = time(NULL); 02952 } 02953 02954 _source = (istream *)NULL; 02955 _source_filename = Filename(); 02956 source_file.close(); 02957 02958 return fpos + (streampos)_data_length; 02959 } 02960 02961 //////////////////////////////////////////////////////////////////// 02962 // Function: Multifile::Subfile::rewrite_index_data_start 02963 // Access: Public 02964 // Description: Seeks within the indicate pfstream back to the index 02965 // record and rewrites just the _data_start and 02966 // _data_length part of the index record. 02967 //////////////////////////////////////////////////////////////////// 02968 void Multifile::Subfile:: 02969 rewrite_index_data_start(ostream &write, Multifile *multifile) { 02970 nassertv(_index_start != (streampos)0); 02971 02972 static const size_t data_start_offset = 4; 02973 size_t data_start_pos = _index_start + (streampos)data_start_offset; 02974 write.seekp(data_start_pos + multifile->_offset); 02975 nassertv(!write.fail()); 02976 02977 StreamWriter writer(write); 02978 writer.add_uint32(multifile->streampos_to_word(_data_start)); 02979 writer.add_uint32(_data_length); 02980 writer.add_uint16(_flags); 02981 if ((_flags & (SF_compressed | SF_encrypted)) != 0) { 02982 writer.add_uint32(_uncompressed_length); 02983 } 02984 if (multifile->_record_timestamp) { 02985 writer.add_uint32(_timestamp); 02986 } else { 02987 writer.add_uint32(0); 02988 } 02989 } 02990 02991 //////////////////////////////////////////////////////////////////// 02992 // Function: Multifile::Subfile::rewrite_index_flags 02993 // Access: Public 02994 // Description: Seeks within the indicated ostream back to the index 02995 // record and rewrites just the _flags part of the 02996 // index record. 02997 //////////////////////////////////////////////////////////////////// 02998 void Multifile::Subfile:: 02999 rewrite_index_flags(ostream &write) { 03000 // If the subfile has never even been recorded to disk, we don't 03001 // need to do anything at all in this function. 03002 if (_index_start != (streampos)0) { 03003 static const size_t flags_offset = 4 + 4 + 4; 03004 size_t flags_pos = _index_start + (streampos)flags_offset; 03005 write.seekp(flags_pos); 03006 nassertv(!write.fail()); 03007 03008 StreamWriter writer(write); 03009 writer.add_uint16(_flags); 03010 } 03011 }