Panda3D

multifile.cxx

00001 // Filename: multifile.cxx
00002 // Created by:  mike (09Jan97)
00003 //
00004 ////////////////////////////////////////////////////////////////////
00005 //
00006 // PANDA 3D SOFTWARE
00007 // Copyright (c) Carnegie Mellon University.  All rights reserved.
00008 //
00009 // All use of this software is subject to the terms of the revised BSD
00010 // license.  You should have received a copy of this license along
00011 // with this source code in a file named "LICENSE."
00012 //
00013 ////////////////////////////////////////////////////////////////////
00014 
00015 #include "multifile.h"
00016 
00017 #include "config_express.h"
00018 #include "streamWriter.h"
00019 #include "streamReader.h"
00020 #include "datagram.h"
00021 #include "zStream.h"
00022 #include "encryptStream.h"
00023 #include "virtualFileSystem.h"
00024 #include "virtualFile.h"
00025 
00026 #include <algorithm>
00027 #include <time.h>
00028 
00029 // This sequence of bytes begins each Multifile to identify it as a
00030 // Multifile.
00031 const char Multifile::_header[] = "pmf\0\n\r";
00032 const size_t Multifile::_header_size = 6;
00033 
00034 // These numbers identify the version of the Multifile.  Generally, a
00035 // change in the major version is intolerable; while a Multifile with
00036 // an older minor version may still be read.
00037 const int Multifile::_current_major_ver = 1;
00038 
00039 const int Multifile::_current_minor_ver = 1;
00040 // Bumped to version 1.1 on 6/8/06 to add timestamps.
00041 
00042 // To confirm that the supplied password matches, we write the
00043 // Mutifile magic header at the beginning of the encrypted stream.
00044 // I suppose this does compromise the encryption security a tiny
00045 // bit by making it easy for crackers to validate that a
00046 // particular password guess matches or doesn't match, but the
00047 // encryption algorithm doesn't depend on this being difficult
00048 // anyway.
00049 const char Multifile::_encrypt_header[] = "crypty";
00050 const size_t Multifile::_encrypt_header_size = 6;
00051 
00052 
00053 
00054 //
00055 // A Multifile consists of the following elements:
00056 //
00057 // (1) A header.  This is always the first n bytes of the Multifile,
00058 // and contains a magic number to identify the file, as well as
00059 // version numbers and any file-specific parameters.
00060 //
00061 //   char[6]    The string Multifile::_header, a magic number.
00062 //   int16      The file's major version number
00063 //   int16      The file's minor version number
00064 //   uint32     Scale factor.  This scales all address references within
00065 //              the file.  Normally 1, this may be set larger to
00066 //              support Multifiles larger than 4GB.
00067 //   uint32     An overall modification timestamp for the entire multifile.
00068 
00069 //
00070 // (2) Zero or more index entries, one for each subfile within the
00071 // Multifile.  These entries are of variable length.  The first one of
00072 // these immediately follows the header, and the first word of each
00073 // index entry contains the address of the next index entry.  A zero
00074 // "next" address marks the end of the chain.  These may appear at any
00075 // point within the Multifile; they do not necessarily appear in
00076 // sequential order at the beginning of the file (although they will
00077 // after the file has been "packed").
00078 //
00079 //   uint32     The address of the next entry.  0 to mark the end.
00080 //   uint32     The address of this subfile's data record.
00081 //   uint32     The length in bytes of this subfile's data record.
00082 //   uint16     The Subfile::_flags member.
00083 //  [uint32]    The original, uncompressed and unencrypted length of the
00084 //               subfile, if it is compressed or encrypted.  This field
00085 //               is only present if one or both of the SF_compressed
00086 //               or SF_encrypted bits are set in _flags.
00087 //   uint32     A modification timestamp for the subfile.
00088 //   uint16     The length in bytes of the subfile's name.
00089 //   char[n]    The subfile's name.
00090 //
00091 // (3) Zero or more data entries, one for each subfile.  These may
00092 // appear at any point within the Multifile; they do not necessarily
00093 // follow each index entry, nor are they necessarily all grouped
00094 // together at the end (although they will be all grouped together at
00095 // the end after the file has been "packed").  These are just blocks
00096 // of literal data.
00097 //
00098 
00099 ////////////////////////////////////////////////////////////////////
00100 //     Function: Multifile::Constructor
00101 //       Access: Published
00102 //  Description:
00103 ////////////////////////////////////////////////////////////////////
00104 Multifile::
00105 Multifile() :
00106   _read_filew(_read_file),
00107   _read_write_filew(_read_write_file)
00108 {
00109   ConfigVariableInt multifile_encryption_iteration_count
00110     ("multifile-encryption-iteration-count", 0,
00111      PRC_DESC("This is a special value of encryption-iteration-count used to encrypt "
00112               "subfiles within a multifile.  It has a default value of 0 (just one "
00113               "application), on the assumption that the files from a multifile must "
00114               "be loaded quickly, without paying the cost of an expensive hash on "
00115               "each subfile in order to decrypt it."));
00116   
00117   _read = (IStreamWrapper *)NULL;
00118   _write = (ostream *)NULL;
00119   _offset = 0;
00120   _owns_stream = false;
00121   _next_index = 0;
00122   _last_index = 0;
00123   _last_data_byte = 0;
00124   _needs_repack = false;
00125   _timestamp = 0;
00126   _timestamp_dirty = false;
00127   _record_timestamp = true;
00128   _scale_factor = 1;
00129   _new_scale_factor = 1;
00130   _encryption_flag = false;
00131   _encryption_iteration_count = multifile_encryption_iteration_count;
00132   _file_major_ver = 0;
00133   _file_minor_ver = 0;
00134 
00135 #ifdef HAVE_OPENSSL
00136   // Get these values from the config file via an EncryptStreamBuf.
00137   EncryptStreamBuf tbuf;
00138   _encryption_algorithm = tbuf.get_algorithm();
00139   _encryption_key_length = tbuf.get_key_length();
00140 #endif
00141 }
00142 
00143 ////////////////////////////////////////////////////////////////////
00144 //     Function: Multifile::Destructor
00145 //       Access: Published
00146 //  Description:
00147 ////////////////////////////////////////////////////////////////////
00148 Multifile::
00149 ~Multifile() {
00150   close();
00151 }
00152 
00153 ////////////////////////////////////////////////////////////////////
00154 //     Function: Multifile::Copy Constructor
00155 //       Access: Private
00156 //  Description: Don't try to copy Multifiles.
00157 ////////////////////////////////////////////////////////////////////
00158 Multifile::
00159 Multifile(const Multifile &copy) :
00160   _read_filew(_read_file),
00161   _read_write_filew(_read_write_file)
00162 {
00163   nassertv(false);
00164 }
00165 
00166 ////////////////////////////////////////////////////////////////////
00167 //     Function: Multifile::Copy Assignment Operator
00168 //       Access: Private
00169 //  Description: Don't try to copy Multifiles.
00170 ////////////////////////////////////////////////////////////////////
00171 void Multifile::
00172 operator = (const Multifile &copy) {
00173   nassertv(false);
00174 }
00175 
00176 ////////////////////////////////////////////////////////////////////
00177 //     Function: Multifile::open_read
00178 //       Access: Published
00179 //  Description: Opens the named Multifile on disk for reading.  The
00180 //               Multifile index is read in, and the list of subfiles
00181 //               becomes available; individual subfiles may then be
00182 //               extracted or read, but the list of subfiles may not
00183 //               be modified.
00184 //
00185 //               Also see the version of open_read() which accepts an
00186 //               istream.  Returns true on success, false on failure.
00187 ////////////////////////////////////////////////////////////////////
00188 bool Multifile::
00189 open_read(const Filename &multifile_name, const streampos &offset) {
00190   close();
00191   Filename fname = multifile_name;
00192   fname.set_binary();
00193 
00194   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00195   PT(VirtualFile) vfile = vfs->get_file(fname);
00196   if (vfile == NULL) {
00197     return false;
00198   }
00199   istream *multifile_stream = vfile->open_read_file(false);
00200   if (multifile_stream == NULL) {
00201     return false;
00202   }
00203 
00204   _timestamp = vfile->get_timestamp();
00205   _timestamp_dirty = true;
00206   _read = new IStreamWrapper(multifile_stream, true);
00207   _owns_stream = true;
00208   _multifile_name = multifile_name;
00209   _offset = offset;
00210   return read_index();
00211 }
00212 
00213 ////////////////////////////////////////////////////////////////////
00214 //     Function: Multifile::open_read
00215 //       Access: Public
00216 //  Description: Opens an anonymous Multifile for reading using an
00217 //               istream.  There must be seek functionality via
00218 //               seekg() and tellg() on the istream.
00219 //
00220 //               If owns_pointer is true, then the Multifile assumes
00221 //               ownership of the stream pointer and will delete it
00222 //               when the multifile is closed, including if this
00223 //               function returns false.
00224 ////////////////////////////////////////////////////////////////////
00225 bool Multifile::
00226 open_read(IStreamWrapper *multifile_stream, bool owns_pointer,
00227           const streampos &offset) {
00228   close();
00229   _timestamp = time(NULL);
00230   _timestamp_dirty = true;
00231   _read = multifile_stream;
00232   _owns_stream = owns_pointer;
00233   _offset = offset;
00234   return read_index();
00235 }
00236 
00237 ////////////////////////////////////////////////////////////////////
00238 //     Function: Multifile::open_write
00239 //       Access: Published
00240 //  Description: Opens the named Multifile on disk for writing.  If
00241 //               there already exists a file by that name, it is
00242 //               truncated.  The Multifile is then prepared for
00243 //               accepting a brand new set of subfiles, which will be
00244 //               written to the indicated filename.  Individual
00245 //               subfiles may not be extracted or read.
00246 //
00247 //               Also see the version of open_write() which accepts an
00248 //               ostream.  Returns true on success, false on failure.
00249 ////////////////////////////////////////////////////////////////////
00250 bool Multifile::
00251 open_write(const Filename &multifile_name) {
00252   close();
00253   Filename fname = multifile_name;
00254   fname.set_binary();
00255   if (!fname.open_write(_write_file, true)) {
00256     return false;
00257   }
00258   _timestamp = time(NULL);
00259   _timestamp_dirty = true;
00260   _write = &_write_file;
00261   _multifile_name = multifile_name;
00262   return true;
00263 }
00264 
00265 ////////////////////////////////////////////////////////////////////
00266 //     Function: Multifile::open_write
00267 //       Access: Public
00268 //  Description: Opens an anonymous Multifile for writing using an
00269 //               ostream.  There must be seek functionality via
00270 //               seekp() and tellp() on the pstream.
00271 //
00272 //               If owns_pointer is true, then the Multifile assumes
00273 //               ownership of the stream pointer and will delete it
00274 //               when the multifile is closed, including if this
00275 //               function returns false.
00276 ////////////////////////////////////////////////////////////////////
00277 bool Multifile::
00278 open_write(ostream *multifile_stream, bool owns_pointer) {
00279   close();
00280   _timestamp = time(NULL);
00281   _timestamp_dirty = true;
00282   _write = multifile_stream;
00283   _owns_stream = owns_pointer;
00284   _write->seekp(0, ios::beg);
00285   return true;
00286 }
00287 
00288 ////////////////////////////////////////////////////////////////////
00289 //     Function: Multifile::open_read_write
00290 //       Access: Published
00291 //  Description: Opens the named Multifile on disk for reading and
00292 //               writing.  If there already exists a file by that
00293 //               name, its index is read.  Subfiles may be added or
00294 //               removed, and the resulting changes will be written to
00295 //               the named file.
00296 //
00297 //               Also see the version of open_read_write() which
00298 //               accepts an iostream.  Returns true on success, false
00299 //               on failure.
00300 ////////////////////////////////////////////////////////////////////
00301 bool Multifile::
00302 open_read_write(const Filename &multifile_name) {
00303   close();
00304   Filename fname = multifile_name;
00305   fname.set_binary();
00306   bool exists = fname.exists();
00307   if (!fname.open_read_write(_read_write_file)) {
00308     return false;
00309   }
00310   if (exists) {
00311     _timestamp = fname.get_timestamp();
00312   } else {
00313     _timestamp = time(NULL);
00314   }
00315   _timestamp_dirty = true;
00316   _read = &_read_write_filew;
00317   _write = &_read_write_file;
00318   _multifile_name = multifile_name;
00319 
00320   if (exists) {
00321     return read_index();
00322   } else {
00323     return true;
00324   }
00325 }
00326 
00327 ////////////////////////////////////////////////////////////////////
00328 //     Function: Multifile::open_read_write
00329 //       Access: Public
00330 //  Description: Opens an anonymous Multifile for reading and writing
00331 //               using an iostream.  There must be seek functionality
00332 //               via seekg()/seekp() and tellg()/tellp() on the
00333 //               iostream.
00334 //
00335 //               If owns_pointer is true, then the Multifile assumes
00336 //               ownership of the stream pointer and will delete it
00337 //               when the multifile is closed, including if this
00338 //               function returns false.
00339 ////////////////////////////////////////////////////////////////////
00340 bool Multifile::
00341 open_read_write(iostream *multifile_stream, bool owns_pointer) {
00342   close();
00343   _timestamp = time(NULL);
00344   _timestamp_dirty = true;
00345 
00346   // We don't support locking when opening a file in read-write mode,
00347   // because we don't bother with locking on write.  But we need to
00348   // have an IStreamWrapper to assign to the _read member, so we
00349   // create one on-the-fly here.
00350   _read = new StreamWrapper(multifile_stream, owns_pointer);
00351   _write = multifile_stream;
00352   _owns_stream = true;  // Because we own the StreamWrapper, above.
00353   _write->seekp(0, ios::beg);
00354 
00355   // Check whether the read stream is empty.
00356   multifile_stream->seekg(0, ios::end);
00357   if (multifile_stream->tellg() == (streampos)0) {
00358     // The read stream is empty, which is always valid.
00359     return true;
00360   }
00361 
00362   // The read stream is not empty, so we'd better have a valid
00363   // Multifile.
00364   return read_index();
00365 }
00366 
00367 ////////////////////////////////////////////////////////////////////
00368 //     Function: Multifile::close
00369 //       Access: Published
00370 //  Description: Closes the Multifile if it is open.  All changes are
00371 //               flushed to disk, and the file becomes invalid for
00372 //               further operations until the next call to open().
00373 ////////////////////////////////////////////////////////////////////
00374 void Multifile::
00375 close() {
00376   if (_new_scale_factor != _scale_factor) {
00377     // If we have changed the scale factor recently, we need to force
00378     // a repack.
00379     repack();
00380   } else {
00381     flush();
00382   }
00383 
00384   if (_owns_stream) {
00385     // We prefer to delete the IStreamWrapper over the ostream, if
00386     // possible.
00387     if (_read != (IStreamWrapper *)NULL) {
00388       delete _read;
00389     } else if (_write != (ostream *)NULL) {
00390       delete _write;
00391     }
00392   }
00393 
00394   _read = (IStreamWrapper *)NULL;
00395   _write = (ostream *)NULL;
00396   _offset = 0;
00397   _owns_stream = false;
00398   _next_index = 0;
00399   _last_index = 0;
00400   _needs_repack = false;
00401   _timestamp = 0;
00402   _timestamp_dirty = false;
00403   _scale_factor = 1;
00404   _new_scale_factor = 1;
00405   _encryption_flag = false;
00406   _file_major_ver = 0;
00407   _file_minor_ver = 0;
00408 
00409   _read_file.close();
00410   _write_file.close();
00411   _read_write_file.close();
00412   _multifile_name = Filename();
00413 
00414   clear_subfiles();
00415 }
00416 
00417 ////////////////////////////////////////////////////////////////////
00418 //     Function: Multifile::set_scale_factor
00419 //       Access: Published
00420 //  Description: Changes the internal scale factor for this Multifile.
00421 //
00422 //               This is normally 1, but it may be set to any
00423 //               arbitrary value (greater than zero) to support
00424 //               Multifile archives that exceed 4GB, if necessary.
00425 //               (Individual subfiles may still not exceed 4GB.)
00426 //
00427 //               All addresses within the file are rounded up to the
00428 //               next multiple of _scale_factor, and zeros are written
00429 //               to the file to fill the resulting gaps.  Then the
00430 //               address is divided by _scale_factor and written out
00431 //               as a 32-bit integer.  Thus, setting a scale factor of
00432 //               2 supports up to 8GB files, 3 supports 12GB files,
00433 //               etc.
00434 //
00435 //               Calling this function on an already-existing
00436 //               Multifile will have no immediate effect until a
00437 //               future call to repack() or close() (or until the
00438 //               Multifile is destructed).
00439 ////////////////////////////////////////////////////////////////////
00440 void Multifile::
00441 set_scale_factor(size_t scale_factor) {
00442   nassertv(is_write_valid());
00443   nassertv(scale_factor != (size_t)0);
00444 
00445   if (_next_index == (streampos)0) {
00446     // If it's a brand new Multifile, we can go ahead and set it
00447     // immediately.
00448     _scale_factor = scale_factor;
00449   } else {
00450     // Otherwise, we'd better have read access so we can repack it
00451     // later.
00452     nassertv(is_read_valid());
00453   }
00454 
00455   // Setting the _new_scale_factor different from the _scale_factor
00456   // will force a repack operation on close.
00457   _new_scale_factor = scale_factor;
00458 }
00459 
00460 ////////////////////////////////////////////////////////////////////
00461 //     Function: Multifile::add_subfile
00462 //       Access: Published
00463 //  Description: Adds a file on disk as a subfile to the Multifile.
00464 //               The file named by filename will be read and added to
00465 //               the Multifile at the next call to flush().  If there
00466 //               already exists a subfile with the indicated name, it
00467 //               is replaced without examining its contents (but see
00468 //               also update_subfile).
00469 //
00470 //               Returns the subfile name on success (it might have
00471 //               been modified slightly), or empty string on failure.
00472 ////////////////////////////////////////////////////////////////////
00473 string Multifile::
00474 add_subfile(const string &subfile_name, const Filename &filename,
00475             int compression_level) {
00476   nassertr(is_write_valid(), string());
00477 
00478   if (!filename.exists()) {
00479     return string();
00480   }
00481   string name = standardize_subfile_name(subfile_name);
00482   if (!name.empty()) {
00483     Subfile *subfile = new Subfile;
00484     subfile->_name = name;
00485     subfile->_source_filename = filename;
00486     subfile->_source_filename.set_binary();
00487     
00488     add_new_subfile(subfile, compression_level);
00489   }
00490 
00491   _timestamp = time(NULL);
00492   _timestamp_dirty = true;
00493 
00494   return name;
00495 }
00496 
00497 ////////////////////////////////////////////////////////////////////
00498 //     Function: Multifile::add_subfile
00499 //       Access: Public
00500 //  Description: Adds a file from a stream as a subfile to the Multifile.
00501 //               The indicated istream will be read and its contents
00502 //               added to the Multifile at the next call to flush().
00503 //
00504 //               Note that the istream must remain untouched and
00505 //               unused by any other code until flush() is called.  At
00506 //               that time, the Multifile will read the entire
00507 //               contents of the istream from the current file
00508 //               position to the end of the file.  Subsequently, the
00509 //               Multifile will *not* close or delete the istream.  It
00510 //               is the caller's responsibility to ensure that the
00511 //               istream pointer does not destruct during the lifetime
00512 //               of the Multifile.
00513 //
00514 //               Returns the subfile name on success (it might have
00515 //               been modified slightly), or empty string on failure.
00516 ////////////////////////////////////////////////////////////////////
00517 string Multifile::
00518 add_subfile(const string &subfile_name, istream *subfile_data,
00519             int compression_level) {
00520   nassertr(is_write_valid(), string());
00521 
00522   string name = standardize_subfile_name(subfile_name);
00523   if (!name.empty()) {
00524     Subfile *subfile = new Subfile;
00525     subfile->_name = name;
00526     subfile->_source = subfile_data;
00527     add_new_subfile(subfile, compression_level);
00528   }
00529 
00530   return name;
00531 }
00532 
00533 ////////////////////////////////////////////////////////////////////
00534 //     Function: Multifile::update_subfile
00535 //       Access: Published
00536 //  Description: Adds a file on disk to the subfile.  If a subfile
00537 //               already exists with the same name, its contents are
00538 //               compared byte-for-byte to the disk file, and it is
00539 //               replaced only if it is different; otherwise, the
00540 //               multifile is left unchanged.
00541 ////////////////////////////////////////////////////////////////////
00542 string Multifile::
00543 update_subfile(const string &subfile_name, const Filename &filename,
00544                int compression_level) {
00545   nassertr(is_write_valid(), string());
00546 
00547   if (!filename.exists()) {
00548     return string();
00549   }
00550   string name = standardize_subfile_name(subfile_name);
00551   if (!name.empty()) {
00552     int index = find_subfile(name);
00553     if (index >= 0) {
00554       // The subfile already exists; compare it to the source file.
00555       if (compare_subfile(index, filename)) {
00556         // The files are identical; do nothing.
00557         return name;
00558       }
00559     }
00560 
00561     // The subfile does not already exist or it is different from the
00562     // source file.  Add the new source file.
00563     Subfile *subfile = new Subfile;
00564     subfile->_name = name;
00565     subfile->_source_filename = filename;
00566     subfile->_source_filename.set_binary();
00567 
00568     add_new_subfile(subfile, compression_level);
00569   }
00570 
00571   _timestamp = time(NULL);
00572   _timestamp_dirty = true;
00573 
00574   return name;
00575 }
00576 
00577 #ifdef HAVE_OPENSSL
00578 ////////////////////////////////////////////////////////////////////
00579 //     Function: Multifile::add_signature
00580 //       Access: Published
00581 //  Description: Adds a new signature to the Multifile.  This
00582 //               signature associates the indicated certificate with
00583 //               the current contents of the Multifile.  When the
00584 //               Multifile is read later, the signature will still be
00585 //               present only if the Multifile is unchanged; any
00586 //               subsequent changes to the Multifile will
00587 //               automatically invalidate and remove the signature.
00588 //
00589 //               The chain filename may be empty if the certificate
00590 //               does not require an authenticating certificate chain
00591 //               (e.g. because it is self-signed).
00592 //
00593 //               The specified private key must match the certificate,
00594 //               and the Multifile must be open in read-write mode.
00595 //               The private key is only used for generating the
00596 //               signature; it is not written to the Multifile and
00597 //               cannot be retrieved from the Multifile later.
00598 //               (However, the certificate *can* be retrieved from the
00599 //               Multifile later, to identify the entity that created
00600 //               the signature.)
00601 //
00602 //               This implicitly causes a repack() operation if one is
00603 //               needed.  Returns true on success, false on failure.
00604 //
00605 //               This flavor of add_signature() reads the certificate
00606 //               and private key from a PEM-formatted file, for
00607 //               instance as generated by the openssl command.  If the
00608 //               private key file is password-encrypted, the third
00609 //               parameter will be used as the password to decrypt it.
00610 ////////////////////////////////////////////////////////////////////
00611 bool Multifile::
00612 add_signature(const Filename &certificate, const Filename &chain,
00613               const Filename &pkey, const string &password) {
00614   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00615 
00616   if (chain.empty() && pkey.empty()) {
00617     // If the second two filenames are empty, assume we're going for
00618     // the composite mode, where everything's stuffed into the first
00619     // file.
00620     return add_signature(certificate, password);
00621   }
00622 
00623   CertChain cert_chain;
00624 
00625   // Read the certificate file from VFS.  First, read the complete
00626   // file into memory.
00627   string certificate_data;
00628   if (!vfs->read_file(certificate, certificate_data, true)) {
00629     express_cat.info()
00630       << "Could not read " << certificate << ".\n";
00631     return false;
00632   }
00633 
00634   // Create an in-memory BIO to read the "file" from the buffer.
00635   BIO *certificate_mbio = BIO_new_mem_buf((void *)certificate_data.data(), certificate_data.size());
00636   X509 *x509 = PEM_read_bio_X509(certificate_mbio, NULL, NULL, (void *)"");
00637   BIO_free(certificate_mbio);
00638   if (x509 == NULL) {
00639     express_cat.info()
00640       << "Could not read certificate in " << certificate << ".\n";
00641     return false;
00642   }
00643 
00644   // Store the first X509--the actual certificate--as the first record
00645   // in our CertChain object.
00646   cert_chain.push_back(CertRecord(x509));
00647 
00648   // Read the rest of the certificates in the chain file.
00649   if (!chain.empty()) {
00650     string chain_data;
00651     if (!vfs->read_file(chain, chain_data, true)) {
00652       express_cat.info()
00653         << "Could not read " << chain << ".\n";
00654       return false;
00655     }
00656 
00657     BIO *chain_mbio = BIO_new_mem_buf((void *)chain_data.data(), chain_data.size());
00658     X509 *c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)"");
00659     while (c != NULL) {
00660       cert_chain.push_back(c);
00661       c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)"");
00662     }
00663     BIO_free(chain_mbio);
00664 
00665     if (cert_chain.size() == 1) {
00666       express_cat.info()
00667         << "Could not read certificate chain in " << chain << ".\n";
00668       return false;
00669     }
00670   }
00671 
00672   // Now do the same thing with the private key.  This one may be
00673   // password-encrypted on disk.
00674   string pkey_data;
00675   if (!vfs->read_file(pkey, pkey_data, true)) {
00676     express_cat.info()
00677       << "Could not read " << pkey << ".\n";
00678     return false;
00679   }
00680 
00681   BIO *pkey_mbio = BIO_new_mem_buf((void *)pkey_data.data(), pkey_data.size());
00682   EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, NULL, NULL,
00683                                                (void *)password.c_str());
00684   BIO_free(pkey_mbio);
00685   if (evp_pkey == NULL) {
00686     express_cat.info()
00687       << "Could not read private key in " << pkey << ".\n";
00688     return false;
00689   }
00690 
00691   bool result = add_signature(cert_chain, evp_pkey);
00692 
00693   EVP_PKEY_free(evp_pkey);
00694 
00695   return result;
00696 }
00697 #endif  // HAVE_OPENSSL
00698 
00699 #ifdef HAVE_OPENSSL
00700 ////////////////////////////////////////////////////////////////////
00701 //     Function: Multifile::add_signature
00702 //       Access: Published
00703 //  Description: Adds a new signature to the Multifile.  This
00704 //               signature associates the indicated certificate with
00705 //               the current contents of the Multifile.  When the
00706 //               Multifile is read later, the signature will still be
00707 //               present only if the Multifile is unchanged; any
00708 //               subsequent changes to the Multifile will
00709 //               automatically invalidate and remove the signature.
00710 //
00711 //               This flavor of add_signature() reads the certificate,
00712 //               private key, and certificate chain from the same
00713 //               PEM-formatted file.  It takes the first private key
00714 //               found as the intended key, and then uses the first
00715 //               certificate found that matches that key as the
00716 //               signing certificate.  Any other certificates in the
00717 //               file are taken to be part of the chain.
00718 ////////////////////////////////////////////////////////////////////
00719 bool Multifile::
00720 add_signature(const Filename &composite, const string &password) {
00721   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00722 
00723   // First, read the complete file into memory.
00724   string composite_data;
00725   if (!vfs->read_file(composite, composite_data, true)) {
00726     express_cat.info()
00727       << "Could not read " << composite << ".\n";
00728     return false;
00729   }
00730 
00731   // Get the private key.
00732   BIO *pkey_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size());
00733   EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, NULL, NULL,
00734                                                (void *)password.c_str());
00735   BIO_free(pkey_mbio);
00736   if (evp_pkey == NULL) {
00737     express_cat.info()
00738       << "Could not read private key in " << composite << ".\n";
00739     return false;
00740   }
00741 
00742   // Now read all of the certificates.
00743   CertChain cert_chain;
00744 
00745   BIO *chain_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size());
00746   X509 *c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)"");
00747   while (c != NULL) {
00748     cert_chain.push_back(c);
00749     c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)"");
00750   }
00751   BIO_free(chain_mbio);
00752 
00753   if (cert_chain.empty()) {
00754     express_cat.info()
00755       << "Could not read certificates in " << composite << ".\n";
00756     return false;
00757   }
00758 
00759   // Now find the certificate that matches the signature, and move it
00760   // to the front of the chain.
00761   size_t i;
00762   bool found_match = false;
00763   for (i = 0; i < cert_chain.size(); ++i) {
00764     X509 *c = cert_chain[i]._cert;
00765     if (X509_check_private_key(c, evp_pkey)) {
00766       found_match = true;
00767       if (i != 0) {
00768         // Move this entry to the beginning.
00769         cert_chain.insert(cert_chain.begin(), cert_chain[i]);
00770         cert_chain.erase(cert_chain.begin() + i + 1);
00771       }
00772       break;
00773     }
00774   }
00775 
00776   if (!found_match) {
00777     express_cat.info()
00778       << "No certificates in " << composite << " match key.\n";
00779     return false;
00780   }
00781 
00782   bool result = add_signature(cert_chain, evp_pkey);
00783 
00784   EVP_PKEY_free(evp_pkey);
00785 
00786   return result;
00787 }
00788 #endif  // HAVE_OPENSSL
00789 
00790 #ifdef HAVE_OPENSSL
00791 ////////////////////////////////////////////////////////////////////
00792 //     Function: Multifile::add_signature
00793 //       Access: Published
00794 //  Description: Adds a new signature to the Multifile.  This
00795 //               signature associates the indicated certificate with
00796 //               the current contents of the Multifile.  When the
00797 //               Multifile is read later, the signature will still be
00798 //               present only if the Multifile is unchanged; any
00799 //               subsequent changes to the Multifile will
00800 //               automatically invalidate and remove the signature.
00801 //
00802 //               If chain is non-NULL, it represents the certificate
00803 //               chain that validates the certificate.
00804 //
00805 //               The specified private key must match the certificate,
00806 //               and the Multifile must be open in read-write mode.
00807 //               The private key is only used for generating the
00808 //               signature; it is not written to the Multifile and
00809 //               cannot be retrieved from the Multifile later.
00810 //               (However, the certificate *can* be retrieved from the
00811 //               Multifile later, to identify the entity that created
00812 //               the signature.)
00813 //
00814 //               This implicitly causes a repack() operation if one is
00815 //               needed.  Returns true on success, false on failure.
00816 ////////////////////////////////////////////////////////////////////
00817 bool Multifile::
00818 add_signature(X509 *certificate, STACK_OF(X509) *chain, EVP_PKEY *pkey) {
00819   // Convert the certificate and chain into our own CertChain
00820   // structure.
00821   CertChain cert_chain;
00822   cert_chain.push_back(CertRecord(certificate));
00823   if (chain != NULL) {
00824     int num = sk_X509_num(chain);
00825     for (int i = 0; i < num; ++i) {
00826       cert_chain.push_back(CertRecord((X509 *)sk_X509_value(chain, i)));
00827     }
00828   }
00829 
00830   return add_signature(cert_chain, pkey);
00831 }
00832 #endif  // HAVE_OPENSSL
00833 
00834 #ifdef HAVE_OPENSSL
00835 ////////////////////////////////////////////////////////////////////
00836 //     Function: Multifile::add_signature
00837 //       Access: Published
00838 //  Description: Adds a new signature to the Multifile.  This
00839 //               signature associates the indicated certificate with
00840 //               the current contents of the Multifile.  When the
00841 //               Multifile is read later, the signature will still be
00842 //               present only if the Multifile is unchanged; any
00843 //               subsequent changes to the Multifile will
00844 //               automatically invalidate and remove the signature.
00845 //
00846 //               The signature certificate is the first certificate on
00847 //               the CertChain object.  Any remaining certificates are
00848 //               support certificates to authenticate the first one.
00849 //
00850 //               The specified private key must match the certificate,
00851 //               and the Multifile must be open in read-write mode.
00852 //               The private key is only used for generating the
00853 //               signature; it is not written to the Multifile and
00854 //               cannot be retrieved from the Multifile later.
00855 //               (However, the certificate *can* be retrieved from the
00856 //               Multifile later, to identify the entity that created
00857 //               the signature.)
00858 //
00859 //               This implicitly causes a repack() operation if one is
00860 //               needed.  Returns true on success, false on failure.
00861 ////////////////////////////////////////////////////////////////////
00862 bool Multifile::
00863 add_signature(const Multifile::CertChain &cert_chain, EVP_PKEY *pkey) {
00864   if (_needs_repack) {
00865     if (!repack()) {
00866       return false;
00867     }
00868   } else {
00869     if (!flush()) {
00870       return false;
00871     }
00872   }
00873 
00874   if (cert_chain.empty()) {
00875     express_cat.info()
00876       << "No certificate given.\n";
00877     return false;
00878   }
00879 
00880   if (pkey == NULL) {
00881     express_cat.info()
00882       << "No private key given.\n";
00883     return false;
00884   }
00885 
00886   if (!X509_check_private_key(cert_chain[0]._cert, pkey)) {
00887     express_cat.info()
00888       << "Private key does not match certificate.\n"; 
00889     return false;
00890   }
00891   
00892   // Now encode that list of certs to a stream in DER form.
00893   stringstream der_stream;
00894   StreamWriter der_writer(der_stream);
00895   der_writer.add_uint32(cert_chain.size());
00896 
00897   CertChain::const_iterator ci;
00898   for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) {
00899     X509 *cert = (*ci)._cert;
00900 
00901     int der_len = i2d_X509(cert, NULL);
00902     unsigned char *der_buf = new unsigned char[der_len];
00903     unsigned char *p = der_buf;
00904     i2d_X509(cert, &p);
00905     der_writer.append_data(der_buf, der_len);
00906     delete[] der_buf;
00907   }
00908 
00909   // Create a temporary Subfile for writing out the signature.
00910   der_stream.seekg(0);
00911   Subfile *subfile = new Subfile;
00912   subfile->_pkey = pkey;
00913   subfile->_flags |= SF_signature;
00914   subfile->_source = &der_stream;
00915 
00916   // Write the new Subfile at the end.  The cert_special subfiles
00917   // always go at the end, because they're not the part of the file
00918   // that's signed.
00919   nassertr(_new_subfiles.empty(), false);
00920   _new_subfiles.push_back(subfile);
00921   bool result = flush();
00922   
00923   delete subfile;
00924 
00925   return result;
00926 }
00927 #endif  // HAVE_OPENSSL
00928 
00929 #ifdef HAVE_OPENSSL
00930 ////////////////////////////////////////////////////////////////////
00931 //     Function: Multifile::get_num_signatures
00932 //       Access: Published
00933 //  Description: Returns the number of matching signatures found on
00934 //               the Multifile.  These signatures may be iterated via
00935 //               get_signature() and related methods.
00936 //
00937 //               A signature on this list is guaranteed to match the
00938 //               Multifile contents, proving that the Multifile has
00939 //               been unmodified since the signature was applied.
00940 //               However, this does not guarantee that the certificate
00941 //               itself is actually from who it says it is from; only
00942 //               that it matches the Multifile contents.  See
00943 //               validate_signature_certificate() to authenticate a
00944 //               particular certificate.
00945 ////////////////////////////////////////////////////////////////////
00946 int Multifile::
00947 get_num_signatures() const {
00948   ((Multifile *)this)->check_signatures();
00949   return _signatures.size();
00950 }
00951 #endif  // HAVE_OPENSSL
00952 
00953 #ifdef HAVE_OPENSSL
00954 ////////////////////////////////////////////////////////////////////
00955 //     Function: Multifile::get_signature
00956 //       Access: Published
00957 //  Description: Returns the nth signature found on the Multifile.
00958 //               See the comments in get_num_signatures().
00959 ////////////////////////////////////////////////////////////////////
00960 const Multifile::CertChain &Multifile::
00961 get_signature(int n) const {
00962   static CertChain error_chain;
00963   nassertr(n >= 0 && n < (int)_signatures.size(), error_chain);
00964   return _signatures[n];
00965 }
00966 #endif  // HAVE_OPENSSL
00967 
00968 #ifdef HAVE_OPENSSL
00969 ////////////////////////////////////////////////////////////////////
00970 //     Function: Multifile::get_signature_subject_name
00971 //       Access: Published
00972 //  Description: Returns the "subject name" for the nth signature found
00973 //               on the Multifile.  This is a string formatted
00974 //               according to RFC2253 that should more-or-less
00975 //               identify a particular certificate; when paired with
00976 //               the public key (see get_signature_public_key()), it
00977 //               can uniquely identify a certificate.  See the
00978 //               comments in get_num_signatures().
00979 ////////////////////////////////////////////////////////////////////
00980 string Multifile::
00981 get_signature_subject_name(int n) const {
00982   const CertChain &cert_chain = get_signature(n);
00983   nassertr(!cert_chain.empty(), string());
00984 
00985   X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert);
00986   if (xname != NULL) {
00987     // We use "print" to dump the output to a memory BIO.  Is
00988     // there an easier way to extract the X509_NAME text?  Curse
00989     // these incomplete docs.
00990     BIO *mbio = BIO_new(BIO_s_mem());
00991     X509_NAME_print_ex(mbio, xname, 0, XN_FLAG_RFC2253);
00992 
00993     char *pp;
00994     long pp_size = BIO_get_mem_data(mbio, &pp);
00995     string name(pp, pp_size);
00996     BIO_free(mbio);
00997     return name;
00998   }
00999 
01000   return string();
01001 }
01002 #endif  // HAVE_OPENSSL
01003 
01004 #ifdef HAVE_OPENSSL
01005 ////////////////////////////////////////////////////////////////////
01006 //     Function: Multifile::get_signature_friendly_name
01007 //       Access: Published
01008 //  Description: Returns a "friendly name" for the nth signature found
01009 //               on the Multifile.  This attempts to extract out the
01010 //               most meaningful part of the subject name.  It returns
01011 //               the emailAddress, if it is defined; otherwise, it
01012 //               returns the commonName.
01013 
01014 //               See the comments in get_num_signatures().
01015 ////////////////////////////////////////////////////////////////////
01016 string Multifile::
01017 get_signature_friendly_name(int n) const {
01018   const CertChain &cert_chain = get_signature(n);
01019   nassertr(!cert_chain.empty(), string());
01020 
01021   static const int nid_choices[] = {
01022     NID_pkcs9_emailAddress,
01023     NID_commonName,
01024     -1,
01025   };
01026 
01027   // Choose the first NID that exists on the cert.
01028   for (int ni = 0; nid_choices[ni] != -1; ++ni) {
01029     int nid = nid_choices[ni];
01030 
01031     // A complex OpenSSL interface to extract out the name in utf-8.
01032     X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert);
01033     if (xname != NULL) {
01034       int pos = X509_NAME_get_index_by_NID(xname, nid, -1);
01035       if (pos != -1) {
01036         // We just get the first common name.  I guess it's possible to
01037         // have more than one; not sure what that means in this context.
01038         X509_NAME_ENTRY *xentry = X509_NAME_get_entry(xname, pos);
01039         if (xentry != NULL) {
01040           ASN1_STRING *data = X509_NAME_ENTRY_get_data(xentry);
01041           if (data != NULL) {
01042             // We use "print" to dump the output to a memory BIO.  Is
01043             // there an easier way to decode the ASN1_STRING?  Curse
01044             // these incomplete docs.
01045             BIO *mbio = BIO_new(BIO_s_mem());
01046             ASN1_STRING_print_ex(mbio, data, ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB);
01047             
01048             char *pp;
01049             long pp_size = BIO_get_mem_data(mbio, &pp);
01050             string name(pp, pp_size);
01051             BIO_free(mbio);
01052             return name;
01053           }
01054         }
01055       }
01056     }
01057   }
01058 
01059   return string();
01060 }
01061 #endif  // HAVE_OPENSSL
01062 
01063 #ifdef HAVE_OPENSSL
01064 ////////////////////////////////////////////////////////////////////
01065 //     Function: Multifile::get_signature_public_key
01066 //       Access: Published
01067 //  Description: Returns the public key used for the nth signature
01068 //               found on the Multifile.  This is encoded in DER form
01069 //               and returned as a string of hex digits.
01070 //
01071 //               This can be used, in conjunction with the subject
01072 //               name (see get_signature_subject_name()), to uniquely
01073 //               identify a particular certificate and its subsequent
01074 //               reissues.  See the comments in get_num_signatures().
01075 ////////////////////////////////////////////////////////////////////
01076 string Multifile::
01077 get_signature_public_key(int n) const {
01078   const CertChain &cert_chain = get_signature(n);
01079   nassertr(!cert_chain.empty(), string());
01080 
01081   EVP_PKEY *pkey = X509_get_pubkey(cert_chain[0]._cert);
01082   if (pkey != NULL) {
01083     int key_len = i2d_PublicKey(pkey, NULL);
01084     unsigned char *key_buf = new unsigned char[key_len];
01085     unsigned char *p = key_buf;
01086     i2d_PublicKey(pkey, &p);
01087     string result;
01088     for (int i = 0; i < key_len; ++i) {
01089       result += tohex(key_buf[i] >> 4);
01090       result += tohex(key_buf[i]);
01091     }
01092     delete[] key_buf;
01093     return result;
01094   }
01095 
01096   return string();
01097 }
01098 #endif  // HAVE_OPENSSL
01099 
01100 #ifdef HAVE_OPENSSL
01101 ////////////////////////////////////////////////////////////////////
01102 //     Function: Multifile::write_signature_certificate
01103 //       Access: Published
01104 //  Description: Writes the certificate for the nth signature, in
01105 //               verbose form, to the indicated stream.  See the
01106 //               comments in get_num_signatures().
01107 ////////////////////////////////////////////////////////////////////
01108 void Multifile::
01109 write_signature_certificate(int n, ostream &out) const {
01110   const CertChain &cert_chain = get_signature(n);
01111   nassertv(!cert_chain.empty());
01112 
01113   BIO *mbio = BIO_new(BIO_s_mem());
01114   X509_print(mbio, cert_chain[0]._cert);
01115 
01116   char *pp;
01117   long pp_size = BIO_get_mem_data(mbio, &pp);
01118   out.write(pp, pp_size);
01119   BIO_free(mbio);
01120 }
01121 #endif  // HAVE_OPENSSL
01122 
01123 #ifdef HAVE_OPENSSL
01124 ////////////////////////////////////////////////////////////////////
01125 //     Function: Multifile::validate_signature_certificate
01126 //       Access: Published
01127 //  Description: Checks that the certificate used for the nth
01128 //               signature is a valid, authorized certificate with
01129 //               some known certificate authority.  Returns 0 if it
01130 //               is valid, -1 if there is some error, or the
01131 //               corresponding OpenSSL error code if it is invalid,
01132 //               out-of-date, or self-signed.
01133 ////////////////////////////////////////////////////////////////////
01134 int Multifile::
01135 validate_signature_certificate(int n) const {
01136   int verify_result = -1;
01137 
01138   const CertChain &chain = get_signature(n);
01139   nassertr(!chain.empty(), false);
01140 
01141   OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
01142 
01143   // Copy our CertChain structure into an X509 pointer and
01144   // accompanying STACK_OF(X509) pointer.
01145   X509 *x509 = chain[0]._cert;
01146   STACK_OF(X509) *stack = NULL;
01147   if (chain.size() > 1) {
01148     stack = sk_X509_new(NULL);
01149     for (size_t n = 1; n < chain.size(); ++n) {
01150       sk_X509_push(stack, chain[n]._cert);
01151     }
01152   }
01153 
01154   // Create the X509_STORE_CTX for verifying the cert and chain.
01155   X509_STORE_CTX *ctx = X509_STORE_CTX_new();
01156   X509_STORE_CTX_init(ctx, sslw->get_x509_store(), x509, stack);
01157   X509_STORE_CTX_set_cert(ctx, x509);
01158 
01159   if (X509_verify_cert(ctx)) {
01160     verify_result = 0;
01161   } else {
01162     verify_result = X509_STORE_CTX_get_error(ctx);
01163   }
01164 
01165   if (express_cat.is_debug()) {
01166     express_cat.debug()
01167       << get_signature_subject_name(n) << ": validate " << verify_result
01168       << "\n";
01169   }
01170 
01171   sk_X509_free(stack);
01172   X509_STORE_CTX_cleanup(ctx);
01173   X509_STORE_CTX_free(ctx);
01174 
01175   return verify_result;
01176 }
01177 #endif // HAVE_OPENSSL
01178 
01179 ////////////////////////////////////////////////////////////////////
01180 //     Function: Multifile::flush
01181 //       Access: Published
01182 //  Description: Writes all contents of the Multifile to disk.  Until
01183 //               flush() is called, add_subfile() and remove_subfile()
01184 //               do not actually do anything to disk.  At this point,
01185 //               all of the recently-added subfiles are read and their
01186 //               contents are added to the end of the Multifile, and
01187 //               the recently-removed subfiles are marked gone from
01188 //               the Multifile.
01189 //
01190 //               This may result in a suboptimal index.  To guarantee
01191 //               that the index is written at the beginning of the
01192 //               file, call repack() instead of flush().
01193 //
01194 //               It is not necessary to call flush() explicitly unless
01195 //               you are concerned about reading the recently-added
01196 //               subfiles immediately.
01197 //
01198 //               Returns true on success, false on failure.
01199 ////////////////////////////////////////////////////////////////////
01200 bool Multifile::
01201 flush() {
01202   if (!is_write_valid()) {
01203     return false;
01204   }
01205 
01206   bool new_file = (_next_index == (streampos)0);
01207   if (new_file) {
01208     // If we don't have an index yet, we don't have a header.  Write
01209     // the header.
01210     if (!write_header()) {
01211       return false;
01212     }
01213 
01214   } else {
01215     if (_file_minor_ver != _current_minor_ver) {
01216       // If we *do* have an index already, but this is an old version
01217       // multifile, we have to completely rewrite it anyway.
01218       return repack();
01219     }
01220   }
01221 
01222   nassertr(_write != (ostream *)NULL, false);
01223 
01224   // First, mark out all of the removed subfiles.
01225   PendingSubfiles::iterator pi;
01226   for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
01227     Subfile *subfile = (*pi);
01228     subfile->rewrite_index_flags(*_write);
01229     delete subfile;
01230   }
01231   _removed_subfiles.clear();
01232 
01233   bool wrote_ok = true;
01234 
01235   if (!_new_subfiles.empty() || new_file) {
01236     // Add a few more files to the end.  We always add subfiles at the
01237     // end of the multifile, so go there first.
01238     sort(_new_subfiles.begin(), _new_subfiles.end(), IndirectLess<Subfile>());
01239     if (_last_index != (streampos)0) {
01240       _write->seekp(0, ios::end);
01241       if (_write->fail()) {
01242         express_cat.info()
01243           << "Unable to seek Multifile " << _multifile_name << ".\n";
01244         return false;
01245       }
01246       _next_index = _write->tellp();
01247       _next_index = pad_to_streampos(_next_index);
01248 
01249       // And update the forward link from the last_index to point to
01250       // this new index location.
01251       _write->seekp(_last_index);
01252       StreamWriter writer(_write, false);
01253       writer.add_uint32(streampos_to_word(_next_index));
01254     }
01255 
01256     _write->seekp(_next_index);
01257     nassertr(_next_index == _write->tellp(), false);
01258     
01259     // Ok, here we are at the end of the file.  Write out the
01260     // recently-added subfiles here.  First, count up the index size.
01261     for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
01262       Subfile *subfile = (*pi);
01263       _last_index = _next_index;
01264       _next_index = subfile->write_index(*_write, _next_index, this);
01265       nassertr(_next_index == _write->tellp(), false);
01266       _next_index = pad_to_streampos(_next_index);
01267       nassertr(_next_index == _write->tellp(), false);
01268     }
01269     
01270     // Now we're at the end of the index.  Write a 0 here to mark the
01271     // end.
01272     StreamWriter writer(_write, false);
01273     writer.add_uint32(0);
01274     _next_index += 4;
01275     nassertr(_next_index == _write->tellp(), false);
01276     _next_index = pad_to_streampos(_next_index);
01277 
01278     // All right, now write out each subfile's data.
01279     for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
01280       Subfile *subfile = (*pi);
01281 
01282       if (_read != (IStreamWrapper *)NULL) {
01283         _read->acquire();
01284         _next_index = subfile->write_data(*_write, _read->get_istream(),
01285                                           _next_index, this);
01286         _read->release();
01287 
01288       } else {
01289         _next_index = subfile->write_data(*_write, NULL, _next_index, this);
01290       }
01291 
01292       nassertr(_next_index == _write->tellp(), false);
01293       _next_index = pad_to_streampos(_next_index);
01294       if (subfile->is_data_invalid()) {
01295         wrote_ok = false;
01296       }
01297 
01298       if (!subfile->is_cert_special()) {
01299         _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos());
01300       }
01301       nassertr(_next_index == _write->tellp(), false);
01302     }
01303     
01304     // Now go back and fill in the proper addresses for the data start.
01305     // We didn't do it in the first pass, because we don't really want
01306     // to keep all those file handles open, and so we didn't have to
01307     // determine each file's length ahead of time.
01308     for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
01309       Subfile *subfile = (*pi);
01310       subfile->rewrite_index_data_start(*_write, this);
01311     }
01312 
01313     _new_subfiles.clear();
01314   }
01315 
01316   // Also update the overall timestamp.
01317   if (_timestamp_dirty) {
01318     nassertr(!_write->fail(), false);
01319     static const size_t timestamp_pos = _header_prefix.size() + _header_size + 2 + 2 + 4;
01320     _write->seekp(timestamp_pos);
01321     nassertr(!_write->fail(), false);
01322     
01323     StreamWriter writer(*_write);
01324     if (_record_timestamp) {
01325       writer.add_uint32(_timestamp);
01326     } else {
01327       writer.add_uint32(0);
01328     }
01329     _timestamp_dirty = false;
01330   }
01331 
01332   _write->flush();
01333   if (!wrote_ok || _write->fail()) {
01334     express_cat.info()
01335       << "Unable to update Multifile " << _multifile_name << ".\n";
01336     close();
01337     return false;
01338   }
01339 
01340   return true;
01341 }
01342 
01343 ////////////////////////////////////////////////////////////////////
01344 //     Function: Multifile::repack
01345 //       Access: Published
01346 //  Description: Forces a complete rewrite of the Multifile and all of
01347 //               its contents, so that its index will appear at the
01348 //               beginning of the file with all of the subfiles listed
01349 //               in alphabetical order.  This is considered optimal
01350 //               for reading, and is the standard configuration; but
01351 //               it is not essential to do this.
01352 //
01353 //               It is only valid to call this if the Multifile was
01354 //               opened using open_read_write() and an explicit
01355 //               filename, rather than an iostream.  Also, we must
01356 //               have write permission to the directory containing the
01357 //               Multifile.
01358 //
01359 //               Returns true on success, false on failure.
01360 ////////////////////////////////////////////////////////////////////
01361 bool Multifile::
01362 repack() {
01363   if (_next_index == (streampos)0) {
01364     // If the Multifile hasn't yet been written, this is really just a
01365     // flush operation.
01366     _needs_repack = false;
01367     return flush();
01368   }
01369 
01370   nassertr(is_write_valid() && is_read_valid(), false);
01371   nassertr(!_multifile_name.empty(), false);
01372 
01373   // First, we open a temporary filename to copy the Multifile to.
01374   Filename dirname = _multifile_name.get_dirname();
01375   if (dirname.empty()) {
01376     dirname = ".";
01377   }
01378   Filename temp_filename = Filename::temporary(dirname, "mftemp");
01379   temp_filename.set_binary();
01380   pofstream temp;
01381   if (!temp_filename.open_write(temp)) {
01382     express_cat.info()
01383       << "Unable to open temporary file " << temp_filename << "\n";
01384     return false;
01385   }
01386 
01387   // Now we scrub our internal structures so it looks like we're a
01388   // brand new Multifile.
01389   PendingSubfiles::iterator pi;
01390   for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
01391     Subfile *subfile = (*pi);
01392     delete subfile;
01393   }
01394   _removed_subfiles.clear();
01395   _new_subfiles.clear();
01396   copy(_subfiles.begin(), _subfiles.end(), back_inserter(_new_subfiles));
01397   _next_index = 0;
01398   _last_index = 0;
01399   _last_data_byte = 0;
01400   _scale_factor = _new_scale_factor;
01401 
01402   // And we write our contents to our new temporary file.
01403   _write = &temp;
01404   if (!flush()) {
01405     temp.close();
01406     temp_filename.unlink();
01407     return false;
01408   }
01409 
01410   // Now close everything, and move the temporary file back over our
01411   // original file.
01412   Filename orig_name = _multifile_name;
01413   temp.close();
01414   close();
01415   orig_name.unlink();
01416   if (!temp_filename.rename_to(orig_name)) {
01417     express_cat.info()
01418       << "Unable to rename temporary file " << temp_filename << " to "
01419       << orig_name << ".\n";
01420     return false;
01421   }
01422 
01423   if (!open_read_write(orig_name)) {
01424     express_cat.info()
01425       << "Unable to read newly repacked " << _multifile_name 
01426       << ".\n";
01427     return false;
01428   }
01429 
01430   return true;
01431 }
01432 
01433 ////////////////////////////////////////////////////////////////////
01434 //     Function: Multifile::get_num_subfiles
01435 //       Access: Published
01436 //  Description: Returns the number of subfiles within the Multifile.
01437 //               The subfiles may be accessed in alphabetical order by
01438 //               iterating through [0 .. get_num_subfiles()).
01439 ////////////////////////////////////////////////////////////////////
01440 int Multifile::
01441 get_num_subfiles() const {
01442   return _subfiles.size();
01443 }
01444 
01445 ////////////////////////////////////////////////////////////////////
01446 //     Function: Multifile::find_subfile
01447 //       Access: Published
01448 //  Description: Returns the index of the subfile with the indicated
01449 //               name, or -1 if the named subfile is not within the
01450 //               Multifile.
01451 ////////////////////////////////////////////////////////////////////
01452 int Multifile::
01453 find_subfile(const string &subfile_name) const {
01454   Subfile find_subfile;
01455   find_subfile._name = standardize_subfile_name(subfile_name);
01456   Subfiles::const_iterator fi;
01457   fi = _subfiles.find(&find_subfile);
01458   if (fi == _subfiles.end()) {
01459     // Not present.
01460     return -1;
01461   }
01462   return (fi - _subfiles.begin());
01463 }
01464 
01465 ////////////////////////////////////////////////////////////////////
01466 //     Function: Multifile::has_directory
01467 //       Access: Published
01468 //  Description: Returns true if the indicated subfile name is the
01469 //               directory prefix to one or more files within the
01470 //               Multifile.  That is, the Multifile contains at least
01471 //               one file named "subfile_name/...".
01472 ////////////////////////////////////////////////////////////////////
01473 bool Multifile::
01474 has_directory(const string &subfile_name) const {
01475   string prefix = subfile_name;
01476   if (!prefix.empty()) {
01477     prefix += '/';
01478   }
01479   Subfile find_subfile;
01480   find_subfile._name = prefix;
01481   Subfiles::const_iterator fi;
01482   fi = _subfiles.upper_bound(&find_subfile);
01483   if (fi == _subfiles.end()) {
01484     // Not present.
01485     return false;
01486   }
01487 
01488   // At least one subfile exists whose name sorts after prefix.  If it
01489   // contains prefix as the initial substring, then we have a match.
01490   Subfile *subfile = (*fi);
01491   return (subfile->_name.length() > prefix.length() &&
01492           subfile->_name.substr(0, prefix.length()) == prefix);
01493 }
01494 
01495 ////////////////////////////////////////////////////////////////////
01496 //     Function: Multifile::scan_directory
01497 //       Access: Published
01498 //  Description: Considers subfile_name to be the name of a
01499 //               subdirectory within the Multifile, but not a file
01500 //               itself; fills the given vector up with the sorted list
01501 //               of subdirectories or files within the named
01502 //               directory.
01503 //
01504 //               Note that directories do not exist explicitly within
01505 //               a Multifile; this just checks for the existence of
01506 //               files with the given initial prefix.
01507 //
01508 //               Returns true if successful, false otherwise.
01509 ////////////////////////////////////////////////////////////////////
01510 bool Multifile::
01511 scan_directory(vector_string &contents, const string &subfile_name) const {
01512   string prefix = subfile_name;
01513   if (!prefix.empty()) {
01514     prefix += '/';
01515   }
01516   Subfile find_subfile;
01517   find_subfile._name = prefix;
01518   Subfiles::const_iterator fi;
01519   fi = _subfiles.upper_bound(&find_subfile);
01520 
01521   string previous = "";
01522   while (fi != _subfiles.end()) {
01523     Subfile *subfile = (*fi);
01524     if (!(subfile->_name.length() > prefix.length() &&
01525           subfile->_name.substr(0, prefix.length()) == prefix)) {
01526       // We've reached the end of the list of subfiles beneath the
01527       // indicated directory prefix.
01528       return true;
01529     }
01530 
01531     size_t slash = subfile->_name.find('/', prefix.length());
01532     string basename = subfile->_name.substr(prefix.length(), slash - prefix.length());
01533     if (basename != previous) {
01534       contents.push_back(basename);
01535       previous = basename;
01536     }
01537     ++fi;
01538   }
01539 
01540   return true;
01541 }
01542 
01543 ////////////////////////////////////////////////////////////////////
01544 //     Function: Multifile::remove_subfile
01545 //       Access: Published
01546 //  Description: Removes the nth subfile from the Multifile.  This
01547 //               will cause all subsequent index numbers to decrease
01548 //               by one.  The file will not actually be removed from
01549 //               the disk until the next call to flush().
01550 //
01551 //               Note that this does not actually remove the data from
01552 //               the indicated subfile; it simply removes it from the
01553 //               index.  The Multifile will not be reduced in size
01554 //               after this operation, until the next call to
01555 //               repack().
01556 ////////////////////////////////////////////////////////////////////
01557 void Multifile::
01558 remove_subfile(int index) {
01559   nassertv(is_write_valid());
01560   nassertv(index >= 0 && index < (int)_subfiles.size());
01561   Subfile *subfile = _subfiles[index];
01562   subfile->_flags |= SF_deleted;
01563   _removed_subfiles.push_back(subfile);
01564   _subfiles.erase(_subfiles.begin() + index);
01565 
01566   _timestamp = time(NULL);
01567   _timestamp_dirty = true;
01568 
01569   _needs_repack = true;
01570 }
01571 
01572 ////////////////////////////////////////////////////////////////////
01573 //     Function: Multifile::get_subfile_name
01574 //       Access: Published
01575 //  Description: Returns the name of the nth subfile.
01576 ////////////////////////////////////////////////////////////////////
01577 const string &Multifile::
01578 get_subfile_name(int index) const {
01579 #ifndef NDEBUG
01580   static string empty_string;
01581   nassertr(index >= 0 && index < (int)_subfiles.size(), empty_string);
01582 #endif
01583   return _subfiles[index]->_name;
01584 }
01585 
01586 ////////////////////////////////////////////////////////////////////
01587 //     Function: Multifile::get_subfile_length
01588 //       Access: Published
01589 //  Description: Returns the uncompressed data length of the nth
01590 //               subfile.  This might return 0 if the subfile has
01591 //               recently been added and flush() has not yet been
01592 //               called.
01593 ////////////////////////////////////////////////////////////////////
01594 size_t Multifile::
01595 get_subfile_length(int index) const {
01596   nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
01597   return _subfiles[index]->_uncompressed_length;
01598 }
01599 
01600 ////////////////////////////////////////////////////////////////////
01601 //     Function: Multifile::get_subfile_timestamp
01602 //       Access: Published
01603 //  Description: Returns the modification time of the nth
01604 //               subfile.  If this is called on an older .mf file,
01605 //               which did not store individual timestamps in the file
01606 //               (or if get_record_timestamp() is false), this will
01607 //               return the modification time of the overall
01608 //               multifile.
01609 ////////////////////////////////////////////////////////////////////
01610 time_t Multifile::
01611 get_subfile_timestamp(int index) const {
01612   nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
01613   if (!get_record_timestamp()) {
01614     return get_timestamp();
01615   } else {
01616     return _subfiles[index]->_timestamp;
01617   }
01618 }
01619 
01620 ////////////////////////////////////////////////////////////////////
01621 //     Function: Multifile::is_subfile_compressed
01622 //       Access: Published
01623 //  Description: Returns true if the indicated subfile has been
01624 //               compressed when stored within the archive, false
01625 //               otherwise.
01626 ////////////////////////////////////////////////////////////////////
01627 bool Multifile::
01628 is_subfile_compressed(int index) const {
01629   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01630   return (_subfiles[index]->_flags & SF_compressed) != 0;
01631 }
01632 
01633 ////////////////////////////////////////////////////////////////////
01634 //     Function: Multifile::is_subfile_encrypted
01635 //       Access: Published
01636 //  Description: Returns true if the indicated subfile has been
01637 //               encrypted when stored within the archive, false
01638 //               otherwise.
01639 ////////////////////////////////////////////////////////////////////
01640 bool Multifile::
01641 is_subfile_encrypted(int index) const {
01642   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01643   return (_subfiles[index]->_flags & SF_encrypted) != 0;
01644 }
01645 
01646 ////////////////////////////////////////////////////////////////////
01647 //     Function: Multifile::get_index_end
01648 //       Access: Published
01649 //  Description: Returns the first byte that is guaranteed to follow
01650 //               any index byte already written to disk in the
01651 //               Multifile.
01652 //
01653 //               This number is largely meaningless in many cases, but
01654 //               if needs_repack() is false, and the file is flushed,
01655 //               this will indicate the number of bytes in the header
01656 //               + index.  Everything at this byte position and later
01657 //               will be actual data.
01658 ////////////////////////////////////////////////////////////////////
01659 streampos Multifile::
01660 get_index_end() const {
01661   return normalize_streampos(_next_index + (streampos)4);
01662 }
01663 
01664 ////////////////////////////////////////////////////////////////////
01665 //     Function: Multifile::get_subfile_internal_start
01666 //       Access: Published
01667 //  Description: Returns the starting byte position within the
01668 //               Multifile at which the indicated subfile begins.
01669 //               This may be used, with get_subfile_internal_length(),
01670 //               for low-level access to the subfile, but usually it
01671 //               is better to use open_read_subfile() instead (which
01672 //               automatically decrypts and/or uncompresses the
01673 //               subfile data).
01674 ////////////////////////////////////////////////////////////////////
01675 streampos Multifile::
01676 get_subfile_internal_start(int index) const {
01677   nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
01678   return _subfiles[index]->_data_start;
01679 }
01680 
01681 ////////////////////////////////////////////////////////////////////
01682 //     Function: Multifile::get_subfile_internal_length
01683 //       Access: Published
01684 //  Description: Returns the number of bytes the indicated subfile
01685 //               consumes within the archive.  For compressed
01686 //               subfiles, this will generally be smaller than
01687 //               get_subfile_length(); for encrypted (but
01688 //               noncompressed) subfiles, it may be slightly
01689 //               different, for noncompressed and nonencrypted
01690 //               subfiles, it will be equal.
01691 ////////////////////////////////////////////////////////////////////
01692 size_t Multifile::
01693 get_subfile_internal_length(int index) const {
01694   nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
01695   return _subfiles[index]->_data_length;
01696 }
01697 
01698 ////////////////////////////////////////////////////////////////////
01699 //     Function: Multifile::open_read_subfile
01700 //       Access: Published
01701 //  Description: Returns an istream that may be used to read the
01702 //               indicated subfile.  You may seek() within this
01703 //               istream to your heart's content; even though it will
01704 //               be a reference to the already-opened pfstream of the
01705 //               Multifile itself, byte 0 appears to be the beginning
01706 //               of the subfile and EOF appears to be the end of the
01707 //               subfile.
01708 //
01709 //               The returned istream will have been allocated via
01710 //               new; you should pass the pointer to
01711 //               close_read_subfile() when you are finished with it to
01712 //               delete it and release its resources.
01713 //
01714 //               Any future calls to repack() or close() (or the
01715 //               Multifile destructor) will invalidate all currently
01716 //               open subfile pointers.
01717 //
01718 //               The return value will be NULL if the stream cannot be
01719 //               opened for some reason.
01720 ////////////////////////////////////////////////////////////////////
01721 istream *Multifile::
01722 open_read_subfile(int index) {
01723   nassertr(is_read_valid(), NULL);
01724   nassertr(index >= 0 && index < (int)_subfiles.size(), NULL);
01725   Subfile *subfile = _subfiles[index];
01726 
01727   if (subfile->_source != (istream *)NULL ||
01728       !subfile->_source_filename.empty()) {
01729     // The subfile has not yet been copied into the physical
01730     // Multifile.  Force a flush operation to incorporate it.
01731     flush();
01732 
01733     // That shouldn't change the subfile index or delete the subfile
01734     // pointer.
01735     nassertr(subfile == _subfiles[index], NULL);
01736   }
01737 
01738   return open_read_subfile(subfile);
01739 }
01740 
01741 ////////////////////////////////////////////////////////////////////
01742 //     Function: Multifile::close_read_subfile
01743 //       Access: Published, Static
01744 //  Description: Closes a file opened by a previous call to
01745 //               open_read_subfile().  This really just deletes the
01746 //               istream pointer, but it is recommended to use this
01747 //               interface instead of deleting it explicitly, to help
01748 //               work around compiler issues.
01749 ////////////////////////////////////////////////////////////////////
01750 void Multifile::
01751 close_read_subfile(istream *stream) {
01752   if (stream != (istream *)NULL) {
01753     // For some reason--compiler bug in gcc 3.2?--explicitly deleting
01754     // the stream pointer does not call the appropriate global delete
01755     // function; instead apparently calling the system delete
01756     // function.  So we call the delete function by hand instead.
01757 #if !defined(WIN32_VC) && !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW)
01758     stream->~istream();
01759     (*global_operator_delete)(stream);
01760 #else
01761     delete stream;
01762 #endif
01763   }
01764 }
01765 
01766 ////////////////////////////////////////////////////////////////////
01767 //     Function: Multifile::extract_subfile
01768 //       Access: Published
01769 //  Description: Extracts the nth subfile into a file with the given
01770 //               name.
01771 ////////////////////////////////////////////////////////////////////
01772 bool Multifile::
01773 extract_subfile(int index, const Filename &filename) {
01774   nassertr(is_read_valid(), false);
01775   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01776   
01777   Filename fname = filename;
01778   fname.set_binary();
01779   fname.make_dir();
01780   pofstream out;
01781   if (!fname.open_write(out, true)) {
01782     express_cat.info()
01783       << "Unable to write to file " << filename << "\n";
01784     return false;
01785   }
01786 
01787   return extract_subfile_to(index, out);
01788 }
01789 
01790 ////////////////////////////////////////////////////////////////////
01791 //     Function: Multifile::extract_subfile_to
01792 //       Access: Public
01793 //  Description: Extracts the nth subfile to the indicated ostream.
01794 ////////////////////////////////////////////////////////////////////
01795 bool Multifile::
01796 extract_subfile_to(int index, ostream &out) {
01797   nassertr(is_read_valid(), false);
01798   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01799 
01800   istream *in = open_read_subfile(index);
01801   if (in == (istream *)NULL) {
01802     return false;
01803   }
01804 
01805   static const size_t buffer_size = 4096;
01806   char buffer[buffer_size];
01807 
01808   in->read(buffer, buffer_size);
01809   size_t count = in->gcount();
01810   while (count != 0) {
01811     out.write(buffer, count);
01812     in->read(buffer, buffer_size);
01813     count = in->gcount();
01814   }
01815 
01816   bool failed = (in->fail() && !in->eof());
01817   close_read_subfile(in);
01818   nassertr(!failed, false);
01819 
01820   return (!out.fail());
01821 }
01822 
01823 ////////////////////////////////////////////////////////////////////
01824 //     Function: Multifile::compare_subfile
01825 //       Access: Published
01826 //  Description: Performs a byte-for-byte comparison of the indicated
01827 //               file on disk with the nth subfile.  Returns true if
01828 //               the files are equivalent, or false if they are
01829 //               different (or the file is missing).
01830 ////////////////////////////////////////////////////////////////////
01831 bool Multifile::
01832 compare_subfile(int index, const Filename &filename) {
01833   nassertr(is_read_valid(), false);
01834   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01835 
01836   if (!filename.exists()) {
01837     express_cat.info()
01838       << "File is missing: " << filename << "\n";
01839     return false;
01840   }
01841 
01842   istream *in1 = open_read_subfile(index);
01843   if (in1 == (istream *)NULL) {
01844     return false;
01845   }
01846 
01847   pifstream in2;
01848   Filename bin_filename = Filename::binary_filename(filename);
01849   if (!bin_filename.open_read(in2)) {
01850     express_cat.info()
01851       << "Cannot read " << filename << "\n";
01852     return false;
01853   }
01854 
01855   // Check the file size.
01856   in2.seekg(0, ios::end);
01857   streampos file_size = in2.tellg();
01858 
01859   if (file_size != (streampos)get_subfile_length(index)) {
01860     // The files have different sizes.
01861     close_read_subfile(in1);
01862     return false;
01863   }
01864 
01865   // Check the file data, byte-for-byte.
01866   in2.seekg(0);
01867   int byte1 = in1->get();
01868   int byte2 = in2.get();
01869   while (!in1->fail() && !in1->eof() &&
01870          !in2.fail() && !in2.eof()) {
01871     if (byte1 != byte2) { 
01872       close_read_subfile(in1);
01873       return false;
01874     }
01875     byte1 = in1->get();
01876     byte2 = in2.get();
01877   }
01878 
01879   bool failed = (in1->fail() && !in1->eof()) || (in2.fail() && !in2.eof());
01880   close_read_subfile(in1);
01881 
01882   nassertr(!failed, false);
01883 
01884   return true;
01885 }
01886 
01887 ////////////////////////////////////////////////////////////////////
01888 //     Function: Multifile::output
01889 //       Access: Published
01890 //  Description: 
01891 ////////////////////////////////////////////////////////////////////
01892 void Multifile::
01893 output(ostream &out) const {
01894   out << "Multifile " << _multifile_name << ", " << get_num_subfiles()
01895       << " subfiles.\n";
01896 }
01897 
01898 ////////////////////////////////////////////////////////////////////
01899 //     Function: Multifile::ls
01900 //       Access: Published
01901 //  Description: Shows a list of all subfiles within the Multifile.
01902 ////////////////////////////////////////////////////////////////////
01903 void Multifile::
01904 ls(ostream &out) const {
01905   int num_subfiles = get_num_subfiles();
01906   for (int i = 0; i < num_subfiles; i++) {
01907     string subfile_name = get_subfile_name(i);
01908     out << subfile_name << "\n";
01909   }
01910 }
01911 
01912 ////////////////////////////////////////////////////////////////////
01913 //     Function: Multifile::set_header_prefix
01914 //       Access: Published
01915 //  Description: Sets the string which is written to the Multifile
01916 //               before the Multifile header.  This string must begin
01917 //               with a hash mark and end with a newline character;
01918 //               and if it includes embedded newline characters, each
01919 //               one must be followed by a hash mark.  If these
01920 //               conditions are not initially true, the string will be
01921 //               modified as necessary to make it so.
01922 //
01923 //               This is primarily useful as a simple hack to allow
01924 //               p3d applications to be run directly from the command
01925 //               line on Unix-like systems.
01926 //
01927 //               The return value is true if successful, or false on
01928 //               failure (for instance, because the header prefix
01929 //               violates the above rules).
01930 ////////////////////////////////////////////////////////////////////
01931 void Multifile::
01932 set_header_prefix(const string &header_prefix) {
01933   string new_header_prefix = header_prefix;
01934 
01935   if (!new_header_prefix.empty()) {
01936     // It must begin with a hash mark.
01937     if (new_header_prefix[0] != '#') {
01938       new_header_prefix = string("#") + new_header_prefix;
01939     }
01940     
01941     // It must end with a newline.
01942     if (new_header_prefix[new_header_prefix.size() - 1] != '\n') {
01943       new_header_prefix += string("\n");
01944     }
01945     
01946     // Embedded newlines must be followed by a hash mark.
01947     size_t newline = new_header_prefix.find('\n');
01948     while (newline < new_header_prefix.size() - 1) {
01949       if (new_header_prefix[newline + 1] != '#') {
01950         new_header_prefix = new_header_prefix.substr(0, newline + 1) + string("#") + new_header_prefix.substr(newline + 1);
01951       }
01952       newline = new_header_prefix.find('#', newline);
01953     }
01954   }
01955 
01956   if (_header_prefix != new_header_prefix) {
01957     _header_prefix = new_header_prefix;
01958     _needs_repack = true;
01959   }
01960 }
01961   
01962   
01963 ////////////////////////////////////////////////////////////////////
01964 //     Function: Multifile::read_subfile
01965 //       Access: Public
01966 //  Description: Fills a string with the entire contents of
01967 //               the indicated subfile.
01968 ////////////////////////////////////////////////////////////////////
01969 bool Multifile::
01970 read_subfile(int index, string &result) {
01971   result = string();
01972 
01973   // We use a temporary pvector, because dynamic accumulation of a
01974   // pvector seems to be many times faster than that of a string, at
01975   // least on the Windows implementation of STL.
01976   pvector<unsigned char> pv;
01977   if (!read_subfile(index, pv)) {
01978     return false;
01979   }
01980 
01981   if (!pv.empty()) {
01982     result.append((const char *)&pv[0], pv.size());
01983   }
01984 
01985   return true;
01986 }
01987 
01988 ////////////////////////////////////////////////////////////////////
01989 //     Function: Multifile::read_subfile
01990 //       Access: Public
01991 //  Description: Fills a pvector with the entire contents of
01992 //               the indicated subfile.
01993 ////////////////////////////////////////////////////////////////////
01994 bool Multifile::
01995 read_subfile(int index, pvector<unsigned char> &result) {
01996   nassertr(is_read_valid(), false);
01997   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01998   result.clear();
01999 
02000   // Now look up the particular Subfile we are reading.
02001   nassertr(is_read_valid(), NULL);
02002   nassertr(index >= 0 && index < (int)_subfiles.size(), NULL);
02003   Subfile *subfile = _subfiles[index];
02004 
02005   if (subfile->_source != (istream *)NULL ||
02006       !subfile->_source_filename.empty()) {
02007     // The subfile has not yet been copied into the physical
02008     // Multifile.  Force a flush operation to incorporate it.
02009     flush();
02010 
02011     // That shouldn't change the subfile index or delete the subfile
02012     // pointer.
02013     nassertr(subfile == _subfiles[index], NULL);
02014   }
02015 
02016   result.reserve(subfile->_uncompressed_length);
02017 
02018   bool success = true;
02019   if (subfile->_flags & (SF_encrypted | SF_compressed)) {
02020     // If the subfile is encrypted or compressed, we can't read it
02021     // directly.  Fall back to the generic implementation.
02022     istream *in = open_read_subfile(index);
02023     if (in == (istream *)NULL) {
02024       return false;
02025     }
02026 
02027     success = VirtualFile::simple_read_file(in, result);
02028     close_read_subfile(in);
02029 
02030   } else {
02031     // But if the subfile is just a plain file, we can just read the
02032     // data directly from the Multifile, without paying the cost of an
02033     // ISubStream.
02034     static const size_t buffer_size = 4096;
02035     char buffer[buffer_size];
02036 
02037     streamsize pos = _offset + subfile->_data_start;
02038     size_t max_bytes = subfile->_data_length;
02039     streamsize count = 0;
02040     bool eof = true;
02041 
02042     streamsize num_bytes = (streamsize)min(buffer_size, max_bytes);
02043     _read->seek_read(pos, buffer, num_bytes, count, eof);
02044     while (count != 0) {
02045       thread_consider_yield();
02046       nassertr(count <= max_bytes, false);
02047       result.insert(result.end(), buffer, buffer + (size_t)count);
02048       max_bytes -= (size_t)count;
02049       pos += count;
02050 
02051       num_bytes = (streamsize)min(buffer_size, max_bytes);
02052       _read->seek_read(pos, buffer, num_bytes, count, eof);
02053     }
02054 
02055     success = !eof;
02056   }
02057 
02058   if (!success) {
02059     ostringstream message;
02060     message << "I/O error reading from " << get_multifile_name() << " at "
02061             << get_subfile_name(index);
02062     nassert_raise(message.str());
02063     return false;
02064   }
02065 
02066   return true;
02067 }
02068 
02069 ////////////////////////////////////////////////////////////////////
02070 //     Function: Multifile::pad_to_streampos
02071 //       Access: Private
02072 //  Description: Assumes the _write pointer is at the indicated fpos,
02073 //               rounds the fpos up to the next legitimate address
02074 //               (using normalize_streampos()), and writes enough
02075 //               zeroes to the stream to fill the gap.  Returns the
02076 //               new fpos.
02077 ////////////////////////////////////////////////////////////////////
02078 streampos Multifile::
02079 pad_to_streampos(streampos fpos) {
02080   nassertr(_write != (ostream *)NULL, fpos);
02081   nassertr(_write->tellp() == fpos, fpos);
02082   streampos new_fpos = normalize_streampos(fpos);
02083   while (fpos < new_fpos) {
02084     _write->put(0);
02085     fpos += 1; // VC++ doesn't define streampos++ (!)
02086   }
02087   nassertr(_write->tellp() == fpos, fpos);
02088   return fpos;
02089 }
02090 
02091 ////////////////////////////////////////////////////////////////////
02092 //     Function: Multifile::add_new_subfile
02093 //       Access: Private
02094 //  Description: Adds a newly-allocated Subfile pointer to the
02095 //               Multifile.
02096 ////////////////////////////////////////////////////////////////////
02097 void Multifile::
02098 add_new_subfile(Subfile *subfile, int compression_level) {
02099   if (compression_level != 0) {
02100 #ifndef HAVE_ZLIB
02101     express_cat.warning()
02102       << "zlib not compiled in; cannot generated compressed multifiles.\n";
02103     compression_level = 0;
02104 #else  // HAVE_ZLIB
02105     subfile->_flags |= SF_compressed;
02106     subfile->_compression_level = compression_level;
02107 #endif  // HAVE_ZLIB
02108   }
02109 
02110 #ifdef HAVE_OPENSSL
02111   if (_encryption_flag) {
02112     subfile->_flags |= SF_encrypted;
02113   }
02114 #endif  // HAVE_OPENSSL
02115 
02116   if (_next_index != (streampos)0) {
02117     // If we're adding a Subfile to an already-existing Multifile, we
02118     // will eventually need to repack the file.
02119     _needs_repack = true;
02120   }
02121 
02122   pair<Subfiles::iterator, bool> insert_result = _subfiles.insert(subfile);
02123   if (!insert_result.second) {
02124     // Hmm, unable to insert.  There must already be a subfile by that
02125     // name.  Remove the old one.
02126     Subfile *old_subfile = (*insert_result.first);
02127     old_subfile->_flags |= SF_deleted;
02128 
02129     // Maybe it was just added to the _new_subfiles list.  In this case, remove it from that list.
02130     PendingSubfiles::iterator ni = find(_new_subfiles.begin(), _new_subfiles.end(), old_subfile);
02131     if (ni != _new_subfiles.end()) {
02132       _new_subfiles.erase(ni);
02133 
02134     } else {
02135       // Otherwise, add it to the _removed_subfiles list, so we can remove the old one.
02136       _removed_subfiles.push_back(old_subfile);
02137     }
02138 
02139     (*insert_result.first) = subfile;
02140   }
02141 
02142   _new_subfiles.push_back(subfile);
02143 }
02144 
02145 ////////////////////////////////////////////////////////////////////
02146 //     Function: Multifile::open_read_subfile
02147 //       Access: Private
02148 //  Description: This variant of open_read_subfile() is used
02149 //               internally only, and accepts a pointer to the
02150 //               internal Subfile object, which is assumed to be valid
02151 //               and written to the multifile.
02152 ////////////////////////////////////////////////////////////////////
02153 istream *Multifile::
02154 open_read_subfile(Subfile *subfile) {
02155   nassertr(subfile->_source == (istream *)NULL &&
02156            subfile->_source_filename.empty(), NULL);
02157 
02158   // Return an ISubStream object that references into the open
02159   // Multifile istream.
02160   nassertr(subfile->_data_start != (streampos)0, NULL);
02161   istream *stream = 
02162     new ISubStream(_read, _offset + subfile->_data_start,
02163                    _offset + subfile->_data_start + (streampos)subfile->_data_length); 
02164   
02165   if ((subfile->_flags & SF_encrypted) != 0) {
02166 #ifndef HAVE_OPENSSL
02167     express_cat.error()
02168       << "OpenSSL not compiled in; cannot read encrypted multifiles.\n";
02169     delete stream;
02170     return NULL;
02171 #else  // HAVE_OPENSSL
02172     // The subfile is encrypted.  So actually, return an
02173     // IDecryptStream that wraps around the ISubStream.
02174     IDecryptStream *wrapper = 
02175       new IDecryptStream(stream, true, _encryption_password);
02176     stream = wrapper;
02177 
02178     // Validate the password by confirming that the encryption header
02179     // matches.
02180     char this_header[_encrypt_header_size];
02181     stream->read(this_header, _encrypt_header_size);
02182     if (stream->fail() || stream->gcount() != (unsigned)_encrypt_header_size ||
02183         memcmp(this_header, _encrypt_header, _encrypt_header_size) != 0) {
02184       express_cat.error()
02185         << "Unable to decrypt subfile " << subfile->_name << ".\n";
02186       delete stream;
02187       return NULL;
02188     }
02189 #endif  // HAVE_OPENSSL
02190   }
02191 
02192   if ((subfile->_flags & SF_compressed) != 0) {
02193 #ifndef HAVE_ZLIB
02194     express_cat.error()
02195       << "zlib not compiled in; cannot read compressed multifiles.\n";
02196     delete stream;
02197     return NULL;
02198 #else  // HAVE_ZLIB
02199     // Oops, the subfile is compressed.  So actually, return an
02200     // IDecompressStream that wraps around the ISubStream.
02201     IDecompressStream *wrapper = new IDecompressStream(stream, true);
02202     stream = wrapper;
02203 #endif  // HAVE_ZLIB
02204   }
02205 
02206   if (stream->fail()) {
02207     // Hmm, some inexplicable problem.
02208     delete stream;
02209     return NULL;
02210   }
02211 
02212   return stream;
02213 }
02214 
02215 ////////////////////////////////////////////////////////////////////
02216 //     Function: Multifile::standardize_subfile_name
02217 //       Access: Private
02218 //  Description: Returns the standard form of the subfile name.
02219 ////////////////////////////////////////////////////////////////////
02220 string Multifile::
02221 standardize_subfile_name(const string &subfile_name) const {
02222   Filename name = subfile_name;
02223   name.standardize();
02224   if (name.empty() || name == "/") {
02225     // Invalid empty name.
02226     return string();
02227   }
02228 
02229   if (name[0] == '/') {
02230     return name.get_fullpath().substr(1);
02231   } else if (name.length() > 2 && name[0] == '.' && name[1] == '/') {
02232     return name.get_fullpath().substr(2);
02233   } else {
02234     return name.get_fullpath();
02235   }
02236 }
02237 
02238 ////////////////////////////////////////////////////////////////////
02239 //     Function: Multifile::clear_subfiles
02240 //       Access: Private
02241 //  Description: Removes the set of subfiles from the tables and frees
02242 //               their associated memory.
02243 ////////////////////////////////////////////////////////////////////
02244 void Multifile::
02245 clear_subfiles() {
02246   PendingSubfiles::iterator pi;
02247   for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
02248     Subfile *subfile = (*pi);
02249     subfile->rewrite_index_flags(*_write);
02250     delete subfile;
02251   }
02252   _removed_subfiles.clear();
02253 
02254   // We don't have to delete the ones in _new_subfiles, because these
02255   // also appear in _subfiles.
02256   _new_subfiles.clear();
02257 
02258 #ifdef HAVE_OPENSSL
02259   for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) {
02260     Subfile *subfile = (*pi);
02261     delete subfile;
02262   }
02263   _cert_special.clear();
02264 
02265   _signatures.clear();
02266 #endif  // HAVE_OPENSSL
02267 
02268   Subfiles::iterator fi;
02269   for (fi = _subfiles.begin(); fi != _subfiles.end(); ++fi) {
02270     Subfile *subfile = (*fi);
02271     delete subfile;
02272   }
02273   _subfiles.clear();
02274 }
02275 
02276 ////////////////////////////////////////////////////////////////////
02277 //     Function: Multifile::read_index
02278 //       Access: Private
02279 //  Description: Reads the Multifile header and index.  Returns true
02280 //               if successful, false if the Multifile is not valid.
02281 ////////////////////////////////////////////////////////////////////
02282 bool Multifile::
02283 read_index() {
02284   nassertr(_read != (IStreamWrapper *)NULL, false);
02285 
02286   // We acquire the IStreamWrapper lock for the duration of this
02287   // method.
02288   _read->acquire();
02289   istream *read = _read->get_istream();
02290 
02291   char this_header[_header_size];
02292   read->seekg(_offset);
02293 
02294   // Here's a special case: if the multifile begins with a hash
02295   // character, then we continue reading and discarding lines of ASCII
02296   // text, until we come across a nonempty line that does not begin
02297   // with a hash character.  This allows a P3D application (which is a
02298   // multifile) to be run directly on the command line on Unix-based
02299   // systems.
02300   _header_prefix = string();
02301   int ch = read->get();
02302 
02303   if (ch == '#') {
02304     while (ch != EOF && ch == '#') {
02305       // Skip to the end of the line.
02306       while (ch != EOF && ch != '\n') {
02307         _header_prefix += ch;
02308         ch = read->get();
02309       }
02310       // Skip to the first non-whitespace character of the line.
02311       while (ch != EOF && (isspace(ch) || ch == '\r')) {
02312         _header_prefix += ch;
02313         ch = read->get();
02314       }
02315     }
02316   }
02317 
02318   // Now read the actual Multifile header.
02319   this_header[0] = ch;
02320   read->read(this_header + 1, _header_size - 1);
02321   if (read->fail() || read->gcount() != (unsigned)(_header_size - 1)) {
02322     express_cat.info()
02323       << "Unable to read Multifile header " << _multifile_name << ".\n";
02324     _read->release();
02325     close();
02326     return false;
02327   }
02328 
02329   if (memcmp(this_header, _header, _header_size) != 0) {
02330     express_cat.info()
02331       << _multifile_name << " is not a Multifile.\n";
02332     _read->release();
02333     close();
02334     return false;
02335   }
02336 
02337   // Now get the version numbers out.
02338   StreamReader reader(read, false);
02339   _file_major_ver = reader.get_int16();
02340   _file_minor_ver = reader.get_int16();
02341   _scale_factor = reader.get_uint32();
02342   _new_scale_factor = _scale_factor;
02343 
02344   if (read->eof() || read->fail()) {
02345     express_cat.info()
02346       << _multifile_name << " header is truncated.\n";
02347     _read->release();
02348     close();
02349     return false;
02350   }
02351 
02352   if (_file_major_ver != _current_major_ver ||
02353       (_file_major_ver == _current_major_ver && 
02354        _file_minor_ver > _current_minor_ver)) {
02355     express_cat.info()
02356       << _multifile_name << " has version " << _file_major_ver << "."
02357       << _file_minor_ver << ", expecting version " 
02358       << _current_major_ver << "." << _current_minor_ver << ".\n";
02359     _read->release();
02360     close();
02361     return false;
02362   }
02363 
02364   _record_timestamp = true;
02365   if (_file_minor_ver >= 1) {
02366     time_t read_timestamp = reader.get_uint32();
02367     if (read_timestamp == 0) {
02368       // If we read a 0 timestamp from the file, that implies that we
02369       // don't want to record a timestamp in this particular file.
02370       _record_timestamp = false;
02371     } else {
02372       _timestamp = read_timestamp;
02373     }
02374     _timestamp_dirty = false;
02375   }
02376 
02377   // Now read the index out.
02378   _next_index = read->tellg() - _offset;
02379   _next_index = normalize_streampos(_next_index);  
02380   read->seekg(_next_index + _offset);
02381   _last_index = 0;
02382   _last_data_byte = 0;
02383   streampos index_forward;
02384   streamoff bytes_skipped = 0;
02385   bool read_cert_special = false;
02386 
02387   Subfile *subfile = new Subfile;
02388   index_forward = subfile->read_index(*read, _next_index, this);
02389   while (index_forward != (streampos)0) {
02390     _last_index = _next_index;
02391     if (subfile->is_deleted()) {
02392       // Ignore deleted Subfiles in the index.
02393       _needs_repack = true;
02394       delete subfile;
02395     } else if (subfile->is_cert_special()) {
02396       // Certificate chains and signature files get stored in a
02397       // special list.
02398       _cert_special.push_back(subfile);
02399       read_cert_special = true;
02400     } else {
02401       _subfiles.push_back(subfile);
02402     }
02403     if (!subfile->is_cert_special()) {
02404       if (bytes_skipped != 0) {
02405         // If the index entries don't follow exactly sequentially
02406         // (except for the cert special files), the file ought to be
02407         // repacked.
02408         _needs_repack = true;
02409       }
02410       if (read_cert_special) {
02411         // If we read a normal subfile following a cert_special entry,
02412         // the file ought to be repacked (certificates have to go at
02413         // the end).
02414         _needs_repack = true;
02415       }
02416       _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos());
02417     }
02418     streampos curr_pos = normalize_streampos(read->tellg() - _offset);
02419     bytes_skipped = index_forward - curr_pos;
02420     read->seekg(index_forward + _offset);
02421     _next_index = index_forward;
02422     subfile = new Subfile;
02423     index_forward = subfile->read_index(*read, _next_index, this);
02424   }
02425   if (subfile->is_index_invalid()) {
02426     express_cat.info()
02427       << "Error reading index for " << _multifile_name << ".\n";
02428     _read->release();
02429     close();
02430     delete subfile;
02431     return false;
02432   }
02433 
02434   // Check if the list is already sorted.  If it is not, we need a
02435   // repack.
02436   for (size_t si = 1; si < _subfiles.size() && !_needs_repack; ++si) {
02437     if (*_subfiles[si] < *_subfiles[si - 1]) {
02438       _needs_repack = true;
02439     }
02440   }
02441 
02442   if (_needs_repack) {
02443     // At least sort them now.
02444     size_t before_size = _subfiles.size();
02445     _subfiles.sort();
02446     size_t after_size = _subfiles.size();
02447     
02448     // If these don't match, the same filename appeared twice in the
02449     // index, which shouldn't be possible.
02450     nassertr(before_size == after_size, true);
02451   }
02452 
02453   delete subfile;
02454   _read->release();
02455   return true;
02456 }  
02457 
02458 ////////////////////////////////////////////////////////////////////
02459 //     Function: Multifile::write_header
02460 //       Access: Private
02461 //  Description: Writes just the header part of the Multifile, not the
02462 //               index.
02463 ////////////////////////////////////////////////////////////////////
02464 bool Multifile::
02465 write_header() {
02466   _file_major_ver = _current_major_ver;
02467   _file_minor_ver = _current_minor_ver;
02468 
02469   nassertr(_write != (ostream *)NULL, false);
02470   nassertr(_write->tellp() == (streampos)0, false);
02471   _write->write(_header_prefix.data(), _header_prefix.size());
02472   _write->write(_header, _header_size);
02473   StreamWriter writer(_write, false);
02474   writer.add_int16(_current_major_ver);
02475   writer.add_int16(_current_minor_ver);
02476   writer.add_uint32(_scale_factor);
02477 
02478   if (_record_timestamp) {
02479     writer.add_uint32(_timestamp);
02480   } else {
02481     writer.add_uint32(0);
02482     _timestamp_dirty = false;
02483   }
02484 
02485   _next_index = _write->tellp();
02486   _next_index = pad_to_streampos(_next_index);
02487   _last_index = 0;
02488 
02489   if (_write->fail()) {
02490     express_cat.info()
02491       << "Unable to write header for " << _multifile_name << ".\n";
02492     close();
02493     return false;
02494   }
02495 
02496   return true;
02497 }
02498 
02499 ////////////////////////////////////////////////////////////////////
02500 //     Function: Multifile::check_signatures
02501 //       Access: Private
02502 //  Description: Walks through the list of _cert_special entries in
02503 //               the Multifile, moving any valid signatures found to
02504 //               _signatures.  After this call, _cert_special will be
02505 //               empty.
02506 //
02507 //               This does not check the validity of the certificates
02508 //               themselves.  It only checks that they correctly sign
02509 //               the Multifile contents.
02510 ////////////////////////////////////////////////////////////////////
02511 void Multifile::
02512 check_signatures() {
02513 #ifdef HAVE_OPENSSL
02514   PendingSubfiles::iterator pi;
02515 
02516   for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) {
02517     Subfile *subfile = (*pi);
02518     nassertv((subfile->_flags & SF_signature) != 0);
02519 
02520     // Extract the signature data and certificate separately.
02521     istream *stream = open_read_subfile(subfile);
02522     nassertv(stream != NULL);
02523     StreamReader reader(*stream);
02524     size_t sig_size = reader.get_uint32();
02525     string sig_string = reader.extract_bytes(sig_size);
02526 
02527     size_t num_certs = reader.get_uint32();
02528 
02529     // Read the remaining buffer of certificate data.
02530     pvector<unsigned char> buffer;
02531     bool success = VirtualFile::simple_read_file(stream, buffer);
02532     nassertv(success);
02533     close_read_subfile(stream);
02534 
02535     // Now convert each of the certificates to an X509 object, and
02536     // store it in our CertChain.
02537     CertChain chain;
02538     EVP_PKEY *pkey = NULL;
02539     if (!buffer.empty()) {
02540 #if OPENSSL_VERSION_NUMBER >= 0x00908000L
02541       // Beginning in 0.9.8, d2i_X509() accepted a const unsigned char **.
02542       const unsigned char *bp, *bp_end;
02543 #else
02544       // Prior to 0.9.8, d2i_X509() accepted an unsigned char **.
02545       unsigned char *bp, *bp_end;
02546 #endif
02547       bp = (unsigned char *)&buffer[0];
02548       bp_end = bp + buffer.size();
02549       X509 *x509 = d2i_X509(NULL, &bp, bp_end - bp);
02550       while (num_certs > 0 && x509 != NULL) {
02551         chain.push_back(CertRecord(x509));
02552         --num_certs;
02553         x509 = d2i_X509(NULL, &bp, bp_end - bp);
02554       }
02555       if (num_certs != 0 || x509 != NULL) {
02556         express_cat.warning() 
02557           << "Extra data in signature record.\n";
02558       }
02559     }
02560     
02561     if (!chain.empty()) {
02562       pkey = X509_get_pubkey(chain[0]._cert);
02563     }
02564     
02565     if (pkey != NULL) {
02566       EVP_MD_CTX *md_ctx;
02567 #ifdef SSL_097
02568       md_ctx = EVP_MD_CTX_create();
02569 #else
02570       md_ctx = new EVP_MD_CTX;
02571 #endif
02572       EVP_VerifyInit(md_ctx, EVP_sha1());
02573       
02574       nassertv(_read != NULL);
02575       _read->acquire();
02576       istream *read = _read->get_istream();
02577       
02578       // Read and hash the multifile contents, but only up till
02579       // _last_data_byte.
02580       read->seekg(_offset);
02581       streampos bytes_remaining = _last_data_byte;
02582       static const size_t buffer_size = 4096;
02583       char buffer[buffer_size];
02584       read->read(buffer, min((streampos)buffer_size, bytes_remaining));
02585       size_t count = read->gcount();
02586       while (count != 0) {
02587         nassertv(count <= buffer_size);
02588         EVP_VerifyUpdate(md_ctx, buffer, count);
02589         bytes_remaining -= count;
02590         read->read(buffer, min((streampos)buffer_size, bytes_remaining));
02591         count = read->gcount();
02592       }
02593       nassertv(bytes_remaining == (streampos)0);
02594       _read->release();
02595       
02596       // Now check that the signature matches the hash.
02597       int verify_result = 
02598         EVP_VerifyFinal(md_ctx, 
02599                         (unsigned char *)sig_string.data(), 
02600                         sig_string.size(), pkey);
02601       if (verify_result == 1) {
02602         // The signature matches; save the certificate and its chain.
02603         _signatures.push_back(chain);
02604       } else {
02605         // Bad match.
02606         _needs_repack = true;
02607       }
02608     }
02609   }
02610 #endif  // HAVE_OPENSSL
02611 
02612   _cert_special.clear();
02613 }
02614 
02615 ////////////////////////////////////////////////////////////////////
02616 //     Function: Multifile::Subfile::read_index
02617 //       Access: Public
02618 //  Description: Reads the index record for the Subfile from the
02619 //               indicated istream.  Assumes the istream has already
02620 //               been positioned to the indicated stream position,
02621 //               fpos, the start of the index record.  Returns the
02622 //               position within the file of the next index record.
02623 ////////////////////////////////////////////////////////////////////
02624 streampos Multifile::Subfile::
02625 read_index(istream &read, streampos fpos, Multifile *multifile) {
02626   nassertr(read.tellg() - multifile->_offset == fpos, fpos);
02627 
02628   // First, get the next stream position.  We do this separately,
02629   // because if it is zero, we don't get anything else.
02630   StreamReader reader(read);
02631 
02632   streampos next_index = multifile->word_to_streampos(reader.get_uint32());
02633   if (read.eof() || read.fail()) {
02634     _flags |= SF_index_invalid;
02635     return 0;
02636   }
02637 
02638   if (next_index == (streampos)0) {
02639     return 0;
02640   }
02641 
02642   // Now get the rest of the index.
02643 
02644   _index_start = fpos;
02645   _index_length = 0;
02646   
02647   _data_start = multifile->word_to_streampos(reader.get_uint32());
02648   _data_length = reader.get_uint32();
02649   _flags = reader.get_uint16();
02650   if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
02651     _uncompressed_length = reader.get_uint32();
02652   } else {
02653     _uncompressed_length = _data_length;
02654   }
02655   if (multifile->_file_minor_ver < 1) {
02656     _timestamp = multifile->get_timestamp();
02657   } else {
02658     _timestamp = reader.get_uint32();
02659     if (_timestamp == 0) {
02660       _timestamp = multifile->get_timestamp();
02661     }
02662   }
02663 
02664   size_t name_length = reader.get_uint16();
02665   if (read.eof() || read.fail()) {
02666     _flags |= SF_index_invalid;
02667     return 0;
02668   }
02669 
02670   // And finally, get the rest of the name.
02671   char *name_buffer = (char *)PANDA_MALLOC_ARRAY(name_length);
02672   nassertr(name_buffer != (char *)NULL, next_index);
02673   for (size_t ni = 0; ni < name_length; ni++) {
02674     name_buffer[ni] = read.get() ^ 0xff;
02675   }
02676   _name = string(name_buffer, name_length);
02677   PANDA_FREE_ARRAY(name_buffer);
02678 
02679   if (read.eof() || read.fail()) {
02680     _flags |= SF_index_invalid;
02681     return 0;
02682   }
02683 
02684   _index_length = read.tellg() - fpos - multifile->_offset;
02685   return next_index;
02686 }
02687 
02688 ////////////////////////////////////////////////////////////////////
02689 //     Function: Multifile::Subfile::write_index
02690 //       Access: Public
02691 //  Description: Writes the index record for the Subfile to the
02692 //               indicated ostream.  Assumes the istream has already
02693 //               been positioned to the indicated stream position,
02694 //               fpos, the start of the index record, and that this is
02695 //               the effective end of the file.  Returns the position
02696 //               within the file of the next index record.
02697 //
02698 //               The _index_start member is updated by this operation.
02699 ////////////////////////////////////////////////////////////////////
02700 streampos Multifile::Subfile::
02701 write_index(ostream &write, streampos fpos, Multifile *multifile) {
02702   nassertr(write.tellp() - multifile->_offset == fpos, fpos);
02703 
02704   _index_start = fpos;
02705   _index_length = 0;
02706 
02707   // This will be the contents of this particular index record.  We
02708   // build it up first since it will be variable length.
02709   Datagram dg;
02710   dg.add_uint32(multifile->streampos_to_word(_data_start));
02711   dg.add_uint32(_data_length);
02712   dg.add_uint16(_flags);
02713   if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
02714     dg.add_uint32(_uncompressed_length);
02715   }
02716   dg.add_uint32(_timestamp);
02717   dg.add_uint16(_name.length());
02718 
02719   // For no real good reason, we'll invert all the bits in the name.
02720   // The only reason we do this is to make it inconvenient for a
02721   // casual browser of the Multifile to discover the names of the
02722   // files stored within it.  Naturally, this isn't real obfuscation
02723   // or security.
02724   string::iterator ni;
02725   for (ni = _name.begin(); ni != _name.end(); ++ni) {
02726     dg.add_int8((*ni) ^ 0xff);
02727   }
02728 
02729   size_t this_index_size = 4 + dg.get_length();
02730 
02731   // Plus, we will write out the next index address first.
02732   streampos next_index = fpos + (streampos)this_index_size; 
02733 
02734   Datagram idg;
02735   idg.add_uint32(multifile->streampos_to_word(next_index));
02736 
02737   write.write((const char *)idg.get_data(), idg.get_length());
02738   write.write((const char *)dg.get_data(), dg.get_length());
02739 
02740   _index_length = write.tellp() - fpos - multifile->_offset;
02741   return next_index;
02742 }
02743 
02744 ////////////////////////////////////////////////////////////////////
02745 //     Function: Multifile::Subfile::write_data
02746 //       Access: Public
02747 //  Description: Writes the data record for the Subfile to the
02748 //               indicated ostream: the actual contents of the
02749 //               Subfile.  Assumes the istream has already been
02750 //               positioned to the indicated stream position, fpos,
02751 //               the start of the data record, and that this is the
02752 //               effective end of the file.  Returns the position
02753 //               within the file of the next data record.
02754 //
02755 //               The _data_start, _data_length, and
02756 //               _uncompressed_length members are updated by this
02757 //               operation.
02758 //
02759 //               If the "read" pointer is non-NULL, it is the readable
02760 //               istream of a Multifile in which the Subfile might
02761 //               already be packed.  This is used for reading the
02762 //               contents of the Subfile during a repack() operation.
02763 ////////////////////////////////////////////////////////////////////
02764 streampos Multifile::Subfile::
02765 write_data(ostream &write, istream *read, streampos fpos,
02766            Multifile *multifile) {
02767   nassertr(write.tellp() - multifile->_offset == fpos, fpos);
02768 
02769   istream *source = _source;
02770   pifstream source_file;
02771   if (source == (istream *)NULL && !_source_filename.empty()) {
02772     // If we have a filename, open it up and read that.
02773     if (!_source_filename.open_read(source_file)) {
02774       // Unable to open the source file.
02775       express_cat.info()
02776         << "Unable to read " << _source_filename << ".\n";
02777       _flags |= SF_data_invalid;
02778       _data_length = 0;
02779       _uncompressed_length = 0;
02780     } else {
02781       source = &source_file;
02782     }
02783   }
02784 
02785   if (source == (istream *)NULL) {
02786     // We don't have any source data.  Perhaps we're reading from an
02787     // already-packed Subfile (e.g. during repack()).
02788     if (read == (istream *)NULL) {
02789       // No, we're just screwed.
02790       express_cat.info()
02791         << "No source for subfile " << _name << ".\n";
02792       _flags |= SF_data_invalid;
02793     } else {
02794       // Read the data from the original Multifile.
02795       read->seekg(_data_start + multifile->_offset);
02796       for (size_t p = 0; p < _data_length; p++) {
02797         int byte = read->get();
02798         if (read->eof() || read->fail()) {
02799           // Unexpected EOF or other failure on the source file.
02800           express_cat.info()
02801             << "Unexpected EOF for subfile " << _name << ".\n";
02802           _flags |= SF_data_invalid;
02803           break;
02804         }
02805         write.put(byte);
02806       }
02807     }
02808   } else {
02809     // We do have source data.  Copy it in, and also measure its
02810     // length.
02811     ostream *putter = &write;
02812     bool delete_putter = false;
02813 
02814 #ifndef HAVE_OPENSSL
02815     // Without OpenSSL, we can't support encryption.  The flag had
02816     // better not be set.
02817     nassertr((_flags & SF_encrypted) == 0, fpos);
02818 
02819 #else  // HAVE_OPENSSL
02820     if ((_flags & SF_encrypted) != 0) {
02821       // Write it encrypted.
02822       OEncryptStream *encrypt = new OEncryptStream;
02823       encrypt->set_iteration_count(multifile->_encryption_iteration_count);
02824       encrypt->open(putter, delete_putter, multifile->_encryption_password);
02825 
02826       putter = encrypt;
02827       delete_putter = true;
02828 
02829       // Also write the encrypt_header to the beginning of the
02830       // encrypted stream, so we can validate the password on
02831       // decryption.
02832       putter->write(_encrypt_header, _encrypt_header_size);
02833     }
02834 #endif  // HAVE_OPENSSL
02835 
02836 #ifndef HAVE_ZLIB
02837     // Without ZLIB, we can't support compression.  The flag had
02838     // better not be set.
02839     nassertr((_flags & SF_compressed) == 0, fpos);
02840 #else  // HAVE_ZLIB
02841     if ((_flags & SF_compressed) != 0) {
02842       // Write it compressed.
02843       putter = new OCompressStream(putter, delete_putter, _compression_level);
02844       delete_putter = true;
02845     }
02846 #endif  // HAVE_ZLIB
02847 
02848     streampos write_start = fpos;
02849     _uncompressed_length = 0;
02850 
02851 #ifndef HAVE_OPENSSL
02852     // We also need OpenSSL for signatures.
02853     nassertr((_flags & SF_signature) == 0, fpos);
02854 
02855 #else  // HAVE_OPENSSL
02856     if ((_flags & SF_signature) != 0) {
02857       // If it's a special signature record, precede the record data
02858       // (the certificate itself) with the signature data generated
02859       // against the multifile contents.
02860 
02861       // In order to generate a signature, we need to have a valid
02862       // read pointer.
02863       nassertr(read != NULL, fpos);
02864 
02865       // And we also need to have a private key.
02866       nassertr(_pkey != NULL, fpos);
02867 
02868       EVP_MD_CTX *md_ctx;
02869 #ifdef SSL_097
02870       md_ctx = EVP_MD_CTX_create();
02871 #else
02872       md_ctx = new EVP_MD_CTX;
02873 #endif
02874       EVP_SignInit(md_ctx, EVP_sha1());
02875 
02876       // Read and hash the multifile contents, but only up till
02877       // _last_data_byte.
02878       nassertr(multifile->_last_data_byte < fpos, fpos);
02879       read->seekg(multifile->_offset);
02880       streampos bytes_remaining = multifile->_last_data_byte;
02881       static const size_t buffer_size = 4096;
02882       char buffer[buffer_size];
02883       read->read(buffer, min((streampos)buffer_size, bytes_remaining));
02884       size_t count = read->gcount();
02885       while (count != 0) {
02886         nassertr(count <= buffer_size, fpos);
02887         EVP_SignUpdate(md_ctx, buffer, count);
02888         bytes_remaining -= count;
02889         read->read(buffer, min((streampos)buffer_size, bytes_remaining));
02890         count = read->gcount();
02891       }
02892       nassertr(bytes_remaining == (streampos)0, fpos);
02893 
02894       // Now generate and write out the signature.
02895       unsigned int max_size = EVP_PKEY_size(_pkey);
02896       unsigned char *sig_data = new unsigned char[max_size];
02897       unsigned int sig_size;
02898       if (!EVP_SignFinal(md_ctx, sig_data, &sig_size, _pkey)) {
02899         OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
02900         sslw->notify_ssl_errors();
02901       }
02902       nassertr(sig_size <= max_size, fpos);
02903 
02904       StreamWriter writer(*putter);
02905       writer.add_uint32(sig_size);
02906       putter->write((char *)sig_data, sig_size);
02907       _uncompressed_length += 4 + sig_size;
02908 
02909       delete[] sig_data;
02910       
02911 #ifdef SSL_097
02912       EVP_MD_CTX_destroy(md_ctx);
02913 #else
02914       delete md_ctx;
02915 #endif
02916     }
02917 #endif  // HAVE_OPENSSL
02918 
02919     // Finally, we can write out the data itself.
02920     static const size_t buffer_size = 4096;
02921     char buffer[buffer_size];
02922     
02923     source->read(buffer, buffer_size);
02924     size_t count = source->gcount();
02925     while (count != 0) {
02926       _uncompressed_length += count;
02927       putter->write(buffer, count);
02928       source->read(buffer, buffer_size);
02929       count = source->gcount();
02930     }
02931 
02932     if (delete_putter) {
02933       delete putter;
02934     }
02935 
02936     streampos write_end = write.tellp() - multifile->_offset;
02937     _data_length = (size_t)(write_end - write_start);
02938   }
02939 
02940   // We can't set _data_start until down here, after we have read the
02941   // Subfile.  (In case we are running during repack()).
02942   _data_start = fpos;
02943 
02944   // Get the modification timestamp for this subfile.  This is read
02945   // from the source file, if we have a filename; otherwise, it's the
02946   // current time.
02947   if (!_source_filename.empty()) {
02948     _timestamp = _source_filename.get_timestamp();
02949   }
02950   if (_timestamp == 0) {
02951     _timestamp = time(NULL);
02952   }
02953 
02954   _source = (istream *)NULL;
02955   _source_filename = Filename();
02956   source_file.close();
02957 
02958   return fpos + (streampos)_data_length;
02959 }
02960 
02961 ////////////////////////////////////////////////////////////////////
02962 //     Function: Multifile::Subfile::rewrite_index_data_start
02963 //       Access: Public
02964 //  Description: Seeks within the indicate pfstream back to the index
02965 //               record and rewrites just the _data_start and
02966 //               _data_length part of the index record.
02967 ////////////////////////////////////////////////////////////////////
02968 void Multifile::Subfile::
02969 rewrite_index_data_start(ostream &write, Multifile *multifile) {
02970   nassertv(_index_start != (streampos)0);
02971 
02972   static const size_t data_start_offset = 4;
02973   size_t data_start_pos = _index_start + (streampos)data_start_offset;
02974   write.seekp(data_start_pos + multifile->_offset);
02975   nassertv(!write.fail());
02976 
02977   StreamWriter writer(write);
02978   writer.add_uint32(multifile->streampos_to_word(_data_start));
02979   writer.add_uint32(_data_length);
02980   writer.add_uint16(_flags);
02981   if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
02982     writer.add_uint32(_uncompressed_length);
02983   }
02984   if (multifile->_record_timestamp) {
02985     writer.add_uint32(_timestamp);
02986   } else {
02987     writer.add_uint32(0);
02988   }
02989 }
02990 
02991 ////////////////////////////////////////////////////////////////////
02992 //     Function: Multifile::Subfile::rewrite_index_flags
02993 //       Access: Public
02994 //  Description: Seeks within the indicated ostream back to the index
02995 //               record and rewrites just the _flags part of the
02996 //               index record.
02997 ////////////////////////////////////////////////////////////////////
02998 void Multifile::Subfile::
02999 rewrite_index_flags(ostream &write) {
03000   // If the subfile has never even been recorded to disk, we don't
03001   // need to do anything at all in this function.
03002   if (_index_start != (streampos)0) {
03003     static const size_t flags_offset = 4 + 4 + 4;
03004     size_t flags_pos = _index_start + (streampos)flags_offset;
03005     write.seekp(flags_pos);
03006     nassertv(!write.fail());
03007     
03008     StreamWriter writer(write);
03009     writer.add_uint16(_flags);
03010   }
03011 }
 All Classes Functions Variables Enumerations