Panda3D
 All Classes Functions Variables Enumerations
multifile.cxx
00001 // Filename: multifile.cxx
00002 // Created by:  mike (09Jan97)
00003 //
00004 ////////////////////////////////////////////////////////////////////
00005 //
00006 // PANDA 3D SOFTWARE
00007 // Copyright (c) Carnegie Mellon University.  All rights reserved.
00008 //
00009 // All use of this software is subject to the terms of the revised BSD
00010 // license.  You should have received a copy of this license along
00011 // with this source code in a file named "LICENSE."
00012 //
00013 ////////////////////////////////////////////////////////////////////
00014 
00015 #include "multifile.h"
00016 
00017 #include "config_express.h"
00018 #include "streamWriter.h"
00019 #include "streamReader.h"
00020 #include "datagram.h"
00021 #include "zStream.h"
00022 #include "encryptStream.h"
00023 #include "virtualFileSystem.h"
00024 #include "virtualFile.h"
00025 
00026 #include <algorithm>
00027 #include <iterator>
00028 #include <time.h>
00029 
00030 // This sequence of bytes begins each Multifile to identify it as a
00031 // Multifile.
00032 const char Multifile::_header[] = "pmf\0\n\r";
00033 const size_t Multifile::_header_size = 6;
00034 
00035 // These numbers identify the version of the Multifile.  Generally, a
00036 // change in the major version is intolerable; while a Multifile with
00037 // an older minor version may still be read.
00038 const int Multifile::_current_major_ver = 1;
00039 
00040 const int Multifile::_current_minor_ver = 1;
00041 // Bumped to version 1.1 on 6/8/06 to add timestamps.
00042 
00043 // To confirm that the supplied password matches, we write the
00044 // Mutifile magic header at the beginning of the encrypted stream.
00045 // I suppose this does compromise the encryption security a tiny
00046 // bit by making it easy for crackers to validate that a
00047 // particular password guess matches or doesn't match, but the
00048 // encryption algorithm doesn't depend on this being difficult
00049 // anyway.
00050 const char Multifile::_encrypt_header[] = "crypty";
00051 const size_t Multifile::_encrypt_header_size = 6;
00052 
00053 
00054 
00055 //
00056 // A Multifile consists of the following elements:
00057 //
00058 // (1) A header.  This is always the first n bytes of the Multifile,
00059 // and contains a magic number to identify the file, as well as
00060 // version numbers and any file-specific parameters.
00061 //
00062 //   char[6]    The string Multifile::_header, a magic number.
00063 //   int16      The file's major version number
00064 //   int16      The file's minor version number
00065 //   uint32     Scale factor.  This scales all address references within
00066 //              the file.  Normally 1, this may be set larger to
00067 //              support Multifiles larger than 4GB.
00068 //   uint32     An overall modification timestamp for the entire multifile.
00069 
00070 //
00071 // (2) Zero or more index entries, one for each subfile within the
00072 // Multifile.  These entries are of variable length.  The first one of
00073 // these immediately follows the header, and the first word of each
00074 // index entry contains the address of the next index entry.  A zero
00075 // "next" address marks the end of the chain.  These may appear at any
00076 // point within the Multifile; they do not necessarily appear in
00077 // sequential order at the beginning of the file (although they will
00078 // after the file has been "packed").
00079 //
00080 //   uint32     The address of the next entry.  0 to mark the end.
00081 //   uint32     The address of this subfile's data record.
00082 //   uint32     The length in bytes of this subfile's data record.
00083 //   uint16     The Subfile::_flags member.
00084 //  [uint32]    The original, uncompressed and unencrypted length of the
00085 //               subfile, if it is compressed or encrypted.  This field
00086 //               is only present if one or both of the SF_compressed
00087 //               or SF_encrypted bits are set in _flags.
00088 //   uint32     A modification timestamp for the subfile.
00089 //   uint16     The length in bytes of the subfile's name.
00090 //   char[n]    The subfile's name.
00091 //
00092 // (3) Zero or more data entries, one for each subfile.  These may
00093 // appear at any point within the Multifile; they do not necessarily
00094 // follow each index entry, nor are they necessarily all grouped
00095 // together at the end (although they will be all grouped together at
00096 // the end after the file has been "packed").  These are just blocks
00097 // of literal data.
00098 //
00099 
00100 ////////////////////////////////////////////////////////////////////
00101 //     Function: Multifile::Constructor
00102 //       Access: Published
00103 //  Description:
00104 ////////////////////////////////////////////////////////////////////
00105 Multifile::
00106 Multifile() :
00107   _read_filew(_read_file),
00108   _read_write_filew(_read_write_file)
00109 {
00110   ConfigVariableInt multifile_encryption_iteration_count
00111     ("multifile-encryption-iteration-count", 0,
00112      PRC_DESC("This is a special value of encryption-iteration-count used to encrypt "
00113               "subfiles within a multifile.  It has a default value of 0 (just one "
00114               "application), on the assumption that the files from a multifile must "
00115               "be loaded quickly, without paying the cost of an expensive hash on "
00116               "each subfile in order to decrypt it."));
00117   
00118   _read = (IStreamWrapper *)NULL;
00119   _write = (ostream *)NULL;
00120   _offset = 0;
00121   _owns_stream = false;
00122   _next_index = 0;
00123   _last_index = 0;
00124   _last_data_byte = 0;
00125   _needs_repack = false;
00126   _timestamp = 0;
00127   _timestamp_dirty = false;
00128   _record_timestamp = true;
00129   _scale_factor = 1;
00130   _new_scale_factor = 1;
00131   _encryption_flag = false;
00132   _encryption_iteration_count = multifile_encryption_iteration_count;
00133   _file_major_ver = 0;
00134   _file_minor_ver = 0;
00135 
00136 #ifdef HAVE_OPENSSL
00137   // Get these values from the config file via an EncryptStreamBuf.
00138   EncryptStreamBuf tbuf;
00139   _encryption_algorithm = tbuf.get_algorithm();
00140   _encryption_key_length = tbuf.get_key_length();
00141 #endif
00142 }
00143 
00144 ////////////////////////////////////////////////////////////////////
00145 //     Function: Multifile::Destructor
00146 //       Access: Published
00147 //  Description:
00148 ////////////////////////////////////////////////////////////////////
00149 Multifile::
00150 ~Multifile() {
00151   close();
00152 }
00153 
00154 ////////////////////////////////////////////////////////////////////
00155 //     Function: Multifile::Copy Constructor
00156 //       Access: Private
00157 //  Description: Don't try to copy Multifiles.
00158 ////////////////////////////////////////////////////////////////////
00159 Multifile::
00160 Multifile(const Multifile &copy) :
00161   _read_filew(_read_file),
00162   _read_write_filew(_read_write_file)
00163 {
00164   nassertv(false);
00165 }
00166 
00167 ////////////////////////////////////////////////////////////////////
00168 //     Function: Multifile::Copy Assignment Operator
00169 //       Access: Private
00170 //  Description: Don't try to copy Multifiles.
00171 ////////////////////////////////////////////////////////////////////
00172 void Multifile::
00173 operator = (const Multifile &copy) {
00174   nassertv(false);
00175 }
00176 
00177 ////////////////////////////////////////////////////////////////////
00178 //     Function: Multifile::open_read
00179 //       Access: Published
00180 //  Description: Opens the named Multifile on disk for reading.  The
00181 //               Multifile index is read in, and the list of subfiles
00182 //               becomes available; individual subfiles may then be
00183 //               extracted or read, but the list of subfiles may not
00184 //               be modified.
00185 //
00186 //               Also see the version of open_read() which accepts an
00187 //               istream.  Returns true on success, false on failure.
00188 ////////////////////////////////////////////////////////////////////
00189 bool Multifile::
00190 open_read(const Filename &multifile_name, const streampos &offset) {
00191   close();
00192   Filename fname = multifile_name;
00193   fname.set_binary();
00194 
00195   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00196   PT(VirtualFile) vfile = vfs->get_file(fname);
00197   if (vfile == NULL) {
00198     return false;
00199   }
00200   istream *multifile_stream = vfile->open_read_file(false);
00201   if (multifile_stream == NULL) {
00202     return false;
00203   }
00204 
00205   _timestamp = vfile->get_timestamp();
00206   _timestamp_dirty = true;
00207   _read = new IStreamWrapper(multifile_stream, true);
00208   _owns_stream = true;
00209   _multifile_name = multifile_name;
00210   _offset = offset;
00211   return read_index();
00212 }
00213 
00214 ////////////////////////////////////////////////////////////////////
00215 //     Function: Multifile::open_read
00216 //       Access: Public
00217 //  Description: Opens an anonymous Multifile for reading using an
00218 //               istream.  There must be seek functionality via
00219 //               seekg() and tellg() on the istream.
00220 //
00221 //               If owns_pointer is true, then the Multifile assumes
00222 //               ownership of the stream pointer and will delete it
00223 //               when the multifile is closed, including if this
00224 //               function returns false.
00225 ////////////////////////////////////////////////////////////////////
00226 bool Multifile::
00227 open_read(IStreamWrapper *multifile_stream, bool owns_pointer,
00228           const streampos &offset) {
00229   close();
00230   _timestamp = time(NULL);
00231   _timestamp_dirty = true;
00232   _read = multifile_stream;
00233   _owns_stream = owns_pointer;
00234   _offset = offset;
00235   return read_index();
00236 }
00237 
00238 ////////////////////////////////////////////////////////////////////
00239 //     Function: Multifile::open_write
00240 //       Access: Published
00241 //  Description: Opens the named Multifile on disk for writing.  If
00242 //               there already exists a file by that name, it is
00243 //               truncated.  The Multifile is then prepared for
00244 //               accepting a brand new set of subfiles, which will be
00245 //               written to the indicated filename.  Individual
00246 //               subfiles may not be extracted or read.
00247 //
00248 //               Also see the version of open_write() which accepts an
00249 //               ostream.  Returns true on success, false on failure.
00250 ////////////////////////////////////////////////////////////////////
00251 bool Multifile::
00252 open_write(const Filename &multifile_name) {
00253   close();
00254   Filename fname = multifile_name;
00255   fname.set_binary();
00256   if (!fname.open_write(_write_file, true)) {
00257     return false;
00258   }
00259   _timestamp = time(NULL);
00260   _timestamp_dirty = true;
00261   _write = &_write_file;
00262   _multifile_name = multifile_name;
00263   return true;
00264 }
00265 
00266 ////////////////////////////////////////////////////////////////////
00267 //     Function: Multifile::open_write
00268 //       Access: Public
00269 //  Description: Opens an anonymous Multifile for writing using an
00270 //               ostream.  There must be seek functionality via
00271 //               seekp() and tellp() on the pstream.
00272 //
00273 //               If owns_pointer is true, then the Multifile assumes
00274 //               ownership of the stream pointer and will delete it
00275 //               when the multifile is closed, including if this
00276 //               function returns false.
00277 ////////////////////////////////////////////////////////////////////
00278 bool Multifile::
00279 open_write(ostream *multifile_stream, bool owns_pointer) {
00280   close();
00281   _timestamp = time(NULL);
00282   _timestamp_dirty = true;
00283   _write = multifile_stream;
00284   _owns_stream = owns_pointer;
00285   _write->seekp(0, ios::beg);
00286   return true;
00287 }
00288 
00289 ////////////////////////////////////////////////////////////////////
00290 //     Function: Multifile::open_read_write
00291 //       Access: Published
00292 //  Description: Opens the named Multifile on disk for reading and
00293 //               writing.  If there already exists a file by that
00294 //               name, its index is read.  Subfiles may be added or
00295 //               removed, and the resulting changes will be written to
00296 //               the named file.
00297 //
00298 //               Also see the version of open_read_write() which
00299 //               accepts an iostream.  Returns true on success, false
00300 //               on failure.
00301 ////////////////////////////////////////////////////////////////////
00302 bool Multifile::
00303 open_read_write(const Filename &multifile_name) {
00304   close();
00305   Filename fname = multifile_name;
00306   fname.set_binary();
00307   bool exists = fname.exists();
00308   if (!fname.open_read_write(_read_write_file)) {
00309     return false;
00310   }
00311   if (exists) {
00312     _timestamp = fname.get_timestamp();
00313   } else {
00314     _timestamp = time(NULL);
00315   }
00316   _timestamp_dirty = true;
00317   _read = &_read_write_filew;
00318   _write = &_read_write_file;
00319   _multifile_name = multifile_name;
00320 
00321   if (exists) {
00322     return read_index();
00323   } else {
00324     return true;
00325   }
00326 }
00327 
00328 ////////////////////////////////////////////////////////////////////
00329 //     Function: Multifile::open_read_write
00330 //       Access: Public
00331 //  Description: Opens an anonymous Multifile for reading and writing
00332 //               using an iostream.  There must be seek functionality
00333 //               via seekg()/seekp() and tellg()/tellp() on the
00334 //               iostream.
00335 //
00336 //               If owns_pointer is true, then the Multifile assumes
00337 //               ownership of the stream pointer and will delete it
00338 //               when the multifile is closed, including if this
00339 //               function returns false.
00340 ////////////////////////////////////////////////////////////////////
00341 bool Multifile::
00342 open_read_write(iostream *multifile_stream, bool owns_pointer) {
00343   close();
00344   _timestamp = time(NULL);
00345   _timestamp_dirty = true;
00346 
00347   // We don't support locking when opening a file in read-write mode,
00348   // because we don't bother with locking on write.  But we need to
00349   // have an IStreamWrapper to assign to the _read member, so we
00350   // create one on-the-fly here.
00351   _read = new StreamWrapper(multifile_stream, owns_pointer);
00352   _write = multifile_stream;
00353   _owns_stream = true;  // Because we own the StreamWrapper, above.
00354   _write->seekp(0, ios::beg);
00355 
00356   // Check whether the read stream is empty.
00357   multifile_stream->seekg(0, ios::end);
00358   if (multifile_stream->tellg() == (streampos)0) {
00359     // The read stream is empty, which is always valid.
00360     return true;
00361   }
00362 
00363   // The read stream is not empty, so we'd better have a valid
00364   // Multifile.
00365   return read_index();
00366 }
00367 
00368 ////////////////////////////////////////////////////////////////////
00369 //     Function: Multifile::close
00370 //       Access: Published
00371 //  Description: Closes the Multifile if it is open.  All changes are
00372 //               flushed to disk, and the file becomes invalid for
00373 //               further operations until the next call to open().
00374 ////////////////////////////////////////////////////////////////////
00375 void Multifile::
00376 close() {
00377   if (_new_scale_factor != _scale_factor) {
00378     // If we have changed the scale factor recently, we need to force
00379     // a repack.
00380     repack();
00381   } else {
00382     flush();
00383   }
00384 
00385   if (_owns_stream) {
00386     // We prefer to delete the IStreamWrapper over the ostream, if
00387     // possible.
00388     if (_read != (IStreamWrapper *)NULL) {
00389       delete _read;
00390     } else if (_write != (ostream *)NULL) {
00391       delete _write;
00392     }
00393   }
00394 
00395   _read = (IStreamWrapper *)NULL;
00396   _write = (ostream *)NULL;
00397   _offset = 0;
00398   _owns_stream = false;
00399   _next_index = 0;
00400   _last_index = 0;
00401   _needs_repack = false;
00402   _timestamp = 0;
00403   _timestamp_dirty = false;
00404   _scale_factor = 1;
00405   _new_scale_factor = 1;
00406   _encryption_flag = false;
00407   _file_major_ver = 0;
00408   _file_minor_ver = 0;
00409 
00410   _read_file.close();
00411   _write_file.close();
00412   _read_write_file.close();
00413   _multifile_name = Filename();
00414 
00415   clear_subfiles();
00416 }
00417 
00418 ////////////////////////////////////////////////////////////////////
00419 //     Function: Multifile::set_scale_factor
00420 //       Access: Published
00421 //  Description: Changes the internal scale factor for this Multifile.
00422 //
00423 //               This is normally 1, but it may be set to any
00424 //               arbitrary value (greater than zero) to support
00425 //               Multifile archives that exceed 4GB, if necessary.
00426 //               (Individual subfiles may still not exceed 4GB.)
00427 //
00428 //               All addresses within the file are rounded up to the
00429 //               next multiple of _scale_factor, and zeros are written
00430 //               to the file to fill the resulting gaps.  Then the
00431 //               address is divided by _scale_factor and written out
00432 //               as a 32-bit integer.  Thus, setting a scale factor of
00433 //               2 supports up to 8GB files, 3 supports 12GB files,
00434 //               etc.
00435 //
00436 //               Calling this function on an already-existing
00437 //               Multifile will have no immediate effect until a
00438 //               future call to repack() or close() (or until the
00439 //               Multifile is destructed).
00440 ////////////////////////////////////////////////////////////////////
00441 void Multifile::
00442 set_scale_factor(size_t scale_factor) {
00443   nassertv(is_write_valid());
00444   nassertv(scale_factor != (size_t)0);
00445 
00446   if (_next_index == (streampos)0) {
00447     // If it's a brand new Multifile, we can go ahead and set it
00448     // immediately.
00449     _scale_factor = scale_factor;
00450   } else {
00451     // Otherwise, we'd better have read access so we can repack it
00452     // later.
00453     nassertv(is_read_valid());
00454   }
00455 
00456   // Setting the _new_scale_factor different from the _scale_factor
00457   // will force a repack operation on close.
00458   _new_scale_factor = scale_factor;
00459 }
00460 
00461 ////////////////////////////////////////////////////////////////////
00462 //     Function: Multifile::add_subfile
00463 //       Access: Published
00464 //  Description: Adds a file on disk as a subfile to the Multifile.
00465 //               The file named by filename will be read and added to
00466 //               the Multifile at the next call to flush().  If there
00467 //               already exists a subfile with the indicated name, it
00468 //               is replaced without examining its contents (but see
00469 //               also update_subfile).
00470 //
00471 //               Filename::set_binary() or set_text() must have been
00472 //               called previously to specify the nature of the source
00473 //               file.  If set_text() was called, the text flag will
00474 //               be set on the subfile.
00475 //
00476 //               Returns the subfile name on success (it might have
00477 //               been modified slightly), or empty string on failure.
00478 ////////////////////////////////////////////////////////////////////
00479 string Multifile::
00480 add_subfile(const string &subfile_name, const Filename &filename,
00481             int compression_level) {
00482   nassertr(is_write_valid(), string());
00483 
00484   Filename fname = filename;
00485   if (multifile_always_binary) {
00486     fname.set_binary();
00487   }
00488 
00489   nassertr(fname.is_binary_or_text(), string());
00490 
00491   if (!fname.exists()) {
00492     return string();
00493   }
00494   string name = standardize_subfile_name(subfile_name);
00495   if (!name.empty()) {
00496     Subfile *subfile = new Subfile;
00497     subfile->_name = name;
00498     subfile->_source_filename = fname;
00499     if (fname.is_text()) {
00500       subfile->_flags |= SF_text;
00501     }
00502     
00503     add_new_subfile(subfile, compression_level);
00504   }
00505 
00506   _timestamp = time(NULL);
00507   _timestamp_dirty = true;
00508 
00509   return name;
00510 }
00511 
00512 ////////////////////////////////////////////////////////////////////
00513 //     Function: Multifile::add_subfile
00514 //       Access: Public
00515 //  Description: Adds a file from a stream as a subfile to the Multifile.
00516 //               The indicated istream will be read and its contents
00517 //               added to the Multifile at the next call to flush().
00518 //               The file will be added as a binary subfile.
00519 //
00520 //               Note that the istream must remain untouched and
00521 //               unused by any other code until flush() is called.  At
00522 //               that time, the Multifile will read the entire
00523 //               contents of the istream from the current file
00524 //               position to the end of the file.  Subsequently, the
00525 //               Multifile will *not* close or delete the istream.  It
00526 //               is the caller's responsibility to ensure that the
00527 //               istream pointer does not destruct during the lifetime
00528 //               of the Multifile.
00529 //
00530 //               Returns the subfile name on success (it might have
00531 //               been modified slightly), or empty string on failure.
00532 ////////////////////////////////////////////////////////////////////
00533 string Multifile::
00534 add_subfile(const string &subfile_name, istream *subfile_data,
00535             int compression_level) {
00536   nassertr(is_write_valid(), string());
00537 
00538   string name = standardize_subfile_name(subfile_name);
00539   if (!name.empty()) {
00540     Subfile *subfile = new Subfile;
00541     subfile->_name = name;
00542     subfile->_source = subfile_data;
00543     add_new_subfile(subfile, compression_level);
00544   }
00545 
00546   return name;
00547 }
00548 
00549 ////////////////////////////////////////////////////////////////////
00550 //     Function: Multifile::update_subfile
00551 //       Access: Published
00552 //  Description: Adds a file on disk to the subfile.  If a subfile
00553 //               already exists with the same name, its contents are
00554 //               compared byte-for-byte to the disk file, and it is
00555 //               replaced only if it is different; otherwise, the
00556 //               multifile is left unchanged.
00557 //
00558 //               Filename::set_binary() or set_text() must have been
00559 //               called previously to specify the nature of the source
00560 //               file.  If set_text() was called, the text flag will
00561 //               be set on the subfile.
00562 ////////////////////////////////////////////////////////////////////
00563 string Multifile::
00564 update_subfile(const string &subfile_name, const Filename &filename,
00565                int compression_level) {
00566   nassertr(is_write_valid(), string());
00567 
00568   Filename fname = filename;
00569   if (multifile_always_binary) {
00570     fname.set_binary();
00571   }
00572 
00573   nassertr(fname.is_binary_or_text(), string());
00574 
00575   if (!fname.exists()) {
00576     return string();
00577   }
00578   string name = standardize_subfile_name(subfile_name);
00579   if (!name.empty()) {
00580     int index = find_subfile(name);
00581     if (index >= 0) {
00582       // The subfile already exists; compare it to the source file.
00583       if (compare_subfile(index, fname)) {
00584         // The files are identical; do nothing.
00585         return name;
00586       }
00587     }
00588 
00589     // The subfile does not already exist or it is different from the
00590     // source file.  Add the new source file.
00591     Subfile *subfile = new Subfile;
00592     subfile->_name = name;
00593     subfile->_source_filename = fname;
00594     if (fname.is_text()) {
00595       subfile->_flags |= SF_text;
00596     }
00597 
00598     add_new_subfile(subfile, compression_level);
00599   }
00600 
00601   _timestamp = time(NULL);
00602   _timestamp_dirty = true;
00603 
00604   return name;
00605 }
00606 
00607 #ifdef HAVE_OPENSSL
00608 ////////////////////////////////////////////////////////////////////
00609 //     Function: Multifile::add_signature
00610 //       Access: Published
00611 //  Description: Adds a new signature to the Multifile.  This
00612 //               signature associates the indicated certificate with
00613 //               the current contents of the Multifile.  When the
00614 //               Multifile is read later, the signature will still be
00615 //               present only if the Multifile is unchanged; any
00616 //               subsequent changes to the Multifile will
00617 //               automatically invalidate and remove the signature.
00618 //
00619 //               The chain filename may be empty if the certificate
00620 //               does not require an authenticating certificate chain
00621 //               (e.g. because it is self-signed).
00622 //
00623 //               The specified private key must match the certificate,
00624 //               and the Multifile must be open in read-write mode.
00625 //               The private key is only used for generating the
00626 //               signature; it is not written to the Multifile and
00627 //               cannot be retrieved from the Multifile later.
00628 //               (However, the certificate *can* be retrieved from the
00629 //               Multifile later, to identify the entity that created
00630 //               the signature.)
00631 //
00632 //               This implicitly causes a repack() operation if one is
00633 //               needed.  Returns true on success, false on failure.
00634 //
00635 //               This flavor of add_signature() reads the certificate
00636 //               and private key from a PEM-formatted file, for
00637 //               instance as generated by the openssl command.  If the
00638 //               private key file is password-encrypted, the third
00639 //               parameter will be used as the password to decrypt it.
00640 ////////////////////////////////////////////////////////////////////
00641 bool Multifile::
00642 add_signature(const Filename &certificate, const Filename &chain,
00643               const Filename &pkey, const string &password) {
00644   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00645 
00646   if (chain.empty() && pkey.empty()) {
00647     // If the second two filenames are empty, assume we're going for
00648     // the composite mode, where everything's stuffed into the first
00649     // file.
00650     return add_signature(certificate, password);
00651   }
00652 
00653   CertChain cert_chain;
00654 
00655   // Read the certificate file from VFS.  First, read the complete
00656   // file into memory.
00657   string certificate_data;
00658   if (!vfs->read_file(certificate, certificate_data, true)) {
00659     express_cat.info()
00660       << "Could not read " << certificate << ".\n";
00661     return false;
00662   }
00663 
00664   // Create an in-memory BIO to read the "file" from the buffer.
00665   BIO *certificate_mbio = BIO_new_mem_buf((void *)certificate_data.data(), certificate_data.size());
00666   X509 *x509 = PEM_read_bio_X509(certificate_mbio, NULL, NULL, (void *)"");
00667   BIO_free(certificate_mbio);
00668   if (x509 == NULL) {
00669     express_cat.info()
00670       << "Could not read certificate in " << certificate << ".\n";
00671     return false;
00672   }
00673 
00674   // Store the first X509--the actual certificate--as the first record
00675   // in our CertChain object.
00676   cert_chain.push_back(CertRecord(x509));
00677 
00678   // Read the rest of the certificates in the chain file.
00679   if (!chain.empty()) {
00680     string chain_data;
00681     if (!vfs->read_file(chain, chain_data, true)) {
00682       express_cat.info()
00683         << "Could not read " << chain << ".\n";
00684       return false;
00685     }
00686 
00687     BIO *chain_mbio = BIO_new_mem_buf((void *)chain_data.data(), chain_data.size());
00688     X509 *c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)"");
00689     while (c != NULL) {
00690       cert_chain.push_back(c);
00691       c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)"");
00692     }
00693     BIO_free(chain_mbio);
00694 
00695     if (cert_chain.size() == 1) {
00696       express_cat.info()
00697         << "Could not read certificate chain in " << chain << ".\n";
00698       return false;
00699     }
00700   }
00701 
00702   // Now do the same thing with the private key.  This one may be
00703   // password-encrypted on disk.
00704   string pkey_data;
00705   if (!vfs->read_file(pkey, pkey_data, true)) {
00706     express_cat.info()
00707       << "Could not read " << pkey << ".\n";
00708     return false;
00709   }
00710 
00711   BIO *pkey_mbio = BIO_new_mem_buf((void *)pkey_data.data(), pkey_data.size());
00712   EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, NULL, NULL,
00713                                                (void *)password.c_str());
00714   BIO_free(pkey_mbio);
00715   if (evp_pkey == NULL) {
00716     express_cat.info()
00717       << "Could not read private key in " << pkey << ".\n";
00718     return false;
00719   }
00720 
00721   bool result = add_signature(cert_chain, evp_pkey);
00722 
00723   EVP_PKEY_free(evp_pkey);
00724 
00725   return result;
00726 }
00727 #endif  // HAVE_OPENSSL
00728 
00729 #ifdef HAVE_OPENSSL
00730 ////////////////////////////////////////////////////////////////////
00731 //     Function: Multifile::add_signature
00732 //       Access: Published
00733 //  Description: Adds a new signature to the Multifile.  This
00734 //               signature associates the indicated certificate with
00735 //               the current contents of the Multifile.  When the
00736 //               Multifile is read later, the signature will still be
00737 //               present only if the Multifile is unchanged; any
00738 //               subsequent changes to the Multifile will
00739 //               automatically invalidate and remove the signature.
00740 //
00741 //               This flavor of add_signature() reads the certificate,
00742 //               private key, and certificate chain from the same
00743 //               PEM-formatted file.  It takes the first private key
00744 //               found as the intended key, and then uses the first
00745 //               certificate found that matches that key as the
00746 //               signing certificate.  Any other certificates in the
00747 //               file are taken to be part of the chain.
00748 ////////////////////////////////////////////////////////////////////
00749 bool Multifile::
00750 add_signature(const Filename &composite, const string &password) {
00751   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00752 
00753   // First, read the complete file into memory.
00754   string composite_data;
00755   if (!vfs->read_file(composite, composite_data, true)) {
00756     express_cat.info()
00757       << "Could not read " << composite << ".\n";
00758     return false;
00759   }
00760 
00761   // Get the private key.
00762   BIO *pkey_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size());
00763   EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, NULL, NULL,
00764                                                (void *)password.c_str());
00765   BIO_free(pkey_mbio);
00766   if (evp_pkey == NULL) {
00767     express_cat.info()
00768       << "Could not read private key in " << composite << ".\n";
00769     return false;
00770   }
00771 
00772   // Now read all of the certificates.
00773   CertChain cert_chain;
00774 
00775   BIO *chain_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size());
00776   X509 *c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)"");
00777   while (c != NULL) {
00778     cert_chain.push_back(c);
00779     c = PEM_read_bio_X509(chain_mbio, NULL, NULL, (void *)"");
00780   }
00781   BIO_free(chain_mbio);
00782 
00783   if (cert_chain.empty()) {
00784     express_cat.info()
00785       << "Could not read certificates in " << composite << ".\n";
00786     return false;
00787   }
00788 
00789   // Now find the certificate that matches the signature, and move it
00790   // to the front of the chain.
00791   size_t i;
00792   bool found_match = false;
00793   for (i = 0; i < cert_chain.size(); ++i) {
00794     X509 *c = cert_chain[i]._cert;
00795     if (X509_check_private_key(c, evp_pkey)) {
00796       found_match = true;
00797       if (i != 0) {
00798         // Move this entry to the beginning.
00799         cert_chain.insert(cert_chain.begin(), cert_chain[i]);
00800         cert_chain.erase(cert_chain.begin() + i + 1);
00801       }
00802       break;
00803     }
00804   }
00805 
00806   if (!found_match) {
00807     express_cat.info()
00808       << "No certificates in " << composite << " match key.\n";
00809     return false;
00810   }
00811 
00812   bool result = add_signature(cert_chain, evp_pkey);
00813 
00814   EVP_PKEY_free(evp_pkey);
00815 
00816   return result;
00817 }
00818 #endif  // HAVE_OPENSSL
00819 
00820 #ifdef HAVE_OPENSSL
00821 ////////////////////////////////////////////////////////////////////
00822 //     Function: Multifile::add_signature
00823 //       Access: Published
00824 //  Description: Adds a new signature to the Multifile.  This
00825 //               signature associates the indicated certificate with
00826 //               the current contents of the Multifile.  When the
00827 //               Multifile is read later, the signature will still be
00828 //               present only if the Multifile is unchanged; any
00829 //               subsequent changes to the Multifile will
00830 //               automatically invalidate and remove the signature.
00831 //
00832 //               If chain is non-NULL, it represents the certificate
00833 //               chain that validates the certificate.
00834 //
00835 //               The specified private key must match the certificate,
00836 //               and the Multifile must be open in read-write mode.
00837 //               The private key is only used for generating the
00838 //               signature; it is not written to the Multifile and
00839 //               cannot be retrieved from the Multifile later.
00840 //               (However, the certificate *can* be retrieved from the
00841 //               Multifile later, to identify the entity that created
00842 //               the signature.)
00843 //
00844 //               This implicitly causes a repack() operation if one is
00845 //               needed.  Returns true on success, false on failure.
00846 ////////////////////////////////////////////////////////////////////
00847 bool Multifile::
00848 add_signature(X509 *certificate, STACK_OF(X509) *chain, EVP_PKEY *pkey) {
00849   // Convert the certificate and chain into our own CertChain
00850   // structure.
00851   CertChain cert_chain;
00852   cert_chain.push_back(CertRecord(certificate));
00853   if (chain != NULL) {
00854     int num = sk_X509_num(chain);
00855     for (int i = 0; i < num; ++i) {
00856       cert_chain.push_back(CertRecord((X509 *)sk_X509_value(chain, i)));
00857     }
00858   }
00859 
00860   return add_signature(cert_chain, pkey);
00861 }
00862 #endif  // HAVE_OPENSSL
00863 
00864 #ifdef HAVE_OPENSSL
00865 ////////////////////////////////////////////////////////////////////
00866 //     Function: Multifile::add_signature
00867 //       Access: Published
00868 //  Description: Adds a new signature to the Multifile.  This
00869 //               signature associates the indicated certificate with
00870 //               the current contents of the Multifile.  When the
00871 //               Multifile is read later, the signature will still be
00872 //               present only if the Multifile is unchanged; any
00873 //               subsequent changes to the Multifile will
00874 //               automatically invalidate and remove the signature.
00875 //
00876 //               The signature certificate is the first certificate on
00877 //               the CertChain object.  Any remaining certificates are
00878 //               support certificates to authenticate the first one.
00879 //
00880 //               The specified private key must match the certificate,
00881 //               and the Multifile must be open in read-write mode.
00882 //               The private key is only used for generating the
00883 //               signature; it is not written to the Multifile and
00884 //               cannot be retrieved from the Multifile later.
00885 //               (However, the certificate *can* be retrieved from the
00886 //               Multifile later, to identify the entity that created
00887 //               the signature.)
00888 //
00889 //               This implicitly causes a repack() operation if one is
00890 //               needed.  Returns true on success, false on failure.
00891 ////////////////////////////////////////////////////////////////////
00892 bool Multifile::
00893 add_signature(const Multifile::CertChain &cert_chain, EVP_PKEY *pkey) {
00894   if (_needs_repack) {
00895     if (!repack()) {
00896       return false;
00897     }
00898   } else {
00899     if (!flush()) {
00900       return false;
00901     }
00902   }
00903 
00904   if (cert_chain.empty()) {
00905     express_cat.info()
00906       << "No certificate given.\n";
00907     return false;
00908   }
00909 
00910   if (pkey == NULL) {
00911     express_cat.info()
00912       << "No private key given.\n";
00913     return false;
00914   }
00915 
00916   if (!X509_check_private_key(cert_chain[0]._cert, pkey)) {
00917     express_cat.info()
00918       << "Private key does not match certificate.\n"; 
00919     return false;
00920   }
00921   
00922   // Now encode that list of certs to a stream in DER form.
00923   stringstream der_stream;
00924   StreamWriter der_writer(der_stream);
00925   der_writer.add_uint32(cert_chain.size());
00926 
00927   CertChain::const_iterator ci;
00928   for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) {
00929     X509 *cert = (*ci)._cert;
00930 
00931     int der_len = i2d_X509(cert, NULL);
00932     unsigned char *der_buf = new unsigned char[der_len];
00933     unsigned char *p = der_buf;
00934     i2d_X509(cert, &p);
00935     der_writer.append_data(der_buf, der_len);
00936     delete[] der_buf;
00937   }
00938 
00939   // Create a temporary Subfile for writing out the signature.
00940   der_stream.seekg(0);
00941   Subfile *subfile = new Subfile;
00942   subfile->_pkey = pkey;
00943   subfile->_flags |= SF_signature;
00944   subfile->_source = &der_stream;
00945 
00946   // Write the new Subfile at the end.  The cert_special subfiles
00947   // always go at the end, because they're not the part of the file
00948   // that's signed.
00949   nassertr(_new_subfiles.empty(), false);
00950   _new_subfiles.push_back(subfile);
00951   bool result = flush();
00952   
00953   delete subfile;
00954 
00955   return result;
00956 }
00957 #endif  // HAVE_OPENSSL
00958 
00959 #ifdef HAVE_OPENSSL
00960 ////////////////////////////////////////////////////////////////////
00961 //     Function: Multifile::get_num_signatures
00962 //       Access: Published
00963 //  Description: Returns the number of matching signatures found on
00964 //               the Multifile.  These signatures may be iterated via
00965 //               get_signature() and related methods.
00966 //
00967 //               A signature on this list is guaranteed to match the
00968 //               Multifile contents, proving that the Multifile has
00969 //               been unmodified since the signature was applied.
00970 //               However, this does not guarantee that the certificate
00971 //               itself is actually from who it says it is from; only
00972 //               that it matches the Multifile contents.  See
00973 //               validate_signature_certificate() to authenticate a
00974 //               particular certificate.
00975 ////////////////////////////////////////////////////////////////////
00976 int Multifile::
00977 get_num_signatures() const {
00978   ((Multifile *)this)->check_signatures();
00979   return _signatures.size();
00980 }
00981 #endif  // HAVE_OPENSSL
00982 
00983 #ifdef HAVE_OPENSSL
00984 ////////////////////////////////////////////////////////////////////
00985 //     Function: Multifile::get_signature
00986 //       Access: Published
00987 //  Description: Returns the nth signature found on the Multifile.
00988 //               See the comments in get_num_signatures().
00989 ////////////////////////////////////////////////////////////////////
00990 const Multifile::CertChain &Multifile::
00991 get_signature(int n) const {
00992   ((Multifile *)this)->check_signatures();
00993   static CertChain error_chain;
00994   nassertr(n >= 0 && n < (int)_signatures.size(), error_chain);
00995   return _signatures[n];
00996 }
00997 #endif  // HAVE_OPENSSL
00998 
00999 #ifdef HAVE_OPENSSL
01000 ////////////////////////////////////////////////////////////////////
01001 //     Function: Multifile::get_signature_subject_name
01002 //       Access: Published
01003 //  Description: Returns the "subject name" for the nth signature found
01004 //               on the Multifile.  This is a string formatted
01005 //               according to RFC2253 that should more-or-less
01006 //               identify a particular certificate; when paired with
01007 //               the public key (see get_signature_public_key()), it
01008 //               can uniquely identify a certificate.  See the
01009 //               comments in get_num_signatures().
01010 ////////////////////////////////////////////////////////////////////
01011 string Multifile::
01012 get_signature_subject_name(int n) const {
01013   const CertChain &cert_chain = get_signature(n);
01014   nassertr(!cert_chain.empty(), string());
01015 
01016   X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert);
01017   if (xname != NULL) {
01018     // We use "print" to dump the output to a memory BIO.  Is
01019     // there an easier way to extract the X509_NAME text?  Curse
01020     // these incomplete docs.
01021     BIO *mbio = BIO_new(BIO_s_mem());
01022     X509_NAME_print_ex(mbio, xname, 0, XN_FLAG_RFC2253);
01023 
01024     char *pp;
01025     long pp_size = BIO_get_mem_data(mbio, &pp);
01026     string name(pp, pp_size);
01027     BIO_free(mbio);
01028     return name;
01029   }
01030 
01031   return string();
01032 }
01033 #endif  // HAVE_OPENSSL
01034 
01035 #ifdef HAVE_OPENSSL
01036 ////////////////////////////////////////////////////////////////////
01037 //     Function: Multifile::get_signature_friendly_name
01038 //       Access: Published
01039 //  Description: Returns a "friendly name" for the nth signature found
01040 //               on the Multifile.  This attempts to extract out the
01041 //               most meaningful part of the subject name.  It returns
01042 //               the emailAddress, if it is defined; otherwise, it
01043 //               returns the commonName.
01044 
01045 //               See the comments in get_num_signatures().
01046 ////////////////////////////////////////////////////////////////////
01047 string Multifile::
01048 get_signature_friendly_name(int n) const {
01049   const CertChain &cert_chain = get_signature(n);
01050   nassertr(!cert_chain.empty(), string());
01051 
01052   static const int nid_choices[] = {
01053     NID_pkcs9_emailAddress,
01054     NID_subject_alt_name,
01055     NID_commonName,
01056     -1,
01057   };
01058 
01059   // Choose the first NID that exists on the cert.
01060   for (int ni = 0; nid_choices[ni] != -1; ++ni) {
01061     int nid = nid_choices[ni];
01062 
01063     // A complex OpenSSL interface to extract out the name in utf-8.
01064     X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert);
01065     if (xname != NULL) {
01066       int pos = X509_NAME_get_index_by_NID(xname, nid, -1);
01067       if (pos != -1) {
01068         // We just get the first common name.  I guess it's possible to
01069         // have more than one; not sure what that means in this context.
01070         X509_NAME_ENTRY *xentry = X509_NAME_get_entry(xname, pos);
01071         if (xentry != NULL) {
01072           ASN1_STRING *data = X509_NAME_ENTRY_get_data(xentry);
01073           if (data != NULL) {
01074             // We use "print" to dump the output to a memory BIO.  Is
01075             // there an easier way to decode the ASN1_STRING?  Curse
01076             // these incomplete docs.
01077             BIO *mbio = BIO_new(BIO_s_mem());
01078             ASN1_STRING_print_ex(mbio, data, ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB);
01079             
01080             char *pp;
01081             long pp_size = BIO_get_mem_data(mbio, &pp);
01082             string name(pp, pp_size);
01083             BIO_free(mbio);
01084             return name;
01085           }
01086         }
01087       }
01088     }
01089   }
01090 
01091   return string();
01092 }
01093 #endif  // HAVE_OPENSSL
01094 
01095 #ifdef HAVE_OPENSSL
01096 ////////////////////////////////////////////////////////////////////
01097 //     Function: Multifile::get_signature_public_key
01098 //       Access: Published
01099 //  Description: Returns the public key used for the nth signature
01100 //               found on the Multifile.  This is encoded in DER form
01101 //               and returned as a string of hex digits.
01102 //
01103 //               This can be used, in conjunction with the subject
01104 //               name (see get_signature_subject_name()), to uniquely
01105 //               identify a particular certificate and its subsequent
01106 //               reissues.  See the comments in get_num_signatures().
01107 ////////////////////////////////////////////////////////////////////
01108 string Multifile::
01109 get_signature_public_key(int n) const {
01110   const CertChain &cert_chain = get_signature(n);
01111   nassertr(!cert_chain.empty(), string());
01112 
01113   EVP_PKEY *pkey = X509_get_pubkey(cert_chain[0]._cert);
01114   if (pkey != NULL) {
01115     int key_len = i2d_PublicKey(pkey, NULL);
01116     unsigned char *key_buf = new unsigned char[key_len];
01117     unsigned char *p = key_buf;
01118     i2d_PublicKey(pkey, &p);
01119     string result;
01120     for (int i = 0; i < key_len; ++i) {
01121       result += tohex(key_buf[i] >> 4);
01122       result += tohex(key_buf[i]);
01123     }
01124     delete[] key_buf;
01125     return result;
01126   }
01127 
01128   return string();
01129 }
01130 #endif  // HAVE_OPENSSL
01131 
01132 #ifdef HAVE_OPENSSL
01133 ////////////////////////////////////////////////////////////////////
01134 //     Function: Multifile::print_signature_certificate
01135 //       Access: Published
01136 //  Description: Writes the certificate for the nth signature, in
01137 //               user-readable verbose form, to the indicated stream.
01138 //               See the comments in get_num_signatures().
01139 ////////////////////////////////////////////////////////////////////
01140 void Multifile::
01141 print_signature_certificate(int n, ostream &out) const {
01142   const CertChain &cert_chain = get_signature(n);
01143   nassertv(!cert_chain.empty());
01144 
01145   BIO *mbio = BIO_new(BIO_s_mem());
01146   X509_print(mbio, cert_chain[0]._cert);
01147 
01148   char *pp;
01149   long pp_size = BIO_get_mem_data(mbio, &pp);
01150   out.write(pp, pp_size);
01151   BIO_free(mbio);
01152 }
01153 #endif  // HAVE_OPENSSL
01154 
01155 #ifdef HAVE_OPENSSL
01156 ////////////////////////////////////////////////////////////////////
01157 //     Function: Multifile::write_signature_certificate
01158 //       Access: Published
01159 //  Description: Writes the certificate for the nth signature, in
01160 //               PEM form, to the indicated stream.  See the comments
01161 //               in get_num_signatures().
01162 ////////////////////////////////////////////////////////////////////
01163 void Multifile::
01164 write_signature_certificate(int n, ostream &out) const {
01165   const CertChain &cert_chain = get_signature(n);
01166   nassertv(!cert_chain.empty());
01167 
01168   BIO *mbio = BIO_new(BIO_s_mem());
01169 
01170   CertChain::const_iterator ci;
01171   for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) {
01172     X509 *c = (*ci)._cert;
01173     X509_print(mbio, c);
01174     PEM_write_bio_X509(mbio, c);
01175   }
01176 
01177   char *pp;
01178   long pp_size = BIO_get_mem_data(mbio, &pp);
01179   out.write(pp, pp_size);
01180   BIO_free(mbio);
01181 }
01182 #endif  // HAVE_OPENSSL
01183 
01184 #ifdef HAVE_OPENSSL
01185 ////////////////////////////////////////////////////////////////////
01186 //     Function: Multifile::validate_signature_certificate
01187 //       Access: Published
01188 //  Description: Checks that the certificate used for the nth
01189 //               signature is a valid, authorized certificate with
01190 //               some known certificate authority.  Returns 0 if it
01191 //               is valid, -1 if there is some error, or the
01192 //               corresponding OpenSSL error code if it is invalid,
01193 //               out-of-date, or self-signed.
01194 ////////////////////////////////////////////////////////////////////
01195 int Multifile::
01196 validate_signature_certificate(int n) const {
01197   int verify_result = -1;
01198 
01199   const CertChain &chain = get_signature(n);
01200   nassertr(!chain.empty(), false);
01201 
01202   OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
01203 
01204   // Copy our CertChain structure into an X509 pointer and
01205   // accompanying STACK_OF(X509) pointer.
01206   X509 *x509 = chain[0]._cert;
01207   STACK_OF(X509) *stack = NULL;
01208   if (chain.size() > 1) {
01209     stack = sk_X509_new(NULL);
01210     for (size_t n = 1; n < chain.size(); ++n) {
01211       sk_X509_push(stack, chain[n]._cert);
01212     }
01213   }
01214 
01215   // Create the X509_STORE_CTX for verifying the cert and chain.
01216   X509_STORE_CTX *ctx = X509_STORE_CTX_new();
01217   X509_STORE_CTX_init(ctx, sslw->get_x509_store(), x509, stack);
01218   X509_STORE_CTX_set_cert(ctx, x509);
01219 
01220   if (X509_verify_cert(ctx)) {
01221     verify_result = 0;
01222   } else {
01223     verify_result = X509_STORE_CTX_get_error(ctx);
01224   }
01225 
01226   if (express_cat.is_debug()) {
01227     express_cat.debug()
01228       << get_signature_subject_name(n) << ": validate " << verify_result
01229       << "\n";
01230   }
01231 
01232   sk_X509_free(stack);
01233   X509_STORE_CTX_cleanup(ctx);
01234   X509_STORE_CTX_free(ctx);
01235 
01236   return verify_result;
01237 }
01238 #endif // HAVE_OPENSSL
01239 
01240 ////////////////////////////////////////////////////////////////////
01241 //     Function: Multifile::flush
01242 //       Access: Published
01243 //  Description: Writes all contents of the Multifile to disk.  Until
01244 //               flush() is called, add_subfile() and remove_subfile()
01245 //               do not actually do anything to disk.  At this point,
01246 //               all of the recently-added subfiles are read and their
01247 //               contents are added to the end of the Multifile, and
01248 //               the recently-removed subfiles are marked gone from
01249 //               the Multifile.
01250 //
01251 //               This may result in a suboptimal index.  To guarantee
01252 //               that the index is written at the beginning of the
01253 //               file, call repack() instead of flush().
01254 //
01255 //               It is not necessary to call flush() explicitly unless
01256 //               you are concerned about reading the recently-added
01257 //               subfiles immediately.
01258 //
01259 //               Returns true on success, false on failure.
01260 ////////////////////////////////////////////////////////////////////
01261 bool Multifile::
01262 flush() {
01263   if (!is_write_valid()) {
01264     return false;
01265   }
01266 
01267   bool new_file = (_next_index == (streampos)0);
01268   if (new_file) {
01269     // If we don't have an index yet, we don't have a header.  Write
01270     // the header.
01271     if (!write_header()) {
01272       return false;
01273     }
01274 
01275   } else {
01276     if (_file_minor_ver != _current_minor_ver) {
01277       // If we *do* have an index already, but this is an old version
01278       // multifile, we have to completely rewrite it anyway.
01279       return repack();
01280     }
01281   }
01282 
01283   nassertr(_write != (ostream *)NULL, false);
01284 
01285   // First, mark out all of the removed subfiles.
01286   PendingSubfiles::iterator pi;
01287   for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
01288     Subfile *subfile = (*pi);
01289     subfile->rewrite_index_flags(*_write);
01290     delete subfile;
01291   }
01292   _removed_subfiles.clear();
01293 
01294   bool wrote_ok = true;
01295 
01296   if (!_new_subfiles.empty() || new_file) {
01297     // Add a few more files to the end.  We always add subfiles at the
01298     // end of the multifile, so go there first.
01299     sort(_new_subfiles.begin(), _new_subfiles.end(), IndirectLess<Subfile>());
01300     if (_last_index != (streampos)0) {
01301       _write->seekp(0, ios::end);
01302       if (_write->fail()) {
01303         express_cat.info()
01304           << "Unable to seek Multifile " << _multifile_name << ".\n";
01305         return false;
01306       }
01307       _next_index = _write->tellp();
01308       _next_index = pad_to_streampos(_next_index);
01309 
01310       // And update the forward link from the last_index to point to
01311       // this new index location.
01312       _write->seekp(_last_index);
01313       StreamWriter writer(_write, false);
01314       writer.add_uint32(streampos_to_word(_next_index));
01315     }
01316 
01317     _write->seekp(_next_index);
01318     nassertr(_next_index == _write->tellp(), false);
01319     
01320     // Ok, here we are at the end of the file.  Write out the
01321     // recently-added subfiles here.  First, count up the index size.
01322     for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
01323       Subfile *subfile = (*pi);
01324       _last_index = _next_index;
01325       _next_index = subfile->write_index(*_write, _next_index, this);
01326       nassertr(_next_index == _write->tellp(), false);
01327       _next_index = pad_to_streampos(_next_index);
01328       nassertr(_next_index == _write->tellp(), false);
01329     }
01330     
01331     // Now we're at the end of the index.  Write a 0 here to mark the
01332     // end.
01333     StreamWriter writer(_write, false);
01334     writer.add_uint32(0);
01335     _next_index += 4;
01336     nassertr(_next_index == _write->tellp(), false);
01337     _next_index = pad_to_streampos(_next_index);
01338 
01339     // All right, now write out each subfile's data.
01340     for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
01341       Subfile *subfile = (*pi);
01342 
01343       if (_read != (IStreamWrapper *)NULL) {
01344         _read->acquire();
01345         _next_index = subfile->write_data(*_write, _read->get_istream(),
01346                                           _next_index, this);
01347         _read->release();
01348 
01349       } else {
01350         _next_index = subfile->write_data(*_write, NULL, _next_index, this);
01351       }
01352 
01353       nassertr(_next_index == _write->tellp(), false);
01354       _next_index = pad_to_streampos(_next_index);
01355       if (subfile->is_data_invalid()) {
01356         wrote_ok = false;
01357       }
01358 
01359       if (!subfile->is_cert_special()) {
01360         _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos());
01361       }
01362       nassertr(_next_index == _write->tellp(), false);
01363     }
01364     
01365     // Now go back and fill in the proper addresses for the data start.
01366     // We didn't do it in the first pass, because we don't really want
01367     // to keep all those file handles open, and so we didn't have to
01368     // determine each file's length ahead of time.
01369     for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
01370       Subfile *subfile = (*pi);
01371       subfile->rewrite_index_data_start(*_write, this);
01372     }
01373 
01374     _new_subfiles.clear();
01375   }
01376 
01377   // Also update the overall timestamp.
01378   if (_timestamp_dirty) {
01379     nassertr(!_write->fail(), false);
01380     static const size_t timestamp_pos = _header_prefix.size() + _header_size + 2 + 2 + 4;
01381     _write->seekp(timestamp_pos);
01382     nassertr(!_write->fail(), false);
01383     
01384     StreamWriter writer(*_write);
01385     if (_record_timestamp) {
01386       writer.add_uint32(_timestamp);
01387     } else {
01388       writer.add_uint32(0);
01389     }
01390     _timestamp_dirty = false;
01391   }
01392 
01393   _write->flush();
01394   if (!wrote_ok || _write->fail()) {
01395     express_cat.info()
01396       << "Unable to update Multifile " << _multifile_name << ".\n";
01397     close();
01398     return false;
01399   }
01400 
01401   return true;
01402 }
01403 
01404 ////////////////////////////////////////////////////////////////////
01405 //     Function: Multifile::repack
01406 //       Access: Published
01407 //  Description: Forces a complete rewrite of the Multifile and all of
01408 //               its contents, so that its index will appear at the
01409 //               beginning of the file with all of the subfiles listed
01410 //               in alphabetical order.  This is considered optimal
01411 //               for reading, and is the standard configuration; but
01412 //               it is not essential to do this.
01413 //
01414 //               It is only valid to call this if the Multifile was
01415 //               opened using open_read_write() and an explicit
01416 //               filename, rather than an iostream.  Also, we must
01417 //               have write permission to the directory containing the
01418 //               Multifile.
01419 //
01420 //               Returns true on success, false on failure.
01421 ////////////////////////////////////////////////////////////////////
01422 bool Multifile::
01423 repack() {
01424   if (_next_index == (streampos)0) {
01425     // If the Multifile hasn't yet been written, this is really just a
01426     // flush operation.
01427     _needs_repack = false;
01428     return flush();
01429   }
01430 
01431   nassertr(is_write_valid() && is_read_valid(), false);
01432   nassertr(!_multifile_name.empty(), false);
01433 
01434   // First, we open a temporary filename to copy the Multifile to.
01435   Filename dirname = _multifile_name.get_dirname();
01436   if (dirname.empty()) {
01437     dirname = ".";
01438   }
01439   Filename temp_filename = Filename::temporary(dirname, "mftemp");
01440   temp_filename.set_binary();
01441   pofstream temp;
01442   if (!temp_filename.open_write(temp)) {
01443     express_cat.info()
01444       << "Unable to open temporary file " << temp_filename << "\n";
01445     return false;
01446   }
01447 
01448   // Now we scrub our internal structures so it looks like we're a
01449   // brand new Multifile.
01450   PendingSubfiles::iterator pi;
01451   for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
01452     Subfile *subfile = (*pi);
01453     delete subfile;
01454   }
01455   _removed_subfiles.clear();
01456   _new_subfiles.clear();
01457   copy(_subfiles.begin(), _subfiles.end(), back_inserter(_new_subfiles));
01458   _next_index = 0;
01459   _last_index = 0;
01460   _last_data_byte = 0;
01461   _scale_factor = _new_scale_factor;
01462 
01463   // And we write our contents to our new temporary file.
01464   _write = &temp;
01465   if (!flush()) {
01466     temp.close();
01467     temp_filename.unlink();
01468     return false;
01469   }
01470 
01471   // Now close everything, and move the temporary file back over our
01472   // original file.
01473   Filename orig_name = _multifile_name;
01474   temp.close();
01475   close();
01476   orig_name.unlink();
01477   if (!temp_filename.rename_to(orig_name)) {
01478     express_cat.info()
01479       << "Unable to rename temporary file " << temp_filename << " to "
01480       << orig_name << ".\n";
01481     return false;
01482   }
01483 
01484   if (!open_read_write(orig_name)) {
01485     express_cat.info()
01486       << "Unable to read newly repacked " << _multifile_name 
01487       << ".\n";
01488     return false;
01489   }
01490 
01491   return true;
01492 }
01493 
01494 ////////////////////////////////////////////////////////////////////
01495 //     Function: Multifile::get_num_subfiles
01496 //       Access: Published
01497 //  Description: Returns the number of subfiles within the Multifile.
01498 //               The subfiles may be accessed in alphabetical order by
01499 //               iterating through [0 .. get_num_subfiles()).
01500 ////////////////////////////////////////////////////////////////////
01501 int Multifile::
01502 get_num_subfiles() const {
01503   return _subfiles.size();
01504 }
01505 
01506 ////////////////////////////////////////////////////////////////////
01507 //     Function: Multifile::find_subfile
01508 //       Access: Published
01509 //  Description: Returns the index of the subfile with the indicated
01510 //               name, or -1 if the named subfile is not within the
01511 //               Multifile.
01512 ////////////////////////////////////////////////////////////////////
01513 int Multifile::
01514 find_subfile(const string &subfile_name) const {
01515   Subfile find_subfile;
01516   find_subfile._name = standardize_subfile_name(subfile_name);
01517   Subfiles::const_iterator fi;
01518   fi = _subfiles.find(&find_subfile);
01519   if (fi == _subfiles.end()) {
01520     // Not present.
01521     return -1;
01522   }
01523   return (fi - _subfiles.begin());
01524 }
01525 
01526 ////////////////////////////////////////////////////////////////////
01527 //     Function: Multifile::has_directory
01528 //       Access: Published
01529 //  Description: Returns true if the indicated subfile name is the
01530 //               directory prefix to one or more files within the
01531 //               Multifile.  That is, the Multifile contains at least
01532 //               one file named "subfile_name/...".
01533 ////////////////////////////////////////////////////////////////////
01534 bool Multifile::
01535 has_directory(const string &subfile_name) const {
01536   string prefix = subfile_name;
01537   if (!prefix.empty()) {
01538     prefix += '/';
01539   }
01540   Subfile find_subfile;
01541   find_subfile._name = prefix;
01542   Subfiles::const_iterator fi;
01543   fi = _subfiles.upper_bound(&find_subfile);
01544   if (fi == _subfiles.end()) {
01545     // Not present.
01546     return false;
01547   }
01548 
01549   // At least one subfile exists whose name sorts after prefix.  If it
01550   // contains prefix as the initial substring, then we have a match.
01551   Subfile *subfile = (*fi);
01552   return (subfile->_name.length() > prefix.length() &&
01553           subfile->_name.substr(0, prefix.length()) == prefix);
01554 }
01555 
01556 ////////////////////////////////////////////////////////////////////
01557 //     Function: Multifile::scan_directory
01558 //       Access: Published
01559 //  Description: Considers subfile_name to be the name of a
01560 //               subdirectory within the Multifile, but not a file
01561 //               itself; fills the given vector up with the sorted list
01562 //               of subdirectories or files within the named
01563 //               directory.
01564 //
01565 //               Note that directories do not exist explicitly within
01566 //               a Multifile; this just checks for the existence of
01567 //               files with the given initial prefix.
01568 //
01569 //               Returns true if successful, false otherwise.
01570 ////////////////////////////////////////////////////////////////////
01571 bool Multifile::
01572 scan_directory(vector_string &contents, const string &subfile_name) const {
01573   string prefix = subfile_name;
01574   if (!prefix.empty()) {
01575     prefix += '/';
01576   }
01577   Subfile find_subfile;
01578   find_subfile._name = prefix;
01579   Subfiles::const_iterator fi;
01580   fi = _subfiles.upper_bound(&find_subfile);
01581 
01582   string previous = "";
01583   while (fi != _subfiles.end()) {
01584     Subfile *subfile = (*fi);
01585     if (!(subfile->_name.length() > prefix.length() &&
01586           subfile->_name.substr(0, prefix.length()) == prefix)) {
01587       // We've reached the end of the list of subfiles beneath the
01588       // indicated directory prefix.
01589       return true;
01590     }
01591 
01592     size_t slash = subfile->_name.find('/', prefix.length());
01593     string basename = subfile->_name.substr(prefix.length(), slash - prefix.length());
01594     if (basename != previous) {
01595       contents.push_back(basename);
01596       previous = basename;
01597     }
01598     ++fi;
01599   }
01600 
01601   return true;
01602 }
01603 
01604 ////////////////////////////////////////////////////////////////////
01605 //     Function: Multifile::remove_subfile
01606 //       Access: Published
01607 //  Description: Removes the nth subfile from the Multifile.  This
01608 //               will cause all subsequent index numbers to decrease
01609 //               by one.  The file will not actually be removed from
01610 //               the disk until the next call to flush().
01611 //
01612 //               Note that this does not actually remove the data from
01613 //               the indicated subfile; it simply removes it from the
01614 //               index.  The Multifile will not be reduced in size
01615 //               after this operation, until the next call to
01616 //               repack().
01617 ////////////////////////////////////////////////////////////////////
01618 void Multifile::
01619 remove_subfile(int index) {
01620   nassertv(is_write_valid());
01621   nassertv(index >= 0 && index < (int)_subfiles.size());
01622   Subfile *subfile = _subfiles[index];
01623   subfile->_flags |= SF_deleted;
01624   _removed_subfiles.push_back(subfile);
01625   _subfiles.erase(_subfiles.begin() + index);
01626 
01627   _timestamp = time(NULL);
01628   _timestamp_dirty = true;
01629 
01630   _needs_repack = true;
01631 }
01632 
01633 ////////////////////////////////////////////////////////////////////
01634 //     Function: Multifile::get_subfile_name
01635 //       Access: Published
01636 //  Description: Returns the name of the nth subfile.
01637 ////////////////////////////////////////////////////////////////////
01638 const string &Multifile::
01639 get_subfile_name(int index) const {
01640 #ifndef NDEBUG
01641   static string empty_string;
01642   nassertr(index >= 0 && index < (int)_subfiles.size(), empty_string);
01643 #endif
01644   return _subfiles[index]->_name;
01645 }
01646 
01647 ////////////////////////////////////////////////////////////////////
01648 //     Function: Multifile::get_subfile_length
01649 //       Access: Published
01650 //  Description: Returns the uncompressed data length of the nth
01651 //               subfile.  This might return 0 if the subfile has
01652 //               recently been added and flush() has not yet been
01653 //               called.
01654 ////////////////////////////////////////////////////////////////////
01655 size_t Multifile::
01656 get_subfile_length(int index) const {
01657   nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
01658   return _subfiles[index]->_uncompressed_length;
01659 }
01660 
01661 ////////////////////////////////////////////////////////////////////
01662 //     Function: Multifile::get_subfile_timestamp
01663 //       Access: Published
01664 //  Description: Returns the modification time of the nth
01665 //               subfile.  If this is called on an older .mf file,
01666 //               which did not store individual timestamps in the file
01667 //               (or if get_record_timestamp() is false), this will
01668 //               return the modification time of the overall
01669 //               multifile.
01670 ////////////////////////////////////////////////////////////////////
01671 time_t Multifile::
01672 get_subfile_timestamp(int index) const {
01673   nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
01674   if (!get_record_timestamp()) {
01675     return get_timestamp();
01676   } else {
01677     return _subfiles[index]->_timestamp;
01678   }
01679 }
01680 
01681 ////////////////////////////////////////////////////////////////////
01682 //     Function: Multifile::is_subfile_compressed
01683 //       Access: Published
01684 //  Description: Returns true if the indicated subfile has been
01685 //               compressed when stored within the archive, false
01686 //               otherwise.
01687 ////////////////////////////////////////////////////////////////////
01688 bool Multifile::
01689 is_subfile_compressed(int index) const {
01690   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01691   return (_subfiles[index]->_flags & SF_compressed) != 0;
01692 }
01693 
01694 ////////////////////////////////////////////////////////////////////
01695 //     Function: Multifile::is_subfile_encrypted
01696 //       Access: Published
01697 //  Description: Returns true if the indicated subfile has been
01698 //               encrypted when stored within the archive, false
01699 //               otherwise.
01700 ////////////////////////////////////////////////////////////////////
01701 bool Multifile::
01702 is_subfile_encrypted(int index) const {
01703   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01704   return (_subfiles[index]->_flags & SF_encrypted) != 0;
01705 }
01706 
01707 ////////////////////////////////////////////////////////////////////
01708 //     Function: Multifile::is_subfile_text
01709 //       Access: Published
01710 //  Description: Returns true if the indicated subfile represents text
01711 //               data, or false if it represents binary data.  If the
01712 //               file is text data, it may have been processed by
01713 //               end-of-line conversion when it was added.  (But the
01714 //               actual bits in the multifile will represent the
01715 //               standard Unix end-of-line convention, e.g. \n instead
01716 //               of \r\n.)
01717 ////////////////////////////////////////////////////////////////////
01718 bool Multifile::
01719 is_subfile_text(int index) const {
01720   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01721   return (_subfiles[index]->_flags & SF_text) != 0;
01722 }
01723 
01724 ////////////////////////////////////////////////////////////////////
01725 //     Function: Multifile::get_index_end
01726 //       Access: Published
01727 //  Description: Returns the first byte that is guaranteed to follow
01728 //               any index byte already written to disk in the
01729 //               Multifile.
01730 //
01731 //               This number is largely meaningless in many cases, but
01732 //               if needs_repack() is false, and the file is flushed,
01733 //               this will indicate the number of bytes in the header
01734 //               + index.  Everything at this byte position and later
01735 //               will be actual data.
01736 ////////////////////////////////////////////////////////////////////
01737 streampos Multifile::
01738 get_index_end() const {
01739   return normalize_streampos(_next_index + (streampos)4);
01740 }
01741 
01742 ////////////////////////////////////////////////////////////////////
01743 //     Function: Multifile::get_subfile_internal_start
01744 //       Access: Published
01745 //  Description: Returns the starting byte position within the
01746 //               Multifile at which the indicated subfile begins.
01747 //               This may be used, with get_subfile_internal_length(),
01748 //               for low-level access to the subfile, but usually it
01749 //               is better to use open_read_subfile() instead (which
01750 //               automatically decrypts and/or uncompresses the
01751 //               subfile data).
01752 ////////////////////////////////////////////////////////////////////
01753 streampos Multifile::
01754 get_subfile_internal_start(int index) const {
01755   nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
01756   return _subfiles[index]->_data_start;
01757 }
01758 
01759 ////////////////////////////////////////////////////////////////////
01760 //     Function: Multifile::get_subfile_internal_length
01761 //       Access: Published
01762 //  Description: Returns the number of bytes the indicated subfile
01763 //               consumes within the archive.  For compressed
01764 //               subfiles, this will generally be smaller than
01765 //               get_subfile_length(); for encrypted (but
01766 //               noncompressed) subfiles, it may be slightly
01767 //               different, for noncompressed and nonencrypted
01768 //               subfiles, it will be equal.
01769 ////////////////////////////////////////////////////////////////////
01770 size_t Multifile::
01771 get_subfile_internal_length(int index) const {
01772   nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
01773   return _subfiles[index]->_data_length;
01774 }
01775 
01776 ////////////////////////////////////////////////////////////////////
01777 //     Function: Multifile::open_read_subfile
01778 //       Access: Published
01779 //  Description: Returns an istream that may be used to read the
01780 //               indicated subfile.  You may seek() within this
01781 //               istream to your heart's content; even though it will
01782 //               be a reference to the already-opened pfstream of the
01783 //               Multifile itself, byte 0 appears to be the beginning
01784 //               of the subfile and EOF appears to be the end of the
01785 //               subfile.
01786 //
01787 //               The returned istream will have been allocated via
01788 //               new; you should pass the pointer to
01789 //               close_read_subfile() when you are finished with it to
01790 //               delete it and release its resources.
01791 //
01792 //               Any future calls to repack() or close() (or the
01793 //               Multifile destructor) will invalidate all currently
01794 //               open subfile pointers.
01795 //
01796 //               The return value will be NULL if the stream cannot be
01797 //               opened for some reason.
01798 ////////////////////////////////////////////////////////////////////
01799 istream *Multifile::
01800 open_read_subfile(int index) {
01801   nassertr(is_read_valid(), NULL);
01802   nassertr(index >= 0 && index < (int)_subfiles.size(), NULL);
01803   Subfile *subfile = _subfiles[index];
01804 
01805   if (subfile->_source != (istream *)NULL ||
01806       !subfile->_source_filename.empty()) {
01807     // The subfile has not yet been copied into the physical
01808     // Multifile.  Force a flush operation to incorporate it.
01809     flush();
01810 
01811     // That shouldn't change the subfile index or delete the subfile
01812     // pointer.
01813     nassertr(subfile == _subfiles[index], NULL);
01814   }
01815 
01816   return open_read_subfile(subfile);
01817 }
01818 
01819 ////////////////////////////////////////////////////////////////////
01820 //     Function: Multifile::close_read_subfile
01821 //       Access: Published, Static
01822 //  Description: Closes a file opened by a previous call to
01823 //               open_read_subfile().  This really just deletes the
01824 //               istream pointer, but it is recommended to use this
01825 //               interface instead of deleting it explicitly, to help
01826 //               work around compiler issues.
01827 ////////////////////////////////////////////////////////////////////
01828 void Multifile::
01829 close_read_subfile(istream *stream) {
01830   if (stream != (istream *)NULL) {
01831     // For some reason--compiler bug in gcc 3.2?--explicitly deleting
01832     // the stream pointer does not call the appropriate global delete
01833     // function; instead apparently calling the system delete
01834     // function.  So we call the delete function by hand instead.
01835 #if !defined(WIN32_VC) && !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW)
01836     stream->~istream();
01837     (*global_operator_delete)(stream);
01838 #else
01839     delete stream;
01840 #endif
01841   }
01842 }
01843 
01844 ////////////////////////////////////////////////////////////////////
01845 //     Function: Multifile::extract_subfile
01846 //       Access: Published
01847 //  Description: Extracts the nth subfile into a file with the given
01848 //               name.
01849 ////////////////////////////////////////////////////////////////////
01850 bool Multifile::
01851 extract_subfile(int index, const Filename &filename) {
01852   nassertr(is_read_valid(), false);
01853   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01854   
01855   Filename fname = filename;
01856   if (multifile_always_binary) {
01857     fname.set_binary();
01858   }
01859 
01860   if (!fname.is_binary_or_text()) {
01861     // If we haven't specified binary or text, infer it from the type
01862     // of the subfile.
01863     if ((_subfiles[index]->_flags & SF_text) != 0) {
01864       fname.set_text();
01865     } else {
01866       fname.set_binary();
01867     }
01868   }
01869   fname.make_dir();
01870   pofstream out;
01871   if (!fname.open_write(out, true)) {
01872     express_cat.info()
01873       << "Unable to write to file " << filename << "\n";
01874     return false;
01875   }
01876 
01877   return extract_subfile_to(index, out);
01878 }
01879 
01880 ////////////////////////////////////////////////////////////////////
01881 //     Function: Multifile::extract_subfile_to
01882 //       Access: Public
01883 //  Description: Extracts the nth subfile to the indicated ostream.
01884 ////////////////////////////////////////////////////////////////////
01885 bool Multifile::
01886 extract_subfile_to(int index, ostream &out) {
01887   nassertr(is_read_valid(), false);
01888   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01889 
01890   istream *in = open_read_subfile(index);
01891   if (in == (istream *)NULL) {
01892     return false;
01893   }
01894 
01895   static const size_t buffer_size = 4096;
01896   char buffer[buffer_size];
01897 
01898   in->read(buffer, buffer_size);
01899   size_t count = in->gcount();
01900   while (count != 0) {
01901     out.write(buffer, count);
01902     in->read(buffer, buffer_size);
01903     count = in->gcount();
01904   }
01905 
01906   bool failed = (in->fail() && !in->eof());
01907   close_read_subfile(in);
01908   nassertr(!failed, false);
01909 
01910   return (!out.fail());
01911 }
01912 
01913 ////////////////////////////////////////////////////////////////////
01914 //     Function: Multifile::compare_subfile
01915 //       Access: Published
01916 //  Description: Performs a byte-for-byte comparison of the indicated
01917 //               file on disk with the nth subfile.  Returns true if
01918 //               the files are equivalent, or false if they are
01919 //               different (or the file is missing).
01920 //
01921 //               If Filename::set_binary() or set_text() has already
01922 //               been called, it specifies the nature of the source
01923 //               file.  If this is different from the text flag of the
01924 //               subfile, the comparison will always return false.
01925 //               If this has not been specified, it will be set from
01926 //               the text flag of the subfile.
01927 ////////////////////////////////////////////////////////////////////
01928 bool Multifile::
01929 compare_subfile(int index, const Filename &filename) {
01930   nassertr(is_read_valid(), false);
01931   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
01932 
01933   if (!filename.exists()) {
01934     express_cat.info()
01935       << "File is missing: " << filename << "\n";
01936     return false;
01937   }
01938 
01939   Filename fname = filename;
01940   if (fname.is_binary()) {
01941     // If we've specified a binary file, it had better be a binary
01942     // subfile.
01943     if ((_subfiles[index]->_flags & SF_text) != 0) {
01944       if (express_cat.is_debug()) {
01945         express_cat.debug()
01946           << "File is not binary: " << filename << "\n";
01947       }
01948       return false;
01949     }
01950 
01951   } else if (fname.is_text()) {
01952     // If we've specified a text file, it had better be a text
01953     // subfile.
01954     if ((_subfiles[index]->_flags & SF_text) == 0) {
01955       if (express_cat.is_debug()) {
01956         express_cat.debug()
01957           << "File is not text: " << filename << "\n";
01958       }
01959       return false;
01960     }
01961 
01962   } else {
01963     // If we haven't specified binary or text, infer it from the type
01964     // of the subfile.
01965     if ((_subfiles[index]->_flags & SF_text) != 0) {
01966       fname.set_text();
01967     } else {
01968       fname.set_binary();
01969     }
01970   }
01971 
01972   istream *in1 = open_read_subfile(index);
01973   if (in1 == (istream *)NULL) {
01974     return false;
01975   }
01976 
01977   pifstream in2;
01978 
01979   if (!fname.open_read(in2)) {
01980     express_cat.info()
01981       << "Cannot read " << filename << "\n";
01982     return false;
01983   }
01984 
01985   if (fname.is_binary()) {
01986     // Check the file size.
01987     in2.seekg(0, ios::end);
01988     streampos file_size = in2.tellg();
01989     
01990     if (file_size != (streampos)get_subfile_length(index)) {
01991       // The files have different sizes.
01992       close_read_subfile(in1);
01993       return false;
01994     }
01995   }
01996 
01997   // Check the file data, byte-for-byte.
01998   in2.seekg(0);
01999   int byte1 = in1->get();
02000   int byte2 = in2.get();
02001   while (!in1->fail() && !in1->eof() &&
02002          !in2.fail() && !in2.eof()) {
02003     if (byte1 != byte2) { 
02004       close_read_subfile(in1);
02005       return false;
02006     }
02007     byte1 = in1->get();
02008     byte2 = in2.get();
02009   }
02010 
02011   bool failed = (in1->fail() && !in1->eof()) || (in2.fail() && !in2.eof());
02012   close_read_subfile(in1);
02013 
02014   nassertr(!failed, false);
02015 
02016   return true;
02017 }
02018 
02019 ////////////////////////////////////////////////////////////////////
02020 //     Function: Multifile::output
02021 //       Access: Published
02022 //  Description: 
02023 ////////////////////////////////////////////////////////////////////
02024 void Multifile::
02025 output(ostream &out) const {
02026   out << "Multifile " << _multifile_name << ", " << get_num_subfiles()
02027       << " subfiles.\n";
02028 }
02029 
02030 ////////////////////////////////////////////////////////////////////
02031 //     Function: Multifile::ls
02032 //       Access: Published
02033 //  Description: Shows a list of all subfiles within the Multifile.
02034 ////////////////////////////////////////////////////////////////////
02035 void Multifile::
02036 ls(ostream &out) const {
02037   int num_subfiles = get_num_subfiles();
02038   for (int i = 0; i < num_subfiles; i++) {
02039     string subfile_name = get_subfile_name(i);
02040     out << subfile_name << "\n";
02041   }
02042 }
02043 
02044 ////////////////////////////////////////////////////////////////////
02045 //     Function: Multifile::set_header_prefix
02046 //       Access: Published
02047 //  Description: Sets the string which is written to the Multifile
02048 //               before the Multifile header.  This string must begin
02049 //               with a hash mark and end with a newline character;
02050 //               and if it includes embedded newline characters, each
02051 //               one must be followed by a hash mark.  If these
02052 //               conditions are not initially true, the string will be
02053 //               modified as necessary to make it so.
02054 //
02055 //               This is primarily useful as a simple hack to allow
02056 //               p3d applications to be run directly from the command
02057 //               line on Unix-like systems.
02058 //
02059 //               The return value is true if successful, or false on
02060 //               failure (for instance, because the header prefix
02061 //               violates the above rules).
02062 ////////////////////////////////////////////////////////////////////
02063 void Multifile::
02064 set_header_prefix(const string &header_prefix) {
02065   string new_header_prefix = header_prefix;
02066 
02067   if (!new_header_prefix.empty()) {
02068     // It must begin with a hash mark.
02069     if (new_header_prefix[0] != '#') {
02070       new_header_prefix = string("#") + new_header_prefix;
02071     }
02072     
02073     // It must end with a newline.
02074     if (new_header_prefix[new_header_prefix.size() - 1] != '\n') {
02075       new_header_prefix += string("\n");
02076     }
02077     
02078     // Embedded newlines must be followed by a hash mark.
02079     size_t newline = new_header_prefix.find('\n');
02080     while (newline < new_header_prefix.size() - 1) {
02081       if (new_header_prefix[newline + 1] != '#') {
02082         new_header_prefix = new_header_prefix.substr(0, newline + 1) + string("#") + new_header_prefix.substr(newline + 1);
02083       }
02084       newline = new_header_prefix.find('#', newline);
02085     }
02086   }
02087 
02088   if (_header_prefix != new_header_prefix) {
02089     _header_prefix = new_header_prefix;
02090     _needs_repack = true;
02091   }
02092 }
02093   
02094   
02095 ////////////////////////////////////////////////////////////////////
02096 //     Function: Multifile::read_subfile
02097 //       Access: Public
02098 //  Description: Fills a string with the entire contents of
02099 //               the indicated subfile.
02100 ////////////////////////////////////////////////////////////////////
02101 bool Multifile::
02102 read_subfile(int index, string &result) {
02103   result = string();
02104 
02105   // We use a temporary pvector, because dynamic accumulation of a
02106   // pvector seems to be many times faster than that of a string, at
02107   // least on the Windows implementation of STL.
02108   pvector<unsigned char> pv;
02109   if (!read_subfile(index, pv)) {
02110     return false;
02111   }
02112 
02113   if (!pv.empty()) {
02114     result.append((const char *)&pv[0], pv.size());
02115   }
02116 
02117   return true;
02118 }
02119 
02120 ////////////////////////////////////////////////////////////////////
02121 //     Function: Multifile::read_subfile
02122 //       Access: Public
02123 //  Description: Fills a pvector with the entire contents of
02124 //               the indicated subfile.
02125 ////////////////////////////////////////////////////////////////////
02126 bool Multifile::
02127 read_subfile(int index, pvector<unsigned char> &result) {
02128   nassertr(is_read_valid(), false);
02129   nassertr(index >= 0 && index < (int)_subfiles.size(), false);
02130   result.clear();
02131 
02132   // Now look up the particular Subfile we are reading.
02133   nassertr(is_read_valid(), NULL);
02134   nassertr(index >= 0 && index < (int)_subfiles.size(), NULL);
02135   Subfile *subfile = _subfiles[index];
02136 
02137   if (subfile->_source != (istream *)NULL ||
02138       !subfile->_source_filename.empty()) {
02139     // The subfile has not yet been copied into the physical
02140     // Multifile.  Force a flush operation to incorporate it.
02141     flush();
02142 
02143     // That shouldn't change the subfile index or delete the subfile
02144     // pointer.
02145     nassertr(subfile == _subfiles[index], NULL);
02146   }
02147 
02148   result.reserve(subfile->_uncompressed_length);
02149 
02150   bool success = true;
02151   if (subfile->_flags & (SF_encrypted | SF_compressed)) {
02152     // If the subfile is encrypted or compressed, we can't read it
02153     // directly.  Fall back to the generic implementation.
02154     istream *in = open_read_subfile(index);
02155     if (in == (istream *)NULL) {
02156       return false;
02157     }
02158 
02159     success = VirtualFile::simple_read_file(in, result);
02160     close_read_subfile(in);
02161 
02162   } else {
02163     // But if the subfile is just a plain file, we can just read the
02164     // data directly from the Multifile, without paying the cost of an
02165     // ISubStream.
02166     static const size_t buffer_size = 4096;
02167     char buffer[buffer_size];
02168 
02169     streamsize pos = _offset + subfile->_data_start;
02170     size_t max_bytes = subfile->_data_length;
02171     streamsize count = 0;
02172     bool eof = true;
02173 
02174     streamsize num_bytes = (streamsize)min(buffer_size, max_bytes);
02175     _read->seek_read(pos, buffer, num_bytes, count, eof);
02176     while (count != 0) {
02177       thread_consider_yield();
02178       nassertr(count <= (streamsize)max_bytes, false);
02179       result.insert(result.end(), buffer, buffer + (size_t)count);
02180       max_bytes -= (size_t)count;
02181       pos += count;
02182 
02183       num_bytes = (streamsize)min(buffer_size, max_bytes);
02184       _read->seek_read(pos, buffer, num_bytes, count, eof);
02185     }
02186 
02187     success = !eof;
02188   }
02189 
02190   if (!success) {
02191     ostringstream message;
02192     message << "I/O error reading from " << get_multifile_name() << " at "
02193             << get_subfile_name(index);
02194     nassert_raise(message.str());
02195     return false;
02196   }
02197 
02198   return true;
02199 }
02200 
02201 ////////////////////////////////////////////////////////////////////
02202 //     Function: Multifile::pad_to_streampos
02203 //       Access: Private
02204 //  Description: Assumes the _write pointer is at the indicated fpos,
02205 //               rounds the fpos up to the next legitimate address
02206 //               (using normalize_streampos()), and writes enough
02207 //               zeroes to the stream to fill the gap.  Returns the
02208 //               new fpos.
02209 ////////////////////////////////////////////////////////////////////
02210 streampos Multifile::
02211 pad_to_streampos(streampos fpos) {
02212   nassertr(_write != (ostream *)NULL, fpos);
02213   nassertr(_write->tellp() == fpos, fpos);
02214   streampos new_fpos = normalize_streampos(fpos);
02215   while (fpos < new_fpos) {
02216     _write->put(0);
02217     fpos += 1; // VC++ doesn't define streampos++ (!)
02218   }
02219   nassertr(_write->tellp() == fpos, fpos);
02220   return fpos;
02221 }
02222 
02223 ////////////////////////////////////////////////////////////////////
02224 //     Function: Multifile::add_new_subfile
02225 //       Access: Private
02226 //  Description: Adds a newly-allocated Subfile pointer to the
02227 //               Multifile.
02228 ////////////////////////////////////////////////////////////////////
02229 void Multifile::
02230 add_new_subfile(Subfile *subfile, int compression_level) {
02231   if (compression_level != 0) {
02232 #ifndef HAVE_ZLIB
02233     express_cat.warning()
02234       << "zlib not compiled in; cannot generated compressed multifiles.\n";
02235     compression_level = 0;
02236 #else  // HAVE_ZLIB
02237     subfile->_flags |= SF_compressed;
02238     subfile->_compression_level = compression_level;
02239 #endif  // HAVE_ZLIB
02240   }
02241 
02242 #ifdef HAVE_OPENSSL
02243   if (_encryption_flag) {
02244     subfile->_flags |= SF_encrypted;
02245   }
02246 #endif  // HAVE_OPENSSL
02247 
02248   if (_next_index != (streampos)0) {
02249     // If we're adding a Subfile to an already-existing Multifile, we
02250     // will eventually need to repack the file.
02251     _needs_repack = true;
02252   }
02253 
02254   pair<Subfiles::iterator, bool> insert_result = _subfiles.insert(subfile);
02255   if (!insert_result.second) {
02256     // Hmm, unable to insert.  There must already be a subfile by that
02257     // name.  Remove the old one.
02258     Subfile *old_subfile = (*insert_result.first);
02259     old_subfile->_flags |= SF_deleted;
02260 
02261     // Maybe it was just added to the _new_subfiles list.  In this case, remove it from that list.
02262     PendingSubfiles::iterator ni = find(_new_subfiles.begin(), _new_subfiles.end(), old_subfile);
02263     if (ni != _new_subfiles.end()) {
02264       _new_subfiles.erase(ni);
02265 
02266     } else {
02267       // Otherwise, add it to the _removed_subfiles list, so we can remove the old one.
02268       _removed_subfiles.push_back(old_subfile);
02269     }
02270 
02271     (*insert_result.first) = subfile;
02272   }
02273 
02274   _new_subfiles.push_back(subfile);
02275 }
02276 
02277 ////////////////////////////////////////////////////////////////////
02278 //     Function: Multifile::open_read_subfile
02279 //       Access: Private
02280 //  Description: This variant of open_read_subfile() is used
02281 //               internally only, and accepts a pointer to the
02282 //               internal Subfile object, which is assumed to be valid
02283 //               and written to the multifile.
02284 ////////////////////////////////////////////////////////////////////
02285 istream *Multifile::
02286 open_read_subfile(Subfile *subfile) {
02287   nassertr(subfile->_source == (istream *)NULL &&
02288            subfile->_source_filename.empty(), NULL);
02289 
02290   // Return an ISubStream object that references into the open
02291   // Multifile istream.
02292   nassertr(subfile->_data_start != (streampos)0, NULL);
02293   istream *stream = 
02294     new ISubStream(_read, _offset + subfile->_data_start,
02295                    _offset + subfile->_data_start + (streampos)subfile->_data_length); 
02296   
02297   if ((subfile->_flags & SF_encrypted) != 0) {
02298 #ifndef HAVE_OPENSSL
02299     express_cat.error()
02300       << "OpenSSL not compiled in; cannot read encrypted multifiles.\n";
02301     delete stream;
02302     return NULL;
02303 #else  // HAVE_OPENSSL
02304     // The subfile is encrypted.  So actually, return an
02305     // IDecryptStream that wraps around the ISubStream.
02306     IDecryptStream *wrapper = 
02307       new IDecryptStream(stream, true, _encryption_password);
02308     stream = wrapper;
02309 
02310     // Validate the password by confirming that the encryption header
02311     // matches.
02312     char this_header[_encrypt_header_size];
02313     stream->read(this_header, _encrypt_header_size);
02314     if (stream->fail() || stream->gcount() != (unsigned)_encrypt_header_size ||
02315         memcmp(this_header, _encrypt_header, _encrypt_header_size) != 0) {
02316       express_cat.error()
02317         << "Unable to decrypt subfile " << subfile->_name << ".\n";
02318       delete stream;
02319       return NULL;
02320     }
02321 #endif  // HAVE_OPENSSL
02322   }
02323 
02324   if ((subfile->_flags & SF_compressed) != 0) {
02325 #ifndef HAVE_ZLIB
02326     express_cat.error()
02327       << "zlib not compiled in; cannot read compressed multifiles.\n";
02328     delete stream;
02329     return NULL;
02330 #else  // HAVE_ZLIB
02331     // Oops, the subfile is compressed.  So actually, return an
02332     // IDecompressStream that wraps around the ISubStream.
02333     IDecompressStream *wrapper = new IDecompressStream(stream, true);
02334     stream = wrapper;
02335 #endif  // HAVE_ZLIB
02336   }
02337 
02338   if (stream->fail()) {
02339     // Hmm, some inexplicable problem.
02340     delete stream;
02341     return NULL;
02342   }
02343 
02344   return stream;
02345 }
02346 
02347 ////////////////////////////////////////////////////////////////////
02348 //     Function: Multifile::standardize_subfile_name
02349 //       Access: Private
02350 //  Description: Returns the standard form of the subfile name.
02351 ////////////////////////////////////////////////////////////////////
02352 string Multifile::
02353 standardize_subfile_name(const string &subfile_name) const {
02354   Filename name = subfile_name;
02355   name.standardize();
02356   if (name.empty() || name == "/") {
02357     // Invalid empty name.
02358     return string();
02359   }
02360 
02361   if (name[0] == '/') {
02362     return name.get_fullpath().substr(1);
02363   } else if (name.length() > 2 && name[0] == '.' && name[1] == '/') {
02364     return name.get_fullpath().substr(2);
02365   } else {
02366     return name.get_fullpath();
02367   }
02368 }
02369 
02370 ////////////////////////////////////////////////////////////////////
02371 //     Function: Multifile::clear_subfiles
02372 //       Access: Private
02373 //  Description: Removes the set of subfiles from the tables and frees
02374 //               their associated memory.
02375 ////////////////////////////////////////////////////////////////////
02376 void Multifile::
02377 clear_subfiles() {
02378   PendingSubfiles::iterator pi;
02379   for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
02380     Subfile *subfile = (*pi);
02381     subfile->rewrite_index_flags(*_write);
02382     delete subfile;
02383   }
02384   _removed_subfiles.clear();
02385 
02386   // We don't have to delete the ones in _new_subfiles, because these
02387   // also appear in _subfiles.
02388   _new_subfiles.clear();
02389 
02390 #ifdef HAVE_OPENSSL
02391   for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) {
02392     Subfile *subfile = (*pi);
02393     delete subfile;
02394   }
02395   _cert_special.clear();
02396 
02397   _signatures.clear();
02398 #endif  // HAVE_OPENSSL
02399 
02400   Subfiles::iterator fi;
02401   for (fi = _subfiles.begin(); fi != _subfiles.end(); ++fi) {
02402     Subfile *subfile = (*fi);
02403     delete subfile;
02404   }
02405   _subfiles.clear();
02406 }
02407 
02408 ////////////////////////////////////////////////////////////////////
02409 //     Function: Multifile::read_index
02410 //       Access: Private
02411 //  Description: Reads the Multifile header and index.  Returns true
02412 //               if successful, false if the Multifile is not valid.
02413 ////////////////////////////////////////////////////////////////////
02414 bool Multifile::
02415 read_index() {
02416   nassertr(_read != (IStreamWrapper *)NULL, false);
02417 
02418   // We acquire the IStreamWrapper lock for the duration of this
02419   // method.
02420   _read->acquire();
02421   istream *read = _read->get_istream();
02422 
02423   char this_header[_header_size];
02424   read->seekg(_offset);
02425 
02426   // Here's a special case: if the multifile begins with a hash
02427   // character, then we continue reading and discarding lines of ASCII
02428   // text, until we come across a nonempty line that does not begin
02429   // with a hash character.  This allows a P3D application (which is a
02430   // multifile) to be run directly on the command line on Unix-based
02431   // systems.
02432   _header_prefix = string();
02433   int ch = read->get();
02434 
02435   if (ch == '#') {
02436     while (ch != EOF && ch == '#') {
02437       // Skip to the end of the line.
02438       while (ch != EOF && ch != '\n') {
02439         _header_prefix += ch;
02440         ch = read->get();
02441       }
02442       // Skip to the first non-whitespace character of the line.
02443       while (ch != EOF && (isspace(ch) || ch == '\r')) {
02444         _header_prefix += ch;
02445         ch = read->get();
02446       }
02447     }
02448   }
02449 
02450   // Now read the actual Multifile header.
02451   this_header[0] = ch;
02452   read->read(this_header + 1, _header_size - 1);
02453   if (read->fail() || read->gcount() != (unsigned)(_header_size - 1)) {
02454     express_cat.info()
02455       << "Unable to read Multifile header " << _multifile_name << ".\n";
02456     _read->release();
02457     close();
02458     return false;
02459   }
02460 
02461   if (memcmp(this_header, _header, _header_size) != 0) {
02462     express_cat.info()
02463       << _multifile_name << " is not a Multifile.\n";
02464     _read->release();
02465     close();
02466     return false;
02467   }
02468 
02469   // Now get the version numbers out.
02470   StreamReader reader(read, false);
02471   _file_major_ver = reader.get_int16();
02472   _file_minor_ver = reader.get_int16();
02473   _scale_factor = reader.get_uint32();
02474   _new_scale_factor = _scale_factor;
02475 
02476   if (read->eof() || read->fail()) {
02477     express_cat.info()
02478       << _multifile_name << " header is truncated.\n";
02479     _read->release();
02480     close();
02481     return false;
02482   }
02483 
02484   if (_file_major_ver != _current_major_ver ||
02485       (_file_major_ver == _current_major_ver && 
02486        _file_minor_ver > _current_minor_ver)) {
02487     express_cat.info()
02488       << _multifile_name << " has version " << _file_major_ver << "."
02489       << _file_minor_ver << ", expecting version " 
02490       << _current_major_ver << "." << _current_minor_ver << ".\n";
02491     _read->release();
02492     close();
02493     return false;
02494   }
02495 
02496   _record_timestamp = true;
02497   if (_file_minor_ver >= 1) {
02498     time_t read_timestamp = reader.get_uint32();
02499     if (read_timestamp == 0) {
02500       // If we read a 0 timestamp from the file, that implies that we
02501       // don't want to record a timestamp in this particular file.
02502       _record_timestamp = false;
02503     } else {
02504       _timestamp = read_timestamp;
02505     }
02506     _timestamp_dirty = false;
02507   }
02508 
02509   // Now read the index out.
02510   _next_index = read->tellg() - _offset;
02511   _next_index = normalize_streampos(_next_index);  
02512   read->seekg(_next_index + _offset);
02513   _last_index = 0;
02514   _last_data_byte = 0;
02515   streampos index_forward;
02516   streamoff bytes_skipped = 0;
02517   bool read_cert_special = false;
02518 
02519   Subfile *subfile = new Subfile;
02520   index_forward = subfile->read_index(*read, _next_index, this);
02521   while (index_forward != (streampos)0) {
02522     _last_index = _next_index;
02523     if (subfile->is_deleted()) {
02524       // Ignore deleted Subfiles in the index.
02525       _needs_repack = true;
02526       delete subfile;
02527     } else if (subfile->is_cert_special()) {
02528       // Certificate chains and signature files get stored in a
02529       // special list.
02530       _cert_special.push_back(subfile);
02531       read_cert_special = true;
02532     } else {
02533       _subfiles.push_back(subfile);
02534     }
02535     if (!subfile->is_cert_special()) {
02536       if (bytes_skipped != 0) {
02537         // If the index entries don't follow exactly sequentially
02538         // (except for the cert special files), the file ought to be
02539         // repacked.
02540         _needs_repack = true;
02541       }
02542       if (read_cert_special) {
02543         // If we read a normal subfile following a cert_special entry,
02544         // the file ought to be repacked (certificates have to go at
02545         // the end).
02546         _needs_repack = true;
02547       }
02548       _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos());
02549     }
02550     streampos curr_pos = normalize_streampos(read->tellg() - _offset);
02551     bytes_skipped = index_forward - curr_pos;
02552     read->seekg(index_forward + _offset);
02553     _next_index = index_forward;
02554     subfile = new Subfile;
02555     index_forward = subfile->read_index(*read, _next_index, this);
02556   }
02557   if (subfile->is_index_invalid()) {
02558     express_cat.info()
02559       << "Error reading index for " << _multifile_name << ".\n";
02560     _read->release();
02561     close();
02562     delete subfile;
02563     return false;
02564   }
02565 
02566   // Check if the list is already sorted.  If it is not, we need a
02567   // repack.
02568   for (size_t si = 1; si < _subfiles.size() && !_needs_repack; ++si) {
02569     if (*_subfiles[si] < *_subfiles[si - 1]) {
02570       _needs_repack = true;
02571     }
02572   }
02573 
02574   if (_needs_repack) {
02575     // At least sort them now.
02576     size_t before_size = _subfiles.size();
02577     _subfiles.sort();
02578     size_t after_size = _subfiles.size();
02579     
02580     // If these don't match, the same filename appeared twice in the
02581     // index, which shouldn't be possible.
02582     nassertr(before_size == after_size, true);
02583   }
02584 
02585   delete subfile;
02586   _read->release();
02587   return true;
02588 }  
02589 
02590 ////////////////////////////////////////////////////////////////////
02591 //     Function: Multifile::write_header
02592 //       Access: Private
02593 //  Description: Writes just the header part of the Multifile, not the
02594 //               index.
02595 ////////////////////////////////////////////////////////////////////
02596 bool Multifile::
02597 write_header() {
02598   _file_major_ver = _current_major_ver;
02599   _file_minor_ver = _current_minor_ver;
02600 
02601   nassertr(_write != (ostream *)NULL, false);
02602   nassertr(_write->tellp() == (streampos)0, false);
02603   _write->write(_header_prefix.data(), _header_prefix.size());
02604   _write->write(_header, _header_size);
02605   StreamWriter writer(_write, false);
02606   writer.add_int16(_current_major_ver);
02607   writer.add_int16(_current_minor_ver);
02608   writer.add_uint32(_scale_factor);
02609 
02610   if (_record_timestamp) {
02611     writer.add_uint32(_timestamp);
02612   } else {
02613     writer.add_uint32(0);
02614     _timestamp_dirty = false;
02615   }
02616 
02617   _next_index = _write->tellp();
02618   _next_index = pad_to_streampos(_next_index);
02619   _last_index = 0;
02620 
02621   if (_write->fail()) {
02622     express_cat.info()
02623       << "Unable to write header for " << _multifile_name << ".\n";
02624     close();
02625     return false;
02626   }
02627 
02628   return true;
02629 }
02630 
02631 ////////////////////////////////////////////////////////////////////
02632 //     Function: Multifile::check_signatures
02633 //       Access: Private
02634 //  Description: Walks through the list of _cert_special entries in
02635 //               the Multifile, moving any valid signatures found to
02636 //               _signatures.  After this call, _cert_special will be
02637 //               empty.
02638 //
02639 //               This does not check the validity of the certificates
02640 //               themselves.  It only checks that they correctly sign
02641 //               the Multifile contents.
02642 ////////////////////////////////////////////////////////////////////
02643 void Multifile::
02644 check_signatures() {
02645 #ifdef HAVE_OPENSSL
02646   PendingSubfiles::iterator pi;
02647 
02648   for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) {
02649     Subfile *subfile = (*pi);
02650     nassertv((subfile->_flags & SF_signature) != 0);
02651 
02652     // Extract the signature data and certificate separately.
02653     istream *stream = open_read_subfile(subfile);
02654     nassertv(stream != NULL);
02655     StreamReader reader(*stream);
02656     size_t sig_size = reader.get_uint32();
02657     string sig_string = reader.extract_bytes(sig_size);
02658 
02659     size_t num_certs = reader.get_uint32();
02660 
02661     // Read the remaining buffer of certificate data.
02662     pvector<unsigned char> buffer;
02663     bool success = VirtualFile::simple_read_file(stream, buffer);
02664     nassertv(success);
02665     close_read_subfile(stream);
02666 
02667     // Now convert each of the certificates to an X509 object, and
02668     // store it in our CertChain.
02669     CertChain chain;
02670     EVP_PKEY *pkey = NULL;
02671     if (!buffer.empty()) {
02672 #if OPENSSL_VERSION_NUMBER >= 0x00908000L
02673       // Beginning in 0.9.8, d2i_X509() accepted a const unsigned char **.
02674       const unsigned char *bp, *bp_end;
02675 #else
02676       // Prior to 0.9.8, d2i_X509() accepted an unsigned char **.
02677       unsigned char *bp, *bp_end;
02678 #endif
02679       bp = (unsigned char *)&buffer[0];
02680       bp_end = bp + buffer.size();
02681       X509 *x509 = d2i_X509(NULL, &bp, bp_end - bp);
02682       while (num_certs > 0 && x509 != NULL) {
02683         chain.push_back(CertRecord(x509));
02684         --num_certs;
02685         x509 = d2i_X509(NULL, &bp, bp_end - bp);
02686       }
02687       if (num_certs != 0 || x509 != NULL) {
02688         express_cat.warning() 
02689           << "Extra data in signature record.\n";
02690       }
02691     }
02692     
02693     if (!chain.empty()) {
02694       pkey = X509_get_pubkey(chain[0]._cert);
02695     }
02696     
02697     if (pkey != NULL) {
02698       EVP_MD_CTX *md_ctx;
02699 #ifdef SSL_097
02700       md_ctx = EVP_MD_CTX_create();
02701 #else
02702       md_ctx = new EVP_MD_CTX;
02703 #endif
02704       EVP_VerifyInit(md_ctx, EVP_sha1());
02705       
02706       nassertv(_read != NULL);
02707       _read->acquire();
02708       istream *read = _read->get_istream();
02709       
02710       // Read and hash the multifile contents, but only up till
02711       // _last_data_byte.
02712       read->seekg(_offset);
02713       streampos bytes_remaining = _last_data_byte;
02714       static const size_t buffer_size = 4096;
02715       char buffer[buffer_size];
02716       read->read(buffer, min((streampos)buffer_size, bytes_remaining));
02717       size_t count = read->gcount();
02718       while (count != 0) {
02719         nassertv(count <= buffer_size);
02720         EVP_VerifyUpdate(md_ctx, buffer, count);
02721         bytes_remaining -= count;
02722         read->read(buffer, min((streampos)buffer_size, bytes_remaining));
02723         count = read->gcount();
02724       }
02725       nassertv(bytes_remaining == (streampos)0);
02726       _read->release();
02727       
02728       // Now check that the signature matches the hash.
02729       int verify_result = 
02730         EVP_VerifyFinal(md_ctx, 
02731                         (unsigned char *)sig_string.data(), 
02732                         sig_string.size(), pkey);
02733       if (verify_result == 1) {
02734         // The signature matches; save the certificate and its chain.
02735         _signatures.push_back(chain);
02736       } else {
02737         // Bad match.
02738         _needs_repack = true;
02739       }
02740     }
02741   }
02742 #endif  // HAVE_OPENSSL
02743 
02744   _cert_special.clear();
02745 }
02746 
02747 ////////////////////////////////////////////////////////////////////
02748 //     Function: Multifile::Subfile::read_index
02749 //       Access: Public
02750 //  Description: Reads the index record for the Subfile from the
02751 //               indicated istream.  Assumes the istream has already
02752 //               been positioned to the indicated stream position,
02753 //               fpos, the start of the index record.  Returns the
02754 //               position within the file of the next index record.
02755 ////////////////////////////////////////////////////////////////////
02756 streampos Multifile::Subfile::
02757 read_index(istream &read, streampos fpos, Multifile *multifile) {
02758   nassertr(read.tellg() - multifile->_offset == fpos, fpos);
02759 
02760   // First, get the next stream position.  We do this separately,
02761   // because if it is zero, we don't get anything else.
02762   StreamReader reader(read);
02763 
02764   streampos next_index = multifile->word_to_streampos(reader.get_uint32());
02765   if (read.eof() || read.fail()) {
02766     _flags |= SF_index_invalid;
02767     return 0;
02768   }
02769 
02770   if (next_index == (streampos)0) {
02771     return 0;
02772   }
02773 
02774   // Now get the rest of the index.
02775 
02776   _index_start = fpos;
02777   _index_length = 0;
02778   
02779   _data_start = multifile->word_to_streampos(reader.get_uint32());
02780   _data_length = reader.get_uint32();
02781   _flags = reader.get_uint16();
02782   if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
02783     _uncompressed_length = reader.get_uint32();
02784   } else {
02785     _uncompressed_length = _data_length;
02786   }
02787   if (multifile->_file_minor_ver < 1) {
02788     _timestamp = multifile->get_timestamp();
02789   } else {
02790     _timestamp = reader.get_uint32();
02791     if (_timestamp == 0) {
02792       _timestamp = multifile->get_timestamp();
02793     }
02794   }
02795 
02796   size_t name_length = reader.get_uint16();
02797   if (read.eof() || read.fail()) {
02798     _flags |= SF_index_invalid;
02799     return 0;
02800   }
02801 
02802   // And finally, get the rest of the name.
02803   char *name_buffer = (char *)PANDA_MALLOC_ARRAY(name_length);
02804   nassertr(name_buffer != (char *)NULL, next_index);
02805   for (size_t ni = 0; ni < name_length; ni++) {
02806     name_buffer[ni] = read.get() ^ 0xff;
02807   }
02808   _name = string(name_buffer, name_length);
02809   PANDA_FREE_ARRAY(name_buffer);
02810 
02811   if (read.eof() || read.fail()) {
02812     _flags |= SF_index_invalid;
02813     return 0;
02814   }
02815 
02816   _index_length = read.tellg() - fpos - multifile->_offset;
02817   return next_index;
02818 }
02819 
02820 ////////////////////////////////////////////////////////////////////
02821 //     Function: Multifile::Subfile::write_index
02822 //       Access: Public
02823 //  Description: Writes the index record for the Subfile to the
02824 //               indicated ostream.  Assumes the istream has already
02825 //               been positioned to the indicated stream position,
02826 //               fpos, the start of the index record, and that this is
02827 //               the effective end of the file.  Returns the position
02828 //               within the file of the next index record.
02829 //
02830 //               The _index_start member is updated by this operation.
02831 ////////////////////////////////////////////////////////////////////
02832 streampos Multifile::Subfile::
02833 write_index(ostream &write, streampos fpos, Multifile *multifile) {
02834   nassertr(write.tellp() - multifile->_offset == fpos, fpos);
02835 
02836   _index_start = fpos;
02837   _index_length = 0;
02838 
02839   // This will be the contents of this particular index record.  We
02840   // build it up first since it will be variable length.
02841   Datagram dg;
02842   dg.add_uint32(multifile->streampos_to_word(_data_start));
02843   dg.add_uint32(_data_length);
02844   dg.add_uint16(_flags);
02845   if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
02846     dg.add_uint32(_uncompressed_length);
02847   }
02848   dg.add_uint32(_timestamp);
02849   dg.add_uint16(_name.length());
02850 
02851   // For no real good reason, we'll invert all the bits in the name.
02852   // The only reason we do this is to make it inconvenient for a
02853   // casual browser of the Multifile to discover the names of the
02854   // files stored within it.  Naturally, this isn't real obfuscation
02855   // or security.
02856   string::iterator ni;
02857   for (ni = _name.begin(); ni != _name.end(); ++ni) {
02858     dg.add_int8((*ni) ^ 0xff);
02859   }
02860 
02861   size_t this_index_size = 4 + dg.get_length();
02862 
02863   // Plus, we will write out the next index address first.
02864   streampos next_index = fpos + (streampos)this_index_size; 
02865 
02866   Datagram idg;
02867   idg.add_uint32(multifile->streampos_to_word(next_index));
02868 
02869   write.write((const char *)idg.get_data(), idg.get_length());
02870   write.write((const char *)dg.get_data(), dg.get_length());
02871 
02872   _index_length = write.tellp() - fpos - multifile->_offset;
02873   return next_index;
02874 }
02875 
02876 ////////////////////////////////////////////////////////////////////
02877 //     Function: Multifile::Subfile::write_data
02878 //       Access: Public
02879 //  Description: Writes the data record for the Subfile to the
02880 //               indicated ostream: the actual contents of the
02881 //               Subfile.  Assumes the istream has already been
02882 //               positioned to the indicated stream position, fpos,
02883 //               the start of the data record, and that this is the
02884 //               effective end of the file.  Returns the position
02885 //               within the file of the next data record.
02886 //
02887 //               The _data_start, _data_length, and
02888 //               _uncompressed_length members are updated by this
02889 //               operation.
02890 //
02891 //               If the "read" pointer is non-NULL, it is the readable
02892 //               istream of a Multifile in which the Subfile might
02893 //               already be packed.  This is used for reading the
02894 //               contents of the Subfile during a repack() operation.
02895 ////////////////////////////////////////////////////////////////////
02896 streampos Multifile::Subfile::
02897 write_data(ostream &write, istream *read, streampos fpos,
02898            Multifile *multifile) {
02899   nassertr(write.tellp() - multifile->_offset == fpos, fpos);
02900 
02901   istream *source = _source;
02902   pifstream source_file;
02903   if (source == (istream *)NULL && !_source_filename.empty()) {
02904     // If we have a filename, open it up and read that.
02905     if (!_source_filename.open_read(source_file)) {
02906       // Unable to open the source file.
02907       express_cat.info()
02908         << "Unable to read " << _source_filename << ".\n";
02909       _flags |= SF_data_invalid;
02910       _data_length = 0;
02911       _uncompressed_length = 0;
02912     } else {
02913       source = &source_file;
02914     }
02915   }
02916 
02917   if (source == (istream *)NULL) {
02918     // We don't have any source data.  Perhaps we're reading from an
02919     // already-packed Subfile (e.g. during repack()).
02920     if (read == (istream *)NULL) {
02921       // No, we're just screwed.
02922       express_cat.info()
02923         << "No source for subfile " << _name << ".\n";
02924       _flags |= SF_data_invalid;
02925     } else {
02926       // Read the data from the original Multifile.
02927       read->seekg(_data_start + multifile->_offset);
02928       for (size_t p = 0; p < _data_length; p++) {
02929         int byte = read->get();
02930         if (read->eof() || read->fail()) {
02931           // Unexpected EOF or other failure on the source file.
02932           express_cat.info()
02933             << "Unexpected EOF for subfile " << _name << ".\n";
02934           _flags |= SF_data_invalid;
02935           break;
02936         }
02937         write.put(byte);
02938       }
02939     }
02940   } else {
02941     // We do have source data.  Copy it in, and also measure its
02942     // length.
02943     ostream *putter = &write;
02944     bool delete_putter = false;
02945 
02946 #ifndef HAVE_OPENSSL
02947     // Without OpenSSL, we can't support encryption.  The flag had
02948     // better not be set.
02949     nassertr((_flags & SF_encrypted) == 0, fpos);
02950 
02951 #else  // HAVE_OPENSSL
02952     if ((_flags & SF_encrypted) != 0) {
02953       // Write it encrypted.
02954       OEncryptStream *encrypt = new OEncryptStream;
02955       encrypt->set_iteration_count(multifile->_encryption_iteration_count);
02956       encrypt->open(putter, delete_putter, multifile->_encryption_password);
02957 
02958       putter = encrypt;
02959       delete_putter = true;
02960 
02961       // Also write the encrypt_header to the beginning of the
02962       // encrypted stream, so we can validate the password on
02963       // decryption.
02964       putter->write(_encrypt_header, _encrypt_header_size);
02965     }
02966 #endif  // HAVE_OPENSSL
02967 
02968 #ifndef HAVE_ZLIB
02969     // Without ZLIB, we can't support compression.  The flag had
02970     // better not be set.
02971     nassertr((_flags & SF_compressed) == 0, fpos);
02972 #else  // HAVE_ZLIB
02973     if ((_flags & SF_compressed) != 0) {
02974       // Write it compressed.
02975       putter = new OCompressStream(putter, delete_putter, _compression_level);
02976       delete_putter = true;
02977     }
02978 #endif  // HAVE_ZLIB
02979 
02980     streampos write_start = fpos;
02981     _uncompressed_length = 0;
02982 
02983 #ifndef HAVE_OPENSSL
02984     // We also need OpenSSL for signatures.
02985     nassertr((_flags & SF_signature) == 0, fpos);
02986 
02987 #else  // HAVE_OPENSSL
02988     if ((_flags & SF_signature) != 0) {
02989       // If it's a special signature record, precede the record data
02990       // (the certificate itself) with the signature data generated
02991       // against the multifile contents.
02992 
02993       // In order to generate a signature, we need to have a valid
02994       // read pointer.
02995       nassertr(read != NULL, fpos);
02996 
02997       // And we also need to have a private key.
02998       nassertr(_pkey != NULL, fpos);
02999 
03000       EVP_MD_CTX *md_ctx;
03001 #ifdef SSL_097
03002       md_ctx = EVP_MD_CTX_create();
03003 #else
03004       md_ctx = new EVP_MD_CTX;
03005 #endif
03006       EVP_SignInit(md_ctx, EVP_sha1());
03007 
03008       // Read and hash the multifile contents, but only up till
03009       // _last_data_byte.
03010       nassertr(multifile->_last_data_byte < fpos, fpos);
03011       read->seekg(multifile->_offset);
03012       streampos bytes_remaining = multifile->_last_data_byte;
03013       static const size_t buffer_size = 4096;
03014       char buffer[buffer_size];
03015       read->read(buffer, min((streampos)buffer_size, bytes_remaining));
03016       size_t count = read->gcount();
03017       while (count != 0) {
03018         nassertr(count <= buffer_size, fpos);
03019         EVP_SignUpdate(md_ctx, buffer, count);
03020         bytes_remaining -= count;
03021         read->read(buffer, min((streampos)buffer_size, bytes_remaining));
03022         count = read->gcount();
03023       }
03024       nassertr(bytes_remaining == (streampos)0, fpos);
03025 
03026       // Now generate and write out the signature.
03027       unsigned int max_size = EVP_PKEY_size(_pkey);
03028       unsigned char *sig_data = new unsigned char[max_size];
03029       unsigned int sig_size;
03030       if (!EVP_SignFinal(md_ctx, sig_data, &sig_size, _pkey)) {
03031         OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
03032         sslw->notify_ssl_errors();
03033       }
03034       nassertr(sig_size <= max_size, fpos);
03035 
03036       StreamWriter writer(*putter);
03037       writer.add_uint32(sig_size);
03038       putter->write((char *)sig_data, sig_size);
03039       _uncompressed_length += 4 + sig_size;
03040 
03041       delete[] sig_data;
03042       
03043 #ifdef SSL_097
03044       EVP_MD_CTX_destroy(md_ctx);
03045 #else
03046       delete md_ctx;
03047 #endif
03048     }
03049 #endif  // HAVE_OPENSSL
03050 
03051     // Finally, we can write out the data itself.
03052     static const size_t buffer_size = 4096;
03053     char buffer[buffer_size];
03054     
03055     source->read(buffer, buffer_size);
03056     size_t count = source->gcount();
03057     while (count != 0) {
03058       _uncompressed_length += count;
03059       putter->write(buffer, count);
03060       source->read(buffer, buffer_size);
03061       count = source->gcount();
03062     }
03063 
03064     if (delete_putter) {
03065       delete putter;
03066     }
03067 
03068     streampos write_end = write.tellp() - multifile->_offset;
03069     _data_length = (size_t)(write_end - write_start);
03070   }
03071 
03072   // We can't set _data_start until down here, after we have read the
03073   // Subfile.  (In case we are running during repack()).
03074   _data_start = fpos;
03075 
03076   // Get the modification timestamp for this subfile.  This is read
03077   // from the source file, if we have a filename; otherwise, it's the
03078   // current time.
03079   if (!_source_filename.empty()) {
03080     _timestamp = _source_filename.get_timestamp();
03081   }
03082   if (_timestamp == 0) {
03083     _timestamp = time(NULL);
03084   }
03085 
03086   _source = (istream *)NULL;
03087   _source_filename = Filename();
03088   source_file.close();
03089 
03090   return fpos + (streampos)_data_length;
03091 }
03092 
03093 ////////////////////////////////////////////////////////////////////
03094 //     Function: Multifile::Subfile::rewrite_index_data_start
03095 //       Access: Public
03096 //  Description: Seeks within the indicate pfstream back to the index
03097 //               record and rewrites just the _data_start and
03098 //               _data_length part of the index record.
03099 ////////////////////////////////////////////////////////////////////
03100 void Multifile::Subfile::
03101 rewrite_index_data_start(ostream &write, Multifile *multifile) {
03102   nassertv(_index_start != (streampos)0);
03103 
03104   static const size_t data_start_offset = 4;
03105   size_t data_start_pos = _index_start + (streampos)data_start_offset;
03106   write.seekp(data_start_pos + multifile->_offset);
03107   nassertv(!write.fail());
03108 
03109   StreamWriter writer(write);
03110   writer.add_uint32(multifile->streampos_to_word(_data_start));
03111   writer.add_uint32(_data_length);
03112   writer.add_uint16(_flags);
03113   if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
03114     writer.add_uint32(_uncompressed_length);
03115   }
03116   if (multifile->_record_timestamp) {
03117     writer.add_uint32(_timestamp);
03118   } else {
03119     writer.add_uint32(0);
03120   }
03121 }
03122 
03123 ////////////////////////////////////////////////////////////////////
03124 //     Function: Multifile::Subfile::rewrite_index_flags
03125 //       Access: Public
03126 //  Description: Seeks within the indicated ostream back to the index
03127 //               record and rewrites just the _flags part of the
03128 //               index record.
03129 ////////////////////////////////////////////////////////////////////
03130 void Multifile::Subfile::
03131 rewrite_index_flags(ostream &write) {
03132   // If the subfile has never even been recorded to disk, we don't
03133   // need to do anything at all in this function.
03134   if (_index_start != (streampos)0) {
03135     static const size_t flags_offset = 4 + 4 + 4;
03136     size_t flags_pos = _index_start + (streampos)flags_offset;
03137     write.seekp(flags_pos);
03138     nassertv(!write.fail());
03139     
03140     StreamWriter writer(write);
03141     writer.add_uint16(_flags);
03142   }
03143 }
 All Classes Functions Variables Enumerations