Panda3D
multifile.cxx
Go to the documentation of this file.
1 /**
2  * PANDA 3D SOFTWARE
3  * Copyright (c) Carnegie Mellon University. All rights reserved.
4  *
5  * All use of this software is subject to the terms of the revised BSD
6  * license. You should have received a copy of this license along
7  * with this source code in a file named "LICENSE."
8  *
9  * @file multifile.cxx
10  * @author mike
11  * @date 1997-01-09
12  */
13 
14 #include "multifile.h"
15 
16 #include "config_express.h"
17 #include "streamWriter.h"
18 #include "streamReader.h"
19 #include "datagram.h"
20 #include "zStream.h"
21 #include "encryptStream.h"
22 #include "virtualFileSystem.h"
23 #include "virtualFile.h"
24 
25 #include <algorithm>
26 #include <iterator>
27 #include <time.h>
28 
29 #include "openSSLWrapper.h"
30 
31 using std::ios;
32 using std::iostream;
33 using std::istream;
34 using std::max;
35 using std::min;
36 using std::ostream;
37 using std::ostringstream;
38 using std::streamoff;
39 using std::streampos;
40 using std::streamsize;
41 using std::stringstream;
42 using std::string;
43 
44 // This sequence of bytes begins each Multifile to identify it as a Multifile.
45 const char Multifile::_header[] = "pmf\0\n\r";
46 const size_t Multifile::_header_size = 6;
47 
48 // These numbers identify the version of the Multifile. Generally, a change
49 // in the major version is intolerable; while a Multifile with an older minor
50 // version may still be read.
51 const int Multifile::_current_major_ver = 1;
52 
53 const int Multifile::_current_minor_ver = 1;
54 // Bumped to version 1.1 on 6806 to add timestamps.
55 
56 // To confirm that the supplied password matches, we write the Mutifile magic
57 // header at the beginning of the encrypted stream. I suppose this does
58 // compromise the encryption security a tiny bit by making it easy for
59 // crackers to validate that a particular password guess matches or doesn't
60 // match, but the encryption algorithm doesn't depend on this being difficult
61 // anyway.
62 const char Multifile::_encrypt_header[] = "crypty";
63 const size_t Multifile::_encrypt_header_size = 6;
64 
65 
66 
67 /*
68  * A Multifile consists of the following elements: (1) A header. This is
69  * always the first n bytes of the Multifile, and contains a magic number to
70  * identify the file, as well as version numbers and any file-specific
71  * parameters. char[6] The string Multifile::_header, a magic number.
72  * int16 The file's major version number int16 The file's minor
73  * version number uint32 Scale factor. This scales all address references
74  * within the file. Normally 1, this may be set larger to support Multifiles
75  * larger than 4GB. uint32 An overall modification timestamp for the
76  * entire multifile.
77  */
78 
79 /*
80  * (2) Zero or more index entries, one for each subfile within the Multifile.
81  * These entries are of variable length. The first one of these immediately
82  * follows the header, and the first word of each index entry contains the
83  * address of the next index entry. A zero "next" address marks the end of
84  * the chain. These may appear at any point within the Multifile; they do not
85  * necessarily appear in sequential order at the beginning of the file
86  * (although they will after the file has been "packed"). uint32 The
87  * address of the next entry. 0 to mark the end. uint32 The address of
88  * this subfile's data record. uint32 The length in bytes of this
89  * subfile's data record. uint16 The Subfile::_flags member. [uint32]
90  * The original, uncompressed and unencrypted length of the subfile, if it is
91  * compressed or encrypted. This field is only present if one or both of the
92  * SF_compressed or SF_encrypted bits are set in _flags. uint32 A
93  * modification timestamp for the subfile. uint16 The length in bytes of
94  * the subfile's name. char[n] The subfile's name. (3) Zero or more data
95  * entries, one for each subfile. These may appear at any point within the
96  * Multifile; they do not necessarily follow each index entry, nor are they
97  * necessarily all grouped together at the end (although they will be all
98  * grouped together at the end after the file has been "packed"). These are
99  * just blocks of literal data.
100  */
101 
102 /**
103  *
104  */
105 Multifile::
106 Multifile() :
107  _read_filew(_read_file),
108  _read_write_filew(_read_write_file)
109 {
110  ConfigVariableInt multifile_encryption_iteration_count
111  ("multifile-encryption-iteration-count", 0,
112  PRC_DESC("This is a special value of encryption-iteration-count used to encrypt "
113  "subfiles within a multifile. It has a default value of 0 (just one "
114  "application), on the assumption that the files from a multifile must "
115  "be loaded quickly, without paying the cost of an expensive hash on "
116  "each subfile in order to decrypt it."));
117 
118  _read = nullptr;
119  _write = nullptr;
120  _offset = 0;
121  _owns_stream = false;
122  _next_index = 0;
123  _last_index = 0;
124  _last_data_byte = 0;
125  _needs_repack = false;
126  _timestamp = 0;
127  _timestamp_dirty = false;
128  _record_timestamp = true;
129  _scale_factor = 1;
130  _new_scale_factor = 1;
131  _encryption_flag = false;
132  _encryption_iteration_count = multifile_encryption_iteration_count;
133  _file_major_ver = 0;
134  _file_minor_ver = 0;
135 
136 #ifdef HAVE_OPENSSL
137  // Get these values from the config file via an EncryptStreamBuf.
138  EncryptStreamBuf tbuf;
139  _encryption_algorithm = tbuf.get_algorithm();
140  _encryption_key_length = tbuf.get_key_length();
141 #endif
142 }
143 
144 /**
145  *
146  */
147 Multifile::
148 ~Multifile() {
149  close();
150 }
151 
152 /**
153  * Opens the named Multifile on disk for reading. The Multifile index is read
154  * in, and the list of subfiles becomes available; individual subfiles may
155  * then be extracted or read, but the list of subfiles may not be modified.
156  *
157  * Also see the version of open_read() which accepts an istream. Returns true
158  * on success, false on failure.
159  */
160 bool Multifile::
161 open_read(const Filename &multifile_name, const streampos &offset) {
162  close();
163  Filename fname = multifile_name;
164  fname.set_binary();
165 
167  PT(VirtualFile) vfile = vfs->get_file(fname);
168  if (vfile == nullptr) {
169  return false;
170  }
171  istream *multifile_stream = vfile->open_read_file(false);
172  if (multifile_stream == nullptr) {
173  return false;
174  }
175 
176  _timestamp = vfile->get_timestamp();
177  _timestamp_dirty = true;
178  _read = new IStreamWrapper(multifile_stream, true);
179  _owns_stream = true;
180  _multifile_name = multifile_name;
181  _offset = offset;
182  return read_index();
183 }
184 
185 /**
186  * Opens an anonymous Multifile for reading using an istream. There must be
187  * seek functionality via seekg() and tellg() on the istream.
188  *
189  * If owns_pointer is true, then the Multifile assumes ownership of the stream
190  * pointer and will delete it when the multifile is closed, including if this
191  * function returns false.
192  */
193 bool Multifile::
194 open_read(IStreamWrapper *multifile_stream, bool owns_pointer,
195  const streampos &offset) {
196  close();
197  _timestamp = time(nullptr);
198  _timestamp_dirty = true;
199  _read = multifile_stream;
200  _owns_stream = owns_pointer;
201  _offset = offset;
202  return read_index();
203 }
204 
205 /**
206  * Opens the named Multifile on disk for writing. If there already exists a
207  * file by that name, it is truncated. The Multifile is then prepared for
208  * accepting a brand new set of subfiles, which will be written to the
209  * indicated filename. Individual subfiles may not be extracted or read.
210  *
211  * Also see the version of open_write() which accepts an ostream. Returns
212  * true on success, false on failure.
213  */
214 bool Multifile::
215 open_write(const Filename &multifile_name) {
216  close();
217  Filename fname = multifile_name;
218  fname.set_binary();
219  if (!fname.open_write(_write_file, true)) {
220  return false;
221  }
222  _timestamp = time(nullptr);
223  _timestamp_dirty = true;
224  _write = &_write_file;
225  _multifile_name = multifile_name;
226  return true;
227 }
228 
229 /**
230  * Opens an anonymous Multifile for writing using an ostream. There must be
231  * seek functionality via seekp() and tellp() on the pstream.
232  *
233  * If owns_pointer is true, then the Multifile assumes ownership of the stream
234  * pointer and will delete it when the multifile is closed, including if this
235  * function returns false.
236  */
237 bool Multifile::
238 open_write(ostream *multifile_stream, bool owns_pointer) {
239  close();
240  _timestamp = time(nullptr);
241  _timestamp_dirty = true;
242  _write = multifile_stream;
243  _owns_stream = owns_pointer;
244  _write->seekp(0, ios::beg);
245  return true;
246 }
247 
248 /**
249  * Opens the named Multifile on disk for reading and writing. If there
250  * already exists a file by that name, its index is read. Subfiles may be
251  * added or removed, and the resulting changes will be written to the named
252  * file.
253  *
254  * Also see the version of open_read_write() which accepts an iostream.
255  * Returns true on success, false on failure.
256  */
257 bool Multifile::
258 open_read_write(const Filename &multifile_name) {
259  close();
260  Filename fname = multifile_name;
261  fname.set_binary();
262  bool exists = fname.exists();
263  if (!fname.open_read_write(_read_write_file)) {
264  return false;
265  }
266  if (exists) {
267  _timestamp = fname.get_timestamp();
268  } else {
269  _timestamp = time(nullptr);
270  }
271  _timestamp_dirty = true;
272  _read = &_read_write_filew;
273  _write = &_read_write_file;
274  _multifile_name = multifile_name;
275 
276  if (exists) {
277  return read_index();
278  } else {
279  return true;
280  }
281 }
282 
283 /**
284  * Opens an anonymous Multifile for reading and writing using an iostream.
285  * There must be seek functionality via seekg()/seekp() and tellg()/tellp() on
286  * the iostream.
287  *
288  * If owns_pointer is true, then the Multifile assumes ownership of the stream
289  * pointer and will delete it when the multifile is closed, including if this
290  * function returns false.
291  */
292 bool Multifile::
293 open_read_write(iostream *multifile_stream, bool owns_pointer) {
294  close();
295  _timestamp = time(nullptr);
296  _timestamp_dirty = true;
297 
298  // We don't support locking when opening a file in read-write mode, because
299  // we don't bother with locking on write. But we need to have an
300  // IStreamWrapper to assign to the _read member, so we create one on-the-fly
301  // here.
302  _read = new StreamWrapper(multifile_stream, owns_pointer);
303  _write = multifile_stream;
304  _owns_stream = true; // Because we own the StreamWrapper, above.
305  _write->seekp(0, ios::beg);
306 
307  // Check whether the read stream is empty.
308  multifile_stream->seekg(0, ios::end);
309  if (multifile_stream->tellg() == (streampos)0) {
310  // The read stream is empty, which is always valid.
311  return true;
312  }
313 
314  // The read stream is not empty, so we'd better have a valid Multifile.
315  return read_index();
316 }
317 
318 /**
319  * Closes the Multifile if it is open. All changes are flushed to disk, and
320  * the file becomes invalid for further operations until the next call to
321  * open().
322  */
323 void Multifile::
324 close() {
325  if (_new_scale_factor != _scale_factor) {
326  // If we have changed the scale factor recently, we need to force a
327  // repack.
328  repack();
329  } else {
330  flush();
331  }
332 
333  if (_owns_stream) {
334  // We prefer to delete the IStreamWrapper over the ostream, if possible.
335  if (_read != nullptr) {
336  // Only delete it if no SubStream is still referencing it.
337  if (!_read->unref()) {
338  delete _read;
339  }
340  } else if (_write != nullptr) {
341  delete _write;
342  }
343  }
344 
345  _read = nullptr;
346  _write = nullptr;
347  _offset = 0;
348  _owns_stream = false;
349  _next_index = 0;
350  _last_index = 0;
351  _needs_repack = false;
352  _timestamp = 0;
353  _timestamp_dirty = false;
354  _scale_factor = 1;
355  _new_scale_factor = 1;
356  _encryption_flag = false;
357  _file_major_ver = 0;
358  _file_minor_ver = 0;
359 
360  _read_file.close();
361  _write_file.close();
362  _read_write_file.close();
363  _multifile_name = Filename();
364 
365  clear_subfiles();
366 }
367 
368 /**
369  * Changes the internal scale factor for this Multifile.
370  *
371  * This is normally 1, but it may be set to any arbitrary value (greater than
372  * zero) to support Multifile archives that exceed 4GB, if necessary.
373  * (Individual subfiles may still not exceed 4GB.)
374  *
375  * All addresses within the file are rounded up to the next multiple of
376  * _scale_factor, and zeros are written to the file to fill the resulting
377  * gaps. Then the address is divided by _scale_factor and written out as a
378  * 32-bit integer. Thus, setting a scale factor of 2 supports up to 8GB
379  * files, 3 supports 12GB files, etc.
380  *
381  * Calling this function on an already-existing Multifile will have no
382  * immediate effect until a future call to repack() or close() (or until the
383  * Multifile is destructed).
384  */
385 void Multifile::
386 set_scale_factor(size_t scale_factor) {
387  nassertv(is_write_valid());
388  nassertv(scale_factor != (size_t)0);
389 
390  if (_next_index == (streampos)0) {
391  // If it's a brand new Multifile, we can go ahead and set it immediately.
392  _scale_factor = scale_factor;
393  } else {
394  // Otherwise, we'd better have read access so we can repack it later.
395  nassertv(is_read_valid());
396  }
397 
398  // Setting the _new_scale_factor different from the _scale_factor will force
399  // a repack operation on close.
400  _new_scale_factor = scale_factor;
401 }
402 
403 /**
404  * Adds a file on disk as a subfile to the Multifile. The file named by
405  * filename will be read and added to the Multifile at the next call to
406  * flush(). If there already exists a subfile with the indicated name, it is
407  * replaced without examining its contents (but see also update_subfile).
408  *
409  * Either Filename:::set_binary() or set_text() must have been called
410  * previously to specify the nature of the source file. If set_text() was
411  * called, the text flag will be set on the subfile.
412  *
413  * Returns the subfile name on success (it might have been modified slightly),
414  * or empty string on failure.
415  */
416 string Multifile::
417 add_subfile(const string &subfile_name, const Filename &filename,
418  int compression_level) {
419  nassertr(is_write_valid(), string());
420 
421  Filename fname = filename;
422  if (multifile_always_binary) {
423  fname.set_binary();
424  }
425 
426  nassertr(fname.is_binary_or_text(), string());
427 
428  if (!fname.exists()) {
429  return string();
430  }
431  string name = standardize_subfile_name(subfile_name);
432  if (!name.empty()) {
433  Subfile *subfile = new Subfile;
434  subfile->_name = name;
435  subfile->_source_filename = fname;
436  if (fname.is_text()) {
437  subfile->_flags |= SF_text;
438  }
439 
440  add_new_subfile(subfile, compression_level);
441  }
442 
443  _timestamp = time(nullptr);
444  _timestamp_dirty = true;
445 
446  return name;
447 }
448 
449 /**
450  * Adds a file from a stream as a subfile to the Multifile. The indicated
451  * istream will be read and its contents added to the Multifile at the next
452  * call to flush(). The file will be added as a binary subfile.
453  *
454  * Note that the istream must remain untouched and unused by any other code
455  * until flush() is called. At that time, the Multifile will read the entire
456  * contents of the istream from the current file position to the end of the
457  * file. Subsequently, the Multifile will *not* close or delete the istream.
458  * It is the caller's responsibility to ensure that the istream pointer does
459  * not destruct during the lifetime of the Multifile.
460  *
461  * Returns the subfile name on success (it might have been modified slightly),
462  * or empty string on failure.
463  */
464 string Multifile::
465 add_subfile(const string &subfile_name, istream *subfile_data,
466  int compression_level) {
467  nassertr(is_write_valid(), string());
468 
469  string name = standardize_subfile_name(subfile_name);
470  if (!name.empty()) {
471  Subfile *subfile = new Subfile;
472  subfile->_name = name;
473  subfile->_source = subfile_data;
474  add_new_subfile(subfile, compression_level);
475  }
476 
477  return name;
478 }
479 
480 /**
481  * Adds a file on disk to the subfile. If a subfile already exists with the
482  * same name, its contents are compared byte-for-byte to the disk file, and it
483  * is replaced only if it is different; otherwise, the multifile is left
484  * unchanged.
485  *
486  * Either Filename:::set_binary() or set_text() must have been called
487  * previously to specify the nature of the source file. If set_text() was
488  * called, the text flag will be set on the subfile.
489  */
490 string Multifile::
491 update_subfile(const string &subfile_name, const Filename &filename,
492  int compression_level) {
493  nassertr(is_write_valid(), string());
494 
495  Filename fname = filename;
496  if (multifile_always_binary) {
497  fname.set_binary();
498  }
499 
500  nassertr(fname.is_binary_or_text(), string());
501 
502  if (!fname.exists()) {
503  return string();
504  }
505  string name = standardize_subfile_name(subfile_name);
506  if (!name.empty()) {
507  int index = find_subfile(name);
508  if (index >= 0) {
509  // The subfile already exists; compare it to the source file.
510  if (compare_subfile(index, fname)) {
511  // The files are identical; do nothing.
512  return name;
513  }
514  }
515 
516  // The subfile does not already exist or it is different from the source
517  // file. Add the new source file.
518  Subfile *subfile = new Subfile;
519  subfile->_name = name;
520  subfile->_source_filename = fname;
521  if (fname.is_text()) {
522  subfile->_flags |= SF_text;
523  }
524 
525  add_new_subfile(subfile, compression_level);
526  }
527 
528  _timestamp = time(nullptr);
529  _timestamp_dirty = true;
530 
531  return name;
532 }
533 
534 #ifdef HAVE_OPENSSL
535 /**
536  * Ownership of the X509 object is passed into the CertRecord; it will be
537  * freed when the CertRecord destructs.
538  */
539 Multifile::CertRecord::
540 CertRecord(X509 *cert) :
541  _cert(cert)
542 {
543 }
544 
545 /**
546  *
547  */
548 Multifile::CertRecord::
549 CertRecord(const Multifile::CertRecord &copy) :
550  _cert(X509_dup(copy._cert))
551 {
552 }
553 
554 /**
555  *
556  */
557 Multifile::CertRecord::
558 ~CertRecord() {
559  X509_free(_cert);
560 }
561 
562 /**
563  *
564  */
565 void Multifile::CertRecord::
566 operator = (const Multifile::CertRecord &other) {
567  X509_free(_cert);
568  _cert = X509_dup(other._cert);
569 }
570 #endif // HAVE_OPENSSL
571 
572 #ifdef HAVE_OPENSSL
573 /**
574  * Adds a new signature to the Multifile. This signature associates the
575  * indicated certificate with the current contents of the Multifile. When the
576  * Multifile is read later, the signature will still be present only if the
577  * Multifile is unchanged; any subsequent changes to the Multifile will
578  * automatically invalidate and remove the signature.
579  *
580  * The chain filename may be empty if the certificate does not require an
581  * authenticating certificate chain (e.g. because it is self-signed).
582  *
583  * The specified private key must match the certificate, and the Multifile
584  * must be open in read-write mode. The private key is only used for
585  * generating the signature; it is not written to the Multifile and cannot be
586  * retrieved from the Multifile later. (However, the certificate *can* be
587  * retrieved from the Multifile later, to identify the entity that created the
588  * signature.)
589  *
590  * This implicitly causes a repack() operation if one is needed. Returns true
591  * on success, false on failure.
592  *
593  * This flavor of add_signature() reads the certificate and private key from a
594  * PEM-formatted file, for instance as generated by the openssl command. If
595  * the private key file is password-encrypted, the third parameter will be
596  * used as the password to decrypt it.
597  */
598 bool Multifile::
599 add_signature(const Filename &certificate, const Filename &chain,
600  const Filename &pkey, const string &password) {
602 
603  if (chain.empty() && pkey.empty()) {
604  // If the second two filenames are empty, assume we're going for the
605  // composite mode, where everything's stuffed into the first file.
606  return add_signature(certificate, password);
607  }
608 
609  CertChain cert_chain;
610 
611  // Read the certificate file from VFS. First, read the complete file into
612  // memory.
613  string certificate_data;
614  if (!vfs->read_file(certificate, certificate_data, true)) {
615  express_cat.info()
616  << "Could not read " << certificate << ".\n";
617  return false;
618  }
619 
620  // Create an in-memory BIO to read the "file" from the buffer.
621  BIO *certificate_mbio = BIO_new_mem_buf((void *)certificate_data.data(), certificate_data.size());
622  X509 *x509 = PEM_read_bio_X509(certificate_mbio, nullptr, nullptr, (void *)"");
623  BIO_free(certificate_mbio);
624  if (x509 == nullptr) {
625  express_cat.info()
626  << "Could not read certificate in " << certificate << ".\n";
627  return false;
628  }
629 
630  // Store the first X509--the actual certificate--as the first record in our
631  // CertChain object.
632  cert_chain.push_back(CertRecord(x509));
633 
634  // Read the rest of the certificates in the chain file.
635  if (!chain.empty()) {
636  string chain_data;
637  if (!vfs->read_file(chain, chain_data, true)) {
638  express_cat.info()
639  << "Could not read " << chain << ".\n";
640  return false;
641  }
642 
643  BIO *chain_mbio = BIO_new_mem_buf((void *)chain_data.data(), chain_data.size());
644  X509 *c = PEM_read_bio_X509(chain_mbio, nullptr, nullptr, (void *)"");
645  while (c != nullptr) {
646  cert_chain.push_back(c);
647  c = PEM_read_bio_X509(chain_mbio, nullptr, nullptr, (void *)"");
648  }
649  BIO_free(chain_mbio);
650 
651  if (cert_chain.size() == 1) {
652  express_cat.info()
653  << "Could not read certificate chain in " << chain << ".\n";
654  return false;
655  }
656  }
657 
658  // Now do the same thing with the private key. This one may be password-
659  // encrypted on disk.
660  string pkey_data;
661  if (!vfs->read_file(pkey, pkey_data, true)) {
662  express_cat.info()
663  << "Could not read " << pkey << ".\n";
664  return false;
665  }
666 
667  BIO *pkey_mbio = BIO_new_mem_buf((void *)pkey_data.data(), pkey_data.size());
668  EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, nullptr, nullptr,
669  (void *)password.c_str());
670  BIO_free(pkey_mbio);
671  if (evp_pkey == nullptr) {
672  express_cat.info()
673  << "Could not read private key in " << pkey << ".\n";
674  return false;
675  }
676 
677  bool result = add_signature(cert_chain, evp_pkey);
678 
679  EVP_PKEY_free(evp_pkey);
680 
681  return result;
682 }
683 #endif // HAVE_OPENSSL
684 
685 #ifdef HAVE_OPENSSL
686 /**
687  * Adds a new signature to the Multifile. This signature associates the
688  * indicated certificate with the current contents of the Multifile. When the
689  * Multifile is read later, the signature will still be present only if the
690  * Multifile is unchanged; any subsequent changes to the Multifile will
691  * automatically invalidate and remove the signature.
692  *
693  * This flavor of add_signature() reads the certificate, private key, and
694  * certificate chain from the same PEM-formatted file. It takes the first
695  * private key found as the intended key, and then uses the first certificate
696  * found that matches that key as the signing certificate. Any other
697  * certificates in the file are taken to be part of the chain.
698  */
699 bool Multifile::
700 add_signature(const Filename &composite, const string &password) {
702 
703  // First, read the complete file into memory.
704  string composite_data;
705  if (!vfs->read_file(composite, composite_data, true)) {
706  express_cat.info()
707  << "Could not read " << composite << ".\n";
708  return false;
709  }
710 
711  // Get the private key.
712  BIO *pkey_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size());
713  EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, nullptr, nullptr,
714  (void *)password.c_str());
715  BIO_free(pkey_mbio);
716  if (evp_pkey == nullptr) {
717  express_cat.info()
718  << "Could not read private key in " << composite << ".\n";
719  return false;
720  }
721 
722  // Now read all of the certificates.
723  CertChain cert_chain;
724 
725  BIO *chain_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size());
726  X509 *c = PEM_read_bio_X509(chain_mbio, nullptr, nullptr, (void *)"");
727  while (c != nullptr) {
728  cert_chain.push_back(c);
729  c = PEM_read_bio_X509(chain_mbio, nullptr, nullptr, (void *)"");
730  }
731  BIO_free(chain_mbio);
732 
733  if (cert_chain.empty()) {
734  express_cat.info()
735  << "Could not read certificates in " << composite << ".\n";
736  return false;
737  }
738 
739  // Now find the certificate that matches the signature, and move it to the
740  // front of the chain.
741  size_t i;
742  bool found_match = false;
743  for (i = 0; i < cert_chain.size(); ++i) {
744  X509 *c = cert_chain[i]._cert;
745  if (X509_check_private_key(c, evp_pkey)) {
746  found_match = true;
747  if (i != 0) {
748  // Move this entry to the beginning.
749  cert_chain.insert(cert_chain.begin(), cert_chain[i]);
750  cert_chain.erase(cert_chain.begin() + i + 1);
751  }
752  break;
753  }
754  }
755 
756  if (!found_match) {
757  express_cat.info()
758  << "No certificates in " << composite << " match key.\n";
759  return false;
760  }
761 
762  bool result = add_signature(cert_chain, evp_pkey);
763 
764  EVP_PKEY_free(evp_pkey);
765 
766  return result;
767 }
768 #endif // HAVE_OPENSSL
769 
770 #ifdef HAVE_OPENSSL
771 /**
772  * Adds a new signature to the Multifile. This signature associates the
773  * indicated certificate with the current contents of the Multifile. When the
774  * Multifile is read later, the signature will still be present only if the
775  * Multifile is unchanged; any subsequent changes to the Multifile will
776  * automatically invalidate and remove the signature.
777  *
778  * The signature certificate is the first certificate on the CertChain object.
779  * Any remaining certificates are support certificates to authenticate the
780  * first one.
781  *
782  * The specified private key must match the certificate, and the Multifile
783  * must be open in read-write mode. The private key is only used for
784  * generating the signature; it is not written to the Multifile and cannot be
785  * retrieved from the Multifile later. (However, the certificate *can* be
786  * retrieved from the Multifile later, to identify the entity that created the
787  * signature.)
788  *
789  * This implicitly causes a repack() operation if one is needed. Returns true
790  * on success, false on failure.
791  */
792 bool Multifile::
793 add_signature(const Multifile::CertChain &cert_chain, EVP_PKEY *pkey) {
794  if (_needs_repack) {
795  if (!repack()) {
796  return false;
797  }
798  } else {
799  if (!flush()) {
800  return false;
801  }
802  }
803 
804  if (cert_chain.empty()) {
805  express_cat.info()
806  << "No certificate given.\n";
807  return false;
808  }
809 
810  if (pkey == nullptr) {
811  express_cat.info()
812  << "No private key given.\n";
813  return false;
814  }
815 
816  if (!X509_check_private_key(cert_chain[0]._cert, pkey)) {
817  express_cat.info()
818  << "Private key does not match certificate.\n";
819  return false;
820  }
821 
822  // Now encode that list of certs to a stream in DER form.
823  stringstream der_stream;
824  StreamWriter der_writer(der_stream);
825  der_writer.add_uint32((uint32_t)cert_chain.size());
826 
827  CertChain::const_iterator ci;
828  for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) {
829  X509 *cert = (*ci)._cert;
830 
831  int der_len = i2d_X509(cert, nullptr);
832  unsigned char *der_buf = new unsigned char[der_len];
833  unsigned char *p = der_buf;
834  i2d_X509(cert, &p);
835  der_writer.append_data(der_buf, der_len);
836  delete[] der_buf;
837  }
838 
839  // Create a temporary Subfile for writing out the signature.
840  der_stream.seekg(0);
841  Subfile *subfile = new Subfile;
842  subfile->_pkey = pkey;
843  subfile->_flags |= SF_signature;
844  subfile->_source = &der_stream;
845 
846  // Write the new Subfile at the end. The cert_special subfiles always go at
847  // the end, because they're not the part of the file that's signed.
848  nassertr(_new_subfiles.empty(), false);
849  _new_subfiles.push_back(subfile);
850  bool result = flush();
851 
852  delete subfile;
853 
854  return result;
855 }
856 #endif // HAVE_OPENSSL
857 
858 #ifdef HAVE_OPENSSL
859 /**
860  * Returns the number of matching signatures found on the Multifile. These
861  * signatures may be iterated via get_signature() and related methods.
862  *
863  * A signature on this list is guaranteed to match the Multifile contents,
864  * proving that the Multifile has been unmodified since the signature was
865  * applied. However, this does not guarantee that the certificate itself is
866  * actually from who it says it is from; only that it matches the Multifile
867  * contents. See validate_signature_certificate() to authenticate a
868  * particular certificate.
869  */
870 int Multifile::
871 get_num_signatures() const {
872  ((Multifile *)this)->check_signatures();
873  return _signatures.size();
874 }
875 #endif // HAVE_OPENSSL
876 
877 #ifdef HAVE_OPENSSL
878 /**
879  * Returns the nth signature found on the Multifile. See the comments in
880  * get_num_signatures().
881  */
882 const Multifile::CertChain &Multifile::
883 get_signature(int n) const {
884  ((Multifile *)this)->check_signatures();
885  static CertChain error_chain;
886  nassertr(n >= 0 && n < (int)_signatures.size(), error_chain);
887  return _signatures[n];
888 }
889 #endif // HAVE_OPENSSL
890 
891 #ifdef HAVE_OPENSSL
892 /**
893  * Returns the "subject name" for the nth signature found on the Multifile.
894  * This is a string formatted according to RFC2253 that should more-or-less
895  * identify a particular certificate; when paired with the public key (see
896  * get_signature_public_key()), it can uniquely identify a certificate. See
897  * the comments in get_num_signatures().
898  */
899 string Multifile::
900 get_signature_subject_name(int n) const {
901  const CertChain &cert_chain = get_signature(n);
902  nassertr(!cert_chain.empty(), string());
903 
904  X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert);
905  if (xname != nullptr) {
906  // We use "print" to dump the output to a memory BIO. Is there an easier
907  // way to extract the X509_NAME text? Curse these incomplete docs.
908  BIO *mbio = BIO_new(BIO_s_mem());
909  X509_NAME_print_ex(mbio, xname, 0, XN_FLAG_RFC2253);
910 
911  char *pp;
912  long pp_size = BIO_get_mem_data(mbio, &pp);
913  string name(pp, pp_size);
914  BIO_free(mbio);
915  return name;
916  }
917 
918  return string();
919 }
920 #endif // HAVE_OPENSSL
921 
922 #ifdef HAVE_OPENSSL
923 /**
924  * Returns a "friendly name" for the nth signature found on the Multifile.
925  * This attempts to extract out the most meaningful part of the subject name.
926  * It returns the emailAddress, if it is defined; otherwise, it returns the
927  * commonName.
928  *
929  * See the comments in get_num_signatures().
930  */
931 string Multifile::
932 get_signature_friendly_name(int n) const {
933  const CertChain &cert_chain = get_signature(n);
934  nassertr(!cert_chain.empty(), string());
935 
936  static const int nid_choices[] = {
937  NID_pkcs9_emailAddress,
938  NID_subject_alt_name,
939  NID_commonName,
940  -1,
941  };
942 
943  // Choose the first NID that exists on the cert.
944  for (int ni = 0; nid_choices[ni] != -1; ++ni) {
945  int nid = nid_choices[ni];
946 
947  // A complex OpenSSL interface to extract out the name in utf-8.
948  X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert);
949  if (xname != nullptr) {
950  int pos = X509_NAME_get_index_by_NID(xname, nid, -1);
951  if (pos != -1) {
952  // We just get the first common name. I guess it's possible to have
953  // more than one; not sure what that means in this context.
954  X509_NAME_ENTRY *xentry = X509_NAME_get_entry(xname, pos);
955  if (xentry != nullptr) {
956  ASN1_STRING *data = X509_NAME_ENTRY_get_data(xentry);
957  if (data != nullptr) {
958  // We use "print" to dump the output to a memory BIO. Is there an
959  // easier way to decode the ASN1_STRING? Curse these incomplete
960  // docs.
961  BIO *mbio = BIO_new(BIO_s_mem());
962  ASN1_STRING_print_ex(mbio, data, ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB);
963 
964  char *pp;
965  long pp_size = BIO_get_mem_data(mbio, &pp);
966  string name(pp, pp_size);
967  BIO_free(mbio);
968  return name;
969  }
970  }
971  }
972  }
973  }
974 
975  return string();
976 }
977 #endif // HAVE_OPENSSL
978 
979 #ifdef HAVE_OPENSSL
980 /**
981  * Returns the public key used for the nth signature found on the Multifile.
982  * This is encoded in DER form and returned as a string of hex digits.
983  *
984  * This can be used, in conjunction with the subject name (see
985  * get_signature_subject_name()), to uniquely identify a particular
986  * certificate and its subsequent reissues. See the comments in
987  * get_num_signatures().
988  */
989 string Multifile::
990 get_signature_public_key(int n) const {
991  const CertChain &cert_chain = get_signature(n);
992  nassertr(!cert_chain.empty(), string());
993 
994  EVP_PKEY *pkey = X509_get_pubkey(cert_chain[0]._cert);
995  if (pkey != nullptr) {
996  int key_len = i2d_PublicKey(pkey, nullptr);
997  unsigned char *key_buf = new unsigned char[key_len];
998  unsigned char *p = key_buf;
999  i2d_PublicKey(pkey, &p);
1000  string result;
1001  for (int i = 0; i < key_len; ++i) {
1002  result += tohex(key_buf[i] >> 4);
1003  result += tohex(key_buf[i]);
1004  }
1005  delete[] key_buf;
1006  return result;
1007  }
1008 
1009  return string();
1010 }
1011 #endif // HAVE_OPENSSL
1012 
1013 #ifdef HAVE_OPENSSL
1014 /**
1015  * Writes the certificate for the nth signature, in user-readable verbose
1016  * form, to the indicated stream. See the comments in get_num_signatures().
1017  */
1018 void Multifile::
1019 print_signature_certificate(int n, ostream &out) const {
1020  const CertChain &cert_chain = get_signature(n);
1021  nassertv(!cert_chain.empty());
1022 
1023  BIO *mbio = BIO_new(BIO_s_mem());
1024  X509_print(mbio, cert_chain[0]._cert);
1025 
1026  char *pp;
1027  long pp_size = BIO_get_mem_data(mbio, &pp);
1028  out.write(pp, pp_size);
1029  BIO_free(mbio);
1030 }
1031 #endif // HAVE_OPENSSL
1032 
1033 #ifdef HAVE_OPENSSL
1034 /**
1035  * Writes the certificate for the nth signature, in PEM form, to the indicated
1036  * stream. See the comments in get_num_signatures().
1037  */
1038 void Multifile::
1039 write_signature_certificate(int n, ostream &out) const {
1040  const CertChain &cert_chain = get_signature(n);
1041  nassertv(!cert_chain.empty());
1042 
1043  BIO *mbio = BIO_new(BIO_s_mem());
1044 
1045  CertChain::const_iterator ci;
1046  for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) {
1047  X509 *c = (*ci)._cert;
1048  X509_print(mbio, c);
1049  PEM_write_bio_X509(mbio, c);
1050  }
1051 
1052  char *pp;
1053  long pp_size = BIO_get_mem_data(mbio, &pp);
1054  out.write(pp, pp_size);
1055  BIO_free(mbio);
1056 }
1057 #endif // HAVE_OPENSSL
1058 
1059 #ifdef HAVE_OPENSSL
1060 /**
1061  * Checks that the certificate used for the nth signature is a valid,
1062  * authorized certificate with some known certificate authority. Returns 0 if
1063  * it is valid, -1 if there is some error, or the corresponding OpenSSL error
1064  * code if it is invalid, out-of-date, or self-signed.
1065  */
1066 int Multifile::
1067 validate_signature_certificate(int n) const {
1068  int verify_result = -1;
1069 
1070  const CertChain &chain = get_signature(n);
1071  nassertr(!chain.empty(), false);
1072 
1073  OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
1074 
1075  // Copy our CertChain structure into an X509 pointer and accompanying
1076  // STACK_OF(X509) pointer.
1077  X509 *x509 = chain[0]._cert;
1078  STACK_OF(X509) *stack = nullptr;
1079  if (chain.size() > 1) {
1080  stack = sk_X509_new(nullptr);
1081  for (size_t n = 1; n < chain.size(); ++n) {
1082  sk_X509_push(stack, chain[n]._cert);
1083  }
1084  }
1085 
1086  // Create the X509_STORE_CTX for verifying the cert and chain.
1087  X509_STORE_CTX *ctx = X509_STORE_CTX_new();
1088  X509_STORE_CTX_init(ctx, sslw->get_x509_store(), x509, stack);
1089  X509_STORE_CTX_set_cert(ctx, x509);
1090 
1091  if (X509_verify_cert(ctx)) {
1092  verify_result = 0;
1093  } else {
1094  verify_result = X509_STORE_CTX_get_error(ctx);
1095  }
1096 
1097  if (express_cat.is_debug()) {
1098  express_cat.debug()
1099  << get_signature_subject_name(n) << ": validate " << verify_result
1100  << "\n";
1101  }
1102 
1103  sk_X509_free(stack);
1104  X509_STORE_CTX_cleanup(ctx);
1105  X509_STORE_CTX_free(ctx);
1106 
1107  return verify_result;
1108 }
1109 #endif // HAVE_OPENSSL
1110 
1111 /**
1112  * Writes all contents of the Multifile to disk. Until flush() is called,
1113  * add_subfile() and remove_subfile() do not actually do anything to disk. At
1114  * this point, all of the recently-added subfiles are read and their contents
1115  * are added to the end of the Multifile, and the recently-removed subfiles
1116  * are marked gone from the Multifile.
1117  *
1118  * This may result in a suboptimal index. To guarantee that the index is
1119  * written at the beginning of the file, call repack() instead of flush().
1120  *
1121  * It is not necessary to call flush() explicitly unless you are concerned
1122  * about reading the recently-added subfiles immediately.
1123  *
1124  * Returns true on success, false on failure.
1125  */
1126 bool Multifile::
1128  if (!is_write_valid()) {
1129  return false;
1130  }
1131 
1132  bool new_file = (_next_index == (streampos)0);
1133  if (new_file) {
1134  // If we don't have an index yet, we don't have a header. Write the
1135  // header.
1136  if (!write_header()) {
1137  return false;
1138  }
1139 
1140  } else {
1141  if (_file_minor_ver != _current_minor_ver) {
1142  // If we *do* have an index already, but this is an old version
1143  // multifile, we have to completely rewrite it anyway.
1144  return repack();
1145  }
1146  }
1147 
1148  nassertr(_write != nullptr, false);
1149 
1150  // First, mark out all of the removed subfiles.
1151  PendingSubfiles::iterator pi;
1152  for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
1153  Subfile *subfile = (*pi);
1154  subfile->rewrite_index_flags(*_write);
1155  delete subfile;
1156  }
1157  _removed_subfiles.clear();
1158 
1159  bool wrote_ok = true;
1160 
1161  if (!_new_subfiles.empty() || new_file) {
1162  // Add a few more files to the end. We always add subfiles at the end of
1163  // the multifile, so go there first.
1164  sort(_new_subfiles.begin(), _new_subfiles.end(), IndirectLess<Subfile>());
1165  if (_last_index != (streampos)0) {
1166  _write->seekp(0, ios::end);
1167  if (_write->fail()) {
1168  express_cat.info()
1169  << "Unable to seek Multifile " << _multifile_name << ".\n";
1170  return false;
1171  }
1172  _next_index = _write->tellp();
1173  _next_index = pad_to_streampos(_next_index);
1174 
1175  // And update the forward link from the last_index to point to this new
1176  // index location.
1177  _write->seekp(_last_index);
1178  StreamWriter writer(_write, false);
1179  writer.add_uint32(streampos_to_word(_next_index));
1180  }
1181 
1182  _write->seekp(_next_index);
1183  nassertr(_next_index == _write->tellp(), false);
1184 
1185  // Ok, here we are at the end of the file. Write out the recently-added
1186  // subfiles here. First, count up the index size.
1187  for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
1188  Subfile *subfile = (*pi);
1189  _last_index = _next_index;
1190  _next_index = subfile->write_index(*_write, _next_index, this);
1191  nassertr(_next_index == _write->tellp(), false);
1192  _next_index = pad_to_streampos(_next_index);
1193  nassertr(_next_index == _write->tellp(), false);
1194  }
1195 
1196  // Now we're at the end of the index. Write a 0 here to mark the end.
1197  StreamWriter writer(_write, false);
1198  writer.add_uint32(0);
1199  _next_index += 4;
1200  nassertr(_next_index == _write->tellp(), false);
1201  _next_index = pad_to_streampos(_next_index);
1202 
1203  // All right, now write out each subfile's data.
1204  for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
1205  Subfile *subfile = (*pi);
1206 
1207  if (_read != nullptr) {
1208  _read->acquire();
1209  _next_index = subfile->write_data(*_write, _read->get_istream(),
1210  _next_index, this);
1211  _read->release();
1212 
1213  } else {
1214  _next_index = subfile->write_data(*_write, nullptr, _next_index, this);
1215  }
1216 
1217  nassertr(_next_index == _write->tellp(), false);
1218  _next_index = pad_to_streampos(_next_index);
1219  if (subfile->is_data_invalid()) {
1220  wrote_ok = false;
1221  }
1222 
1223  if (!subfile->is_cert_special()) {
1224  _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos());
1225  }
1226  nassertr(_next_index == _write->tellp(), false);
1227  }
1228 
1229  // Now go back and fill in the proper addresses for the data start. We
1230  // didn't do it in the first pass, because we don't really want to keep
1231  // all those file handles open, and so we didn't have to determine each
1232  // file's length ahead of time.
1233  for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
1234  Subfile *subfile = (*pi);
1235  subfile->rewrite_index_data_start(*_write, this);
1236  }
1237 
1238  _new_subfiles.clear();
1239  }
1240 
1241  // Also update the overall timestamp.
1242  if (_timestamp_dirty) {
1243  nassertr(!_write->fail(), false);
1244  static const size_t timestamp_pos = _header_prefix.size() + _header_size + 2 + 2 + 4;
1245  _write->seekp(timestamp_pos);
1246  nassertr(!_write->fail(), false);
1247 
1248  StreamWriter writer(*_write);
1249  if (_record_timestamp) {
1250  writer.add_uint32(_timestamp);
1251  } else {
1252  writer.add_uint32(0);
1253  }
1254  _timestamp_dirty = false;
1255  }
1256 
1257  _write->flush();
1258  if (!wrote_ok || _write->fail()) {
1259  express_cat.info()
1260  << "Unable to update Multifile " << _multifile_name << ".\n";
1261  close();
1262  return false;
1263  }
1264 
1265  return true;
1266 }
1267 
1268 /**
1269  * Forces a complete rewrite of the Multifile and all of its contents, so that
1270  * its index will appear at the beginning of the file with all of the subfiles
1271  * listed in alphabetical order. This is considered optimal for reading, and
1272  * is the standard configuration; but it is not essential to do this.
1273  *
1274  * It is only valid to call this if the Multifile was opened using
1275  * open_read_write() and an explicit filename, rather than an iostream. Also,
1276  * we must have write permission to the directory containing the Multifile.
1277  *
1278  * Returns true on success, false on failure.
1279  */
1280 bool Multifile::
1282  if (_next_index == (streampos)0) {
1283  // If the Multifile hasn't yet been written, this is really just a flush
1284  // operation.
1285  _needs_repack = false;
1286  return flush();
1287  }
1288 
1289  nassertr(is_write_valid() && is_read_valid(), false);
1290  nassertr(!_multifile_name.empty(), false);
1291 
1292  // First, we open a temporary filename to copy the Multifile to.
1293  Filename dirname = _multifile_name.get_dirname();
1294  if (dirname.empty()) {
1295  dirname = ".";
1296  }
1297  Filename temp_filename = Filename::temporary(dirname, "mftemp");
1298  temp_filename.set_binary();
1299  pofstream temp;
1300  if (!temp_filename.open_write(temp)) {
1301  express_cat.info()
1302  << "Unable to open temporary file " << temp_filename << "\n";
1303  return false;
1304  }
1305 
1306  // Now we scrub our internal structures so it looks like we're a brand new
1307  // Multifile.
1308  PendingSubfiles::iterator pi;
1309  for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
1310  Subfile *subfile = (*pi);
1311  delete subfile;
1312  }
1313  _removed_subfiles.clear();
1314  _new_subfiles.clear();
1315  std::copy(_subfiles.begin(), _subfiles.end(), std::back_inserter(_new_subfiles));
1316  _next_index = 0;
1317  _last_index = 0;
1318  _last_data_byte = 0;
1319  _scale_factor = _new_scale_factor;
1320 
1321  // And we write our contents to our new temporary file.
1322  _write = &temp;
1323  if (!flush()) {
1324  temp.close();
1325  temp_filename.unlink();
1326  return false;
1327  }
1328 
1329  // Now close everything, and move the temporary file back over our original
1330  // file.
1331  Filename orig_name = _multifile_name;
1332  temp.close();
1333  close();
1334  orig_name.unlink();
1335  if (!temp_filename.rename_to(orig_name)) {
1336  express_cat.info()
1337  << "Unable to rename temporary file " << temp_filename << " to "
1338  << orig_name << ".\n";
1339  return false;
1340  }
1341 
1342  if (!open_read_write(orig_name)) {
1343  express_cat.info()
1344  << "Unable to read newly repacked " << _multifile_name
1345  << ".\n";
1346  return false;
1347  }
1348 
1349  return true;
1350 }
1351 
1352 /**
1353  * Returns the number of subfiles within the Multifile. The subfiles may be
1354  * accessed in alphabetical order by iterating through [0 ..
1355  * get_num_subfiles()).
1356  */
1357 int Multifile::
1358 get_num_subfiles() const {
1359  return _subfiles.size();
1360 }
1361 
1362 /**
1363  * Returns the index of the subfile with the indicated name, or -1 if the
1364  * named subfile is not within the Multifile.
1365  */
1366 int Multifile::
1367 find_subfile(const string &subfile_name) const {
1368  Subfile find_subfile;
1369  find_subfile._name = standardize_subfile_name(subfile_name);
1370  Subfiles::const_iterator fi;
1371  fi = _subfiles.find(&find_subfile);
1372  if (fi == _subfiles.end()) {
1373  // Not present.
1374  return -1;
1375  }
1376  return (fi - _subfiles.begin());
1377 }
1378 
1379 /**
1380  * Returns true if the indicated subfile name is the directory prefix to one
1381  * or more files within the Multifile. That is, the Multifile contains at
1382  * least one file named "subfile_name/...".
1383  */
1384 bool Multifile::
1385 has_directory(const string &subfile_name) const {
1386  string prefix = subfile_name;
1387  if (!prefix.empty()) {
1388  prefix += '/';
1389  }
1390  Subfile find_subfile;
1391  find_subfile._name = prefix;
1392  Subfiles::const_iterator fi;
1393  fi = _subfiles.upper_bound(&find_subfile);
1394  if (fi == _subfiles.end()) {
1395  // Not present.
1396  return false;
1397  }
1398 
1399  // At least one subfile exists whose name sorts after prefix. If it
1400  // contains prefix as the initial substring, then we have a match.
1401  Subfile *subfile = (*fi);
1402  return (subfile->_name.length() > prefix.length() &&
1403  subfile->_name.substr(0, prefix.length()) == prefix);
1404 }
1405 
1406 /**
1407  * Considers subfile_name to be the name of a subdirectory within the
1408  * Multifile, but not a file itself; fills the given vector up with the sorted
1409  * list of subdirectories or files within the named directory.
1410  *
1411  * Note that directories do not exist explicitly within a Multifile; this just
1412  * checks for the existence of files with the given initial prefix.
1413  *
1414  * Returns true if successful, false otherwise.
1415  */
1416 bool Multifile::
1417 scan_directory(vector_string &contents, const string &subfile_name) const {
1418  string prefix = subfile_name;
1419  if (!prefix.empty()) {
1420  prefix += '/';
1421  }
1422  Subfile find_subfile;
1423  find_subfile._name = prefix;
1424  Subfiles::const_iterator fi;
1425  fi = _subfiles.upper_bound(&find_subfile);
1426 
1427  string previous = "";
1428  while (fi != _subfiles.end()) {
1429  Subfile *subfile = (*fi);
1430  if (!(subfile->_name.length() > prefix.length() &&
1431  subfile->_name.substr(0, prefix.length()) == prefix)) {
1432  // We've reached the end of the list of subfiles beneath the indicated
1433  // directory prefix.
1434  return true;
1435  }
1436 
1437  size_t slash = subfile->_name.find('/', prefix.length());
1438  string basename = subfile->_name.substr(prefix.length(), slash - prefix.length());
1439  if (basename != previous) {
1440  contents.push_back(basename);
1441  previous = basename;
1442  }
1443  ++fi;
1444  }
1445 
1446  return true;
1447 }
1448 
1449 /**
1450  * Removes the nth subfile from the Multifile. This will cause all subsequent
1451  * index numbers to decrease by one. The file will not actually be removed
1452  * from the disk until the next call to flush().
1453  *
1454  * Note that this does not actually remove the data from the indicated
1455  * subfile; it simply removes it from the index. The Multifile will not be
1456  * reduced in size after this operation, until the next call to repack().
1457  */
1458 void Multifile::
1459 remove_subfile(int index) {
1460  nassertv(is_write_valid());
1461  nassertv(index >= 0 && index < (int)_subfiles.size());
1462  Subfile *subfile = _subfiles[index];
1463  subfile->_flags |= SF_deleted;
1464  _removed_subfiles.push_back(subfile);
1465  _subfiles.erase(_subfiles.begin() + index);
1466 
1467  _timestamp = time(nullptr);
1468  _timestamp_dirty = true;
1469 
1470  _needs_repack = true;
1471 }
1472 
1473 /**
1474  * Returns the name of the nth subfile.
1475  */
1476 const string &Multifile::
1477 get_subfile_name(int index) const {
1478 #ifndef NDEBUG
1479  static string empty_string;
1480  nassertr(index >= 0 && index < (int)_subfiles.size(), empty_string);
1481 #endif
1482  return _subfiles[index]->_name;
1483 }
1484 
1485 /**
1486  * Returns the uncompressed data length of the nth subfile. This might return
1487  * 0 if the subfile has recently been added and flush() has not yet been
1488  * called.
1489  */
1490 size_t Multifile::
1491 get_subfile_length(int index) const {
1492  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
1493  return _subfiles[index]->_uncompressed_length;
1494 }
1495 
1496 /**
1497  * Returns the modification time of the nth subfile. If this is called on an
1498  * older .mf file, which did not store individual timestamps in the file (or
1499  * if get_record_timestamp() is false), this will return the modification time
1500  * of the overall multifile.
1501  */
1502 time_t Multifile::
1503 get_subfile_timestamp(int index) const {
1504  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
1505  if (!get_record_timestamp()) {
1506  return get_timestamp();
1507  } else {
1508  return _subfiles[index]->_timestamp;
1509  }
1510 }
1511 
1512 /**
1513  * Returns true if the indicated subfile has been compressed when stored
1514  * within the archive, false otherwise.
1515  */
1516 bool Multifile::
1517 is_subfile_compressed(int index) const {
1518  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1519  return (_subfiles[index]->_flags & SF_compressed) != 0;
1520 }
1521 
1522 /**
1523  * Returns true if the indicated subfile has been encrypted when stored within
1524  * the archive, false otherwise.
1525  */
1526 bool Multifile::
1527 is_subfile_encrypted(int index) const {
1528  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1529  return (_subfiles[index]->_flags & SF_encrypted) != 0;
1530 }
1531 
1532 /**
1533  * Returns true if the indicated subfile represents text data, or false if it
1534  * represents binary data. If the file is text data, it may have been
1535  * processed by end-of-line conversion when it was added. (But the actual
1536  * bits in the multifile will represent the standard Unix end-of-line
1537  * convention, e.g. \n instead of \r\n.)
1538  */
1539 bool Multifile::
1540 is_subfile_text(int index) const {
1541  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1542  return (_subfiles[index]->_flags & SF_text) != 0;
1543 }
1544 
1545 /**
1546  * Returns the first byte that is guaranteed to follow any index byte already
1547  * written to disk in the Multifile.
1548  *
1549  * This number is largely meaningless in many cases, but if needs_repack() is
1550  * false, and the file is flushed, this will indicate the number of bytes in
1551  * the header + index. Everything at this byte position and later will be
1552  * actual data.
1553  */
1554 streampos Multifile::
1555 get_index_end() const {
1556  return normalize_streampos(_next_index + (streampos)4);
1557 }
1558 
1559 /**
1560  * Returns the starting byte position within the Multifile at which the
1561  * indicated subfile begins. This may be used, with
1562  * get_subfile_internal_length(), for low-level access to the subfile, but
1563  * usually it is better to use open_read_subfile() instead (which
1564  * automatically decrypts and/or uncompresses the subfile data).
1565  */
1566 streampos Multifile::
1567 get_subfile_internal_start(int index) const {
1568  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
1569  return _subfiles[index]->_data_start;
1570 }
1571 
1572 /**
1573  * Returns the number of bytes the indicated subfile consumes within the
1574  * archive. For compressed subfiles, this will generally be smaller than
1575  * get_subfile_length(); for encrypted (but noncompressed) subfiles, it may be
1576  * slightly different, for noncompressed and nonencrypted subfiles, it will be
1577  * equal.
1578  */
1579 size_t Multifile::
1581  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
1582  return _subfiles[index]->_data_length;
1583 }
1584 
1585 /**
1586  * Returns an istream that may be used to read the indicated subfile. You may
1587  * seek() within this istream to your heart's content; even though it will be
1588  * a reference to the already-opened pfstream of the Multifile itself, byte 0
1589  * appears to be the beginning of the subfile and EOF appears to be the end of
1590  * the subfile.
1591  *
1592  * The returned istream will have been allocated via new; you should pass the
1593  * pointer to close_read_subfile() when you are finished with it to delete it
1594  * and release its resources.
1595  *
1596  * Any future calls to repack() or close() (or the Multifile destructor) will
1597  * invalidate all currently open subfile pointers.
1598  *
1599  * The return value will be NULL if the stream cannot be opened for some
1600  * reason.
1601  */
1602 istream *Multifile::
1603 open_read_subfile(int index) {
1604  nassertr(is_read_valid(), nullptr);
1605  nassertr(index >= 0 && index < (int)_subfiles.size(), nullptr);
1606  Subfile *subfile = _subfiles[index];
1607 
1608  if (subfile->_source != nullptr ||
1609  !subfile->_source_filename.empty()) {
1610  // The subfile has not yet been copied into the physical Multifile. Force
1611  // a flush operation to incorporate it.
1612  flush();
1613 
1614  // That shouldn't change the subfile index or delete the subfile pointer.
1615  nassertr(subfile == _subfiles[index], nullptr);
1616  }
1617 
1618  return open_read_subfile(subfile);
1619 }
1620 
1621 /**
1622  * Closes a file opened by a previous call to open_read_subfile(). This
1623  * really just deletes the istream pointer, but it is recommended to use this
1624  * interface instead of deleting it explicitly, to help work around compiler
1625  * issues.
1626  */
1627 void Multifile::
1628 close_read_subfile(istream *stream) {
1629  if (stream != nullptr) {
1630  // For some reason--compiler bug in gcc 3.2?--explicitly deleting the
1631  // stream pointer does not call the appropriate global delete function;
1632  // instead apparently calling the system delete function. So we call the
1633  // delete function by hand instead.
1634 #if !defined(WIN32_VC) && !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW)
1635  stream->~istream();
1636  (*global_operator_delete)(stream);
1637 #else
1638  delete stream;
1639 #endif
1640  }
1641 }
1642 
1643 /**
1644  * Extracts the nth subfile into a file with the given name.
1645  */
1646 bool Multifile::
1647 extract_subfile(int index, const Filename &filename) {
1648  nassertr(is_read_valid(), false);
1649  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1650 
1651  Filename fname = filename;
1652  if (multifile_always_binary) {
1653  fname.set_binary();
1654  }
1655 
1656  if (!fname.is_binary_or_text()) {
1657  // If we haven't specified binary or text, infer it from the type of the
1658  // subfile.
1659  if ((_subfiles[index]->_flags & SF_text) != 0) {
1660  fname.set_text();
1661  } else {
1662  fname.set_binary();
1663  }
1664  }
1665  fname.make_dir();
1666  pofstream out;
1667  if (!fname.open_write(out, true)) {
1668  express_cat.info()
1669  << "Unable to write to file " << filename << "\n";
1670  return false;
1671  }
1672 
1673  return extract_subfile_to(index, out);
1674 }
1675 
1676 /**
1677  * Extracts the nth subfile to the indicated ostream.
1678  */
1679 bool Multifile::
1680 extract_subfile_to(int index, ostream &out) {
1681  nassertr(is_read_valid(), false);
1682  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1683 
1684  istream *in = open_read_subfile(index);
1685  if (in == nullptr) {
1686  return false;
1687  }
1688 
1689  static const size_t buffer_size = 4096;
1690  char buffer[buffer_size];
1691 
1692  in->read(buffer, buffer_size);
1693  size_t count = in->gcount();
1694  while (count != 0) {
1695  out.write(buffer, count);
1696  in->read(buffer, buffer_size);
1697  count = in->gcount();
1698  }
1699 
1700  bool failed = (in->fail() && !in->eof());
1701  close_read_subfile(in);
1702  nassertr(!failed, false);
1703 
1704  return (!out.fail());
1705 }
1706 
1707 /**
1708  * Performs a byte-for-byte comparison of the indicated file on disk with the
1709  * nth subfile. Returns true if the files are equivalent, or false if they
1710  * are different (or the file is missing).
1711  *
1712  * If Filename::set_binary() or set_text() has already been called, it
1713  * specifies the nature of the source file. If this is different from the
1714  * text flag of the subfile, the comparison will always return false. If this
1715  * has not been specified, it will be set from the text flag of the subfile.
1716  */
1717 bool Multifile::
1718 compare_subfile(int index, const Filename &filename) {
1719  nassertr(is_read_valid(), false);
1720  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1721 
1722  if (!filename.exists()) {
1723  express_cat.info()
1724  << "File is missing: " << filename << "\n";
1725  return false;
1726  }
1727 
1728  Filename fname = filename;
1729  if (fname.is_binary()) {
1730  // If we've specified a binary file, it had better be a binary subfile.
1731  if ((_subfiles[index]->_flags & SF_text) != 0) {
1732  if (express_cat.is_debug()) {
1733  express_cat.debug()
1734  << "File is not binary: " << filename << "\n";
1735  }
1736  return false;
1737  }
1738 
1739  } else if (fname.is_text()) {
1740  // If we've specified a text file, it had better be a text subfile.
1741  if ((_subfiles[index]->_flags & SF_text) == 0) {
1742  if (express_cat.is_debug()) {
1743  express_cat.debug()
1744  << "File is not text: " << filename << "\n";
1745  }
1746  return false;
1747  }
1748 
1749  } else {
1750  // If we haven't specified binary or text, infer it from the type of the
1751  // subfile.
1752  if ((_subfiles[index]->_flags & SF_text) != 0) {
1753  fname.set_text();
1754  } else {
1755  fname.set_binary();
1756  }
1757  }
1758 
1759  istream *in1 = open_read_subfile(index);
1760  if (in1 == nullptr) {
1761  return false;
1762  }
1763 
1764  pifstream in2;
1765 
1766  if (!fname.open_read(in2)) {
1767  express_cat.info()
1768  << "Cannot read " << filename << "\n";
1769  return false;
1770  }
1771 
1772  if (fname.is_binary()) {
1773  // Check the file size.
1774  in2.seekg(0, ios::end);
1775  streampos file_size = in2.tellg();
1776 
1777  if (file_size != (streampos)get_subfile_length(index)) {
1778  // The files have different sizes.
1779  close_read_subfile(in1);
1780  return false;
1781  }
1782  }
1783 
1784  // Check the file data, byte-for-byte.
1785  in2.seekg(0);
1786  int byte1 = in1->get();
1787  int byte2 = in2.get();
1788  while (!in1->fail() && !in1->eof() &&
1789  !in2.fail() && !in2.eof()) {
1790  if (byte1 != byte2) {
1791  close_read_subfile(in1);
1792  return false;
1793  }
1794  byte1 = in1->get();
1795  byte2 = in2.get();
1796  }
1797 
1798  bool failed = (in1->fail() && !in1->eof()) || (in2.fail() && !in2.eof());
1799  close_read_subfile(in1);
1800 
1801  nassertr(!failed, false);
1802 
1803  return true;
1804 }
1805 
1806 /**
1807  *
1808  */
1809 void Multifile::
1810 output(ostream &out) const {
1811  out << "Multifile " << _multifile_name << ", " << get_num_subfiles()
1812  << " subfiles.\n";
1813 }
1814 
1815 /**
1816  * Shows a list of all subfiles within the Multifile.
1817  */
1818 void Multifile::
1819 ls(ostream &out) const {
1820  int num_subfiles = get_num_subfiles();
1821  for (int i = 0; i < num_subfiles; i++) {
1822  string subfile_name = get_subfile_name(i);
1823  out << subfile_name << "\n";
1824  }
1825 }
1826 
1827 /**
1828  * Sets the string which is written to the Multifile before the Multifile
1829  * header. This string must begin with a hash mark and end with a newline
1830  * character; and if it includes embedded newline characters, each one must be
1831  * followed by a hash mark. If these conditions are not initially true, the
1832  * string will be modified as necessary to make it so.
1833  *
1834  * This is primarily useful as a simple hack to allow p3d applications to be
1835  * run directly from the command line on Unix-like systems.
1836  *
1837  * The return value is true if successful, or false on failure (for instance,
1838  * because the header prefix violates the above rules).
1839  */
1840 void Multifile::
1841 set_header_prefix(const string &header_prefix) {
1842  string new_header_prefix = header_prefix;
1843 
1844  if (!new_header_prefix.empty()) {
1845  // It must begin with a hash mark.
1846  if (new_header_prefix[0] != '#') {
1847  new_header_prefix = string("#") + new_header_prefix;
1848  }
1849 
1850  // It must end with a newline.
1851  if (new_header_prefix[new_header_prefix.size() - 1] != '\n') {
1852  new_header_prefix += string("\n");
1853  }
1854 
1855  // Embedded newlines must be followed by a hash mark.
1856  size_t newline = new_header_prefix.find('\n');
1857  while (newline < new_header_prefix.size() - 1) {
1858  if (new_header_prefix[newline + 1] != '#') {
1859  new_header_prefix = new_header_prefix.substr(0, newline + 1) + string("#") + new_header_prefix.substr(newline + 1);
1860  }
1861  newline = new_header_prefix.find('#', newline);
1862  }
1863  }
1864 
1865  if (_header_prefix != new_header_prefix) {
1866  _header_prefix = new_header_prefix;
1867  _needs_repack = true;
1868  }
1869 }
1870 
1871 
1872 /**
1873  * Fills a string with the entire contents of the indicated subfile.
1874  */
1875 bool Multifile::
1876 read_subfile(int index, string &result) {
1877  result = string();
1878 
1879  // We use a temporary pvector, because dynamic accumulation of a pvector
1880  // seems to be many times faster than that of a string, at least on the
1881  // Windows implementation of STL.
1882  vector_uchar pv;
1883  if (!read_subfile(index, pv)) {
1884  return false;
1885  }
1886 
1887  if (!pv.empty()) {
1888  result.append((const char *)&pv[0], pv.size());
1889  }
1890 
1891  return true;
1892 }
1893 
1894 /**
1895  * Fills a pvector with the entire contents of the indicated subfile.
1896  */
1897 bool Multifile::
1898 read_subfile(int index, vector_uchar &result) {
1899  nassertr(is_read_valid(), false);
1900  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1901  result.clear();
1902 
1903  // Now look up the particular Subfile we are reading.
1904  nassertr(is_read_valid(), false);
1905  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1906  Subfile *subfile = _subfiles[index];
1907 
1908  if (subfile->_source != nullptr ||
1909  !subfile->_source_filename.empty()) {
1910  // The subfile has not yet been copied into the physical Multifile. Force
1911  // a flush operation to incorporate it.
1912  flush();
1913 
1914  // That shouldn't change the subfile index or delete the subfile pointer.
1915  nassertr(subfile == _subfiles[index], false);
1916  }
1917 
1918  result.reserve(subfile->_uncompressed_length);
1919 
1920  bool success = true;
1921  if (subfile->_flags & (SF_encrypted | SF_compressed)) {
1922  // If the subfile is encrypted or compressed, we can't read it directly.
1923  // Fall back to the generic implementation.
1924  istream *in = open_read_subfile(index);
1925  if (in == nullptr) {
1926  return false;
1927  }
1928 
1929  success = VirtualFile::simple_read_file(in, result);
1930  close_read_subfile(in);
1931 
1932  } else {
1933  // But if the subfile is just a plain file, we can just read the data
1934  // directly from the Multifile, without paying the cost of an ISubStream.
1935  static const size_t buffer_size = 4096;
1936  char buffer[buffer_size];
1937 
1938  streamsize pos = _offset + subfile->_data_start;
1939  size_t max_bytes = subfile->_data_length;
1940  streamsize count = 0;
1941  bool eof = true;
1942 
1943  streamsize num_bytes = (streamsize)min(buffer_size, max_bytes);
1944  _read->seek_read(pos, buffer, num_bytes, count, eof);
1945  while (count != 0) {
1946  thread_consider_yield();
1947  nassertr(count <= (streamsize)max_bytes, false);
1948  result.insert(result.end(), buffer, buffer + (size_t)count);
1949  max_bytes -= (size_t)count;
1950  pos += count;
1951 
1952  num_bytes = (streamsize)min(buffer_size, max_bytes);
1953  _read->seek_read(pos, buffer, num_bytes, count, eof);
1954  }
1955 
1956  success = !eof;
1957  }
1958 
1959  if (!success) {
1960  ostringstream message;
1961  message << "I/O error reading from " << get_multifile_name() << " at "
1962  << get_subfile_name(index);
1963  nassert_raise(message.str());
1964  return false;
1965  }
1966 
1967  return true;
1968 }
1969 
1970 /**
1971  * Assumes the _write pointer is at the indicated fpos, rounds the fpos up to
1972  * the next legitimate address (using normalize_streampos()), and writes
1973  * enough zeroes to the stream to fill the gap. Returns the new fpos.
1974  */
1975 streampos Multifile::
1976 pad_to_streampos(streampos fpos) {
1977  nassertr(_write != nullptr, fpos);
1978  nassertr(_write->tellp() == fpos, fpos);
1979  streampos new_fpos = normalize_streampos(fpos);
1980  while (fpos < new_fpos) {
1981  _write->put(0);
1982  fpos += 1; // VC++ doesn't define streampos++ (!)
1983  }
1984  nassertr(_write->tellp() == fpos, fpos);
1985  return fpos;
1986 }
1987 
1988 /**
1989  * Adds a newly-allocated Subfile pointer to the Multifile.
1990  */
1991 void Multifile::
1992 add_new_subfile(Subfile *subfile, int compression_level) {
1993  if (compression_level != 0) {
1994 #ifndef HAVE_ZLIB
1995  express_cat.warning()
1996  << "zlib not compiled in; cannot generated compressed multifiles.\n";
1997  compression_level = 0;
1998 #else // HAVE_ZLIB
1999  subfile->_flags |= SF_compressed;
2000  subfile->_compression_level = compression_level;
2001 #endif // HAVE_ZLIB
2002  }
2003 
2004 #ifdef HAVE_OPENSSL
2005  if (_encryption_flag) {
2006  subfile->_flags |= SF_encrypted;
2007  }
2008 #endif // HAVE_OPENSSL
2009 
2010  if (_next_index != (streampos)0) {
2011  // If we're adding a Subfile to an already-existing Multifile, we will
2012  // eventually need to repack the file.
2013  _needs_repack = true;
2014  }
2015 
2016  std::pair<Subfiles::iterator, bool> insert_result = _subfiles.insert(subfile);
2017  if (!insert_result.second) {
2018  // Hmm, unable to insert. There must already be a subfile by that name.
2019  // Remove the old one.
2020  Subfile *old_subfile = (*insert_result.first);
2021  old_subfile->_flags |= SF_deleted;
2022 
2023  // Maybe it was just added to the _new_subfiles list. In this case,
2024  // remove it from that list.
2025  PendingSubfiles::iterator ni = find(_new_subfiles.begin(), _new_subfiles.end(), old_subfile);
2026  if (ni != _new_subfiles.end()) {
2027  _new_subfiles.erase(ni);
2028 
2029  } else {
2030  // Otherwise, add it to the _removed_subfiles list, so we can remove the
2031  // old one.
2032  _removed_subfiles.push_back(old_subfile);
2033  }
2034 
2035  (*insert_result.first) = subfile;
2036  }
2037 
2038  _new_subfiles.push_back(subfile);
2039 }
2040 
2041 /**
2042  * This variant of open_read_subfile() is used internally only, and accepts a
2043  * pointer to the internal Subfile object, which is assumed to be valid and
2044  * written to the multifile.
2045  */
2046 istream *Multifile::
2047 open_read_subfile(Subfile *subfile) {
2048  nassertr(subfile->_source == nullptr &&
2049  subfile->_source_filename.empty(), nullptr);
2050 
2051  // Return an ISubStream object that references into the open Multifile
2052  // istream.
2053  nassertr(subfile->_data_start != (streampos)0, nullptr);
2054  istream *stream =
2055  new ISubStream(_read, _offset + subfile->_data_start,
2056  _offset + subfile->_data_start + (streampos)subfile->_data_length);
2057 
2058  if ((subfile->_flags & SF_encrypted) != 0) {
2059 #ifndef HAVE_OPENSSL
2060  express_cat.error()
2061  << "OpenSSL not compiled in; cannot read encrypted multifiles.\n";
2062  delete stream;
2063  return nullptr;
2064 #else // HAVE_OPENSSL
2065  // The subfile is encrypted. So actually, return an IDecryptStream that
2066  // wraps around the ISubStream.
2067  IDecryptStream *wrapper =
2068  new IDecryptStream(stream, true, _encryption_password);
2069  stream = wrapper;
2070 
2071  // Validate the password by confirming that the encryption header matches.
2072  char this_header[_encrypt_header_size];
2073  stream->read(this_header, _encrypt_header_size);
2074  if (stream->fail() || stream->gcount() != (unsigned)_encrypt_header_size ||
2075  memcmp(this_header, _encrypt_header, _encrypt_header_size) != 0) {
2076  express_cat.error()
2077  << "Unable to decrypt subfile " << subfile->_name << ".\n";
2078  delete stream;
2079  return nullptr;
2080  }
2081 #endif // HAVE_OPENSSL
2082  }
2083 
2084  if ((subfile->_flags & SF_compressed) != 0) {
2085 #ifndef HAVE_ZLIB
2086  express_cat.error()
2087  << "zlib not compiled in; cannot read compressed multifiles.\n";
2088  delete stream;
2089  return nullptr;
2090 #else // HAVE_ZLIB
2091  // Oops, the subfile is compressed. So actually, return an
2092  // IDecompressStream that wraps around the ISubStream.
2093  IDecompressStream *wrapper = new IDecompressStream(stream, true);
2094  stream = wrapper;
2095 #endif // HAVE_ZLIB
2096  }
2097 
2098  if (stream->fail()) {
2099  // Hmm, some inexplicable problem.
2100  delete stream;
2101  return nullptr;
2102  }
2103 
2104  return stream;
2105 }
2106 
2107 /**
2108  * Returns the standard form of the subfile name.
2109  */
2110 string Multifile::
2111 standardize_subfile_name(const string &subfile_name) const {
2112  Filename name = subfile_name;
2113  name.standardize();
2114  if (name.empty() || name == "/") {
2115  // Invalid empty name.
2116  return string();
2117  }
2118 
2119  if (name[0] == '/') {
2120  return name.get_fullpath().substr(1);
2121  } else if (name.length() > 2 && name[0] == '.' && name[1] == '/') {
2122  return name.get_fullpath().substr(2);
2123  } else {
2124  return name.get_fullpath();
2125  }
2126 }
2127 
2128 /**
2129  * Removes the set of subfiles from the tables and frees their associated
2130  * memory.
2131  */
2132 void Multifile::
2133 clear_subfiles() {
2134  PendingSubfiles::iterator pi;
2135  for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
2136  Subfile *subfile = (*pi);
2137  subfile->rewrite_index_flags(*_write);
2138  delete subfile;
2139  }
2140  _removed_subfiles.clear();
2141 
2142  // We don't have to delete the ones in _new_subfiles, because these also
2143  // appear in _subfiles.
2144  _new_subfiles.clear();
2145 
2146 #ifdef HAVE_OPENSSL
2147  for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) {
2148  Subfile *subfile = (*pi);
2149  delete subfile;
2150  }
2151  _cert_special.clear();
2152 
2153  _signatures.clear();
2154 #endif // HAVE_OPENSSL
2155 
2156  Subfiles::iterator fi;
2157  for (fi = _subfiles.begin(); fi != _subfiles.end(); ++fi) {
2158  Subfile *subfile = (*fi);
2159  delete subfile;
2160  }
2161  _subfiles.clear();
2162 }
2163 
2164 /**
2165  * Reads the Multifile header and index. Returns true if successful, false if
2166  * the Multifile is not valid.
2167  */
2168 bool Multifile::
2169 read_index() {
2170  nassertr(_read != nullptr, false);
2171 
2172  // We acquire the IStreamWrapper lock for the duration of this method.
2173  _read->acquire();
2174  istream *read = _read->get_istream();
2175 
2176  char this_header[_header_size];
2177  read->seekg(_offset);
2178 
2179  // Here's a special case: if the multifile begins with a hash character,
2180  // then we continue reading and discarding lines of ASCII text, until we
2181  // come across a nonempty line that does not begin with a hash character.
2182  // This allows a P3D application (which is a multifile) to be run directly
2183  // on the command line on Unix-based systems.
2184  _header_prefix = string();
2185  int ch = read->get();
2186 
2187  if (ch == '#') {
2188  while (ch != EOF && ch == '#') {
2189  // Skip to the end of the line.
2190  while (ch != EOF && ch != '\n') {
2191  _header_prefix += ch;
2192  ch = read->get();
2193  }
2194  // Skip to the first non-whitespace character of the line.
2195  while (ch != EOF && (isspace(ch) || ch == '\r')) {
2196  _header_prefix += ch;
2197  ch = read->get();
2198  }
2199  }
2200  }
2201 
2202  // Now read the actual Multifile header.
2203  this_header[0] = ch;
2204  read->read(this_header + 1, _header_size - 1);
2205  if (read->fail() || read->gcount() != (unsigned)(_header_size - 1)) {
2206  express_cat.info()
2207  << "Unable to read Multifile header " << _multifile_name << ".\n";
2208  _read->release();
2209  close();
2210  return false;
2211  }
2212 
2213  if (memcmp(this_header, _header, _header_size) != 0) {
2214  express_cat.info()
2215  << _multifile_name << " is not a Multifile.\n";
2216  _read->release();
2217  close();
2218  return false;
2219  }
2220 
2221  // Now get the version numbers out.
2222  StreamReader reader(read, false);
2223  _file_major_ver = reader.get_int16();
2224  _file_minor_ver = reader.get_int16();
2225  _scale_factor = reader.get_uint32();
2226  _new_scale_factor = _scale_factor;
2227 
2228  if (read->eof() || read->fail()) {
2229  express_cat.info()
2230  << _multifile_name << " header is truncated.\n";
2231  _read->release();
2232  close();
2233  return false;
2234  }
2235 
2236  if (_file_major_ver != _current_major_ver ||
2237  (_file_major_ver == _current_major_ver &&
2238  _file_minor_ver > _current_minor_ver)) {
2239  express_cat.info()
2240  << _multifile_name << " has version " << _file_major_ver << "."
2241  << _file_minor_ver << ", expecting version "
2242  << _current_major_ver << "." << _current_minor_ver << ".\n";
2243  _read->release();
2244  close();
2245  return false;
2246  }
2247 
2248  _record_timestamp = true;
2249  if (_file_minor_ver >= 1) {
2250  time_t read_timestamp = reader.get_uint32();
2251  if (read_timestamp == 0) {
2252  // If we read a 0 timestamp from the file, that implies that we don't
2253  // want to record a timestamp in this particular file.
2254  _record_timestamp = false;
2255  } else {
2256  _timestamp = read_timestamp;
2257  }
2258  _timestamp_dirty = false;
2259  }
2260 
2261  // Now read the index out.
2262  streampos curr_pos = read->tellg() - _offset;
2263  _next_index = normalize_streampos(curr_pos);
2264  if (_next_index > curr_pos) {
2265  read->ignore(_next_index - curr_pos);
2266  }
2267  _last_index = 0;
2268  _last_data_byte = 0;
2269  streampos index_forward;
2270  streamoff bytes_skipped = 0;
2271  bool read_cert_special = false;
2272 
2273  Subfile *subfile = new Subfile;
2274  index_forward = subfile->read_index(*read, _next_index, this);
2275  while (index_forward != (streampos)0) {
2276  _last_index = _next_index;
2277  if (subfile->is_deleted()) {
2278  // Ignore deleted Subfiles in the index.
2279  _needs_repack = true;
2280  delete subfile;
2281  } else if (subfile->is_cert_special()) {
2282  // Certificate chains and signature files get stored in a special list.
2283  _cert_special.push_back(subfile);
2284  read_cert_special = true;
2285  } else {
2286  _subfiles.push_back(subfile);
2287  }
2288  if (!subfile->is_cert_special()) {
2289  if (bytes_skipped != 0) {
2290  // If the index entries don't follow exactly sequentially (except for
2291  // the cert special files), the file ought to be repacked.
2292  _needs_repack = true;
2293  }
2294  if (read_cert_special) {
2295  // If we read a normal subfile following a cert_special entry, the
2296  // file ought to be repacked (certificates have to go at the end).
2297  _needs_repack = true;
2298  }
2299  _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos());
2300  }
2301  streampos curr_pos = read->tellg() - _offset;
2302  bytes_skipped = index_forward - normalize_streampos(curr_pos);
2303  _next_index = index_forward;
2304  if (_next_index > curr_pos) {
2305  read->ignore(_next_index - curr_pos);
2306  }
2307  subfile = new Subfile;
2308  index_forward = subfile->read_index(*read, _next_index, this);
2309  }
2310  if (subfile->is_index_invalid()) {
2311  express_cat.info()
2312  << "Error reading index for " << _multifile_name << ".\n";
2313  _read->release();
2314  close();
2315  delete subfile;
2316  return false;
2317  }
2318 
2319  // Check if the list is already sorted. If it is not, we need a repack.
2320  for (size_t si = 1; si < _subfiles.size() && !_needs_repack; ++si) {
2321  if (*_subfiles[si] < *_subfiles[si - 1]) {
2322  _needs_repack = true;
2323  }
2324  }
2325 
2326  if (_needs_repack) {
2327  // At least sort them now.
2328  size_t before_size = _subfiles.size();
2329  _subfiles.sort();
2330  size_t after_size = _subfiles.size();
2331 
2332  // If these don't match, the same filename appeared twice in the index,
2333  // which shouldn't be possible.
2334  nassertr(before_size == after_size, true);
2335  }
2336 
2337  delete subfile;
2338  _read->release();
2339  return true;
2340 }
2341 
2342 /**
2343  * Writes just the header part of the Multifile, not the index.
2344  */
2345 bool Multifile::
2346 write_header() {
2347  _file_major_ver = _current_major_ver;
2348  _file_minor_ver = _current_minor_ver;
2349 
2350  nassertr(_write != nullptr, false);
2351  nassertr(_write->tellp() == (streampos)0, false);
2352  _write->write(_header_prefix.data(), _header_prefix.size());
2353  _write->write(_header, _header_size);
2354  StreamWriter writer(_write, false);
2355  writer.add_int16(_current_major_ver);
2356  writer.add_int16(_current_minor_ver);
2357  writer.add_uint32(_scale_factor);
2358 
2359  if (_record_timestamp) {
2360  writer.add_uint32(_timestamp);
2361  } else {
2362  writer.add_uint32(0);
2363  _timestamp_dirty = false;
2364  }
2365 
2366  _next_index = _write->tellp();
2367  _next_index = pad_to_streampos(_next_index);
2368  _last_index = 0;
2369 
2370  if (_write->fail()) {
2371  express_cat.info()
2372  << "Unable to write header for " << _multifile_name << ".\n";
2373  close();
2374  return false;
2375  }
2376 
2377  return true;
2378 }
2379 
2380 /**
2381  * Walks through the list of _cert_special entries in the Multifile, moving
2382  * any valid signatures found to _signatures. After this call, _cert_special
2383  * will be empty.
2384  *
2385  * This does not check the validity of the certificates themselves. It only
2386  * checks that they correctly sign the Multifile contents.
2387  */
2388 void Multifile::
2389 check_signatures() {
2390 #ifdef HAVE_OPENSSL
2391  PendingSubfiles::iterator pi;
2392 
2393  for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) {
2394  Subfile *subfile = (*pi);
2395  nassertv((subfile->_flags & SF_signature) != 0);
2396 
2397  // Extract the signature data and certificate separately.
2398  istream *stream = open_read_subfile(subfile);
2399  nassertv(stream != nullptr);
2400  StreamReader reader(*stream);
2401  size_t sig_size = reader.get_uint32();
2402  vector_uchar sig_data = reader.extract_bytes(sig_size);
2403 
2404  size_t num_certs = reader.get_uint32();
2405 
2406  // Read the remaining buffer of certificate data.
2407  vector_uchar buffer;
2408  bool success = VirtualFile::simple_read_file(stream, buffer);
2409  nassertv(success);
2410  close_read_subfile(stream);
2411 
2412  // Now convert each of the certificates to an X509 object, and store it in
2413  // our CertChain.
2414  CertChain chain;
2415  EVP_PKEY *pkey = nullptr;
2416  if (!buffer.empty()) {
2417 #if OPENSSL_VERSION_NUMBER >= 0x00908000L
2418  // Beginning in 0.9.8, d2i_X509() accepted a const unsigned char **.
2419  const unsigned char *bp, *bp_end;
2420 #else
2421  // Prior to 0.9.8, d2i_X509() accepted an unsigned char **.
2422  unsigned char *bp, *bp_end;
2423 #endif
2424  bp = (unsigned char *)&buffer[0];
2425  bp_end = bp + buffer.size();
2426  X509 *x509 = d2i_X509(nullptr, &bp, bp_end - bp);
2427  while (num_certs > 0 && x509 != nullptr) {
2428  chain.push_back(CertRecord(x509));
2429  --num_certs;
2430  x509 = d2i_X509(nullptr, &bp, bp_end - bp);
2431  }
2432  if (num_certs != 0 || x509 != nullptr) {
2433  express_cat.warning()
2434  << "Extra data in signature record.\n";
2435  }
2436  }
2437 
2438  if (!chain.empty()) {
2439  pkey = X509_get_pubkey(chain[0]._cert);
2440  }
2441 
2442  if (pkey != nullptr) {
2443  EVP_MD_CTX *md_ctx = EVP_MD_CTX_create();
2444  EVP_VerifyInit(md_ctx, EVP_sha1());
2445 
2446  nassertv(_read != nullptr);
2447  _read->acquire();
2448  istream *read = _read->get_istream();
2449 
2450  // Read and hash the multifile contents, but only up till
2451  // _last_data_byte.
2452  read->seekg(_offset);
2453  streampos bytes_remaining = _last_data_byte;
2454  static const size_t buffer_size = 4096;
2455  char buffer[buffer_size];
2456  read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2457  size_t count = read->gcount();
2458  while (count != 0) {
2459  nassertv(count <= buffer_size);
2460  EVP_VerifyUpdate(md_ctx, buffer, count);
2461  bytes_remaining -= count;
2462  read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2463  count = read->gcount();
2464  }
2465  nassertv(bytes_remaining == (streampos)0);
2466  _read->release();
2467 
2468  // Now check that the signature matches the hash.
2469  int verify_result =
2470  EVP_VerifyFinal(md_ctx, sig_data.data(), sig_data.size(), pkey);
2471  if (verify_result == 1) {
2472  // The signature matches; save the certificate and its chain.
2473  _signatures.push_back(chain);
2474  } else {
2475  // Bad match.
2476  _needs_repack = true;
2477  }
2478  }
2479  }
2480 #endif // HAVE_OPENSSL
2481 
2482  _cert_special.clear();
2483 }
2484 
2485 /**
2486  * Reads the index record for the Subfile from the indicated istream. Assumes
2487  * the istream has already been positioned to the indicated stream position,
2488  * fpos, the start of the index record. Returns the position within the file
2489  * of the next index record.
2490  */
2491 streampos Multifile::Subfile::
2492 read_index(istream &read, streampos fpos, Multifile *multifile) {
2493  nassertr(read.tellg() - multifile->_offset == fpos, fpos);
2494 
2495  // First, get the next stream position. We do this separately, because if
2496  // it is zero, we don't get anything else.
2497  StreamReader reader(read);
2498 
2499  streampos next_index = multifile->word_to_streampos(reader.get_uint32());
2500  if (read.eof() || read.fail()) {
2501  _flags |= SF_index_invalid;
2502  return 0;
2503  }
2504 
2505  if (next_index == (streampos)0) {
2506  return 0;
2507  }
2508 
2509  // Now get the rest of the index.
2510 
2511  _index_start = fpos;
2512  _index_length = 0;
2513 
2514  _data_start = multifile->word_to_streampos(reader.get_uint32());
2515  _data_length = reader.get_uint32();
2516  _flags = reader.get_uint16();
2517  if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
2518  _uncompressed_length = reader.get_uint32();
2519  } else {
2520  _uncompressed_length = _data_length;
2521  }
2522  if (multifile->_file_minor_ver < 1) {
2523  _timestamp = multifile->get_timestamp();
2524  } else {
2525  _timestamp = reader.get_uint32();
2526  if (_timestamp == 0) {
2527  _timestamp = multifile->get_timestamp();
2528  }
2529  }
2530 
2531  size_t name_length = reader.get_uint16();
2532  if (read.eof() || read.fail()) {
2533  _flags |= SF_index_invalid;
2534  return 0;
2535  }
2536 
2537  // And finally, get the rest of the name.
2538  char *name_buffer = (char *)PANDA_MALLOC_ARRAY(name_length);
2539  nassertr(name_buffer != nullptr, next_index);
2540  for (size_t ni = 0; ni < name_length; ni++) {
2541  name_buffer[ni] = read.get() ^ 0xff;
2542  }
2543  _name = string(name_buffer, name_length);
2544  PANDA_FREE_ARRAY(name_buffer);
2545 
2546  if (read.eof() || read.fail()) {
2547  _flags |= SF_index_invalid;
2548  return 0;
2549  }
2550 
2551  _index_length = read.tellg() - fpos - multifile->_offset;
2552  return next_index;
2553 }
2554 
2555 /**
2556  * Writes the index record for the Subfile to the indicated ostream. Assumes
2557  * the istream has already been positioned to the indicated stream position,
2558  * fpos, the start of the index record, and that this is the effective end of
2559  * the file. Returns the position within the file of the next index record.
2560  *
2561  * The _index_start member is updated by this operation.
2562  */
2563 streampos Multifile::Subfile::
2564 write_index(ostream &write, streampos fpos, Multifile *multifile) {
2565  nassertr(write.tellp() - multifile->_offset == fpos, fpos);
2566 
2567  _index_start = fpos;
2568  _index_length = 0;
2569 
2570  // This will be the contents of this particular index record. We build it
2571  // up first since it will be variable length.
2572  Datagram dg;
2573  dg.add_uint32(multifile->streampos_to_word(_data_start));
2574  dg.add_uint32(_data_length);
2575  dg.add_uint16(_flags);
2576  if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
2577  dg.add_uint32(_uncompressed_length);
2578  }
2579  dg.add_uint32(_timestamp);
2580  dg.add_uint16(_name.length());
2581 
2582  // For no real good reason, we'll invert all the bits in the name. The only
2583  // reason we do this is to make it inconvenient for a casual browser of the
2584  // Multifile to discover the names of the files stored within it.
2585  // Naturally, this isn't real obfuscation or security.
2586  string::iterator ni;
2587  for (ni = _name.begin(); ni != _name.end(); ++ni) {
2588  dg.add_int8((*ni) ^ 0xff);
2589  }
2590 
2591  size_t this_index_size = 4 + dg.get_length();
2592 
2593  // Plus, we will write out the next index address first.
2594  streampos next_index = fpos + (streampos)this_index_size;
2595 
2596  Datagram idg;
2597  idg.add_uint32(multifile->streampos_to_word(next_index));
2598 
2599  write.write((const char *)idg.get_data(), idg.get_length());
2600  write.write((const char *)dg.get_data(), dg.get_length());
2601 
2602  _index_length = write.tellp() - fpos - multifile->_offset;
2603  return next_index;
2604 }
2605 
2606 /**
2607  * Writes the data record for the Subfile to the indicated ostream: the actual
2608  * contents of the Subfile. Assumes the istream has already been positioned
2609  * to the indicated stream position, fpos, the start of the data record, and
2610  * that this is the effective end of the file. Returns the position within
2611  * the file of the next data record.
2612  *
2613  * The _data_start, _data_length, and _uncompressed_length members are updated
2614  * by this operation.
2615  *
2616  * If the "read" pointer is non-NULL, it is the readable istream of a
2617  * Multifile in which the Subfile might already be packed. This is used for
2618  * reading the contents of the Subfile during a repack() operation.
2619  */
2620 streampos Multifile::Subfile::
2621 write_data(ostream &write, istream *read, streampos fpos,
2622  Multifile *multifile) {
2623  nassertr(write.tellp() - multifile->_offset == fpos, fpos);
2624 
2625  istream *source = _source;
2626  pifstream source_file;
2627  if (source == nullptr && !_source_filename.empty()) {
2628  // If we have a filename, open it up and read that.
2629  if (!_source_filename.open_read(source_file)) {
2630  // Unable to open the source file.
2631  express_cat.info()
2632  << "Unable to read " << _source_filename << ".\n";
2633  _flags |= SF_data_invalid;
2634  _data_length = 0;
2635  _uncompressed_length = 0;
2636  } else {
2637  source = &source_file;
2638  }
2639  }
2640 
2641  if (source == nullptr) {
2642  // We don't have any source data. Perhaps we're reading from an already-
2643  // packed Subfile (e.g. during repack()).
2644  if (read == nullptr) {
2645  // No, we're just screwed.
2646  express_cat.info()
2647  << "No source for subfile " << _name << ".\n";
2648  _flags |= SF_data_invalid;
2649  } else {
2650  // Read the data from the original Multifile.
2651  read->seekg(_data_start + multifile->_offset);
2652  for (size_t p = 0; p < _data_length; p++) {
2653  int byte = read->get();
2654  if (read->eof() || read->fail()) {
2655  // Unexpected EOF or other failure on the source file.
2656  express_cat.info()
2657  << "Unexpected EOF for subfile " << _name << ".\n";
2658  _flags |= SF_data_invalid;
2659  break;
2660  }
2661  write.put(byte);
2662  }
2663  }
2664  } else {
2665  // We do have source data. Copy it in, and also measure its length.
2666  ostream *putter = &write;
2667  bool delete_putter = false;
2668 
2669 #ifndef HAVE_OPENSSL
2670  // Without OpenSSL, we can't support encryption. The flag had better not
2671  // be set.
2672  nassertr((_flags & SF_encrypted) == 0, fpos);
2673 
2674 #else // HAVE_OPENSSL
2675  if ((_flags & SF_encrypted) != 0) {
2676  // Write it encrypted.
2677  OEncryptStream *encrypt = new OEncryptStream;
2678  encrypt->set_iteration_count(multifile->_encryption_iteration_count);
2679  encrypt->open(putter, delete_putter, multifile->_encryption_password);
2680 
2681  putter = encrypt;
2682  delete_putter = true;
2683 
2684  // Also write the encrypt_header to the beginning of the encrypted
2685  // stream, so we can validate the password on decryption.
2686  putter->write(_encrypt_header, _encrypt_header_size);
2687  }
2688 #endif // HAVE_OPENSSL
2689 
2690 #ifndef HAVE_ZLIB
2691  // Without ZLIB, we can't support compression. The flag had better not be
2692  // set.
2693  nassertr((_flags & SF_compressed) == 0, fpos);
2694 #else // HAVE_ZLIB
2695  if ((_flags & SF_compressed) != 0) {
2696  // Write it compressed.
2697  putter = new OCompressStream(putter, delete_putter, _compression_level);
2698  delete_putter = true;
2699  }
2700 #endif // HAVE_ZLIB
2701 
2702  streampos write_start = fpos;
2703  _uncompressed_length = 0;
2704 
2705 #ifndef HAVE_OPENSSL
2706  // We also need OpenSSL for signatures.
2707  nassertr((_flags & SF_signature) == 0, fpos);
2708 
2709 #else // HAVE_OPENSSL
2710  if ((_flags & SF_signature) != 0) {
2711  // If it's a special signature record, precede the record data (the
2712  // certificate itself) with the signature data generated against the
2713  // multifile contents.
2714 
2715  // In order to generate a signature, we need to have a valid read
2716  // pointer.
2717  nassertr(read != nullptr, fpos);
2718 
2719  // And we also need to have a private key.
2720  nassertr(_pkey != nullptr, fpos);
2721 
2722  EVP_MD_CTX *md_ctx = EVP_MD_CTX_create();
2723  EVP_SignInit(md_ctx, EVP_sha1());
2724 
2725  // Read and hash the multifile contents, but only up till
2726  // _last_data_byte.
2727  nassertr(multifile->_last_data_byte < fpos, fpos);
2728  read->seekg(multifile->_offset);
2729  streampos bytes_remaining = multifile->_last_data_byte;
2730  static const size_t buffer_size = 4096;
2731  char buffer[buffer_size];
2732  read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2733  size_t count = read->gcount();
2734  while (count != 0) {
2735  nassertr(count <= buffer_size, fpos);
2736  EVP_SignUpdate(md_ctx, buffer, count);
2737  bytes_remaining -= count;
2738  read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2739  count = read->gcount();
2740  }
2741  nassertr(bytes_remaining == (streampos)0, fpos);
2742 
2743  // Now generate and write out the signature.
2744  unsigned int max_size = EVP_PKEY_size(_pkey);
2745  unsigned char *sig_data = new unsigned char[max_size];
2746  unsigned int sig_size;
2747  if (!EVP_SignFinal(md_ctx, sig_data, &sig_size, _pkey)) {
2748  OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
2749  sslw->notify_ssl_errors();
2750  }
2751  nassertr(sig_size <= max_size, fpos);
2752 
2753  StreamWriter writer(*putter);
2754  writer.add_uint32(sig_size);
2755  putter->write((char *)sig_data, sig_size);
2756  _uncompressed_length += 4 + sig_size;
2757 
2758  delete[] sig_data;
2759 
2760  EVP_MD_CTX_destroy(md_ctx);
2761  }
2762 #endif // HAVE_OPENSSL
2763 
2764  // Finally, we can write out the data itself.
2765  static const size_t buffer_size = 4096;
2766  char buffer[buffer_size];
2767 
2768  source->read(buffer, buffer_size);
2769  size_t count = source->gcount();
2770  while (count != 0) {
2771  _uncompressed_length += count;
2772  putter->write(buffer, count);
2773  source->read(buffer, buffer_size);
2774  count = source->gcount();
2775  }
2776 
2777  if (delete_putter) {
2778  delete putter;
2779  }
2780 
2781  streampos write_end = write.tellp() - multifile->_offset;
2782  _data_length = (size_t)(write_end - write_start);
2783  }
2784 
2785  // We can't set _data_start until down here, after we have read the Subfile.
2786  // (In case we are running during repack()).
2787  _data_start = fpos;
2788 
2789  // Get the modification timestamp for this subfile. This is read from the
2790  // source file, if we have a filename; otherwise, it's the current time.
2791  if (!_source_filename.empty()) {
2792  _timestamp = _source_filename.get_timestamp();
2793  }
2794  if (_timestamp == 0) {
2795  _timestamp = time(nullptr);
2796  }
2797 
2798  _source = nullptr;
2799  _source_filename = Filename();
2800  source_file.close();
2801 
2802  return fpos + (streampos)_data_length;
2803 }
2804 
2805 /**
2806  * Seeks within the indicate pfstream back to the index record and rewrites
2807  * just the _data_start and _data_length part of the index record.
2808  */
2809 void Multifile::Subfile::
2810 rewrite_index_data_start(ostream &write, Multifile *multifile) {
2811  nassertv(_index_start != (streampos)0);
2812 
2813  static const size_t data_start_offset = 4;
2814  size_t data_start_pos = _index_start + (streampos)data_start_offset;
2815  write.seekp(data_start_pos + multifile->_offset);
2816  nassertv(!write.fail());
2817 
2818  StreamWriter writer(write);
2819  writer.add_uint32(multifile->streampos_to_word(_data_start));
2820  writer.add_uint32(_data_length);
2821  writer.add_uint16(_flags);
2822  if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
2823  writer.add_uint32(_uncompressed_length);
2824  }
2825  if (multifile->_record_timestamp) {
2826  writer.add_uint32(_timestamp);
2827  } else {
2828  writer.add_uint32(0);
2829  }
2830 }
2831 
2832 /**
2833  * Seeks within the indicated ostream back to the index record and rewrites
2834  * just the _flags part of the index record.
2835  */
2836 void Multifile::Subfile::
2837 rewrite_index_flags(ostream &write) {
2838  // If the subfile has never even been recorded to disk, we don't need to do
2839  // anything at all in this function.
2840  if (_index_start != (streampos)0) {
2841  static const size_t flags_offset = 4 + 4 + 4;
2842  size_t flags_pos = _index_start + (streampos)flags_offset;
2843  write.seekp(flags_pos);
2844  nassertv(!write.fail());
2845 
2846  StreamWriter writer(write);
2847  writer.add_uint16(_flags);
2848  }
2849 }
A StreamWriter object is used to write sequential binary data directly to an ostream.
Definition: streamWriter.h:29
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
bool open_write(std::ofstream &stream, bool truncate=true) const
Opens the indicated ifstream for writing the file, if possible.
Definition: filename.cxx:1899
std::string get_dirname() const
Returns the directory part of the filename.
Definition: filename.I:358
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
size_t get_subfile_internal_length(int index) const
Returns the number of bytes the indicated subfile consumes within the archive.
Definition: multifile.cxx:1580
time_t get_timestamp() const
Returns a time_t value that represents the time the file was last modified, to within whatever precis...
Definition: filename.cxx:1497
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
std::istream * open_read_subfile(int index)
Returns an istream that may be used to read the indicated subfile.
Definition: multifile.cxx:1603
bool open_write(const Filename &multifile_name)
Opens the named Multifile on disk for writing.
Definition: multifile.cxx:215
size_type_0 size() const
Returns the number of elements in the ordered vector.
void clear()
Removes all elements from the ordered vector.
time_t get_timestamp() const
Returns the modification timestamp of the overall Multifile.
Definition: multifile.I:66
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS&#39;s file system.
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:414
iterator_0 begin()
Returns the iterator that marks the first element in the ordered vector.
void set_text()
Indicates that the filename represents a text file.
Definition: filename.I:424
bool open_read(std::ifstream &stream) const
Opens the indicated ifstream for reading the file, if possible.
Definition: filename.cxx:1863
bool is_subfile_compressed(int index) const
Returns true if the indicated subfile has been compressed when stored within the archive, false otherwise.
Definition: multifile.cxx:1517
std::streampos get_index_end() const
Returns the first byte that is guaranteed to follow any index byte already written to disk in the Mul...
Definition: multifile.cxx:1555
This class provides a locking wrapper around a combination ostream/istream pointer.
iterator_0 end()
Returns the iterator that marks the end of the ordered vector.
The abstract base class for a file or directory within the VirtualFileSystem.
Definition: virtualFile.h:35
bool is_write_valid() const
Returns true if the Multifile has been opened for write mode and there have been no errors...
Definition: multifile.I:46
bool is_subfile_encrypted(int index) const
Returns true if the indicated subfile has been encrypted when stored within the archive, false otherwise.
Definition: multifile.cxx:1527
void standardize()
Converts the filename to standard form by replacing consecutive slashes with a single slash...
Definition: filename.cxx:900
bool is_binary() const
Returns true if the Filename has been indicated to represent a binary file via a previous call to set...
Definition: filename.I:435
static void close_read_subfile(std::istream *stream)
Closes a file opened by a previous call to open_read_subfile().
Definition: multifile.cxx:1628
void add_uint32(uint32_t value)
Adds an unsigned 32-bit integer to the datagram.
Definition: datagram.I:94
bool repack()
Forces a complete rewrite of the Multifile and all of its contents, so that its index will appear at ...
Definition: multifile.cxx:1281
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
void release()
Releases the internal lock.
Definition: streamWrapper.I:53
void add_uint16(uint16_t value)
Adds an unsigned 16-bit integer to the datagram.
Definition: datagram.I:85
void add_int8(int8_t value)
Adds a signed 8-bit integer to the datagram.
Definition: datagram.I:42
size_t get_subfile_length(int index) const
Returns the uncompressed data length of the nth subfile.
Definition: multifile.cxx:1491
An istream object that presents a subwindow into another istream.
Definition: subStream.h:30
bool scan_directory(vector_string &contents, const std::string &subfile_name) const
Considers subfile_name to be the name of a subdirectory within the Multifile, but not a file itself; ...
Definition: multifile.cxx:1417
int find_subfile(const std::string &subfile_name) const
Returns the index of the subfile with the indicated name, or -1 if the named subfile is not within th...
Definition: multifile.cxx:1367
bool has_directory(const std::string &subfile_name) const
Returns true if the indicated subfile name is the directory prefix to one or more files within the Mu...
Definition: multifile.cxx:1385
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:39
get_subfile_name
Returns the name of the nth subfile.
Definition: multifile.h:117
std::string update_subfile(const std::string &subfile_name, const Filename &filename, int compression_level)
Adds a file on disk to the subfile.
Definition: multifile.cxx:491
This class provides a locking wrapper around an arbitrary istream pointer.
Definition: streamWrapper.h:59
time_t get_subfile_timestamp(int index) const
Returns the modification time of the nth subfile.
Definition: multifile.cxx:1503
std::string get_fullpath() const
Returns the entire filename: directory, basename, extension.
Definition: filename.I:338
void set_scale_factor(size_t scale_factor)
Changes the internal scale factor for this Multifile.
Definition: multifile.cxx:386
bool open_read_write(const Filename &multifile_name)
Opens the named Multifile on disk for reading and writing.
Definition: multifile.cxx:258
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
void sort()
Maps to sort_unique().
bool is_read_valid() const
Returns true if the Multifile has been opened for read mode and there have been no errors...
Definition: multifile.I:37
bool extract_subfile_to(int index, std::ostream &out)
Extracts the nth subfile to the indicated ostream.
Definition: multifile.cxx:1680
bool flush()
Writes all contents of the Multifile to disk.
Definition: multifile.cxx:1127
bool get_record_timestamp() const
Returns the flag indicating whether timestamps should be recorded within the Multifile or not...
Definition: multifile.I:93
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
bool unlink() const
Permanently deletes the file associated with the filename, if possible.
Definition: filename.cxx:2319
void ls(std::ostream &out=std::cout) const
Shows a list of all subfiles within the Multifile.
Definition: multifile.cxx:1819
bool compare_subfile(int index, const Filename &filename)
Performs a byte-for-byte comparison of the indicated file on disk with the nth subfile.
Definition: multifile.cxx:1718
bool extract_subfile(int index, const Filename &filename)
Extracts the nth subfile into a file with the given name.
Definition: multifile.cxx:1647
PointerTo< VirtualFile > get_file(const Filename &filename, bool status_only=false) const
Looks up the file by the indicated name in the file system.
void remove_subfile(int index)
Removes the nth subfile from the Multifile.
Definition: multifile.cxx:1459
void push_back(const value_type_0 &key)
Adds the new element to the end of the vector without regard for proper sorting.
bool is_text() const
Returns true if the Filename has been indicated to represent a text file via a previous call to set_t...
Definition: filename.I:456
get_num_subfiles
Returns the number of subfiles within the Multifile.
Definition: multifile.h:117
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
void acquire()
Acquires the internal lock.
Definition: streamWrapper.I:38
A file that contains a set of files.
Definition: multifile.h:37
void set_header_prefix(const std::string &header_prefix)
Sets the string which is written to the Multifile before the Multifile header.
Definition: multifile.cxx:1841
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
void close()
Closes the Multifile if it is open.
Definition: multifile.cxx:324
This is a convenience class to specialize ConfigVariable as an integer type.
static Filename temporary(const std::string &dirname, const std::string &prefix, const std::string &suffix=std::string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix...
Definition: filename.cxx:424
bool open_read_write(std::fstream &stream, bool truncate=false) const
Opens the indicated fstream for read/write access to the file, if possible.
Definition: filename.cxx:1977
const Filename & get_multifile_name() const
Returns the filename of the Multifile, if it is available.
Definition: multifile.I:18
static bool simple_read_file(std::istream *stream, vector_uchar &result)
Fills up the indicated pvector with the contents of the just-opened file.
get_istream
Returns the istream this object is wrapping.
Definition: streamWrapper.h:67
void add_uint32(uint32_t value)
Adds an unsigned 32-bit integer to the stream.
Definition: streamWriter.I:147
bool is_subfile_text(int index) const
Returns true if the indicated subfile represents text data, or false if it represents binary data...
Definition: multifile.cxx:1540
A class to read sequential binary data directly from an istream.
Definition: streamReader.h:28
void seek_read(std::streamsize pos, char *buffer, std::streamsize num_bytes, std::streamsize &read_bytes, bool &eof)
Atomically seeks to a particular offset from the beginning of the file, and reads a number of bytes f...
An ordered list of data elements, formatted in memory for transmission over a socket or writing to a ...
Definition: datagram.h:38
std::string add_subfile(const std::string &subfile_name, const Filename &filename, int compression_level)
Adds a file on disk as a subfile to the Multifile.
Definition: multifile.cxx:417
vector_uchar read_subfile(int index)
Returns a vector_uchar that contains the entire contents of the indicated subfile.
Definition: multifile.I:285
std::streampos get_subfile_internal_start(int index) const
Returns the starting byte position within the Multifile at which the indicated subfile begins...
Definition: multifile.cxx:1567
bool rename_to(const Filename &other) const
Renames the file to the indicated new filename.
Definition: filename.cxx:2339
bool is_binary_or_text() const
Returns true either is_binary() or is_text() is true; that is, that the filename has been specified a...
Definition: filename.I:445
bool exists() const
Returns true if the filename exists on the disk, false otherwise.
Definition: filename.cxx:1267
bool unref() const
Decrements the reference count.
Definition: streamWrapper.I:75
size_t get_length() const
Returns the number of bytes in the datagram.
Definition: datagram.I:335
const void * get_data() const
Returns a pointer to the beginning of the datagram&#39;s data.
Definition: datagram.I:327
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.