Panda3D
multifile.cxx
Go to the documentation of this file.
1 /**
2  * PANDA 3D SOFTWARE
3  * Copyright (c) Carnegie Mellon University. All rights reserved.
4  *
5  * All use of this software is subject to the terms of the revised BSD
6  * license. You should have received a copy of this license along
7  * with this source code in a file named "LICENSE."
8  *
9  * @file multifile.cxx
10  * @author mike
11  * @date 1997-01-09
12  */
13 
14 #include "multifile.h"
15 
16 #include "config_express.h"
17 #include "streamWriter.h"
18 #include "streamReader.h"
19 #include "datagram.h"
20 #include "zStream.h"
21 #include "encryptStream.h"
22 #include "virtualFileSystem.h"
23 #include "virtualFile.h"
24 
25 #include <algorithm>
26 #include <iterator>
27 #include <time.h>
28 
29 #include "openSSLWrapper.h"
30 
31 using std::ios;
32 using std::iostream;
33 using std::istream;
34 using std::max;
35 using std::min;
36 using std::ostream;
37 using std::ostringstream;
38 using std::streamoff;
39 using std::streampos;
40 using std::streamsize;
41 using std::stringstream;
42 using std::string;
43 
44 // This sequence of bytes begins each Multifile to identify it as a Multifile.
45 const char Multifile::_header[] = "pmf\0\n\r";
46 const size_t Multifile::_header_size = 6;
47 
48 // These numbers identify the version of the Multifile. Generally, a change
49 // in the major version is intolerable; while a Multifile with an older minor
50 // version may still be read.
51 const int Multifile::_current_major_ver = 1;
52 
53 const int Multifile::_current_minor_ver = 1;
54 // Bumped to version 1.1 on 6806 to add timestamps.
55 
56 // To confirm that the supplied password matches, we write the Mutifile magic
57 // header at the beginning of the encrypted stream. I suppose this does
58 // compromise the encryption security a tiny bit by making it easy for
59 // crackers to validate that a particular password guess matches or doesn't
60 // match, but the encryption algorithm doesn't depend on this being difficult
61 // anyway.
62 const char Multifile::_encrypt_header[] = "crypty";
63 const size_t Multifile::_encrypt_header_size = 6;
64 
65 
66 
67 /*
68  * A Multifile consists of the following elements: (1) A header. This is
69  * always the first n bytes of the Multifile, and contains a magic number to
70  * identify the file, as well as version numbers and any file-specific
71  * parameters. char[6] The string Multifile::_header, a magic number.
72  * int16 The file's major version number int16 The file's minor
73  * version number uint32 Scale factor. This scales all address references
74  * within the file. Normally 1, this may be set larger to support Multifiles
75  * larger than 4GB. uint32 An overall modification timestamp for the
76  * entire multifile.
77  */
78 
79 /*
80  * (2) Zero or more index entries, one for each subfile within the Multifile.
81  * These entries are of variable length. The first one of these immediately
82  * follows the header, and the first word of each index entry contains the
83  * address of the next index entry. A zero "next" address marks the end of
84  * the chain. These may appear at any point within the Multifile; they do not
85  * necessarily appear in sequential order at the beginning of the file
86  * (although they will after the file has been "packed"). uint32 The
87  * address of the next entry. 0 to mark the end. uint32 The address of
88  * this subfile's data record. uint32 The length in bytes of this
89  * subfile's data record. uint16 The Subfile::_flags member. [uint32]
90  * The original, uncompressed and unencrypted length of the subfile, if it is
91  * compressed or encrypted. This field is only present if one or both of the
92  * SF_compressed or SF_encrypted bits are set in _flags. uint32 A
93  * modification timestamp for the subfile. uint16 The length in bytes of
94  * the subfile's name. char[n] The subfile's name. (3) Zero or more data
95  * entries, one for each subfile. These may appear at any point within the
96  * Multifile; they do not necessarily follow each index entry, nor are they
97  * necessarily all grouped together at the end (although they will be all
98  * grouped together at the end after the file has been "packed"). These are
99  * just blocks of literal data.
100  */
101 
102 /**
103  *
104  */
105 Multifile::
106 Multifile() :
107  _read_filew(_read_file),
108  _read_write_filew(_read_write_file)
109 {
110  ConfigVariableInt multifile_encryption_iteration_count
111  ("multifile-encryption-iteration-count", 0,
112  PRC_DESC("This is a special value of encryption-iteration-count used to encrypt "
113  "subfiles within a multifile. It has a default value of 0 (just one "
114  "application), on the assumption that the files from a multifile must "
115  "be loaded quickly, without paying the cost of an expensive hash on "
116  "each subfile in order to decrypt it."));
117 
118  _read = nullptr;
119  _write = nullptr;
120  _offset = 0;
121  _owns_stream = false;
122  _next_index = 0;
123  _last_index = 0;
124  _last_data_byte = 0;
125  _needs_repack = false;
126  _timestamp = 0;
127  _timestamp_dirty = false;
128  _record_timestamp = true;
129  _scale_factor = 1;
130  _new_scale_factor = 1;
131  _encryption_flag = false;
132  _encryption_iteration_count = multifile_encryption_iteration_count;
133  _file_major_ver = 0;
134  _file_minor_ver = 0;
135 
136 #ifdef HAVE_OPENSSL
137  // Get these values from the config file via an EncryptStreamBuf.
138  EncryptStreamBuf tbuf;
139  _encryption_algorithm = tbuf.get_algorithm();
140  _encryption_key_length = tbuf.get_key_length();
141 #endif
142 }
143 
144 /**
145  *
146  */
147 Multifile::
148 ~Multifile() {
149  close();
150 }
151 
152 /**
153  * Opens the named Multifile on disk for reading. The Multifile index is read
154  * in, and the list of subfiles becomes available; individual subfiles may
155  * then be extracted or read, but the list of subfiles may not be modified.
156  *
157  * Also see the version of open_read() which accepts an istream. Returns true
158  * on success, false on failure.
159  */
160 bool Multifile::
161 open_read(const Filename &multifile_name, const streampos &offset) {
162  close();
163  Filename fname = multifile_name;
164  fname.set_binary();
165 
167  PT(VirtualFile) vfile = vfs->get_file(fname);
168  if (vfile == nullptr) {
169  return false;
170  }
171  istream *multifile_stream = vfile->open_read_file(false);
172  if (multifile_stream == nullptr) {
173  return false;
174  }
175 
176  _timestamp = vfile->get_timestamp();
177  _timestamp_dirty = true;
178  _read = new IStreamWrapper(multifile_stream, true);
179  _owns_stream = true;
180  _multifile_name = multifile_name;
181  _offset = offset;
182  return read_index();
183 }
184 
185 /**
186  * Opens an anonymous Multifile for reading using an istream. There must be
187  * seek functionality via seekg() and tellg() on the istream.
188  *
189  * If owns_pointer is true, then the Multifile assumes ownership of the stream
190  * pointer and will delete it when the multifile is closed, including if this
191  * function returns false.
192  */
193 bool Multifile::
194 open_read(IStreamWrapper *multifile_stream, bool owns_pointer,
195  const streampos &offset) {
196  close();
197  _timestamp = time(nullptr);
198  _timestamp_dirty = true;
199  _read = multifile_stream;
200  _owns_stream = owns_pointer;
201  _offset = offset;
202  return read_index();
203 }
204 
205 /**
206  * Opens the named Multifile on disk for writing. If there already exists a
207  * file by that name, it is truncated. The Multifile is then prepared for
208  * accepting a brand new set of subfiles, which will be written to the
209  * indicated filename. Individual subfiles may not be extracted or read.
210  *
211  * Also see the version of open_write() which accepts an ostream. Returns
212  * true on success, false on failure.
213  */
215 open_write(const Filename &multifile_name) {
216  close();
217  Filename fname = multifile_name;
218  fname.set_binary();
219  if (!fname.open_write(_write_file, true)) {
220  return false;
221  }
222  _timestamp = time(nullptr);
223  _timestamp_dirty = true;
224  _write = &_write_file;
225  _multifile_name = multifile_name;
226  return true;
227 }
228 
229 /**
230  * Opens an anonymous Multifile for writing using an ostream. There must be
231  * seek functionality via seekp() and tellp() on the pstream.
232  *
233  * If owns_pointer is true, then the Multifile assumes ownership of the stream
234  * pointer and will delete it when the multifile is closed, including if this
235  * function returns false.
236  */
238 open_write(ostream *multifile_stream, bool owns_pointer) {
239  close();
240  _timestamp = time(nullptr);
241  _timestamp_dirty = true;
242  _write = multifile_stream;
243  _owns_stream = owns_pointer;
244  _write->seekp(0, ios::beg);
245  return true;
246 }
247 
248 /**
249  * Opens the named Multifile on disk for reading and writing. If there
250  * already exists a file by that name, its index is read. Subfiles may be
251  * added or removed, and the resulting changes will be written to the named
252  * file.
253  *
254  * Also see the version of open_read_write() which accepts an iostream.
255  * Returns true on success, false on failure.
256  */
258 open_read_write(const Filename &multifile_name) {
259  close();
260  Filename fname = multifile_name;
261  fname.set_binary();
262  bool exists = fname.exists();
263  if (!fname.open_read_write(_read_write_file)) {
264  return false;
265  }
266  if (exists) {
267  _timestamp = fname.get_timestamp();
268  } else {
269  _timestamp = time(nullptr);
270  }
271  _timestamp_dirty = true;
272  _read = &_read_write_filew;
273  _write = &_read_write_file;
274  _multifile_name = multifile_name;
275 
276  if (exists) {
277  return read_index();
278  } else {
279  return true;
280  }
281 }
282 
283 /**
284  * Opens an anonymous Multifile for reading and writing using an iostream.
285  * There must be seek functionality via seekg()/seekp() and tellg()/tellp() on
286  * the iostream.
287  *
288  * If owns_pointer is true, then the Multifile assumes ownership of the stream
289  * pointer and will delete it when the multifile is closed, including if this
290  * function returns false.
291  */
292 bool Multifile::
293 open_read_write(iostream *multifile_stream, bool owns_pointer) {
294  close();
295  _timestamp = time(nullptr);
296  _timestamp_dirty = true;
297 
298  // We don't support locking when opening a file in read-write mode, because
299  // we don't bother with locking on write. But we need to have an
300  // IStreamWrapper to assign to the _read member, so we create one on-the-fly
301  // here.
302  _read = new StreamWrapper(multifile_stream, owns_pointer);
303  _write = multifile_stream;
304  _owns_stream = true; // Because we own the StreamWrapper, above.
305  _write->seekp(0, ios::beg);
306 
307  // Check whether the read stream is empty.
308  multifile_stream->seekg(0, ios::end);
309  if (multifile_stream->tellg() == (streampos)0) {
310  // The read stream is empty, which is always valid.
311  return true;
312  }
313 
314  // The read stream is not empty, so we'd better have a valid Multifile.
315  return read_index();
316 }
317 
318 /**
319  * Closes the Multifile if it is open. All changes are flushed to disk, and
320  * the file becomes invalid for further operations until the next call to
321  * open().
322  */
324 close() {
325  if (_new_scale_factor != _scale_factor) {
326  // If we have changed the scale factor recently, we need to force a
327  // repack.
328  repack();
329  } else {
330  flush();
331  }
332 
333  if (_owns_stream) {
334  // We prefer to delete the IStreamWrapper over the ostream, if possible.
335  if (_read != nullptr) {
336  // Only delete it if no SubStream is still referencing it.
337  if (!_read->unref()) {
338  delete _read;
339  }
340  } else if (_write != nullptr) {
341  delete _write;
342  }
343  }
344 
345  _read = nullptr;
346  _write = nullptr;
347  _offset = 0;
348  _owns_stream = false;
349  _next_index = 0;
350  _last_index = 0;
351  _needs_repack = false;
352  _timestamp = 0;
353  _timestamp_dirty = false;
354  _scale_factor = 1;
355  _new_scale_factor = 1;
356  _encryption_flag = false;
357  _file_major_ver = 0;
358  _file_minor_ver = 0;
359 
360  _read_file.close();
361  _write_file.close();
362  _read_write_file.close();
363  _multifile_name = Filename();
364 
365  clear_subfiles();
366 }
367 
368 /**
369  * Changes the internal scale factor for this Multifile.
370  *
371  * This is normally 1, but it may be set to any arbitrary value (greater than
372  * zero) to support Multifile archives that exceed 4GB, if necessary.
373  * (Individual subfiles may still not exceed 4GB.)
374  *
375  * All addresses within the file are rounded up to the next multiple of
376  * _scale_factor, and zeros are written to the file to fill the resulting
377  * gaps. Then the address is divided by _scale_factor and written out as a
378  * 32-bit integer. Thus, setting a scale factor of 2 supports up to 8GB
379  * files, 3 supports 12GB files, etc.
380  *
381  * Calling this function on an already-existing Multifile will have no
382  * immediate effect until a future call to repack() or close() (or until the
383  * Multifile is destructed).
384  */
386 set_scale_factor(size_t scale_factor) {
387  nassertv(is_write_valid());
388  nassertv(scale_factor != (size_t)0);
389 
390  if (_next_index == (streampos)0) {
391  // If it's a brand new Multifile, we can go ahead and set it immediately.
392  _scale_factor = scale_factor;
393  } else {
394  // Otherwise, we'd better have read access so we can repack it later.
395  nassertv(is_read_valid());
396  }
397 
398  // Setting the _new_scale_factor different from the _scale_factor will force
399  // a repack operation on close.
400  _new_scale_factor = scale_factor;
401 }
402 
403 /**
404  * Adds a file on disk as a subfile to the Multifile. The file named by
405  * filename will be read and added to the Multifile at the next call to
406  * flush(). If there already exists a subfile with the indicated name, it is
407  * replaced without examining its contents (but see also update_subfile).
408  *
409  * Either Filename:::set_binary() or set_text() must have been called
410  * previously to specify the nature of the source file. If set_text() was
411  * called, the text flag will be set on the subfile.
412  *
413  * Returns the subfile name on success (it might have been modified slightly),
414  * or empty string on failure.
415  */
417 add_subfile(const string &subfile_name, const Filename &filename,
418  int compression_level) {
419  nassertr(is_write_valid(), string());
420 
421  Filename fname = filename;
422  if (multifile_always_binary) {
423  fname.set_binary();
424  }
425 
426  nassertr(fname.is_binary_or_text(), string());
427 
428  if (!fname.exists()) {
429  return string();
430  }
431  string name = standardize_subfile_name(subfile_name);
432  if (!name.empty()) {
433  Subfile *subfile = new Subfile;
434  subfile->_name = name;
435  subfile->_source_filename = fname;
436  if (fname.is_text()) {
437  subfile->_flags |= SF_text;
438  }
439 
440  add_new_subfile(subfile, compression_level);
441  }
442 
443  _timestamp = time(nullptr);
444  _timestamp_dirty = true;
445 
446  return name;
447 }
448 
449 /**
450  * Adds a file from a stream as a subfile to the Multifile. The indicated
451  * istream will be read and its contents added to the Multifile at the next
452  * call to flush(). The file will be added as a binary subfile.
453  *
454  * Note that the istream must remain untouched and unused by any other code
455  * until flush() is called. At that time, the Multifile will read the entire
456  * contents of the istream from the current file position to the end of the
457  * file. Subsequently, the Multifile will *not* close or delete the istream.
458  * It is the caller's responsibility to ensure that the istream pointer does
459  * not destruct during the lifetime of the Multifile.
460  *
461  * Returns the subfile name on success (it might have been modified slightly),
462  * or empty string on failure.
463  */
465 add_subfile(const string &subfile_name, istream *subfile_data,
466  int compression_level) {
467  nassertr(is_write_valid(), string());
468 
469  string name = standardize_subfile_name(subfile_name);
470  if (!name.empty()) {
471  Subfile *subfile = new Subfile;
472  subfile->_name = name;
473  subfile->_source = subfile_data;
474  add_new_subfile(subfile, compression_level);
475  }
476 
477  return name;
478 }
479 
480 /**
481  * Adds a file on disk to the subfile. If a subfile already exists with the
482  * same name, its contents are compared byte-for-byte to the disk file, and it
483  * is replaced only if it is different; otherwise, the multifile is left
484  * unchanged.
485  *
486  * Either Filename:::set_binary() or set_text() must have been called
487  * previously to specify the nature of the source file. If set_text() was
488  * called, the text flag will be set on the subfile.
489  */
491 update_subfile(const string &subfile_name, const Filename &filename,
492  int compression_level) {
493  nassertr(is_write_valid(), string());
494 
495  Filename fname = filename;
496  if (multifile_always_binary) {
497  fname.set_binary();
498  }
499 
500  nassertr(fname.is_binary_or_text(), string());
501 
502  if (!fname.exists()) {
503  return string();
504  }
505  string name = standardize_subfile_name(subfile_name);
506  if (!name.empty()) {
507  int index = find_subfile(name);
508  if (index >= 0) {
509  // The subfile already exists; compare it to the source file.
510  if (compare_subfile(index, fname)) {
511  // The files are identical; do nothing.
512  return name;
513  }
514  }
515 
516  // The subfile does not already exist or it is different from the source
517  // file. Add the new source file.
518  Subfile *subfile = new Subfile;
519  subfile->_name = name;
520  subfile->_source_filename = fname;
521  if (fname.is_text()) {
522  subfile->_flags |= SF_text;
523  }
524 
525  add_new_subfile(subfile, compression_level);
526  }
527 
528  _timestamp = time(nullptr);
529  _timestamp_dirty = true;
530 
531  return name;
532 }
533 
534 #ifdef HAVE_OPENSSL
535 /**
536  * Ownership of the X509 object is passed into the CertRecord; it will be
537  * freed when the CertRecord destructs.
538  */
539 Multifile::CertRecord::
540 CertRecord(X509 *cert) :
541  _cert(cert)
542 {
543 }
544 
545 /**
546  *
547  */
548 Multifile::CertRecord::
549 CertRecord(const Multifile::CertRecord &copy) :
550  _cert(X509_dup(copy._cert))
551 {
552 }
553 
554 /**
555  *
556  */
557 Multifile::CertRecord::
558 ~CertRecord() {
559  X509_free(_cert);
560 }
561 
562 /**
563  *
564  */
565 void Multifile::CertRecord::
566 operator = (const Multifile::CertRecord &other) {
567  X509_free(_cert);
568  _cert = X509_dup(other._cert);
569 }
570 #endif // HAVE_OPENSSL
571 
572 #ifdef HAVE_OPENSSL
573 /**
574  * Adds a new signature to the Multifile. This signature associates the
575  * indicated certificate with the current contents of the Multifile. When the
576  * Multifile is read later, the signature will still be present only if the
577  * Multifile is unchanged; any subsequent changes to the Multifile will
578  * automatically invalidate and remove the signature.
579  *
580  * The chain filename may be empty if the certificate does not require an
581  * authenticating certificate chain (e.g. because it is self-signed).
582  *
583  * The specified private key must match the certificate, and the Multifile
584  * must be open in read-write mode. The private key is only used for
585  * generating the signature; it is not written to the Multifile and cannot be
586  * retrieved from the Multifile later. (However, the certificate *can* be
587  * retrieved from the Multifile later, to identify the entity that created the
588  * signature.)
589  *
590  * This implicitly causes a repack() operation if one is needed. Returns true
591  * on success, false on failure.
592  *
593  * This flavor of add_signature() reads the certificate and private key from a
594  * PEM-formatted file, for instance as generated by the openssl command. If
595  * the private key file is password-encrypted, the third parameter will be
596  * used as the password to decrypt it.
597  */
598 bool Multifile::
599 add_signature(const Filename &certificate, const Filename &chain,
600  const Filename &pkey, const string &password) {
602 
603  if (chain.empty() && pkey.empty()) {
604  // If the second two filenames are empty, assume we're going for the
605  // composite mode, where everything's stuffed into the first file.
606  return add_signature(certificate, password);
607  }
608 
609  CertChain cert_chain;
610 
611  // Read the certificate file from VFS. First, read the complete file into
612  // memory.
613  string certificate_data;
614  if (!vfs->read_file(certificate, certificate_data, true)) {
615  express_cat.info()
616  << "Could not read " << certificate << ".\n";
617  return false;
618  }
619 
620  // Create an in-memory BIO to read the "file" from the buffer.
621  BIO *certificate_mbio = BIO_new_mem_buf((void *)certificate_data.data(), certificate_data.size());
622  X509 *x509 = PEM_read_bio_X509(certificate_mbio, nullptr, nullptr, (void *)"");
623  BIO_free(certificate_mbio);
624  if (x509 == nullptr) {
625  express_cat.info()
626  << "Could not read certificate in " << certificate << ".\n";
627  return false;
628  }
629 
630  // Store the first X509--the actual certificate--as the first record in our
631  // CertChain object.
632  cert_chain.push_back(CertRecord(x509));
633 
634  // Read the rest of the certificates in the chain file.
635  if (!chain.empty()) {
636  string chain_data;
637  if (!vfs->read_file(chain, chain_data, true)) {
638  express_cat.info()
639  << "Could not read " << chain << ".\n";
640  return false;
641  }
642 
643  BIO *chain_mbio = BIO_new_mem_buf((void *)chain_data.data(), chain_data.size());
644  X509 *c = PEM_read_bio_X509(chain_mbio, nullptr, nullptr, (void *)"");
645  while (c != nullptr) {
646  cert_chain.push_back(c);
647  c = PEM_read_bio_X509(chain_mbio, nullptr, nullptr, (void *)"");
648  }
649  BIO_free(chain_mbio);
650 
651  if (cert_chain.size() == 1) {
652  express_cat.info()
653  << "Could not read certificate chain in " << chain << ".\n";
654  return false;
655  }
656  }
657 
658  // Now do the same thing with the private key. This one may be password-
659  // encrypted on disk.
660  string pkey_data;
661  if (!vfs->read_file(pkey, pkey_data, true)) {
662  express_cat.info()
663  << "Could not read " << pkey << ".\n";
664  return false;
665  }
666 
667  BIO *pkey_mbio = BIO_new_mem_buf((void *)pkey_data.data(), pkey_data.size());
668  EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, nullptr, nullptr,
669  (void *)password.c_str());
670  BIO_free(pkey_mbio);
671  if (evp_pkey == nullptr) {
672  express_cat.info()
673  << "Could not read private key in " << pkey << ".\n";
674  return false;
675  }
676 
677  bool result = add_signature(cert_chain, evp_pkey);
678 
679  EVP_PKEY_free(evp_pkey);
680 
681  return result;
682 }
683 #endif // HAVE_OPENSSL
684 
685 #ifdef HAVE_OPENSSL
686 /**
687  * Adds a new signature to the Multifile. This signature associates the
688  * indicated certificate with the current contents of the Multifile. When the
689  * Multifile is read later, the signature will still be present only if the
690  * Multifile is unchanged; any subsequent changes to the Multifile will
691  * automatically invalidate and remove the signature.
692  *
693  * This flavor of add_signature() reads the certificate, private key, and
694  * certificate chain from the same PEM-formatted file. It takes the first
695  * private key found as the intended key, and then uses the first certificate
696  * found that matches that key as the signing certificate. Any other
697  * certificates in the file are taken to be part of the chain.
698  */
699 bool Multifile::
700 add_signature(const Filename &composite, const string &password) {
702 
703  // First, read the complete file into memory.
704  string composite_data;
705  if (!vfs->read_file(composite, composite_data, true)) {
706  express_cat.info()
707  << "Could not read " << composite << ".\n";
708  return false;
709  }
710 
711  // Get the private key.
712  BIO *pkey_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size());
713  EVP_PKEY *evp_pkey = PEM_read_bio_PrivateKey(pkey_mbio, nullptr, nullptr,
714  (void *)password.c_str());
715  BIO_free(pkey_mbio);
716  if (evp_pkey == nullptr) {
717  express_cat.info()
718  << "Could not read private key in " << composite << ".\n";
719  return false;
720  }
721 
722  // Now read all of the certificates.
723  CertChain cert_chain;
724 
725  BIO *chain_mbio = BIO_new_mem_buf((void *)composite_data.data(), composite_data.size());
726  X509 *c = PEM_read_bio_X509(chain_mbio, nullptr, nullptr, (void *)"");
727  while (c != nullptr) {
728  cert_chain.push_back(c);
729  c = PEM_read_bio_X509(chain_mbio, nullptr, nullptr, (void *)"");
730  }
731  BIO_free(chain_mbio);
732 
733  if (cert_chain.empty()) {
734  express_cat.info()
735  << "Could not read certificates in " << composite << ".\n";
736  return false;
737  }
738 
739  // Now find the certificate that matches the signature, and move it to the
740  // front of the chain.
741  size_t i;
742  bool found_match = false;
743  for (i = 0; i < cert_chain.size(); ++i) {
744  X509 *c = cert_chain[i]._cert;
745  if (X509_check_private_key(c, evp_pkey)) {
746  found_match = true;
747  if (i != 0) {
748  // Move this entry to the beginning.
749  cert_chain.insert(cert_chain.begin(), cert_chain[i]);
750  cert_chain.erase(cert_chain.begin() + i + 1);
751  }
752  break;
753  }
754  }
755 
756  if (!found_match) {
757  express_cat.info()
758  << "No certificates in " << composite << " match key.\n";
759  return false;
760  }
761 
762  bool result = add_signature(cert_chain, evp_pkey);
763 
764  EVP_PKEY_free(evp_pkey);
765 
766  return result;
767 }
768 #endif // HAVE_OPENSSL
769 
770 #ifdef HAVE_OPENSSL
771 /**
772  * Adds a new signature to the Multifile. This signature associates the
773  * indicated certificate with the current contents of the Multifile. When the
774  * Multifile is read later, the signature will still be present only if the
775  * Multifile is unchanged; any subsequent changes to the Multifile will
776  * automatically invalidate and remove the signature.
777  *
778  * The signature certificate is the first certificate on the CertChain object.
779  * Any remaining certificates are support certificates to authenticate the
780  * first one.
781  *
782  * The specified private key must match the certificate, and the Multifile
783  * must be open in read-write mode. The private key is only used for
784  * generating the signature; it is not written to the Multifile and cannot be
785  * retrieved from the Multifile later. (However, the certificate *can* be
786  * retrieved from the Multifile later, to identify the entity that created the
787  * signature.)
788  *
789  * This implicitly causes a repack() operation if one is needed. Returns true
790  * on success, false on failure.
791  */
792 bool Multifile::
793 add_signature(const Multifile::CertChain &cert_chain, EVP_PKEY *pkey) {
794  if (_needs_repack) {
795  if (!repack()) {
796  return false;
797  }
798  } else {
799  if (!flush()) {
800  return false;
801  }
802  }
803 
804  if (cert_chain.empty()) {
805  express_cat.info()
806  << "No certificate given.\n";
807  return false;
808  }
809 
810  if (pkey == nullptr) {
811  express_cat.info()
812  << "No private key given.\n";
813  return false;
814  }
815 
816  if (!X509_check_private_key(cert_chain[0]._cert, pkey)) {
817  express_cat.info()
818  << "Private key does not match certificate.\n";
819  return false;
820  }
821 
822  // Now encode that list of certs to a stream in DER form.
823  stringstream der_stream;
824  StreamWriter der_writer(der_stream);
825  der_writer.add_uint32((uint32_t)cert_chain.size());
826 
827  CertChain::const_iterator ci;
828  for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) {
829  X509 *cert = (*ci)._cert;
830 
831  int der_len = i2d_X509(cert, nullptr);
832  unsigned char *der_buf = new unsigned char[der_len];
833  unsigned char *p = der_buf;
834  i2d_X509(cert, &p);
835  der_writer.append_data(der_buf, der_len);
836  delete[] der_buf;
837  }
838 
839  // Create a temporary Subfile for writing out the signature.
840  der_stream.seekg(0);
841  Subfile *subfile = new Subfile;
842  subfile->_pkey = pkey;
843  subfile->_flags |= SF_signature;
844  subfile->_source = &der_stream;
845 
846  // Write the new Subfile at the end. The cert_special subfiles always go at
847  // the end, because they're not the part of the file that's signed.
848  nassertr(_new_subfiles.empty(), false);
849  _new_subfiles.push_back(subfile);
850  bool result = flush();
851 
852  delete subfile;
853 
854  return result;
855 }
856 #endif // HAVE_OPENSSL
857 
858 #ifdef HAVE_OPENSSL
859 /**
860  * Returns the number of matching signatures found on the Multifile. These
861  * signatures may be iterated via get_signature() and related methods.
862  *
863  * A signature on this list is guaranteed to match the Multifile contents,
864  * proving that the Multifile has been unmodified since the signature was
865  * applied. However, this does not guarantee that the certificate itself is
866  * actually from who it says it is from; only that it matches the Multifile
867  * contents. See validate_signature_certificate() to authenticate a
868  * particular certificate.
869  */
870 int Multifile::
871 get_num_signatures() const {
872  ((Multifile *)this)->check_signatures();
873  return _signatures.size();
874 }
875 #endif // HAVE_OPENSSL
876 
877 #ifdef HAVE_OPENSSL
878 /**
879  * Returns the nth signature found on the Multifile. See the comments in
880  * get_num_signatures().
881  */
882 const Multifile::CertChain &Multifile::
883 get_signature(int n) const {
884  ((Multifile *)this)->check_signatures();
885  static CertChain error_chain;
886  nassertr(n >= 0 && n < (int)_signatures.size(), error_chain);
887  return _signatures[n];
888 }
889 #endif // HAVE_OPENSSL
890 
891 #ifdef HAVE_OPENSSL
892 /**
893  * Returns the "subject name" for the nth signature found on the Multifile.
894  * This is a string formatted according to RFC2253 that should more-or-less
895  * identify a particular certificate; when paired with the public key (see
896  * get_signature_public_key()), it can uniquely identify a certificate. See
897  * the comments in get_num_signatures().
898  */
899 string Multifile::
900 get_signature_subject_name(int n) const {
901  const CertChain &cert_chain = get_signature(n);
902  nassertr(!cert_chain.empty(), string());
903 
904  X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert);
905  if (xname != nullptr) {
906  // We use "print" to dump the output to a memory BIO. Is there an easier
907  // way to extract the X509_NAME text? Curse these incomplete docs.
908  BIO *mbio = BIO_new(BIO_s_mem());
909  X509_NAME_print_ex(mbio, xname, 0, XN_FLAG_RFC2253);
910 
911  char *pp;
912  long pp_size = BIO_get_mem_data(mbio, &pp);
913  string name(pp, pp_size);
914  BIO_free(mbio);
915  return name;
916  }
917 
918  return string();
919 }
920 #endif // HAVE_OPENSSL
921 
922 #ifdef HAVE_OPENSSL
923 /**
924  * Returns a "friendly name" for the nth signature found on the Multifile.
925  * This attempts to extract out the most meaningful part of the subject name.
926  * It returns the emailAddress, if it is defined; otherwise, it returns the
927  * commonName.
928  *
929  * See the comments in get_num_signatures().
930  */
931 string Multifile::
932 get_signature_friendly_name(int n) const {
933  const CertChain &cert_chain = get_signature(n);
934  nassertr(!cert_chain.empty(), string());
935 
936  static const int nid_choices[] = {
937  NID_pkcs9_emailAddress,
938  NID_subject_alt_name,
939  NID_commonName,
940  -1,
941  };
942 
943  // Choose the first NID that exists on the cert.
944  for (int ni = 0; nid_choices[ni] != -1; ++ni) {
945  int nid = nid_choices[ni];
946 
947  // A complex OpenSSL interface to extract out the name in utf-8.
948  X509_NAME *xname = X509_get_subject_name(cert_chain[0]._cert);
949  if (xname != nullptr) {
950  int pos = X509_NAME_get_index_by_NID(xname, nid, -1);
951  if (pos != -1) {
952  // We just get the first common name. I guess it's possible to have
953  // more than one; not sure what that means in this context.
954  X509_NAME_ENTRY *xentry = X509_NAME_get_entry(xname, pos);
955  if (xentry != nullptr) {
956  ASN1_STRING *data = X509_NAME_ENTRY_get_data(xentry);
957  if (data != nullptr) {
958  // We use "print" to dump the output to a memory BIO. Is there an
959  // easier way to decode the ASN1_STRING? Curse these incomplete
960  // docs.
961  BIO *mbio = BIO_new(BIO_s_mem());
962  ASN1_STRING_print_ex(mbio, data, ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB);
963 
964  char *pp;
965  long pp_size = BIO_get_mem_data(mbio, &pp);
966  string name(pp, pp_size);
967  BIO_free(mbio);
968  return name;
969  }
970  }
971  }
972  }
973  }
974 
975  return string();
976 }
977 #endif // HAVE_OPENSSL
978 
979 #ifdef HAVE_OPENSSL
980 /**
981  * Returns the public key used for the nth signature found on the Multifile.
982  * This is encoded in DER form and returned as a string of hex digits.
983  *
984  * This can be used, in conjunction with the subject name (see
985  * get_signature_subject_name()), to uniquely identify a particular
986  * certificate and its subsequent reissues. See the comments in
987  * get_num_signatures().
988  */
989 string Multifile::
990 get_signature_public_key(int n) const {
991  const CertChain &cert_chain = get_signature(n);
992  nassertr(!cert_chain.empty(), string());
993 
994  EVP_PKEY *pkey = X509_get_pubkey(cert_chain[0]._cert);
995  if (pkey != nullptr) {
996  int key_len = i2d_PublicKey(pkey, nullptr);
997  unsigned char *key_buf = new unsigned char[key_len];
998  unsigned char *p = key_buf;
999  i2d_PublicKey(pkey, &p);
1000  string result;
1001  for (int i = 0; i < key_len; ++i) {
1002  result += tohex(key_buf[i] >> 4);
1003  result += tohex(key_buf[i]);
1004  }
1005  delete[] key_buf;
1006  return result;
1007  }
1008 
1009  return string();
1010 }
1011 #endif // HAVE_OPENSSL
1012 
1013 #ifdef HAVE_OPENSSL
1014 /**
1015  * Writes the certificate for the nth signature, in user-readable verbose
1016  * form, to the indicated stream. See the comments in get_num_signatures().
1017  */
1018 void Multifile::
1019 print_signature_certificate(int n, ostream &out) const {
1020  const CertChain &cert_chain = get_signature(n);
1021  nassertv(!cert_chain.empty());
1022 
1023  BIO *mbio = BIO_new(BIO_s_mem());
1024  X509_print(mbio, cert_chain[0]._cert);
1025 
1026  char *pp;
1027  long pp_size = BIO_get_mem_data(mbio, &pp);
1028  out.write(pp, pp_size);
1029  BIO_free(mbio);
1030 }
1031 #endif // HAVE_OPENSSL
1032 
1033 #ifdef HAVE_OPENSSL
1034 /**
1035  * Writes the certificate for the nth signature, in PEM form, to the indicated
1036  * stream. See the comments in get_num_signatures().
1037  */
1038 void Multifile::
1039 write_signature_certificate(int n, ostream &out) const {
1040  const CertChain &cert_chain = get_signature(n);
1041  nassertv(!cert_chain.empty());
1042 
1043  BIO *mbio = BIO_new(BIO_s_mem());
1044 
1045  CertChain::const_iterator ci;
1046  for (ci = cert_chain.begin(); ci != cert_chain.end(); ++ci) {
1047  X509 *c = (*ci)._cert;
1048  X509_print(mbio, c);
1049  PEM_write_bio_X509(mbio, c);
1050  }
1051 
1052  char *pp;
1053  long pp_size = BIO_get_mem_data(mbio, &pp);
1054  out.write(pp, pp_size);
1055  BIO_free(mbio);
1056 }
1057 #endif // HAVE_OPENSSL
1058 
1059 #ifdef HAVE_OPENSSL
1060 /**
1061  * Checks that the certificate used for the nth signature is a valid,
1062  * authorized certificate with some known certificate authority. Returns 0 if
1063  * it is valid, -1 if there is some error, or the corresponding OpenSSL error
1064  * code if it is invalid, out-of-date, or self-signed.
1065  */
1066 int Multifile::
1067 validate_signature_certificate(int n) const {
1068  int verify_result = -1;
1069 
1070  const CertChain &chain = get_signature(n);
1071  nassertr(!chain.empty(), false);
1072 
1073  OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
1074 
1075  // Copy our CertChain structure into an X509 pointer and accompanying
1076  // STACK_OF(X509) pointer.
1077  X509 *x509 = chain[0]._cert;
1078  STACK_OF(X509) *stack = nullptr;
1079  if (chain.size() > 1) {
1080  stack = sk_X509_new(nullptr);
1081  for (size_t n = 1; n < chain.size(); ++n) {
1082  sk_X509_push(stack, chain[n]._cert);
1083  }
1084  }
1085 
1086  // Create the X509_STORE_CTX for verifying the cert and chain.
1087  X509_STORE_CTX *ctx = X509_STORE_CTX_new();
1088  X509_STORE_CTX_init(ctx, sslw->get_x509_store(), x509, stack);
1089  X509_STORE_CTX_set_cert(ctx, x509);
1090 
1091  if (X509_verify_cert(ctx)) {
1092  verify_result = 0;
1093  } else {
1094  verify_result = X509_STORE_CTX_get_error(ctx);
1095  }
1096 
1097  if (express_cat.is_debug()) {
1098  express_cat.debug()
1099  << get_signature_subject_name(n) << ": validate " << verify_result
1100  << "\n";
1101  }
1102 
1103  sk_X509_free(stack);
1104  X509_STORE_CTX_cleanup(ctx);
1105  X509_STORE_CTX_free(ctx);
1106 
1107  return verify_result;
1108 }
1109 #endif // HAVE_OPENSSL
1110 
1111 /**
1112  * Writes all contents of the Multifile to disk. Until flush() is called,
1113  * add_subfile() and remove_subfile() do not actually do anything to disk. At
1114  * this point, all of the recently-added subfiles are read and their contents
1115  * are added to the end of the Multifile, and the recently-removed subfiles
1116  * are marked gone from the Multifile.
1117  *
1118  * This may result in a suboptimal index. To guarantee that the index is
1119  * written at the beginning of the file, call repack() instead of flush().
1120  *
1121  * It is not necessary to call flush() explicitly unless you are concerned
1122  * about reading the recently-added subfiles immediately.
1123  *
1124  * Returns true on success, false on failure.
1125  */
1127 flush() {
1128  if (!is_write_valid()) {
1129  return false;
1130  }
1131 
1132  bool new_file = (_next_index == (streampos)0);
1133  if (new_file) {
1134  // If we don't have an index yet, we don't have a header. Write the
1135  // header.
1136  if (!write_header()) {
1137  return false;
1138  }
1139 
1140  } else {
1141  if (_file_minor_ver != _current_minor_ver) {
1142  // If we *do* have an index already, but this is an old version
1143  // multifile, we have to completely rewrite it anyway.
1144  return repack();
1145  }
1146  }
1147 
1148  nassertr(_write != nullptr, false);
1149 
1150  // First, mark out all of the removed subfiles.
1151  PendingSubfiles::iterator pi;
1152  for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
1153  Subfile *subfile = (*pi);
1154  subfile->rewrite_index_flags(*_write);
1155  delete subfile;
1156  }
1157  _removed_subfiles.clear();
1158 
1159  bool wrote_ok = true;
1160 
1161  if (!_new_subfiles.empty() || new_file) {
1162  // Add a few more files to the end. We always add subfiles at the end of
1163  // the multifile, so go there first.
1164  sort(_new_subfiles.begin(), _new_subfiles.end(), IndirectLess<Subfile>());
1165  if (_last_index != (streampos)0) {
1166  _write->seekp(0, ios::end);
1167  if (_write->fail()) {
1168  express_cat.info()
1169  << "Unable to seek Multifile " << _multifile_name << ".\n";
1170  return false;
1171  }
1172  _next_index = _write->tellp();
1173  _next_index = pad_to_streampos(_next_index);
1174 
1175  // And update the forward link from the last_index to point to this new
1176  // index location.
1177  _write->seekp(_last_index);
1178  StreamWriter writer(_write, false);
1179  writer.add_uint32(streampos_to_word(_next_index));
1180  }
1181 
1182  _write->seekp(_next_index);
1183  nassertr(_next_index == _write->tellp(), false);
1184 
1185  // Ok, here we are at the end of the file. Write out the recently-added
1186  // subfiles here. First, count up the index size.
1187  for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
1188  Subfile *subfile = (*pi);
1189  _last_index = _next_index;
1190  _next_index = subfile->write_index(*_write, _next_index, this);
1191  nassertr(_next_index == _write->tellp(), false);
1192  _next_index = pad_to_streampos(_next_index);
1193  nassertr(_next_index == _write->tellp(), false);
1194  }
1195 
1196  // Now we're at the end of the index. Write a 0 here to mark the end.
1197  StreamWriter writer(_write, false);
1198  writer.add_uint32(0);
1199  _next_index += 4;
1200  nassertr(_next_index == _write->tellp(), false);
1201  _next_index = pad_to_streampos(_next_index);
1202 
1203  // All right, now write out each subfile's data.
1204  for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
1205  Subfile *subfile = (*pi);
1206 
1207  if (_read != nullptr) {
1208  _read->acquire();
1209  _next_index = subfile->write_data(*_write, _read->get_istream(),
1210  _next_index, this);
1211  _read->release();
1212 
1213  } else {
1214  _next_index = subfile->write_data(*_write, nullptr, _next_index, this);
1215  }
1216 
1217  nassertr(_next_index == _write->tellp(), false);
1218  _next_index = pad_to_streampos(_next_index);
1219  if (subfile->is_data_invalid()) {
1220  wrote_ok = false;
1221  }
1222 
1223  if (!subfile->is_cert_special()) {
1224  _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos());
1225  }
1226  nassertr(_next_index == _write->tellp(), false);
1227  }
1228 
1229  // Now go back and fill in the proper addresses for the data start. We
1230  // didn't do it in the first pass, because we don't really want to keep
1231  // all those file handles open, and so we didn't have to determine each
1232  // file's length ahead of time.
1233  for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
1234  Subfile *subfile = (*pi);
1235  subfile->rewrite_index_data_start(*_write, this);
1236  }
1237 
1238  _new_subfiles.clear();
1239  }
1240 
1241  // Also update the overall timestamp.
1242  if (_timestamp_dirty) {
1243  nassertr(!_write->fail(), false);
1244  static const size_t timestamp_pos = _header_prefix.size() + _header_size + 2 + 2 + 4;
1245  _write->seekp(timestamp_pos);
1246  nassertr(!_write->fail(), false);
1247 
1248  StreamWriter writer(*_write);
1249  if (_record_timestamp) {
1250  writer.add_uint32(_timestamp);
1251  } else {
1252  writer.add_uint32(0);
1253  }
1254  _timestamp_dirty = false;
1255  }
1256 
1257  _write->flush();
1258  if (!wrote_ok || _write->fail()) {
1259  express_cat.info()
1260  << "Unable to update Multifile " << _multifile_name << ".\n";
1261  close();
1262  return false;
1263  }
1264 
1265  return true;
1266 }
1267 
1268 /**
1269  * Forces a complete rewrite of the Multifile and all of its contents, so that
1270  * its index will appear at the beginning of the file with all of the subfiles
1271  * listed in alphabetical order. This is considered optimal for reading, and
1272  * is the standard configuration; but it is not essential to do this.
1273  *
1274  * It is only valid to call this if the Multifile was opened using
1275  * open_read_write() and an explicit filename, rather than an iostream. Also,
1276  * we must have write permission to the directory containing the Multifile.
1277  *
1278  * Returns true on success, false on failure.
1279  */
1281 repack() {
1282  if (_next_index == (streampos)0) {
1283  // If the Multifile hasn't yet been written, this is really just a flush
1284  // operation.
1285  _needs_repack = false;
1286  return flush();
1287  }
1288 
1289  nassertr(is_write_valid() && is_read_valid(), false);
1290  nassertr(!_multifile_name.empty(), false);
1291 
1292  // First, we open a temporary filename to copy the Multifile to.
1293  Filename dirname = _multifile_name.get_dirname();
1294  if (dirname.empty()) {
1295  dirname = ".";
1296  }
1297  Filename temp_filename = Filename::temporary(dirname, "mftemp");
1298  temp_filename.set_binary();
1299  pofstream temp;
1300  if (!temp_filename.open_write(temp)) {
1301  express_cat.info()
1302  << "Unable to open temporary file " << temp_filename << "\n";
1303  return false;
1304  }
1305 
1306  // Now we scrub our internal structures so it looks like we're a brand new
1307  // Multifile.
1308  PendingSubfiles::iterator pi;
1309  for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
1310  Subfile *subfile = (*pi);
1311  delete subfile;
1312  }
1313  _removed_subfiles.clear();
1314  _new_subfiles.clear();
1315  std::copy(_subfiles.begin(), _subfiles.end(), std::back_inserter(_new_subfiles));
1316  _next_index = 0;
1317  _last_index = 0;
1318  _last_data_byte = 0;
1319  _scale_factor = _new_scale_factor;
1320 
1321  // And we write our contents to our new temporary file.
1322  _write = &temp;
1323  if (!flush()) {
1324  temp.close();
1325  temp_filename.unlink();
1326  return false;
1327  }
1328 
1329  // Now close everything, and move the temporary file back over our original
1330  // file.
1331  Filename orig_name = _multifile_name;
1332  temp.close();
1333  close();
1334  orig_name.unlink();
1335  if (!temp_filename.rename_to(orig_name)) {
1336  express_cat.info()
1337  << "Unable to rename temporary file " << temp_filename << " to "
1338  << orig_name << ".\n";
1339  return false;
1340  }
1341 
1342  if (!open_read_write(orig_name)) {
1343  express_cat.info()
1344  << "Unable to read newly repacked " << _multifile_name
1345  << ".\n";
1346  return false;
1347  }
1348 
1349  return true;
1350 }
1351 
1352 /**
1353  * Returns the number of subfiles within the Multifile. The subfiles may be
1354  * accessed in alphabetical order by iterating through [0 ..
1355  * get_num_subfiles()).
1356  */
1357 int Multifile::
1358 get_num_subfiles() const {
1359  return _subfiles.size();
1360 }
1361 
1362 /**
1363  * Returns the index of the subfile with the indicated name, or -1 if the
1364  * named subfile is not within the Multifile.
1365  */
1367 find_subfile(const string &subfile_name) const {
1368  Subfile find_subfile;
1369  find_subfile._name = standardize_subfile_name(subfile_name);
1370  Subfiles::const_iterator fi;
1371  fi = _subfiles.find(&find_subfile);
1372  if (fi == _subfiles.end()) {
1373  // Not present.
1374  return -1;
1375  }
1376  return (fi - _subfiles.begin());
1377 }
1378 
1379 /**
1380  * Returns true if the indicated subfile name is the directory prefix to one
1381  * or more files within the Multifile. That is, the Multifile contains at
1382  * least one file named "subfile_name/...".
1383  */
1385 has_directory(const string &subfile_name) const {
1386  string prefix = subfile_name;
1387  if (!prefix.empty()) {
1388  prefix += '/';
1389  }
1390  Subfile find_subfile;
1391  find_subfile._name = prefix;
1392  Subfiles::const_iterator fi;
1393  fi = _subfiles.upper_bound(&find_subfile);
1394  if (fi == _subfiles.end()) {
1395  // Not present.
1396  return false;
1397  }
1398 
1399  // At least one subfile exists whose name sorts after prefix. If it
1400  // contains prefix as the initial substring, then we have a match.
1401  Subfile *subfile = (*fi);
1402  return (subfile->_name.length() > prefix.length() &&
1403  subfile->_name.substr(0, prefix.length()) == prefix);
1404 }
1405 
1406 /**
1407  * Considers subfile_name to be the name of a subdirectory within the
1408  * Multifile, but not a file itself; fills the given vector up with the sorted
1409  * list of subdirectories or files within the named directory.
1410  *
1411  * Note that directories do not exist explicitly within a Multifile; this just
1412  * checks for the existence of files with the given initial prefix.
1413  *
1414  * Returns true if successful, false otherwise.
1415  */
1417 scan_directory(vector_string &contents, const string &subfile_name) const {
1418  string prefix = subfile_name;
1419  if (!prefix.empty()) {
1420  prefix += '/';
1421  }
1422  Subfile find_subfile;
1423  find_subfile._name = prefix;
1424  Subfiles::const_iterator fi;
1425  fi = _subfiles.upper_bound(&find_subfile);
1426 
1427  string previous = "";
1428  while (fi != _subfiles.end()) {
1429  Subfile *subfile = (*fi);
1430  if (!(subfile->_name.length() > prefix.length() &&
1431  subfile->_name.substr(0, prefix.length()) == prefix)) {
1432  // We've reached the end of the list of subfiles beneath the indicated
1433  // directory prefix.
1434  return true;
1435  }
1436 
1437  size_t slash = subfile->_name.find('/', prefix.length());
1438  string basename = subfile->_name.substr(prefix.length(), slash - prefix.length());
1439  if (basename != previous) {
1440  contents.push_back(basename);
1441  previous = basename;
1442  }
1443  ++fi;
1444  }
1445 
1446  return true;
1447 }
1448 
1449 /**
1450  * Removes the nth subfile from the Multifile. This will cause all subsequent
1451  * index numbers to decrease by one. The file will not actually be removed
1452  * from the disk until the next call to flush().
1453  *
1454  * Note that this does not actually remove the data from the indicated
1455  * subfile; it simply removes it from the index. The Multifile will not be
1456  * reduced in size after this operation, until the next call to repack().
1457  */
1459 remove_subfile(int index) {
1460  nassertv(is_write_valid());
1461  nassertv(index >= 0 && index < (int)_subfiles.size());
1462  Subfile *subfile = _subfiles[index];
1463  subfile->_flags |= SF_deleted;
1464  _removed_subfiles.push_back(subfile);
1465  _subfiles.erase(_subfiles.begin() + index);
1466 
1467  _timestamp = time(nullptr);
1468  _timestamp_dirty = true;
1469 
1470  _needs_repack = true;
1471 }
1472 
1473 /**
1474  * Returns the name of the nth subfile.
1475  */
1476 const string &Multifile::
1477 get_subfile_name(int index) const {
1478 #ifndef NDEBUG
1479  static string empty_string;
1480  nassertr(index >= 0 && index < (int)_subfiles.size(), empty_string);
1481 #endif
1482  return _subfiles[index]->_name;
1483 }
1484 
1485 /**
1486  * Returns the uncompressed data length of the nth subfile. This might return
1487  * 0 if the subfile has recently been added and flush() has not yet been
1488  * called.
1489  */
1491 get_subfile_length(int index) const {
1492  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
1493  return _subfiles[index]->_uncompressed_length;
1494 }
1495 
1496 /**
1497  * Returns the modification time of the nth subfile. If this is called on an
1498  * older .mf file, which did not store individual timestamps in the file (or
1499  * if get_record_timestamp() is false), this will return the modification time
1500  * of the overall multifile.
1501  */
1503 get_subfile_timestamp(int index) const {
1504  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
1505  if (!get_record_timestamp()) {
1506  return get_timestamp();
1507  } else {
1508  return _subfiles[index]->_timestamp;
1509  }
1510 }
1511 
1512 /**
1513  * Returns true if the indicated subfile has been compressed when stored
1514  * within the archive, false otherwise.
1515  */
1517 is_subfile_compressed(int index) const {
1518  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1519  return (_subfiles[index]->_flags & SF_compressed) != 0;
1520 }
1521 
1522 /**
1523  * Returns true if the indicated subfile has been encrypted when stored within
1524  * the archive, false otherwise.
1525  */
1527 is_subfile_encrypted(int index) const {
1528  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1529  return (_subfiles[index]->_flags & SF_encrypted) != 0;
1530 }
1531 
1532 /**
1533  * Returns true if the indicated subfile represents text data, or false if it
1534  * represents binary data. If the file is text data, it may have been
1535  * processed by end-of-line conversion when it was added. (But the actual
1536  * bits in the multifile will represent the standard Unix end-of-line
1537  * convention, e.g. \n instead of \r\n.)
1538  */
1540 is_subfile_text(int index) const {
1541  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1542  return (_subfiles[index]->_flags & SF_text) != 0;
1543 }
1544 
1545 /**
1546  * Returns the first byte that is guaranteed to follow any index byte already
1547  * written to disk in the Multifile.
1548  *
1549  * This number is largely meaningless in many cases, but if needs_repack() is
1550  * false, and the file is flushed, this will indicate the number of bytes in
1551  * the header + index. Everything at this byte position and later will be
1552  * actual data.
1553  */
1554 streampos Multifile::
1555 get_index_end() const {
1556  return normalize_streampos(_next_index + (streampos)4);
1557 }
1558 
1559 /**
1560  * Returns the starting byte position within the Multifile at which the
1561  * indicated subfile begins. This may be used, with
1562  * get_subfile_internal_length(), for low-level access to the subfile, but
1563  * usually it is better to use open_read_subfile() instead (which
1564  * automatically decrypts and/or uncompresses the subfile data).
1565  */
1566 streampos Multifile::
1567 get_subfile_internal_start(int index) const {
1568  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
1569  return _subfiles[index]->_data_start;
1570 }
1571 
1572 /**
1573  * Returns the number of bytes the indicated subfile consumes within the
1574  * archive. For compressed subfiles, this will generally be smaller than
1575  * get_subfile_length(); for encrypted (but noncompressed) subfiles, it may be
1576  * slightly different, for noncompressed and nonencrypted subfiles, it will be
1577  * equal.
1578  */
1580 get_subfile_internal_length(int index) const {
1581  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
1582  return _subfiles[index]->_data_length;
1583 }
1584 
1585 /**
1586  * Returns an istream that may be used to read the indicated subfile. You may
1587  * seek() within this istream to your heart's content; even though it will be
1588  * a reference to the already-opened pfstream of the Multifile itself, byte 0
1589  * appears to be the beginning of the subfile and EOF appears to be the end of
1590  * the subfile.
1591  *
1592  * The returned istream will have been allocated via new; you should pass the
1593  * pointer to close_read_subfile() when you are finished with it to delete it
1594  * and release its resources.
1595  *
1596  * Any future calls to repack() or close() (or the Multifile destructor) will
1597  * invalidate all currently open subfile pointers.
1598  *
1599  * The return value will be NULL if the stream cannot be opened for some
1600  * reason.
1601  */
1602 istream *Multifile::
1603 open_read_subfile(int index) {
1604  nassertr(is_read_valid(), nullptr);
1605  nassertr(index >= 0 && index < (int)_subfiles.size(), nullptr);
1606  Subfile *subfile = _subfiles[index];
1607 
1608  if (subfile->_source != nullptr ||
1609  !subfile->_source_filename.empty()) {
1610  // The subfile has not yet been copied into the physical Multifile. Force
1611  // a flush operation to incorporate it.
1612  flush();
1613 
1614  // That shouldn't change the subfile index or delete the subfile pointer.
1615  nassertr(subfile == _subfiles[index], nullptr);
1616  }
1617 
1618  return open_read_subfile(subfile);
1619 }
1620 
1621 /**
1622  * Closes a file opened by a previous call to open_read_subfile(). This
1623  * really just deletes the istream pointer, but it is recommended to use this
1624  * interface instead of deleting it explicitly, to help work around compiler
1625  * issues.
1626  */
1628 close_read_subfile(istream *stream) {
1629  if (stream != nullptr) {
1630  // For some reason--compiler bug in gcc 3.2?--explicitly deleting the
1631  // stream pointer does not call the appropriate global delete function;
1632  // instead apparently calling the system delete function. So we call the
1633  // delete function by hand instead.
1634 #if !defined(WIN32_VC) && !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW)
1635  stream->~istream();
1636  (*global_operator_delete)(stream);
1637 #else
1638  delete stream;
1639 #endif
1640  }
1641 }
1642 
1643 /**
1644  * Extracts the nth subfile into a file with the given name.
1645  */
1647 extract_subfile(int index, const Filename &filename) {
1648  nassertr(is_read_valid(), false);
1649  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1650 
1651  Filename fname = filename;
1652  if (multifile_always_binary) {
1653  fname.set_binary();
1654  }
1655 
1656  if (!fname.is_binary_or_text()) {
1657  // If we haven't specified binary or text, infer it from the type of the
1658  // subfile.
1659  if ((_subfiles[index]->_flags & SF_text) != 0) {
1660  fname.set_text();
1661  } else {
1662  fname.set_binary();
1663  }
1664  }
1665  fname.make_dir();
1666  pofstream out;
1667  if (!fname.open_write(out, true)) {
1668  express_cat.info()
1669  << "Unable to write to file " << filename << "\n";
1670  return false;
1671  }
1672 
1673  return extract_subfile_to(index, out);
1674 }
1675 
1676 /**
1677  * Extracts the nth subfile to the indicated ostream.
1678  */
1680 extract_subfile_to(int index, ostream &out) {
1681  nassertr(is_read_valid(), false);
1682  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1683 
1684  istream *in = open_read_subfile(index);
1685  if (in == nullptr) {
1686  return false;
1687  }
1688 
1689  static const size_t buffer_size = 4096;
1690  char buffer[buffer_size];
1691 
1692  in->read(buffer, buffer_size);
1693  size_t count = in->gcount();
1694  while (count != 0) {
1695  out.write(buffer, count);
1696  in->read(buffer, buffer_size);
1697  count = in->gcount();
1698  }
1699 
1700  bool failed = (in->fail() && !in->eof());
1701  close_read_subfile(in);
1702  nassertr(!failed, false);
1703 
1704  return (!out.fail());
1705 }
1706 
1707 /**
1708  * Performs a byte-for-byte comparison of the indicated file on disk with the
1709  * nth subfile. Returns true if the files are equivalent, or false if they
1710  * are different (or the file is missing).
1711  *
1712  * If Filename::set_binary() or set_text() has already been called, it
1713  * specifies the nature of the source file. If this is different from the
1714  * text flag of the subfile, the comparison will always return false. If this
1715  * has not been specified, it will be set from the text flag of the subfile.
1716  */
1718 compare_subfile(int index, const Filename &filename) {
1719  nassertr(is_read_valid(), false);
1720  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1721 
1722  if (!filename.exists()) {
1723  express_cat.info()
1724  << "File is missing: " << filename << "\n";
1725  return false;
1726  }
1727 
1728  Filename fname = filename;
1729  if (fname.is_binary()) {
1730  // If we've specified a binary file, it had better be a binary subfile.
1731  if ((_subfiles[index]->_flags & SF_text) != 0) {
1732  if (express_cat.is_debug()) {
1733  express_cat.debug()
1734  << "File is not binary: " << filename << "\n";
1735  }
1736  return false;
1737  }
1738 
1739  } else if (fname.is_text()) {
1740  // If we've specified a text file, it had better be a text subfile.
1741  if ((_subfiles[index]->_flags & SF_text) == 0) {
1742  if (express_cat.is_debug()) {
1743  express_cat.debug()
1744  << "File is not text: " << filename << "\n";
1745  }
1746  return false;
1747  }
1748 
1749  } else {
1750  // If we haven't specified binary or text, infer it from the type of the
1751  // subfile.
1752  if ((_subfiles[index]->_flags & SF_text) != 0) {
1753  fname.set_text();
1754  } else {
1755  fname.set_binary();
1756  }
1757  }
1758 
1759  istream *in1 = open_read_subfile(index);
1760  if (in1 == nullptr) {
1761  return false;
1762  }
1763 
1764  pifstream in2;
1765 
1766  if (!fname.open_read(in2)) {
1767  express_cat.info()
1768  << "Cannot read " << filename << "\n";
1769  return false;
1770  }
1771 
1772  if (fname.is_binary()) {
1773  // Check the file size.
1774  in2.seekg(0, ios::end);
1775  streampos file_size = in2.tellg();
1776 
1777  if (file_size != (streampos)get_subfile_length(index)) {
1778  // The files have different sizes.
1779  close_read_subfile(in1);
1780  return false;
1781  }
1782  }
1783 
1784  // Check the file data, byte-for-byte.
1785  in2.seekg(0);
1786  int byte1 = in1->get();
1787  int byte2 = in2.get();
1788  while (!in1->fail() && !in2.fail()) {
1789  if (byte1 != byte2) {
1790  close_read_subfile(in1);
1791  return false;
1792  }
1793  byte1 = in1->get();
1794  byte2 = in2.get();
1795  }
1796 
1797  bool failed = (in1->fail() && !in1->eof()) || (in2.fail() && !in2.eof());
1798  close_read_subfile(in1);
1799 
1800  nassertr(!failed, false);
1801 
1802  return true;
1803 }
1804 
1805 /**
1806  *
1807  */
1808 void Multifile::
1809 output(ostream &out) const {
1810  out << "Multifile " << _multifile_name << ", " << get_num_subfiles()
1811  << " subfiles.\n";
1812 }
1813 
1814 /**
1815  * Shows a list of all subfiles within the Multifile.
1816  */
1818 ls(ostream &out) const {
1819  int num_subfiles = get_num_subfiles();
1820  for (int i = 0; i < num_subfiles; i++) {
1821  string subfile_name = get_subfile_name(i);
1822  out << subfile_name << "\n";
1823  }
1824 }
1825 
1826 /**
1827  * Sets the string which is written to the Multifile before the Multifile
1828  * header. This string must begin with a hash mark and end with a newline
1829  * character; and if it includes embedded newline characters, each one must be
1830  * followed by a hash mark. If these conditions are not initially true, the
1831  * string will be modified as necessary to make it so.
1832  *
1833  * This is primarily useful as a simple hack to allow p3d applications to be
1834  * run directly from the command line on Unix-like systems.
1835  *
1836  * The return value is true if successful, or false on failure (for instance,
1837  * because the header prefix violates the above rules).
1838  */
1840 set_header_prefix(const string &header_prefix) {
1841  string new_header_prefix = header_prefix;
1842 
1843  if (!new_header_prefix.empty()) {
1844  // It must begin with a hash mark.
1845  if (new_header_prefix[0] != '#') {
1846  new_header_prefix = string("#") + new_header_prefix;
1847  }
1848 
1849  // It must end with a newline.
1850  if (new_header_prefix[new_header_prefix.size() - 1] != '\n') {
1851  new_header_prefix += string("\n");
1852  }
1853 
1854  // Embedded newlines must be followed by a hash mark.
1855  size_t newline = new_header_prefix.find('\n');
1856  while (newline < new_header_prefix.size() - 1) {
1857  if (new_header_prefix[newline + 1] != '#') {
1858  new_header_prefix = new_header_prefix.substr(0, newline + 1) + string("#") + new_header_prefix.substr(newline + 1);
1859  }
1860  newline = new_header_prefix.find('#', newline);
1861  }
1862  }
1863 
1864  if (_header_prefix != new_header_prefix) {
1865  _header_prefix = new_header_prefix;
1866  _needs_repack = true;
1867  }
1868 }
1869 
1870 
1871 /**
1872  * Fills a string with the entire contents of the indicated subfile.
1873  */
1875 read_subfile(int index, string &result) {
1876  result = string();
1877 
1878  // We use a temporary pvector, because dynamic accumulation of a pvector
1879  // seems to be many times faster than that of a string, at least on the
1880  // Windows implementation of STL.
1881  vector_uchar pv;
1882  if (!read_subfile(index, pv)) {
1883  return false;
1884  }
1885 
1886  if (!pv.empty()) {
1887  result.append((const char *)&pv[0], pv.size());
1888  }
1889 
1890  return true;
1891 }
1892 
1893 /**
1894  * Fills a pvector with the entire contents of the indicated subfile.
1895  */
1897 read_subfile(int index, vector_uchar &result) {
1898  nassertr(is_read_valid(), false);
1899  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1900  result.clear();
1901 
1902  // Now look up the particular Subfile we are reading.
1903  nassertr(is_read_valid(), false);
1904  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
1905  Subfile *subfile = _subfiles[index];
1906 
1907  if (subfile->_source != nullptr ||
1908  !subfile->_source_filename.empty()) {
1909  // The subfile has not yet been copied into the physical Multifile. Force
1910  // a flush operation to incorporate it.
1911  flush();
1912 
1913  // That shouldn't change the subfile index or delete the subfile pointer.
1914  nassertr(subfile == _subfiles[index], false);
1915  }
1916 
1917  result.reserve(subfile->_uncompressed_length);
1918 
1919  bool success = true;
1920  if (subfile->_flags & (SF_encrypted | SF_compressed)) {
1921  // If the subfile is encrypted or compressed, we can't read it directly.
1922  // Fall back to the generic implementation.
1923  istream *in = open_read_subfile(index);
1924  if (in == nullptr) {
1925  return false;
1926  }
1927 
1928  success = VirtualFile::simple_read_file(in, result);
1929  close_read_subfile(in);
1930 
1931  } else {
1932  // But if the subfile is just a plain file, we can just read the data
1933  // directly from the Multifile, without paying the cost of an ISubStream.
1934  static const size_t buffer_size = 4096;
1935  char buffer[buffer_size];
1936 
1937  streamsize pos = _offset + subfile->_data_start;
1938  size_t max_bytes = subfile->_data_length;
1939  streamsize count = 0;
1940  bool eof = true;
1941 
1942  streamsize num_bytes = (streamsize)min(buffer_size, max_bytes);
1943  _read->seek_read(pos, buffer, num_bytes, count, eof);
1944  while (count != 0) {
1945  thread_consider_yield();
1946  nassertr(count <= (streamsize)max_bytes, false);
1947  result.insert(result.end(), buffer, buffer + (size_t)count);
1948  max_bytes -= (size_t)count;
1949  pos += count;
1950 
1951  num_bytes = (streamsize)min(buffer_size, max_bytes);
1952  _read->seek_read(pos, buffer, num_bytes, count, eof);
1953  }
1954 
1955  success = !eof;
1956  }
1957 
1958  if (!success) {
1959  ostringstream message;
1960  message << "I/O error reading from " << get_multifile_name() << " at "
1961  << get_subfile_name(index);
1962  nassert_raise(message.str());
1963  return false;
1964  }
1965 
1966  return true;
1967 }
1968 
1969 /**
1970  * Assumes the _write pointer is at the indicated fpos, rounds the fpos up to
1971  * the next legitimate address (using normalize_streampos()), and writes
1972  * enough zeroes to the stream to fill the gap. Returns the new fpos.
1973  */
1974 streampos Multifile::
1975 pad_to_streampos(streampos fpos) {
1976  nassertr(_write != nullptr, fpos);
1977  nassertr(_write->tellp() == fpos, fpos);
1978  streampos new_fpos = normalize_streampos(fpos);
1979  while (fpos < new_fpos) {
1980  _write->put(0);
1981  fpos += 1; // VC++ doesn't define streampos++ (!)
1982  }
1983  nassertr(_write->tellp() == fpos, fpos);
1984  return fpos;
1985 }
1986 
1987 /**
1988  * Adds a newly-allocated Subfile pointer to the Multifile.
1989  */
1990 void Multifile::
1991 add_new_subfile(Subfile *subfile, int compression_level) {
1992  if (compression_level != 0) {
1993 #ifndef HAVE_ZLIB
1994  express_cat.warning()
1995  << "zlib not compiled in; cannot generated compressed multifiles.\n";
1996  compression_level = 0;
1997 #else // HAVE_ZLIB
1998  subfile->_flags |= SF_compressed;
1999  subfile->_compression_level = compression_level;
2000 #endif // HAVE_ZLIB
2001  }
2002 
2003 #ifdef HAVE_OPENSSL
2004  if (_encryption_flag) {
2005  subfile->_flags |= SF_encrypted;
2006  }
2007 #endif // HAVE_OPENSSL
2008 
2009  if (_next_index != (streampos)0) {
2010  // If we're adding a Subfile to an already-existing Multifile, we will
2011  // eventually need to repack the file.
2012  _needs_repack = true;
2013  }
2014 
2015  std::pair<Subfiles::iterator, bool> insert_result = _subfiles.insert(subfile);
2016  if (!insert_result.second) {
2017  // Hmm, unable to insert. There must already be a subfile by that name.
2018  // Remove the old one.
2019  Subfile *old_subfile = (*insert_result.first);
2020  old_subfile->_flags |= SF_deleted;
2021 
2022  // Maybe it was just added to the _new_subfiles list. In this case,
2023  // remove it from that list.
2024  PendingSubfiles::iterator ni = find(_new_subfiles.begin(), _new_subfiles.end(), old_subfile);
2025  if (ni != _new_subfiles.end()) {
2026  _new_subfiles.erase(ni);
2027 
2028  } else {
2029  // Otherwise, add it to the _removed_subfiles list, so we can remove the
2030  // old one.
2031  _removed_subfiles.push_back(old_subfile);
2032  }
2033 
2034  (*insert_result.first) = subfile;
2035  }
2036 
2037  _new_subfiles.push_back(subfile);
2038 }
2039 
2040 /**
2041  * This variant of open_read_subfile() is used internally only, and accepts a
2042  * pointer to the internal Subfile object, which is assumed to be valid and
2043  * written to the multifile.
2044  */
2045 istream *Multifile::
2046 open_read_subfile(Subfile *subfile) {
2047  nassertr(subfile->_source == nullptr &&
2048  subfile->_source_filename.empty(), nullptr);
2049 
2050  // Return an ISubStream object that references into the open Multifile
2051  // istream.
2052  nassertr(subfile->_data_start != (streampos)0, nullptr);
2053  istream *stream =
2054  new ISubStream(_read, _offset + subfile->_data_start,
2055  _offset + subfile->_data_start + (streampos)subfile->_data_length);
2056 
2057  if ((subfile->_flags & SF_encrypted) != 0) {
2058 #ifndef HAVE_OPENSSL
2059  express_cat.error()
2060  << "OpenSSL not compiled in; cannot read encrypted multifiles.\n";
2061  delete stream;
2062  return nullptr;
2063 #else // HAVE_OPENSSL
2064  // The subfile is encrypted. So actually, return an IDecryptStream that
2065  // wraps around the ISubStream.
2066  IDecryptStream *wrapper =
2067  new IDecryptStream(stream, true, _encryption_password);
2068  stream = wrapper;
2069 
2070  // Validate the password by confirming that the encryption header matches.
2071  char this_header[_encrypt_header_size];
2072  stream->read(this_header, _encrypt_header_size);
2073  if (stream->fail() || stream->gcount() != (unsigned)_encrypt_header_size ||
2074  memcmp(this_header, _encrypt_header, _encrypt_header_size) != 0) {
2075  express_cat.error()
2076  << "Unable to decrypt subfile " << subfile->_name << ".\n";
2077  delete stream;
2078  return nullptr;
2079  }
2080 #endif // HAVE_OPENSSL
2081  }
2082 
2083  if ((subfile->_flags & SF_compressed) != 0) {
2084 #ifndef HAVE_ZLIB
2085  express_cat.error()
2086  << "zlib not compiled in; cannot read compressed multifiles.\n";
2087  delete stream;
2088  return nullptr;
2089 #else // HAVE_ZLIB
2090  // Oops, the subfile is compressed. So actually, return an
2091  // IDecompressStream that wraps around the ISubStream.
2092  IDecompressStream *wrapper = new IDecompressStream(stream, true);
2093  stream = wrapper;
2094 #endif // HAVE_ZLIB
2095  }
2096 
2097  if (stream->fail()) {
2098  // Hmm, some inexplicable problem.
2099  delete stream;
2100  return nullptr;
2101  }
2102 
2103  return stream;
2104 }
2105 
2106 /**
2107  * Returns the standard form of the subfile name.
2108  */
2109 string Multifile::
2110 standardize_subfile_name(const string &subfile_name) const {
2111  Filename name = subfile_name;
2112  name.standardize();
2113  if (name.empty() || name == "/") {
2114  // Invalid empty name.
2115  return string();
2116  }
2117 
2118  if (name[0] == '/') {
2119  return name.get_fullpath().substr(1);
2120  } else if (name.length() > 2 && name[0] == '.' && name[1] == '/') {
2121  return name.get_fullpath().substr(2);
2122  } else {
2123  return name.get_fullpath();
2124  }
2125 }
2126 
2127 /**
2128  * Removes the set of subfiles from the tables and frees their associated
2129  * memory.
2130  */
2131 void Multifile::
2132 clear_subfiles() {
2133  PendingSubfiles::iterator pi;
2134  for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
2135  Subfile *subfile = (*pi);
2136  subfile->rewrite_index_flags(*_write);
2137  delete subfile;
2138  }
2139  _removed_subfiles.clear();
2140 
2141  // We don't have to delete the ones in _new_subfiles, because these also
2142  // appear in _subfiles.
2143  _new_subfiles.clear();
2144 
2145 #ifdef HAVE_OPENSSL
2146  for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) {
2147  Subfile *subfile = (*pi);
2148  delete subfile;
2149  }
2150  _cert_special.clear();
2151 
2152  _signatures.clear();
2153 #endif // HAVE_OPENSSL
2154 
2155  Subfiles::iterator fi;
2156  for (fi = _subfiles.begin(); fi != _subfiles.end(); ++fi) {
2157  Subfile *subfile = (*fi);
2158  delete subfile;
2159  }
2160  _subfiles.clear();
2161 }
2162 
2163 /**
2164  * Reads the Multifile header and index. Returns true if successful, false if
2165  * the Multifile is not valid.
2166  */
2167 bool Multifile::
2168 read_index() {
2169  nassertr(_read != nullptr, false);
2170 
2171  // We acquire the IStreamWrapper lock for the duration of this method.
2172  _read->acquire();
2173  istream *read = _read->get_istream();
2174 
2175  char this_header[_header_size];
2176  read->seekg(_offset);
2177 
2178  // Here's a special case: if the multifile begins with a hash character,
2179  // then we continue reading and discarding lines of ASCII text, until we
2180  // come across a nonempty line that does not begin with a hash character.
2181  // This allows a P3D application (which is a multifile) to be run directly
2182  // on the command line on Unix-based systems.
2183  _header_prefix = string();
2184  int ch = read->get();
2185 
2186  if (ch == '#') {
2187  while (ch != EOF && ch == '#') {
2188  // Skip to the end of the line.
2189  while (ch != EOF && ch != '\n') {
2190  _header_prefix += ch;
2191  ch = read->get();
2192  }
2193  // Skip to the first non-whitespace character of the line.
2194  while (ch != EOF && (isspace(ch) || ch == '\r')) {
2195  _header_prefix += ch;
2196  ch = read->get();
2197  }
2198  }
2199  }
2200 
2201  // Now read the actual Multifile header.
2202  this_header[0] = ch;
2203  read->read(this_header + 1, _header_size - 1);
2204  if (read->fail() || read->gcount() != (unsigned)(_header_size - 1)) {
2205  express_cat.info()
2206  << "Unable to read Multifile header " << _multifile_name << ".\n";
2207  _read->release();
2208  close();
2209  return false;
2210  }
2211 
2212  if (memcmp(this_header, _header, _header_size) != 0) {
2213  express_cat.info()
2214  << _multifile_name << " is not a Multifile.\n";
2215  _read->release();
2216  close();
2217  return false;
2218  }
2219 
2220  // Now get the version numbers out.
2221  StreamReader reader(read, false);
2222  _file_major_ver = reader.get_int16();
2223  _file_minor_ver = reader.get_int16();
2224  _scale_factor = reader.get_uint32();
2225  _new_scale_factor = _scale_factor;
2226 
2227  if (read->eof() || read->fail()) {
2228  express_cat.info()
2229  << _multifile_name << " header is truncated.\n";
2230  _read->release();
2231  close();
2232  return false;
2233  }
2234 
2235  if (_file_major_ver != _current_major_ver ||
2236  (_file_major_ver == _current_major_ver &&
2237  _file_minor_ver > _current_minor_ver)) {
2238  express_cat.info()
2239  << _multifile_name << " has version " << _file_major_ver << "."
2240  << _file_minor_ver << ", expecting version "
2241  << _current_major_ver << "." << _current_minor_ver << ".\n";
2242  _read->release();
2243  close();
2244  return false;
2245  }
2246 
2247  _record_timestamp = true;
2248  if (_file_minor_ver >= 1) {
2249  time_t read_timestamp = reader.get_uint32();
2250  if (read_timestamp == 0) {
2251  // If we read a 0 timestamp from the file, that implies that we don't
2252  // want to record a timestamp in this particular file.
2253  _record_timestamp = false;
2254  } else {
2255  _timestamp = read_timestamp;
2256  }
2257  _timestamp_dirty = false;
2258  }
2259 
2260  // Now read the index out.
2261  streampos curr_pos = read->tellg() - _offset;
2262  _next_index = normalize_streampos(curr_pos);
2263  if (_next_index > curr_pos) {
2264  read->ignore(_next_index - curr_pos);
2265  }
2266  _last_index = 0;
2267  _last_data_byte = 0;
2268  streampos index_forward;
2269  streamoff bytes_skipped = 0;
2270  bool read_cert_special = false;
2271 
2272  Subfile *subfile = new Subfile;
2273  index_forward = subfile->read_index(*read, _next_index, this);
2274  while (index_forward != (streampos)0) {
2275  _last_index = _next_index;
2276  if (subfile->is_deleted()) {
2277  // Ignore deleted Subfiles in the index.
2278  _needs_repack = true;
2279  delete subfile;
2280  } else if (subfile->is_cert_special()) {
2281  // Certificate chains and signature files get stored in a special list.
2282  _cert_special.push_back(subfile);
2283  read_cert_special = true;
2284  } else {
2285  _subfiles.push_back(subfile);
2286  }
2287  if (!subfile->is_cert_special()) {
2288  if (bytes_skipped != 0) {
2289  // If the index entries don't follow exactly sequentially (except for
2290  // the cert special files), the file ought to be repacked.
2291  _needs_repack = true;
2292  }
2293  if (read_cert_special) {
2294  // If we read a normal subfile following a cert_special entry, the
2295  // file ought to be repacked (certificates have to go at the end).
2296  _needs_repack = true;
2297  }
2298  _last_data_byte = max(_last_data_byte, subfile->get_last_byte_pos());
2299  }
2300  streampos curr_pos = read->tellg() - _offset;
2301  bytes_skipped = index_forward - normalize_streampos(curr_pos);
2302  _next_index = index_forward;
2303  if (_next_index > curr_pos) {
2304  read->ignore(_next_index - curr_pos);
2305  }
2306  subfile = new Subfile;
2307  index_forward = subfile->read_index(*read, _next_index, this);
2308  }
2309  if (subfile->is_index_invalid()) {
2310  express_cat.info()
2311  << "Error reading index for " << _multifile_name << ".\n";
2312  _read->release();
2313  close();
2314  delete subfile;
2315  return false;
2316  }
2317 
2318  // Check if the list is already sorted. If it is not, we need a repack.
2319  for (size_t si = 1; si < _subfiles.size() && !_needs_repack; ++si) {
2320  if (*_subfiles[si] < *_subfiles[si - 1]) {
2321  _needs_repack = true;
2322  }
2323  }
2324 
2325  if (_needs_repack) {
2326  // At least sort them now.
2327  size_t before_size = _subfiles.size();
2328  _subfiles.sort();
2329  size_t after_size = _subfiles.size();
2330 
2331  // If these don't match, the same filename appeared twice in the index,
2332  // which shouldn't be possible.
2333  nassertr(before_size == after_size, true);
2334  }
2335 
2336  delete subfile;
2337  _read->release();
2338  return true;
2339 }
2340 
2341 /**
2342  * Writes just the header part of the Multifile, not the index.
2343  */
2344 bool Multifile::
2345 write_header() {
2346  _file_major_ver = _current_major_ver;
2347  _file_minor_ver = _current_minor_ver;
2348 
2349  nassertr(_write != nullptr, false);
2350  nassertr(_write->tellp() == (streampos)0, false);
2351  _write->write(_header_prefix.data(), _header_prefix.size());
2352  _write->write(_header, _header_size);
2353  StreamWriter writer(_write, false);
2354  writer.add_int16(_current_major_ver);
2355  writer.add_int16(_current_minor_ver);
2356  writer.add_uint32(_scale_factor);
2357 
2358  if (_record_timestamp) {
2359  writer.add_uint32(_timestamp);
2360  } else {
2361  writer.add_uint32(0);
2362  _timestamp_dirty = false;
2363  }
2364 
2365  _next_index = _write->tellp();
2366  _next_index = pad_to_streampos(_next_index);
2367  _last_index = 0;
2368 
2369  if (_write->fail()) {
2370  express_cat.info()
2371  << "Unable to write header for " << _multifile_name << ".\n";
2372  close();
2373  return false;
2374  }
2375 
2376  return true;
2377 }
2378 
2379 /**
2380  * Walks through the list of _cert_special entries in the Multifile, moving
2381  * any valid signatures found to _signatures. After this call, _cert_special
2382  * will be empty.
2383  *
2384  * This does not check the validity of the certificates themselves. It only
2385  * checks that they correctly sign the Multifile contents.
2386  */
2387 void Multifile::
2388 check_signatures() {
2389 #ifdef HAVE_OPENSSL
2390  PendingSubfiles::iterator pi;
2391 
2392  for (pi = _cert_special.begin(); pi != _cert_special.end(); ++pi) {
2393  Subfile *subfile = (*pi);
2394  nassertv((subfile->_flags & SF_signature) != 0);
2395 
2396  // Extract the signature data and certificate separately.
2397  istream *stream = open_read_subfile(subfile);
2398  nassertv(stream != nullptr);
2399  StreamReader reader(*stream);
2400  size_t sig_size = reader.get_uint32();
2401  vector_uchar sig_data = reader.extract_bytes(sig_size);
2402 
2403  size_t num_certs = reader.get_uint32();
2404 
2405  // Read the remaining buffer of certificate data.
2406  vector_uchar buffer;
2407  bool success = VirtualFile::simple_read_file(stream, buffer);
2408  nassertv(success);
2409  close_read_subfile(stream);
2410 
2411  // Now convert each of the certificates to an X509 object, and store it in
2412  // our CertChain.
2413  CertChain chain;
2414  EVP_PKEY *pkey = nullptr;
2415  if (!buffer.empty()) {
2416 #if OPENSSL_VERSION_NUMBER >= 0x00908000L
2417  // Beginning in 0.9.8, d2i_X509() accepted a const unsigned char **.
2418  const unsigned char *bp, *bp_end;
2419 #else
2420  // Prior to 0.9.8, d2i_X509() accepted an unsigned char **.
2421  unsigned char *bp, *bp_end;
2422 #endif
2423  bp = (unsigned char *)&buffer[0];
2424  bp_end = bp + buffer.size();
2425  X509 *x509 = d2i_X509(nullptr, &bp, bp_end - bp);
2426  while (num_certs > 0 && x509 != nullptr) {
2427  chain.push_back(CertRecord(x509));
2428  --num_certs;
2429  x509 = d2i_X509(nullptr, &bp, bp_end - bp);
2430  }
2431  if (num_certs != 0 || x509 != nullptr) {
2432  express_cat.warning()
2433  << "Extra data in signature record.\n";
2434  }
2435  }
2436 
2437  if (!chain.empty()) {
2438  pkey = X509_get_pubkey(chain[0]._cert);
2439  }
2440 
2441  if (pkey != nullptr) {
2442  EVP_MD_CTX *md_ctx = EVP_MD_CTX_create();
2443  EVP_VerifyInit(md_ctx, EVP_sha1());
2444 
2445  nassertv(_read != nullptr);
2446  _read->acquire();
2447  istream *read = _read->get_istream();
2448 
2449  // Read and hash the multifile contents, but only up till
2450  // _last_data_byte.
2451  read->seekg(_offset);
2452  streampos bytes_remaining = _last_data_byte;
2453  static const size_t buffer_size = 4096;
2454  char buffer[buffer_size];
2455  read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2456  size_t count = read->gcount();
2457  while (count != 0) {
2458  nassertv(count <= buffer_size);
2459  EVP_VerifyUpdate(md_ctx, buffer, count);
2460  bytes_remaining -= count;
2461  read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2462  count = read->gcount();
2463  }
2464  nassertv(bytes_remaining == (streampos)0);
2465  _read->release();
2466 
2467  // Now check that the signature matches the hash.
2468  int verify_result =
2469  EVP_VerifyFinal(md_ctx, sig_data.data(), sig_data.size(), pkey);
2470  if (verify_result == 1) {
2471  // The signature matches; save the certificate and its chain.
2472  _signatures.push_back(chain);
2473  } else {
2474  // Bad match.
2475  _needs_repack = true;
2476  }
2477  }
2478  }
2479 #endif // HAVE_OPENSSL
2480 
2481  _cert_special.clear();
2482 }
2483 
2484 /**
2485  * Reads the index record for the Subfile from the indicated istream. Assumes
2486  * the istream has already been positioned to the indicated stream position,
2487  * fpos, the start of the index record. Returns the position within the file
2488  * of the next index record.
2489  */
2490 streampos Multifile::Subfile::
2491 read_index(istream &read, streampos fpos, Multifile *multifile) {
2492  nassertr(read.tellg() - multifile->_offset == fpos, fpos);
2493 
2494  // First, get the next stream position. We do this separately, because if
2495  // it is zero, we don't get anything else.
2496  StreamReader reader(read);
2497 
2498  streampos next_index = multifile->word_to_streampos(reader.get_uint32());
2499  if (read.fail()) {
2500  _flags |= SF_index_invalid;
2501  return 0;
2502  }
2503 
2504  if (next_index == (streampos)0) {
2505  return 0;
2506  }
2507 
2508  // Now get the rest of the index.
2509 
2510  _index_start = fpos;
2511  _index_length = 0;
2512 
2513  _data_start = multifile->word_to_streampos(reader.get_uint32());
2514  _data_length = reader.get_uint32();
2515  _flags = reader.get_uint16();
2516  if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
2517  _uncompressed_length = reader.get_uint32();
2518  } else {
2519  _uncompressed_length = _data_length;
2520  }
2521  if (multifile->_file_minor_ver < 1) {
2522  _timestamp = multifile->get_timestamp();
2523  } else {
2524  _timestamp = reader.get_uint32();
2525  if (_timestamp == 0) {
2526  _timestamp = multifile->get_timestamp();
2527  }
2528  }
2529 
2530  size_t name_length = reader.get_uint16();
2531  if (read.fail()) {
2532  _flags |= SF_index_invalid;
2533  return 0;
2534  }
2535 
2536  // And finally, get the rest of the name.
2537  char *name_buffer = (char *)PANDA_MALLOC_ARRAY(name_length);
2538  nassertr(name_buffer != nullptr, next_index);
2539  for (size_t ni = 0; ni < name_length; ni++) {
2540  name_buffer[ni] = read.get() ^ 0xff;
2541  }
2542  _name = string(name_buffer, name_length);
2543  PANDA_FREE_ARRAY(name_buffer);
2544 
2545  if (read.fail()) {
2546  _flags |= SF_index_invalid;
2547  return 0;
2548  }
2549 
2550  _index_length = read.tellg() - fpos - multifile->_offset;
2551  return next_index;
2552 }
2553 
2554 /**
2555  * Writes the index record for the Subfile to the indicated ostream. Assumes
2556  * the istream has already been positioned to the indicated stream position,
2557  * fpos, the start of the index record, and that this is the effective end of
2558  * the file. Returns the position within the file of the next index record.
2559  *
2560  * The _index_start member is updated by this operation.
2561  */
2562 streampos Multifile::Subfile::
2563 write_index(ostream &write, streampos fpos, Multifile *multifile) {
2564  nassertr(write.tellp() - multifile->_offset == fpos, fpos);
2565 
2566  _index_start = fpos;
2567  _index_length = 0;
2568 
2569  // This will be the contents of this particular index record. We build it
2570  // up first since it will be variable length.
2571  Datagram dg;
2572  dg.add_uint32(multifile->streampos_to_word(_data_start));
2573  dg.add_uint32(_data_length);
2574  dg.add_uint16(_flags);
2575  if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
2576  dg.add_uint32(_uncompressed_length);
2577  }
2578  dg.add_uint32(_timestamp);
2579  dg.add_uint16(_name.length());
2580 
2581  // For no real good reason, we'll invert all the bits in the name. The only
2582  // reason we do this is to make it inconvenient for a casual browser of the
2583  // Multifile to discover the names of the files stored within it.
2584  // Naturally, this isn't real obfuscation or security.
2585  string::iterator ni;
2586  for (ni = _name.begin(); ni != _name.end(); ++ni) {
2587  dg.add_int8((*ni) ^ 0xff);
2588  }
2589 
2590  size_t this_index_size = 4 + dg.get_length();
2591 
2592  // Plus, we will write out the next index address first.
2593  streampos next_index = fpos + (streampos)this_index_size;
2594 
2595  Datagram idg;
2596  idg.add_uint32(multifile->streampos_to_word(next_index));
2597 
2598  write.write((const char *)idg.get_data(), idg.get_length());
2599  write.write((const char *)dg.get_data(), dg.get_length());
2600 
2601  _index_length = write.tellp() - fpos - multifile->_offset;
2602  return next_index;
2603 }
2604 
2605 /**
2606  * Writes the data record for the Subfile to the indicated ostream: the actual
2607  * contents of the Subfile. Assumes the istream has already been positioned
2608  * to the indicated stream position, fpos, the start of the data record, and
2609  * that this is the effective end of the file. Returns the position within
2610  * the file of the next data record.
2611  *
2612  * The _data_start, _data_length, and _uncompressed_length members are updated
2613  * by this operation.
2614  *
2615  * If the "read" pointer is non-NULL, it is the readable istream of a
2616  * Multifile in which the Subfile might already be packed. This is used for
2617  * reading the contents of the Subfile during a repack() operation.
2618  */
2619 streampos Multifile::Subfile::
2620 write_data(ostream &write, istream *read, streampos fpos,
2621  Multifile *multifile) {
2622  nassertr(write.tellp() - multifile->_offset == fpos, fpos);
2623 
2624  istream *source = _source;
2625  pifstream source_file;
2626  if (source == nullptr && !_source_filename.empty()) {
2627  // If we have a filename, open it up and read that.
2628  if (!_source_filename.open_read(source_file)) {
2629  // Unable to open the source file.
2630  express_cat.info()
2631  << "Unable to read " << _source_filename << ".\n";
2632  _flags |= SF_data_invalid;
2633  _data_length = 0;
2634  _uncompressed_length = 0;
2635  } else {
2636  source = &source_file;
2637  }
2638  }
2639 
2640  if (source == nullptr) {
2641  // We don't have any source data. Perhaps we're reading from an already-
2642  // packed Subfile (e.g. during repack()).
2643  if (read == nullptr) {
2644  // No, we're just screwed.
2645  express_cat.info()
2646  << "No source for subfile " << _name << ".\n";
2647  _flags |= SF_data_invalid;
2648  } else {
2649  // Read the data from the original Multifile.
2650  read->seekg(_data_start + multifile->_offset);
2651  for (size_t p = 0; p < _data_length; p++) {
2652  int byte = read->get();
2653  if (read->eof() || read->fail()) {
2654  // Unexpected EOF or other failure on the source file.
2655  express_cat.info()
2656  << "Unexpected EOF for subfile " << _name << ".\n";
2657  _flags |= SF_data_invalid;
2658  break;
2659  }
2660  write.put(byte);
2661  }
2662  }
2663  } else {
2664  // We do have source data. Copy it in, and also measure its length.
2665  ostream *putter = &write;
2666  bool delete_putter = false;
2667 
2668 #ifndef HAVE_OPENSSL
2669  // Without OpenSSL, we can't support encryption. The flag had better not
2670  // be set.
2671  nassertr((_flags & SF_encrypted) == 0, fpos);
2672 
2673 #else // HAVE_OPENSSL
2674  if ((_flags & SF_encrypted) != 0) {
2675  // Write it encrypted.
2676  OEncryptStream *encrypt = new OEncryptStream;
2677  encrypt->set_iteration_count(multifile->_encryption_iteration_count);
2678  encrypt->open(putter, delete_putter, multifile->_encryption_password);
2679 
2680  putter = encrypt;
2681  delete_putter = true;
2682 
2683  // Also write the encrypt_header to the beginning of the encrypted
2684  // stream, so we can validate the password on decryption.
2685  putter->write(_encrypt_header, _encrypt_header_size);
2686  }
2687 #endif // HAVE_OPENSSL
2688 
2689 #ifndef HAVE_ZLIB
2690  // Without ZLIB, we can't support compression. The flag had better not be
2691  // set.
2692  nassertr((_flags & SF_compressed) == 0, fpos);
2693 #else // HAVE_ZLIB
2694  if ((_flags & SF_compressed) != 0) {
2695  // Write it compressed.
2696  putter = new OCompressStream(putter, delete_putter, _compression_level);
2697  delete_putter = true;
2698  }
2699 #endif // HAVE_ZLIB
2700 
2701  streampos write_start = fpos;
2702  _uncompressed_length = 0;
2703 
2704 #ifndef HAVE_OPENSSL
2705  // We also need OpenSSL for signatures.
2706  nassertr((_flags & SF_signature) == 0, fpos);
2707 
2708 #else // HAVE_OPENSSL
2709  if ((_flags & SF_signature) != 0) {
2710  // If it's a special signature record, precede the record data (the
2711  // certificate itself) with the signature data generated against the
2712  // multifile contents.
2713 
2714  // In order to generate a signature, we need to have a valid read
2715  // pointer.
2716  nassertr(read != nullptr, fpos);
2717 
2718  // And we also need to have a private key.
2719  nassertr(_pkey != nullptr, fpos);
2720 
2721  EVP_MD_CTX *md_ctx = EVP_MD_CTX_create();
2722  EVP_SignInit(md_ctx, EVP_sha1());
2723 
2724  // Read and hash the multifile contents, but only up till
2725  // _last_data_byte.
2726  nassertr(multifile->_last_data_byte < fpos, fpos);
2727  read->seekg(multifile->_offset);
2728  streampos bytes_remaining = multifile->_last_data_byte;
2729  static const size_t buffer_size = 4096;
2730  char buffer[buffer_size];
2731  read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2732  size_t count = read->gcount();
2733  while (count != 0) {
2734  nassertr(count <= buffer_size, fpos);
2735  EVP_SignUpdate(md_ctx, buffer, count);
2736  bytes_remaining -= count;
2737  read->read(buffer, min((streampos)buffer_size, bytes_remaining));
2738  count = read->gcount();
2739  }
2740  nassertr(bytes_remaining == (streampos)0, fpos);
2741 
2742  // Now generate and write out the signature.
2743  unsigned int max_size = EVP_PKEY_size(_pkey);
2744  unsigned char *sig_data = new unsigned char[max_size];
2745  unsigned int sig_size;
2746  if (!EVP_SignFinal(md_ctx, sig_data, &sig_size, _pkey)) {
2747  OpenSSLWrapper *sslw = OpenSSLWrapper::get_global_ptr();
2748  sslw->notify_ssl_errors();
2749  }
2750  nassertr(sig_size <= max_size, fpos);
2751 
2752  StreamWriter writer(*putter);
2753  writer.add_uint32(sig_size);
2754  putter->write((char *)sig_data, sig_size);
2755  _uncompressed_length += 4 + sig_size;
2756 
2757  delete[] sig_data;
2758 
2759  EVP_MD_CTX_destroy(md_ctx);
2760  }
2761 #endif // HAVE_OPENSSL
2762 
2763  // Finally, we can write out the data itself.
2764  static const size_t buffer_size = 4096;
2765  char buffer[buffer_size];
2766 
2767  source->read(buffer, buffer_size);
2768  size_t count = source->gcount();
2769  while (count != 0) {
2770  _uncompressed_length += count;
2771  putter->write(buffer, count);
2772  source->read(buffer, buffer_size);
2773  count = source->gcount();
2774  }
2775 
2776  if (delete_putter) {
2777  delete putter;
2778  }
2779 
2780  streampos write_end = write.tellp() - multifile->_offset;
2781  _data_length = (size_t)(write_end - write_start);
2782  }
2783 
2784  // We can't set _data_start until down here, after we have read the Subfile.
2785  // (In case we are running during repack()).
2786  _data_start = fpos;
2787 
2788  // Get the modification timestamp for this subfile. This is read from the
2789  // source file, if we have a filename; otherwise, it's the current time.
2790  if (!_source_filename.empty()) {
2791  _timestamp = _source_filename.get_timestamp();
2792  }
2793  if (_timestamp == 0) {
2794  _timestamp = time(nullptr);
2795  }
2796 
2797  _source = nullptr;
2798  _source_filename = Filename();
2799  source_file.close();
2800 
2801  return fpos + (streampos)_data_length;
2802 }
2803 
2804 /**
2805  * Seeks within the indicate pfstream back to the index record and rewrites
2806  * just the _data_start and _data_length part of the index record.
2807  */
2808 void Multifile::Subfile::
2809 rewrite_index_data_start(ostream &write, Multifile *multifile) {
2810  nassertv(_index_start != (streampos)0);
2811 
2812  static const size_t data_start_offset = 4;
2813  size_t data_start_pos = _index_start + (streampos)data_start_offset;
2814  write.seekp(data_start_pos + multifile->_offset);
2815  nassertv(!write.fail());
2816 
2817  StreamWriter writer(write);
2818  writer.add_uint32(multifile->streampos_to_word(_data_start));
2819  writer.add_uint32(_data_length);
2820  writer.add_uint16(_flags);
2821  if ((_flags & (SF_compressed | SF_encrypted)) != 0) {
2822  writer.add_uint32(_uncompressed_length);
2823  }
2824  if (multifile->_record_timestamp) {
2825  writer.add_uint32(_timestamp);
2826  } else {
2827  writer.add_uint32(0);
2828  }
2829 }
2830 
2831 /**
2832  * Seeks within the indicated ostream back to the index record and rewrites
2833  * just the _flags part of the index record.
2834  */
2835 void Multifile::Subfile::
2836 rewrite_index_flags(ostream &write) {
2837  // If the subfile has never even been recorded to disk, we don't need to do
2838  // anything at all in this function.
2839  if (_index_start != (streampos)0) {
2840  static const size_t flags_offset = 4 + 4 + 4;
2841  size_t flags_pos = _index_start + (streampos)flags_offset;
2842  write.seekp(flags_pos);
2843  nassertv(!write.fail());
2844 
2845  StreamWriter writer(write);
2846  writer.add_uint16(_flags);
2847  }
2848 }
This is a convenience class to specialize ConfigVariable as an integer type.
An ordered list of data elements, formatted in memory for transmission over a socket or writing to a ...
Definition: datagram.h:38
void add_uint32(uint32_t value)
Adds an unsigned 32-bit integer to the datagram.
Definition: datagram.I:94
size_t get_length() const
Returns the number of bytes in the datagram.
Definition: datagram.I:335
void add_uint16(uint16_t value)
Adds an unsigned 16-bit integer to the datagram.
Definition: datagram.I:85
const void * get_data() const
Returns a pointer to the beginning of the datagram's data.
Definition: datagram.I:327
void add_int8(int8_t value)
Adds a signed 8-bit integer to the datagram.
Definition: datagram.I:42
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:39
bool is_text() const
Returns true if the Filename has been indicated to represent a text file via a previous call to set_t...
Definition: filename.I:456
bool open_read(std::ifstream &stream) const
Opens the indicated ifstream for reading the file, if possible.
Definition: filename.cxx:1863
bool rename_to(const Filename &other) const
Renames the file to the indicated new filename.
Definition: filename.cxx:2339
bool open_read_write(std::fstream &stream, bool truncate=false) const
Opens the indicated fstream for read/write access to the file, if possible.
Definition: filename.cxx:1977
bool is_binary() const
Returns true if the Filename has been indicated to represent a binary file via a previous call to set...
Definition: filename.I:435
bool unlink() const
Permanently deletes the file associated with the filename, if possible.
Definition: filename.cxx:2319
void standardize()
Converts the filename to standard form by replacing consecutive slashes with a single slash,...
Definition: filename.cxx:900
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:414
bool is_binary_or_text() const
Returns true either is_binary() or is_text() is true; that is, that the filename has been specified a...
Definition: filename.I:445
bool make_dir() const
Creates all the directories in the path to the file specified in the filename, except for the basenam...
Definition: filename.cxx:2484
void set_text()
Indicates that the filename represents a text file.
Definition: filename.I:424
time_t get_timestamp() const
Returns a time_t value that represents the time the file was last modified, to within whatever precis...
Definition: filename.cxx:1497
std::string get_fullpath() const
Returns the entire filename: directory, basename, extension.
Definition: filename.I:338
static Filename temporary(const std::string &dirname, const std::string &prefix, const std::string &suffix=std::string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix.
Definition: filename.cxx:424
bool open_write(std::ofstream &stream, bool truncate=true) const
Opens the indicated ifstream for writing the file, if possible.
Definition: filename.cxx:1899
std::string get_dirname() const
Returns the directory part of the filename.
Definition: filename.I:358
bool exists() const
Returns true if the filename exists on the disk, false otherwise.
Definition: filename.cxx:1267
This class provides a locking wrapper around an arbitrary istream pointer.
Definition: streamWrapper.h:59
void seek_read(std::streamsize pos, char *buffer, std::streamsize num_bytes, std::streamsize &read_bytes, bool &eof)
Atomically seeks to a particular offset from the beginning of the file, and reads a number of bytes f...
get_istream
Returns the istream this object is wrapping.
Definition: streamWrapper.h:67
An istream object that presents a subwindow into another istream.
Definition: subStream.h:30
An STL function object class, this is intended to be used on any ordered collection of pointers to cl...
Definition: indirectLess.h:25
A file that contains a set of files.
Definition: multifile.h:37
void set_header_prefix(const std::string &header_prefix)
Sets the string which is written to the Multifile before the Multifile header.
Definition: multifile.cxx:1840
const Filename & get_multifile_name() const
Returns the filename of the Multifile, if it is available.
Definition: multifile.I:18
bool compare_subfile(int index, const Filename &filename)
Performs a byte-for-byte comparison of the indicated file on disk with the nth subfile.
Definition: multifile.cxx:1718
bool is_subfile_text(int index) const
Returns true if the indicated subfile represents text data, or false if it represents binary data.
Definition: multifile.cxx:1540
bool open_read_write(const Filename &multifile_name)
Opens the named Multifile on disk for reading and writing.
Definition: multifile.cxx:258
std::string update_subfile(const std::string &subfile_name, const Filename &filename, int compression_level)
Adds a file on disk to the subfile.
Definition: multifile.cxx:491
bool has_directory(const std::string &subfile_name) const
Returns true if the indicated subfile name is the directory prefix to one or more files within the Mu...
Definition: multifile.cxx:1385
bool extract_subfile_to(int index, std::ostream &out)
Extracts the nth subfile to the indicated ostream.
Definition: multifile.cxx:1680
std::streampos get_index_end() const
Returns the first byte that is guaranteed to follow any index byte already written to disk in the Mul...
Definition: multifile.cxx:1555
void close()
Closes the Multifile if it is open.
Definition: multifile.cxx:324
bool scan_directory(vector_string &contents, const std::string &subfile_name) const
Considers subfile_name to be the name of a subdirectory within the Multifile, but not a file itself; ...
Definition: multifile.cxx:1417
get_num_subfiles
Returns the number of subfiles within the Multifile.
Definition: multifile.h:118
bool flush()
Writes all contents of the Multifile to disk.
Definition: multifile.cxx:1127
bool is_subfile_compressed(int index) const
Returns true if the indicated subfile has been compressed when stored within the archive,...
Definition: multifile.cxx:1517
static void close_read_subfile(std::istream *stream)
Closes a file opened by a previous call to open_read_subfile().
Definition: multifile.cxx:1628
int find_subfile(const std::string &subfile_name) const
Returns the index of the subfile with the indicated name, or -1 if the named subfile is not within th...
Definition: multifile.cxx:1367
time_t get_subfile_timestamp(int index) const
Returns the modification time of the nth subfile.
Definition: multifile.cxx:1503
bool get_record_timestamp() const
Returns the flag indicating whether timestamps should be recorded within the Multifile or not.
Definition: multifile.I:104
bool is_write_valid() const
Returns true if the Multifile has been opened for write mode and there have been no errors,...
Definition: multifile.I:46
void set_scale_factor(size_t scale_factor)
Changes the internal scale factor for this Multifile.
Definition: multifile.cxx:386
size_t get_subfile_length(int index) const
Returns the uncompressed data length of the nth subfile.
Definition: multifile.cxx:1491
bool is_subfile_encrypted(int index) const
Returns true if the indicated subfile has been encrypted when stored within the archive,...
Definition: multifile.cxx:1527
time_t get_timestamp() const
Returns the modification timestamp of the overall Multifile.
Definition: multifile.I:66
bool repack()
Forces a complete rewrite of the Multifile and all of its contents, so that its index will appear at ...
Definition: multifile.cxx:1281
std::streampos get_subfile_internal_start(int index) const
Returns the starting byte position within the Multifile at which the indicated subfile begins.
Definition: multifile.cxx:1567
std::istream * open_read_subfile(int index)
Returns an istream that may be used to read the indicated subfile.
Definition: multifile.cxx:1603
bool open_write(const Filename &multifile_name)
Opens the named Multifile on disk for writing.
Definition: multifile.cxx:215
size_t get_subfile_internal_length(int index) const
Returns the number of bytes the indicated subfile consumes within the archive.
Definition: multifile.cxx:1580
bool is_read_valid() const
Returns true if the Multifile has been opened for read mode and there have been no errors,...
Definition: multifile.I:37
void remove_subfile(int index)
Removes the nth subfile from the Multifile.
Definition: multifile.cxx:1459
get_subfile_name
Returns the name of the nth subfile.
Definition: multifile.h:118
vector_uchar read_subfile(int index)
Returns a vector_uchar that contains the entire contents of the indicated subfile.
Definition: multifile.I:296
void ls(std::ostream &out=std::cout) const
Shows a list of all subfiles within the Multifile.
Definition: multifile.cxx:1818
std::string add_subfile(const std::string &subfile_name, const Filename &filename, int compression_level)
Adds a file on disk as a subfile to the Multifile.
Definition: multifile.cxx:417
bool extract_subfile(int index, const Filename &filename)
Extracts the nth subfile into a file with the given name.
Definition: multifile.cxx:1647
A class to read sequential binary data directly from an istream.
Definition: streamReader.h:28
bool unref() const
Decrements the reference count.
Definition: streamWrapper.I:75
void release()
Releases the internal lock.
Definition: streamWrapper.I:53
void acquire()
Acquires the internal lock.
Definition: streamWrapper.I:38
This class provides a locking wrapper around a combination ostream/istream pointer.
A StreamWriter object is used to write sequential binary data directly to an ostream.
Definition: streamWriter.h:29
void add_uint32(uint32_t value)
Adds an unsigned 32-bit integer to the stream.
Definition: streamWriter.I:147
A hierarchy of directories and files that appears to be one continuous file system,...
PointerTo< VirtualFile > get_file(const Filename &filename, bool status_only=false) const
Looks up the file by the indicated name in the file system.
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
The abstract base class for a file or directory within the VirtualFileSystem.
Definition: virtualFile.h:35
static bool simple_read_file(std::istream *stream, vector_uchar &result)
Fills up the indicated pvector with the contents of the just-opened file.
iterator_0 begin()
Returns the iterator that marks the first element in the ordered vector.
void push_back(const value_type_0 &key)
Adds the new element to the end of the vector without regard for proper sorting.
size_type_0 size() const
Returns the number of elements in the ordered vector.
iterator_0 end()
Returns the iterator that marks the end of the ordered vector.
void clear()
Removes all elements from the ordered vector.
void sort()
Maps to sort_unique().
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.