Panda3D
bamCache.cxx
1 // Filename: bamCache.cxx
2 // Created by: drose (09Jun06)
3 //
4 ////////////////////////////////////////////////////////////////////
5 //
6 // PANDA 3D SOFTWARE
7 // Copyright (c) Carnegie Mellon University. All rights reserved.
8 //
9 // All use of this software is subject to the terms of the revised BSD
10 // license. You should have received a copy of this license along
11 // with this source code in a file named "LICENSE."
12 //
13 ////////////////////////////////////////////////////////////////////
14 
15 #include "bamCache.h"
16 #include "bamCacheIndex.h"
17 #include "bamReader.h"
18 #include "bamWriter.h"
19 #include "hashVal.h"
20 #include "datagramInputFile.h"
21 #include "datagramOutputFile.h"
22 #include "config_util.h"
23 #include "bam.h"
24 #include "typeRegistry.h"
25 #include "string_utils.h"
26 #include "configVariableInt.h"
27 #include "configVariableString.h"
28 #include "configVariableFilename.h"
29 #include "virtualFileSystem.h"
30 
31 BamCache *BamCache::_global_ptr = NULL;
32 
33 ////////////////////////////////////////////////////////////////////
34 // Function: BamCache::Constructor
35 // Access: Published
36 // Description:
37 ////////////////////////////////////////////////////////////////////
38 BamCache::
39 BamCache() :
40  _active(true),
41  _read_only(false),
42  _index(new BamCacheIndex),
43  _index_stale_since(0)
44 {
45  ConfigVariableFilename model_cache_dir
46  ("model-cache-dir", Filename(),
47  PRC_DESC("The full path to a directory, local to this computer, in which "
48  "model and texture files will be cached on load. If a directory "
49  "name is specified here, files may be loaded from the cache "
50  "instead of from their actual pathnames, which may save load time, "
51  "especially if you are loading egg files instead of bam files, "
52  "or if you are loading models from a shared network drive. "
53  "If this is the empty string, no cache will be used."));
54 
55  ConfigVariableInt model_cache_flush
56  ("model-cache-flush", 30,
57  PRC_DESC("This is the amount of time, in seconds, between automatic "
58  "flushes of the model-cache index."));
59 
60  ConfigVariableBool model_cache_models
61  ("model-cache-models", true,
62  PRC_DESC("If this is set to true, models will be cached in the "
63  "model cache, as bam files."));
64 
65  ConfigVariableBool model_cache_textures
66  ("model-cache-textures", true,
67  PRC_DESC("If this is set to true, textures will also be cached in the "
68  "model cache, as txo files."));
69 
70  ConfigVariableBool model_cache_compressed_textures
71  ("model-cache-compressed-textures", false,
72  PRC_DESC("If this is set to true, compressed textures will be cached "
73  "in the model cache, in their compressed form as downloaded "
74  "by the GSG. This may be set in conjunction with "
75  "model-cache-textures, or it may be independent."));
76 
77  ConfigVariableInt model_cache_max_kbytes
78  ("model-cache-max-kbytes", 10485760,
79  PRC_DESC("This is the maximum size of the model cache, in kilobytes."));
80 
81  _cache_models = model_cache_models;
82  _cache_textures = model_cache_textures;
83  _cache_compressed_textures = model_cache_compressed_textures;
84 
85  _flush_time = model_cache_flush;
86  _max_kbytes = model_cache_max_kbytes;
87 
88  if (!model_cache_dir.empty()) {
89  set_root(model_cache_dir);
90  }
91 }
92 
93 ////////////////////////////////////////////////////////////////////
94 // Function: BamCache::Destructor
95 // Access: Published
96 // Description:
97 ////////////////////////////////////////////////////////////////////
98 BamCache::
99 ~BamCache() {
100  flush_index();
101  delete _index;
102  _index = NULL;
103 }
104 
105 ////////////////////////////////////////////////////////////////////
106 // Function: BamCache::set_root
107 // Access: Published
108 // Description: Changes the current root pathname of the cache. This
109 // specifies where the cache files are stored on disk.
110 // This should name a directory that is on a disk local
111 // to the machine (not on a network-mounted disk), for
112 // instance, /tmp/panda-cache or /c/panda-cache.
113 //
114 // If the directory does not already exist, it will be
115 // created as a result of this call.
116 ////////////////////////////////////////////////////////////////////
117 void BamCache::
118 set_root(const Filename &root) {
119  ReMutexHolder holder(_lock);
120  flush_index();
121  _root = root;
122 
123  // The root filename must be a directory.
125  if (!vfs->is_directory(_root)) {
126  vfs->make_directory_full(_root);
127  }
128 
129  delete _index;
130  _index = new BamCacheIndex;
131  _index_stale_since = 0;
132  read_index();
133  check_cache_size();
134 
135  nassertv(vfs->is_directory(_root));
136 }
137 
138 ////////////////////////////////////////////////////////////////////
139 // Function: BamCache::lookup
140 // Access: Published
141 // Description: Looks up a file in the cache.
142 //
143 // If the file is cacheable, then regardless of whether
144 // the file is found in the cache or not, this returns a
145 // BamCacheRecord. On the other hand, if the file
146 // cannot be cached, returns NULL.
147 //
148 // If record->has_data() returns true, then the file was
149 // found in the cache, and you may call
150 // record->extract_data() to get the object. If
151 // record->has_data() returns false, then the file was
152 // not found in the cache or the cache was stale; and
153 // you should reload the source file (calling
154 // record->add_dependent_file() for each file loaded,
155 // including the original source file), and then call
156 // record->set_data() to record the resulting loaded
157 // object; and finally, you should call store() to write
158 // the cached record to disk.
159 ////////////////////////////////////////////////////////////////////
160 PT(BamCacheRecord) BamCache::
161 lookup(const Filename &source_filename, const string &cache_extension) {
162  ReMutexHolder holder(_lock);
164 
166 
167  Filename source_pathname(source_filename);
168  source_pathname.make_absolute(vfs->get_cwd());
169 
170  Filename rel_pathname(source_pathname);
171  rel_pathname.make_relative_to(_root, false);
172  if (rel_pathname.is_local()) {
173  // If the source pathname is already within the cache directory,
174  // don't cache it further.
175  return NULL;
176  }
177 
178  Filename cache_filename = hash_filename(source_pathname.get_fullpath());
179  cache_filename.set_extension(cache_extension);
180 
181  return find_and_read_record(source_pathname, cache_filename);
182 }
183 
184 ////////////////////////////////////////////////////////////////////
185 // Function: BamCache::store
186 // Access: Published
187 // Description: Flushes a cache entry to disk. You must have
188 // retrieved the cache record via a prior call to
189 // lookup(), and then stored the data via
190 // record->set_data(). Returns true on success, false
191 // on failure.
192 ////////////////////////////////////////////////////////////////////
193 bool BamCache::
196  ReMutexHolder holder(_lock);
197  nassertr(!record->_cache_pathname.empty(), false);
198  nassertr(record->has_data(), false);
199 
200  if (_read_only) {
201  return false;
202  }
203 
205 
206 #ifndef NDEBUG
207  // Ensure that the cache_pathname is within the _root directory tree.
208  Filename rel_pathname(record->_cache_pathname);
209  rel_pathname.make_relative_to(_root, false);
210  nassertr(rel_pathname.is_local(), false);
211 #endif // NDEBUG
212 
213  record->_recorded_time = time(NULL);
214 
215  Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
216 
217  // We actually do the write to a temporary filename first, and then
218  // move it into place, so that no one attempts to read the file
219  // while it is in the process of being written.
220  Thread *current_thread = Thread::get_current_thread();
221  string extension = current_thread->get_unique_id() + string(".tmp");
222  Filename temp_pathname = cache_pathname;
223  temp_pathname.set_extension(extension);
224  temp_pathname.set_binary();
225 
226  DatagramOutputFile dout;
227  if (!dout.open(temp_pathname)) {
228  util_cat.error()
229  << "Could not write cache file: " << temp_pathname << "\n";
230  vfs->delete_file(temp_pathname);
231  emergency_read_only();
232  return false;
233  }
234 
235  if (!dout.write_header(_bam_header)) {
236  util_cat.error()
237  << "Unable to write to " << temp_pathname << "\n";
238  vfs->delete_file(temp_pathname);
239  return false;
240  }
241 
242  {
243  BamWriter writer(&dout);
244  if (!writer.init()) {
245  util_cat.error()
246  << "Unable to write Bam header to " << temp_pathname << "\n";
247  vfs->delete_file(temp_pathname);
248  return false;
249  }
250 
251  TypeRegistry *type_registry = TypeRegistry::ptr();
252  TypeHandle texture_type = type_registry->find_type("Texture");
253  if (record->get_data()->is_of_type(texture_type)) {
254  // Texture objects write the actual texture image.
255  writer.set_file_texture_mode(BamWriter::BTM_rawdata);
256  } else {
257  // Any other kinds of objects write texture references.
258  writer.set_file_texture_mode(BamWriter::BTM_fullpath);
259  }
260 
261  if (!writer.write_object(record)) {
262  util_cat.error()
263  << "Unable to write object to " << temp_pathname << "\n";
264  vfs->delete_file(temp_pathname);
265  return false;
266  }
267 
268  if (!writer.write_object(record->get_data())) {
269  util_cat.error()
270  << "Unable to write object data to " << temp_pathname << "\n";
271  vfs->delete_file(temp_pathname);
272  return false;
273  }
274 
275  // Now that we are done with the BamWriter, it's important to let
276  // it destruct now and clean itself up, or it might get mad if we
277  // delete any TypedWritables below that haven't been written yet.
278  }
279 
280  record->_record_size = dout.get_file_pos();
281  dout.close();
282 
283  // Now move the file into place.
284  if (!vfs->rename_file(temp_pathname, cache_pathname) && vfs->exists(temp_pathname)) {
285  vfs->delete_file(cache_pathname);
286  if (!vfs->rename_file(temp_pathname, cache_pathname)) {
287  util_cat.error()
288  << "Unable to rename " << temp_pathname << " to "
289  << cache_pathname << "\n";
290  vfs->delete_file(temp_pathname);
291  return false;
292  }
293  }
294 
295  add_to_index(record);
296 
297  return true;
298 }
299 
300 ////////////////////////////////////////////////////////////////////
301 // Function: BamCache::emergency_read_only
302 // Access: Private
303 // Description: Called when an attempt to write to the cache dir
304 // has failed, usually for lack of disk space or
305 // because of incorrect file permissions. Outputs
306 // an error and puts the BamCache into read-only
307 // mode.
308 ////////////////////////////////////////////////////////////////////
309 void BamCache::
310 emergency_read_only() {
311  util_cat.error() <<
312  "Could not write to the Bam Cache. Disabling future attempts.\n";
313  _read_only = true;
314 }
315 
316 ////////////////////////////////////////////////////////////////////
317 // Function: BamCache::consider_flush_index
318 // Access: Published
319 // Description: Flushes the index if enough time has elapsed since
320 // the index was last flushed.
321 ////////////////////////////////////////////////////////////////////
322 void BamCache::
324 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
325  if (!_lock.try_acquire()) {
326  // If we can't grab the lock, no big deal. We don't want to hold up
327  // the frame waiting for a cache operation. We can try again later.
328  return;
329  }
330 #endif
331 
332  if (_index_stale_since != 0) {
333  int elapsed = (int)time(NULL) - (int)_index_stale_since;
334  if (elapsed > _flush_time) {
335  flush_index();
336  }
337  }
338 
339 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
340  _lock.release();
341 #endif
342 }
343 
344 ////////////////////////////////////////////////////////////////////
345 // Function: BamCache::flush_index
346 // Access: Published
347 // Description: Ensures the index is written to disk.
348 ////////////////////////////////////////////////////////////////////
349 void BamCache::
351  ReMutexHolder holder(_lock);
352  if (_index_stale_since == 0) {
353  // Never mind.
354  return;
355  }
356 
357  while (true) {
358  if (_read_only) {
359  return;
360  }
361 
362  Filename temp_pathname = Filename::temporary(_root, "index-", ".boo");
363 
364  if (!do_write_index(temp_pathname, _index)) {
365  emergency_read_only();
366  return;
367  }
368 
369  // Now atomically write the name of this index file to the index
370  // reference file.
372  Filename index_ref_pathname(_root, Filename("index_name.txt"));
373  string old_index = _index_ref_contents;
374  string new_index = temp_pathname.get_basename() + "\n";
375  string orig_index;
376 
377  if (vfs->atomic_compare_and_exchange_contents(index_ref_pathname, orig_index, old_index, new_index)) {
378  // We successfully wrote our version of the index, and no other
379  // process beat us to it. Our index is now the official one.
380  // Remove the old index.
381  vfs->delete_file(_index_pathname);
382  _index_pathname = temp_pathname;
383  _index_ref_contents = new_index;
384  _index_stale_since = 0;
385  return;
386  }
387 
388  // Shoot, some other process updated the index while we were
389  // trying to update it, and they beat us to it. We have to merge,
390  // and try again.
391  vfs->delete_file(temp_pathname);
392  _index_pathname = Filename(_root, Filename(trim(orig_index)));
393  _index_ref_contents = orig_index;
394  read_index();
395  }
396  check_cache_size();
397 }
398 
399 ////////////////////////////////////////////////////////////////////
400 // Function: BamCache::list_index
401 // Access: Published
402 // Description: Writes the contents of the index to standard output.
403 ////////////////////////////////////////////////////////////////////
404 void BamCache::
405 list_index(ostream &out, int indent_level) const {
406  _index->write(out, indent_level);
407 }
408 
409 ////////////////////////////////////////////////////////////////////
410 // Function: BamCache::read_index
411 // Access: Private
412 // Description: Reads, or re-reads the index file from disk. If
413 // _index_stale_since is nonzero, the index file is read
414 // and then merged with our current index.
415 ////////////////////////////////////////////////////////////////////
416 void BamCache::
417 read_index() {
418  if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
419  // Couldn't read the index ref; rebuild the index.
420  rebuild_index();
421  return;
422  }
423 
424  while (true) {
425  BamCacheIndex *new_index = do_read_index(_index_pathname);
426  if (new_index != (BamCacheIndex *)NULL) {
427  merge_index(new_index);
428  return;
429  }
430 
431  // We couldn't read the index. Maybe it's been removed already.
432  // See if the index_pathname has changed.
433  Filename old_index_pathname = _index_pathname;
434  if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
435  // Couldn't read the index ref; rebuild the index.
436  rebuild_index();
437  return;
438  }
439 
440  if (old_index_pathname == _index_pathname) {
441  // Nope, we just couldn't read it. Delete it and build a new
442  // one.
444  vfs->delete_file(_index_pathname);
445  rebuild_index();
446  flush_index();
447  return;
448  }
449  }
450 }
451 
452 ////////////////////////////////////////////////////////////////////
453 // Function: BamCache::read_index_pathname
454 // Access: Private
455 // Description: Atomically reads the current index filename from the
456 // index reference file. The index filename moves
457 // around as different processes update the index.
458 ////////////////////////////////////////////////////////////////////
459 bool BamCache::
460 read_index_pathname(Filename &index_pathname, string &index_ref_contents) const {
462  index_ref_contents.clear();
463  Filename index_ref_pathname(_root, Filename("index_name.txt"));
464  if (!vfs->atomic_read_contents(index_ref_pathname, index_ref_contents)) {
465  return false;
466  }
467 
468  string trimmed = trim(index_ref_contents);
469  if (trimmed.empty()) {
470  index_pathname = Filename();
471  } else {
472  index_pathname = Filename(_root, Filename(trimmed));
473  }
474  return true;
475 }
476 
477 ////////////////////////////////////////////////////////////////////
478 // Function: BamCache::merge_index
479 // Access: Private
480 // Description: The supplied index file has been updated by some other
481 // process. Merge it with our current index.
482 //
483 // Ownership of the pointer is transferred with this
484 // call. The caller should assume that new_index will
485 // be deleted by this method.
486 ////////////////////////////////////////////////////////////////////
487 void BamCache::
488 merge_index(BamCacheIndex *new_index) {
489  if (_index_stale_since == 0) {
490  // If our index isn't stale, just replace it.
491  delete _index;
492  _index = new_index;
493  return;
494  }
495 
496  BamCacheIndex *old_index = _index;
497  old_index->release_records();
498  new_index->release_records();
499  _index = new BamCacheIndex;
500 
501  BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
502  BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
503 
504  while (ai != old_index->_records.end() &&
505  bi != new_index->_records.end()) {
506  if ((*ai).first < (*bi).first) {
507  // Here is an entry we have in our index, not present in the new
508  // index.
509  PT(BamCacheRecord) record = (*ai).second;
510  Filename cache_pathname(_root, record->get_cache_filename());
511  if (cache_pathname.exists()) {
512  // The file exists; keep it.
513  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
514  }
515  ++ai;
516 
517  } else if ((*bi).first < (*ai).first) {
518  // Here is an entry in the new index, not present in our index.
519  PT(BamCacheRecord) record = (*bi).second;
520  Filename cache_pathname(_root, record->get_cache_filename());
521  if (cache_pathname.exists()) {
522  // The file exists; keep it.
523  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
524  }
525  ++bi;
526 
527  } else {
528  // Here is an entry we have in both.
529  PT(BamCacheRecord) a_record = (*ai).second;
530  PT(BamCacheRecord) b_record = (*bi).second;
531  if (*a_record == *b_record) {
532  // They're the same entry. It doesn't really matter which one
533  // we keep.
534  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
535 
536  } else {
537  // They're different. Just throw them both away, and re-read
538  // the current data from the cache file.
539 
540  Filename cache_pathname(_root, a_record->get_cache_filename());
541 
542  if (cache_pathname.exists()) {
543  PT(BamCacheRecord) record = do_read_record(cache_pathname, false);
544  if (record != (BamCacheRecord *)NULL) {
545  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
546  }
547  }
548  }
549 
550  ++ai;
551  ++bi;
552  }
553  }
554 
555  while (ai != old_index->_records.end()) {
556  // Here is an entry we have in our index, not present in the new
557  // index.
558  PT(BamCacheRecord) record = (*ai).second;
559  Filename cache_pathname(_root, record->get_cache_filename());
560  if (cache_pathname.exists()) {
561  // The file exists; keep it.
562  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
563  }
564  ++ai;
565  }
566 
567  while (bi != new_index->_records.end()) {
568  // Here is an entry in the new index, not present in our index.
569  PT(BamCacheRecord) record = (*bi).second;
570  Filename cache_pathname(_root, record->get_cache_filename());
571  if (cache_pathname.exists()) {
572  // The file exists; keep it.
573  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
574  }
575  ++bi;
576  }
577 
578  _index->process_new_records();
579 }
580 
581 ////////////////////////////////////////////////////////////////////
582 // Function: BamCache::rebuild_index
583 // Access: Private
584 // Description: Regenerates the index from scratch by scanning the
585 // directory.
586 ////////////////////////////////////////////////////////////////////
587 void BamCache::
588 rebuild_index() {
590 
591  PT(VirtualFileList) contents = vfs->scan_directory(_root);
592  if (contents == NULL) {
593  util_cat.error()
594  << "Unable to read directory " << _root << ", caching disabled.\n";
595  set_active(false);
596  return;
597  }
598 
599  delete _index;
600  _index = new BamCacheIndex;
601 
602  int num_files = contents->get_num_files();
603  for (int ci = 0; ci < num_files; ++ci) {
604  VirtualFile *file = contents->get_file(ci);
605  Filename filename = file->get_filename();
606  if (filename.get_extension() == "bam" ||
607  filename.get_extension() == "txo") {
608  Filename pathname(_root, filename);
609 
610  PT(BamCacheRecord) record = do_read_record(pathname, false);
611  if (record == (BamCacheRecord *)NULL) {
612  // Well, it was invalid, so blow it away.
613  if (util_cat.is_debug()) {
614  util_cat.debug()
615  << "Deleting invalid " << pathname << "\n";
616  }
617  file->delete_file();
618 
619  } else {
620  record->_record_access_time = record->_recorded_time;
621 
622  bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second;
623  if (!inserted) {
624  util_cat.info()
625  << "Multiple cache files defining " << record->get_source_pathname() << "\n";
626  file->delete_file();
627  }
628  }
629  }
630  }
631  _index->process_new_records();
632 
633  _index_stale_since = time(NULL);
634  check_cache_size();
635  flush_index();
636 }
637 
638 ////////////////////////////////////////////////////////////////////
639 // Function: BamCache::add_to_index
640 // Access: Private
641 // Description: Updates the index entry for the indicated record.
642 // Note that a copy of the record is made first.
643 ////////////////////////////////////////////////////////////////////
644 void BamCache::
645 add_to_index(const BamCacheRecord *record) {
646  PT(BamCacheRecord) new_record = record->make_copy();
647 
648  if (_index->add_record(new_record)) {
649  mark_index_stale();
650  check_cache_size();
651  }
652 }
653 
654 ////////////////////////////////////////////////////////////////////
655 // Function: BamCache::remove_from_index
656 // Access: Private
657 // Description: Removes the index entry for the indicated record, if
658 // there is one.
659 ////////////////////////////////////////////////////////////////////
660 void BamCache::
661 remove_from_index(const Filename &source_pathname) {
662  if (_index->remove_record(source_pathname)) {
663  mark_index_stale();
664  }
665 }
666 
667 ////////////////////////////////////////////////////////////////////
668 // Function: BamCache::check_cache_size
669 // Access: Private
670 // Description: If the cache size has exceeded its specified size
671 // limit, removes an old file.
672 ////////////////////////////////////////////////////////////////////
673 void BamCache::
674 check_cache_size() {
675  if (_index->_cache_size == 0) {
676  // 0 means no limit.
677  return;
678  }
679 
680  if (_index->_cache_size / 1024 > _max_kbytes) {
681  while (_index->_cache_size / 1024 > _max_kbytes) {
682  PT(BamCacheRecord) record = _index->evict_old_file();
683  if (record == NULL) {
684  // Never mind; the cache is empty.
685  break;
686  }
688  Filename cache_pathname(_root, record->get_cache_filename());
689  if (util_cat.is_debug()) {
690  util_cat.debug()
691  << "Deleting " << cache_pathname
692  << " to keep cache size below " << _max_kbytes << "K\n";
693  }
694  vfs->delete_file(cache_pathname);
695  }
696  mark_index_stale();
697  }
698 }
699 
700 ////////////////////////////////////////////////////////////////////
701 // Function: BamCache::do_read_index
702 // Access: Private, Static
703 // Description: Reads the index data from the specified filename.
704 // Returns a newly-allocated BamCacheIndex object on
705 // success, or NULL on failure.
706 ////////////////////////////////////////////////////////////////////
707 BamCacheIndex *BamCache::
708 do_read_index(const Filename &index_pathname) {
709  if (index_pathname.empty()) {
710  return NULL;
711  }
712 
713  DatagramInputFile din;
714  if (!din.open(index_pathname)) {
715  util_cat.debug()
716  << "Could not read index file: " << index_pathname << "\n";
717  return NULL;
718  }
719 
720  string head;
721  if (!din.read_header(head, _bam_header.size())) {
722  util_cat.debug()
723  << index_pathname << " is not an index file.\n";
724  return NULL;
725  }
726 
727  if (head != _bam_header) {
728  util_cat.debug()
729  << index_pathname << " is not an index file.\n";
730  return NULL;
731  }
732 
733  BamReader reader(&din);
734  if (!reader.init()) {
735  return NULL;
736  }
737 
738  TypedWritable *object = reader.read_object();
739 
740  if (object == (TypedWritable *)NULL) {
741  util_cat.error()
742  << "Cache index " << index_pathname << " is empty.\n";
743  return NULL;
744 
745  } else if (!object->is_of_type(BamCacheIndex::get_class_type())) {
746  util_cat.error()
747  << "Cache index " << index_pathname << " contains a "
748  << object->get_type() << ", not a BamCacheIndex.\n";
749  return NULL;
750  }
751 
752  BamCacheIndex *index = DCAST(BamCacheIndex, object);
753  if (!reader.resolve()) {
754  util_cat.error()
755  << "Unable to fully resolve cache index file.\n";
756  return NULL;
757  }
758 
759  return index;
760 }
761 
762 ////////////////////////////////////////////////////////////////////
763 // Function: BamCache::do_write_index
764 // Access: Private, Static
765 // Description: Writes the given index data to the specified filename.
766 ////////////////////////////////////////////////////////////////////
767 bool BamCache::
768 do_write_index(const Filename &index_pathname, const BamCacheIndex *index) {
770  DatagramOutputFile dout;
771  if (!dout.open(index_pathname)) {
772  util_cat.error()
773  << "Could not write index file: " << index_pathname << "\n";
774  vfs->delete_file(index_pathname);
775  return false;
776  }
777 
778  if (!dout.write_header(_bam_header)) {
779  util_cat.error()
780  << "Unable to write to " << index_pathname << "\n";
781  vfs->delete_file(index_pathname);
782  return false;
783  }
784 
785  {
786  BamWriter writer(&dout);
787  if (!writer.init()) {
788  vfs->delete_file(index_pathname);
789  return false;
790  }
791 
792  if (!writer.write_object(index)) {
793  vfs->delete_file(index_pathname);
794  return false;
795  }
796  }
797 
798  return true;
799 }
800 
801 ////////////////////////////////////////////////////////////////////
802 // Function: BamCache::find_and_read_record
803 // Access: Private
804 // Description: Looks for the existing cache file that corresponds
805 // to the indicated filename. Normally, this is the
806 // specified cache filename exactly; but in the case of
807 // a hash collision, it may be a variant of the cache
808 // filename.
809 ////////////////////////////////////////////////////////////////////
810 PT(BamCacheRecord) BamCache::
811 find_and_read_record(const Filename &source_pathname,
812  const Filename &cache_filename) {
813  int pass = 0;
814  while (true) {
815  PT(BamCacheRecord) record =
816  read_record(source_pathname, cache_filename, pass);
817  if (record != (BamCacheRecord *)NULL) {
818  add_to_index(record);
819  return record;
820  }
821  ++pass;
822  }
823 }
824 
825 ////////////////////////////////////////////////////////////////////
826 // Function: BamCache::read_record
827 // Access: Private
828 // Description: Reads the indicated cache file and returns its
829 // associated record if it can be read and it matches
830 // the source filename.
831 ////////////////////////////////////////////////////////////////////
832 PT(BamCacheRecord) BamCache::
833 read_record(const Filename &source_pathname,
834  const Filename &cache_filename,
835  int pass) {
837  Filename cache_pathname(_root, cache_filename);
838  if (pass != 0) {
839  ostringstream strm;
840  strm << cache_pathname.get_basename_wo_extension() << "_" << pass;
841  cache_pathname.set_basename_wo_extension(strm.str());
842  }
843 
844  if (!cache_pathname.exists()) {
845  // There is no such cache file already. Declare it.
846  if (util_cat.is_debug()) {
847  util_cat.debug()
848  << "Declaring new cache file " << cache_pathname << " for " << source_pathname << "\n";
849  }
850  PT(BamCacheRecord) record =
851  new BamCacheRecord(source_pathname, cache_filename);
852  record->_cache_pathname = cache_pathname;
853  return record;
854  }
855 
856  if (util_cat.is_debug()) {
857  util_cat.debug()
858  << "Reading cache file " << cache_pathname << " for " << source_pathname << "\n";
859  }
860 
861  PT(BamCacheRecord) record = do_read_record(cache_pathname, true);
862  if (record == (BamCacheRecord *)NULL) {
863  // Well, it was invalid, so blow it away, and make a new one.
864  if (util_cat.is_debug()) {
865  util_cat.debug()
866  << "Deleting invalid cache file " << cache_pathname << "\n";
867  }
868  vfs->delete_file(cache_pathname);
869  remove_from_index(source_pathname);
870 
871  PT(BamCacheRecord) record =
872  new BamCacheRecord(source_pathname, cache_filename);
873  record->_cache_pathname = cache_pathname;
874  return record;
875  }
876 
877  if (record->get_source_pathname() != source_pathname) {
878  // This might be just a hash conflict.
879  if (util_cat.is_debug()) {
880  util_cat.debug()
881  << "Cache file " << cache_pathname << " references "
882  << record->get_source_pathname() << ", not "
883  << source_pathname << "\n";
884  }
885  return NULL;
886  }
887 
888  if (!record->has_data()) {
889  // If we didn't find any data, the caller will have to reload it.
890  record->clear_dependent_files();
891  }
892 
893  record->_cache_pathname = cache_pathname;
894  return record;
895 }
896 
897 ////////////////////////////////////////////////////////////////////
898 // Function: BamCache::do_read_record
899 // Access: Private, Static
900 // Description: Actually reads a record from the file.
901 ////////////////////////////////////////////////////////////////////
902 PT(BamCacheRecord) BamCache::
903 do_read_record(const Filename &cache_pathname, bool read_data) {
904  DatagramInputFile din;
905  if (!din.open(cache_pathname)) {
906  if (util_cat.is_debug()) {
907  util_cat.debug()
908  << "Could not read cache file: " << cache_pathname << "\n";
909  }
910  return NULL;
911  }
912 
913  string head;
914  if (!din.read_header(head, _bam_header.size())) {
915  if (util_cat.is_debug()) {
916  util_cat.debug()
917  << cache_pathname << " is not a cache file.\n";
918  }
919  return NULL;
920  }
921 
922  if (head != _bam_header) {
923  if (util_cat.is_debug()) {
924  util_cat.debug()
925  << cache_pathname << " is not a cache file.\n";
926  }
927  return NULL;
928  }
929 
930  BamReader reader(&din);
931  if (!reader.init()) {
932  return NULL;
933  }
934 
935  TypedWritable *object = reader.read_object();
936  if (object == (TypedWritable *)NULL) {
937  if (util_cat.is_debug()) {
938  util_cat.debug()
939  << cache_pathname << " is empty.\n";
940  }
941  return NULL;
942 
943  } else if (!object->is_of_type(BamCacheRecord::get_class_type())) {
944  if (util_cat.is_debug()) {
945  util_cat.debug()
946  << "Cache file " << cache_pathname << " contains a "
947  << object->get_type() << ", not a BamCacheRecord.\n";
948  }
949  return NULL;
950  }
951 
952  PT(BamCacheRecord) record = DCAST(BamCacheRecord, object);
953  if (!reader.resolve()) {
954  if (util_cat.is_debug()) {
955  util_cat.debug()
956  << "Unable to fully resolve cache record in " << cache_pathname << "\n";
957  }
958  return NULL;
959  }
960 
961  // From this point below, we have validated that the selected
962  // filename is indeed a cache record for the indicated source file,
963  // and therefore the cache record will be returned.
964 
965  // We still need to decide whether the cache record is stale.
966  if (read_data && record->dependents_unchanged()) {
967  // The cache record doesn't appear to be stale. Load the cached
968  // object.
969  TypedWritable *ptr;
970  ReferenceCount *ref_ptr;
971 
972  if (reader.read_object(ptr, ref_ptr)) {
973  if (!reader.resolve()) {
974  if (util_cat.is_debug()) {
975  util_cat.debug()
976  << "Unable to fully resolve cached object in " << cache_pathname << "\n";
977  }
978  delete object;
979  } else {
980  // The object is valid. Store it in the record.
981  record->set_data(ptr, ref_ptr);
982  }
983  }
984  }
985 
986  // Also get the total file size.
987  PT(VirtualFile) vfile = din.get_vfile();
988  istream &in = din.get_stream();
989  in.clear();
990  record->_record_size = vfile->get_file_size(&in);
991 
992  // And the last access time is now, duh.
993  record->_record_access_time = time(NULL);
994 
995  return record;
996 }
997 
998 ////////////////////////////////////////////////////////////////////
999 // Function: BamCache::hash_filename
1000 // Access: Private, Static
1001 // Description: Returns the appropriate filename to use for a cache
1002 // file, given the fullpath string to the source
1003 // filename.
1004 ////////////////////////////////////////////////////////////////////
1005 string BamCache::
1006 hash_filename(const string &filename) {
1007 #ifdef HAVE_OPENSSL
1008  // With OpenSSl, use the MD5 hash of the filename.
1009  HashVal hv;
1010  hv.hash_string(filename);
1011  ostringstream strm;
1012  hv.output_hex(strm);
1013  return strm.str();
1014 
1015 #else // HAVE_OPENSSL
1016  // Without OpenSSL, don't get fancy; just build a simple hash.
1017  unsigned int hash = 0;
1018  for (string::const_iterator si = filename.begin();
1019  si != filename.end();
1020  ++si) {
1021  hash = (hash * 9109) + (unsigned int)(*si);
1022  }
1023 
1024  ostringstream strm;
1025  strm << hex << setw(8) << setfill('0') << hash;
1026  return strm.str();
1027 
1028 #endif // HAVE_OPENSSL
1029 }
1030 
1031 ////////////////////////////////////////////////////////////////////
1032 // Function: BamCache::make_global
1033 // Access: Private, Static
1034 // Description: Constructs the global BamCache object.
1035 ////////////////////////////////////////////////////////////////////
1036 void BamCache::
1037 make_global() {
1038  _global_ptr = new BamCache;
1039 
1040  if (_global_ptr->_root.empty()) {
1041  _global_ptr->set_active(false);
1042  }
1043 }
void set_data(TypedWritable *ptr, ReferenceCount *ref_ptr)
Stores a new data object on the record.
virtual streampos get_file_pos()
Returns the current file position within the data stream, if any, or 0 if the file position is not me...
bool atomic_compare_and_exchange_contents(const Filename &filename, string &orig_contents, const string &old_contents, const string &new_contents)
See Filename::atomic_compare_and_exchange_contents().
void set_active(bool flag)
Changes the state of the active flag.
Definition: bamCache.I:29
string get_basename() const
Returns the basename part of the filename.
Definition: filename.I:436
void set_extension(const string &s)
Replaces the file extension.
Definition: filename.cxx:837
bool rename_file(const Filename &orig_filename, const Filename &new_filename)
Attempts to move or rename the indicated file or directory.
TypedWritable * get_data() const
Returns a pointer to the data stored in the record, or NULL if there is no data.
string get_fullpath() const
Returns the entire filename: directory, basename, extension.
Definition: filename.I:398
This is the fundamental interface for extracting binary objects from a Bam file, as generated by a Ba...
Definition: bamReader.h:122
TypeHandle find_type(const string &name) const
Looks for a previously-registered type of the given name.
bool resolve()
This may be called at any time during processing of the Bam file to resolve all the known pointers so...
Definition: bamReader.cxx:359
This class maintains a cache of Bam and/or Txo objects generated from model files and texture images ...
Definition: bamCache.h:47
This is a convenience class to specialize ConfigVariable as a Filename type.
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS&#39;s file system.
This is a convenience class to specialize ConfigVariable as a boolean type.
bool try_acquire() const
Returns immediately, with a true value indicating the mutex has been acquired, and false indicating i...
TypedWritable * read_object()
Reads a single object from the Bam file.
Definition: bamReader.cxx:249
This class can be used to read a binary file that consists of an arbitrary header followed by a numbe...
void release() const
Releases the reMutex.
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:494
Base class for objects that can be written to and read from Bam files.
Definition: typedWritable.h:37
bool write_header(const string &header)
Writes a sequence of bytes to the beginning of the datagram file.
void list_index(ostream &out, int indent_level=0) const
Writes the contents of the index to standard output.
Definition: bamCache.cxx:405
void flush_index()
Ensures the index is written to disk.
Definition: bamCache.cxx:350
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer...
Definition: hashVal.h:32
This is the fundamental interface for writing binary objects to a Bam file, to be extracted later by ...
Definition: bamWriter.h:73
bool make_directory_full(const Filename &filename)
Attempts to create a directory within the file system.
void set_basename_wo_extension(const string &s)
Replaces the basename part of the filename, without the file extension.
Definition: filename.cxx:813
bool open(const FileReference *file)
Opens the indicated filename for reading.
The abstract base class for a file or directory within the VirtualFileSystem.
Definition: virtualFile.h:37
string get_extension() const
Returns the file extension.
Definition: filename.I:477
bool read_header(string &header, size_t num_bytes)
Reads a sequence of bytes from the beginning of the datagram file.
static Thread * get_current_thread()
Returns a pointer to the currently-executing Thread object.
Definition: thread.I:145
bool open(const FileReference *file)
Opens the indicated filename for writing.
istream & get_stream()
Returns the istream represented by the input file.
Type get_type() const
Returns the type of the file represented by the filename, as previously set by set_type().
Definition: filename.I:583
static Filename temporary(const string &dirname, const string &prefix, const string &suffix=string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix...
Definition: filename.cxx:439
virtual VirtualFile * get_vfile()
Returns the VirtualFile that provides the source for these datagrams, if any, or NULL if the datagram...
This represents the in-memory index that records the list of files stored in the BamCache.
Definition: bamCacheIndex.h:37
void set_file_texture_mode(BamTextureMode file_texture_mode)
Changes the BamTextureMode preference for the Bam file currently being written.
Definition: bamWriter.I:94
bool atomic_read_contents(const Filename &filename, string &contents) const
See Filename::atomic_read_contents().
bool store(BamCacheRecord *record)
Flushes a cache entry to disk.
Definition: bamCache.cxx:194
void set_root(const Filename &root)
Changes the current root pathname of the cache.
Definition: bamCache.cxx:118
A list of VirtualFiles, as returned by VirtualFile::scan_directory().
bool exists(const Filename &filename) const
Convenience function; returns true if the named file exists.
bool init()
Initializes the BamReader prior to reading any objects from its source.
Definition: bamReader.cxx:94
bool is_directory(const Filename &filename) const
Convenience function; returns true if the named file exists and is a directory.
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:44
void output_hex(ostream &out) const
Outputs the HashVal as a 32-digit hexadecimal number.
Definition: hashVal.cxx:31
An instance of this class is written to the front of a Bam or Txo file to make the file a cached inst...
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
virtual bool delete_file()
Attempts to delete this file or directory.
Definition: virtualFile.cxx:77
bool write_object(const TypedWritable *obj)
Writes a single object to the Bam file, so that the BamReader::read_object() can later correctly rest...
Definition: bamWriter.cxx:152
Similar to MutexHolder, but for a reentrant mutex.
Definition: reMutexHolder.h:27
string get_basename_wo_extension() const
Returns the basename part of the filename, without the file extension.
Definition: filename.I:460
bool is_local() const
Returns true if the filename is local, e.g.
Definition: filename.I:664
bool make_relative_to(Filename directory, bool allow_backups=true)
Adjusts this filename, which must be a fully-specified pathname beginning with a slash, to make it a relative filename, relative to the fully-specified directory indicated (which must also begin with, and may or may not end with, a slash–a terminating slash is ignored).
Definition: filename.cxx:1766
static TypeRegistry * ptr()
Returns the pointer to the global TypeRegistry object.
This class can be used to write a binary file that consists of an arbitrary header followed by a numb...
void make_absolute()
Converts the filename to a fully-qualified pathname from the root (if it is a relative pathname)...
Definition: filename.cxx:1019
A base class for all things that want to be reference-counted.
void clear_dependent_files()
Empties the list of files that contribute to the data in this record.
The TypeRegistry class maintains all the assigned TypeHandles in a given system.
Definition: typeRegistry.h:39
A thread; that is, a lightweight process.
Definition: thread.h:51
string get_unique_id() const
Returns a string that is guaranteed to be unique to this thread, across all processes on the machine...
Definition: thread.I:72
This is a convenience class to specialize ConfigVariable as an integer type.
bool is_of_type(TypeHandle handle) const
Returns true if the current object is or derives from the indicated type.
Definition: typedObject.I:63
Filename get_cwd() const
Returns the current directory name.
bool dependents_unchanged() const
Returns true if all of the dependent files are still the same as when the cache was recorded...
bool delete_file(const Filename &filename)
Attempts to delete the indicated file or directory.
TypeHandle is the identifier used to differentiate C++ class types.
Definition: typeHandle.h:85
void close()
Closes the file.
bool init()
Initializes the BamWriter prior to writing any objects to its output stream.
Definition: bamWriter.cxx:98
void consider_flush_index()
Flushes the index if enough time has elapsed since the index was last flushed.
Definition: bamCache.cxx:323
bool exists() const
Returns true if the filename exists on the disk, false otherwise.
Definition: filename.cxx:1356