Panda3D
 All Classes Functions Variables Enumerations
bamCache.cxx
1 // Filename: bamCache.cxx
2 // Created by: drose (09Jun06)
3 //
4 ////////////////////////////////////////////////////////////////////
5 //
6 // PANDA 3D SOFTWARE
7 // Copyright (c) Carnegie Mellon University. All rights reserved.
8 //
9 // All use of this software is subject to the terms of the revised BSD
10 // license. You should have received a copy of this license along
11 // with this source code in a file named "LICENSE."
12 //
13 ////////////////////////////////////////////////////////////////////
14 
15 #include "bamCache.h"
16 #include "bamCacheIndex.h"
17 #include "bamReader.h"
18 #include "bamWriter.h"
19 #include "hashVal.h"
20 #include "datagramInputFile.h"
21 #include "datagramOutputFile.h"
22 #include "config_util.h"
23 #include "bam.h"
24 #include "typeRegistry.h"
25 #include "string_utils.h"
26 #include "configVariableInt.h"
27 #include "configVariableString.h"
28 #include "configVariableFilename.h"
29 #include "virtualFileSystem.h"
30 
31 BamCache *BamCache::_global_ptr = NULL;
32 
33 ////////////////////////////////////////////////////////////////////
34 // Function: BamCache::Constructor
35 // Access: Published
36 // Description:
37 ////////////////////////////////////////////////////////////////////
38 BamCache::
39 BamCache() :
40  _active(true),
41  _read_only(false),
42  _index(new BamCacheIndex),
43  _index_stale_since(0)
44 {
45  ConfigVariableFilename model_cache_dir
46  ("model-cache-dir", Filename(),
47  PRC_DESC("The full path to a directory, local to this computer, in which "
48  "model and texture files will be cached on load. If a directory "
49  "name is specified here, files may be loaded from the cache "
50  "instead of from their actual pathnames, which may save load time, "
51  "especially if you are loading egg files instead of bam files, "
52  "or if you are loading models from a shared network drive. "
53  "If this is the empty string, no cache will be used."));
54 
55  ConfigVariableInt model_cache_flush
56  ("model-cache-flush", 30,
57  PRC_DESC("This is the amount of time, in seconds, between automatic "
58  "flushes of the model-cache index."));
59 
60  ConfigVariableBool model_cache_models
61  ("model-cache-models", true,
62  PRC_DESC("If this is set to true, models will be cached in the "
63  "model cache, as bam files."));
64 
65  ConfigVariableBool model_cache_textures
66  ("model-cache-textures", true,
67  PRC_DESC("If this is set to true, textures will also be cached in the "
68  "model cache, as txo files."));
69 
70  ConfigVariableBool model_cache_compressed_textures
71  ("model-cache-compressed-textures", false,
72  PRC_DESC("If this is set to true, compressed textures will be cached "
73  "in the model cache, in their compressed form as downloaded "
74  "by the GSG. This may be set in conjunction with "
75  "model-cache-textures, or it may be independent."));
76 
77  ConfigVariableInt model_cache_max_kbytes
78  ("model-cache-max-kbytes", 10485760,
79  PRC_DESC("This is the maximum size of the model cache, in kilobytes."));
80 
81  _cache_models = model_cache_models;
82  _cache_textures = model_cache_textures;
83  _cache_compressed_textures = model_cache_compressed_textures;
84 
85  _flush_time = model_cache_flush;
86  _max_kbytes = model_cache_max_kbytes;
87 
88  if (!model_cache_dir.empty()) {
89  set_root(model_cache_dir);
90  }
91 }
92 
93 ////////////////////////////////////////////////////////////////////
94 // Function: BamCache::Destructor
95 // Access: Published
96 // Description:
97 ////////////////////////////////////////////////////////////////////
98 BamCache::
99 ~BamCache() {
100  flush_index();
101  delete _index;
102  _index = NULL;
103 }
104 
105 ////////////////////////////////////////////////////////////////////
106 // Function: BamCache::set_root
107 // Access: Published
108 // Description: Changes the current root pathname of the cache. This
109 // specifies where the cache files are stored on disk.
110 // This should name a directory that is on a disk local
111 // to the machine (not on a network-mounted disk), for
112 // instance, /tmp/panda-cache or /c/panda-cache.
113 //
114 // If the directory does not already exist, it will be
115 // created as a result of this call.
116 ////////////////////////////////////////////////////////////////////
117 void BamCache::
118 set_root(const Filename &root) {
119  ReMutexHolder holder(_lock);
120  flush_index();
121  _root = root;
122 
123  // The root filename must be a directory.
125  if (!vfs->is_directory(_root)) {
126  vfs->make_directory_full(_root);
127  }
128 
129  delete _index;
130  _index = new BamCacheIndex;
131  _index_stale_since = 0;
132  read_index();
133  check_cache_size();
134 
135  nassertv(vfs->is_directory(_root));
136 }
137 
138 ////////////////////////////////////////////////////////////////////
139 // Function: BamCache::lookup
140 // Access: Published
141 // Description: Looks up a file in the cache.
142 //
143 // If the file is cacheable, then regardless of whether
144 // the file is found in the cache or not, this returns a
145 // BamCacheRecord. On the other hand, if the file
146 // cannot be cached, returns NULL.
147 //
148 // If record->has_data() returns true, then the file was
149 // found in the cache, and you may call
150 // record->extract_data() to get the object. If
151 // record->has_data() returns false, then the file was
152 // not found in the cache or the cache was stale; and
153 // you should reload the source file (calling
154 // record->add_dependent_file() for each file loaded,
155 // including the original source file), and then call
156 // record->set_data() to record the resulting loaded
157 // object; and finally, you should call store() to write
158 // the cached record to disk.
159 ////////////////////////////////////////////////////////////////////
161 lookup(const Filename &source_filename, const string &cache_extension) {
162  ReMutexHolder holder(_lock);
163  consider_flush_index();
164 
166 
167  Filename source_pathname(source_filename);
168  source_pathname.make_absolute(vfs->get_cwd());
169 
170  Filename rel_pathname(source_pathname);
171  rel_pathname.make_relative_to(_root, false);
172  if (rel_pathname.is_local()) {
173  // If the source pathname is already within the cache directory,
174  // don't cache it further.
175  return NULL;
176  }
177 
178  Filename cache_filename = hash_filename(source_pathname.get_fullpath());
179  cache_filename.set_extension(cache_extension);
180 
181  return find_and_read_record(source_pathname, cache_filename);
182 }
183 
184 ////////////////////////////////////////////////////////////////////
185 // Function: BamCache::store
186 // Access: Published
187 // Description: Flushes a cache entry to disk. You must have
188 // retrieved the cache record via a prior call to
189 // lookup(), and then stored the data via
190 // record->set_data(). Returns true on success, false
191 // on failure.
192 ////////////////////////////////////////////////////////////////////
193 bool BamCache::
196  ReMutexHolder holder(_lock);
197  nassertr(!record->_cache_pathname.empty(), false);
198  nassertr(record->has_data(), false);
199 
200  if (_read_only) {
201  return false;
202  }
203 
205 
206 #ifndef NDEBUG
207  // Ensure that the cache_pathname is within the _root directory tree.
208  Filename rel_pathname(record->_cache_pathname);
209  rel_pathname.make_relative_to(_root, false);
210  nassertr(rel_pathname.is_local(), false);
211 #endif // NDEBUG
212 
213  record->_recorded_time = time(NULL);
214 
215  Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
216 
217  // We actually do the write to a temporary filename first, and then
218  // move it into place, so that no one attempts to read the file
219  // while it is in the process of being written.
220  Thread *current_thread = Thread::get_current_thread();
221  string extension = current_thread->get_unique_id() + string(".tmp");
222  Filename temp_pathname = cache_pathname;
223  temp_pathname.set_extension(extension);
224  temp_pathname.set_binary();
225 
226  DatagramOutputFile dout;
227  if (!dout.open(temp_pathname)) {
228  util_cat.error()
229  << "Could not write cache file: " << temp_pathname << "\n";
230  vfs->delete_file(temp_pathname);
231  emergency_read_only();
232  return false;
233  }
234 
235  if (!dout.write_header(_bam_header)) {
236  util_cat.error()
237  << "Unable to write to " << temp_pathname << "\n";
238  vfs->delete_file(temp_pathname);
239  return false;
240  }
241 
242  {
243  BamWriter writer(&dout);
244  if (!writer.init()) {
245  util_cat.error()
246  << "Unable to write Bam header to " << temp_pathname << "\n";
247  vfs->delete_file(temp_pathname);
248  return false;
249  }
250 
251  TypeRegistry *type_registry = TypeRegistry::ptr();
252  TypeHandle texture_type = type_registry->find_type("Texture");
253  if (record->get_data()->is_of_type(texture_type)) {
254  // Texture objects write the actual texture image.
255  writer.set_file_texture_mode(BamWriter::BTM_rawdata);
256  } else {
257  // Any other kinds of objects write texture references.
258  writer.set_file_texture_mode(BamWriter::BTM_fullpath);
259  }
260 
261  if (!writer.write_object(record)) {
262  util_cat.error()
263  << "Unable to write object to " << temp_pathname << "\n";
264  vfs->delete_file(temp_pathname);
265  return false;
266  }
267 
268  if (!writer.write_object(record->get_data())) {
269  util_cat.error()
270  << "Unable to write object data to " << temp_pathname << "\n";
271  vfs->delete_file(temp_pathname);
272  return false;
273  }
274 
275  // Now that we are done with the BamWriter, it's important to let
276  // it destruct now and clean itself up, or it might get mad if we
277  // delete any TypedWritables below that haven't been written yet.
278  }
279 
280  record->_record_size = dout.get_file_pos();
281  dout.close();
282 
283  // Now move the file into place.
284  if (!vfs->rename_file(temp_pathname, cache_pathname) && vfs->exists(temp_pathname)) {
285  vfs->delete_file(cache_pathname);
286  if (!vfs->rename_file(temp_pathname, cache_pathname)) {
287  util_cat.error()
288  << "Unable to rename " << temp_pathname << " to "
289  << cache_pathname << "\n";
290  vfs->delete_file(temp_pathname);
291  return false;
292  }
293  }
294 
295  add_to_index(record);
296 
297  return true;
298 }
299 
300 ////////////////////////////////////////////////////////////////////
301 // Function: BamCache::emergency_read_only
302 // Access: Private
303 // Description: Called when an attempt to write to the cache dir
304 // has failed, usually for lack of disk space or
305 // because of incorrect file permissions. Outputs
306 // an error and puts the BamCache into read-only
307 // mode.
308 ////////////////////////////////////////////////////////////////////
309 void BamCache::
310 emergency_read_only() {
311  util_cat.error() <<
312  "Could not write to the Bam Cache. Disabling future attempts.\n";
313  _read_only = true;
314 }
315 
316 ////////////////////////////////////////////////////////////////////
317 // Function: BamCache::consider_flush_index
318 // Access: Published
319 // Description: Flushes the index if enough time has elapsed since
320 // the index was last flushed.
321 ////////////////////////////////////////////////////////////////////
322 void BamCache::
324  ReMutexHolder holder(_lock);
325  if (_index_stale_since != 0) {
326  int elapsed = (int)time(NULL) - (int)_index_stale_since;
327  if (elapsed > _flush_time) {
328  flush_index();
329  }
330  }
331 }
332 
333 ////////////////////////////////////////////////////////////////////
334 // Function: BamCache::flush_index
335 // Access: Published
336 // Description: Ensures the index is written to disk.
337 ////////////////////////////////////////////////////////////////////
338 void BamCache::
340  ReMutexHolder holder(_lock);
341  if (_index_stale_since == 0) {
342  // Never mind.
343  return;
344  }
345 
346  while (true) {
347  if (_read_only) {
348  return;
349  }
350 
351  Filename temp_pathname = Filename::temporary(_root, "index-", ".boo");
352 
353  if (!do_write_index(temp_pathname, _index)) {
354  emergency_read_only();
355  return;
356  }
357 
358  // Now atomically write the name of this index file to the index
359  // reference file.
361  Filename index_ref_pathname(_root, Filename("index_name.txt"));
362  string old_index = _index_ref_contents;
363  string new_index = temp_pathname.get_basename() + "\n";
364  string orig_index;
365 
366  if (vfs->atomic_compare_and_exchange_contents(index_ref_pathname, orig_index, old_index, new_index)) {
367  // We successfully wrote our version of the index, and no other
368  // process beat us to it. Our index is now the official one.
369  // Remove the old index.
370  vfs->delete_file(_index_pathname);
371  _index_pathname = temp_pathname;
372  _index_ref_contents = new_index;
373  _index_stale_since = 0;
374  return;
375  }
376 
377  // Shoot, some other process updated the index while we were
378  // trying to update it, and they beat us to it. We have to merge,
379  // and try again.
380  vfs->delete_file(temp_pathname);
381  _index_pathname = Filename(_root, Filename(trim(orig_index)));
382  _index_ref_contents = orig_index;
383  read_index();
384  }
385  check_cache_size();
386 }
387 
388 ////////////////////////////////////////////////////////////////////
389 // Function: BamCache::list_index
390 // Access: Published
391 // Description: Writes the contents of the index to standard output.
392 ////////////////////////////////////////////////////////////////////
393 void BamCache::
394 list_index(ostream &out, int indent_level) const {
395  _index->write(out, indent_level);
396 }
397 
398 ////////////////////////////////////////////////////////////////////
399 // Function: BamCache::read_index
400 // Access: Private
401 // Description: Reads, or re-reads the index file from disk. If
402 // _index_stale_since is nonzero, the index file is read
403 // and then merged with our current index.
404 ////////////////////////////////////////////////////////////////////
405 void BamCache::
406 read_index() {
407  if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
408  // Couldn't read the index ref; rebuild the index.
409  rebuild_index();
410  return;
411  }
412 
413  while (true) {
414  BamCacheIndex *new_index = do_read_index(_index_pathname);
415  if (new_index != (BamCacheIndex *)NULL) {
416  merge_index(new_index);
417  return;
418  }
419 
420  // We couldn't read the index. Maybe it's been removed already.
421  // See if the index_pathname has changed.
422  Filename old_index_pathname = _index_pathname;
423  if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
424  // Couldn't read the index ref; rebuild the index.
425  rebuild_index();
426  return;
427  }
428 
429  if (old_index_pathname == _index_pathname) {
430  // Nope, we just couldn't read it. Delete it and build a new
431  // one.
433  vfs->delete_file(_index_pathname);
434  rebuild_index();
435  flush_index();
436  return;
437  }
438  }
439 }
440 
441 ////////////////////////////////////////////////////////////////////
442 // Function: BamCache::read_index_pathname
443 // Access: Private
444 // Description: Atomically reads the current index filename from the
445 // index reference file. The index filename moves
446 // around as different processes update the index.
447 ////////////////////////////////////////////////////////////////////
448 bool BamCache::
449 read_index_pathname(Filename &index_pathname, string &index_ref_contents) const {
451  index_ref_contents.clear();
452  Filename index_ref_pathname(_root, Filename("index_name.txt"));
453  if (!vfs->atomic_read_contents(index_ref_pathname, index_ref_contents)) {
454  return false;
455  }
456 
457  string trimmed = trim(index_ref_contents);
458  if (trimmed.empty()) {
459  index_pathname = Filename();
460  } else {
461  index_pathname = Filename(_root, Filename(trimmed));
462  }
463  return true;
464 }
465 
466 ////////////////////////////////////////////////////////////////////
467 // Function: BamCache::merge_index
468 // Access: Private
469 // Description: The supplied index file has been updated by some other
470 // process. Merge it with our current index.
471 //
472 // Ownership of the pointer is transferred with this
473 // call. The caller should assume that new_index will
474 // be deleted by this method.
475 ////////////////////////////////////////////////////////////////////
476 void BamCache::
477 merge_index(BamCacheIndex *new_index) {
478  if (_index_stale_since == 0) {
479  // If our index isn't stale, just replace it.
480  delete _index;
481  _index = new_index;
482  return;
483  }
484 
485  BamCacheIndex *old_index = _index;
486  old_index->release_records();
487  new_index->release_records();
488  _index = new BamCacheIndex;
489 
490  BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
491  BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
492 
493  while (ai != old_index->_records.end() &&
494  bi != new_index->_records.end()) {
495  if ((*ai).first < (*bi).first) {
496  // Here is an entry we have in our index, not present in the new
497  // index.
498  PT(BamCacheRecord) record = (*ai).second;
499  Filename cache_pathname(_root, record->get_cache_filename());
500  if (cache_pathname.exists()) {
501  // The file exists; keep it.
502  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
503  }
504  ++ai;
505 
506  } else if ((*bi).first < (*ai).first) {
507  // Here is an entry in the new index, not present in our index.
508  PT(BamCacheRecord) record = (*bi).second;
509  Filename cache_pathname(_root, record->get_cache_filename());
510  if (cache_pathname.exists()) {
511  // The file exists; keep it.
512  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
513  }
514  ++bi;
515 
516  } else {
517  // Here is an entry we have in both.
518  PT(BamCacheRecord) a_record = (*ai).second;
519  PT(BamCacheRecord) b_record = (*bi).second;
520  if (*a_record == *b_record) {
521  // They're the same entry. It doesn't really matter which one
522  // we keep.
523  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
524 
525  } else {
526  // They're different. Just throw them both away, and re-read
527  // the current data from the cache file.
528 
529  Filename cache_pathname(_root, a_record->get_cache_filename());
530 
531  if (cache_pathname.exists()) {
532  PT(BamCacheRecord) record = do_read_record(cache_pathname, false);
533  if (record != (BamCacheRecord *)NULL) {
534  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
535  }
536  }
537  }
538 
539  ++ai;
540  ++bi;
541  }
542  }
543 
544  while (ai != old_index->_records.end()) {
545  // Here is an entry we have in our index, not present in the new
546  // index.
547  PT(BamCacheRecord) record = (*ai).second;
548  Filename cache_pathname(_root, record->get_cache_filename());
549  if (cache_pathname.exists()) {
550  // The file exists; keep it.
551  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
552  }
553  ++ai;
554  }
555 
556  while (bi != new_index->_records.end()) {
557  // Here is an entry in the new index, not present in our index.
558  PT(BamCacheRecord) record = (*bi).second;
559  Filename cache_pathname(_root, record->get_cache_filename());
560  if (cache_pathname.exists()) {
561  // The file exists; keep it.
562  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
563  }
564  ++bi;
565  }
566 
567  _index->process_new_records();
568 }
569 
570 ////////////////////////////////////////////////////////////////////
571 // Function: BamCache::rebuild_index
572 // Access: Private
573 // Description: Regenerates the index from scratch by scanning the
574 // directory.
575 ////////////////////////////////////////////////////////////////////
576 void BamCache::
577 rebuild_index() {
579 
580  PT(VirtualFileList) contents = vfs->scan_directory(_root);
581  if (contents == NULL) {
582  util_cat.error()
583  << "Unable to read directory " << _root << ", caching disabled.\n";
584  set_active(false);
585  return;
586  }
587 
588  delete _index;
589  _index = new BamCacheIndex;
590 
591  int num_files = contents->get_num_files();
592  for (int ci = 0; ci < num_files; ++ci) {
593  VirtualFile *file = contents->get_file(ci);
594  Filename filename = file->get_filename();
595  if (filename.get_extension() == "bam" ||
596  filename.get_extension() == "txo") {
597  Filename pathname(_root, filename);
598 
599  PT(BamCacheRecord) record = do_read_record(pathname, false);
600  if (record == (BamCacheRecord *)NULL) {
601  // Well, it was invalid, so blow it away.
602  if (util_cat.is_debug()) {
603  util_cat.debug()
604  << "Deleting invalid " << pathname << "\n";
605  }
606  file->delete_file();
607 
608  } else {
609  record->_record_access_time = record->_recorded_time;
610 
611  bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second;
612  if (!inserted) {
613  util_cat.info()
614  << "Multiple cache files defining " << record->get_source_pathname() << "\n";
615  file->delete_file();
616  }
617  }
618  }
619  }
620  _index->process_new_records();
621 
622  _index_stale_since = time(NULL);
623  check_cache_size();
624  flush_index();
625 }
626 
627 ////////////////////////////////////////////////////////////////////
628 // Function: BamCache::add_to_index
629 // Access: Private
630 // Description: Updates the index entry for the indicated record.
631 // Note that a copy of the record is made first.
632 ////////////////////////////////////////////////////////////////////
633 void BamCache::
634 add_to_index(const BamCacheRecord *record) {
635  PT(BamCacheRecord) new_record = record->make_copy();
636 
637  if (_index->add_record(new_record)) {
638  mark_index_stale();
639  check_cache_size();
640  }
641 }
642 
643 ////////////////////////////////////////////////////////////////////
644 // Function: BamCache::remove_from_index
645 // Access: Private
646 // Description: Removes the index entry for the indicated record, if
647 // there is one.
648 ////////////////////////////////////////////////////////////////////
649 void BamCache::
650 remove_from_index(const Filename &source_pathname) {
651  if (_index->remove_record(source_pathname)) {
652  mark_index_stale();
653  }
654 }
655 
656 ////////////////////////////////////////////////////////////////////
657 // Function: BamCache::check_cache_size
658 // Access: Private
659 // Description: If the cache size has exceeded its specified size
660 // limit, removes an old file.
661 ////////////////////////////////////////////////////////////////////
662 void BamCache::
663 check_cache_size() {
664  if (_index->_cache_size == 0) {
665  // 0 means no limit.
666  return;
667  }
668 
669  if (_index->_cache_size / 1024 > _max_kbytes) {
670  while (_index->_cache_size / 1024 > _max_kbytes) {
671  PT(BamCacheRecord) record = _index->evict_old_file();
672  if (record == NULL) {
673  // Never mind; the cache is empty.
674  break;
675  }
677  Filename cache_pathname(_root, record->get_cache_filename());
678  if (util_cat.is_debug()) {
679  util_cat.debug()
680  << "Deleting " << cache_pathname
681  << " to keep cache size below " << _max_kbytes << "K\n";
682  }
683  vfs->delete_file(cache_pathname);
684  }
685  mark_index_stale();
686  }
687 }
688 
689 ////////////////////////////////////////////////////////////////////
690 // Function: BamCache::do_read_index
691 // Access: Private, Static
692 // Description: Reads the index data from the specified filename.
693 // Returns a newly-allocated BamCacheIndex object on
694 // success, or NULL on failure.
695 ////////////////////////////////////////////////////////////////////
696 BamCacheIndex *BamCache::
697 do_read_index(const Filename &index_pathname) {
698  if (index_pathname.empty()) {
699  return NULL;
700  }
701 
702  DatagramInputFile din;
703  if (!din.open(index_pathname)) {
704  util_cat.debug()
705  << "Could not read index file: " << index_pathname << "\n";
706  return NULL;
707  }
708 
709  string head;
710  if (!din.read_header(head, _bam_header.size())) {
711  util_cat.debug()
712  << index_pathname << " is not an index file.\n";
713  return NULL;
714  }
715 
716  if (head != _bam_header) {
717  util_cat.debug()
718  << index_pathname << " is not an index file.\n";
719  return NULL;
720  }
721 
722  BamReader reader(&din);
723  if (!reader.init()) {
724  return NULL;
725  }
726 
727  TypedWritable *object = reader.read_object();
728 
729  if (object == (TypedWritable *)NULL) {
730  util_cat.error()
731  << "Cache index " << index_pathname << " is empty.\n";
732  return NULL;
733 
734  } else if (!object->is_of_type(BamCacheIndex::get_class_type())) {
735  util_cat.error()
736  << "Cache index " << index_pathname << " contains a "
737  << object->get_type() << ", not a BamCacheIndex.\n";
738  return NULL;
739  }
740 
741  BamCacheIndex *index = DCAST(BamCacheIndex, object);
742  if (!reader.resolve()) {
743  util_cat.error()
744  << "Unable to fully resolve cache index file.\n";
745  return NULL;
746  }
747 
748  return index;
749 }
750 
751 ////////////////////////////////////////////////////////////////////
752 // Function: BamCache::do_write_index
753 // Access: Private, Static
754 // Description: Writes the given index data to the specified filename.
755 ////////////////////////////////////////////////////////////////////
756 bool BamCache::
757 do_write_index(const Filename &index_pathname, const BamCacheIndex *index) {
759  DatagramOutputFile dout;
760  if (!dout.open(index_pathname)) {
761  util_cat.error()
762  << "Could not write index file: " << index_pathname << "\n";
763  vfs->delete_file(index_pathname);
764  return false;
765  }
766 
767  if (!dout.write_header(_bam_header)) {
768  util_cat.error()
769  << "Unable to write to " << index_pathname << "\n";
770  vfs->delete_file(index_pathname);
771  return false;
772  }
773 
774  {
775  BamWriter writer(&dout);
776  if (!writer.init()) {
777  vfs->delete_file(index_pathname);
778  return false;
779  }
780 
781  if (!writer.write_object(index)) {
782  vfs->delete_file(index_pathname);
783  return false;
784  }
785  }
786 
787  return true;
788 }
789 
790 ////////////////////////////////////////////////////////////////////
791 // Function: BamCache::find_and_read_record
792 // Access: Private
793 // Description: Looks for the existing cache file that corresponds
794 // to the indicated filename. Normally, this is the
795 // specified cache filename exactly; but in the case of
796 // a hash collision, it may be a variant of the cache
797 // filename.
798 ////////////////////////////////////////////////////////////////////
800 find_and_read_record(const Filename &source_pathname,
801  const Filename &cache_filename) {
802  int pass = 0;
803  while (true) {
804  PT(BamCacheRecord) record =
805  read_record(source_pathname, cache_filename, pass);
806  if (record != (BamCacheRecord *)NULL) {
807  add_to_index(record);
808  return record;
809  }
810  ++pass;
811  }
812 }
813 
814 ////////////////////////////////////////////////////////////////////
815 // Function: BamCache::read_record
816 // Access: Private
817 // Description: Reads the indicated cache file and returns its
818 // associated record if it can be read and it matches
819 // the source filename.
820 ////////////////////////////////////////////////////////////////////
822 read_record(const Filename &source_pathname,
823  const Filename &cache_filename,
824  int pass) {
826  Filename cache_pathname(_root, cache_filename);
827  if (pass != 0) {
828  ostringstream strm;
829  strm << cache_pathname.get_basename_wo_extension() << "_" << pass;
830  cache_pathname.set_basename_wo_extension(strm.str());
831  }
832 
833  if (!cache_pathname.exists()) {
834  // There is no such cache file already. Declare it.
835  if (util_cat.is_debug()) {
836  util_cat.debug()
837  << "Declaring new cache file " << cache_pathname << " for " << source_pathname << "\n";
838  }
839  PT(BamCacheRecord) record =
840  new BamCacheRecord(source_pathname, cache_filename);
841  record->_cache_pathname = cache_pathname;
842  return record;
843  }
844 
845  if (util_cat.is_debug()) {
846  util_cat.debug()
847  << "Reading cache file " << cache_pathname << " for " << source_pathname << "\n";
848  }
849 
850  PT(BamCacheRecord) record = do_read_record(cache_pathname, true);
851  if (record == (BamCacheRecord *)NULL) {
852  // Well, it was invalid, so blow it away, and make a new one.
853  if (util_cat.is_debug()) {
854  util_cat.debug()
855  << "Deleting invalid cache file " << cache_pathname << "\n";
856  }
857  vfs->delete_file(cache_pathname);
858  remove_from_index(source_pathname);
859 
860  PT(BamCacheRecord) record =
861  new BamCacheRecord(source_pathname, cache_filename);
862  record->_cache_pathname = cache_pathname;
863  return record;
864  }
865 
866  if (record->get_source_pathname() != source_pathname) {
867  // This might be just a hash conflict.
868  if (util_cat.is_debug()) {
869  util_cat.debug()
870  << "Cache file " << cache_pathname << " references "
871  << record->get_source_pathname() << ", not "
872  << source_pathname << "\n";
873  }
874  return NULL;
875  }
876 
877  if (!record->has_data()) {
878  // If we didn't find any data, the caller will have to reload it.
879  record->clear_dependent_files();
880  }
881 
882  record->_cache_pathname = cache_pathname;
883  return record;
884 }
885 
886 ////////////////////////////////////////////////////////////////////
887 // Function: BamCache::do_read_record
888 // Access: Private, Static
889 // Description: Actually reads a record from the file.
890 ////////////////////////////////////////////////////////////////////
892 do_read_record(const Filename &cache_pathname, bool read_data) {
893  DatagramInputFile din;
894  if (!din.open(cache_pathname)) {
895  if (util_cat.is_debug()) {
896  util_cat.debug()
897  << "Could not read cache file: " << cache_pathname << "\n";
898  }
899  return NULL;
900  }
901 
902  string head;
903  if (!din.read_header(head, _bam_header.size())) {
904  if (util_cat.is_debug()) {
905  util_cat.debug()
906  << cache_pathname << " is not a cache file.\n";
907  }
908  return NULL;
909  }
910 
911  if (head != _bam_header) {
912  if (util_cat.is_debug()) {
913  util_cat.debug()
914  << cache_pathname << " is not a cache file.\n";
915  }
916  return NULL;
917  }
918 
919  BamReader reader(&din);
920  if (!reader.init()) {
921  return NULL;
922  }
923 
924  TypedWritable *object = reader.read_object();
925  if (object == (TypedWritable *)NULL) {
926  if (util_cat.is_debug()) {
927  util_cat.debug()
928  << cache_pathname << " is empty.\n";
929  }
930  return NULL;
931 
932  } else if (!object->is_of_type(BamCacheRecord::get_class_type())) {
933  if (util_cat.is_debug()) {
934  util_cat.debug()
935  << "Cache file " << cache_pathname << " contains a "
936  << object->get_type() << ", not a BamCacheRecord.\n";
937  }
938  return NULL;
939  }
940 
941  PT(BamCacheRecord) record = DCAST(BamCacheRecord, object);
942  if (!reader.resolve()) {
943  if (util_cat.is_debug()) {
944  util_cat.debug()
945  << "Unable to fully resolve cache record in " << cache_pathname << "\n";
946  }
947  return NULL;
948  }
949 
950  // From this point below, we have validated that the selected
951  // filename is indeed a cache record for the indicated source file,
952  // and therefore the cache record will be returned.
953 
954  // We still need to decide whether the cache record is stale.
955  if (read_data && record->dependents_unchanged()) {
956  // The cache record doesn't appear to be stale. Load the cached
957  // object.
958  TypedWritable *ptr;
959  ReferenceCount *ref_ptr;
960 
961  if (reader.read_object(ptr, ref_ptr)) {
962  if (!reader.resolve()) {
963  if (util_cat.is_debug()) {
964  util_cat.debug()
965  << "Unable to fully resolve cached object in " << cache_pathname << "\n";
966  }
967  delete object;
968  } else {
969  // The object is valid. Store it in the record.
970  record->set_data(ptr, ref_ptr);
971  }
972  }
973  }
974 
975  // Also get the total file size.
976  PT(VirtualFile) vfile = din.get_vfile();
977  istream &in = din.get_stream();
978  in.clear();
979  record->_record_size = vfile->get_file_size(&in);
980 
981  // And the last access time is now, duh.
982  record->_record_access_time = time(NULL);
983 
984  return record;
985 }
986 
987 ////////////////////////////////////////////////////////////////////
988 // Function: BamCache::hash_filename
989 // Access: Private, Static
990 // Description: Returns the appropriate filename to use for a cache
991 // file, given the fullpath string to the source
992 // filename.
993 ////////////////////////////////////////////////////////////////////
994 string BamCache::
995 hash_filename(const string &filename) {
996 #ifdef HAVE_OPENSSL
997  // With OpenSSl, use the MD5 hash of the filename.
998  HashVal hv;
999  hv.hash_string(filename);
1000  ostringstream strm;
1001  hv.output_hex(strm);
1002  return strm.str();
1003 
1004 #else // HAVE_OPENSSL
1005  // Without OpenSSL, don't get fancy; just build a simple hash.
1006  unsigned int hash = 0;
1007  for (string::const_iterator si = filename.begin();
1008  si != filename.end();
1009  ++si) {
1010  hash = (hash * 9109) + (unsigned int)(*si);
1011  }
1012 
1013  ostringstream strm;
1014  strm << hex << setw(8) << setfill('0') << hash;
1015  return strm.str();
1016 
1017 #endif // HAVE_OPENSSL
1018 }
1019 
1020 ////////////////////////////////////////////////////////////////////
1021 // Function: BamCache::make_global
1022 // Access: Private, Static
1023 // Description: Constructs the global BamCache object.
1024 ////////////////////////////////////////////////////////////////////
1025 void BamCache::
1026 make_global() {
1027  _global_ptr = new BamCache;
1028 
1029  if (_global_ptr->_root.empty()) {
1030  _global_ptr->set_active(false);
1031  }
1032 }
virtual streampos get_file_pos()
Returns the current file position within the data stream, if any, or 0 if the file position is not me...
bool atomic_compare_and_exchange_contents(const Filename &filename, string &orig_contents, const string &old_contents, const string &new_contents)
See Filename::atomic_compare_and_exchange_contents().
void set_active(bool flag)
Changes the state of the active flag.
Definition: bamCache.I:29
void set_extension(const string &s)
Replaces the file extension.
Definition: filename.cxx:837
bool rename_file(const Filename &orig_filename, const Filename &new_filename)
Attempts to move or rename the indicated file or directory.
TypeHandle find_type(const string &name) const
Looks for a previously-registered type of the given name.
bool is_directory(const Filename &filename) const
Convenience function; returns true if the named file exists and is a directory.
This is the fundamental interface for extracting binary objects from a Bam file, as generated by a Ba...
Definition: bamReader.h:122
bool atomic_read_contents(const Filename &filename, string &contents) const
See Filename::atomic_read_contents().
This class maintains a cache of Bam and/or Txo objects generated from model files and texture images ...
Definition: bamCache.h:47
This is a convenience class to specialize ConfigVariable as a Filename type.
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS&#39;s file system.
This is a convenience class to specialize ConfigVariable as a boolean type.
This class can be used to read a binary file that consists of an arbitrary header followed by a numbe...
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:494
Base class for objects that can be written to and read from Bam files.
Definition: typedWritable.h:37
bool is_of_type(TypeHandle handle) const
Returns true if the current object is or derives from the indicated type.
Definition: typedObject.I:63
bool write_header(const string &header)
Writes a sequence of bytes to the beginning of the datagram file.
void flush_index()
Ensures the index is written to disk.
Definition: bamCache.cxx:339
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer...
Definition: hashVal.h:32
This is the fundamental interface for writing binary objects to a Bam file, to be extracted later by ...
Definition: bamWriter.h:73
bool make_directory_full(const Filename &filename)
Attempts to create a directory within the file system.
const Filename & get_cache_filename() const
Returns the name of the cache file as hashed from the source_pathname.
bool open(const FileReference *file)
Opens the indicated filename for reading.
The abstract base class for a file or directory within the VirtualFileSystem.
Definition: virtualFile.h:37
bool read_header(string &header, size_t num_bytes)
Reads a sequence of bytes from the beginning of the datagram file.
bool is_local() const
Returns true if the filename is local, e.g.
Definition: filename.I:664
static Thread * get_current_thread()
Returns a pointer to the currently-executing Thread object.
Definition: thread.I:145
bool open(const FileReference *file)
Opens the indicated filename for writing.
bool has_data() const
Returns true if this cache record has an in-memory data object associated–that is, the object stored in the cache.
static Filename temporary(const string &dirname, const string &prefix, const string &suffix=string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix...
Definition: filename.cxx:439
This represents the in-memory index that records the list of files stored in the BamCache.
Definition: bamCacheIndex.h:37
void set_file_texture_mode(BamTextureMode file_texture_mode)
Changes the BamTextureMode preference for the Bam file currently being written.
Definition: bamWriter.I:94
string get_unique_id() const
Returns a string that is guaranteed to be unique to this thread, across all processes on the machine...
Definition: thread.I:72
Filename get_cwd() const
Returns the current directory name.
bool store(BamCacheRecord *record)
Flushes a cache entry to disk.
Definition: bamCache.cxx:194
void set_root(const Filename &root)
Changes the current root pathname of the cache.
Definition: bamCache.cxx:118
A list of VirtualFiles, as returned by VirtualFile::scan_directory().
Type get_type() const
Returns the type of the file represented by the filename, as previously set by set_type().
Definition: filename.I:583
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:44
An instance of this class is written to the front of a Bam or Txo file to make the file a cached inst...
void list_index(ostream &out, int indent_level=0) const
Writes the contents of the index to standard output.
Definition: bamCache.cxx:394
void output_hex(ostream &out) const
Outputs the HashVal as a 32-digit hexadecimal number.
Definition: hashVal.cxx:31
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
virtual bool delete_file()
Attempts to delete this file or directory.
Definition: virtualFile.cxx:77
bool write_object(const TypedWritable *obj)
Writes a single object to the Bam file, so that the BamReader::read_object() can later correctly rest...
Definition: bamWriter.cxx:152
TypedWritable * get_data() const
Returns a pointer to the data stored in the record, or NULL if there is no data.
Similar to MutexHolder, but for a reentrant mutex.
Definition: reMutexHolder.h:27
bool make_relative_to(Filename directory, bool allow_backups=true)
Adjusts this filename, which must be a fully-specified pathname beginning with a slash, to make it a relative filename, relative to the fully-specified directory indicated (which must also begin with, and may or may not end with, a slash–a terminating slash is ignored).
Definition: filename.cxx:1766
const Filename & get_source_pathname() const
Returns the full pathname to the source file that originally generated this cache request...
static TypeRegistry * ptr()
Returns the pointer to the global TypeRegistry object.
This class can be used to write a binary file that consists of an arbitrary header followed by a numb...
bool exists(const Filename &filename) const
Convenience function; returns true if the named file exists.
A base class for all things that want to be reference-counted.
string get_basename() const
Returns the basename part of the filename.
Definition: filename.I:436
The TypeRegistry class maintains all the assigned TypeHandles in a given system.
Definition: typeRegistry.h:39
A thread; that is, a lightweight process.
Definition: thread.h:51
This is a convenience class to specialize ConfigVariable as an integer type.
bool delete_file(const Filename &filename)
Attempts to delete the indicated file or directory.
TypeHandle is the identifier used to differentiate C++ class types.
Definition: typeHandle.h:85
void close()
Closes the file.
string get_extension() const
Returns the file extension.
Definition: filename.I:477
bool init()
Initializes the BamWriter prior to writing any objects to its output stream.
Definition: bamWriter.cxx:98
void consider_flush_index()
Flushes the index if enough time has elapsed since the index was last flushed.
Definition: bamCache.cxx:323