Panda3D
bamCache.cxx
Go to the documentation of this file.
1 /**
2  * PANDA 3D SOFTWARE
3  * Copyright (c) Carnegie Mellon University. All rights reserved.
4  *
5  * All use of this software is subject to the terms of the revised BSD
6  * license. You should have received a copy of this license along
7  * with this source code in a file named "LICENSE."
8  *
9  * @file bamCache.cxx
10  * @author drose
11  * @date 2006-06-09
12  */
13 
14 #include "bamCache.h"
15 #include "bamCacheIndex.h"
16 #include "bamReader.h"
17 #include "bamWriter.h"
18 #include "hashVal.h"
19 #include "datagramInputFile.h"
20 #include "datagramOutputFile.h"
21 #include "config_putil.h"
22 #include "bam.h"
23 #include "typeRegistry.h"
24 #include "string_utils.h"
25 #include "configVariableInt.h"
26 #include "configVariableString.h"
27 #include "configVariableFilename.h"
28 #include "virtualFileSystem.h"
29 
30 using std::istream;
31 using std::ostream;
32 using std::ostringstream;
33 using std::string;
34 
35 BamCache *BamCache::_global_ptr = nullptr;
36 
37 /**
38  *
39  */
40 BamCache::
41 BamCache() :
42  _active(true),
43  _read_only(false),
44  _index(new BamCacheIndex),
45  _index_stale_since(0)
46 {
47  ConfigVariableFilename model_cache_dir
48  ("model-cache-dir", Filename(),
49  PRC_DESC("The full path to a directory, local to this computer, in which "
50  "model and texture files will be cached on load. If a directory "
51  "name is specified here, files may be loaded from the cache "
52  "instead of from their actual pathnames, which may save load time, "
53  "especially if you are loading egg files instead of bam files, "
54  "or if you are loading models from a shared network drive. "
55  "If this is the empty string, no cache will be used."));
56 
57  ConfigVariableInt model_cache_flush
58  ("model-cache-flush", 30,
59  PRC_DESC("This is the amount of time, in seconds, between automatic "
60  "flushes of the model-cache index."));
61 
62  ConfigVariableBool model_cache_models
63  ("model-cache-models", true,
64  PRC_DESC("If this is set to true, models will be cached in the "
65  "model cache, as bam files."));
66 
67  ConfigVariableBool model_cache_textures
68  ("model-cache-textures", true,
69  PRC_DESC("If this is set to true, textures will also be cached in the "
70  "model cache, as txo files."));
71 
72  ConfigVariableBool model_cache_compressed_textures
73  ("model-cache-compressed-textures", false,
74  PRC_DESC("If this is set to true, compressed textures will be cached "
75  "in the model cache, in their compressed form as downloaded "
76  "by the GSG. This may be set in conjunction with "
77  "model-cache-textures, or it may be independent."));
78 
79  ConfigVariableBool model_cache_compiled_shaders
80  ("model-cache-compiled-shaders", false,
81  PRC_DESC("If this is set to true, compiled shaders will be cached "
82  "in the model cache, in their binary form as downloaded "
83  "by the GSG."));
84 
85  ConfigVariableInt model_cache_max_kbytes
86  ("model-cache-max-kbytes", 10485760,
87  PRC_DESC("This is the maximum size of the model cache, in kilobytes."));
88 
89  _cache_models = model_cache_models;
90  _cache_textures = model_cache_textures;
91  _cache_compressed_textures = model_cache_compressed_textures;
92  _cache_compiled_shaders = model_cache_compiled_shaders;
93 
94  _flush_time = model_cache_flush;
95  _max_kbytes = model_cache_max_kbytes;
96 
97  if (!model_cache_dir.empty()) {
98  set_root(model_cache_dir);
99  }
100 }
101 
102 /**
103  *
104  */
105 BamCache::
106 ~BamCache() {
107  flush_index();
108  delete _index;
109  _index = nullptr;
110 }
111 
112 /**
113  * Changes the current root pathname of the cache. This specifies where the
114  * cache files are stored on disk. This should name a directory that is on a
115  * disk local to the machine (not on a network-mounted disk), for instance,
116  * /tmp/panda-cache or /c/panda-cache.
117  *
118  * If the directory does not already exist, it will be created as a result of
119  * this call.
120  */
121 void BamCache::
122 set_root(const Filename &root) {
123  ReMutexHolder holder(_lock);
124  flush_index();
125  _root = root;
126 
127  // The root filename must be a directory.
129  if (!vfs->is_directory(_root)) {
130  vfs->make_directory_full(_root);
131  }
132 
133  delete _index;
134  _index = new BamCacheIndex;
135  _index_stale_since = 0;
136 
137  if (!vfs->is_directory(_root)) {
138  util_cat.error()
139  << "Unable to make directory " << _root << ", caching disabled.\n";
140  _active = false;
141  return;
142  }
143 
144  read_index();
145  check_cache_size();
146 }
147 
148 /**
149  * Looks up a file in the cache.
150  *
151  * If the file is cacheable, then regardless of whether the file is found in
152  * the cache or not, this returns a BamCacheRecord. On the other hand, if the
153  * file cannot be cached, returns NULL.
154  *
155  * If record->has_data() returns true, then the file was found in the cache,
156  * and you may call record->extract_data() to get the object. If
157  * record->has_data() returns false, then the file was not found in the cache
158  * or the cache was stale; and you should reload the source file (calling
159  * record->add_dependent_file() for each file loaded, including the original
160  * source file), and then call record->set_data() to record the resulting
161  * loaded object; and finally, you should call store() to write the cached
162  * record to disk.
163  */
164 PT(BamCacheRecord) BamCache::
165 lookup(const Filename &source_filename, const string &cache_extension) {
166  ReMutexHolder holder(_lock);
168 
170 
171  Filename source_pathname(source_filename);
172  source_pathname.make_absolute(vfs->get_cwd());
173 
174  Filename rel_pathname(source_pathname);
175  rel_pathname.make_relative_to(_root, false);
176  if (rel_pathname.is_local()) {
177  // If the source pathname is already within the cache directory, don't
178  // cache it further.
179  return nullptr;
180  }
181 
182  Filename cache_filename = hash_filename(source_pathname.get_fullpath());
183  cache_filename.set_extension(cache_extension);
184 
185  return find_and_read_record(source_pathname, cache_filename);
186 }
187 
188 /**
189  * Flushes a cache entry to disk. You must have retrieved the cache record
190  * via a prior call to lookup(), and then stored the data via
191  * record->set_data(). Returns true on success, false on failure.
192  */
194 store(BamCacheRecord *record) {
196  ReMutexHolder holder(_lock);
197  nassertr(!record->_cache_pathname.empty(), false);
198  nassertr(record->has_data(), false);
199 
200  if (_read_only) {
201  return false;
202  }
203 
205 
206 #ifndef NDEBUG
207  // Ensure that the cache_pathname is within the _root directory tree.
208  Filename rel_pathname(record->_cache_pathname);
209  rel_pathname.make_relative_to(_root, false);
210  nassertr(rel_pathname.is_local(), false);
211 #endif // NDEBUG
212 
213  record->_recorded_time = time(nullptr);
214 
215  Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
216 
217  // We actually do the write to a temporary filename first, and then move it
218  // into place, so that no one attempts to read the file while it is in the
219  // process of being written.
220  Thread *current_thread = Thread::get_current_thread();
221  string extension = current_thread->get_unique_id() + string(".tmp");
222  Filename temp_pathname = cache_pathname;
223  temp_pathname.set_extension(extension);
224  temp_pathname.set_binary();
225 
226  DatagramOutputFile dout;
227  if (!dout.open(temp_pathname)) {
228  util_cat.error()
229  << "Could not write cache file: " << temp_pathname << "\n";
230  vfs->delete_file(temp_pathname);
231  emergency_read_only();
232  return false;
233  }
234 
235  if (!dout.write_header(_bam_header)) {
236  util_cat.error()
237  << "Unable to write to " << temp_pathname << "\n";
238  vfs->delete_file(temp_pathname);
239  return false;
240  }
241 
242  {
243  BamWriter writer(&dout);
244  if (!writer.init()) {
245  util_cat.error()
246  << "Unable to write Bam header to " << temp_pathname << "\n";
247  vfs->delete_file(temp_pathname);
248  return false;
249  }
250 
251  TypeRegistry *type_registry = TypeRegistry::ptr();
252  TypeHandle texture_type = type_registry->find_type("Texture");
253  if (record->get_data()->is_of_type(texture_type)) {
254  // Texture objects write the actual texture image.
255  writer.set_file_texture_mode(BamWriter::BTM_rawdata);
256  } else {
257  // Any other kinds of objects write texture references.
258  writer.set_file_texture_mode(BamWriter::BTM_fullpath);
259  }
260 
261  // This is necessary for relative NodePaths to work.
262  TypeHandle node_type = type_registry->find_type("PandaNode");
263  if (record->get_data()->is_of_type(node_type)) {
264  writer.set_root_node(record->get_data());
265  }
266 
267  if (!writer.write_object(record)) {
268  util_cat.error()
269  << "Unable to write object to " << temp_pathname << "\n";
270  vfs->delete_file(temp_pathname);
271  return false;
272  }
273 
274  if (!writer.write_object(record->get_data())) {
275  util_cat.error()
276  << "Unable to write object data to " << temp_pathname << "\n";
277  vfs->delete_file(temp_pathname);
278  return false;
279  }
280 
281  // Now that we are done with the BamWriter, it's important to let it
282  // destruct now and clean itself up, or it might get mad if we delete any
283  // TypedWritables below that haven't been written yet.
284  }
285 
286  record->_record_size = dout.get_file_pos();
287  dout.close();
288 
289  // Now move the file into place.
290  if (!vfs->rename_file(temp_pathname, cache_pathname) && vfs->exists(temp_pathname)) {
291  vfs->delete_file(cache_pathname);
292  if (!vfs->rename_file(temp_pathname, cache_pathname)) {
293  util_cat.error()
294  << "Unable to rename " << temp_pathname << " to "
295  << cache_pathname << "\n";
296  vfs->delete_file(temp_pathname);
297  return false;
298  }
299  }
300 
301  add_to_index(record);
302 
303  return true;
304 }
305 
306 /**
307  * Called when an attempt to write to the cache dir has failed, usually for
308  * lack of disk space or because of incorrect file permissions. Outputs an
309  * error and puts the BamCache into read-only mode.
310  */
311 void BamCache::
312 emergency_read_only() {
313  util_cat.error() <<
314  "Could not write to the Bam Cache. Disabling future attempts.\n";
315  _read_only = true;
316 }
317 
318 /**
319  * Flushes the index if enough time has elapsed since the index was last
320  * flushed.
321  */
324 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
325  if (!_lock.try_lock()) {
326  // If we can't grab the lock, no big deal. We don't want to hold up
327  // the frame waiting for a cache operation. We can try again later.
328  return;
329  }
330 #endif
331 
332  if (_index_stale_since != 0) {
333  int elapsed = (int)time(nullptr) - (int)_index_stale_since;
334  if (elapsed > _flush_time) {
335  flush_index();
336  }
337  }
338 
339 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
340  _lock.unlock();
341 #endif
342 }
343 
344 /**
345  * Ensures the index is written to disk.
346  */
348 flush_index() {
349  ReMutexHolder holder(_lock);
350  if (_index_stale_since == 0) {
351  // Never mind.
352  return;
353  }
354 
355  while (true) {
356  if (_read_only) {
357  return;
358  }
359 
360  Filename temp_pathname = Filename::temporary(_root, "index-", ".boo");
361 
362  if (!do_write_index(temp_pathname, _index)) {
363  emergency_read_only();
364  return;
365  }
366 
367  // Now atomically write the name of this index file to the index reference
368  // file.
370  Filename index_ref_pathname(_root, Filename("index_name.txt"));
371  string old_index = _index_ref_contents;
372  string new_index = temp_pathname.get_basename() + "\n";
373  string orig_index;
374 
375  if (vfs->atomic_compare_and_exchange_contents(index_ref_pathname, orig_index, old_index, new_index)) {
376  // We successfully wrote our version of the index, and no other process
377  // beat us to it. Our index is now the official one. Remove the old
378  // index.
379  vfs->delete_file(_index_pathname);
380  _index_pathname = temp_pathname;
381  _index_ref_contents = new_index;
382  _index_stale_since = 0;
383  return;
384  }
385 
386  // Shoot, some other process updated the index while we were trying to
387  // update it, and they beat us to it. We have to merge, and try again.
388  vfs->delete_file(temp_pathname);
389  _index_pathname = Filename(_root, Filename(trim(orig_index)));
390  _index_ref_contents = orig_index;
391  read_index();
392  }
393  check_cache_size();
394 }
395 
396 /**
397  * Writes the contents of the index to standard output.
398  */
400 list_index(ostream &out, int indent_level) const {
401  _index->write(out, indent_level);
402 }
403 
404 /**
405  * Reads, or re-reads the index file from disk. If _index_stale_since is
406  * nonzero, the index file is read and then merged with our current index.
407  */
408 void BamCache::
409 read_index() {
410  if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
411  // Couldn't read the index ref; rebuild the index.
412  rebuild_index();
413  return;
414  }
415 
416  while (true) {
417  BamCacheIndex *new_index = do_read_index(_index_pathname);
418  if (new_index != nullptr) {
419  merge_index(new_index);
420  return;
421  }
422 
423  // We couldn't read the index. Maybe it's been removed already. See if
424  // the index_pathname has changed.
425  Filename old_index_pathname = _index_pathname;
426  if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
427  // Couldn't read the index ref; rebuild the index.
428  rebuild_index();
429  return;
430  }
431 
432  if (old_index_pathname == _index_pathname) {
433  // Nope, we just couldn't read it. Delete it and build a new one.
435  vfs->delete_file(_index_pathname);
436  rebuild_index();
437  flush_index();
438  return;
439  }
440  }
441 }
442 
443 /**
444  * Atomically reads the current index filename from the index reference file.
445  * The index filename moves around as different processes update the index.
446  */
447 bool BamCache::
448 read_index_pathname(Filename &index_pathname, string &index_ref_contents) const {
450  index_ref_contents.clear();
451  Filename index_ref_pathname(_root, Filename("index_name.txt"));
452  if (!vfs->atomic_read_contents(index_ref_pathname, index_ref_contents)) {
453  return false;
454  }
455 
456  string trimmed = trim(index_ref_contents);
457  if (trimmed.empty()) {
458  index_pathname = Filename();
459  } else {
460  index_pathname = Filename(_root, Filename(trimmed));
461  }
462  return true;
463 }
464 
465 /**
466  * The supplied index file has been updated by some other process. Merge it
467  * with our current index.
468  *
469  * Ownership of the pointer is transferred with this call. The caller should
470  * assume that new_index will be deleted by this method.
471  */
472 void BamCache::
473 merge_index(BamCacheIndex *new_index) {
474  if (_index_stale_since == 0) {
475  // If our index isn't stale, just replace it.
476  delete _index;
477  _index = new_index;
478  return;
479  }
480 
481  BamCacheIndex *old_index = _index;
482  old_index->release_records();
483  new_index->release_records();
484  _index = new BamCacheIndex;
485 
486  BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
487  BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
488 
489  while (ai != old_index->_records.end() &&
490  bi != new_index->_records.end()) {
491  if ((*ai).first < (*bi).first) {
492  // Here is an entry we have in our index, not present in the new index.
493  PT(BamCacheRecord) record = (*ai).second;
494  Filename cache_pathname(_root, record->get_cache_filename());
495  if (cache_pathname.exists()) {
496  // The file exists; keep it.
497  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
498  }
499  ++ai;
500 
501  } else if ((*bi).first < (*ai).first) {
502  // Here is an entry in the new index, not present in our index.
503  PT(BamCacheRecord) record = (*bi).second;
504  Filename cache_pathname(_root, record->get_cache_filename());
505  if (cache_pathname.exists()) {
506  // The file exists; keep it.
507  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
508  }
509  ++bi;
510 
511  } else {
512  // Here is an entry we have in both.
513  PT(BamCacheRecord) a_record = (*ai).second;
514  PT(BamCacheRecord) b_record = (*bi).second;
515  if (*a_record == *b_record) {
516  // They're the same entry. It doesn't really matter which one we
517  // keep.
518  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
519 
520  } else {
521  // They're different. Just throw them both away, and re-read the
522  // current data from the cache file.
523 
524  Filename cache_pathname(_root, a_record->get_cache_filename());
525 
526  if (cache_pathname.exists()) {
527  PT(BamCacheRecord) record = do_read_record(cache_pathname, false);
528  if (record != nullptr) {
529  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
530  }
531  }
532  }
533 
534  ++ai;
535  ++bi;
536  }
537  }
538 
539  while (ai != old_index->_records.end()) {
540  // Here is an entry we have in our index, not present in the new index.
541  PT(BamCacheRecord) record = (*ai).second;
542  Filename cache_pathname(_root, record->get_cache_filename());
543  if (cache_pathname.exists()) {
544  // The file exists; keep it.
545  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
546  }
547  ++ai;
548  }
549 
550  while (bi != new_index->_records.end()) {
551  // Here is an entry in the new index, not present in our index.
552  PT(BamCacheRecord) record = (*bi).second;
553  Filename cache_pathname(_root, record->get_cache_filename());
554  if (cache_pathname.exists()) {
555  // The file exists; keep it.
556  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
557  }
558  ++bi;
559  }
560 
561  _index->process_new_records();
562 }
563 
564 /**
565  * Regenerates the index from scratch by scanning the directory.
566  */
567 void BamCache::
568 rebuild_index() {
570 
571  PT(VirtualFileList) contents = vfs->scan_directory(_root);
572  if (contents == nullptr) {
573  util_cat.error()
574  << "Unable to read directory " << _root << ", caching disabled.\n";
575  set_active(false);
576  return;
577  }
578 
579  delete _index;
580  _index = new BamCacheIndex;
581 
582  int num_files = contents->get_num_files();
583  for (int ci = 0; ci < num_files; ++ci) {
584  VirtualFile *file = contents->get_file(ci);
585  Filename filename = file->get_filename();
586  if (filename.get_extension() == "bam" ||
587  filename.get_extension() == "txo") {
588  Filename pathname(_root, filename);
589 
590  PT(BamCacheRecord) record = do_read_record(pathname, false);
591  if (record == nullptr) {
592  // Well, it was invalid, so blow it away.
593  if (util_cat.is_debug()) {
594  util_cat.debug()
595  << "Deleting invalid " << pathname << "\n";
596  }
597  file->delete_file();
598 
599  } else {
600  record->_record_access_time = record->_recorded_time;
601 
602  bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second;
603  if (!inserted) {
604  util_cat.info()
605  << "Multiple cache files defining " << record->get_source_pathname() << "\n";
606  file->delete_file();
607  }
608  }
609  }
610  }
611  _index->process_new_records();
612 
613  _index_stale_since = time(nullptr);
614  check_cache_size();
615  flush_index();
616 }
617 
618 /**
619  * Updates the index entry for the indicated record. Note that a copy of the
620  * record is made first.
621  */
622 void BamCache::
623 add_to_index(const BamCacheRecord *record) {
624  PT(BamCacheRecord) new_record = record->make_copy();
625 
626  if (_index->add_record(new_record)) {
627  mark_index_stale();
628  check_cache_size();
629  }
630 }
631 
632 /**
633  * Removes the index entry for the indicated record, if there is one.
634  */
635 void BamCache::
636 remove_from_index(const Filename &source_pathname) {
637  if (_index->remove_record(source_pathname)) {
638  mark_index_stale();
639  }
640 }
641 
642 /**
643  * If the cache size has exceeded its specified size limit, removes an old
644  * file.
645  */
646 void BamCache::
647 check_cache_size() {
648  if (_index->_cache_size == 0) {
649  // 0 means no limit.
650  return;
651  }
652 
653  if (_index->_cache_size / 1024 > _max_kbytes) {
654  while (_index->_cache_size / 1024 > _max_kbytes) {
655  PT(BamCacheRecord) record = _index->evict_old_file();
656  if (record == nullptr) {
657  // Never mind; the cache is empty.
658  break;
659  }
661  Filename cache_pathname(_root, record->get_cache_filename());
662  if (util_cat.is_debug()) {
663  util_cat.debug()
664  << "Deleting " << cache_pathname
665  << " to keep cache size below " << _max_kbytes << "K\n";
666  }
667  vfs->delete_file(cache_pathname);
668  }
669  mark_index_stale();
670  }
671 }
672 
673 /**
674  * Reads the index data from the specified filename. Returns a newly-
675  * allocated BamCacheIndex object on success, or NULL on failure.
676  */
677 BamCacheIndex *BamCache::
678 do_read_index(const Filename &index_pathname) {
679  if (index_pathname.empty()) {
680  return nullptr;
681  }
682 
683  DatagramInputFile din;
684  if (!din.open(index_pathname)) {
685  util_cat.debug()
686  << "Could not read index file: " << index_pathname << "\n";
687  return nullptr;
688  }
689 
690  string head;
691  if (!din.read_header(head, _bam_header.size())) {
692  util_cat.debug()
693  << index_pathname << " is not an index file.\n";
694  return nullptr;
695  }
696 
697  if (head != _bam_header) {
698  util_cat.debug()
699  << index_pathname << " is not an index file.\n";
700  return nullptr;
701  }
702 
703  BamReader reader(&din);
704  if (!reader.init()) {
705  return nullptr;
706  }
707 
708  TypedWritable *object = reader.read_object();
709 
710  if (object == nullptr) {
711  util_cat.error()
712  << "Cache index " << index_pathname << " is empty.\n";
713  return nullptr;
714 
715  } else if (!object->is_of_type(BamCacheIndex::get_class_type())) {
716  util_cat.error()
717  << "Cache index " << index_pathname << " contains a "
718  << object->get_type() << ", not a BamCacheIndex.\n";
719  return nullptr;
720  }
721 
722  BamCacheIndex *index = DCAST(BamCacheIndex, object);
723  if (!reader.resolve()) {
724  util_cat.error()
725  << "Unable to fully resolve cache index file.\n";
726  return nullptr;
727  }
728 
729  return index;
730 }
731 
732 /**
733  * Writes the given index data to the specified filename.
734  */
735 bool BamCache::
736 do_write_index(const Filename &index_pathname, const BamCacheIndex *index) {
738  DatagramOutputFile dout;
739  if (!dout.open(index_pathname)) {
740  util_cat.error()
741  << "Could not write index file: " << index_pathname << "\n";
742  vfs->delete_file(index_pathname);
743  return false;
744  }
745 
746  if (!dout.write_header(_bam_header)) {
747  util_cat.error()
748  << "Unable to write to " << index_pathname << "\n";
749  vfs->delete_file(index_pathname);
750  return false;
751  }
752 
753  {
754  BamWriter writer(&dout);
755  if (!writer.init()) {
756  vfs->delete_file(index_pathname);
757  return false;
758  }
759 
760  if (!writer.write_object(index)) {
761  vfs->delete_file(index_pathname);
762  return false;
763  }
764  }
765 
766  return true;
767 }
768 
769 /**
770  * Looks for the existing cache file that corresponds to the indicated
771  * filename. Normally, this is the specified cache filename exactly; but in
772  * the case of a hash collision, it may be a variant of the cache filename.
773  */
774 PT(BamCacheRecord) BamCache::
775 find_and_read_record(const Filename &source_pathname,
776  const Filename &cache_filename) {
777  int pass = 0;
778  while (true) {
779  PT(BamCacheRecord) record =
780  read_record(source_pathname, cache_filename, pass);
781  if (record != nullptr) {
782  add_to_index(record);
783  return record;
784  }
785  ++pass;
786  }
787 }
788 
789 /**
790  * Reads the indicated cache file and returns its associated record if it can
791  * be read and it matches the source filename.
792  */
793 PT(BamCacheRecord) BamCache::
794 read_record(const Filename &source_pathname,
795  const Filename &cache_filename,
796  int pass) {
798  Filename cache_pathname(_root, cache_filename);
799  if (pass != 0) {
800  ostringstream strm;
801  strm << cache_pathname.get_basename_wo_extension() << "_" << pass;
802  cache_pathname.set_basename_wo_extension(strm.str());
803  }
804 
805  if (!cache_pathname.exists()) {
806  // There is no such cache file already. Declare it.
807  if (util_cat.is_debug()) {
808  util_cat.debug()
809  << "Declaring new cache file " << cache_pathname << " for " << source_pathname << "\n";
810  }
811  PT(BamCacheRecord) record =
812  new BamCacheRecord(source_pathname, cache_filename);
813  record->_cache_pathname = cache_pathname;
814  return record;
815  }
816 
817  if (util_cat.is_debug()) {
818  util_cat.debug()
819  << "Reading cache file " << cache_pathname << " for " << source_pathname << "\n";
820  }
821 
822  PT(BamCacheRecord) record = do_read_record(cache_pathname, true);
823  if (record == nullptr) {
824  // Well, it was invalid, so blow it away, and make a new one.
825  if (util_cat.is_debug()) {
826  util_cat.debug()
827  << "Deleting invalid cache file " << cache_pathname << "\n";
828  }
829  vfs->delete_file(cache_pathname);
830  remove_from_index(source_pathname);
831 
832  PT(BamCacheRecord) record =
833  new BamCacheRecord(source_pathname, cache_filename);
834  record->_cache_pathname = cache_pathname;
835  return record;
836  }
837 
838  if (record->get_source_pathname() != source_pathname) {
839  // This might be just a hash conflict.
840  if (util_cat.is_debug()) {
841  util_cat.debug()
842  << "Cache file " << cache_pathname << " references "
843  << record->get_source_pathname() << ", not "
844  << source_pathname << "\n";
845  }
846  return nullptr;
847  }
848 
849  if (!record->has_data()) {
850  // If we didn't find any data, the caller will have to reload it.
851  record->clear_dependent_files();
852  }
853 
854  record->_cache_pathname = cache_pathname;
855  return record;
856 }
857 
858 /**
859  * Actually reads a record from the file.
860  */
861 PT(BamCacheRecord) BamCache::
862 do_read_record(const Filename &cache_pathname, bool read_data) {
863  DatagramInputFile din;
864  if (!din.open(cache_pathname)) {
865  if (util_cat.is_debug()) {
866  util_cat.debug()
867  << "Could not read cache file: " << cache_pathname << "\n";
868  }
869  return nullptr;
870  }
871 
872  string head;
873  if (!din.read_header(head, _bam_header.size())) {
874  if (util_cat.is_debug()) {
875  util_cat.debug()
876  << cache_pathname << " is not a cache file.\n";
877  }
878  return nullptr;
879  }
880 
881  if (head != _bam_header) {
882  if (util_cat.is_debug()) {
883  util_cat.debug()
884  << cache_pathname << " is not a cache file.\n";
885  }
886  return nullptr;
887  }
888 
889  BamReader reader(&din);
890  if (!reader.init()) {
891  return nullptr;
892  }
893 
894  TypedWritable *object = reader.read_object();
895  if (object == nullptr) {
896  if (util_cat.is_debug()) {
897  util_cat.debug()
898  << cache_pathname << " is empty.\n";
899  }
900  return nullptr;
901 
902  } else if (!object->is_of_type(BamCacheRecord::get_class_type())) {
903  if (util_cat.is_debug()) {
904  util_cat.debug()
905  << "Cache file " << cache_pathname << " contains a "
906  << object->get_type() << ", not a BamCacheRecord.\n";
907  }
908  return nullptr;
909  }
910 
911  PT(BamCacheRecord) record = DCAST(BamCacheRecord, object);
912  if (!reader.resolve()) {
913  if (util_cat.is_debug()) {
914  util_cat.debug()
915  << "Unable to fully resolve cache record in " << cache_pathname << "\n";
916  }
917  return nullptr;
918  }
919 
920  // From this point below, we have validated that the selected filename is
921  // indeed a cache record for the indicated source file, and therefore the
922  // cache record will be returned.
923 
924  // We still need to decide whether the cache record is stale.
925  if (read_data && record->dependents_unchanged()) {
926  // The cache record doesn't appear to be stale. Load the cached object.
927  TypedWritable *ptr;
928  ReferenceCount *ref_ptr;
929 
930  if (reader.read_object(ptr, ref_ptr)) {
931  if (!reader.resolve()) {
932  if (util_cat.is_debug()) {
933  util_cat.debug()
934  << "Unable to fully resolve cached object in " << cache_pathname << "\n";
935  }
936  delete object;
937  } else {
938  // The object is valid. Store it in the record.
939  record->set_data(ptr, ref_ptr);
940  }
941  }
942  }
943 
944  // Also get the total file size.
945  PT(VirtualFile) vfile = din.get_vfile();
946  istream &in = din.get_stream();
947  in.clear();
948  record->_record_size = vfile->get_file_size(&in);
949 
950  // And the last access time is now, duh.
951  record->_record_access_time = time(nullptr);
952 
953  return record;
954 }
955 
956 /**
957  * Returns the appropriate filename to use for a cache file, given the
958  * fullpath string to the source filename.
959  */
960 string BamCache::
961 hash_filename(const string &filename) {
962 #ifdef HAVE_OPENSSL
963  // With OpenSSl, use the MD5 hash of the filename.
964  HashVal hv;
965  hv.hash_string(filename);
966  ostringstream strm;
967  hv.output_hex(strm);
968  return strm.str();
969 
970 #else // HAVE_OPENSSL
971  // Without OpenSSL, don't get fancy; just build a simple hash.
972  unsigned int hash = 0;
973  for (string::const_iterator si = filename.begin();
974  si != filename.end();
975  ++si) {
976  hash = (hash * 9109) + (unsigned int)(*si);
977  }
978 
979  ostringstream strm;
980  strm << std::hex << std::setw(8) << std::setfill('0') << hash;
981  return strm.str();
982 
983 #endif // HAVE_OPENSSL
984 }
985 
986 /**
987  * Constructs the global BamCache object.
988  */
989 void BamCache::
990 make_global() {
991  _global_ptr = new BamCache;
992 
993  if (_global_ptr->_root.empty()) {
994  _global_ptr->set_active(false);
995  }
996 }
bamCache.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
ConfigVariableBool
This is a convenience class to specialize ConfigVariable as a boolean type.
Definition: configVariableBool.h:23
config_putil.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
VirtualFileSystem::get_cwd
Filename get_cwd() const
Returns the current directory name.
Definition: virtualFileSystem.cxx:455
BamCache::consider_flush_index
void consider_flush_index()
Flushes the index if enough time has elapsed since the index was last flushed.
Definition: bamCache.cxx:323
BamCacheRecord::set_data
set_data
Stores a new data object on the record.
Definition: bamCacheRecord.h:77
BamCacheRecord
An instance of this class is written to the front of a Bam or Txo file to make the file a cached inst...
Definition: bamCacheRecord.h:36
DatagramInputFile::read_header
bool read_header(std::string &header, size_t num_bytes)
Reads a sequence of bytes from the beginning of the datagram file.
Definition: datagramInputFile.cxx:104
TypeRegistry::find_type
TypeHandle find_type(const std::string &name) const
Looks for a previously-registered type of the given name.
Definition: typeRegistry.cxx:233
ReMutexDirect::unlock
void unlock()
Alias for release() to match C++11 semantics.
Definition: reMutexDirect.I:62
string_utils.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
ReMutexDirect::try_lock
bool try_lock()
Alias for try_acquire() to match C++11 semantics.
Definition: reMutexDirect.I:48
ReMutexHolder
Similar to MutexHolder, but for a reentrant mutex.
Definition: reMutexHolder.h:25
ReferenceCount
A base class for all things that want to be reference-counted.
Definition: referenceCount.h:38
datagramInputFile.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
DatagramInputFile::get_stream
std::istream & get_stream()
Returns the istream represented by the input file.
Definition: datagramInputFile.I:47
VirtualFileSystem::is_directory
bool is_directory(const Filename &filename) const
Convenience function; returns true if the named file exists and is a directory.
Definition: virtualFileSystem.I:27
BamWriter::set_root_node
set_root_node
Sets the root node of the part of the scene graph we are currently writing out.
Definition: bamWriter.h:96
BamReader
This is the fundamental interface for extracting binary objects from a Bam file, as generated by a Ba...
Definition: bamReader.h:110
DatagramOutputFile::close
void close()
Closes the file.
Definition: datagramOutputFile.cxx:73
BamCache::set_active
set_active
Changes the state of the active flag.
Definition: bamCache.h:88
BamWriter
This is the fundamental interface for writing binary objects to a Bam file, to be extracted later by ...
Definition: bamWriter.h:63
bam.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
DatagramInputFile::open
bool open(const FileReference *file)
Opens the indicated filename for reading.
Definition: datagramInputFile.cxx:33
DatagramInputFile
This class can be used to read a binary file that consists of an arbitrary header followed by a numbe...
Definition: datagramInputFile.h:28
bamReader.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PT
PT(BamCacheRecord) BamCache
Looks up a file in the cache.
Definition: bamCache.cxx:164
VirtualFileSystem::exists
bool exists(const Filename &filename) const
Convenience function; returns true if the named file exists.
Definition: virtualFileSystem.I:18
TypedWritable
Base class for objects that can be written to and read from Bam files.
Definition: typedWritable.h:35
DatagramInputFile::get_vfile
virtual VirtualFile * get_vfile()
Returns the VirtualFile that provides the source for these datagrams, if any, or NULL if the datagram...
Definition: datagramInputFile.cxx:353
Thread::get_unique_id
get_unique_id
Returns a string that is guaranteed to be unique to this thread, across all processes on the machine,...
Definition: thread.h:104
BamCache::list_index
void list_index(std::ostream &out, int indent_level=0) const
Writes the contents of the index to standard output.
Definition: bamCache.cxx:400
DatagramOutputFile::get_file_pos
virtual std::streampos get_file_pos()
Returns the current file position within the data stream, if any, or 0 if the file position is not me...
Definition: datagramOutputFile.cxx:317
Thread::get_current_thread
get_current_thread
Returns a pointer to the currently-executing Thread object.
Definition: thread.h:109
BamWriter::set_file_texture_mode
void set_file_texture_mode(BamTextureMode file_texture_mode)
Changes the BamTextureMode preference for the Bam file currently being written.
Definition: bamWriter.I:101
BamCache::flush_index
void flush_index()
Ensures the index is written to disk.
Definition: bamCache.cxx:348
TypeHandle
TypeHandle is the identifier used to differentiate C++ class types.
Definition: typeHandle.h:81
configVariableInt.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
VirtualFileSystem::atomic_read_contents
bool atomic_read_contents(const Filename &filename, std::string &contents) const
See Filename::atomic_read_contents().
Definition: virtualFileSystem.cxx:1015
VirtualFileSystem::make_directory_full
bool make_directory_full(const Filename &filename)
Attempts to create a directory within the file system.
Definition: virtualFileSystem.cxx:483
HashVal::output_hex
void output_hex(std::ostream &out) const
Outputs the HashVal as a 32-digit hexadecimal number.
Definition: hashVal.cxx:34
VirtualFileSystem::atomic_compare_and_exchange_contents
bool atomic_compare_and_exchange_contents(const Filename &filename, std::string &orig_contents, const std::string &old_contents, const std::string &new_contents)
See Filename::atomic_compare_and_exchange_contents().
Definition: virtualFileSystem.cxx:1000
hashVal.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
VirtualFileSystem
A hierarchy of directories and files that appears to be one continuous file system,...
Definition: virtualFileSystem.h:40
BamWriter::write_object
bool write_object(const TypedWritable *obj)
Writes a single object to the Bam file, so that the BamReader::read_object() can later correctly rest...
Definition: bamWriter.cxx:204
BamCache::store
bool store(BamCacheRecord *record)
Flushes a cache entry to disk.
Definition: bamCache.cxx:194
bamCacheIndex.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
datagramOutputFile.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
VirtualFileList
A list of VirtualFiles, as returned by VirtualFile::scan_directory().
Definition: virtualFileList.h:25
Filename::get_fullpath
std::string get_fullpath() const
Returns the entire filename: directory, basename, extension.
Definition: filename.I:338
DatagramOutputFile::write_header
bool write_header(const std::string &header)
Writes a sequence of bytes to the beginning of the datagram file.
Definition: datagramOutputFile.cxx:96
BamCache::set_root
set_root
Changes the current root pathname of the cache.
Definition: bamCache.h:95
Filename::is_local
bool is_local() const
Returns true if the filename is local, e.g.
Definition: filename.I:549
Filename::temporary
static Filename temporary(const std::string &dirname, const std::string &prefix, const std::string &suffix=std::string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix.
Definition: filename.cxx:424
BamCacheRecord::clear_dependent_files
void clear_dependent_files()
Empties the list of files that contribute to the data in this record.
Definition: bamCacheRecord.cxx:137
DatagramOutputFile
This class can be used to write a binary file that consists of an arbitrary header followed by a numb...
Definition: datagramOutputFile.h:30
Filename::get_type
Type get_type() const
Returns the type of the file represented by the filename, as previously set by set_type().
Definition: filename.I:485
ConfigVariableFilename
This is a convenience class to specialize ConfigVariable as a Filename type.
Definition: configVariableFilename.h:27
BamCacheRecord::get_data
get_data
Returns a pointer to the data stored in the record, or NULL if there is no data.
Definition: bamCacheRecord.h:77
VirtualFileSystem::delete_file
bool delete_file(const Filename &filename)
Attempts to delete the indicated file or directory.
Definition: virtualFileSystem.cxx:575
BamCache
This class maintains a cache of Bam and/or Txo objects generated from model files and texture images ...
Definition: bamCache.h:42
BamCacheIndex
This represents the in-memory index that records the list of files stored in the BamCache.
Definition: bamCacheIndex.h:33
VirtualFileSystem::get_global_ptr
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
Definition: virtualFileSystem.cxx:741
virtualFileSystem.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
TypeRegistry::ptr
static TypeRegistry * ptr()
Returns the pointer to the global TypeRegistry object.
Definition: typeRegistry.I:30
VirtualFile
The abstract base class for a file or directory within the VirtualFileSystem.
Definition: virtualFile.h:35
BamWriter::init
bool init()
Initializes the BamWriter prior to writing any objects to its output stream.
Definition: bamWriter.cxx:147
Filename::set_extension
void set_extension(const std::string &s)
Replaces the file extension.
Definition: filename.cxx:804
Filename::make_absolute
void make_absolute()
Converts the filename to a fully-qualified pathname from the root (if it is a relative pathname),...
Definition: filename.cxx:968
HashVal
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer.
Definition: hashVal.h:31
trim
string trim(const string &str)
Returns a new string representing the contents of the given string with both leading and trailing whi...
Definition: string_utils.cxx:281
TypeRegistry
The TypeRegistry class maintains all the assigned TypeHandles in a given system.
Definition: typeRegistry.h:36
typeRegistry.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
configVariableFilename.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
bamWriter.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
Filename::make_relative_to
bool make_relative_to(Filename directory, bool allow_backups=true)
Adjusts this filename, which must be a fully-specified pathname beginning with a slash,...
Definition: filename.cxx:1640
configVariableString.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
ConfigVariableInt
This is a convenience class to specialize ConfigVariable as an integer type.
Definition: configVariableInt.h:24
Thread
A thread; that is, a lightweight process.
Definition: thread.h:46
VirtualFileSystem::rename_file
bool rename_file(const Filename &orig_filename, const Filename &new_filename)
Attempts to move or rename the indicated file or directory.
Definition: virtualFileSystem.cxx:596
DatagramOutputFile::open
bool open(const FileReference *file)
Opens the indicated filename for writing.
Definition: datagramOutputFile.cxx:28
Filename::get_extension
std::string get_extension() const
Returns the file extension.
Definition: filename.I:400
Filename::get_basename
std::string get_basename() const
Returns the basename part of the filename.
Definition: filename.I:367
VirtualFile::delete_file
virtual bool delete_file()
Attempts to delete this file or directory.
Definition: virtualFile.cxx:69
BamCacheRecord::dependents_unchanged
bool dependents_unchanged() const
Returns true if all of the dependent files are still the same as when the cache was recorded,...
Definition: bamCacheRecord.cxx:84
Filename
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:39
Filename::set_binary
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:414
TypedObject::is_of_type
bool is_of_type(TypeHandle handle) const
Returns true if the current object is or derives from the indicated type.
Definition: typedObject.I:28