Panda3D
bamCache.cxx
Go to the documentation of this file.
1 /**
2  * PANDA 3D SOFTWARE
3  * Copyright (c) Carnegie Mellon University. All rights reserved.
4  *
5  * All use of this software is subject to the terms of the revised BSD
6  * license. You should have received a copy of this license along
7  * with this source code in a file named "LICENSE."
8  *
9  * @file bamCache.cxx
10  * @author drose
11  * @date 2006-06-09
12  */
13 
14 #include "bamCache.h"
15 #include "bamCacheIndex.h"
16 #include "bamReader.h"
17 #include "bamWriter.h"
18 #include "hashVal.h"
19 #include "datagramInputFile.h"
20 #include "datagramOutputFile.h"
21 #include "config_putil.h"
22 #include "bam.h"
23 #include "typeRegistry.h"
24 #include "string_utils.h"
25 #include "configVariableInt.h"
26 #include "configVariableString.h"
27 #include "configVariableFilename.h"
28 #include "virtualFileSystem.h"
29 
30 using std::istream;
31 using std::ostream;
32 using std::ostringstream;
33 using std::string;
34 
35 BamCache *BamCache::_global_ptr = nullptr;
36 
37 /**
38  *
39  */
40 BamCache::
41 BamCache() :
42  _active(true),
43  _read_only(false),
44  _index(new BamCacheIndex),
45  _index_stale_since(0)
46 {
47  ConfigVariableFilename model_cache_dir
48  ("model-cache-dir", Filename(),
49  PRC_DESC("The full path to a directory, local to this computer, in which "
50  "model and texture files will be cached on load. If a directory "
51  "name is specified here, files may be loaded from the cache "
52  "instead of from their actual pathnames, which may save load time, "
53  "especially if you are loading egg files instead of bam files, "
54  "or if you are loading models from a shared network drive. "
55  "If this is the empty string, no cache will be used."));
56 
57  ConfigVariableInt model_cache_flush
58  ("model-cache-flush", 30,
59  PRC_DESC("This is the amount of time, in seconds, between automatic "
60  "flushes of the model-cache index."));
61 
62  ConfigVariableBool model_cache_models
63  ("model-cache-models", true,
64  PRC_DESC("If this is set to true, models will be cached in the "
65  "model cache, as bam files."));
66 
67  ConfigVariableBool model_cache_textures
68  ("model-cache-textures", true,
69  PRC_DESC("If this is set to true, textures will also be cached in the "
70  "model cache, as txo files."));
71 
72  ConfigVariableBool model_cache_compressed_textures
73  ("model-cache-compressed-textures", false,
74  PRC_DESC("If this is set to true, compressed textures will be cached "
75  "in the model cache, in their compressed form as downloaded "
76  "by the GSG. This may be set in conjunction with "
77  "model-cache-textures, or it may be independent."));
78 
79  ConfigVariableBool model_cache_compiled_shaders
80  ("model-cache-compiled-shaders", false,
81  PRC_DESC("If this is set to true, compiled shaders will be cached "
82  "in the model cache, in their binary form as downloaded "
83  "by the GSG."));
84 
85  ConfigVariableInt model_cache_max_kbytes
86  ("model-cache-max-kbytes", 10485760,
87  PRC_DESC("This is the maximum size of the model cache, in kilobytes."));
88 
89  _cache_models = model_cache_models;
90  _cache_textures = model_cache_textures;
91  _cache_compressed_textures = model_cache_compressed_textures;
92  _cache_compiled_shaders = model_cache_compiled_shaders;
93 
94  _flush_time = model_cache_flush;
95  _max_kbytes = model_cache_max_kbytes;
96 
97  if (!model_cache_dir.empty()) {
98  set_root(model_cache_dir);
99  }
100 }
101 
102 /**
103  *
104  */
105 BamCache::
106 ~BamCache() {
107  flush_index();
108  delete _index;
109  _index = nullptr;
110 }
111 
112 /**
113  * Changes the current root pathname of the cache. This specifies where the
114  * cache files are stored on disk. This should name a directory that is on a
115  * disk local to the machine (not on a network-mounted disk), for instance,
116  * /tmp/panda-cache or /c/panda-cache.
117  *
118  * If the directory does not already exist, it will be created as a result of
119  * this call.
120  */
121 void BamCache::
122 set_root(const Filename &root) {
123  ReMutexHolder holder(_lock);
124  flush_index();
125  _root = root;
126 
127  // The root filename must be a directory.
129  if (!vfs->is_directory(_root)) {
130  vfs->make_directory_full(_root);
131  }
132 
133  delete _index;
134  _index = new BamCacheIndex;
135  _index_stale_since = 0;
136  read_index();
137  check_cache_size();
138 
139  nassertv(vfs->is_directory(_root));
140 }
141 
142 /**
143  * Looks up a file in the cache.
144  *
145  * If the file is cacheable, then regardless of whether the file is found in
146  * the cache or not, this returns a BamCacheRecord. On the other hand, if the
147  * file cannot be cached, returns NULL.
148  *
149  * If record->has_data() returns true, then the file was found in the cache,
150  * and you may call record->extract_data() to get the object. If
151  * record->has_data() returns false, then the file was not found in the cache
152  * or the cache was stale; and you should reload the source file (calling
153  * record->add_dependent_file() for each file loaded, including the original
154  * source file), and then call record->set_data() to record the resulting
155  * loaded object; and finally, you should call store() to write the cached
156  * record to disk.
157  */
158 PT(BamCacheRecord) BamCache::
159 lookup(const Filename &source_filename, const string &cache_extension) {
160  ReMutexHolder holder(_lock);
162 
164 
165  Filename source_pathname(source_filename);
166  source_pathname.make_absolute(vfs->get_cwd());
167 
168  Filename rel_pathname(source_pathname);
169  rel_pathname.make_relative_to(_root, false);
170  if (rel_pathname.is_local()) {
171  // If the source pathname is already within the cache directory, don't
172  // cache it further.
173  return nullptr;
174  }
175 
176  Filename cache_filename = hash_filename(source_pathname.get_fullpath());
177  cache_filename.set_extension(cache_extension);
178 
179  return find_and_read_record(source_pathname, cache_filename);
180 }
181 
182 /**
183  * Flushes a cache entry to disk. You must have retrieved the cache record
184  * via a prior call to lookup(), and then stored the data via
185  * record->set_data(). Returns true on success, false on failure.
186  */
187 bool BamCache::
190  ReMutexHolder holder(_lock);
191  nassertr(!record->_cache_pathname.empty(), false);
192  nassertr(record->has_data(), false);
193 
194  if (_read_only) {
195  return false;
196  }
197 
199 
200 #ifndef NDEBUG
201  // Ensure that the cache_pathname is within the _root directory tree.
202  Filename rel_pathname(record->_cache_pathname);
203  rel_pathname.make_relative_to(_root, false);
204  nassertr(rel_pathname.is_local(), false);
205 #endif // NDEBUG
206 
207  record->_recorded_time = time(nullptr);
208 
209  Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
210 
211  // We actually do the write to a temporary filename first, and then move it
212  // into place, so that no one attempts to read the file while it is in the
213  // process of being written.
214  Thread *current_thread = Thread::get_current_thread();
215  string extension = current_thread->get_unique_id() + string(".tmp");
216  Filename temp_pathname = cache_pathname;
217  temp_pathname.set_extension(extension);
218  temp_pathname.set_binary();
219 
220  DatagramOutputFile dout;
221  if (!dout.open(temp_pathname)) {
222  util_cat.error()
223  << "Could not write cache file: " << temp_pathname << "\n";
224  vfs->delete_file(temp_pathname);
225  emergency_read_only();
226  return false;
227  }
228 
229  if (!dout.write_header(_bam_header)) {
230  util_cat.error()
231  << "Unable to write to " << temp_pathname << "\n";
232  vfs->delete_file(temp_pathname);
233  return false;
234  }
235 
236  {
237  BamWriter writer(&dout);
238  if (!writer.init()) {
239  util_cat.error()
240  << "Unable to write Bam header to " << temp_pathname << "\n";
241  vfs->delete_file(temp_pathname);
242  return false;
243  }
244 
245  TypeRegistry *type_registry = TypeRegistry::ptr();
246  TypeHandle texture_type = type_registry->find_type("Texture");
247  if (record->get_data()->is_of_type(texture_type)) {
248  // Texture objects write the actual texture image.
249  writer.set_file_texture_mode(BamWriter::BTM_rawdata);
250  } else {
251  // Any other kinds of objects write texture references.
252  writer.set_file_texture_mode(BamWriter::BTM_fullpath);
253  }
254 
255  // This is necessary for relative NodePaths to work.
256  TypeHandle node_type = type_registry->find_type("PandaNode");
257  if (record->get_data()->is_of_type(node_type)) {
258  writer.set_root_node(record->get_data());
259  }
260 
261  if (!writer.write_object(record)) {
262  util_cat.error()
263  << "Unable to write object to " << temp_pathname << "\n";
264  vfs->delete_file(temp_pathname);
265  return false;
266  }
267 
268  if (!writer.write_object(record->get_data())) {
269  util_cat.error()
270  << "Unable to write object data to " << temp_pathname << "\n";
271  vfs->delete_file(temp_pathname);
272  return false;
273  }
274 
275  // Now that we are done with the BamWriter, it's important to let it
276  // destruct now and clean itself up, or it might get mad if we delete any
277  // TypedWritables below that haven't been written yet.
278  }
279 
280  record->_record_size = dout.get_file_pos();
281  dout.close();
282 
283  // Now move the file into place.
284  if (!vfs->rename_file(temp_pathname, cache_pathname) && vfs->exists(temp_pathname)) {
285  vfs->delete_file(cache_pathname);
286  if (!vfs->rename_file(temp_pathname, cache_pathname)) {
287  util_cat.error()
288  << "Unable to rename " << temp_pathname << " to "
289  << cache_pathname << "\n";
290  vfs->delete_file(temp_pathname);
291  return false;
292  }
293  }
294 
295  add_to_index(record);
296 
297  return true;
298 }
299 
300 /**
301  * Called when an attempt to write to the cache dir has failed, usually for
302  * lack of disk space or because of incorrect file permissions. Outputs an
303  * error and puts the BamCache into read-only mode.
304  */
305 void BamCache::
306 emergency_read_only() {
307  util_cat.error() <<
308  "Could not write to the Bam Cache. Disabling future attempts.\n";
309  _read_only = true;
310 }
311 
312 /**
313  * Flushes the index if enough time has elapsed since the index was last
314  * flushed.
315  */
316 void BamCache::
318 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
319  if (!_lock.try_lock()) {
320  // If we can't grab the lock, no big deal. We don't want to hold up
321  // the frame waiting for a cache operation. We can try again later.
322  return;
323  }
324 #endif
325 
326  if (_index_stale_since != 0) {
327  int elapsed = (int)time(nullptr) - (int)_index_stale_since;
328  if (elapsed > _flush_time) {
329  flush_index();
330  }
331  }
332 
333 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
334  _lock.unlock();
335 #endif
336 }
337 
338 /**
339  * Ensures the index is written to disk.
340  */
341 void BamCache::
343  ReMutexHolder holder(_lock);
344  if (_index_stale_since == 0) {
345  // Never mind.
346  return;
347  }
348 
349  while (true) {
350  if (_read_only) {
351  return;
352  }
353 
354  Filename temp_pathname = Filename::temporary(_root, "index-", ".boo");
355 
356  if (!do_write_index(temp_pathname, _index)) {
357  emergency_read_only();
358  return;
359  }
360 
361  // Now atomically write the name of this index file to the index reference
362  // file.
364  Filename index_ref_pathname(_root, Filename("index_name.txt"));
365  string old_index = _index_ref_contents;
366  string new_index = temp_pathname.get_basename() + "\n";
367  string orig_index;
368 
369  if (vfs->atomic_compare_and_exchange_contents(index_ref_pathname, orig_index, old_index, new_index)) {
370  // We successfully wrote our version of the index, and no other process
371  // beat us to it. Our index is now the official one. Remove the old
372  // index.
373  vfs->delete_file(_index_pathname);
374  _index_pathname = temp_pathname;
375  _index_ref_contents = new_index;
376  _index_stale_since = 0;
377  return;
378  }
379 
380  // Shoot, some other process updated the index while we were trying to
381  // update it, and they beat us to it. We have to merge, and try again.
382  vfs->delete_file(temp_pathname);
383  _index_pathname = Filename(_root, Filename(trim(orig_index)));
384  _index_ref_contents = orig_index;
385  read_index();
386  }
387  check_cache_size();
388 }
389 
390 /**
391  * Writes the contents of the index to standard output.
392  */
393 void BamCache::
394 list_index(ostream &out, int indent_level) const {
395  _index->write(out, indent_level);
396 }
397 
398 /**
399  * Reads, or re-reads the index file from disk. If _index_stale_since is
400  * nonzero, the index file is read and then merged with our current index.
401  */
402 void BamCache::
403 read_index() {
404  if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
405  // Couldn't read the index ref; rebuild the index.
406  rebuild_index();
407  return;
408  }
409 
410  while (true) {
411  BamCacheIndex *new_index = do_read_index(_index_pathname);
412  if (new_index != nullptr) {
413  merge_index(new_index);
414  return;
415  }
416 
417  // We couldn't read the index. Maybe it's been removed already. See if
418  // the index_pathname has changed.
419  Filename old_index_pathname = _index_pathname;
420  if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
421  // Couldn't read the index ref; rebuild the index.
422  rebuild_index();
423  return;
424  }
425 
426  if (old_index_pathname == _index_pathname) {
427  // Nope, we just couldn't read it. Delete it and build a new one.
429  vfs->delete_file(_index_pathname);
430  rebuild_index();
431  flush_index();
432  return;
433  }
434  }
435 }
436 
437 /**
438  * Atomically reads the current index filename from the index reference file.
439  * The index filename moves around as different processes update the index.
440  */
441 bool BamCache::
442 read_index_pathname(Filename &index_pathname, string &index_ref_contents) const {
444  index_ref_contents.clear();
445  Filename index_ref_pathname(_root, Filename("index_name.txt"));
446  if (!vfs->atomic_read_contents(index_ref_pathname, index_ref_contents)) {
447  return false;
448  }
449 
450  string trimmed = trim(index_ref_contents);
451  if (trimmed.empty()) {
452  index_pathname = Filename();
453  } else {
454  index_pathname = Filename(_root, Filename(trimmed));
455  }
456  return true;
457 }
458 
459 /**
460  * The supplied index file has been updated by some other process. Merge it
461  * with our current index.
462  *
463  * Ownership of the pointer is transferred with this call. The caller should
464  * assume that new_index will be deleted by this method.
465  */
466 void BamCache::
467 merge_index(BamCacheIndex *new_index) {
468  if (_index_stale_since == 0) {
469  // If our index isn't stale, just replace it.
470  delete _index;
471  _index = new_index;
472  return;
473  }
474 
475  BamCacheIndex *old_index = _index;
476  old_index->release_records();
477  new_index->release_records();
478  _index = new BamCacheIndex;
479 
480  BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
481  BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
482 
483  while (ai != old_index->_records.end() &&
484  bi != new_index->_records.end()) {
485  if ((*ai).first < (*bi).first) {
486  // Here is an entry we have in our index, not present in the new index.
487  PT(BamCacheRecord) record = (*ai).second;
488  Filename cache_pathname(_root, record->get_cache_filename());
489  if (cache_pathname.exists()) {
490  // The file exists; keep it.
491  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
492  }
493  ++ai;
494 
495  } else if ((*bi).first < (*ai).first) {
496  // Here is an entry in the new index, not present in our index.
497  PT(BamCacheRecord) record = (*bi).second;
498  Filename cache_pathname(_root, record->get_cache_filename());
499  if (cache_pathname.exists()) {
500  // The file exists; keep it.
501  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
502  }
503  ++bi;
504 
505  } else {
506  // Here is an entry we have in both.
507  PT(BamCacheRecord) a_record = (*ai).second;
508  PT(BamCacheRecord) b_record = (*bi).second;
509  if (*a_record == *b_record) {
510  // They're the same entry. It doesn't really matter which one we
511  // keep.
512  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
513 
514  } else {
515  // They're different. Just throw them both away, and re-read the
516  // current data from the cache file.
517 
518  Filename cache_pathname(_root, a_record->get_cache_filename());
519 
520  if (cache_pathname.exists()) {
521  PT(BamCacheRecord) record = do_read_record(cache_pathname, false);
522  if (record != nullptr) {
523  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
524  }
525  }
526  }
527 
528  ++ai;
529  ++bi;
530  }
531  }
532 
533  while (ai != old_index->_records.end()) {
534  // Here is an entry we have in our index, not present in the new index.
535  PT(BamCacheRecord) record = (*ai).second;
536  Filename cache_pathname(_root, record->get_cache_filename());
537  if (cache_pathname.exists()) {
538  // The file exists; keep it.
539  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
540  }
541  ++ai;
542  }
543 
544  while (bi != new_index->_records.end()) {
545  // Here is an entry in the new index, not present in our index.
546  PT(BamCacheRecord) record = (*bi).second;
547  Filename cache_pathname(_root, record->get_cache_filename());
548  if (cache_pathname.exists()) {
549  // The file exists; keep it.
550  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
551  }
552  ++bi;
553  }
554 
555  _index->process_new_records();
556 }
557 
558 /**
559  * Regenerates the index from scratch by scanning the directory.
560  */
561 void BamCache::
562 rebuild_index() {
564 
565  PT(VirtualFileList) contents = vfs->scan_directory(_root);
566  if (contents == nullptr) {
567  util_cat.error()
568  << "Unable to read directory " << _root << ", caching disabled.\n";
569  set_active(false);
570  return;
571  }
572 
573  delete _index;
574  _index = new BamCacheIndex;
575 
576  int num_files = contents->get_num_files();
577  for (int ci = 0; ci < num_files; ++ci) {
578  VirtualFile *file = contents->get_file(ci);
579  Filename filename = file->get_filename();
580  if (filename.get_extension() == "bam" ||
581  filename.get_extension() == "txo") {
582  Filename pathname(_root, filename);
583 
584  PT(BamCacheRecord) record = do_read_record(pathname, false);
585  if (record == nullptr) {
586  // Well, it was invalid, so blow it away.
587  if (util_cat.is_debug()) {
588  util_cat.debug()
589  << "Deleting invalid " << pathname << "\n";
590  }
591  file->delete_file();
592 
593  } else {
594  record->_record_access_time = record->_recorded_time;
595 
596  bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second;
597  if (!inserted) {
598  util_cat.info()
599  << "Multiple cache files defining " << record->get_source_pathname() << "\n";
600  file->delete_file();
601  }
602  }
603  }
604  }
605  _index->process_new_records();
606 
607  _index_stale_since = time(nullptr);
608  check_cache_size();
609  flush_index();
610 }
611 
612 /**
613  * Updates the index entry for the indicated record. Note that a copy of the
614  * record is made first.
615  */
616 void BamCache::
617 add_to_index(const BamCacheRecord *record) {
618  PT(BamCacheRecord) new_record = record->make_copy();
619 
620  if (_index->add_record(new_record)) {
621  mark_index_stale();
622  check_cache_size();
623  }
624 }
625 
626 /**
627  * Removes the index entry for the indicated record, if there is one.
628  */
629 void BamCache::
630 remove_from_index(const Filename &source_pathname) {
631  if (_index->remove_record(source_pathname)) {
632  mark_index_stale();
633  }
634 }
635 
636 /**
637  * If the cache size has exceeded its specified size limit, removes an old
638  * file.
639  */
640 void BamCache::
641 check_cache_size() {
642  if (_index->_cache_size == 0) {
643  // 0 means no limit.
644  return;
645  }
646 
647  if (_index->_cache_size / 1024 > _max_kbytes) {
648  while (_index->_cache_size / 1024 > _max_kbytes) {
649  PT(BamCacheRecord) record = _index->evict_old_file();
650  if (record == nullptr) {
651  // Never mind; the cache is empty.
652  break;
653  }
655  Filename cache_pathname(_root, record->get_cache_filename());
656  if (util_cat.is_debug()) {
657  util_cat.debug()
658  << "Deleting " << cache_pathname
659  << " to keep cache size below " << _max_kbytes << "K\n";
660  }
661  vfs->delete_file(cache_pathname);
662  }
663  mark_index_stale();
664  }
665 }
666 
667 /**
668  * Reads the index data from the specified filename. Returns a newly-
669  * allocated BamCacheIndex object on success, or NULL on failure.
670  */
671 BamCacheIndex *BamCache::
672 do_read_index(const Filename &index_pathname) {
673  if (index_pathname.empty()) {
674  return nullptr;
675  }
676 
677  DatagramInputFile din;
678  if (!din.open(index_pathname)) {
679  util_cat.debug()
680  << "Could not read index file: " << index_pathname << "\n";
681  return nullptr;
682  }
683 
684  string head;
685  if (!din.read_header(head, _bam_header.size())) {
686  util_cat.debug()
687  << index_pathname << " is not an index file.\n";
688  return nullptr;
689  }
690 
691  if (head != _bam_header) {
692  util_cat.debug()
693  << index_pathname << " is not an index file.\n";
694  return nullptr;
695  }
696 
697  BamReader reader(&din);
698  if (!reader.init()) {
699  return nullptr;
700  }
701 
702  TypedWritable *object = reader.read_object();
703 
704  if (object == nullptr) {
705  util_cat.error()
706  << "Cache index " << index_pathname << " is empty.\n";
707  return nullptr;
708 
709  } else if (!object->is_of_type(BamCacheIndex::get_class_type())) {
710  util_cat.error()
711  << "Cache index " << index_pathname << " contains a "
712  << object->get_type() << ", not a BamCacheIndex.\n";
713  return nullptr;
714  }
715 
716  BamCacheIndex *index = DCAST(BamCacheIndex, object);
717  if (!reader.resolve()) {
718  util_cat.error()
719  << "Unable to fully resolve cache index file.\n";
720  return nullptr;
721  }
722 
723  return index;
724 }
725 
726 /**
727  * Writes the given index data to the specified filename.
728  */
729 bool BamCache::
730 do_write_index(const Filename &index_pathname, const BamCacheIndex *index) {
732  DatagramOutputFile dout;
733  if (!dout.open(index_pathname)) {
734  util_cat.error()
735  << "Could not write index file: " << index_pathname << "\n";
736  vfs->delete_file(index_pathname);
737  return false;
738  }
739 
740  if (!dout.write_header(_bam_header)) {
741  util_cat.error()
742  << "Unable to write to " << index_pathname << "\n";
743  vfs->delete_file(index_pathname);
744  return false;
745  }
746 
747  {
748  BamWriter writer(&dout);
749  if (!writer.init()) {
750  vfs->delete_file(index_pathname);
751  return false;
752  }
753 
754  if (!writer.write_object(index)) {
755  vfs->delete_file(index_pathname);
756  return false;
757  }
758  }
759 
760  return true;
761 }
762 
763 /**
764  * Looks for the existing cache file that corresponds to the indicated
765  * filename. Normally, this is the specified cache filename exactly; but in
766  * the case of a hash collision, it may be a variant of the cache filename.
767  */
768 PT(BamCacheRecord) BamCache::
769 find_and_read_record(const Filename &source_pathname,
770  const Filename &cache_filename) {
771  int pass = 0;
772  while (true) {
773  PT(BamCacheRecord) record =
774  read_record(source_pathname, cache_filename, pass);
775  if (record != nullptr) {
776  add_to_index(record);
777  return record;
778  }
779  ++pass;
780  }
781 }
782 
783 /**
784  * Reads the indicated cache file and returns its associated record if it can
785  * be read and it matches the source filename.
786  */
787 PT(BamCacheRecord) BamCache::
788 read_record(const Filename &source_pathname,
789  const Filename &cache_filename,
790  int pass) {
792  Filename cache_pathname(_root, cache_filename);
793  if (pass != 0) {
794  ostringstream strm;
795  strm << cache_pathname.get_basename_wo_extension() << "_" << pass;
796  cache_pathname.set_basename_wo_extension(strm.str());
797  }
798 
799  if (!cache_pathname.exists()) {
800  // There is no such cache file already. Declare it.
801  if (util_cat.is_debug()) {
802  util_cat.debug()
803  << "Declaring new cache file " << cache_pathname << " for " << source_pathname << "\n";
804  }
805  PT(BamCacheRecord) record =
806  new BamCacheRecord(source_pathname, cache_filename);
807  record->_cache_pathname = cache_pathname;
808  return record;
809  }
810 
811  if (util_cat.is_debug()) {
812  util_cat.debug()
813  << "Reading cache file " << cache_pathname << " for " << source_pathname << "\n";
814  }
815 
816  PT(BamCacheRecord) record = do_read_record(cache_pathname, true);
817  if (record == nullptr) {
818  // Well, it was invalid, so blow it away, and make a new one.
819  if (util_cat.is_debug()) {
820  util_cat.debug()
821  << "Deleting invalid cache file " << cache_pathname << "\n";
822  }
823  vfs->delete_file(cache_pathname);
824  remove_from_index(source_pathname);
825 
826  PT(BamCacheRecord) record =
827  new BamCacheRecord(source_pathname, cache_filename);
828  record->_cache_pathname = cache_pathname;
829  return record;
830  }
831 
832  if (record->get_source_pathname() != source_pathname) {
833  // This might be just a hash conflict.
834  if (util_cat.is_debug()) {
835  util_cat.debug()
836  << "Cache file " << cache_pathname << " references "
837  << record->get_source_pathname() << ", not "
838  << source_pathname << "\n";
839  }
840  return nullptr;
841  }
842 
843  if (!record->has_data()) {
844  // If we didn't find any data, the caller will have to reload it.
845  record->clear_dependent_files();
846  }
847 
848  record->_cache_pathname = cache_pathname;
849  return record;
850 }
851 
852 /**
853  * Actually reads a record from the file.
854  */
855 PT(BamCacheRecord) BamCache::
856 do_read_record(const Filename &cache_pathname, bool read_data) {
857  DatagramInputFile din;
858  if (!din.open(cache_pathname)) {
859  if (util_cat.is_debug()) {
860  util_cat.debug()
861  << "Could not read cache file: " << cache_pathname << "\n";
862  }
863  return nullptr;
864  }
865 
866  string head;
867  if (!din.read_header(head, _bam_header.size())) {
868  if (util_cat.is_debug()) {
869  util_cat.debug()
870  << cache_pathname << " is not a cache file.\n";
871  }
872  return nullptr;
873  }
874 
875  if (head != _bam_header) {
876  if (util_cat.is_debug()) {
877  util_cat.debug()
878  << cache_pathname << " is not a cache file.\n";
879  }
880  return nullptr;
881  }
882 
883  BamReader reader(&din);
884  if (!reader.init()) {
885  return nullptr;
886  }
887 
888  TypedWritable *object = reader.read_object();
889  if (object == nullptr) {
890  if (util_cat.is_debug()) {
891  util_cat.debug()
892  << cache_pathname << " is empty.\n";
893  }
894  return nullptr;
895 
896  } else if (!object->is_of_type(BamCacheRecord::get_class_type())) {
897  if (util_cat.is_debug()) {
898  util_cat.debug()
899  << "Cache file " << cache_pathname << " contains a "
900  << object->get_type() << ", not a BamCacheRecord.\n";
901  }
902  return nullptr;
903  }
904 
905  PT(BamCacheRecord) record = DCAST(BamCacheRecord, object);
906  if (!reader.resolve()) {
907  if (util_cat.is_debug()) {
908  util_cat.debug()
909  << "Unable to fully resolve cache record in " << cache_pathname << "\n";
910  }
911  return nullptr;
912  }
913 
914  // From this point below, we have validated that the selected filename is
915  // indeed a cache record for the indicated source file, and therefore the
916  // cache record will be returned.
917 
918  // We still need to decide whether the cache record is stale.
919  if (read_data && record->dependents_unchanged()) {
920  // The cache record doesn't appear to be stale. Load the cached object.
921  TypedWritable *ptr;
922  ReferenceCount *ref_ptr;
923 
924  if (reader.read_object(ptr, ref_ptr)) {
925  if (!reader.resolve()) {
926  if (util_cat.is_debug()) {
927  util_cat.debug()
928  << "Unable to fully resolve cached object in " << cache_pathname << "\n";
929  }
930  delete object;
931  } else {
932  // The object is valid. Store it in the record.
933  record->set_data(ptr, ref_ptr);
934  }
935  }
936  }
937 
938  // Also get the total file size.
939  PT(VirtualFile) vfile = din.get_vfile();
940  istream &in = din.get_stream();
941  in.clear();
942  record->_record_size = vfile->get_file_size(&in);
943 
944  // And the last access time is now, duh.
945  record->_record_access_time = time(nullptr);
946 
947  return record;
948 }
949 
950 /**
951  * Returns the appropriate filename to use for a cache file, given the
952  * fullpath string to the source filename.
953  */
954 string BamCache::
955 hash_filename(const string &filename) {
956 #ifdef HAVE_OPENSSL
957  // With OpenSSl, use the MD5 hash of the filename.
958  HashVal hv;
959  hv.hash_string(filename);
960  ostringstream strm;
961  hv.output_hex(strm);
962  return strm.str();
963 
964 #else // HAVE_OPENSSL
965  // Without OpenSSL, don't get fancy; just build a simple hash.
966  unsigned int hash = 0;
967  for (string::const_iterator si = filename.begin();
968  si != filename.end();
969  ++si) {
970  hash = (hash * 9109) + (unsigned int)(*si);
971  }
972 
973  ostringstream strm;
974  strm << std::hex << std::setw(8) << std::setfill('0') << hash;
975  return strm.str();
976 
977 #endif // HAVE_OPENSSL
978 }
979 
980 /**
981  * Constructs the global BamCache object.
982  */
983 void BamCache::
984 make_global() {
985  _global_ptr = new BamCache;
986 
987  if (_global_ptr->_root.empty()) {
988  _global_ptr->set_active(false);
989  }
990 }
bamCache.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
ConfigVariableBool
This is a convenience class to specialize ConfigVariable as a boolean type.
Definition: configVariableBool.h:23
config_putil.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
VirtualFileSystem::get_cwd
Filename get_cwd() const
Returns the current directory name.
Definition: virtualFileSystem.cxx:455
BamCache::consider_flush_index
void consider_flush_index()
Flushes the index if enough time has elapsed since the index was last flushed.
Definition: bamCache.cxx:317
BamCacheRecord::set_data
set_data
Stores a new data object on the record.
Definition: bamCacheRecord.h:77
BamCacheRecord
An instance of this class is written to the front of a Bam or Txo file to make the file a cached inst...
Definition: bamCacheRecord.h:35
DatagramInputFile::read_header
bool read_header(std::string &header, size_t num_bytes)
Reads a sequence of bytes from the beginning of the datagram file.
Definition: datagramInputFile.cxx:104
TypeRegistry::find_type
TypeHandle find_type(const std::string &name) const
Looks for a previously-registered type of the given name.
Definition: typeRegistry.cxx:233
ReMutexDirect::unlock
void unlock()
Alias for release() to match C++11 semantics.
Definition: reMutexDirect.I:62
string_utils.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
ReMutexDirect::try_lock
bool try_lock()
Alias for try_acquire() to match C++11 semantics.
Definition: reMutexDirect.I:48
ReMutexHolder
Similar to MutexHolder, but for a reentrant mutex.
Definition: reMutexHolder.h:25
ReferenceCount
A base class for all things that want to be reference-counted.
Definition: referenceCount.h:38
datagramInputFile.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
DatagramInputFile::get_stream
std::istream & get_stream()
Returns the istream represented by the input file.
Definition: datagramInputFile.I:47
VirtualFileSystem::is_directory
bool is_directory(const Filename &filename) const
Convenience function; returns true if the named file exists and is a directory.
Definition: virtualFileSystem.I:27
BamWriter::set_root_node
set_root_node
Sets the root node of the part of the scene graph we are currently writing out.
Definition: bamWriter.h:96
BamReader
This is the fundamental interface for extracting binary objects from a Bam file, as generated by a Ba...
Definition: bamReader.h:110
DatagramOutputFile::close
void close()
Closes the file.
Definition: datagramOutputFile.cxx:73
BamCache::set_active
set_active
Changes the state of the active flag.
Definition: bamCache.h:88
BamWriter
This is the fundamental interface for writing binary objects to a Bam file, to be extracted later by ...
Definition: bamWriter.h:63
bam.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
DatagramInputFile::open
bool open(const FileReference *file)
Opens the indicated filename for reading.
Definition: datagramInputFile.cxx:33
DatagramInputFile
This class can be used to read a binary file that consists of an arbitrary header followed by a numbe...
Definition: datagramInputFile.h:28
bamReader.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PT
PT(BamCacheRecord) BamCache
Looks up a file in the cache.
Definition: bamCache.cxx:158
VirtualFileSystem::exists
bool exists(const Filename &filename) const
Convenience function; returns true if the named file exists.
Definition: virtualFileSystem.I:18
TypedWritable
Base class for objects that can be written to and read from Bam files.
Definition: typedWritable.h:35
DatagramInputFile::get_vfile
virtual VirtualFile * get_vfile()
Returns the VirtualFile that provides the source for these datagrams, if any, or NULL if the datagram...
Definition: datagramInputFile.cxx:353
Thread::get_unique_id
get_unique_id
Returns a string that is guaranteed to be unique to this thread, across all processes on the machine,...
Definition: thread.h:104
BamCache::list_index
void list_index(std::ostream &out, int indent_level=0) const
Writes the contents of the index to standard output.
Definition: bamCache.cxx:394
DatagramOutputFile::get_file_pos
virtual std::streampos get_file_pos()
Returns the current file position within the data stream, if any, or 0 if the file position is not me...
Definition: datagramOutputFile.cxx:317
Thread::get_current_thread
get_current_thread
Returns a pointer to the currently-executing Thread object.
Definition: thread.h:109
BamWriter::set_file_texture_mode
void set_file_texture_mode(BamTextureMode file_texture_mode)
Changes the BamTextureMode preference for the Bam file currently being written.
Definition: bamWriter.I:101
BamCache::flush_index
void flush_index()
Ensures the index is written to disk.
Definition: bamCache.cxx:342
TypeHandle
TypeHandle is the identifier used to differentiate C++ class types.
Definition: typeHandle.h:81
configVariableInt.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
VirtualFileSystem::atomic_read_contents
bool atomic_read_contents(const Filename &filename, std::string &contents) const
See Filename::atomic_read_contents().
Definition: virtualFileSystem.cxx:1016
VirtualFileSystem::make_directory_full
bool make_directory_full(const Filename &filename)
Attempts to create a directory within the file system.
Definition: virtualFileSystem.cxx:483
HashVal::output_hex
void output_hex(std::ostream &out) const
Outputs the HashVal as a 32-digit hexadecimal number.
Definition: hashVal.cxx:34
VirtualFileSystem::atomic_compare_and_exchange_contents
bool atomic_compare_and_exchange_contents(const Filename &filename, std::string &orig_contents, const std::string &old_contents, const std::string &new_contents)
See Filename::atomic_compare_and_exchange_contents().
Definition: virtualFileSystem.cxx:1001
hashVal.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
VirtualFileSystem
A hierarchy of directories and files that appears to be one continuous file system,...
Definition: virtualFileSystem.h:40
BamWriter::write_object
bool write_object(const TypedWritable *obj)
Writes a single object to the Bam file, so that the BamReader::read_object() can later correctly rest...
Definition: bamWriter.cxx:204
BamCache::store
bool store(BamCacheRecord *record)
Flushes a cache entry to disk.
Definition: bamCache.cxx:188
bamCacheIndex.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
datagramOutputFile.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
VirtualFileList
A list of VirtualFiles, as returned by VirtualFile::scan_directory().
Definition: virtualFileList.h:25
Filename::get_fullpath
std::string get_fullpath() const
Returns the entire filename: directory, basename, extension.
Definition: filename.I:338
DatagramOutputFile::write_header
bool write_header(const std::string &header)
Writes a sequence of bytes to the beginning of the datagram file.
Definition: datagramOutputFile.cxx:96
BamCache::set_root
set_root
Changes the current root pathname of the cache.
Definition: bamCache.h:95
Filename::is_local
bool is_local() const
Returns true if the filename is local, e.g.
Definition: filename.I:549
Filename::temporary
static Filename temporary(const std::string &dirname, const std::string &prefix, const std::string &suffix=std::string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix.
Definition: filename.cxx:424
BamCacheRecord::clear_dependent_files
void clear_dependent_files()
Empties the list of files that contribute to the data in this record.
Definition: bamCacheRecord.cxx:137
DatagramOutputFile
This class can be used to write a binary file that consists of an arbitrary header followed by a numb...
Definition: datagramOutputFile.h:30
Filename::get_type
Type get_type() const
Returns the type of the file represented by the filename, as previously set by set_type().
Definition: filename.I:485
ConfigVariableFilename
This is a convenience class to specialize ConfigVariable as a Filename type.
Definition: configVariableFilename.h:27
BamCacheRecord::get_data
get_data
Returns a pointer to the data stored in the record, or NULL if there is no data.
Definition: bamCacheRecord.h:77
VirtualFileSystem::delete_file
bool delete_file(const Filename &filename)
Attempts to delete the indicated file or directory.
Definition: virtualFileSystem.cxx:576
BamCache
This class maintains a cache of Bam and/or Txo objects generated from model files and texture images ...
Definition: bamCache.h:42
BamCacheIndex
This represents the in-memory index that records the list of files stored in the BamCache.
Definition: bamCacheIndex.h:33
VirtualFileSystem::get_global_ptr
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
Definition: virtualFileSystem.cxx:742
virtualFileSystem.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
TypeRegistry::ptr
static TypeRegistry * ptr()
Returns the pointer to the global TypeRegistry object.
Definition: typeRegistry.I:30
VirtualFile
The abstract base class for a file or directory within the VirtualFileSystem.
Definition: virtualFile.h:35
BamWriter::init
bool init()
Initializes the BamWriter prior to writing any objects to its output stream.
Definition: bamWriter.cxx:147
Filename::set_extension
void set_extension(const std::string &s)
Replaces the file extension.
Definition: filename.cxx:804
Filename::make_absolute
void make_absolute()
Converts the filename to a fully-qualified pathname from the root (if it is a relative pathname),...
Definition: filename.cxx:968
HashVal
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer.
Definition: hashVal.h:31
trim
string trim(const string &str)
Returns a new string representing the contents of the given string with both leading and trailing whi...
Definition: string_utils.cxx:281
TypeRegistry
The TypeRegistry class maintains all the assigned TypeHandles in a given system.
Definition: typeRegistry.h:36
typeRegistry.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
configVariableFilename.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
bamWriter.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
Filename::make_relative_to
bool make_relative_to(Filename directory, bool allow_backups=true)
Adjusts this filename, which must be a fully-specified pathname beginning with a slash,...
Definition: filename.cxx:1640
configVariableString.h
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
ConfigVariableInt
This is a convenience class to specialize ConfigVariable as an integer type.
Definition: configVariableInt.h:24
Thread
A thread; that is, a lightweight process.
Definition: thread.h:46
VirtualFileSystem::rename_file
bool rename_file(const Filename &orig_filename, const Filename &new_filename)
Attempts to move or rename the indicated file or directory.
Definition: virtualFileSystem.cxx:597
DatagramOutputFile::open
bool open(const FileReference *file)
Opens the indicated filename for writing.
Definition: datagramOutputFile.cxx:28
Filename::get_extension
std::string get_extension() const
Returns the file extension.
Definition: filename.I:400
Filename::get_basename
std::string get_basename() const
Returns the basename part of the filename.
Definition: filename.I:367
VirtualFile::delete_file
virtual bool delete_file()
Attempts to delete this file or directory.
Definition: virtualFile.cxx:69
BamCacheRecord::dependents_unchanged
bool dependents_unchanged() const
Returns true if all of the dependent files are still the same as when the cache was recorded,...
Definition: bamCacheRecord.cxx:84
Filename
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:39
Filename::set_binary
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:414
TypedObject::is_of_type
bool is_of_type(TypeHandle handle) const
Returns true if the current object is or derives from the indicated type.
Definition: typedObject.I:28