Panda3D
bamCache.cxx
Go to the documentation of this file.
1 /**
2  * PANDA 3D SOFTWARE
3  * Copyright (c) Carnegie Mellon University. All rights reserved.
4  *
5  * All use of this software is subject to the terms of the revised BSD
6  * license. You should have received a copy of this license along
7  * with this source code in a file named "LICENSE."
8  *
9  * @file bamCache.cxx
10  * @author drose
11  * @date 2006-06-09
12  */
13 
14 #include "bamCache.h"
15 #include "bamCacheIndex.h"
16 #include "bamReader.h"
17 #include "bamWriter.h"
18 #include "hashVal.h"
19 #include "datagramInputFile.h"
20 #include "datagramOutputFile.h"
21 #include "config_putil.h"
22 #include "bam.h"
23 #include "typeRegistry.h"
24 #include "string_utils.h"
25 #include "configVariableInt.h"
26 #include "configVariableString.h"
27 #include "configVariableFilename.h"
28 #include "virtualFileSystem.h"
29 
30 using std::istream;
31 using std::ostream;
32 using std::ostringstream;
33 using std::string;
34 
35 BamCache *BamCache::_global_ptr = nullptr;
36 
37 /**
38  *
39  */
40 BamCache::
41 BamCache() :
42  _active(true),
43  _read_only(false),
44  _index(new BamCacheIndex),
45  _index_stale_since(0)
46 {
47  ConfigVariableFilename model_cache_dir
48  ("model-cache-dir", Filename(),
49  PRC_DESC("The full path to a directory, local to this computer, in which "
50  "model and texture files will be cached on load. If a directory "
51  "name is specified here, files may be loaded from the cache "
52  "instead of from their actual pathnames, which may save load time, "
53  "especially if you are loading egg files instead of bam files, "
54  "or if you are loading models from a shared network drive. "
55  "If this is the empty string, no cache will be used."));
56 
57  ConfigVariableInt model_cache_flush
58  ("model-cache-flush", 30,
59  PRC_DESC("This is the amount of time, in seconds, between automatic "
60  "flushes of the model-cache index."));
61 
62  ConfigVariableBool model_cache_models
63  ("model-cache-models", true,
64  PRC_DESC("If this is set to true, models will be cached in the "
65  "model cache, as bam files."));
66 
67  ConfigVariableBool model_cache_textures
68  ("model-cache-textures", true,
69  PRC_DESC("If this is set to true, textures will also be cached in the "
70  "model cache, as txo files."));
71 
72  ConfigVariableBool model_cache_compressed_textures
73  ("model-cache-compressed-textures", false,
74  PRC_DESC("If this is set to true, compressed textures will be cached "
75  "in the model cache, in their compressed form as downloaded "
76  "by the GSG. This may be set in conjunction with "
77  "model-cache-textures, or it may be independent."));
78 
79  ConfigVariableBool model_cache_compiled_shaders
80  ("model-cache-compiled-shaders", false,
81  PRC_DESC("If this is set to true, compiled shaders will be cached "
82  "in the model cache, in their binary form as downloaded "
83  "by the GSG."));
84 
85  ConfigVariableInt model_cache_max_kbytes
86  ("model-cache-max-kbytes", 10485760,
87  PRC_DESC("This is the maximum size of the model cache, in kilobytes."));
88 
89  _cache_models = model_cache_models;
90  _cache_textures = model_cache_textures;
91  _cache_compressed_textures = model_cache_compressed_textures;
92  _cache_compiled_shaders = model_cache_compiled_shaders;
93 
94  _flush_time = model_cache_flush;
95  _max_kbytes = model_cache_max_kbytes;
96 
97  if (!model_cache_dir.empty()) {
98  set_root(model_cache_dir);
99  }
100 }
101 
102 /**
103  *
104  */
105 BamCache::
106 ~BamCache() {
107  flush_index();
108  delete _index;
109  _index = nullptr;
110 }
111 
112 /**
113  * Changes the current root pathname of the cache. This specifies where the
114  * cache files are stored on disk. This should name a directory that is on a
115  * disk local to the machine (not on a network-mounted disk), for instance,
116  * /tmp/panda-cache or /c/panda-cache.
117  *
118  * If the directory does not already exist, it will be created as a result of
119  * this call.
120  */
121 void BamCache::
122 set_root(const Filename &root) {
123  ReMutexHolder holder(_lock);
124  flush_index();
125  _root = root;
126 
127  // The root filename must be a directory.
129  if (!vfs->is_directory(_root)) {
130  vfs->make_directory_full(_root);
131  }
132 
133  delete _index;
134  _index = new BamCacheIndex;
135  _index_stale_since = 0;
136 
137  if (!vfs->is_directory(_root)) {
138  util_cat.error()
139  << "Unable to make directory " << _root << ", caching disabled.\n";
140  _active = false;
141  return;
142  }
143 
144  read_index();
145  check_cache_size();
146 }
147 
148 /**
149  * Looks up a file in the cache.
150  *
151  * If the file is cacheable, then regardless of whether the file is found in
152  * the cache or not, this returns a BamCacheRecord. On the other hand, if the
153  * file cannot be cached, returns NULL.
154  *
155  * If record->has_data() returns true, then the file was found in the cache,
156  * and you may call record->extract_data() to get the object. If
157  * record->has_data() returns false, then the file was not found in the cache
158  * or the cache was stale; and you should reload the source file (calling
159  * record->add_dependent_file() for each file loaded, including the original
160  * source file), and then call record->set_data() to record the resulting
161  * loaded object; and finally, you should call store() to write the cached
162  * record to disk.
163  */
164 PT(BamCacheRecord) BamCache::
165 lookup(const Filename &source_filename, const string &cache_extension) {
166  ReMutexHolder holder(_lock);
168 
170 
171  Filename source_pathname(source_filename);
172  source_pathname.make_absolute(vfs->get_cwd());
173 
174  Filename rel_pathname(source_pathname);
175  rel_pathname.make_relative_to(_root, false);
176  if (rel_pathname.is_local()) {
177  // If the source pathname is already within the cache directory, don't
178  // cache it further.
179  return nullptr;
180  }
181 
182  Filename cache_filename = hash_filename(source_pathname.get_fullpath());
183  cache_filename.set_extension(cache_extension);
184 
185  return find_and_read_record(source_pathname, cache_filename);
186 }
187 
188 /**
189  * Flushes a cache entry to disk. You must have retrieved the cache record
190  * via a prior call to lookup(), and then stored the data via
191  * record->set_data(). Returns true on success, false on failure.
192  */
194 store(BamCacheRecord *record) {
196  ReMutexHolder holder(_lock);
197  nassertr(!record->_cache_pathname.empty(), false);
198  nassertr(record->has_data(), false);
199 
200  if (_read_only) {
201  return false;
202  }
203 
205 
206 #ifndef NDEBUG
207  // Ensure that the cache_pathname is within the _root directory tree.
208  Filename rel_pathname(record->_cache_pathname);
209  rel_pathname.make_relative_to(_root, false);
210  nassertr(rel_pathname.is_local(), false);
211 #endif // NDEBUG
212 
213  record->_recorded_time = time(nullptr);
214 
215  Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
216 
217  // We actually do the write to a temporary filename first, and then move it
218  // into place, so that no one attempts to read the file while it is in the
219  // process of being written.
220  Thread *current_thread = Thread::get_current_thread();
221  string extension = current_thread->get_unique_id() + string(".tmp");
222  Filename temp_pathname = cache_pathname;
223  temp_pathname.set_extension(extension);
224  temp_pathname.set_binary();
225 
226  DatagramOutputFile dout;
227  if (!dout.open(temp_pathname)) {
228  util_cat.error()
229  << "Could not write cache file: " << temp_pathname << "\n";
230  vfs->delete_file(temp_pathname);
231  emergency_read_only();
232  return false;
233  }
234 
235  if (!dout.write_header(_bam_header)) {
236  util_cat.error()
237  << "Unable to write to " << temp_pathname << "\n";
238  vfs->delete_file(temp_pathname);
239  return false;
240  }
241 
242  {
243  BamWriter writer(&dout);
244  if (!writer.init()) {
245  util_cat.error()
246  << "Unable to write Bam header to " << temp_pathname << "\n";
247  vfs->delete_file(temp_pathname);
248  return false;
249  }
250 
251  TypeRegistry *type_registry = TypeRegistry::ptr();
252  TypeHandle texture_type = type_registry->find_type("Texture");
253  if (record->get_data()->is_of_type(texture_type)) {
254  // Texture objects write the actual texture image.
255  writer.set_file_texture_mode(BamWriter::BTM_rawdata);
256  } else {
257  // Any other kinds of objects write texture references.
258  writer.set_file_texture_mode(BamWriter::BTM_fullpath);
259  }
260 
261  // This is necessary for relative NodePaths to work.
262  TypeHandle node_type = type_registry->find_type("PandaNode");
263  if (record->get_data()->is_of_type(node_type)) {
264  writer.set_root_node(record->get_data());
265  }
266 
267  if (!writer.write_object(record)) {
268  util_cat.error()
269  << "Unable to write object to " << temp_pathname << "\n";
270  vfs->delete_file(temp_pathname);
271  return false;
272  }
273 
274  if (!writer.write_object(record->get_data())) {
275  util_cat.error()
276  << "Unable to write object data to " << temp_pathname << "\n";
277  vfs->delete_file(temp_pathname);
278  return false;
279  }
280 
281  // Now that we are done with the BamWriter, it's important to let it
282  // destruct now and clean itself up, or it might get mad if we delete any
283  // TypedWritables below that haven't been written yet.
284  }
285 
286  record->_record_size = dout.get_file_pos();
287  dout.close();
288 
289  // Now move the file into place.
290  if (!vfs->rename_file(temp_pathname, cache_pathname) && vfs->exists(temp_pathname)) {
291  vfs->delete_file(cache_pathname);
292  if (!vfs->rename_file(temp_pathname, cache_pathname)) {
293  util_cat.error()
294  << "Unable to rename " << temp_pathname << " to "
295  << cache_pathname << "\n";
296  vfs->delete_file(temp_pathname);
297  return false;
298  }
299  }
300 
301  add_to_index(record);
302 
303  return true;
304 }
305 
306 /**
307  * Called when an attempt to write to the cache dir has failed, usually for
308  * lack of disk space or because of incorrect file permissions. Outputs an
309  * error and puts the BamCache into read-only mode.
310  */
311 void BamCache::
312 emergency_read_only() {
313  util_cat.error() <<
314  "Could not write to the Bam Cache. Disabling future attempts.\n";
315  _read_only = true;
316 }
317 
318 /**
319  * Flushes the index if enough time has elapsed since the index was last
320  * flushed.
321  */
324 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
325  if (!_lock.try_lock()) {
326  // If we can't grab the lock, no big deal. We don't want to hold up
327  // the frame waiting for a cache operation. We can try again later.
328  return;
329  }
330 #endif
331 
332  if (_index_stale_since != 0) {
333  int elapsed = (int)time(nullptr) - (int)_index_stale_since;
334  if (elapsed > _flush_time) {
335  flush_index();
336  }
337  }
338 
339 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
340  _lock.unlock();
341 #endif
342 }
343 
344 /**
345  * Ensures the index is written to disk.
346  */
348 flush_index() {
349  ReMutexHolder holder(_lock);
350  if (_index_stale_since == 0) {
351  // Never mind.
352  return;
353  }
354 
355  while (true) {
356  if (_read_only) {
357  return;
358  }
359 
360  Filename temp_pathname = Filename::temporary(_root, "index-", ".boo");
361 
362  if (!do_write_index(temp_pathname, _index)) {
363  emergency_read_only();
364  return;
365  }
366 
367  // Now atomically write the name of this index file to the index reference
368  // file.
370  Filename index_ref_pathname(_root, Filename("index_name.txt"));
371  string old_index = _index_ref_contents;
372  string new_index = temp_pathname.get_basename() + "\n";
373  string orig_index;
374 
375  if (vfs->atomic_compare_and_exchange_contents(index_ref_pathname, orig_index, old_index, new_index)) {
376  // We successfully wrote our version of the index, and no other process
377  // beat us to it. Our index is now the official one. Remove the old
378  // index.
379  vfs->delete_file(_index_pathname);
380  _index_pathname = temp_pathname;
381  _index_ref_contents = new_index;
382  _index_stale_since = 0;
383  return;
384  }
385 
386  // Shoot, some other process updated the index while we were trying to
387  // update it, and they beat us to it. We have to merge, and try again.
388  vfs->delete_file(temp_pathname);
389  _index_pathname = Filename(_root, Filename(trim(orig_index)));
390  _index_ref_contents = orig_index;
391  read_index();
392  }
393  check_cache_size();
394 }
395 
396 /**
397  * Writes the contents of the index to standard output.
398  */
400 list_index(ostream &out, int indent_level) const {
401  _index->write(out, indent_level);
402 }
403 
404 /**
405  * Reads, or re-reads the index file from disk. If _index_stale_since is
406  * nonzero, the index file is read and then merged with our current index.
407  */
408 void BamCache::
409 read_index() {
410  if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
411  // Couldn't read the index ref; rebuild the index.
412  rebuild_index();
413  return;
414  }
415 
416  while (true) {
417  BamCacheIndex *new_index = do_read_index(_index_pathname);
418  if (new_index != nullptr) {
419  merge_index(new_index);
420  return;
421  }
422 
423  // We couldn't read the index. Maybe it's been removed already. See if
424  // the index_pathname has changed.
425  Filename old_index_pathname = _index_pathname;
426  if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
427  // Couldn't read the index ref; rebuild the index.
428  rebuild_index();
429  return;
430  }
431 
432  if (old_index_pathname == _index_pathname) {
433  // Nope, we just couldn't read it. Delete it and build a new one.
435  vfs->delete_file(_index_pathname);
436  rebuild_index();
437  flush_index();
438  return;
439  }
440  }
441 }
442 
443 /**
444  * Atomically reads the current index filename from the index reference file.
445  * The index filename moves around as different processes update the index.
446  */
447 bool BamCache::
448 read_index_pathname(Filename &index_pathname, string &index_ref_contents) const {
450  index_ref_contents.clear();
451  Filename index_ref_pathname(_root, Filename("index_name.txt"));
452  if (!vfs->atomic_read_contents(index_ref_pathname, index_ref_contents)) {
453  return false;
454  }
455 
456  string trimmed = trim(index_ref_contents);
457  if (trimmed.empty()) {
458  index_pathname = Filename();
459  } else {
460  index_pathname = Filename(_root, Filename(trimmed));
461  }
462  return true;
463 }
464 
465 /**
466  * The supplied index file has been updated by some other process. Merge it
467  * with our current index.
468  *
469  * Ownership of the pointer is transferred with this call. The caller should
470  * assume that new_index will be deleted by this method.
471  */
472 void BamCache::
473 merge_index(BamCacheIndex *new_index) {
474  if (_index_stale_since == 0) {
475  // If our index isn't stale, just replace it.
476  delete _index;
477  _index = new_index;
478  return;
479  }
480 
481  BamCacheIndex *old_index = _index;
482  old_index->release_records();
483  new_index->release_records();
484  _index = new BamCacheIndex;
485 
486  BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
487  BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
488 
489  while (ai != old_index->_records.end() &&
490  bi != new_index->_records.end()) {
491  if ((*ai).first < (*bi).first) {
492  // Here is an entry we have in our index, not present in the new index.
493  PT(BamCacheRecord) record = (*ai).second;
494  Filename cache_pathname(_root, record->get_cache_filename());
495  if (cache_pathname.exists()) {
496  // The file exists; keep it.
497  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
498  }
499  ++ai;
500 
501  } else if ((*bi).first < (*ai).first) {
502  // Here is an entry in the new index, not present in our index.
503  PT(BamCacheRecord) record = (*bi).second;
504  Filename cache_pathname(_root, record->get_cache_filename());
505  if (cache_pathname.exists()) {
506  // The file exists; keep it.
507  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
508  }
509  ++bi;
510 
511  } else {
512  // Here is an entry we have in both.
513  PT(BamCacheRecord) a_record = (*ai).second;
514  PT(BamCacheRecord) b_record = (*bi).second;
515  if (*a_record == *b_record) {
516  // They're the same entry. It doesn't really matter which one we
517  // keep.
518  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
519 
520  } else {
521  // They're different. Just throw them both away, and re-read the
522  // current data from the cache file.
523 
524  Filename cache_pathname(_root, a_record->get_cache_filename());
525 
526  if (cache_pathname.exists()) {
527  PT(BamCacheRecord) record = do_read_record(cache_pathname, false);
528  if (record != nullptr) {
529  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
530  }
531  }
532  }
533 
534  ++ai;
535  ++bi;
536  }
537  }
538 
539  while (ai != old_index->_records.end()) {
540  // Here is an entry we have in our index, not present in the new index.
541  PT(BamCacheRecord) record = (*ai).second;
542  Filename cache_pathname(_root, record->get_cache_filename());
543  if (cache_pathname.exists()) {
544  // The file exists; keep it.
545  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
546  }
547  ++ai;
548  }
549 
550  while (bi != new_index->_records.end()) {
551  // Here is an entry in the new index, not present in our index.
552  PT(BamCacheRecord) record = (*bi).second;
553  Filename cache_pathname(_root, record->get_cache_filename());
554  if (cache_pathname.exists()) {
555  // The file exists; keep it.
556  _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
557  }
558  ++bi;
559  }
560 
561  _index->process_new_records();
562 }
563 
564 /**
565  * Regenerates the index from scratch by scanning the directory.
566  */
567 void BamCache::
568 rebuild_index() {
570 
571  PT(VirtualFileList) contents = vfs->scan_directory(_root);
572  if (contents == nullptr) {
573  util_cat.error()
574  << "Unable to read directory " << _root << ", caching disabled.\n";
575  set_active(false);
576  return;
577  }
578 
579  delete _index;
580  _index = new BamCacheIndex;
581 
582  int num_files = contents->get_num_files();
583  for (int ci = 0; ci < num_files; ++ci) {
584  VirtualFile *file = contents->get_file(ci);
585  Filename filename = file->get_filename();
586  if (filename.get_extension() == "bam" ||
587  filename.get_extension() == "txo") {
588  Filename pathname(_root, filename);
589 
590  PT(BamCacheRecord) record = do_read_record(pathname, false);
591  if (record == nullptr) {
592  // Well, it was invalid, so blow it away.
593  if (util_cat.is_debug()) {
594  util_cat.debug()
595  << "Deleting invalid " << pathname << "\n";
596  }
597  file->delete_file();
598 
599  } else {
600  record->_record_access_time = record->_recorded_time;
601 
602  bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second;
603  if (!inserted) {
604  util_cat.info()
605  << "Multiple cache files defining " << record->get_source_pathname() << "\n";
606  file->delete_file();
607  }
608  }
609  }
610  }
611  _index->process_new_records();
612 
613  _index_stale_since = time(nullptr);
614  check_cache_size();
615  flush_index();
616 }
617 
618 /**
619  * Updates the index entry for the indicated record. Note that a copy of the
620  * record is made first.
621  */
622 void BamCache::
623 add_to_index(const BamCacheRecord *record) {
624  PT(BamCacheRecord) new_record = record->make_copy();
625 
626  if (_index->add_record(new_record)) {
627  mark_index_stale();
628  check_cache_size();
629  }
630 }
631 
632 /**
633  * Removes the index entry for the indicated record, if there is one.
634  */
635 void BamCache::
636 remove_from_index(const Filename &source_pathname) {
637  if (_index->remove_record(source_pathname)) {
638  mark_index_stale();
639  }
640 }
641 
642 /**
643  * If the cache size has exceeded its specified size limit, removes an old
644  * file.
645  */
646 void BamCache::
647 check_cache_size() {
648  if (_index->_cache_size == 0) {
649  // 0 means no limit.
650  return;
651  }
652 
653  if (_index->_cache_size / 1024 > _max_kbytes) {
654  while (_index->_cache_size / 1024 > _max_kbytes) {
655  PT(BamCacheRecord) record = _index->evict_old_file();
656  if (record == nullptr) {
657  // Never mind; the cache is empty.
658  break;
659  }
661  Filename cache_pathname(_root, record->get_cache_filename());
662  if (util_cat.is_debug()) {
663  util_cat.debug()
664  << "Deleting " << cache_pathname
665  << " to keep cache size below " << _max_kbytes << "K\n";
666  }
667  vfs->delete_file(cache_pathname);
668  }
669  mark_index_stale();
670  }
671 }
672 
673 /**
674  * Reads the index data from the specified filename. Returns a newly-
675  * allocated BamCacheIndex object on success, or NULL on failure.
676  */
677 BamCacheIndex *BamCache::
678 do_read_index(const Filename &index_pathname) {
679  if (index_pathname.empty()) {
680  return nullptr;
681  }
682 
683  DatagramInputFile din;
684  if (!din.open(index_pathname)) {
685  util_cat.debug()
686  << "Could not read index file: " << index_pathname << "\n";
687  return nullptr;
688  }
689 
690  string head;
691  if (!din.read_header(head, _bam_header.size())) {
692  util_cat.debug()
693  << index_pathname << " is not an index file.\n";
694  return nullptr;
695  }
696 
697  if (head != _bam_header) {
698  util_cat.debug()
699  << index_pathname << " is not an index file.\n";
700  return nullptr;
701  }
702 
703  BamReader reader(&din);
704  if (!reader.init()) {
705  return nullptr;
706  }
707 
708  TypedWritable *object = reader.read_object();
709 
710  if (object == nullptr) {
711  util_cat.error()
712  << "Cache index " << index_pathname << " is empty.\n";
713  return nullptr;
714 
715  } else if (!object->is_of_type(BamCacheIndex::get_class_type())) {
716  util_cat.error()
717  << "Cache index " << index_pathname << " contains a "
718  << object->get_type() << ", not a BamCacheIndex.\n";
719  return nullptr;
720  }
721 
722  BamCacheIndex *index = DCAST(BamCacheIndex, object);
723  if (!reader.resolve()) {
724  util_cat.error()
725  << "Unable to fully resolve cache index file.\n";
726  return nullptr;
727  }
728 
729  return index;
730 }
731 
732 /**
733  * Writes the given index data to the specified filename.
734  */
735 bool BamCache::
736 do_write_index(const Filename &index_pathname, const BamCacheIndex *index) {
738  DatagramOutputFile dout;
739  if (!dout.open(index_pathname)) {
740  util_cat.error()
741  << "Could not write index file: " << index_pathname << "\n";
742  vfs->delete_file(index_pathname);
743  return false;
744  }
745 
746  if (!dout.write_header(_bam_header)) {
747  util_cat.error()
748  << "Unable to write to " << index_pathname << "\n";
749  vfs->delete_file(index_pathname);
750  return false;
751  }
752 
753  {
754  BamWriter writer(&dout);
755  if (!writer.init()) {
756  vfs->delete_file(index_pathname);
757  return false;
758  }
759 
760  if (!writer.write_object(index)) {
761  vfs->delete_file(index_pathname);
762  return false;
763  }
764  }
765 
766  return true;
767 }
768 
769 /**
770  * Looks for the existing cache file that corresponds to the indicated
771  * filename. Normally, this is the specified cache filename exactly; but in
772  * the case of a hash collision, it may be a variant of the cache filename.
773  */
774 PT(BamCacheRecord) BamCache::
775 find_and_read_record(const Filename &source_pathname,
776  const Filename &cache_filename) {
777  int pass = 0;
778  while (true) {
779  PT(BamCacheRecord) record =
780  read_record(source_pathname, cache_filename, pass);
781  if (record != nullptr) {
782  add_to_index(record);
783  return record;
784  }
785  ++pass;
786  }
787 }
788 
789 /**
790  * Reads the indicated cache file and returns its associated record if it can
791  * be read and it matches the source filename.
792  */
793 PT(BamCacheRecord) BamCache::
794 read_record(const Filename &source_pathname,
795  const Filename &cache_filename,
796  int pass) {
798  Filename cache_pathname(_root, cache_filename);
799  if (pass != 0) {
800  ostringstream strm;
801  strm << cache_pathname.get_basename_wo_extension() << "_" << pass;
802  cache_pathname.set_basename_wo_extension(strm.str());
803  }
804 
805  if (!cache_pathname.exists()) {
806  // There is no such cache file already. Declare it.
807  if (util_cat.is_debug()) {
808  util_cat.debug()
809  << "Declaring new cache file " << cache_pathname << " for " << source_pathname << "\n";
810  }
811  PT(BamCacheRecord) record =
812  new BamCacheRecord(source_pathname, cache_filename);
813  record->_cache_pathname = cache_pathname;
814  return record;
815  }
816 
817  if (util_cat.is_debug()) {
818  util_cat.debug()
819  << "Reading cache file " << cache_pathname << " for " << source_pathname << "\n";
820  }
821 
822  PT(BamCacheRecord) record = do_read_record(cache_pathname, true);
823  if (record == nullptr) {
824  // Well, it was invalid, so blow it away, and make a new one.
825  if (util_cat.is_debug()) {
826  util_cat.debug()
827  << "Deleting invalid cache file " << cache_pathname << "\n";
828  }
829  vfs->delete_file(cache_pathname);
830  remove_from_index(source_pathname);
831 
832  PT(BamCacheRecord) record =
833  new BamCacheRecord(source_pathname, cache_filename);
834  record->_cache_pathname = cache_pathname;
835  return record;
836  }
837 
838  if (record->get_source_pathname() != source_pathname) {
839  // This might be just a hash conflict.
840  if (util_cat.is_debug()) {
841  util_cat.debug()
842  << "Cache file " << cache_pathname << " references "
843  << record->get_source_pathname() << ", not "
844  << source_pathname << "\n";
845  }
846  return nullptr;
847  }
848 
849  if (!record->has_data()) {
850  // If we didn't find any data, the caller will have to reload it.
851  record->clear_dependent_files();
852  }
853 
854  record->_cache_pathname = cache_pathname;
855  return record;
856 }
857 
858 /**
859  * Actually reads a record from the file.
860  */
861 PT(BamCacheRecord) BamCache::
862 do_read_record(const Filename &cache_pathname, bool read_data) {
863  DatagramInputFile din;
864  if (!din.open(cache_pathname)) {
865  if (util_cat.is_debug()) {
866  util_cat.debug()
867  << "Could not read cache file: " << cache_pathname << "\n";
868  }
869  return nullptr;
870  }
871 
872  string head;
873  if (!din.read_header(head, _bam_header.size())) {
874  if (util_cat.is_debug()) {
875  util_cat.debug()
876  << cache_pathname << " is not a cache file.\n";
877  }
878  return nullptr;
879  }
880 
881  if (head != _bam_header) {
882  if (util_cat.is_debug()) {
883  util_cat.debug()
884  << cache_pathname << " is not a cache file.\n";
885  }
886  return nullptr;
887  }
888 
889  BamReader reader(&din);
890  if (!reader.init()) {
891  return nullptr;
892  }
893 
894  TypedWritable *object = reader.read_object();
895  if (object == nullptr) {
896  if (util_cat.is_debug()) {
897  util_cat.debug()
898  << cache_pathname << " is empty.\n";
899  }
900  return nullptr;
901 
902  } else if (!object->is_of_type(BamCacheRecord::get_class_type())) {
903  if (util_cat.is_debug()) {
904  util_cat.debug()
905  << "Cache file " << cache_pathname << " contains a "
906  << object->get_type() << ", not a BamCacheRecord.\n";
907  }
908  return nullptr;
909  }
910 
911  PT(BamCacheRecord) record = DCAST(BamCacheRecord, object);
912  if (!reader.resolve()) {
913  if (util_cat.is_debug()) {
914  util_cat.debug()
915  << "Unable to fully resolve cache record in " << cache_pathname << "\n";
916  }
917  return nullptr;
918  }
919 
920  // From this point below, we have validated that the selected filename is
921  // indeed a cache record for the indicated source file, and therefore the
922  // cache record will be returned.
923 
924  // We still need to decide whether the cache record is stale.
925  if (read_data && record->dependents_unchanged()) {
926  // The cache record doesn't appear to be stale. Load the cached object.
927  TypedWritable *ptr;
928  ReferenceCount *ref_ptr;
929 
930  if (reader.read_object(ptr, ref_ptr)) {
931  if (!reader.resolve()) {
932  if (util_cat.is_debug()) {
933  util_cat.debug()
934  << "Unable to fully resolve cached object in " << cache_pathname << "\n";
935  }
936  delete object;
937  } else {
938  // The object is valid. Store it in the record.
939  record->set_data(ptr, ref_ptr);
940  }
941  }
942  }
943 
944  // Also get the total file size.
945  PT(VirtualFile) vfile = din.get_vfile();
946  istream &in = din.get_stream();
947  in.clear();
948  record->_record_size = vfile->get_file_size(&in);
949 
950  // And the last access time is now, duh.
951  record->_record_access_time = time(nullptr);
952 
953  return record;
954 }
955 
956 /**
957  * Returns the appropriate filename to use for a cache file, given the
958  * fullpath string to the source filename.
959  */
960 string BamCache::
961 hash_filename(const string &filename) {
962 #ifdef HAVE_OPENSSL
963  // With OpenSSl, use the MD5 hash of the filename.
964  HashVal hv;
965  hv.hash_string(filename);
966  ostringstream strm;
967  hv.output_hex(strm);
968  return strm.str();
969 
970 #else // HAVE_OPENSSL
971  // Without OpenSSL, don't get fancy; just build a simple hash.
972  unsigned int hash = 0;
973  for (string::const_iterator si = filename.begin();
974  si != filename.end();
975  ++si) {
976  hash = (hash * 9109) + (unsigned int)(*si);
977  }
978 
979  ostringstream strm;
980  strm << std::hex << std::setw(8) << std::setfill('0') << hash;
981  return strm.str();
982 
983 #endif // HAVE_OPENSSL
984 }
985 
986 /**
987  * Constructs the global BamCache object.
988  */
989 void BamCache::
990 make_global() {
991  _global_ptr = new BamCache;
992 
993  if (_global_ptr->_root.empty()) {
994  _global_ptr->set_active(false);
995  }
996 }
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PT(BamCacheRecord) BamCache
Looks up a file in the cache.
Definition: bamCache.cxx:164
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
This represents the in-memory index that records the list of files stored in the BamCache.
Definition: bamCacheIndex.h:33
An instance of this class is written to the front of a Bam or Txo file to make the file a cached inst...
get_data
Returns a pointer to the data stored in the record, or NULL if there is no data.
set_data
Stores a new data object on the record.
void clear_dependent_files()
Empties the list of files that contribute to the data in this record.
bool dependents_unchanged() const
Returns true if all of the dependent files are still the same as when the cache was recorded,...
This class maintains a cache of Bam and/or Txo objects generated from model files and texture images ...
Definition: bamCache.h:42
void consider_flush_index()
Flushes the index if enough time has elapsed since the index was last flushed.
Definition: bamCache.cxx:323
void list_index(std::ostream &out, int indent_level=0) const
Writes the contents of the index to standard output.
Definition: bamCache.cxx:400
bool store(BamCacheRecord *record)
Flushes a cache entry to disk.
Definition: bamCache.cxx:194
void flush_index()
Ensures the index is written to disk.
Definition: bamCache.cxx:348
set_active
Changes the state of the active flag.
Definition: bamCache.h:88
set_root
Changes the current root pathname of the cache.
Definition: bamCache.h:95
This is the fundamental interface for extracting binary objects from a Bam file, as generated by a Ba...
Definition: bamReader.h:110
This is the fundamental interface for writing binary objects to a Bam file, to be extracted later by ...
Definition: bamWriter.h:63
bool write_object(const TypedWritable *obj)
Writes a single object to the Bam file, so that the BamReader::read_object() can later correctly rest...
Definition: bamWriter.cxx:204
void set_file_texture_mode(BamTextureMode file_texture_mode)
Changes the BamTextureMode preference for the Bam file currently being written.
Definition: bamWriter.I:101
bool init()
Initializes the BamWriter prior to writing any objects to its output stream.
Definition: bamWriter.cxx:147
set_root_node
Sets the root node of the part of the scene graph we are currently writing out.
Definition: bamWriter.h:96
This is a convenience class to specialize ConfigVariable as a boolean type.
This is a convenience class to specialize ConfigVariable as a Filename type.
This is a convenience class to specialize ConfigVariable as an integer type.
This class can be used to read a binary file that consists of an arbitrary header followed by a numbe...
virtual VirtualFile * get_vfile()
Returns the VirtualFile that provides the source for these datagrams, if any, or NULL if the datagram...
bool read_header(std::string &header, size_t num_bytes)
Reads a sequence of bytes from the beginning of the datagram file.
bool open(const FileReference *file)
Opens the indicated filename for reading.
std::istream & get_stream()
Returns the istream represented by the input file.
This class can be used to write a binary file that consists of an arbitrary header followed by a numb...
bool open(const FileReference *file)
Opens the indicated filename for writing.
void close()
Closes the file.
virtual std::streampos get_file_pos()
Returns the current file position within the data stream, if any, or 0 if the file position is not me...
bool write_header(const std::string &header)
Writes a sequence of bytes to the beginning of the datagram file.
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:39
std::string get_basename() const
Returns the basename part of the filename.
Definition: filename.I:367
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:414
std::string get_fullpath() const
Returns the entire filename: directory, basename, extension.
Definition: filename.I:338
static Filename temporary(const std::string &dirname, const std::string &prefix, const std::string &suffix=std::string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix.
Definition: filename.cxx:424
void set_extension(const std::string &s)
Replaces the file extension.
Definition: filename.cxx:804
bool make_relative_to(Filename directory, bool allow_backups=true)
Adjusts this filename, which must be a fully-specified pathname beginning with a slash,...
Definition: filename.cxx:1640
std::string get_extension() const
Returns the file extension.
Definition: filename.I:400
bool is_local() const
Returns true if the filename is local, e.g.
Definition: filename.I:549
void make_absolute()
Converts the filename to a fully-qualified pathname from the root (if it is a relative pathname),...
Definition: filename.cxx:968
Type get_type() const
Returns the type of the file represented by the filename, as previously set by set_type().
Definition: filename.I:485
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer.
Definition: hashVal.h:31
void output_hex(std::ostream &out) const
Outputs the HashVal as a 32-digit hexadecimal number.
Definition: hashVal.cxx:34
bool try_lock()
Alias for try_acquire() to match C++11 semantics.
Definition: reMutexDirect.I:48
void unlock()
Alias for release() to match C++11 semantics.
Definition: reMutexDirect.I:62
Similar to MutexHolder, but for a reentrant mutex.
Definition: reMutexHolder.h:25
A base class for all things that want to be reference-counted.
A thread; that is, a lightweight process.
Definition: thread.h:46
get_current_thread
Returns a pointer to the currently-executing Thread object.
Definition: thread.h:109
get_unique_id
Returns a string that is guaranteed to be unique to this thread, across all processes on the machine,...
Definition: thread.h:104
TypeHandle is the identifier used to differentiate C++ class types.
Definition: typeHandle.h:81
The TypeRegistry class maintains all the assigned TypeHandles in a given system.
Definition: typeRegistry.h:36
static TypeRegistry * ptr()
Returns the pointer to the global TypeRegistry object.
Definition: typeRegistry.I:30
TypeHandle find_type(const std::string &name) const
Looks for a previously-registered type of the given name.
bool is_of_type(TypeHandle handle) const
Returns true if the current object is or derives from the indicated type.
Definition: typedObject.I:28
Base class for objects that can be written to and read from Bam files.
Definition: typedWritable.h:35
A list of VirtualFiles, as returned by VirtualFile::scan_directory().
A hierarchy of directories and files that appears to be one continuous file system,...
bool rename_file(const Filename &orig_filename, const Filename &new_filename)
Attempts to move or rename the indicated file or directory.
Filename get_cwd() const
Returns the current directory name.
bool exists(const Filename &filename) const
Convenience function; returns true if the named file exists.
bool is_directory(const Filename &filename) const
Convenience function; returns true if the named file exists and is a directory.
bool atomic_compare_and_exchange_contents(const Filename &filename, std::string &orig_contents, const std::string &old_contents, const std::string &new_contents)
See Filename::atomic_compare_and_exchange_contents().
bool delete_file(const Filename &filename)
Attempts to delete the indicated file or directory.
bool make_directory_full(const Filename &filename)
Attempts to create a directory within the file system.
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
bool atomic_read_contents(const Filename &filename, std::string &contents) const
See Filename::atomic_read_contents().
The abstract base class for a file or directory within the VirtualFileSystem.
Definition: virtualFile.h:35
virtual bool delete_file()
Attempts to delete this file or directory.
Definition: virtualFile.cxx:69
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
string trim(const string &str)
Returns a new string representing the contents of the given string with both leading and trailing whi...
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.