Panda3D
Loading...
Searching...
No Matches
bamCache.cxx
Go to the documentation of this file.
1/**
2 * PANDA 3D SOFTWARE
3 * Copyright (c) Carnegie Mellon University. All rights reserved.
4 *
5 * All use of this software is subject to the terms of the revised BSD
6 * license. You should have received a copy of this license along
7 * with this source code in a file named "LICENSE."
8 *
9 * @file bamCache.cxx
10 * @author drose
11 * @date 2006-06-09
12 */
13
14#include "bamCache.h"
15#include "bamCacheIndex.h"
16#include "bamReader.h"
17#include "bamWriter.h"
18#include "hashVal.h"
19#include "datagramInputFile.h"
20#include "datagramOutputFile.h"
21#include "config_putil.h"
22#include "bam.h"
23#include "typeRegistry.h"
24#include "string_utils.h"
25#include "configVariableInt.h"
28#include "virtualFileSystem.h"
29
30using std::istream;
31using std::ostream;
32using std::ostringstream;
33using std::string;
34
35BamCache *BamCache::_global_ptr = nullptr;
36
37/**
38 *
39 */
40BamCache::
41BamCache() :
42 _active(true),
43 _read_only(false),
44 _index(new BamCacheIndex),
45 _index_stale_since(0)
46{
47 ConfigVariableFilename model_cache_dir
48 ("model-cache-dir", Filename(),
49 PRC_DESC("The full path to a directory, local to this computer, in which "
50 "model and texture files will be cached on load. If a directory "
51 "name is specified here, files may be loaded from the cache "
52 "instead of from their actual pathnames, which may save load time, "
53 "especially if you are loading egg files instead of bam files, "
54 "or if you are loading models from a shared network drive. "
55 "If this is the empty string, no cache will be used."));
56
57 ConfigVariableInt model_cache_flush
58 ("model-cache-flush", 30,
59 PRC_DESC("This is the amount of time, in seconds, between automatic "
60 "flushes of the model-cache index."));
61
62 ConfigVariableBool model_cache_models
63 ("model-cache-models", true,
64 PRC_DESC("If this is set to true, models will be cached in the "
65 "model cache, as bam files."));
66
67 ConfigVariableBool model_cache_textures
68 ("model-cache-textures", true,
69 PRC_DESC("If this is set to true, textures will also be cached in the "
70 "model cache, as txo files."));
71
72 ConfigVariableBool model_cache_compressed_textures
73 ("model-cache-compressed-textures", false,
74 PRC_DESC("If this is set to true, compressed textures will be cached "
75 "in the model cache, in their compressed form as downloaded "
76 "by the GSG. This may be set in conjunction with "
77 "model-cache-textures, or it may be independent."));
78
79 ConfigVariableBool model_cache_compiled_shaders
80 ("model-cache-compiled-shaders", false,
81 PRC_DESC("If this is set to true, compiled shaders will be cached "
82 "in the model cache, in their binary form as downloaded "
83 "by the GSG."));
84
85 ConfigVariableInt model_cache_max_kbytes
86 ("model-cache-max-kbytes", 10485760,
87 PRC_DESC("This is the maximum size of the model cache, in kilobytes."));
88
89 _cache_models = model_cache_models;
90 _cache_textures = model_cache_textures;
91 _cache_compressed_textures = model_cache_compressed_textures;
92 _cache_compiled_shaders = model_cache_compiled_shaders;
93
94 _flush_time = model_cache_flush;
95 _max_kbytes = model_cache_max_kbytes;
96
97 if (!model_cache_dir.empty()) {
98 set_root(model_cache_dir);
99 }
100}
101
102/**
103 *
104 */
105BamCache::
106~BamCache() {
107 flush_index();
108 delete _index;
109 _index = nullptr;
110}
111
112/**
113 * Changes the current root pathname of the cache. This specifies where the
114 * cache files are stored on disk. This should name a directory that is on a
115 * disk local to the machine (not on a network-mounted disk), for instance,
116 * /tmp/panda-cache or /c/panda-cache.
117 *
118 * If the directory does not already exist, it will be created as a result of
119 * this call.
120 */
121void BamCache::
122set_root(const Filename &root) {
123 ReMutexHolder holder(_lock);
124 flush_index();
125 _root = root;
126
127 // The root filename must be a directory.
129 if (!vfs->is_directory(_root)) {
130 vfs->make_directory_full(_root);
131 }
132
133 delete _index;
134 _index = new BamCacheIndex;
135 _index_stale_since = 0;
136
137 if (!vfs->is_directory(_root)) {
138 util_cat.error()
139 << "Unable to make directory " << _root << ", caching disabled.\n";
140 _active = false;
141 return;
142 }
143
144 read_index();
145 check_cache_size();
146}
147
148/**
149 * Looks up a file in the cache.
150 *
151 * If the file is cacheable, then regardless of whether the file is found in
152 * the cache or not, this returns a BamCacheRecord. On the other hand, if the
153 * file cannot be cached, returns NULL.
154 *
155 * If record->has_data() returns true, then the file was found in the cache,
156 * and you may call record->extract_data() to get the object. If
157 * record->has_data() returns false, then the file was not found in the cache
158 * or the cache was stale; and you should reload the source file (calling
159 * record->add_dependent_file() for each file loaded, including the original
160 * source file), and then call record->set_data() to record the resulting
161 * loaded object; and finally, you should call store() to write the cached
162 * record to disk.
163 */
164PT(BamCacheRecord) BamCache::
165lookup(const Filename &source_filename, const string &cache_extension) {
166 ReMutexHolder holder(_lock);
168
170
171 Filename source_pathname(source_filename);
172 source_pathname.make_absolute(vfs->get_cwd());
173
174 Filename rel_pathname(source_pathname);
175 rel_pathname.make_relative_to(_root, false);
176 if (rel_pathname.is_local()) {
177 // If the source pathname is already within the cache directory, don't
178 // cache it further.
179 return nullptr;
180 }
181
182 Filename cache_filename = hash_filename(source_pathname.get_fullpath());
183 cache_filename.set_extension(cache_extension);
184
185 return find_and_read_record(source_pathname, cache_filename);
186}
187
188/**
189 * Flushes a cache entry to disk. You must have retrieved the cache record
190 * via a prior call to lookup(), and then stored the data via
191 * record->set_data(). Returns true on success, false on failure.
192 */
194store(BamCacheRecord *record) {
196 ReMutexHolder holder(_lock);
197 nassertr(!record->_cache_pathname.empty(), false);
198 nassertr(record->has_data(), false);
199
200 if (_read_only) {
201 return false;
202 }
203
205
206#ifndef NDEBUG
207 // Ensure that the cache_pathname is within the _root directory tree.
208 Filename rel_pathname(record->_cache_pathname);
209 rel_pathname.make_relative_to(_root, false);
210 nassertr(rel_pathname.is_local(), false);
211#endif // NDEBUG
212
213 record->_recorded_time = time(nullptr);
214
215 Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
216
217 // We actually do the write to a temporary filename first, and then move it
218 // into place, so that no one attempts to read the file while it is in the
219 // process of being written.
220 Thread *current_thread = Thread::get_current_thread();
221 string extension = current_thread->get_unique_id() + string(".tmp");
222 Filename temp_pathname = cache_pathname;
223 temp_pathname.set_extension(extension);
224 temp_pathname.set_binary();
225
227 if (!dout.open(temp_pathname)) {
228 util_cat.error()
229 << "Could not write cache file: " << temp_pathname << "\n";
230 vfs->delete_file(temp_pathname);
231 emergency_read_only();
232 return false;
233 }
234
235 if (!dout.write_header(_bam_header)) {
236 util_cat.error()
237 << "Unable to write to " << temp_pathname << "\n";
238 vfs->delete_file(temp_pathname);
239 return false;
240 }
241
242 {
243 BamWriter writer(&dout);
244 if (!writer.init()) {
245 util_cat.error()
246 << "Unable to write Bam header to " << temp_pathname << "\n";
247 vfs->delete_file(temp_pathname);
248 return false;
249 }
250
251 TypeRegistry *type_registry = TypeRegistry::ptr();
252 TypeHandle texture_type = type_registry->find_type("Texture");
253 if (record->get_data()->is_of_type(texture_type)) {
254 // Texture objects write the actual texture image.
255 writer.set_file_texture_mode(BamWriter::BTM_rawdata);
256 } else {
257 // Any other kinds of objects write texture references.
258 writer.set_file_texture_mode(BamWriter::BTM_fullpath);
259 }
260
261 // This is necessary for relative NodePaths to work.
262 TypeHandle node_type = type_registry->find_type("PandaNode");
263 if (record->get_data()->is_of_type(node_type)) {
264 writer.set_root_node(record->get_data());
265 }
266
267 if (!writer.write_object(record)) {
268 util_cat.error()
269 << "Unable to write object to " << temp_pathname << "\n";
270 vfs->delete_file(temp_pathname);
271 return false;
272 }
273
274 if (!writer.write_object(record->get_data())) {
275 util_cat.error()
276 << "Unable to write object data to " << temp_pathname << "\n";
277 vfs->delete_file(temp_pathname);
278 return false;
279 }
280
281 // Now that we are done with the BamWriter, it's important to let it
282 // destruct now and clean itself up, or it might get mad if we delete any
283 // TypedWritables below that haven't been written yet.
284 }
285
286 record->_record_size = dout.get_file_pos();
287 dout.close();
288
289 // Now move the file into place.
290 if (!vfs->rename_file(temp_pathname, cache_pathname) && vfs->exists(temp_pathname)) {
291 vfs->delete_file(cache_pathname);
292 if (!vfs->rename_file(temp_pathname, cache_pathname)) {
293 util_cat.error()
294 << "Unable to rename " << temp_pathname << " to "
295 << cache_pathname << "\n";
296 vfs->delete_file(temp_pathname);
297 return false;
298 }
299 }
300
301 add_to_index(record);
302
303 return true;
304}
305
306/**
307 * Called when an attempt to write to the cache dir has failed, usually for
308 * lack of disk space or because of incorrect file permissions. Outputs an
309 * error and puts the BamCache into read-only mode.
310 */
311void BamCache::
312emergency_read_only() {
313 util_cat.error() <<
314 "Could not write to the Bam Cache. Disabling future attempts.\n";
315 _read_only = true;
316}
317
318/**
319 * Flushes the index if enough time has elapsed since the index was last
320 * flushed.
321 */
324#if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
325 if (!_lock.try_lock()) {
326 // If we can't grab the lock, no big deal. We don't want to hold up
327 // the frame waiting for a cache operation. We can try again later.
328 return;
329 }
330#endif
331
332 if (_index_stale_since != 0) {
333 int elapsed = (int)time(nullptr) - (int)_index_stale_since;
334 if (elapsed > _flush_time) {
335 flush_index();
336 }
337 }
338
339#if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
340 _lock.unlock();
341#endif
342}
343
344/**
345 * Ensures the index is written to disk.
346 */
348flush_index() {
349 ReMutexHolder holder(_lock);
350 if (_index_stale_since == 0) {
351 // Never mind.
352 return;
353 }
354
355 while (true) {
356 if (_read_only) {
357 return;
358 }
359
360 Filename temp_pathname = Filename::temporary(_root, "index-", ".boo");
361
362 if (!do_write_index(temp_pathname, _index)) {
363 emergency_read_only();
364 return;
365 }
366
367 // Now atomically write the name of this index file to the index reference
368 // file.
370 Filename index_ref_pathname(_root, Filename("index_name.txt"));
371 string old_index = _index_ref_contents;
372 string new_index = temp_pathname.get_basename() + "\n";
373 string orig_index;
374
375 if (vfs->atomic_compare_and_exchange_contents(index_ref_pathname, orig_index, old_index, new_index)) {
376 // We successfully wrote our version of the index, and no other process
377 // beat us to it. Our index is now the official one. Remove the old
378 // index.
379 vfs->delete_file(_index_pathname);
380 _index_pathname = temp_pathname;
381 _index_ref_contents = new_index;
382 _index_stale_since = 0;
383 return;
384 }
385
386 // Shoot, some other process updated the index while we were trying to
387 // update it, and they beat us to it. We have to merge, and try again.
388 vfs->delete_file(temp_pathname);
389 _index_pathname = Filename(_root, Filename(trim(orig_index)));
390 _index_ref_contents = orig_index;
391 read_index();
392 }
393 check_cache_size();
394}
395
396/**
397 * Writes the contents of the index to standard output.
398 */
400list_index(ostream &out, int indent_level) const {
401 _index->write(out, indent_level);
402}
403
404/**
405 * Reads, or re-reads the index file from disk. If _index_stale_since is
406 * nonzero, the index file is read and then merged with our current index.
407 */
408void BamCache::
409read_index() {
410 if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
411 // Couldn't read the index ref; rebuild the index.
412 rebuild_index();
413 return;
414 }
415
416 while (true) {
417 BamCacheIndex *new_index = do_read_index(_index_pathname);
418 if (new_index != nullptr) {
419 merge_index(new_index);
420 return;
421 }
422
423 // We couldn't read the index. Maybe it's been removed already. See if
424 // the index_pathname has changed.
425 Filename old_index_pathname = _index_pathname;
426 if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
427 // Couldn't read the index ref; rebuild the index.
428 rebuild_index();
429 return;
430 }
431
432 if (old_index_pathname == _index_pathname) {
433 // Nope, we just couldn't read it. Delete it and build a new one.
435 vfs->delete_file(_index_pathname);
436 rebuild_index();
437 flush_index();
438 return;
439 }
440 }
441}
442
443/**
444 * Atomically reads the current index filename from the index reference file.
445 * The index filename moves around as different processes update the index.
446 */
447bool BamCache::
448read_index_pathname(Filename &index_pathname, string &index_ref_contents) const {
450 index_ref_contents.clear();
451 Filename index_ref_pathname(_root, Filename("index_name.txt"));
452 if (!vfs->atomic_read_contents(index_ref_pathname, index_ref_contents)) {
453 return false;
454 }
455
456 string trimmed = trim(index_ref_contents);
457 if (trimmed.empty()) {
458 index_pathname = Filename();
459 } else {
460 index_pathname = Filename(_root, Filename(trimmed));
461 }
462 return true;
463}
464
465/**
466 * The supplied index file has been updated by some other process. Merge it
467 * with our current index.
468 *
469 * Ownership of the pointer is transferred with this call. The caller should
470 * assume that new_index will be deleted by this method.
471 */
472void BamCache::
473merge_index(BamCacheIndex *new_index) {
474 if (_index_stale_since == 0) {
475 // If our index isn't stale, just replace it.
476 delete _index;
477 _index = new_index;
478 return;
479 }
480
481 BamCacheIndex *old_index = _index;
482 old_index->release_records();
483 new_index->release_records();
484 _index = new BamCacheIndex;
485
486 BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
487 BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
488
489 while (ai != old_index->_records.end() &&
490 bi != new_index->_records.end()) {
491 if ((*ai).first < (*bi).first) {
492 // Here is an entry we have in our index, not present in the new index.
493 PT(BamCacheRecord) record = (*ai).second;
494 Filename cache_pathname(_root, record->get_cache_filename());
495 if (cache_pathname.exists()) {
496 // The file exists; keep it.
497 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
498 }
499 ++ai;
500
501 } else if ((*bi).first < (*ai).first) {
502 // Here is an entry in the new index, not present in our index.
503 PT(BamCacheRecord) record = (*bi).second;
504 Filename cache_pathname(_root, record->get_cache_filename());
505 if (cache_pathname.exists()) {
506 // The file exists; keep it.
507 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
508 }
509 ++bi;
510
511 } else {
512 // Here is an entry we have in both.
513 PT(BamCacheRecord) a_record = (*ai).second;
514 PT(BamCacheRecord) b_record = (*bi).second;
515 if (*a_record == *b_record) {
516 // They're the same entry. It doesn't really matter which one we
517 // keep.
518 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
519
520 } else {
521 // They're different. Just throw them both away, and re-read the
522 // current data from the cache file.
523
524 Filename cache_pathname(_root, a_record->get_cache_filename());
525
526 if (cache_pathname.exists()) {
527 PT(BamCacheRecord) record = do_read_record(cache_pathname, false);
528 if (record != nullptr) {
529 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
530 }
531 }
532 }
533
534 ++ai;
535 ++bi;
536 }
537 }
538
539 while (ai != old_index->_records.end()) {
540 // Here is an entry we have in our index, not present in the new index.
541 PT(BamCacheRecord) record = (*ai).second;
542 Filename cache_pathname(_root, record->get_cache_filename());
543 if (cache_pathname.exists()) {
544 // The file exists; keep it.
545 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
546 }
547 ++ai;
548 }
549
550 while (bi != new_index->_records.end()) {
551 // Here is an entry in the new index, not present in our index.
552 PT(BamCacheRecord) record = (*bi).second;
553 Filename cache_pathname(_root, record->get_cache_filename());
554 if (cache_pathname.exists()) {
555 // The file exists; keep it.
556 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
557 }
558 ++bi;
559 }
560
561 _index->process_new_records();
562}
563
564/**
565 * Regenerates the index from scratch by scanning the directory.
566 */
567void BamCache::
568rebuild_index() {
570
571 PT(VirtualFileList) contents = vfs->scan_directory(_root);
572 if (contents == nullptr) {
573 util_cat.error()
574 << "Unable to read directory " << _root << ", caching disabled.\n";
575 set_active(false);
576 return;
577 }
578
579 delete _index;
580 _index = new BamCacheIndex;
581
582 int num_files = contents->get_num_files();
583 for (int ci = 0; ci < num_files; ++ci) {
584 VirtualFile *file = contents->get_file(ci);
585 Filename filename = file->get_filename();
586 if (filename.get_extension() == "bam" ||
587 filename.get_extension() == "txo") {
588 Filename pathname(_root, filename);
589
590 PT(BamCacheRecord) record = do_read_record(pathname, false);
591 if (record == nullptr) {
592 // Well, it was invalid, so blow it away.
593 if (util_cat.is_debug()) {
594 util_cat.debug()
595 << "Deleting invalid " << pathname << "\n";
596 }
597 file->delete_file();
598
599 } else {
600 record->_record_access_time = record->_recorded_time;
601
602 bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second;
603 if (!inserted) {
604 util_cat.info()
605 << "Multiple cache files defining " << record->get_source_pathname() << "\n";
606 file->delete_file();
607 }
608 }
609 }
610 }
611 _index->process_new_records();
612
613 _index_stale_since = time(nullptr);
614 check_cache_size();
615 flush_index();
616}
617
618/**
619 * Updates the index entry for the indicated record. Note that a copy of the
620 * record is made first.
621 */
622void BamCache::
623add_to_index(const BamCacheRecord *record) {
624 PT(BamCacheRecord) new_record = record->make_copy();
625
626 if (_index->add_record(new_record)) {
627 mark_index_stale();
628 check_cache_size();
629 }
630}
631
632/**
633 * Removes the index entry for the indicated record, if there is one.
634 */
635void BamCache::
636remove_from_index(const Filename &source_pathname) {
637 if (_index->remove_record(source_pathname)) {
638 mark_index_stale();
639 }
640}
641
642/**
643 * If the cache size has exceeded its specified size limit, removes an old
644 * file.
645 */
646void BamCache::
647check_cache_size() {
648 if (_index->_cache_size == 0) {
649 // 0 means no limit.
650 return;
651 }
652
653 if (_index->_cache_size / 1024 > _max_kbytes) {
654 while (_index->_cache_size / 1024 > _max_kbytes) {
655 PT(BamCacheRecord) record = _index->evict_old_file();
656 if (record == nullptr) {
657 // Never mind; the cache is empty.
658 break;
659 }
661 Filename cache_pathname(_root, record->get_cache_filename());
662 if (util_cat.is_debug()) {
663 util_cat.debug()
664 << "Deleting " << cache_pathname
665 << " to keep cache size below " << _max_kbytes << "K\n";
666 }
667 vfs->delete_file(cache_pathname);
668 }
669 mark_index_stale();
670 }
671}
672
673/**
674 * Reads the index data from the specified filename. Returns a newly-
675 * allocated BamCacheIndex object on success, or NULL on failure.
676 */
677BamCacheIndex *BamCache::
678do_read_index(const Filename &index_pathname) {
679 if (index_pathname.empty()) {
680 return nullptr;
681 }
682
684 if (!din.open(index_pathname)) {
685 util_cat.debug()
686 << "Could not read index file: " << index_pathname << "\n";
687 return nullptr;
688 }
689
690 string head;
691 if (!din.read_header(head, _bam_header.size())) {
692 util_cat.debug()
693 << index_pathname << " is not an index file.\n";
694 return nullptr;
695 }
696
697 if (head != _bam_header) {
698 util_cat.debug()
699 << index_pathname << " is not an index file.\n";
700 return nullptr;
701 }
702
703 BamReader reader(&din);
704 if (!reader.init()) {
705 return nullptr;
706 }
707
708 TypedWritable *object = reader.read_object();
709
710 if (object == nullptr) {
711 util_cat.error()
712 << "Cache index " << index_pathname << " is empty.\n";
713 return nullptr;
714
715 } else if (!object->is_of_type(BamCacheIndex::get_class_type())) {
716 util_cat.error()
717 << "Cache index " << index_pathname << " contains a "
718 << object->get_type() << ", not a BamCacheIndex.\n";
719 return nullptr;
720 }
721
722 BamCacheIndex *index = DCAST(BamCacheIndex, object);
723 if (!reader.resolve()) {
724 util_cat.error()
725 << "Unable to fully resolve cache index file.\n";
726 return nullptr;
727 }
728
729 return index;
730}
731
732/**
733 * Writes the given index data to the specified filename.
734 */
735bool BamCache::
736do_write_index(const Filename &index_pathname, const BamCacheIndex *index) {
739 if (!dout.open(index_pathname)) {
740 util_cat.error()
741 << "Could not write index file: " << index_pathname << "\n";
742 vfs->delete_file(index_pathname);
743 return false;
744 }
745
746 if (!dout.write_header(_bam_header)) {
747 util_cat.error()
748 << "Unable to write to " << index_pathname << "\n";
749 vfs->delete_file(index_pathname);
750 return false;
751 }
752
753 {
754 BamWriter writer(&dout);
755 if (!writer.init()) {
756 vfs->delete_file(index_pathname);
757 return false;
758 }
759
760 if (!writer.write_object(index)) {
761 vfs->delete_file(index_pathname);
762 return false;
763 }
764 }
765
766 return true;
767}
768
769/**
770 * Looks for the existing cache file that corresponds to the indicated
771 * filename. Normally, this is the specified cache filename exactly; but in
772 * the case of a hash collision, it may be a variant of the cache filename.
773 */
774PT(BamCacheRecord) BamCache::
775find_and_read_record(const Filename &source_pathname,
776 const Filename &cache_filename) {
777 int pass = 0;
778 while (true) {
779 PT(BamCacheRecord) record =
780 read_record(source_pathname, cache_filename, pass);
781 if (record != nullptr) {
782 add_to_index(record);
783 return record;
784 }
785 ++pass;
786 }
787}
788
789/**
790 * Reads the indicated cache file and returns its associated record if it can
791 * be read and it matches the source filename.
792 */
793PT(BamCacheRecord) BamCache::
794read_record(const Filename &source_pathname,
795 const Filename &cache_filename,
796 int pass) {
798 Filename cache_pathname(_root, cache_filename);
799 if (pass != 0) {
800 ostringstream strm;
801 strm << cache_pathname.get_basename_wo_extension() << "_" << pass;
802 cache_pathname.set_basename_wo_extension(strm.str());
803 }
804
805 if (!cache_pathname.exists()) {
806 // There is no such cache file already. Declare it.
807 if (util_cat.is_debug()) {
808 util_cat.debug()
809 << "Declaring new cache file " << cache_pathname << " for " << source_pathname << "\n";
810 }
811 PT(BamCacheRecord) record =
812 new BamCacheRecord(source_pathname, cache_filename);
813 record->_cache_pathname = cache_pathname;
814 return record;
815 }
816
817 if (util_cat.is_debug()) {
818 util_cat.debug()
819 << "Reading cache file " << cache_pathname << " for " << source_pathname << "\n";
820 }
821
822 PT(BamCacheRecord) record = do_read_record(cache_pathname, true);
823 if (record == nullptr) {
824 // Well, it was invalid, so blow it away, and make a new one.
825 if (util_cat.is_debug()) {
826 util_cat.debug()
827 << "Deleting invalid cache file " << cache_pathname << "\n";
828 }
829 vfs->delete_file(cache_pathname);
830 remove_from_index(source_pathname);
831
832 PT(BamCacheRecord) record =
833 new BamCacheRecord(source_pathname, cache_filename);
834 record->_cache_pathname = cache_pathname;
835 return record;
836 }
837
838 if (record->get_source_pathname() != source_pathname) {
839 // This might be just a hash conflict.
840 if (util_cat.is_debug()) {
841 util_cat.debug()
842 << "Cache file " << cache_pathname << " references "
843 << record->get_source_pathname() << ", not "
844 << source_pathname << "\n";
845 }
846 return nullptr;
847 }
848
849 if (!record->has_data()) {
850 // If we didn't find any data, the caller will have to reload it.
851 record->clear_dependent_files();
852 }
853
854 record->_cache_pathname = cache_pathname;
855 return record;
856}
857
858/**
859 * Actually reads a record from the file.
860 */
861PT(BamCacheRecord) BamCache::
862do_read_record(const Filename &cache_pathname, bool read_data) {
864 if (!din.open(cache_pathname)) {
865 if (util_cat.is_debug()) {
866 util_cat.debug()
867 << "Could not read cache file: " << cache_pathname << "\n";
868 }
869 return nullptr;
870 }
871
872 string head;
873 if (!din.read_header(head, _bam_header.size())) {
874 if (util_cat.is_debug()) {
875 util_cat.debug()
876 << cache_pathname << " is not a cache file.\n";
877 }
878 return nullptr;
879 }
880
881 if (head != _bam_header) {
882 if (util_cat.is_debug()) {
883 util_cat.debug()
884 << cache_pathname << " is not a cache file.\n";
885 }
886 return nullptr;
887 }
888
889 BamReader reader(&din);
890 if (!reader.init()) {
891 return nullptr;
892 }
893
894 TypedWritable *object = reader.read_object();
895 if (object == nullptr) {
896 if (util_cat.is_debug()) {
897 util_cat.debug()
898 << cache_pathname << " is empty.\n";
899 }
900 return nullptr;
901
902 } else if (!object->is_of_type(BamCacheRecord::get_class_type())) {
903 if (util_cat.is_debug()) {
904 util_cat.debug()
905 << "Cache file " << cache_pathname << " contains a "
906 << object->get_type() << ", not a BamCacheRecord.\n";
907 }
908 return nullptr;
909 }
910
911 PT(BamCacheRecord) record = DCAST(BamCacheRecord, object);
912 if (!reader.resolve()) {
913 if (util_cat.is_debug()) {
914 util_cat.debug()
915 << "Unable to fully resolve cache record in " << cache_pathname << "\n";
916 }
917 return nullptr;
918 }
919
920 // From this point below, we have validated that the selected filename is
921 // indeed a cache record for the indicated source file, and therefore the
922 // cache record will be returned.
923
924 // We still need to decide whether the cache record is stale.
925 if (read_data && record->dependents_unchanged()) {
926 // The cache record doesn't appear to be stale. Load the cached object.
927 TypedWritable *ptr;
928 ReferenceCount *ref_ptr;
929
930 if (reader.read_object(ptr, ref_ptr)) {
931 if (!reader.resolve()) {
932 if (util_cat.is_debug()) {
933 util_cat.debug()
934 << "Unable to fully resolve cached object in " << cache_pathname << "\n";
935 }
936 delete object;
937 } else {
938 // The object is valid. Store it in the record.
939 record->set_data(ptr, ref_ptr);
940 }
941 }
942 }
943
944 // Also get the total file size.
945 PT(VirtualFile) vfile = din.get_vfile();
946 istream &in = din.get_stream();
947 in.clear();
948 record->_record_size = vfile->get_file_size(&in);
949
950 // And the last access time is now, duh.
951 record->_record_access_time = time(nullptr);
952
953 return record;
954}
955
956/**
957 * Returns the appropriate filename to use for a cache file, given the
958 * fullpath string to the source filename.
959 */
960string BamCache::
961hash_filename(const string &filename) {
962#ifdef HAVE_OPENSSL
963 // With OpenSSl, use the MD5 hash of the filename.
964 HashVal hv;
965 hv.hash_string(filename);
966 ostringstream strm;
967 hv.output_hex(strm);
968 return strm.str();
969
970#else // HAVE_OPENSSL
971 // Without OpenSSL, don't get fancy; just build a simple hash.
972 unsigned int hash = 0;
973 for (string::const_iterator si = filename.begin();
974 si != filename.end();
975 ++si) {
976 hash = (hash * 9109) + (unsigned int)(*si);
977 }
978
979 ostringstream strm;
980 strm << std::hex << std::setw(8) << std::setfill('0') << hash;
981 return strm.str();
982
983#endif // HAVE_OPENSSL
984}
985
986/**
987 * Constructs the global BamCache object.
988 */
989void BamCache::
990make_global() {
991 _global_ptr = new BamCache;
992
993 if (_global_ptr->_root.empty()) {
994 _global_ptr->set_active(false);
995 }
996}
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
This represents the in-memory index that records the list of files stored in the BamCache.
An instance of this class is written to the front of a Bam or Txo file to make the file a cached inst...
get_data
Returns a pointer to the data stored in the record, or NULL if there is no data.
set_data
Stores a new data object on the record.
void clear_dependent_files()
Empties the list of files that contribute to the data in this record.
bool dependents_unchanged() const
Returns true if all of the dependent files are still the same as when the cache was recorded,...
This class maintains a cache of Bam and/or Txo objects generated from model files and texture images ...
Definition bamCache.h:42
void consider_flush_index()
Flushes the index if enough time has elapsed since the index was last flushed.
Definition bamCache.cxx:323
void list_index(std::ostream &out, int indent_level=0) const
Writes the contents of the index to standard output.
Definition bamCache.cxx:400
bool store(BamCacheRecord *record)
Flushes a cache entry to disk.
Definition bamCache.cxx:194
void flush_index()
Ensures the index is written to disk.
Definition bamCache.cxx:348
set_active
Changes the state of the active flag.
Definition bamCache.h:88
set_root
Changes the current root pathname of the cache.
Definition bamCache.h:95
This is the fundamental interface for extracting binary objects from a Bam file, as generated by a Ba...
Definition bamReader.h:110
This is the fundamental interface for writing binary objects to a Bam file, to be extracted later by ...
Definition bamWriter.h:63
bool write_object(const TypedWritable *obj)
Writes a single object to the Bam file, so that the BamReader::read_object() can later correctly rest...
void set_file_texture_mode(BamTextureMode file_texture_mode)
Changes the BamTextureMode preference for the Bam file currently being written.
Definition bamWriter.I:101
bool init()
Initializes the BamWriter prior to writing any objects to its output stream.
set_root_node
Sets the root node of the part of the scene graph we are currently writing out.
Definition bamWriter.h:100
This is a convenience class to specialize ConfigVariable as a boolean type.
This is a convenience class to specialize ConfigVariable as a Filename type.
This is a convenience class to specialize ConfigVariable as an integer type.
This class can be used to read a binary file that consists of an arbitrary header followed by a numbe...
virtual VirtualFile * get_vfile()
Returns the VirtualFile that provides the source for these datagrams, if any, or NULL if the datagram...
bool read_header(std::string &header, size_t num_bytes)
Reads a sequence of bytes from the beginning of the datagram file.
bool open(const FileReference *file)
Opens the indicated filename for reading.
std::istream & get_stream()
Returns the istream represented by the input file.
This class can be used to write a binary file that consists of an arbitrary header followed by a numb...
bool open(const FileReference *file)
Opens the indicated filename for writing.
bool write_header(const vector_uchar &header)
Writes a sequence of bytes to the beginning of the datagram file.
void close()
Closes the file.
virtual std::streampos get_file_pos()
Returns the current file position within the data stream, if any, or 0 if the file position is not me...
The name of a file, such as a texture file or an Egg file.
Definition filename.h:44
std::string get_basename() const
Returns the basename part of the filename.
Definition filename.I:367
void set_binary()
Indicates that the filename represents a binary file.
Definition filename.I:414
std::string get_fullpath() const
Returns the entire filename: directory, basename, extension.
Definition filename.I:338
void set_extension(const std::string &s)
Replaces the file extension.
Definition filename.cxx:804
bool make_relative_to(Filename directory, bool allow_backups=true)
Adjusts this filename, which must be a fully-specified pathname beginning with a slash,...
std::string get_extension() const
Returns the file extension.
Definition filename.I:400
bool is_local() const
Returns true if the filename is local, e.g.
Definition filename.I:549
void make_absolute()
Converts the filename to a fully-qualified pathname from the root (if it is a relative pathname),...
Definition filename.cxx:968
Type get_type() const
Returns the type of the file represented by the filename, as previously set by set_type().
Definition filename.I:485
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer.
Definition hashVal.h:31
void output_hex(std::ostream &out) const
Outputs the HashVal as a 32-digit hexadecimal number.
Definition hashVal.cxx:34
bool try_lock()
Alias for try_acquire() to match C++11 semantics.
void unlock()
Alias for release() to match C++11 semantics.
Similar to MutexHolder, but for a reentrant mutex.
A base class for all things that want to be reference-counted.
A thread; that is, a lightweight process.
Definition thread.h:46
get_current_thread
Returns a pointer to the currently-executing Thread object.
Definition thread.h:109
get_unique_id
Returns a string that is guaranteed to be unique to this thread, across all processes on the machine,...
Definition thread.h:104
TypeHandle is the identifier used to differentiate C++ class types.
Definition typeHandle.h:81
The TypeRegistry class maintains all the assigned TypeHandles in a given system.
static TypeRegistry * ptr()
Returns the pointer to the global TypeRegistry object.
TypeHandle find_type(const std::string &name) const
Looks for a previously-registered type of the given name.
bool is_of_type(TypeHandle handle) const
Returns true if the current object is or derives from the indicated type.
Definition typedObject.I:28
Base class for objects that can be written to and read from Bam files.
A list of VirtualFiles, as returned by VirtualFile::scan_directory().
A hierarchy of directories and files that appears to be one continuous file system,...
bool rename_file(const Filename &orig_filename, const Filename &new_filename)
Attempts to move or rename the indicated file or directory.
Filename get_cwd() const
Returns the current directory name.
bool exists(const Filename &filename) const
Convenience function; returns true if the named file exists in the virtual file system hierarchy.
bool is_directory(const Filename &filename) const
Convenience function; returns true if the named file exists as a directory in the virtual file system...
bool atomic_compare_and_exchange_contents(const Filename &filename, std::string &orig_contents, const std::string &old_contents, const std::string &new_contents)
See Filename::atomic_compare_and_exchange_contents().
bool delete_file(const Filename &filename)
Attempts to delete the indicated file or directory.
bool make_directory_full(const Filename &filename)
Attempts to create a directory within the file system.
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
bool atomic_read_contents(const Filename &filename, std::string &contents) const
See Filename::atomic_read_contents().
The abstract base class for a file or directory within the VirtualFileSystem.
Definition virtualFile.h:35
virtual bool delete_file()
Attempts to delete this file or directory.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
string trim(const string &str)
Returns a new string representing the contents of the given string with both leading and trailing whi...
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.