32 using std::ostringstream;
35 BamCache *BamCache::_global_ptr =
nullptr;
49 PRC_DESC(
"The full path to a directory, local to this computer, in which "
50 "model and texture files will be cached on load. If a directory "
51 "name is specified here, files may be loaded from the cache "
52 "instead of from their actual pathnames, which may save load time, "
53 "especially if you are loading egg files instead of bam files, "
54 "or if you are loading models from a shared network drive. "
55 "If this is the empty string, no cache will be used."));
58 (
"model-cache-flush", 30,
59 PRC_DESC(
"This is the amount of time, in seconds, between automatic "
60 "flushes of the model-cache index."));
63 (
"model-cache-models",
true,
64 PRC_DESC(
"If this is set to true, models will be cached in the "
65 "model cache, as bam files."));
68 (
"model-cache-textures",
true,
69 PRC_DESC(
"If this is set to true, textures will also be cached in the "
70 "model cache, as txo files."));
73 (
"model-cache-compressed-textures",
false,
74 PRC_DESC(
"If this is set to true, compressed textures will be cached "
75 "in the model cache, in their compressed form as downloaded "
76 "by the GSG. This may be set in conjunction with "
77 "model-cache-textures, or it may be independent."));
80 (
"model-cache-compiled-shaders",
false,
81 PRC_DESC(
"If this is set to true, compiled shaders will be cached "
82 "in the model cache, in their binary form as downloaded "
86 (
"model-cache-max-kbytes", 10485760,
87 PRC_DESC(
"This is the maximum size of the model cache, in kilobytes."));
89 _cache_models = model_cache_models;
90 _cache_textures = model_cache_textures;
91 _cache_compressed_textures = model_cache_compressed_textures;
92 _cache_compiled_shaders = model_cache_compiled_shaders;
94 _flush_time = model_cache_flush;
95 _max_kbytes = model_cache_max_kbytes;
97 if (!model_cache_dir.empty()) {
98 set_root(model_cache_dir);
135 _index_stale_since = 0;
139 <<
"Unable to make directory " << _root <<
", caching disabled.\n";
165 lookup(
const Filename &source_filename,
const string &cache_extension) {
171 Filename source_pathname(source_filename);
174 Filename rel_pathname(source_pathname);
185 return find_and_read_record(source_pathname, cache_filename);
197 nassertr(!record->_cache_pathname.empty(),
false);
198 nassertr(record->has_data(),
false);
208 Filename rel_pathname(record->_cache_pathname);
210 nassertr(rel_pathname.
is_local(),
false);
213 record->_recorded_time = time(
nullptr);
215 Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
221 string extension = current_thread->
get_unique_id() + string(
".tmp");
222 Filename temp_pathname = cache_pathname;
227 if (!dout.
open(temp_pathname)) {
229 <<
"Could not write cache file: " << temp_pathname <<
"\n";
231 emergency_read_only();
237 <<
"Unable to write to " << temp_pathname <<
"\n";
244 if (!writer.
init()) {
246 <<
"Unable to write Bam header to " << temp_pathname <<
"\n";
253 if (record->
get_data()->is_of_type(texture_type)) {
263 if (record->
get_data()->is_of_type(node_type)) {
269 <<
"Unable to write object to " << temp_pathname <<
"\n";
276 <<
"Unable to write object data to " << temp_pathname <<
"\n";
290 if (!vfs->
rename_file(temp_pathname, cache_pathname) && vfs->
exists(temp_pathname)) {
292 if (!vfs->
rename_file(temp_pathname, cache_pathname)) {
294 <<
"Unable to rename " << temp_pathname <<
" to "
295 << cache_pathname <<
"\n";
301 add_to_index(record);
312 emergency_read_only() {
314 "Could not write to the Bam Cache. Disabling future attempts.\n";
324 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
332 if (_index_stale_since != 0) {
333 int elapsed = (int)time(
nullptr) - (int)_index_stale_since;
334 if (elapsed > _flush_time) {
339 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS)
350 if (_index_stale_since == 0) {
362 if (!do_write_index(temp_pathname, _index)) {
363 emergency_read_only();
371 string old_index = _index_ref_contents;
380 _index_pathname = temp_pathname;
381 _index_ref_contents = new_index;
382 _index_stale_since = 0;
390 _index_ref_contents = orig_index;
400 list_index(ostream &out,
int indent_level)
const {
401 _index->write(out, indent_level);
410 if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
418 if (new_index !=
nullptr) {
419 merge_index(new_index);
425 Filename old_index_pathname = _index_pathname;
426 if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
432 if (old_index_pathname == _index_pathname) {
448 read_index_pathname(
Filename &index_pathname,
string &index_ref_contents)
const {
450 index_ref_contents.clear();
456 string trimmed =
trim(index_ref_contents);
457 if (trimmed.empty()) {
474 if (_index_stale_since == 0) {
482 old_index->release_records();
483 new_index->release_records();
486 BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
487 BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
489 while (ai != old_index->_records.end() &&
490 bi != new_index->_records.end()) {
491 if ((*ai).first < (*bi).first) {
494 Filename cache_pathname(_root, record->get_cache_filename());
495 if (cache_pathname.exists()) {
497 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
501 }
else if ((*bi).first < (*ai).first) {
504 Filename cache_pathname(_root, record->get_cache_filename());
505 if (cache_pathname.exists()) {
507 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
515 if (*a_record == *b_record) {
518 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
524 Filename cache_pathname(_root, a_record->get_cache_filename());
526 if (cache_pathname.exists()) {
527 PT(
BamCacheRecord) record = do_read_record(cache_pathname,
false);
528 if (record !=
nullptr) {
529 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
539 while (ai != old_index->_records.end()) {
542 Filename cache_pathname(_root, record->get_cache_filename());
543 if (cache_pathname.exists()) {
545 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
550 while (bi != new_index->_records.end()) {
553 Filename cache_pathname(_root, record->get_cache_filename());
554 if (cache_pathname.exists()) {
556 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
561 _index->process_new_records();
572 if (contents ==
nullptr) {
574 <<
"Unable to read directory " << _root <<
", caching disabled.\n";
582 int num_files = contents->get_num_files();
583 for (
int ci = 0; ci < num_files; ++ci) {
585 Filename filename = file->get_filename();
591 if (record ==
nullptr) {
593 if (util_cat.is_debug()) {
595 <<
"Deleting invalid " << pathname <<
"\n";
600 record->_record_access_time = record->_recorded_time;
602 bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second;
605 <<
"Multiple cache files defining " << record->get_source_pathname() <<
"\n";
611 _index->process_new_records();
613 _index_stale_since = time(
nullptr);
626 if (_index->add_record(new_record)) {
636 remove_from_index(
const Filename &source_pathname) {
637 if (_index->remove_record(source_pathname)) {
648 if (_index->_cache_size == 0) {
653 if (_index->_cache_size / 1024 > _max_kbytes) {
654 while (_index->_cache_size / 1024 > _max_kbytes) {
656 if (record ==
nullptr) {
661 Filename cache_pathname(_root, record->get_cache_filename());
662 if (util_cat.is_debug()) {
664 <<
"Deleting " << cache_pathname
665 <<
" to keep cache size below " << _max_kbytes <<
"K\n";
678 do_read_index(
const Filename &index_pathname) {
679 if (index_pathname.empty()) {
684 if (!din.
open(index_pathname)) {
686 <<
"Could not read index file: " << index_pathname <<
"\n";
693 << index_pathname <<
" is not an index file.\n";
697 if (head != _bam_header) {
699 << index_pathname <<
" is not an index file.\n";
704 if (!reader.init()) {
710 if (
object ==
nullptr) {
712 <<
"Cache index " << index_pathname <<
" is empty.\n";
715 }
else if (!object->
is_of_type(BamCacheIndex::get_class_type())) {
717 <<
"Cache index " << index_pathname <<
" contains a "
718 <<
object->
get_type() <<
", not a BamCacheIndex.\n";
723 if (!reader.resolve()) {
725 <<
"Unable to fully resolve cache index file.\n";
739 if (!dout.
open(index_pathname)) {
741 <<
"Could not write index file: " << index_pathname <<
"\n";
748 <<
"Unable to write to " << index_pathname <<
"\n";
755 if (!writer.init()) {
760 if (!writer.write_object(index)) {
775 find_and_read_record(
const Filename &source_pathname,
780 read_record(source_pathname, cache_filename, pass);
781 if (record !=
nullptr) {
782 add_to_index(record);
794 read_record(
const Filename &source_pathname,
798 Filename cache_pathname(_root, cache_filename);
801 strm << cache_pathname.get_basename_wo_extension() <<
"_" << pass;
802 cache_pathname.set_basename_wo_extension(strm.str());
805 if (!cache_pathname.exists()) {
807 if (util_cat.is_debug()) {
809 <<
"Declaring new cache file " << cache_pathname <<
" for " << source_pathname <<
"\n";
813 record->_cache_pathname = cache_pathname;
817 if (util_cat.is_debug()) {
819 <<
"Reading cache file " << cache_pathname <<
" for " << source_pathname <<
"\n";
823 if (record ==
nullptr) {
825 if (util_cat.is_debug()) {
827 <<
"Deleting invalid cache file " << cache_pathname <<
"\n";
830 remove_from_index(source_pathname);
834 record->_cache_pathname = cache_pathname;
838 if (record->get_source_pathname() != source_pathname) {
840 if (util_cat.is_debug()) {
842 <<
"Cache file " << cache_pathname <<
" references "
843 << record->get_source_pathname() <<
", not "
844 << source_pathname <<
"\n";
849 if (!record->has_data()) {
854 record->_cache_pathname = cache_pathname;
862 do_read_record(
const Filename &cache_pathname,
bool read_data) {
864 if (!din.
open(cache_pathname)) {
865 if (util_cat.is_debug()) {
867 <<
"Could not read cache file: " << cache_pathname <<
"\n";
874 if (util_cat.is_debug()) {
876 << cache_pathname <<
" is not a cache file.\n";
881 if (head != _bam_header) {
882 if (util_cat.is_debug()) {
884 << cache_pathname <<
" is not a cache file.\n";
890 if (!reader.init()) {
895 if (
object ==
nullptr) {
896 if (util_cat.is_debug()) {
898 << cache_pathname <<
" is empty.\n";
902 }
else if (!object->
is_of_type(BamCacheRecord::get_class_type())) {
903 if (util_cat.is_debug()) {
905 <<
"Cache file " << cache_pathname <<
" contains a "
906 <<
object->
get_type() <<
", not a BamCacheRecord.\n";
912 if (!reader.resolve()) {
913 if (util_cat.is_debug()) {
915 <<
"Unable to fully resolve cache record in " << cache_pathname <<
"\n";
930 if (reader.read_object(ptr, ref_ptr)) {
931 if (!reader.resolve()) {
932 if (util_cat.is_debug()) {
934 <<
"Unable to fully resolve cached object in " << cache_pathname <<
"\n";
948 record->_record_size = vfile->get_file_size(&in);
951 record->_record_access_time = time(
nullptr);
961 hash_filename(
const string &filename) {
965 hv.hash_string(filename);
972 unsigned int hash = 0;
973 for (string::const_iterator si = filename.begin();
974 si != filename.end();
976 hash = (hash * 9109) + (
unsigned int)(*si);
980 strm << std::hex << std::setw(8) << std::setfill(
'0') << hash;
993 if (_global_ptr->_root.empty()) {
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PT(BamCacheRecord) BamCache
Looks up a file in the cache.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
This represents the in-memory index that records the list of files stored in the BamCache.
An instance of this class is written to the front of a Bam or Txo file to make the file a cached inst...
get_data
Returns a pointer to the data stored in the record, or NULL if there is no data.
set_data
Stores a new data object on the record.
void clear_dependent_files()
Empties the list of files that contribute to the data in this record.
bool dependents_unchanged() const
Returns true if all of the dependent files are still the same as when the cache was recorded,...
This class maintains a cache of Bam and/or Txo objects generated from model files and texture images ...
void consider_flush_index()
Flushes the index if enough time has elapsed since the index was last flushed.
void list_index(std::ostream &out, int indent_level=0) const
Writes the contents of the index to standard output.
bool store(BamCacheRecord *record)
Flushes a cache entry to disk.
void flush_index()
Ensures the index is written to disk.
set_active
Changes the state of the active flag.
set_root
Changes the current root pathname of the cache.
This is the fundamental interface for extracting binary objects from a Bam file, as generated by a Ba...
This is the fundamental interface for writing binary objects to a Bam file, to be extracted later by ...
bool write_object(const TypedWritable *obj)
Writes a single object to the Bam file, so that the BamReader::read_object() can later correctly rest...
void set_file_texture_mode(BamTextureMode file_texture_mode)
Changes the BamTextureMode preference for the Bam file currently being written.
bool init()
Initializes the BamWriter prior to writing any objects to its output stream.
set_root_node
Sets the root node of the part of the scene graph we are currently writing out.
This is a convenience class to specialize ConfigVariable as a boolean type.
This is a convenience class to specialize ConfigVariable as a Filename type.
This is a convenience class to specialize ConfigVariable as an integer type.
This class can be used to write a binary file that consists of an arbitrary header followed by a numb...
bool open(const FileReference *file)
Opens the indicated filename for writing.
void close()
Closes the file.
virtual std::streampos get_file_pos()
Returns the current file position within the data stream, if any, or 0 if the file position is not me...
bool write_header(const std::string &header)
Writes a sequence of bytes to the beginning of the datagram file.
The name of a file, such as a texture file or an Egg file.
std::string get_basename() const
Returns the basename part of the filename.
void set_binary()
Indicates that the filename represents a binary file.
std::string get_fullpath() const
Returns the entire filename: directory, basename, extension.
static Filename temporary(const std::string &dirname, const std::string &prefix, const std::string &suffix=std::string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix.
void set_extension(const std::string &s)
Replaces the file extension.
bool make_relative_to(Filename directory, bool allow_backups=true)
Adjusts this filename, which must be a fully-specified pathname beginning with a slash,...
std::string get_extension() const
Returns the file extension.
bool is_local() const
Returns true if the filename is local, e.g.
void make_absolute()
Converts the filename to a fully-qualified pathname from the root (if it is a relative pathname),...
Type get_type() const
Returns the type of the file represented by the filename, as previously set by set_type().
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer.
void output_hex(std::ostream &out) const
Outputs the HashVal as a 32-digit hexadecimal number.
bool try_lock()
Alias for try_acquire() to match C++11 semantics.
void unlock()
Alias for release() to match C++11 semantics.
Similar to MutexHolder, but for a reentrant mutex.
A base class for all things that want to be reference-counted.
A thread; that is, a lightweight process.
get_current_thread
Returns a pointer to the currently-executing Thread object.
get_unique_id
Returns a string that is guaranteed to be unique to this thread, across all processes on the machine,...
TypeHandle is the identifier used to differentiate C++ class types.
The TypeRegistry class maintains all the assigned TypeHandles in a given system.
static TypeRegistry * ptr()
Returns the pointer to the global TypeRegistry object.
TypeHandle find_type(const std::string &name) const
Looks for a previously-registered type of the given name.
bool is_of_type(TypeHandle handle) const
Returns true if the current object is or derives from the indicated type.
Base class for objects that can be written to and read from Bam files.
A list of VirtualFiles, as returned by VirtualFile::scan_directory().
A hierarchy of directories and files that appears to be one continuous file system,...
bool rename_file(const Filename &orig_filename, const Filename &new_filename)
Attempts to move or rename the indicated file or directory.
Filename get_cwd() const
Returns the current directory name.
bool exists(const Filename &filename) const
Convenience function; returns true if the named file exists.
bool is_directory(const Filename &filename) const
Convenience function; returns true if the named file exists and is a directory.
bool atomic_compare_and_exchange_contents(const Filename &filename, std::string &orig_contents, const std::string &old_contents, const std::string &new_contents)
See Filename::atomic_compare_and_exchange_contents().
bool delete_file(const Filename &filename)
Attempts to delete the indicated file or directory.
bool make_directory_full(const Filename &filename)
Attempts to create a directory within the file system.
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
bool atomic_read_contents(const Filename &filename, std::string &contents) const
See Filename::atomic_read_contents().
The abstract base class for a file or directory within the VirtualFileSystem.
virtual bool delete_file()
Attempts to delete this file or directory.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
string trim(const string &str)
Returns a new string representing the contents of the given string with both leading and trailing whi...
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.