16 #include "bamCacheIndex.h"
17 #include "bamReader.h"
18 #include "bamWriter.h"
20 #include "datagramInputFile.h"
21 #include "datagramOutputFile.h"
22 #include "config_util.h"
24 #include "typeRegistry.h"
25 #include "string_utils.h"
26 #include "configVariableInt.h"
27 #include "configVariableString.h"
28 #include "configVariableFilename.h"
29 #include "virtualFileSystem.h"
31 BamCache *BamCache::_global_ptr = NULL;
47 PRC_DESC(
"The full path to a directory, local to this computer, in which "
48 "model and texture files will be cached on load. If a directory "
49 "name is specified here, files may be loaded from the cache "
50 "instead of from their actual pathnames, which may save load time, "
51 "especially if you are loading egg files instead of bam files, "
52 "or if you are loading models from a shared network drive. "
53 "If this is the empty string, no cache will be used."));
56 (
"model-cache-flush", 30,
57 PRC_DESC(
"This is the amount of time, in seconds, between automatic "
58 "flushes of the model-cache index."));
61 (
"model-cache-models",
true,
62 PRC_DESC(
"If this is set to true, models will be cached in the "
63 "model cache, as bam files."));
66 (
"model-cache-textures",
true,
67 PRC_DESC(
"If this is set to true, textures will also be cached in the "
68 "model cache, as txo files."));
71 (
"model-cache-compressed-textures",
false,
72 PRC_DESC(
"If this is set to true, compressed textures will be cached "
73 "in the model cache, in their compressed form as downloaded "
74 "by the GSG. This may be set in conjunction with "
75 "model-cache-textures, or it may be independent."));
78 (
"model-cache-max-kbytes", 10485760,
79 PRC_DESC(
"This is the maximum size of the model cache, in kilobytes."));
81 _cache_models = model_cache_models;
82 _cache_textures = model_cache_textures;
83 _cache_compressed_textures = model_cache_compressed_textures;
85 _flush_time = model_cache_flush;
86 _max_kbytes = model_cache_max_kbytes;
88 if (!model_cache_dir.empty()) {
89 set_root(model_cache_dir);
131 _index_stale_since = 0;
161 lookup(const
Filename &source_filename, const
string &cache_extension) {
163 consider_flush_index();
167 Filename source_pathname(source_filename);
168 source_pathname.make_absolute(vfs->
get_cwd());
170 Filename rel_pathname(source_pathname);
171 rel_pathname.make_relative_to(_root,
false);
172 if (rel_pathname.is_local()) {
178 Filename cache_filename = hash_filename(source_pathname.get_fullpath());
181 return find_and_read_record(source_pathname, cache_filename);
197 nassertr(!record->_cache_pathname.empty(),
false);
198 nassertr(record->
has_data(),
false);
208 Filename rel_pathname(record->_cache_pathname);
210 nassertr(rel_pathname.
is_local(),
false);
213 record->_recorded_time = time(NULL);
215 Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
221 string extension = current_thread->
get_unique_id() + string(
".tmp");
222 Filename temp_pathname = cache_pathname;
227 if (!dout.
open(temp_pathname)) {
229 <<
"Could not write cache file: " << temp_pathname <<
"\n";
231 emergency_read_only();
237 <<
"Unable to write to " << temp_pathname <<
"\n";
244 if (!writer.
init()) {
246 <<
"Unable to write Bam header to " << temp_pathname <<
"\n";
263 <<
"Unable to write object to " << temp_pathname <<
"\n";
270 <<
"Unable to write object data to " << temp_pathname <<
"\n";
284 if (!vfs->
rename_file(temp_pathname, cache_pathname) && vfs->
exists(temp_pathname)) {
286 if (!vfs->
rename_file(temp_pathname, cache_pathname)) {
288 <<
"Unable to rename " << temp_pathname <<
" to "
289 << cache_pathname <<
"\n";
295 add_to_index(record);
310 emergency_read_only() {
312 "Could not write to the Bam Cache. Disabling future attempts.\n";
325 if (_index_stale_since != 0) {
326 int elapsed = (int)time(NULL) - (int)_index_stale_since;
327 if (elapsed > _flush_time) {
341 if (_index_stale_since == 0) {
353 if (!do_write_index(temp_pathname, _index)) {
354 emergency_read_only();
362 string old_index = _index_ref_contents;
371 _index_pathname = temp_pathname;
372 _index_ref_contents = new_index;
373 _index_stale_since = 0;
382 _index_ref_contents = orig_index;
395 _index->write(out, indent_level);
407 if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
416 merge_index(new_index);
422 Filename old_index_pathname = _index_pathname;
423 if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
429 if (old_index_pathname == _index_pathname) {
449 read_index_pathname(
Filename &index_pathname,
string &index_ref_contents)
const {
451 index_ref_contents.clear();
457 string trimmed = trim(index_ref_contents);
458 if (trimmed.empty()) {
478 if (_index_stale_since == 0) {
486 old_index->release_records();
487 new_index->release_records();
490 BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
491 BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
493 while (ai != old_index->_records.end() &&
494 bi != new_index->_records.end()) {
495 if ((*ai).first < (*bi).first) {
499 Filename cache_pathname(_root, record->get_cache_filename());
500 if (cache_pathname.exists()) {
502 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->
get_source_pathname(), record));
506 }
else if ((*bi).first < (*ai).first) {
509 Filename cache_pathname(_root, record->get_cache_filename());
510 if (cache_pathname.exists()) {
512 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->
get_source_pathname(), record));
520 if (*a_record == *b_record) {
523 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
529 Filename cache_pathname(_root, a_record->get_cache_filename());
531 if (cache_pathname.exists()) {
532 PT(
BamCacheRecord) record = do_read_record(cache_pathname, false);
534 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->
get_source_pathname(), record));
544 while (ai != old_index->_records.end()) {
548 Filename cache_pathname(_root, record->get_cache_filename());
549 if (cache_pathname.exists()) {
551 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->
get_source_pathname(), record));
556 while (bi != new_index->_records.end()) {
559 Filename cache_pathname(_root, record->get_cache_filename());
560 if (cache_pathname.exists()) {
562 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->
get_source_pathname(), record));
567 _index->process_new_records();
581 if (contents == NULL) {
583 <<
"Unable to read directory " << _root <<
", caching disabled.\n";
591 int num_files = contents->get_num_files();
592 for (
int ci = 0; ci < num_files; ++ci) {
594 Filename filename = file->get_filename();
602 if (util_cat.is_debug()) {
604 <<
"Deleting invalid " << pathname <<
"\n";
609 record->_record_access_time = record->_recorded_time;
611 bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->
get_source_pathname(), record)).second;
620 _index->process_new_records();
622 _index_stale_since = time(NULL);
637 if (_index->add_record(new_record)) {
650 remove_from_index(
const Filename &source_pathname) {
651 if (_index->remove_record(source_pathname)) {
664 if (_index->_cache_size == 0) {
669 if (_index->_cache_size / 1024 > _max_kbytes) {
670 while (_index->_cache_size / 1024 > _max_kbytes) {
672 if (record == NULL) {
678 if (util_cat.is_debug()) {
680 <<
"Deleting " << cache_pathname
681 <<
" to keep cache size below " << _max_kbytes <<
"K\n";
697 do_read_index(
const Filename &index_pathname) {
698 if (index_pathname.empty()) {
703 if (!din.
open(index_pathname)) {
705 <<
"Could not read index file: " << index_pathname <<
"\n";
712 << index_pathname <<
" is not an index file.\n";
716 if (head != _bam_header) {
718 << index_pathname <<
" is not an index file.\n";
723 if (!reader.init()) {
731 <<
"Cache index " << index_pathname <<
" is empty.\n";
734 }
else if (!object->
is_of_type(BamCacheIndex::get_class_type())) {
736 <<
"Cache index " << index_pathname <<
" contains a "
737 <<
object->
get_type() <<
", not a BamCacheIndex.\n";
742 if (!reader.resolve()) {
744 <<
"Unable to fully resolve cache index file.\n";
760 if (!dout.
open(index_pathname)) {
762 <<
"Could not write index file: " << index_pathname <<
"\n";
769 <<
"Unable to write to " << index_pathname <<
"\n";
776 if (!writer.init()) {
781 if (!writer.write_object(index)) {
800 find_and_read_record(const
Filename &source_pathname,
805 read_record(source_pathname, cache_filename, pass);
807 add_to_index(record);
822 read_record(const
Filename &source_pathname,
826 Filename cache_pathname(_root, cache_filename);
829 strm << cache_pathname.get_basename_wo_extension() <<
"_" << pass;
830 cache_pathname.set_basename_wo_extension(strm.str());
833 if (!cache_pathname.exists()) {
835 if (util_cat.is_debug()) {
837 <<
"Declaring new cache file " << cache_pathname <<
" for " << source_pathname <<
"\n";
841 record->_cache_pathname = cache_pathname;
845 if (util_cat.is_debug()) {
847 <<
"Reading cache file " << cache_pathname <<
" for " << source_pathname <<
"\n";
853 if (util_cat.is_debug()) {
855 <<
"Deleting invalid cache file " << cache_pathname <<
"\n";
858 remove_from_index(source_pathname);
862 record->_cache_pathname = cache_pathname;
866 if (record->get_source_pathname() != source_pathname) {
868 if (util_cat.is_debug()) {
870 <<
"Cache file " << cache_pathname <<
" references "
872 << source_pathname <<
"\n";
877 if (!record->has_data()) {
879 record->clear_dependent_files();
882 record->_cache_pathname = cache_pathname;
892 do_read_record(const
Filename &cache_pathname,
bool read_data) {
894 if (!din.
open(cache_pathname)) {
895 if (util_cat.is_debug()) {
897 <<
"Could not read cache file: " << cache_pathname <<
"\n";
904 if (util_cat.is_debug()) {
906 << cache_pathname <<
" is not a cache file.\n";
911 if (head != _bam_header) {
912 if (util_cat.is_debug()) {
914 << cache_pathname <<
" is not a cache file.\n";
920 if (!reader.init()) {
926 if (util_cat.is_debug()) {
928 << cache_pathname <<
" is empty.\n";
932 }
else if (!object->
is_of_type(BamCacheRecord::get_class_type())) {
933 if (util_cat.is_debug()) {
935 <<
"Cache file " << cache_pathname <<
" contains a "
936 <<
object->get_type() <<
", not a BamCacheRecord.\n";
942 if (!reader.resolve()) {
943 if (util_cat.is_debug()) {
945 <<
"Unable to fully resolve cache record in " << cache_pathname <<
"\n";
955 if (read_data && record->dependents_unchanged()) {
961 if (reader.read_object(ptr, ref_ptr)) {
962 if (!reader.resolve()) {
963 if (util_cat.is_debug()) {
965 <<
"Unable to fully resolve cached object in " << cache_pathname <<
"\n";
970 record->set_data(ptr, ref_ptr);
977 istream &in = din.get_stream();
979 record->_record_size = vfile->get_file_size(&in);
982 record->_record_access_time = time(NULL);
995 hash_filename(const
string &filename) {
999 hv.hash_string(filename);
1004 #else // HAVE_OPENSSL
1006 unsigned int hash = 0;
1007 for (string::const_iterator si = filename.begin();
1008 si != filename.end();
1010 hash = (hash * 9109) + (
unsigned int)(*si);
1014 strm << hex << setw(8) << setfill(
'0') << hash;
1017 #endif // HAVE_OPENSSL
1029 if (_global_ptr->_root.empty()) {
virtual streampos get_file_pos()
Returns the current file position within the data stream, if any, or 0 if the file position is not me...
bool atomic_compare_and_exchange_contents(const Filename &filename, string &orig_contents, const string &old_contents, const string &new_contents)
See Filename::atomic_compare_and_exchange_contents().
void set_active(bool flag)
Changes the state of the active flag.
void set_extension(const string &s)
Replaces the file extension.
bool rename_file(const Filename &orig_filename, const Filename &new_filename)
Attempts to move or rename the indicated file or directory.
TypeHandle find_type(const string &name) const
Looks for a previously-registered type of the given name.
bool is_directory(const Filename &filename) const
Convenience function; returns true if the named file exists and is a directory.
This is the fundamental interface for extracting binary objects from a Bam file, as generated by a Ba...
bool atomic_read_contents(const Filename &filename, string &contents) const
See Filename::atomic_read_contents().
This class maintains a cache of Bam and/or Txo objects generated from model files and texture images ...
This is a convenience class to specialize ConfigVariable as a Filename type.
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS's file system.
This is a convenience class to specialize ConfigVariable as a boolean type.
void set_binary()
Indicates that the filename represents a binary file.
Base class for objects that can be written to and read from Bam files.
bool is_of_type(TypeHandle handle) const
Returns true if the current object is or derives from the indicated type.
bool write_header(const string &header)
Writes a sequence of bytes to the beginning of the datagram file.
void flush_index()
Ensures the index is written to disk.
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer...
This is the fundamental interface for writing binary objects to a Bam file, to be extracted later by ...
bool make_directory_full(const Filename &filename)
Attempts to create a directory within the file system.
const Filename & get_cache_filename() const
Returns the name of the cache file as hashed from the source_pathname.
The abstract base class for a file or directory within the VirtualFileSystem.
bool is_local() const
Returns true if the filename is local, e.g.
static Thread * get_current_thread()
Returns a pointer to the currently-executing Thread object.
bool open(const FileReference *file)
Opens the indicated filename for writing.
bool has_data() const
Returns true if this cache record has an in-memory data object associated–that is, the object stored in the cache.
static Filename temporary(const string &dirname, const string &prefix, const string &suffix=string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix...
This represents the in-memory index that records the list of files stored in the BamCache.
void set_file_texture_mode(BamTextureMode file_texture_mode)
Changes the BamTextureMode preference for the Bam file currently being written.
string get_unique_id() const
Returns a string that is guaranteed to be unique to this thread, across all processes on the machine...
Filename get_cwd() const
Returns the current directory name.
bool store(BamCacheRecord *record)
Flushes a cache entry to disk.
void set_root(const Filename &root)
Changes the current root pathname of the cache.
A list of VirtualFiles, as returned by VirtualFile::scan_directory().
Type get_type() const
Returns the type of the file represented by the filename, as previously set by set_type().
The name of a file, such as a texture file or an Egg file.
An instance of this class is written to the front of a Bam or Txo file to make the file a cached inst...
void list_index(ostream &out, int indent_level=0) const
Writes the contents of the index to standard output.
void output_hex(ostream &out) const
Outputs the HashVal as a 32-digit hexadecimal number.
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
virtual bool delete_file()
Attempts to delete this file or directory.
bool write_object(const TypedWritable *obj)
Writes a single object to the Bam file, so that the BamReader::read_object() can later correctly rest...
TypedWritable * get_data() const
Returns a pointer to the data stored in the record, or NULL if there is no data.
Similar to MutexHolder, but for a reentrant mutex.
bool make_relative_to(Filename directory, bool allow_backups=true)
Adjusts this filename, which must be a fully-specified pathname beginning with a slash, to make it a relative filename, relative to the fully-specified directory indicated (which must also begin with, and may or may not end with, a slash–a terminating slash is ignored).
const Filename & get_source_pathname() const
Returns the full pathname to the source file that originally generated this cache request...
static TypeRegistry * ptr()
Returns the pointer to the global TypeRegistry object.
This class can be used to write a binary file that consists of an arbitrary header followed by a numb...
bool exists(const Filename &filename) const
Convenience function; returns true if the named file exists.
A base class for all things that want to be reference-counted.
string get_basename() const
Returns the basename part of the filename.
The TypeRegistry class maintains all the assigned TypeHandles in a given system.
A thread; that is, a lightweight process.
This is a convenience class to specialize ConfigVariable as an integer type.
bool delete_file(const Filename &filename)
Attempts to delete the indicated file or directory.
TypeHandle is the identifier used to differentiate C++ class types.
void close()
Closes the file.
string get_extension() const
Returns the file extension.
bool init()
Initializes the BamWriter prior to writing any objects to its output stream.
void consider_flush_index()
Flushes the index if enough time has elapsed since the index was last flushed.