16 #include "bamCacheIndex.h" 17 #include "bamReader.h" 18 #include "bamWriter.h" 20 #include "datagramInputFile.h" 21 #include "datagramOutputFile.h" 22 #include "config_util.h" 24 #include "typeRegistry.h" 25 #include "string_utils.h" 26 #include "configVariableInt.h" 27 #include "configVariableString.h" 28 #include "configVariableFilename.h" 29 #include "virtualFileSystem.h" 31 BamCache *BamCache::_global_ptr = NULL;
47 PRC_DESC(
"The full path to a directory, local to this computer, in which " 48 "model and texture files will be cached on load. If a directory " 49 "name is specified here, files may be loaded from the cache " 50 "instead of from their actual pathnames, which may save load time, " 51 "especially if you are loading egg files instead of bam files, " 52 "or if you are loading models from a shared network drive. " 53 "If this is the empty string, no cache will be used."));
56 (
"model-cache-flush", 30,
57 PRC_DESC(
"This is the amount of time, in seconds, between automatic " 58 "flushes of the model-cache index."));
61 (
"model-cache-models",
true,
62 PRC_DESC(
"If this is set to true, models will be cached in the " 63 "model cache, as bam files."));
66 (
"model-cache-textures",
true,
67 PRC_DESC(
"If this is set to true, textures will also be cached in the " 68 "model cache, as txo files."));
71 (
"model-cache-compressed-textures",
false,
72 PRC_DESC(
"If this is set to true, compressed textures will be cached " 73 "in the model cache, in their compressed form as downloaded " 74 "by the GSG. This may be set in conjunction with " 75 "model-cache-textures, or it may be independent."));
78 (
"model-cache-max-kbytes", 10485760,
79 PRC_DESC(
"This is the maximum size of the model cache, in kilobytes."));
81 _cache_models = model_cache_models;
82 _cache_textures = model_cache_textures;
83 _cache_compressed_textures = model_cache_compressed_textures;
85 _flush_time = model_cache_flush;
86 _max_kbytes = model_cache_max_kbytes;
88 if (!model_cache_dir.empty()) {
131 _index_stale_since = 0;
161 lookup(
const Filename &source_filename,
const string &cache_extension) {
167 Filename source_pathname(source_filename);
170 Filename rel_pathname(source_pathname);
181 return find_and_read_record(source_pathname, cache_filename);
197 nassertr(!record->_cache_pathname.empty(),
false);
198 nassertr(record->has_data(),
false);
208 Filename rel_pathname(record->_cache_pathname);
210 nassertr(rel_pathname.
is_local(),
false);
213 record->_recorded_time = time(NULL);
215 Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
221 string extension = current_thread->
get_unique_id() + string(
".tmp");
222 Filename temp_pathname = cache_pathname;
227 if (!dout.
open(temp_pathname)) {
229 <<
"Could not write cache file: " << temp_pathname <<
"\n";
231 emergency_read_only();
237 <<
"Unable to write to " << temp_pathname <<
"\n";
244 if (!writer.
init()) {
246 <<
"Unable to write Bam header to " << temp_pathname <<
"\n";
263 <<
"Unable to write object to " << temp_pathname <<
"\n";
270 <<
"Unable to write object data to " << temp_pathname <<
"\n";
284 if (!vfs->
rename_file(temp_pathname, cache_pathname) && vfs->
exists(temp_pathname)) {
286 if (!vfs->
rename_file(temp_pathname, cache_pathname)) {
288 <<
"Unable to rename " << temp_pathname <<
" to " 289 << cache_pathname <<
"\n";
295 add_to_index(record);
310 emergency_read_only() {
312 "Could not write to the Bam Cache. Disabling future attempts.\n";
324 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS) 332 if (_index_stale_since != 0) {
333 int elapsed = (int)time(NULL) - (int)_index_stale_since;
334 if (elapsed > _flush_time) {
339 #if defined(HAVE_THREADS) || defined(DEBUG_THREADS) 352 if (_index_stale_since == 0) {
364 if (!do_write_index(temp_pathname, _index)) {
365 emergency_read_only();
373 string old_index = _index_ref_contents;
382 _index_pathname = temp_pathname;
383 _index_ref_contents = new_index;
384 _index_stale_since = 0;
393 _index_ref_contents = orig_index;
406 _index->write(out, indent_level);
418 if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
427 merge_index(new_index);
433 Filename old_index_pathname = _index_pathname;
434 if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
440 if (old_index_pathname == _index_pathname) {
460 read_index_pathname(
Filename &index_pathname,
string &index_ref_contents)
const {
462 index_ref_contents.clear();
468 string trimmed = trim(index_ref_contents);
469 if (trimmed.empty()) {
489 if (_index_stale_since == 0) {
497 old_index->release_records();
498 new_index->release_records();
501 BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
502 BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
504 while (ai != old_index->_records.end() &&
505 bi != new_index->_records.end()) {
506 if ((*ai).first < (*bi).first) {
510 Filename cache_pathname(_root, record->get_cache_filename());
511 if (cache_pathname.
exists()) {
513 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
517 }
else if ((*bi).first < (*ai).first) {
520 Filename cache_pathname(_root, record->get_cache_filename());
521 if (cache_pathname.
exists()) {
523 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
531 if (*a_record == *b_record) {
534 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
540 Filename cache_pathname(_root, a_record->get_cache_filename());
542 if (cache_pathname.
exists()) {
543 PT(
BamCacheRecord) record = do_read_record(cache_pathname,
false);
545 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
555 while (ai != old_index->_records.end()) {
559 Filename cache_pathname(_root, record->get_cache_filename());
560 if (cache_pathname.
exists()) {
562 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
567 while (bi != new_index->_records.end()) {
570 Filename cache_pathname(_root, record->get_cache_filename());
571 if (cache_pathname.
exists()) {
573 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
578 _index->process_new_records();
592 if (contents == NULL) {
594 <<
"Unable to read directory " << _root <<
", caching disabled.\n";
602 int num_files = contents->get_num_files();
603 for (
int ci = 0; ci < num_files; ++ci) {
605 Filename filename = file->get_filename();
613 if (util_cat.is_debug()) {
615 <<
"Deleting invalid " << pathname <<
"\n";
620 record->_record_access_time = record->_recorded_time;
622 bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second;
625 <<
"Multiple cache files defining " << record->get_source_pathname() <<
"\n";
631 _index->process_new_records();
633 _index_stale_since = time(NULL);
648 if (_index->add_record(new_record)) {
661 remove_from_index(
const Filename &source_pathname) {
662 if (_index->remove_record(source_pathname)) {
675 if (_index->_cache_size == 0) {
680 if (_index->_cache_size / 1024 > _max_kbytes) {
681 while (_index->_cache_size / 1024 > _max_kbytes) {
683 if (record == NULL) {
688 Filename cache_pathname(_root, record->get_cache_filename());
689 if (util_cat.is_debug()) {
691 <<
"Deleting " << cache_pathname
692 <<
" to keep cache size below " << _max_kbytes <<
"K\n";
708 do_read_index(
const Filename &index_pathname) {
709 if (index_pathname.empty()) {
714 if (!din.
open(index_pathname)) {
716 <<
"Could not read index file: " << index_pathname <<
"\n";
723 << index_pathname <<
" is not an index file.\n";
727 if (head != _bam_header) {
729 << index_pathname <<
" is not an index file.\n";
734 if (!reader.
init()) {
742 <<
"Cache index " << index_pathname <<
" is empty.\n";
745 }
else if (!object->
is_of_type(BamCacheIndex::get_class_type())) {
747 <<
"Cache index " << index_pathname <<
" contains a " 748 <<
object->
get_type() <<
", not a BamCacheIndex.\n";
755 <<
"Unable to fully resolve cache index file.\n";
771 if (!dout.
open(index_pathname)) {
773 <<
"Could not write index file: " << index_pathname <<
"\n";
780 <<
"Unable to write to " << index_pathname <<
"\n";
787 if (!writer.
init()) {
811 find_and_read_record(
const Filename &source_pathname,
816 read_record(source_pathname, cache_filename, pass);
818 add_to_index(record);
833 read_record(
const Filename &source_pathname,
837 Filename cache_pathname(_root, cache_filename);
844 if (!cache_pathname.
exists()) {
846 if (util_cat.is_debug()) {
848 <<
"Declaring new cache file " << cache_pathname <<
" for " << source_pathname <<
"\n";
852 record->_cache_pathname = cache_pathname;
856 if (util_cat.is_debug()) {
858 <<
"Reading cache file " << cache_pathname <<
" for " << source_pathname <<
"\n";
864 if (util_cat.is_debug()) {
866 <<
"Deleting invalid cache file " << cache_pathname <<
"\n";
869 remove_from_index(source_pathname);
873 record->_cache_pathname = cache_pathname;
877 if (record->get_source_pathname() != source_pathname) {
879 if (util_cat.is_debug()) {
881 <<
"Cache file " << cache_pathname <<
" references " 882 << record->get_source_pathname() <<
", not " 883 << source_pathname <<
"\n";
888 if (!record->has_data()) {
893 record->_cache_pathname = cache_pathname;
903 do_read_record(
const Filename &cache_pathname,
bool read_data) {
905 if (!din.
open(cache_pathname)) {
906 if (util_cat.is_debug()) {
908 <<
"Could not read cache file: " << cache_pathname <<
"\n";
915 if (util_cat.is_debug()) {
917 << cache_pathname <<
" is not a cache file.\n";
922 if (head != _bam_header) {
923 if (util_cat.is_debug()) {
925 << cache_pathname <<
" is not a cache file.\n";
931 if (!reader.
init()) {
937 if (util_cat.is_debug()) {
939 << cache_pathname <<
" is empty.\n";
943 }
else if (!object->
is_of_type(BamCacheRecord::get_class_type())) {
944 if (util_cat.is_debug()) {
946 <<
"Cache file " << cache_pathname <<
" contains a " 947 <<
object->
get_type() <<
", not a BamCacheRecord.\n";
954 if (util_cat.is_debug()) {
956 <<
"Unable to fully resolve cache record in " << cache_pathname <<
"\n";
974 if (util_cat.is_debug()) {
976 <<
"Unable to fully resolve cached object in " << cache_pathname <<
"\n";
990 record->_record_size = vfile->get_file_size(&in);
993 record->_record_access_time = time(NULL);
1006 hash_filename(
const string &filename) {
1010 hv.hash_string(filename);
1015 #else // HAVE_OPENSSL 1017 unsigned int hash = 0;
1018 for (string::const_iterator si = filename.begin();
1019 si != filename.end();
1021 hash = (hash * 9109) + (
unsigned int)(*si);
1025 strm << hex << setw(8) << setfill(
'0') << hash;
1028 #endif // HAVE_OPENSSL 1040 if (_global_ptr->_root.empty()) {
void set_data(TypedWritable *ptr, ReferenceCount *ref_ptr)
Stores a new data object on the record.
virtual streampos get_file_pos()
Returns the current file position within the data stream, if any, or 0 if the file position is not me...
bool atomic_compare_and_exchange_contents(const Filename &filename, string &orig_contents, const string &old_contents, const string &new_contents)
See Filename::atomic_compare_and_exchange_contents().
void set_active(bool flag)
Changes the state of the active flag.
string get_basename() const
Returns the basename part of the filename.
void set_extension(const string &s)
Replaces the file extension.
bool rename_file(const Filename &orig_filename, const Filename &new_filename)
Attempts to move or rename the indicated file or directory.
TypedWritable * get_data() const
Returns a pointer to the data stored in the record, or NULL if there is no data.
string get_fullpath() const
Returns the entire filename: directory, basename, extension.
This is the fundamental interface for extracting binary objects from a Bam file, as generated by a Ba...
TypeHandle find_type(const string &name) const
Looks for a previously-registered type of the given name.
bool resolve()
This may be called at any time during processing of the Bam file to resolve all the known pointers so...
This class maintains a cache of Bam and/or Txo objects generated from model files and texture images ...
This is a convenience class to specialize ConfigVariable as a Filename type.
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS's file system.
This is a convenience class to specialize ConfigVariable as a boolean type.
bool try_acquire() const
Returns immediately, with a true value indicating the mutex has been acquired, and false indicating i...
TypedWritable * read_object()
Reads a single object from the Bam file.
void release() const
Releases the reMutex.
void set_binary()
Indicates that the filename represents a binary file.
Base class for objects that can be written to and read from Bam files.
bool write_header(const string &header)
Writes a sequence of bytes to the beginning of the datagram file.
void list_index(ostream &out, int indent_level=0) const
Writes the contents of the index to standard output.
void flush_index()
Ensures the index is written to disk.
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer...
This is the fundamental interface for writing binary objects to a Bam file, to be extracted later by ...
bool make_directory_full(const Filename &filename)
Attempts to create a directory within the file system.
void set_basename_wo_extension(const string &s)
Replaces the basename part of the filename, without the file extension.
The abstract base class for a file or directory within the VirtualFileSystem.
string get_extension() const
Returns the file extension.
static Thread * get_current_thread()
Returns a pointer to the currently-executing Thread object.
bool open(const FileReference *file)
Opens the indicated filename for writing.
Type get_type() const
Returns the type of the file represented by the filename, as previously set by set_type().
static Filename temporary(const string &dirname, const string &prefix, const string &suffix=string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix...
This represents the in-memory index that records the list of files stored in the BamCache.
void set_file_texture_mode(BamTextureMode file_texture_mode)
Changes the BamTextureMode preference for the Bam file currently being written.
bool atomic_read_contents(const Filename &filename, string &contents) const
See Filename::atomic_read_contents().
bool store(BamCacheRecord *record)
Flushes a cache entry to disk.
void set_root(const Filename &root)
Changes the current root pathname of the cache.
A list of VirtualFiles, as returned by VirtualFile::scan_directory().
bool exists(const Filename &filename) const
Convenience function; returns true if the named file exists.
bool init()
Initializes the BamReader prior to reading any objects from its source.
bool is_directory(const Filename &filename) const
Convenience function; returns true if the named file exists and is a directory.
The name of a file, such as a texture file or an Egg file.
void output_hex(ostream &out) const
Outputs the HashVal as a 32-digit hexadecimal number.
An instance of this class is written to the front of a Bam or Txo file to make the file a cached inst...
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
virtual bool delete_file()
Attempts to delete this file or directory.
bool write_object(const TypedWritable *obj)
Writes a single object to the Bam file, so that the BamReader::read_object() can later correctly rest...
Similar to MutexHolder, but for a reentrant mutex.
string get_basename_wo_extension() const
Returns the basename part of the filename, without the file extension.
bool is_local() const
Returns true if the filename is local, e.g.
bool make_relative_to(Filename directory, bool allow_backups=true)
Adjusts this filename, which must be a fully-specified pathname beginning with a slash, to make it a relative filename, relative to the fully-specified directory indicated (which must also begin with, and may or may not end with, a slash–a terminating slash is ignored).
static TypeRegistry * ptr()
Returns the pointer to the global TypeRegistry object.
This class can be used to write a binary file that consists of an arbitrary header followed by a numb...
void make_absolute()
Converts the filename to a fully-qualified pathname from the root (if it is a relative pathname)...
A base class for all things that want to be reference-counted.
void clear_dependent_files()
Empties the list of files that contribute to the data in this record.
The TypeRegistry class maintains all the assigned TypeHandles in a given system.
A thread; that is, a lightweight process.
string get_unique_id() const
Returns a string that is guaranteed to be unique to this thread, across all processes on the machine...
This is a convenience class to specialize ConfigVariable as an integer type.
bool is_of_type(TypeHandle handle) const
Returns true if the current object is or derives from the indicated type.
Filename get_cwd() const
Returns the current directory name.
bool dependents_unchanged() const
Returns true if all of the dependent files are still the same as when the cache was recorded...
bool delete_file(const Filename &filename)
Attempts to delete the indicated file or directory.
TypeHandle is the identifier used to differentiate C++ class types.
void close()
Closes the file.
bool init()
Initializes the BamWriter prior to writing any objects to its output stream.
void consider_flush_index()
Flushes the index if enough time has elapsed since the index was last flushed.
bool exists() const
Returns true if the filename exists on the disk, false otherwise.