Panda3D
 All Classes Functions Variables Enumerations
bamCache.cxx
00001 // Filename: bamCache.cxx
00002 // Created by:  drose (09Jun06)
00003 //
00004 ////////////////////////////////////////////////////////////////////
00005 //
00006 // PANDA 3D SOFTWARE
00007 // Copyright (c) Carnegie Mellon University.  All rights reserved.
00008 //
00009 // All use of this software is subject to the terms of the revised BSD
00010 // license.  You should have received a copy of this license along
00011 // with this source code in a file named "LICENSE."
00012 //
00013 ////////////////////////////////////////////////////////////////////
00014 
00015 #include "bamCache.h"
00016 #include "bamCacheIndex.h"
00017 #include "hashVal.h"
00018 #include "datagramInputFile.h"
00019 #include "datagramOutputFile.h"
00020 #include "config_util.h"
00021 #include "bam.h"
00022 #include "typeRegistry.h"
00023 #include "string_utils.h"
00024 #include "configVariableInt.h"
00025 #include "configVariableString.h"
00026 #include "configVariableFilename.h"
00027 #include "virtualFileSystem.h"
00028 
00029 BamCache *BamCache::_global_ptr = NULL;
00030 
00031 ////////////////////////////////////////////////////////////////////
00032 //     Function: BamCache::Constructor
00033 //       Access: Published
00034 //  Description:
00035 ////////////////////////////////////////////////////////////////////
00036 BamCache::
00037 BamCache() :
00038   _active(true),
00039   _read_only(false),
00040   _index(new BamCacheIndex),
00041   _index_stale_since(0)
00042 {
00043   ConfigVariableFilename model_cache_dir
00044     ("model-cache-dir", Filename(), 
00045      PRC_DESC("The full path to a directory, local to this computer, in which "
00046               "model and texture files will be cached on load.  If a directory "
00047               "name is specified here, files may be loaded from the cache "
00048               "instead of from their actual pathnames, which may save load time, "
00049               "especially if you are loading egg files instead of bam files, "
00050               "or if you are loading models from a shared network drive.  "
00051               "If this is the empty string, no cache will be used."));
00052   
00053   ConfigVariableInt model_cache_flush
00054     ("model-cache-flush", 30,
00055      PRC_DESC("This is the amount of time, in seconds, between automatic "
00056               "flushes of the model-cache index."));
00057 
00058   ConfigVariableBool model_cache_models
00059     ("model-cache-models", true,
00060      PRC_DESC("If this is set to true, models will be cached in the "
00061               "model cache, as bam files."));
00062 
00063   ConfigVariableBool model_cache_textures
00064     ("model-cache-textures", true,
00065      PRC_DESC("If this is set to true, textures will also be cached in the "
00066               "model cache, as txo files."));
00067 
00068   ConfigVariableBool model_cache_compressed_textures
00069     ("model-cache-compressed-textures", false,
00070      PRC_DESC("If this is set to true, compressed textures will be cached "
00071               "in the model cache, in their compressed form as downloaded "
00072               "by the GSG.  This may be set in conjunction with "
00073               "model-cache-textures, or it may be independent."));
00074 
00075   ConfigVariableInt model_cache_max_kbytes
00076     ("model-cache-max-kbytes", 1048576,
00077      PRC_DESC("This is the maximum size of the model cache, in kilobytes."));
00078 
00079   _cache_models = model_cache_models;
00080   _cache_textures = model_cache_textures;
00081   _cache_compressed_textures = model_cache_compressed_textures;
00082 
00083   _flush_time = model_cache_flush;
00084   _max_kbytes = model_cache_max_kbytes;
00085 
00086   if (!model_cache_dir.empty()) {
00087     set_root(model_cache_dir);
00088   }
00089 }
00090 
00091 ////////////////////////////////////////////////////////////////////
00092 //     Function: BamCache::Destructor
00093 //       Access: Published
00094 //  Description:
00095 ////////////////////////////////////////////////////////////////////
00096 BamCache::
00097 ~BamCache() {
00098   flush_index();
00099   delete _index;
00100   _index = NULL;
00101 }
00102 
00103 ////////////////////////////////////////////////////////////////////
00104 //     Function: BamCache::set_root
00105 //       Access: Published
00106 //  Description: Changes the current root pathname of the cache.  This
00107 //               specifies where the cache files are stored on disk.
00108 //               This should name a directory that is on a disk local
00109 //               to the machine (not on a network-mounted disk), for
00110 //               instance, /tmp/panda-cache or /c/panda-cache.
00111 //
00112 //               If the directory does not already exist, it will be
00113 //               created as a result of this call.
00114 ////////////////////////////////////////////////////////////////////
00115 void BamCache::
00116 set_root(const Filename &root) {
00117   ReMutexHolder holder(_lock);
00118   flush_index();
00119   _root = root;
00120 
00121   // The root filename must be a directory.
00122   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00123   if (!vfs->is_directory(_root)) {
00124     vfs->make_directory_full(_root);
00125   }
00126 
00127   delete _index;
00128   _index = new BamCacheIndex;
00129   _index_stale_since = 0;
00130   read_index();
00131   check_cache_size();
00132 
00133   nassertv(vfs->is_directory(_root));
00134 }
00135 
00136 ////////////////////////////////////////////////////////////////////
00137 //     Function: BamCache::lookup
00138 //       Access: Published
00139 //  Description: Looks up a file in the cache.  
00140 //
00141 //               If the file is cacheable, then regardless of whether
00142 //               the file is found in the cache or not, this returns a
00143 //               BamCacheRecord.  On the other hand, if the file
00144 //               cannot be cached, returns NULL.
00145 //
00146 //               If record->has_data() returns true, then the file was
00147 //               found in the cache, and you may call
00148 //               record->extract_data() to get the object.  If
00149 //               record->has_data() returns false, then the file was
00150 //               not found in the cache or the cache was stale; and
00151 //               you should reload the source file (calling
00152 //               record->add_dependent_file() for each file loaded,
00153 //               including the original source file), and then call
00154 //               record->set_data() to record the resulting loaded
00155 //               object; and finally, you should call store() to write
00156 //               the cached record to disk.
00157 ////////////////////////////////////////////////////////////////////
00158 PT(BamCacheRecord) BamCache::
00159 lookup(const Filename &source_filename, const string &cache_extension) {
00160   ReMutexHolder holder(_lock);
00161   consider_flush_index();
00162 
00163   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00164   
00165   Filename source_pathname(source_filename);
00166   source_pathname.make_absolute(vfs->get_cwd());
00167 
00168   Filename rel_pathname(source_pathname);
00169   rel_pathname.make_relative_to(_root, false);
00170   if (rel_pathname.is_local()) {
00171     // If the source pathname is already within the cache directory,
00172     // don't cache it further.
00173     return NULL;
00174   }
00175 
00176   Filename cache_filename = hash_filename(source_pathname.get_fullpath());
00177   cache_filename.set_extension(cache_extension);
00178 
00179   return find_and_read_record(source_pathname, cache_filename);
00180 }
00181 
00182 ////////////////////////////////////////////////////////////////////
00183 //     Function: BamCache::store
00184 //       Access: Published
00185 //  Description: Flushes a cache entry to disk.  You must have
00186 //               retrieved the cache record via a prior call to
00187 //               lookup(), and then stored the data via
00188 //               record->set_data().  Returns true on success, false
00189 //               on failure.
00190 ////////////////////////////////////////////////////////////////////
00191 bool BamCache::
00192 store(BamCacheRecord *record) {
00193   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00194   ReMutexHolder holder(_lock);
00195   nassertr(!record->_cache_pathname.empty(), false);
00196   nassertr(record->has_data(), false);
00197 
00198   if (_read_only) {
00199     return false;
00200   }
00201   
00202   consider_flush_index();
00203 
00204 #ifndef NDEBUG
00205   // Ensure that the cache_pathname is within the _root directory tree.
00206   Filename rel_pathname(record->_cache_pathname);
00207   rel_pathname.make_relative_to(_root, false);
00208   nassertr(rel_pathname.is_local(), false);
00209 #endif  // NDEBUG
00210 
00211   record->_recorded_time = time(NULL);
00212 
00213   Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
00214 
00215   // We actually do the write to a temporary filename first, and then
00216   // move it into place, so that no one attempts to read the file
00217   // while it is in the process of being written.
00218   Thread *current_thread = Thread::get_current_thread();
00219   string extension = current_thread->get_unique_id() + string(".tmp");
00220   Filename temp_pathname = cache_pathname;
00221   temp_pathname.set_extension(extension);
00222   temp_pathname.set_binary();
00223 
00224   DatagramOutputFile dout;
00225   if (!dout.open(temp_pathname)) {
00226     util_cat.error()
00227       << "Could not write cache file: " << temp_pathname << "\n";
00228     vfs->delete_file(temp_pathname);
00229     emergency_read_only();
00230     return false;
00231   }
00232   
00233   if (!dout.write_header(_bam_header)) {
00234     util_cat.error()
00235       << "Unable to write to " << temp_pathname << "\n";
00236     vfs->delete_file(temp_pathname);
00237     return false;
00238   }
00239 
00240   {
00241     BamWriter writer(&dout);
00242     if (!writer.init()) {
00243       vfs->delete_file(temp_pathname);
00244       return false;
00245     }
00246     
00247     TypeRegistry *type_registry = TypeRegistry::ptr();
00248     TypeHandle texture_type = type_registry->find_type("Texture");
00249     if (record->get_data()->is_of_type(texture_type)) {
00250       // Texture objects write the actual texture image.
00251       writer.set_file_texture_mode(BamWriter::BTM_rawdata);
00252     } else {
00253       // Any other kinds of objects write texture references.
00254       writer.set_file_texture_mode(BamWriter::BTM_fullpath);
00255     }
00256     
00257     if (!writer.write_object(record)) {
00258       vfs->delete_file(temp_pathname);
00259       return false;
00260     }
00261     
00262     if (!writer.write_object(record->get_data())) {
00263       vfs->delete_file(temp_pathname);
00264       return false;
00265     }
00266 
00267     // Now that we are done with the BamWriter, it's important to let
00268     // it destruct now and clean itself up, or it might get mad if we
00269     // delete any TypedWritables below that haven't been written yet.
00270   }
00271 
00272   record->_record_size = dout.get_file_pos();
00273   dout.close();
00274 
00275   // Now move the file into place.
00276   if (!vfs->rename_file(temp_pathname, cache_pathname) && vfs->exists(temp_pathname)) {
00277     vfs->delete_file(cache_pathname);
00278     if (!vfs->rename_file(temp_pathname, cache_pathname)) {
00279       util_cat.error()
00280         << "Unable to rename " << temp_pathname << " to " 
00281         << cache_pathname << "\n";
00282       vfs->delete_file(temp_pathname);
00283       return false;
00284     }
00285   }
00286 
00287   add_to_index(record);
00288 
00289   return true;
00290 }
00291 
00292 ////////////////////////////////////////////////////////////////////
00293 //     Function: BamCache::emergency_read_only
00294 //       Access: Private
00295 //  Description: Called when an attempt to write to the cache dir
00296 //               has failed, usually for lack of disk space or 
00297 //               because of incorrect file permissions.  Outputs
00298 //               an error and puts the BamCache into read-only
00299 //               mode.
00300 ////////////////////////////////////////////////////////////////////
00301 void BamCache::
00302 emergency_read_only() {
00303   util_cat.error() <<
00304     "Could not write to the Bam Cache.  Disabling future attempts.\n";
00305   _read_only = true;
00306 }
00307 
00308 ////////////////////////////////////////////////////////////////////
00309 //     Function: BamCache::consider_flush_index
00310 //       Access: Published
00311 //  Description: Flushes the index if enough time has elapsed since
00312 //               the index was last flushed.
00313 ////////////////////////////////////////////////////////////////////
00314 void BamCache::
00315 consider_flush_index() {
00316   ReMutexHolder holder(_lock);
00317   if (_index_stale_since != 0) {
00318     int elapsed = (int)time(NULL) - (int)_index_stale_since;
00319     if (elapsed > _flush_time) {
00320       flush_index();
00321     }
00322   }
00323 }
00324 
00325 ////////////////////////////////////////////////////////////////////
00326 //     Function: BamCache::flush_index
00327 //       Access: Published
00328 //  Description: Ensures the index is written to disk.
00329 ////////////////////////////////////////////////////////////////////
00330 void BamCache::
00331 flush_index() {
00332   ReMutexHolder holder(_lock);
00333   if (_index_stale_since == 0) {
00334     // Never mind.
00335     return;
00336   }
00337 
00338   while (true) {
00339     if (_read_only) {
00340       return;
00341     }
00342 
00343     Filename temp_pathname = Filename::temporary(_root, "index-", ".boo");
00344 
00345     if (!do_write_index(temp_pathname, _index)) {
00346       emergency_read_only();
00347       return;
00348     }
00349     
00350     // Now atomically write the name of this index file to the index
00351     // reference file.
00352     VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00353     Filename index_ref_pathname(_root, Filename("index_name.txt"));
00354     string old_index = _index_ref_contents;
00355     string new_index = temp_pathname.get_basename() + "\n";
00356     string orig_index;
00357 
00358     if (vfs->atomic_compare_and_exchange_contents(index_ref_pathname, orig_index, old_index, new_index)) {
00359       // We successfully wrote our version of the index, and no other
00360       // process beat us to it.  Our index is now the official one.
00361       // Remove the old index.
00362       vfs->delete_file(_index_pathname);
00363       _index_pathname = temp_pathname;
00364       _index_ref_contents = new_index;
00365       _index_stale_since = 0;
00366       return;
00367     }
00368 
00369     // Shoot, some other process updated the index while we were
00370     // trying to update it, and they beat us to it.  We have to merge,
00371     // and try again.
00372     vfs->delete_file(temp_pathname);
00373     _index_pathname = Filename(_root, Filename(trim(orig_index)));
00374     _index_ref_contents = orig_index;
00375     read_index();
00376   }
00377   check_cache_size();
00378 }
00379 
00380 ////////////////////////////////////////////////////////////////////
00381 //     Function: BamCache::read_index
00382 //       Access: Private
00383 //  Description: Reads, or re-reads the index file from disk.  If
00384 //               _index_stale_since is nonzero, the index file is read
00385 //               and then merged with our current index.
00386 ////////////////////////////////////////////////////////////////////
00387 void BamCache::
00388 read_index() {
00389   if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
00390     // Couldn't read the index ref; rebuild the index.
00391     rebuild_index();
00392     return;
00393   }
00394 
00395   while (true) {
00396     BamCacheIndex *new_index = do_read_index(_index_pathname);
00397     if (new_index != (BamCacheIndex *)NULL) {
00398       merge_index(new_index);
00399       return;
00400     }
00401 
00402     // We couldn't read the index.  Maybe it's been removed already.
00403     // See if the index_pathname has changed.
00404     Filename old_index_pathname = _index_pathname;
00405     if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
00406       // Couldn't read the index ref; rebuild the index.
00407       rebuild_index();
00408       return;
00409     }
00410 
00411     if (old_index_pathname == _index_pathname) {
00412       // Nope, we just couldn't read it.  Delete it and build a new
00413       // one.
00414       VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00415       vfs->delete_file(_index_pathname);
00416       rebuild_index();
00417       flush_index();
00418       return;
00419     }
00420   }
00421 }
00422 
00423 ////////////////////////////////////////////////////////////////////
00424 //     Function: BamCache::read_index_pathname
00425 //       Access: Private
00426 //  Description: Atomically reads the current index filename from the
00427 //               index reference file.  The index filename moves
00428 //               around as different processes update the index.
00429 ////////////////////////////////////////////////////////////////////
00430 bool BamCache::
00431 read_index_pathname(Filename &index_pathname, string &index_ref_contents) const {
00432   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00433   index_ref_contents.clear();
00434   Filename index_ref_pathname(_root, Filename("index_name.txt"));
00435   if (!vfs->atomic_read_contents(index_ref_pathname, index_ref_contents)) {
00436     return false;
00437   }
00438 
00439   string trimmed = trim(index_ref_contents);
00440   if (trimmed.empty()) {
00441     index_pathname = Filename();
00442   } else {
00443     index_pathname = Filename(_root, Filename(trimmed));
00444   }
00445   return true;
00446 }
00447 
00448 ////////////////////////////////////////////////////////////////////
00449 //     Function: BamCache::merge_index
00450 //       Access: Private
00451 //  Description: The supplied index file has been updated by some other
00452 //               process.  Merge it with our current index.
00453 //
00454 //               Ownership of the pointer is transferred with this
00455 //               call.  The caller should assume that new_index will
00456 //               be deleted by this method.
00457 ////////////////////////////////////////////////////////////////////
00458 void BamCache::
00459 merge_index(BamCacheIndex *new_index) {
00460   if (_index_stale_since == 0) {
00461     // If our index isn't stale, just replace it.
00462     delete _index;
00463     _index = new_index;
00464     return;
00465   }
00466 
00467   BamCacheIndex *old_index = _index;
00468   old_index->release_records();
00469   new_index->release_records();
00470   _index = new BamCacheIndex;
00471 
00472   BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
00473   BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
00474   
00475   while (ai != old_index->_records.end() && 
00476          bi != new_index->_records.end()) {
00477     if ((*ai).first < (*bi).first) {
00478       // Here is an entry we have in our index, not present in the new
00479       // index.
00480       PT(BamCacheRecord) record = (*ai).second;
00481       Filename cache_pathname(_root, record->get_cache_filename());
00482       if (cache_pathname.exists()) {
00483         // The file exists; keep it.
00484         _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
00485       }
00486       ++ai;
00487 
00488     } else if ((*bi).first < (*ai).first) {
00489       // Here is an entry in the new index, not present in our index.
00490       PT(BamCacheRecord) record = (*bi).second;
00491       Filename cache_pathname(_root, record->get_cache_filename());
00492       if (cache_pathname.exists()) {
00493         // The file exists; keep it.
00494         _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
00495       }
00496       ++bi;
00497 
00498     } else {
00499       // Here is an entry we have in both.
00500       PT(BamCacheRecord) a_record = (*ai).second;
00501       PT(BamCacheRecord) b_record = (*bi).second;
00502       if (*a_record == *b_record) {
00503         // They're the same entry.  It doesn't really matter which one
00504         // we keep.
00505         _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
00506 
00507       } else {
00508         // They're different.  Just throw them both away, and re-read
00509         // the current data from the cache file.
00510 
00511         Filename cache_pathname(_root, a_record->get_cache_filename());
00512 
00513         if (cache_pathname.exists()) {
00514           PT(BamCacheRecord) record = do_read_record(cache_pathname, false);
00515           if (record != (BamCacheRecord *)NULL) {
00516             _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
00517           }
00518         }
00519       }
00520 
00521       ++ai;
00522       ++bi;
00523     }
00524   }
00525 
00526   while (ai != old_index->_records.end()) {
00527     // Here is an entry we have in our index, not present in the new
00528     // index.
00529     PT(BamCacheRecord) record = (*ai).second;
00530     Filename cache_pathname(_root, record->get_cache_filename());
00531     if (cache_pathname.exists()) {
00532       // The file exists; keep it.
00533       _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
00534     }
00535     ++ai;
00536   }
00537    
00538   while (bi != new_index->_records.end()) {
00539     // Here is an entry in the new index, not present in our index.
00540     PT(BamCacheRecord) record = (*bi).second;
00541     Filename cache_pathname(_root, record->get_cache_filename());
00542     if (cache_pathname.exists()) {
00543       // The file exists; keep it.
00544       _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
00545     }
00546     ++bi;
00547   }
00548 
00549   _index->process_new_records();
00550 }
00551 
00552 ////////////////////////////////////////////////////////////////////
00553 //     Function: BamCache::rebuild_index
00554 //       Access: Private
00555 //  Description: Regenerates the index from scratch by scanning the
00556 //               directory.
00557 ////////////////////////////////////////////////////////////////////
00558 void BamCache::
00559 rebuild_index() {
00560   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00561 
00562   PT(VirtualFileList) contents = vfs->scan_directory(_root);
00563   if (contents == NULL) {
00564     util_cat.error()
00565       << "Unable to read directory " << _root << ", caching disabled.\n";
00566     set_active(false);
00567     return;
00568   }
00569 
00570   delete _index;
00571   _index = new BamCacheIndex;
00572 
00573   int num_files = contents->get_num_files();
00574   for (int ci = 0; ci < num_files; ++ci) {
00575     VirtualFile *file = contents->get_file(ci);
00576     Filename filename = file->get_filename();
00577     if (filename.get_extension() == "bam" ||
00578         filename.get_extension() == "txo") {
00579       Filename pathname(_root, filename);
00580 
00581       PT(BamCacheRecord) record = do_read_record(pathname, false);
00582       if (record == (BamCacheRecord *)NULL) {
00583         // Well, it was invalid, so blow it away.
00584         file->delete_file();
00585 
00586       } else {
00587         record->_record_access_time = record->_recorded_time;
00588 
00589         bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second;
00590         if (!inserted) {
00591           util_cat.info()
00592             << "Multiple cache files defining " << record->get_source_pathname() << "\n";
00593           file->delete_file();
00594         }
00595       }
00596     }
00597   }
00598   _index->process_new_records();
00599 
00600   _index_stale_since = time(NULL);
00601   check_cache_size();
00602   flush_index();
00603 }
00604 
00605 ////////////////////////////////////////////////////////////////////
00606 //     Function: BamCache::add_to_index
00607 //       Access: Private
00608 //  Description: Updates the index entry for the indicated record.
00609 //               Note that a copy of the record is made first.
00610 ////////////////////////////////////////////////////////////////////
00611 void BamCache::
00612 add_to_index(const BamCacheRecord *record) {
00613   PT(BamCacheRecord) new_record = record->make_copy();
00614 
00615   if (_index->add_record(new_record)) {
00616     mark_index_stale();
00617     check_cache_size();
00618   }
00619 }
00620 
00621 ////////////////////////////////////////////////////////////////////
00622 //     Function: BamCache::remove_from_index
00623 //       Access: Private
00624 //  Description: Removes the index entry for the indicated record, if
00625 //               there is one.
00626 ////////////////////////////////////////////////////////////////////
00627 void BamCache::
00628 remove_from_index(const Filename &source_pathname) {
00629   if (_index->remove_record(source_pathname)) {
00630     mark_index_stale();
00631   }
00632 }
00633 
00634 ////////////////////////////////////////////////////////////////////
00635 //     Function: BamCache::check_cache_size
00636 //       Access: Private
00637 //  Description: If the cache size has exceeded its specified size
00638 //               limit, removes an old file.
00639 ////////////////////////////////////////////////////////////////////
00640 void BamCache::
00641 check_cache_size() {
00642   if (_index->_cache_size == 0) {
00643     // 0 means no limit.
00644     return;
00645   }
00646 
00647   if (_index->_cache_size / 1024 > _max_kbytes) {
00648     while (_index->_cache_size / 1024 > _max_kbytes) {
00649       PT(BamCacheRecord) record = _index->evict_old_file();
00650       if (record == NULL) {
00651         // Never mind; the cache is empty.
00652         break;
00653       }
00654       VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00655       Filename cache_pathname(_root, record->get_cache_filename());
00656       vfs->delete_file(cache_pathname);
00657     }
00658     mark_index_stale();
00659   }
00660 }
00661 
00662 ////////////////////////////////////////////////////////////////////
00663 //     Function: BamCache::do_read_index
00664 //       Access: Private, Static
00665 //  Description: Reads the index data from the specified filename.
00666 //               Returns a newly-allocated BamCacheIndex object on
00667 //               success, or NULL on failure.
00668 ////////////////////////////////////////////////////////////////////
00669 BamCacheIndex *BamCache::
00670 do_read_index(const Filename &index_pathname) {
00671   if (index_pathname.empty()) {
00672     return NULL;
00673   }
00674 
00675   DatagramInputFile din;
00676   if (!din.open(index_pathname)) {
00677     util_cat.debug()
00678       << "Could not read index file: " << index_pathname << "\n";
00679     return NULL;
00680   }
00681   
00682   string head;
00683   if (!din.read_header(head, _bam_header.size())) {
00684     util_cat.debug()
00685       << index_pathname << " is not an index file.\n";
00686     return NULL;
00687   }
00688   
00689   if (head != _bam_header) {
00690     util_cat.debug()
00691       << index_pathname << " is not an index file.\n";
00692     return NULL;
00693   }
00694   
00695   BamReader reader(&din);
00696   if (!reader.init()) {
00697     return NULL;
00698   }
00699 
00700   TypedWritable *object = reader.read_object();
00701 
00702   if (object == (TypedWritable *)NULL) {
00703     util_cat.error()
00704       << "Cache index " << index_pathname << " is empty.\n";
00705     return NULL;
00706 
00707   } else if (!object->is_of_type(BamCacheIndex::get_class_type())) {
00708     util_cat.error()
00709       << "Cache index " << index_pathname << " contains a "
00710       << object->get_type() << ", not a BamCacheIndex.\n";
00711     return NULL;
00712   }
00713 
00714   BamCacheIndex *index = DCAST(BamCacheIndex, object);
00715   if (!reader.resolve()) {
00716     util_cat.error()
00717       << "Unable to fully resolve cache index file.\n";
00718     return NULL;
00719   }
00720 
00721   return index;
00722 }
00723 
00724 ////////////////////////////////////////////////////////////////////
00725 //     Function: BamCache::do_write_index
00726 //       Access: Private, Static
00727 //  Description: Writes the given index data to the specified filename.
00728 ////////////////////////////////////////////////////////////////////
00729 bool BamCache::
00730 do_write_index(const Filename &index_pathname, const BamCacheIndex *index) {
00731   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00732   DatagramOutputFile dout;
00733   if (!dout.open(index_pathname)) {
00734     util_cat.error()
00735       << "Could not write index file: " << index_pathname << "\n";
00736     vfs->delete_file(index_pathname);
00737     return false;
00738   }
00739 
00740   if (!dout.write_header(_bam_header)) {
00741     util_cat.error()
00742       << "Unable to write to " << index_pathname << "\n";
00743     vfs->delete_file(index_pathname);
00744     return false;
00745   }
00746 
00747   {
00748     BamWriter writer(&dout);
00749     if (!writer.init()) {
00750       vfs->delete_file(index_pathname);
00751       return false;
00752     }
00753     
00754     if (!writer.write_object(index)) {
00755       vfs->delete_file(index_pathname);
00756       return false;
00757     }
00758   }
00759 
00760   return true;
00761 }
00762 
00763 ////////////////////////////////////////////////////////////////////
00764 //     Function: BamCache::find_and_read_record
00765 //       Access: Private
00766 //  Description: Looks for the existing cache file that corresponds
00767 //               to the indicated filename.  Normally, this is the
00768 //               specified cache filename exactly; but in the case of
00769 //               a hash collision, it may be a variant of the cache
00770 //               filename.
00771 ////////////////////////////////////////////////////////////////////
00772 PT(BamCacheRecord) BamCache::
00773 find_and_read_record(const Filename &source_pathname, 
00774                      const Filename &cache_filename) {
00775   int pass = 0;
00776   while (true) {
00777     PT(BamCacheRecord) record = 
00778       read_record(source_pathname, cache_filename, pass);
00779     if (record != (BamCacheRecord *)NULL) {
00780       add_to_index(record);
00781       return record;
00782     }
00783     ++pass;
00784   }
00785 }
00786 
00787 ////////////////////////////////////////////////////////////////////
00788 //     Function: BamCache::read_record
00789 //       Access: Private
00790 //  Description: Reads the indicated cache file and returns its
00791 //               associated record if it can be read and it matches
00792 //               the source filename.
00793 ////////////////////////////////////////////////////////////////////
00794 PT(BamCacheRecord) BamCache::
00795 read_record(const Filename &source_pathname, 
00796             const Filename &cache_filename,
00797             int pass) {
00798   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00799   Filename cache_pathname(_root, cache_filename);
00800   if (pass != 0) {
00801     ostringstream strm;
00802     strm << cache_pathname.get_basename_wo_extension() << "_" << pass;
00803     cache_pathname.set_basename_wo_extension(strm.str());
00804   }
00805   
00806   if (!cache_pathname.exists()) {
00807     // There is no such cache file already.  Declare it.
00808     if (util_cat.is_debug()) {
00809       util_cat.debug()
00810         << "Declaring new cache file " << cache_pathname << " for " << source_pathname << "\n";
00811     }
00812     PT(BamCacheRecord) record =
00813       new BamCacheRecord(source_pathname, cache_filename);
00814     record->_cache_pathname = cache_pathname;
00815     return record;
00816   }
00817 
00818   if (util_cat.is_debug()) {
00819     util_cat.debug()
00820       << "Reading cache file " << cache_pathname << " for " << source_pathname << "\n";
00821   }
00822 
00823   PT(BamCacheRecord) record = do_read_record(cache_pathname, true);
00824   if (record == (BamCacheRecord *)NULL) {
00825     // Well, it was invalid, so blow it away, and make a new one.
00826     if (util_cat.is_debug()) {
00827       util_cat.debug()
00828         << "Deleting invalid cache file " << cache_pathname << "\n";
00829     }
00830     vfs->delete_file(cache_pathname);
00831     remove_from_index(source_pathname);
00832 
00833     PT(BamCacheRecord) record =
00834       new BamCacheRecord(source_pathname, cache_filename);
00835     record->_cache_pathname = cache_pathname;
00836     return record;
00837   }
00838 
00839   if (record->get_source_pathname() != source_pathname) {
00840     // This might be just a hash conflict.
00841     if (util_cat.is_debug()) {
00842       util_cat.debug()
00843         << "Cache file " << cache_pathname << " references "
00844         << record->get_source_pathname() << ", not "
00845         << source_pathname << "\n";
00846     }
00847     return NULL;
00848   }
00849 
00850   if (!record->has_data()) {
00851     // If we didn't find any data, the caller will have to reload it.
00852     record->clear_dependent_files();
00853   }
00854 
00855   record->_cache_pathname = cache_pathname;
00856   return record;
00857 }
00858 
00859 ////////////////////////////////////////////////////////////////////
00860 //     Function: BamCache::do_read_record
00861 //       Access: Private, Static
00862 //  Description: Actually reads a record from the file.
00863 ////////////////////////////////////////////////////////////////////
00864 PT(BamCacheRecord) BamCache::
00865 do_read_record(const Filename &cache_pathname, bool read_data) {
00866   DatagramInputFile din;
00867   if (!din.open(cache_pathname)) {
00868     if (util_cat.is_debug()) {
00869       util_cat.debug()
00870         << "Could not read cache file: " << cache_pathname << "\n";
00871     }
00872     return NULL;
00873   }
00874   
00875   string head;
00876   if (!din.read_header(head, _bam_header.size())) {
00877     if (util_cat.is_debug()) {
00878       util_cat.debug()
00879         << cache_pathname << " is not a cache file.\n";
00880     }
00881     return NULL;
00882   }
00883   
00884   if (head != _bam_header) {
00885     if (util_cat.is_debug()) {
00886       util_cat.debug()
00887         << cache_pathname << " is not a cache file.\n";
00888     }
00889     return NULL;
00890   }
00891   
00892   BamReader reader(&din);
00893   if (!reader.init()) {
00894     return NULL;
00895   }
00896   
00897   TypedWritable *object = reader.read_object();
00898   if (object == (TypedWritable *)NULL) {
00899     if (util_cat.is_debug()) {
00900       util_cat.debug()
00901         << cache_pathname << " is empty.\n";
00902     }
00903     return NULL;
00904     
00905   } else if (!object->is_of_type(BamCacheRecord::get_class_type())) {
00906     if (util_cat.is_debug()) {
00907       util_cat.debug()
00908         << "Cache file " << cache_pathname << " contains a "
00909         << object->get_type() << ", not a BamCacheRecord.\n";
00910     }
00911     return NULL;
00912   }
00913   
00914   PT(BamCacheRecord) record = DCAST(BamCacheRecord, object);
00915   if (!reader.resolve()) {
00916     if (util_cat.is_debug()) {
00917       util_cat.debug()
00918         << "Unable to fully resolve cache record in " << cache_pathname << "\n";
00919     }
00920     return NULL;
00921   }
00922 
00923   // From this point below, we have validated that the selected
00924   // filename is indeed a cache record for the indicated source file,
00925   // and therefore the cache record will be returned.
00926 
00927   // We still need to decide whether the cache record is stale.
00928   if (read_data && record->dependents_unchanged()) {
00929     // The cache record doesn't appear to be stale.  Load the cached
00930     // object.
00931     TypedWritable *ptr;
00932     ReferenceCount *ref_ptr;
00933 
00934     if (reader.read_object(ptr, ref_ptr)) {
00935       if (!reader.resolve()) {
00936         if (util_cat.is_debug()) {
00937           util_cat.debug()
00938             << "Unable to fully resolve cached object in " << cache_pathname << "\n";
00939         }
00940         delete object;
00941       } else {
00942         // The object is valid.  Store it in the record.
00943         record->set_data(ptr, ref_ptr);
00944       }
00945     }
00946   }
00947   
00948   // Also get the total file size.
00949   istream &in = din.get_stream();
00950   in.clear();
00951   in.seekg(0, ios::end);
00952   record->_record_size = in.tellg();
00953 
00954   // And the last access time is now, duh.
00955   record->_record_access_time = time(NULL);
00956 
00957   return record;
00958 }
00959 
00960 ////////////////////////////////////////////////////////////////////
00961 //     Function: BamCache::hash_filename
00962 //       Access: Private, Static
00963 //  Description: Returns the appropriate filename to use for a cache
00964 //               file, given the fullpath string to the source
00965 //               filename.
00966 ////////////////////////////////////////////////////////////////////
00967 string BamCache::
00968 hash_filename(const string &filename) {
00969 #ifdef HAVE_OPENSSL
00970   // With OpenSSl, use the MD5 hash of the filename.
00971   HashVal hv;
00972   hv.hash_string(filename);
00973   ostringstream strm;
00974   hv.output_hex(strm);
00975   return strm.str();
00976 
00977 #else  // HAVE_OPENSSL
00978   // Without OpenSSL, don't get fancy; just build a simple hash.
00979   unsigned int hash = 0;
00980   for (string::const_iterator si = filename.begin(); 
00981        si != filename.end(); 
00982        ++si) {
00983     hash = (hash * 9109) + (unsigned int)(*si);
00984   }
00985 
00986   ostringstream strm;
00987   strm << hex << setw(8) << setfill('0') << hash;
00988   return strm.str();
00989 
00990 #endif  // HAVE_OPENSSL
00991 }
00992 
00993 ////////////////////////////////////////////////////////////////////
00994 //     Function: BamCache::make_global
00995 //       Access: Private, Static
00996 //  Description: Constructs the global BamCache object.
00997 ////////////////////////////////////////////////////////////////////
00998 void BamCache::
00999 make_global() {
01000   _global_ptr = new BamCache;
01001 
01002   if (_global_ptr->_root.empty()) {
01003     _global_ptr->set_active(false);
01004   }
01005 }
 All Classes Functions Variables Enumerations