Panda3D

bamCache.cxx

00001 // Filename: bamCache.cxx
00002 // Created by:  drose (09Jun06)
00003 //
00004 ////////////////////////////////////////////////////////////////////
00005 //
00006 // PANDA 3D SOFTWARE
00007 // Copyright (c) Carnegie Mellon University.  All rights reserved.
00008 //
00009 // All use of this software is subject to the terms of the revised BSD
00010 // license.  You should have received a copy of this license along
00011 // with this source code in a file named "LICENSE."
00012 //
00013 ////////////////////////////////////////////////////////////////////
00014 
00015 #include "bamCache.h"
00016 #include "bamCacheIndex.h"
00017 #include "hashVal.h"
00018 #include "datagramInputFile.h"
00019 #include "datagramOutputFile.h"
00020 #include "config_util.h"
00021 #include "bam.h"
00022 #include "typeRegistry.h"
00023 #include "string_utils.h"
00024 #include "configVariableInt.h"
00025 #include "configVariableString.h"
00026 #include "configVariableFilename.h"
00027 #include "virtualFileSystem.h"
00028 
00029 BamCache *BamCache::_global_ptr = NULL;
00030 
00031 ////////////////////////////////////////////////////////////////////
00032 //     Function: BamCache::Constructor
00033 //       Access: Published
00034 //  Description:
00035 ////////////////////////////////////////////////////////////////////
00036 BamCache::
00037 BamCache() :
00038   _active(true),
00039   _read_only(false),
00040   _index(new BamCacheIndex),
00041   _index_stale_since(0)
00042 {
00043   ConfigVariableFilename model_cache_dir
00044     ("model-cache-dir", Filename(), 
00045      PRC_DESC("The full path to a directory, local to this computer, in which "
00046               "model and texture files will be cached on load.  If a directory "
00047               "name is specified here, files may be loaded from the cache "
00048               "instead of from their actual pathnames, which may save load time, "
00049               "especially if you are loading egg files instead of bam files, "
00050               "or if you are loading models from a shared network drive.  "
00051               "If this is the empty string, no cache will be used."));
00052   
00053   ConfigVariableInt model_cache_flush
00054     ("model-cache-flush", 30,
00055      PRC_DESC("This is the amount of time, in seconds, between automatic "
00056               "flushes of the model-cache index."));
00057 
00058   ConfigVariableBool model_cache_models
00059     ("model-cache-models", true,
00060      PRC_DESC("If this is set to true, models will be cached in the "
00061               "model cache, as bam files."));
00062 
00063   ConfigVariableBool model_cache_textures
00064     ("model-cache-textures", true,
00065      PRC_DESC("If this is set to true, textures will also be cached in the "
00066               "model cache, as txo files."));
00067 
00068   ConfigVariableBool model_cache_compressed_textures
00069     ("model-cache-compressed-textures", false,
00070      PRC_DESC("If this is set to true, compressed textures will be cached "
00071               "in the model cache, in their compressed form as downloaded "
00072               "by the GSG.  This may be set in conjunction with "
00073               "model-cache-textures, or it may be independent."));
00074 
00075   ConfigVariableInt model_cache_max_kbytes
00076     ("model-cache-max-kbytes", 1048576,
00077      PRC_DESC("This is the maximum size of the model cache, in kilobytes."));
00078 
00079   _cache_models = model_cache_models;
00080   _cache_textures = model_cache_textures;
00081   _cache_compressed_textures = model_cache_compressed_textures;
00082 
00083   _flush_time = model_cache_flush;
00084   _max_kbytes = model_cache_max_kbytes;
00085 
00086   if (!model_cache_dir.empty()) {
00087     set_root(model_cache_dir);
00088   }
00089 }
00090 
00091 ////////////////////////////////////////////////////////////////////
00092 //     Function: BamCache::Destructor
00093 //       Access: Published
00094 //  Description:
00095 ////////////////////////////////////////////////////////////////////
00096 BamCache::
00097 ~BamCache() {
00098   flush_index();
00099   delete _index;
00100   _index = NULL;
00101 }
00102 
00103 ////////////////////////////////////////////////////////////////////
00104 //     Function: BamCache::set_root
00105 //       Access: Published
00106 //  Description: Changes the current root pathname of the cache.  This
00107 //               specifies where the cache files are stored on disk.
00108 //               This should name a directory that is on a disk local
00109 //               to the machine (not on a network-mounted disk), for
00110 //               instance, /tmp/panda-cache or /c/panda-cache.
00111 //
00112 //               If the directory does not already exist, it will be
00113 //               created as a result of this call.
00114 ////////////////////////////////////////////////////////////////////
00115 void BamCache::
00116 set_root(const Filename &root) {
00117   ReMutexHolder holder(_lock);
00118   flush_index();
00119   _root = root;
00120 
00121   // For now, the filename must be a directory.  Maybe eventually we
00122   // will support writing caches to a Panda multifile (though maybe it
00123   // would be better to implement this kind of thing at a lower level,
00124   // via a writable VFS, in which case the specified root filename
00125   // will still be a "directory").
00126   if (!root.is_directory()) {
00127     Filename dirname(_root, Filename("."));
00128     dirname.make_dir();
00129   }
00130   nassertv(root.is_directory());
00131 
00132   delete _index;
00133   _index = new BamCacheIndex;
00134   _index_stale_since = 0;
00135   read_index();
00136   check_cache_size();
00137 }
00138 
00139 ////////////////////////////////////////////////////////////////////
00140 //     Function: BamCache::lookup
00141 //       Access: Published
00142 //  Description: Looks up a file in the cache.  
00143 //
00144 //               If the file is cacheable, then regardless of whether
00145 //               the file is found in the cache or not, this returns a
00146 //               BamCacheRecord.  On the other hand, if the file
00147 //               cannot be cached, returns NULL.
00148 //
00149 //               If record->has_data() returns true, then the file was
00150 //               found in the cache, and you may call
00151 //               record->extract_data() to get the object.  If
00152 //               record->has_data() returns false, then the file was
00153 //               not found in the cache or the cache was stale; and
00154 //               you should reload the source file (calling
00155 //               record->add_dependent_file() for each file loaded,
00156 //               including the original source file), and then call
00157 //               record->set_data() to record the resulting loaded
00158 //               object; and finally, you should call store() to write
00159 //               the cached record to disk.
00160 ////////////////////////////////////////////////////////////////////
00161 PT(BamCacheRecord) BamCache::
00162 lookup(const Filename &source_filename, const string &cache_extension) {
00163   ReMutexHolder holder(_lock);
00164   consider_flush_index();
00165 
00166   VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
00167   
00168   Filename source_pathname(source_filename);
00169   source_pathname.make_absolute(vfs->get_cwd());
00170 
00171   Filename rel_pathname(source_pathname);
00172   rel_pathname.make_relative_to(_root, false);
00173   if (rel_pathname.is_local()) {
00174     // If the source pathname is already within the cache directory,
00175     // don't cache it further.
00176     return NULL;
00177   }
00178 
00179   Filename cache_filename = hash_filename(source_pathname.get_fullpath());
00180   cache_filename.set_extension(cache_extension);
00181 
00182   return find_and_read_record(source_pathname, cache_filename);
00183 }
00184 
00185 ////////////////////////////////////////////////////////////////////
00186 //     Function: BamCache::store
00187 //       Access: Published
00188 //  Description: Flushes a cache entry to disk.  You must have
00189 //               retrieved the cache record via a prior call to
00190 //               lookup(), and then stored the data via
00191 //               record->set_data().  Returns true on success, false
00192 //               on failure.
00193 ////////////////////////////////////////////////////////////////////
00194 bool BamCache::
00195 store(BamCacheRecord *record) {
00196   ReMutexHolder holder(_lock);
00197   nassertr(!record->_cache_pathname.empty(), false);
00198   nassertr(record->has_data(), false);
00199 
00200   if (_read_only) {
00201     return false;
00202   }
00203   
00204   consider_flush_index();
00205 
00206 #ifndef NDEBUG
00207   // Ensure that the cache_pathname is within the _root directory tree.
00208   Filename rel_pathname(record->_cache_pathname);
00209   rel_pathname.make_relative_to(_root, false);
00210   nassertr(rel_pathname.is_local(), false);
00211 #endif  // NDEBUG
00212 
00213   record->_recorded_time = time(NULL);
00214 
00215   Filename cache_pathname = Filename::binary_filename(record->_cache_pathname);
00216 
00217   // We actually do the write to a temporary filename first, and then
00218   // move it into place, so that no one attempts to read the file
00219   // while it is in the process of being written.
00220   Thread *current_thread = Thread::get_current_thread();
00221   string extension = current_thread->get_unique_id() + string(".tmp");
00222   Filename temp_pathname = cache_pathname;
00223   temp_pathname.set_extension(extension);
00224   temp_pathname.set_binary();
00225 
00226   pofstream temp_file;
00227   if (!temp_pathname.open_write(temp_file)) {
00228     util_cat.error()
00229       << "Could not open cache file: " << temp_pathname << "\n";
00230     emergency_read_only();
00231     return false;
00232   }
00233 
00234   DatagramOutputFile dout;
00235   if (!dout.open(temp_file)) {
00236     util_cat.error()
00237       << "Could not write cache file: " << temp_pathname << "\n";
00238     temp_pathname.unlink();
00239     emergency_read_only();
00240     return false;
00241   }
00242   
00243   if (!dout.write_header(_bam_header)) {
00244     util_cat.error()
00245       << "Unable to write to " << temp_pathname << "\n";
00246     temp_pathname.unlink();
00247     return false;
00248   }
00249 
00250   {
00251     BamWriter writer(&dout, temp_pathname);
00252     if (!writer.init()) {
00253       temp_pathname.unlink();
00254       return false;
00255     }
00256     
00257     TypeRegistry *type_registry = TypeRegistry::ptr();
00258     TypeHandle texture_type = type_registry->find_type("Texture");
00259     if (record->get_data()->is_of_type(texture_type)) {
00260       // Texture objects write the actual texture image.
00261       writer.set_file_texture_mode(BamWriter::BTM_rawdata);
00262     } else {
00263       // Any other kinds of objects write texture references.
00264       writer.set_file_texture_mode(BamWriter::BTM_fullpath);
00265     }
00266     
00267     if (!writer.write_object(record)) {
00268       temp_pathname.unlink();
00269       return false;
00270     }
00271     
00272     if (!writer.write_object(record->get_data())) {
00273       temp_pathname.unlink();
00274       return false;
00275     }
00276 
00277     // Now that we are done with the BamWriter, it's important to let
00278     // it destruct now and clean itself up, or it might get mad if we
00279     // delete any TypedWritables below that haven't been written yet.
00280   }
00281 
00282   record->_record_size = temp_file.tellp();
00283   temp_file.close();
00284 
00285   // Now move the file into place.
00286   if (!temp_pathname.rename_to(cache_pathname) && temp_pathname.exists()) {
00287     cache_pathname.unlink();
00288     if (!temp_pathname.rename_to(cache_pathname)) {
00289       util_cat.error()
00290         << "Unable to rename " << temp_pathname << " to " 
00291         << cache_pathname << "\n";
00292       temp_pathname.unlink();
00293       return false;
00294     }
00295   }
00296 
00297   add_to_index(record);
00298 
00299   return true;
00300 }
00301 
00302 ////////////////////////////////////////////////////////////////////
00303 //     Function: BamCache::emergency_read_only
00304 //       Access: Private
00305 //  Description: Called when an attempt to write to the cache dir
00306 //               has failed, usually for lack of disk space or 
00307 //               because of incorrect file permissions.  Outputs
00308 //               an error and puts the BamCache into read-only
00309 //               mode.
00310 ////////////////////////////////////////////////////////////////////
00311 void BamCache::
00312 emergency_read_only() {
00313   util_cat.error() <<
00314     "Could not write to the Bam Cache.  Disabling future attempts.\n";
00315   _read_only = true;
00316 }
00317 
00318 ////////////////////////////////////////////////////////////////////
00319 //     Function: BamCache::consider_flush_index
00320 //       Access: Published
00321 //  Description: Flushes the index if enough time has elapsed since
00322 //               the index was last flushed.
00323 ////////////////////////////////////////////////////////////////////
00324 void BamCache::
00325 consider_flush_index() {
00326   ReMutexHolder holder(_lock);
00327   if (_index_stale_since != 0) {
00328     int elapsed = (int)time(NULL) - (int)_index_stale_since;
00329     if (elapsed > _flush_time) {
00330       flush_index();
00331     }
00332   }
00333 }
00334 
00335 ////////////////////////////////////////////////////////////////////
00336 //     Function: BamCache::flush_index
00337 //       Access: Published
00338 //  Description: Ensures the index is written to disk.
00339 ////////////////////////////////////////////////////////////////////
00340 void BamCache::
00341 flush_index() {
00342   ReMutexHolder holder(_lock);
00343   if (_index_stale_since == 0) {
00344     // Never mind.
00345     return;
00346   }
00347 
00348   while (true) {
00349     if (_read_only) {
00350       return;
00351     }
00352 
00353     Filename temp_pathname = Filename::temporary(_root, "index-", ".boo");
00354 
00355     if (!do_write_index(temp_pathname, _index)) {
00356       emergency_read_only();
00357       return;
00358     }
00359     
00360     // Now atomically write the name of this index file to the index
00361     // reference file.
00362     Filename index_ref_pathname(_root, Filename("index_name.txt"));
00363     string old_index = _index_ref_contents;
00364     string new_index = temp_pathname.get_basename() + "\n";
00365     string orig_index;
00366     if (index_ref_pathname.atomic_compare_and_exchange_contents(orig_index, old_index, new_index)) {
00367       // We successfully wrote our version of the index, and no other
00368       // process beat us to it.  Our index is now the official one.
00369       // Remove the old index.
00370       _index_pathname.unlink();
00371       _index_pathname = temp_pathname;
00372       _index_ref_contents = new_index;
00373       _index_stale_since = 0;
00374       return;
00375     }
00376 
00377     // Shoot, some other process updated the index while we were
00378     // trying to update it, and they beat us to it.  We have to merge,
00379     // and try again.
00380     temp_pathname.unlink();
00381     _index_pathname = Filename(_root, Filename(trim(orig_index)));
00382     _index_ref_contents = orig_index;
00383     read_index();
00384   }
00385   check_cache_size();
00386 }
00387 
00388 ////////////////////////////////////////////////////////////////////
00389 //     Function: BamCache::read_index
00390 //       Access: Private
00391 //  Description: Reads, or re-reads the index file from disk.  If
00392 //               _index_stale_since is nonzero, the index file is read
00393 //               and then merged with our current index.
00394 ////////////////////////////////////////////////////////////////////
00395 void BamCache::
00396 read_index() {
00397   if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
00398     // Couldn't read the index ref; rebuild the index.
00399     rebuild_index();
00400     return;
00401   }
00402 
00403   while (true) {
00404     BamCacheIndex *new_index = do_read_index(_index_pathname);
00405     if (new_index != (BamCacheIndex *)NULL) {
00406       merge_index(new_index);
00407       return;
00408     }
00409 
00410     // We couldn't read the index.  Maybe it's been removed already.
00411     // See if the index_pathname has changed.
00412     Filename old_index_pathname = _index_pathname;
00413     if (!read_index_pathname(_index_pathname, _index_ref_contents)) {
00414       // Couldn't read the index ref; rebuild the index.
00415       rebuild_index();
00416       return;
00417     }
00418 
00419     if (old_index_pathname == _index_pathname) {
00420       // Nope, we just couldn't read it.  Delete it and build a new
00421       // one.
00422       _index_pathname.unlink();
00423       rebuild_index();
00424       flush_index();
00425       return;
00426     }
00427   }
00428 }
00429 
00430 ////////////////////////////////////////////////////////////////////
00431 //     Function: BamCache::read_index_pathname
00432 //       Access: Private
00433 //  Description: Atomically reads the current index filename from the
00434 //               index reference file.  The index filename moves
00435 //               around as different processes update the index.
00436 ////////////////////////////////////////////////////////////////////
00437 bool BamCache::
00438 read_index_pathname(Filename &index_pathname, string &index_ref_contents) const {
00439   index_ref_contents.clear();
00440   Filename index_ref_pathname(_root, Filename("index_name.txt"));
00441   if (!index_ref_pathname.atomic_read_contents(index_ref_contents)) {
00442     return false;
00443   }
00444 
00445   string trimmed = trim(index_ref_contents);
00446   if (trimmed.empty()) {
00447     index_pathname = Filename();
00448   } else {
00449     index_pathname = Filename(_root, Filename(trimmed));
00450   }
00451   return true;
00452 }
00453 
00454 ////////////////////////////////////////////////////////////////////
00455 //     Function: BamCache::merge_index
00456 //       Access: Private
00457 //  Description: The supplied index file has been updated by some other
00458 //               process.  Merge it with our current index.
00459 //
00460 //               Ownership of the pointer is transferred with this
00461 //               call.  The caller should assume that new_index will
00462 //               be deleted by this method.
00463 ////////////////////////////////////////////////////////////////////
00464 void BamCache::
00465 merge_index(BamCacheIndex *new_index) {
00466   if (_index_stale_since == 0) {
00467     // If our index isn't stale, just replace it.
00468     delete _index;
00469     _index = new_index;
00470     return;
00471   }
00472 
00473   BamCacheIndex *old_index = _index;
00474   old_index->release_records();
00475   new_index->release_records();
00476   _index = new BamCacheIndex;
00477 
00478   BamCacheIndex::Records::const_iterator ai = old_index->_records.begin();
00479   BamCacheIndex::Records::const_iterator bi = new_index->_records.begin();
00480   
00481   while (ai != old_index->_records.end() && 
00482          bi != new_index->_records.end()) {
00483     if ((*ai).first < (*bi).first) {
00484       // Here is an entry we have in our index, not present in the new
00485       // index.
00486       PT(BamCacheRecord) record = (*ai).second;
00487       Filename cache_pathname(_root, record->get_cache_filename());
00488       if (cache_pathname.exists()) {
00489         // The file exists; keep it.
00490         _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
00491       }
00492       ++ai;
00493 
00494     } else if ((*bi).first < (*ai).first) {
00495       // Here is an entry in the new index, not present in our index.
00496       PT(BamCacheRecord) record = (*bi).second;
00497       Filename cache_pathname(_root, record->get_cache_filename());
00498       if (cache_pathname.exists()) {
00499         // The file exists; keep it.
00500         _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
00501       }
00502       ++bi;
00503 
00504     } else {
00505       // Here is an entry we have in both.
00506       PT(BamCacheRecord) a_record = (*ai).second;
00507       PT(BamCacheRecord) b_record = (*bi).second;
00508       if (*a_record == *b_record) {
00509         // They're the same entry.  It doesn't really matter which one
00510         // we keep.
00511         _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record));
00512 
00513       } else {
00514         // They're different.  Just throw them both away, and re-read
00515         // the current data from the cache file.
00516 
00517         Filename cache_pathname(_root, a_record->get_cache_filename());
00518 
00519         if (cache_pathname.exists()) {
00520           PT(BamCacheRecord) record = do_read_record(cache_pathname, false);
00521           if (record != (BamCacheRecord *)NULL) {
00522             _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
00523           }
00524         }
00525       }
00526 
00527       ++ai;
00528       ++bi;
00529     }
00530   }
00531 
00532   while (ai != old_index->_records.end()) {
00533     // Here is an entry we have in our index, not present in the new
00534     // index.
00535     PT(BamCacheRecord) record = (*ai).second;
00536     Filename cache_pathname(_root, record->get_cache_filename());
00537     if (cache_pathname.exists()) {
00538       // The file exists; keep it.
00539       _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
00540     }
00541     ++ai;
00542   }
00543    
00544   while (bi != new_index->_records.end()) {
00545     // Here is an entry in the new index, not present in our index.
00546     PT(BamCacheRecord) record = (*bi).second;
00547     Filename cache_pathname(_root, record->get_cache_filename());
00548     if (cache_pathname.exists()) {
00549       // The file exists; keep it.
00550       _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record));
00551     }
00552     ++bi;
00553   }
00554 
00555   _index->process_new_records();
00556 }
00557 
00558 ////////////////////////////////////////////////////////////////////
00559 //     Function: BamCache::rebuild_index
00560 //       Access: Private
00561 //  Description: Regenerates the index from scratch by scanning the
00562 //               directory.
00563 ////////////////////////////////////////////////////////////////////
00564 void BamCache::
00565 rebuild_index() {
00566   vector_string contents;
00567   if (!_root.scan_directory(contents)) {
00568     util_cat.error()
00569       << "Unable to read directory " << _root << ", caching disabled.\n";
00570     set_active(false);
00571     return;
00572   }
00573 
00574   delete _index;
00575   _index = new BamCacheIndex;
00576 
00577   vector_string::const_iterator ci;
00578   for (ci = contents.begin(); ci != contents.end(); ++ci) {
00579     Filename filename(*ci);
00580     if (filename.get_extension() == "bam" ||
00581         filename.get_extension() == "txo") {
00582       Filename pathname(_root, filename);
00583 
00584       PT(BamCacheRecord) record = do_read_record(pathname, false);
00585       if (record == (BamCacheRecord *)NULL) {
00586         // Well, it was invalid, so blow it away.
00587         pathname.unlink();
00588 
00589       } else {
00590         record->_record_access_time = record->_recorded_time;
00591 
00592         bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second;
00593         if (!inserted) {
00594           util_cat.info()
00595             << "Multiple cache files defining " << record->get_source_pathname() << "\n";
00596           pathname.unlink();
00597         }
00598       }
00599     }
00600   }
00601   _index->process_new_records();
00602 
00603   _index_stale_since = time(NULL);
00604   check_cache_size();
00605   flush_index();
00606 }
00607 
00608 ////////////////////////////////////////////////////////////////////
00609 //     Function: BamCache::add_to_index
00610 //       Access: Private
00611 //  Description: Updates the index entry for the indicated record.
00612 //               Note that a copy of the record is made first.
00613 ////////////////////////////////////////////////////////////////////
00614 void BamCache::
00615 add_to_index(const BamCacheRecord *record) {
00616   PT(BamCacheRecord) new_record = record->make_copy();
00617 
00618   if (_index->add_record(new_record)) {
00619     mark_index_stale();
00620     check_cache_size();
00621   }
00622 }
00623 
00624 ////////////////////////////////////////////////////////////////////
00625 //     Function: BamCache::remove_from_index
00626 //       Access: Private
00627 //  Description: Removes the index entry for the indicated record, if
00628 //               there is one.
00629 ////////////////////////////////////////////////////////////////////
00630 void BamCache::
00631 remove_from_index(const Filename &source_pathname) {
00632   if (_index->remove_record(source_pathname)) {
00633     mark_index_stale();
00634   }
00635 }
00636 
00637 ////////////////////////////////////////////////////////////////////
00638 //     Function: BamCache::check_cache_size
00639 //       Access: Private
00640 //  Description: If the cache size has exceeded its specified size
00641 //               limit, removes an old file.
00642 ////////////////////////////////////////////////////////////////////
00643 void BamCache::
00644 check_cache_size() {
00645   if (_index->_cache_size == 0) {
00646     // 0 means no limit.
00647     return;
00648   }
00649 
00650   if (_index->_cache_size / 1024 > _max_kbytes) {
00651     while (_index->_cache_size / 1024 > _max_kbytes) {
00652       PT(BamCacheRecord) record = _index->evict_old_file();
00653       if (record == NULL) {
00654         // Never mind; the cache is empty.
00655         break;
00656       }
00657       Filename cache_pathname(_root, record->get_cache_filename());
00658       cache_pathname.unlink();
00659     }
00660     mark_index_stale();
00661   }
00662 }
00663 
00664 ////////////////////////////////////////////////////////////////////
00665 //     Function: BamCache::do_read_index
00666 //       Access: Private, Static
00667 //  Description: Reads the index data from the specified filename.
00668 //               Returns a newly-allocated BamCacheIndex object on
00669 //               success, or NULL on failure.
00670 ////////////////////////////////////////////////////////////////////
00671 BamCacheIndex *BamCache::
00672 do_read_index(Filename &index_pathname) {
00673   if (index_pathname.empty()) {
00674     return NULL;
00675   }
00676 
00677   index_pathname.set_binary();
00678   pifstream index_file;
00679   if (!index_pathname.open_read(index_file)) {
00680     util_cat.error()
00681       << "Could not open index file: " << index_pathname << "\n";
00682     return NULL;
00683   }
00684 
00685   DatagramInputFile din;
00686     
00687   if (!din.open(index_file)) {
00688     util_cat.debug()
00689       << "Could not read index file: " << index_pathname << "\n";
00690     return NULL;
00691   }
00692   
00693   string head;
00694   if (!din.read_header(head, _bam_header.size())) {
00695     util_cat.debug()
00696       << index_pathname << " is not an index file.\n";
00697     return NULL;
00698   }
00699   
00700   if (head != _bam_header) {
00701     util_cat.debug()
00702       << index_pathname << " is not an index file.\n";
00703     return NULL;
00704   }
00705   
00706   BamReader reader(&din, index_pathname);
00707   if (!reader.init()) {
00708     return NULL;
00709   }
00710 
00711   TypedWritable *object = reader.read_object();
00712 
00713   if (object == (TypedWritable *)NULL) {
00714     util_cat.error()
00715       << "Cache index " << index_pathname << " is empty.\n";
00716     return NULL;
00717 
00718   } else if (!object->is_of_type(BamCacheIndex::get_class_type())) {
00719     util_cat.error()
00720       << "Cache index " << index_pathname << " contains a "
00721       << object->get_type() << ", not a BamCacheIndex.\n";
00722     return NULL;
00723   }
00724 
00725   BamCacheIndex *index = DCAST(BamCacheIndex, object);
00726   if (!reader.resolve()) {
00727     util_cat.error()
00728       << "Unable to fully resolve cache index file.\n";
00729     return NULL;
00730   }
00731 
00732   return index;
00733 }
00734 
00735 ////////////////////////////////////////////////////////////////////
00736 //     Function: BamCache::do_write_index
00737 //       Access: Private, Static
00738 //  Description: Writes the given index data to the specified filename.
00739 ////////////////////////////////////////////////////////////////////
00740 bool BamCache::
00741 do_write_index(Filename &index_pathname, const BamCacheIndex *index) {
00742   index_pathname.set_binary();
00743   pofstream index_file;
00744   
00745   if (!index_pathname.open_write(index_file)) {
00746     util_cat.error()
00747       << "Could not open index file: " << index_pathname << "\n";
00748     return false;
00749   }
00750   
00751   DatagramOutputFile dout;
00752   if (!dout.open(index_file)) {
00753     util_cat.error()
00754       << "Could not write index file: " << index_pathname << "\n";
00755     index_pathname.unlink();
00756     return false;
00757   }
00758 
00759   if (!dout.write_header(_bam_header)) {
00760     util_cat.error()
00761       << "Unable to write to " << index_pathname << "\n";
00762     index_pathname.unlink();
00763     return false;
00764   }
00765 
00766   {
00767     BamWriter writer(&dout, index_pathname);
00768     if (!writer.init()) {
00769       index_pathname.unlink();
00770       return false;
00771     }
00772     
00773     if (!writer.write_object(index)) {
00774       index_pathname.unlink();
00775       return false;
00776     }
00777   }
00778 
00779   index_file.close();
00780   return true;
00781 }
00782 
00783 ////////////////////////////////////////////////////////////////////
00784 //     Function: BamCache::find_and_read_record
00785 //       Access: Private
00786 //  Description: Looks for the existing cache file that corresponds
00787 //               to the indicated filename.  Normally, this is the
00788 //               specified cache filename exactly; but in the case of
00789 //               a hash collision, it may be a variant of the cache
00790 //               filename.
00791 ////////////////////////////////////////////////////////////////////
00792 PT(BamCacheRecord) BamCache::
00793 find_and_read_record(const Filename &source_pathname, 
00794                      const Filename &cache_filename) {
00795   int pass = 0;
00796   while (true) {
00797     PT(BamCacheRecord) record = 
00798       read_record(source_pathname, cache_filename, pass);
00799     if (record != (BamCacheRecord *)NULL) {
00800       add_to_index(record);
00801       return record;
00802     }
00803     ++pass;
00804   }
00805 }
00806 
00807 ////////////////////////////////////////////////////////////////////
00808 //     Function: BamCache::read_record
00809 //       Access: Private
00810 //  Description: Reads the indicated cache file and returns its
00811 //               associated record if it can be read and it matches
00812 //               the source filename.
00813 ////////////////////////////////////////////////////////////////////
00814 PT(BamCacheRecord) BamCache::
00815 read_record(const Filename &source_pathname, 
00816             const Filename &cache_filename,
00817             int pass) {
00818   Filename cache_pathname(_root, cache_filename);
00819   if (pass != 0) {
00820     ostringstream strm;
00821     strm << cache_pathname.get_basename_wo_extension() << "_" << pass;
00822     cache_pathname.set_basename_wo_extension(strm.str());
00823   }
00824   
00825   if (!cache_pathname.exists()) {
00826     // There is no such cache file already.  Declare it.
00827     PT(BamCacheRecord) record =
00828       new BamCacheRecord(source_pathname, cache_filename);
00829     record->_cache_pathname = cache_pathname;
00830     return record;
00831   }
00832 
00833   PT(BamCacheRecord) record = do_read_record(cache_pathname, true);
00834   if (record == (BamCacheRecord *)NULL) {
00835     // Well, it was invalid, so blow it away, and make a new one.
00836     cache_pathname.unlink();
00837     remove_from_index(source_pathname);
00838 
00839     PT(BamCacheRecord) record =
00840       new BamCacheRecord(source_pathname, cache_filename);
00841     record->_cache_pathname = cache_pathname;
00842     return record;
00843   }
00844 
00845   if (record->get_source_pathname() != source_pathname) {
00846     // This might be just a hash conflict.
00847     util_cat.debug()
00848       << "Cache file " << cache_pathname << " references "
00849       << record->get_source_pathname() << ", not "
00850       << source_pathname << "\n";
00851     return NULL;
00852   }
00853 
00854   if (!record->has_data()) {
00855     // If we didn't find any data, the caller will have to reload it.
00856     record->clear_dependent_files();
00857   }
00858 
00859   record->_cache_pathname = cache_pathname;
00860   return record;
00861 }
00862 
00863 ////////////////////////////////////////////////////////////////////
00864 //     Function: BamCache::do_read_record
00865 //       Access: Private, Static
00866 //  Description: Actually reads a record from the file.
00867 ////////////////////////////////////////////////////////////////////
00868 PT(BamCacheRecord) BamCache::
00869 do_read_record(Filename &cache_pathname, bool read_data) {
00870   cache_pathname.set_binary();
00871   pifstream cache_file;
00872   if (!cache_pathname.open_read(cache_file)) {
00873     util_cat.debug()
00874       << "Could not open cache file: " << cache_pathname << "\n";
00875     return NULL;
00876   }
00877 
00878   DatagramInputFile din;
00879     
00880   if (!din.open(cache_file)) {
00881     util_cat.debug()
00882       << "Could not read cache file: " << cache_pathname << "\n";
00883     return NULL;
00884   }
00885   
00886   string head;
00887   if (!din.read_header(head, _bam_header.size())) {
00888     util_cat.debug()
00889       << cache_pathname << " is not a cache file.\n";
00890     return NULL;
00891   }
00892   
00893   if (head != _bam_header) {
00894     util_cat.debug()
00895       << cache_pathname << " is not a cache file.\n";
00896     return NULL;
00897   }
00898   
00899   BamReader reader(&din, cache_pathname);
00900   if (!reader.init()) {
00901     return NULL;
00902   }
00903   
00904   TypedWritable *object = reader.read_object();
00905   if (object == (TypedWritable *)NULL) {
00906     util_cat.debug()
00907       << cache_pathname << " is empty.\n";
00908     return NULL;
00909     
00910   } else if (!object->is_of_type(BamCacheRecord::get_class_type())) {
00911     util_cat.debug()
00912       << "Cache file " << cache_pathname << " contains a "
00913       << object->get_type() << ", not a BamCacheRecord.\n";
00914     return NULL;
00915   }
00916   
00917   PT(BamCacheRecord) record = DCAST(BamCacheRecord, object);
00918   if (!reader.resolve()) {
00919     util_cat.debug()
00920       << "Unable to fully resolve cache record in " << cache_pathname << "\n";
00921     return NULL;
00922   }
00923 
00924   // From this point below, we have validated that the selected
00925   // filename is indeed a cache record for the indicated source file,
00926   // and therefore the cache record will be returned.
00927 
00928   // We still need to decide whether the cache record is stale.
00929   if (read_data && record->dependents_unchanged()) {
00930     // The cache record doesn't appear to be stale.  Load the cached
00931     // object.
00932     TypedWritable *ptr;
00933     ReferenceCount *ref_ptr;
00934 
00935     if (reader.read_object(ptr, ref_ptr)) {
00936       if (!reader.resolve()) {
00937         util_cat.debug()
00938           << "Unable to fully resolve cached object in " << cache_pathname << "\n";
00939         delete object;
00940       } else {
00941         // The object is valid.  Store it in the record.
00942         record->set_data(ptr, ref_ptr);
00943       }
00944     }
00945   }
00946   
00947   // Also get the file size.
00948   cache_file.clear();
00949   cache_file.seekg(0, ios::end);
00950   record->_record_size = cache_file.tellg();
00951 
00952   // And the last access time is now, duh.
00953   record->_record_access_time = time(NULL);
00954 
00955   return record;
00956 }
00957 
00958 ////////////////////////////////////////////////////////////////////
00959 //     Function: BamCache::hash_filename
00960 //       Access: Private, Static
00961 //  Description: Returns the appropriate filename to use for a cache
00962 //               file, given the fullpath string to the source
00963 //               filename.
00964 ////////////////////////////////////////////////////////////////////
00965 string BamCache::
00966 hash_filename(const string &filename) {
00967 #ifdef HAVE_OPENSSL
00968   // With OpenSSl, use the MD5 hash of the filename.
00969   HashVal hv;
00970   hv.hash_string(filename);
00971   ostringstream strm;
00972   hv.output_hex(strm);
00973   return strm.str();
00974 
00975 #else  // HAVE_OPENSSL
00976   // Without OpenSSL, don't get fancy; just build a simple hash.
00977   unsigned int hash = 0;
00978   for (string::const_iterator si = filename.begin(); 
00979        si != filename.end(); 
00980        ++si) {
00981     hash = (hash * 9109) + (unsigned int)(*si);
00982   }
00983 
00984   ostringstream strm;
00985   strm << hex << setw(8) << setfill('0') << hash;
00986   return strm.str();
00987 
00988 #endif  // HAVE_OPENSSL
00989 }
00990 
00991 ////////////////////////////////////////////////////////////////////
00992 //     Function: BamCache::make_global
00993 //       Access: Private, Static
00994 //  Description: Constructs the global BamCache object.
00995 ////////////////////////////////////////////////////////////////////
00996 void BamCache::
00997 make_global() {
00998   _global_ptr = new BamCache;
00999 
01000   if (_global_ptr->_root.empty()) {
01001     _global_ptr->set_active(false);
01002   }
01003 }
 All Classes Functions Variables Enumerations