Panda3D
|
00001 // Filename: bamCache.cxx 00002 // Created by: drose (09Jun06) 00003 // 00004 //////////////////////////////////////////////////////////////////// 00005 // 00006 // PANDA 3D SOFTWARE 00007 // Copyright (c) Carnegie Mellon University. All rights reserved. 00008 // 00009 // All use of this software is subject to the terms of the revised BSD 00010 // license. You should have received a copy of this license along 00011 // with this source code in a file named "LICENSE." 00012 // 00013 //////////////////////////////////////////////////////////////////// 00014 00015 #include "bamCache.h" 00016 #include "bamCacheIndex.h" 00017 #include "hashVal.h" 00018 #include "datagramInputFile.h" 00019 #include "datagramOutputFile.h" 00020 #include "config_util.h" 00021 #include "bam.h" 00022 #include "typeRegistry.h" 00023 #include "string_utils.h" 00024 #include "configVariableInt.h" 00025 #include "configVariableString.h" 00026 #include "configVariableFilename.h" 00027 #include "virtualFileSystem.h" 00028 00029 BamCache *BamCache::_global_ptr = NULL; 00030 00031 //////////////////////////////////////////////////////////////////// 00032 // Function: BamCache::Constructor 00033 // Access: Published 00034 // Description: 00035 //////////////////////////////////////////////////////////////////// 00036 BamCache:: 00037 BamCache() : 00038 _active(true), 00039 _read_only(false), 00040 _index(new BamCacheIndex), 00041 _index_stale_since(0) 00042 { 00043 ConfigVariableFilename model_cache_dir 00044 ("model-cache-dir", Filename(), 00045 PRC_DESC("The full path to a directory, local to this computer, in which " 00046 "model and texture files will be cached on load. If a directory " 00047 "name is specified here, files may be loaded from the cache " 00048 "instead of from their actual pathnames, which may save load time, " 00049 "especially if you are loading egg files instead of bam files, " 00050 "or if you are loading models from a shared network drive. " 00051 "If this is the empty string, no cache will be used.")); 00052 00053 ConfigVariableInt model_cache_flush 00054 ("model-cache-flush", 30, 00055 PRC_DESC("This is the amount of time, in seconds, between automatic " 00056 "flushes of the model-cache index.")); 00057 00058 ConfigVariableBool model_cache_models 00059 ("model-cache-models", true, 00060 PRC_DESC("If this is set to true, models will be cached in the " 00061 "model cache, as bam files.")); 00062 00063 ConfigVariableBool model_cache_textures 00064 ("model-cache-textures", true, 00065 PRC_DESC("If this is set to true, textures will also be cached in the " 00066 "model cache, as txo files.")); 00067 00068 ConfigVariableBool model_cache_compressed_textures 00069 ("model-cache-compressed-textures", false, 00070 PRC_DESC("If this is set to true, compressed textures will be cached " 00071 "in the model cache, in their compressed form as downloaded " 00072 "by the GSG. This may be set in conjunction with " 00073 "model-cache-textures, or it may be independent.")); 00074 00075 ConfigVariableInt model_cache_max_kbytes 00076 ("model-cache-max-kbytes", 1048576, 00077 PRC_DESC("This is the maximum size of the model cache, in kilobytes.")); 00078 00079 _cache_models = model_cache_models; 00080 _cache_textures = model_cache_textures; 00081 _cache_compressed_textures = model_cache_compressed_textures; 00082 00083 _flush_time = model_cache_flush; 00084 _max_kbytes = model_cache_max_kbytes; 00085 00086 if (!model_cache_dir.empty()) { 00087 set_root(model_cache_dir); 00088 } 00089 } 00090 00091 //////////////////////////////////////////////////////////////////// 00092 // Function: BamCache::Destructor 00093 // Access: Published 00094 // Description: 00095 //////////////////////////////////////////////////////////////////// 00096 BamCache:: 00097 ~BamCache() { 00098 flush_index(); 00099 delete _index; 00100 _index = NULL; 00101 } 00102 00103 //////////////////////////////////////////////////////////////////// 00104 // Function: BamCache::set_root 00105 // Access: Published 00106 // Description: Changes the current root pathname of the cache. This 00107 // specifies where the cache files are stored on disk. 00108 // This should name a directory that is on a disk local 00109 // to the machine (not on a network-mounted disk), for 00110 // instance, /tmp/panda-cache or /c/panda-cache. 00111 // 00112 // If the directory does not already exist, it will be 00113 // created as a result of this call. 00114 //////////////////////////////////////////////////////////////////// 00115 void BamCache:: 00116 set_root(const Filename &root) { 00117 ReMutexHolder holder(_lock); 00118 flush_index(); 00119 _root = root; 00120 00121 // The root filename must be a directory. 00122 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00123 if (!vfs->is_directory(_root)) { 00124 vfs->make_directory_full(_root); 00125 } 00126 00127 delete _index; 00128 _index = new BamCacheIndex; 00129 _index_stale_since = 0; 00130 read_index(); 00131 check_cache_size(); 00132 00133 nassertv(vfs->is_directory(_root)); 00134 } 00135 00136 //////////////////////////////////////////////////////////////////// 00137 // Function: BamCache::lookup 00138 // Access: Published 00139 // Description: Looks up a file in the cache. 00140 // 00141 // If the file is cacheable, then regardless of whether 00142 // the file is found in the cache or not, this returns a 00143 // BamCacheRecord. On the other hand, if the file 00144 // cannot be cached, returns NULL. 00145 // 00146 // If record->has_data() returns true, then the file was 00147 // found in the cache, and you may call 00148 // record->extract_data() to get the object. If 00149 // record->has_data() returns false, then the file was 00150 // not found in the cache or the cache was stale; and 00151 // you should reload the source file (calling 00152 // record->add_dependent_file() for each file loaded, 00153 // including the original source file), and then call 00154 // record->set_data() to record the resulting loaded 00155 // object; and finally, you should call store() to write 00156 // the cached record to disk. 00157 //////////////////////////////////////////////////////////////////// 00158 PT(BamCacheRecord) BamCache:: 00159 lookup(const Filename &source_filename, const string &cache_extension) { 00160 ReMutexHolder holder(_lock); 00161 consider_flush_index(); 00162 00163 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00164 00165 Filename source_pathname(source_filename); 00166 source_pathname.make_absolute(vfs->get_cwd()); 00167 00168 Filename rel_pathname(source_pathname); 00169 rel_pathname.make_relative_to(_root, false); 00170 if (rel_pathname.is_local()) { 00171 // If the source pathname is already within the cache directory, 00172 // don't cache it further. 00173 return NULL; 00174 } 00175 00176 Filename cache_filename = hash_filename(source_pathname.get_fullpath()); 00177 cache_filename.set_extension(cache_extension); 00178 00179 return find_and_read_record(source_pathname, cache_filename); 00180 } 00181 00182 //////////////////////////////////////////////////////////////////// 00183 // Function: BamCache::store 00184 // Access: Published 00185 // Description: Flushes a cache entry to disk. You must have 00186 // retrieved the cache record via a prior call to 00187 // lookup(), and then stored the data via 00188 // record->set_data(). Returns true on success, false 00189 // on failure. 00190 //////////////////////////////////////////////////////////////////// 00191 bool BamCache:: 00192 store(BamCacheRecord *record) { 00193 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00194 ReMutexHolder holder(_lock); 00195 nassertr(!record->_cache_pathname.empty(), false); 00196 nassertr(record->has_data(), false); 00197 00198 if (_read_only) { 00199 return false; 00200 } 00201 00202 consider_flush_index(); 00203 00204 #ifndef NDEBUG 00205 // Ensure that the cache_pathname is within the _root directory tree. 00206 Filename rel_pathname(record->_cache_pathname); 00207 rel_pathname.make_relative_to(_root, false); 00208 nassertr(rel_pathname.is_local(), false); 00209 #endif // NDEBUG 00210 00211 record->_recorded_time = time(NULL); 00212 00213 Filename cache_pathname = Filename::binary_filename(record->_cache_pathname); 00214 00215 // We actually do the write to a temporary filename first, and then 00216 // move it into place, so that no one attempts to read the file 00217 // while it is in the process of being written. 00218 Thread *current_thread = Thread::get_current_thread(); 00219 string extension = current_thread->get_unique_id() + string(".tmp"); 00220 Filename temp_pathname = cache_pathname; 00221 temp_pathname.set_extension(extension); 00222 temp_pathname.set_binary(); 00223 00224 DatagramOutputFile dout; 00225 if (!dout.open(temp_pathname)) { 00226 util_cat.error() 00227 << "Could not write cache file: " << temp_pathname << "\n"; 00228 vfs->delete_file(temp_pathname); 00229 emergency_read_only(); 00230 return false; 00231 } 00232 00233 if (!dout.write_header(_bam_header)) { 00234 util_cat.error() 00235 << "Unable to write to " << temp_pathname << "\n"; 00236 vfs->delete_file(temp_pathname); 00237 return false; 00238 } 00239 00240 { 00241 BamWriter writer(&dout); 00242 if (!writer.init()) { 00243 vfs->delete_file(temp_pathname); 00244 return false; 00245 } 00246 00247 TypeRegistry *type_registry = TypeRegistry::ptr(); 00248 TypeHandle texture_type = type_registry->find_type("Texture"); 00249 if (record->get_data()->is_of_type(texture_type)) { 00250 // Texture objects write the actual texture image. 00251 writer.set_file_texture_mode(BamWriter::BTM_rawdata); 00252 } else { 00253 // Any other kinds of objects write texture references. 00254 writer.set_file_texture_mode(BamWriter::BTM_fullpath); 00255 } 00256 00257 if (!writer.write_object(record)) { 00258 vfs->delete_file(temp_pathname); 00259 return false; 00260 } 00261 00262 if (!writer.write_object(record->get_data())) { 00263 vfs->delete_file(temp_pathname); 00264 return false; 00265 } 00266 00267 // Now that we are done with the BamWriter, it's important to let 00268 // it destruct now and clean itself up, or it might get mad if we 00269 // delete any TypedWritables below that haven't been written yet. 00270 } 00271 00272 record->_record_size = dout.get_file_pos(); 00273 dout.close(); 00274 00275 // Now move the file into place. 00276 if (!vfs->rename_file(temp_pathname, cache_pathname) && vfs->exists(temp_pathname)) { 00277 vfs->delete_file(cache_pathname); 00278 if (!vfs->rename_file(temp_pathname, cache_pathname)) { 00279 util_cat.error() 00280 << "Unable to rename " << temp_pathname << " to " 00281 << cache_pathname << "\n"; 00282 vfs->delete_file(temp_pathname); 00283 return false; 00284 } 00285 } 00286 00287 add_to_index(record); 00288 00289 return true; 00290 } 00291 00292 //////////////////////////////////////////////////////////////////// 00293 // Function: BamCache::emergency_read_only 00294 // Access: Private 00295 // Description: Called when an attempt to write to the cache dir 00296 // has failed, usually for lack of disk space or 00297 // because of incorrect file permissions. Outputs 00298 // an error and puts the BamCache into read-only 00299 // mode. 00300 //////////////////////////////////////////////////////////////////// 00301 void BamCache:: 00302 emergency_read_only() { 00303 util_cat.error() << 00304 "Could not write to the Bam Cache. Disabling future attempts.\n"; 00305 _read_only = true; 00306 } 00307 00308 //////////////////////////////////////////////////////////////////// 00309 // Function: BamCache::consider_flush_index 00310 // Access: Published 00311 // Description: Flushes the index if enough time has elapsed since 00312 // the index was last flushed. 00313 //////////////////////////////////////////////////////////////////// 00314 void BamCache:: 00315 consider_flush_index() { 00316 ReMutexHolder holder(_lock); 00317 if (_index_stale_since != 0) { 00318 int elapsed = (int)time(NULL) - (int)_index_stale_since; 00319 if (elapsed > _flush_time) { 00320 flush_index(); 00321 } 00322 } 00323 } 00324 00325 //////////////////////////////////////////////////////////////////// 00326 // Function: BamCache::flush_index 00327 // Access: Published 00328 // Description: Ensures the index is written to disk. 00329 //////////////////////////////////////////////////////////////////// 00330 void BamCache:: 00331 flush_index() { 00332 ReMutexHolder holder(_lock); 00333 if (_index_stale_since == 0) { 00334 // Never mind. 00335 return; 00336 } 00337 00338 while (true) { 00339 if (_read_only) { 00340 return; 00341 } 00342 00343 Filename temp_pathname = Filename::temporary(_root, "index-", ".boo"); 00344 00345 if (!do_write_index(temp_pathname, _index)) { 00346 emergency_read_only(); 00347 return; 00348 } 00349 00350 // Now atomically write the name of this index file to the index 00351 // reference file. 00352 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00353 Filename index_ref_pathname(_root, Filename("index_name.txt")); 00354 string old_index = _index_ref_contents; 00355 string new_index = temp_pathname.get_basename() + "\n"; 00356 string orig_index; 00357 00358 if (vfs->atomic_compare_and_exchange_contents(index_ref_pathname, orig_index, old_index, new_index)) { 00359 // We successfully wrote our version of the index, and no other 00360 // process beat us to it. Our index is now the official one. 00361 // Remove the old index. 00362 vfs->delete_file(_index_pathname); 00363 _index_pathname = temp_pathname; 00364 _index_ref_contents = new_index; 00365 _index_stale_since = 0; 00366 return; 00367 } 00368 00369 // Shoot, some other process updated the index while we were 00370 // trying to update it, and they beat us to it. We have to merge, 00371 // and try again. 00372 vfs->delete_file(temp_pathname); 00373 _index_pathname = Filename(_root, Filename(trim(orig_index))); 00374 _index_ref_contents = orig_index; 00375 read_index(); 00376 } 00377 check_cache_size(); 00378 } 00379 00380 //////////////////////////////////////////////////////////////////// 00381 // Function: BamCache::read_index 00382 // Access: Private 00383 // Description: Reads, or re-reads the index file from disk. If 00384 // _index_stale_since is nonzero, the index file is read 00385 // and then merged with our current index. 00386 //////////////////////////////////////////////////////////////////// 00387 void BamCache:: 00388 read_index() { 00389 if (!read_index_pathname(_index_pathname, _index_ref_contents)) { 00390 // Couldn't read the index ref; rebuild the index. 00391 rebuild_index(); 00392 return; 00393 } 00394 00395 while (true) { 00396 BamCacheIndex *new_index = do_read_index(_index_pathname); 00397 if (new_index != (BamCacheIndex *)NULL) { 00398 merge_index(new_index); 00399 return; 00400 } 00401 00402 // We couldn't read the index. Maybe it's been removed already. 00403 // See if the index_pathname has changed. 00404 Filename old_index_pathname = _index_pathname; 00405 if (!read_index_pathname(_index_pathname, _index_ref_contents)) { 00406 // Couldn't read the index ref; rebuild the index. 00407 rebuild_index(); 00408 return; 00409 } 00410 00411 if (old_index_pathname == _index_pathname) { 00412 // Nope, we just couldn't read it. Delete it and build a new 00413 // one. 00414 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00415 vfs->delete_file(_index_pathname); 00416 rebuild_index(); 00417 flush_index(); 00418 return; 00419 } 00420 } 00421 } 00422 00423 //////////////////////////////////////////////////////////////////// 00424 // Function: BamCache::read_index_pathname 00425 // Access: Private 00426 // Description: Atomically reads the current index filename from the 00427 // index reference file. The index filename moves 00428 // around as different processes update the index. 00429 //////////////////////////////////////////////////////////////////// 00430 bool BamCache:: 00431 read_index_pathname(Filename &index_pathname, string &index_ref_contents) const { 00432 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00433 index_ref_contents.clear(); 00434 Filename index_ref_pathname(_root, Filename("index_name.txt")); 00435 if (!vfs->atomic_read_contents(index_ref_pathname, index_ref_contents)) { 00436 return false; 00437 } 00438 00439 string trimmed = trim(index_ref_contents); 00440 if (trimmed.empty()) { 00441 index_pathname = Filename(); 00442 } else { 00443 index_pathname = Filename(_root, Filename(trimmed)); 00444 } 00445 return true; 00446 } 00447 00448 //////////////////////////////////////////////////////////////////// 00449 // Function: BamCache::merge_index 00450 // Access: Private 00451 // Description: The supplied index file has been updated by some other 00452 // process. Merge it with our current index. 00453 // 00454 // Ownership of the pointer is transferred with this 00455 // call. The caller should assume that new_index will 00456 // be deleted by this method. 00457 //////////////////////////////////////////////////////////////////// 00458 void BamCache:: 00459 merge_index(BamCacheIndex *new_index) { 00460 if (_index_stale_since == 0) { 00461 // If our index isn't stale, just replace it. 00462 delete _index; 00463 _index = new_index; 00464 return; 00465 } 00466 00467 BamCacheIndex *old_index = _index; 00468 old_index->release_records(); 00469 new_index->release_records(); 00470 _index = new BamCacheIndex; 00471 00472 BamCacheIndex::Records::const_iterator ai = old_index->_records.begin(); 00473 BamCacheIndex::Records::const_iterator bi = new_index->_records.begin(); 00474 00475 while (ai != old_index->_records.end() && 00476 bi != new_index->_records.end()) { 00477 if ((*ai).first < (*bi).first) { 00478 // Here is an entry we have in our index, not present in the new 00479 // index. 00480 PT(BamCacheRecord) record = (*ai).second; 00481 Filename cache_pathname(_root, record->get_cache_filename()); 00482 if (cache_pathname.exists()) { 00483 // The file exists; keep it. 00484 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); 00485 } 00486 ++ai; 00487 00488 } else if ((*bi).first < (*ai).first) { 00489 // Here is an entry in the new index, not present in our index. 00490 PT(BamCacheRecord) record = (*bi).second; 00491 Filename cache_pathname(_root, record->get_cache_filename()); 00492 if (cache_pathname.exists()) { 00493 // The file exists; keep it. 00494 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); 00495 } 00496 ++bi; 00497 00498 } else { 00499 // Here is an entry we have in both. 00500 PT(BamCacheRecord) a_record = (*ai).second; 00501 PT(BamCacheRecord) b_record = (*bi).second; 00502 if (*a_record == *b_record) { 00503 // They're the same entry. It doesn't really matter which one 00504 // we keep. 00505 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record)); 00506 00507 } else { 00508 // They're different. Just throw them both away, and re-read 00509 // the current data from the cache file. 00510 00511 Filename cache_pathname(_root, a_record->get_cache_filename()); 00512 00513 if (cache_pathname.exists()) { 00514 PT(BamCacheRecord) record = do_read_record(cache_pathname, false); 00515 if (record != (BamCacheRecord *)NULL) { 00516 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); 00517 } 00518 } 00519 } 00520 00521 ++ai; 00522 ++bi; 00523 } 00524 } 00525 00526 while (ai != old_index->_records.end()) { 00527 // Here is an entry we have in our index, not present in the new 00528 // index. 00529 PT(BamCacheRecord) record = (*ai).second; 00530 Filename cache_pathname(_root, record->get_cache_filename()); 00531 if (cache_pathname.exists()) { 00532 // The file exists; keep it. 00533 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); 00534 } 00535 ++ai; 00536 } 00537 00538 while (bi != new_index->_records.end()) { 00539 // Here is an entry in the new index, not present in our index. 00540 PT(BamCacheRecord) record = (*bi).second; 00541 Filename cache_pathname(_root, record->get_cache_filename()); 00542 if (cache_pathname.exists()) { 00543 // The file exists; keep it. 00544 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); 00545 } 00546 ++bi; 00547 } 00548 00549 _index->process_new_records(); 00550 } 00551 00552 //////////////////////////////////////////////////////////////////// 00553 // Function: BamCache::rebuild_index 00554 // Access: Private 00555 // Description: Regenerates the index from scratch by scanning the 00556 // directory. 00557 //////////////////////////////////////////////////////////////////// 00558 void BamCache:: 00559 rebuild_index() { 00560 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00561 00562 PT(VirtualFileList) contents = vfs->scan_directory(_root); 00563 if (contents == NULL) { 00564 util_cat.error() 00565 << "Unable to read directory " << _root << ", caching disabled.\n"; 00566 set_active(false); 00567 return; 00568 } 00569 00570 delete _index; 00571 _index = new BamCacheIndex; 00572 00573 int num_files = contents->get_num_files(); 00574 for (int ci = 0; ci < num_files; ++ci) { 00575 VirtualFile *file = contents->get_file(ci); 00576 Filename filename = file->get_filename(); 00577 if (filename.get_extension() == "bam" || 00578 filename.get_extension() == "txo") { 00579 Filename pathname(_root, filename); 00580 00581 PT(BamCacheRecord) record = do_read_record(pathname, false); 00582 if (record == (BamCacheRecord *)NULL) { 00583 // Well, it was invalid, so blow it away. 00584 file->delete_file(); 00585 00586 } else { 00587 record->_record_access_time = record->_recorded_time; 00588 00589 bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second; 00590 if (!inserted) { 00591 util_cat.info() 00592 << "Multiple cache files defining " << record->get_source_pathname() << "\n"; 00593 file->delete_file(); 00594 } 00595 } 00596 } 00597 } 00598 _index->process_new_records(); 00599 00600 _index_stale_since = time(NULL); 00601 check_cache_size(); 00602 flush_index(); 00603 } 00604 00605 //////////////////////////////////////////////////////////////////// 00606 // Function: BamCache::add_to_index 00607 // Access: Private 00608 // Description: Updates the index entry for the indicated record. 00609 // Note that a copy of the record is made first. 00610 //////////////////////////////////////////////////////////////////// 00611 void BamCache:: 00612 add_to_index(const BamCacheRecord *record) { 00613 PT(BamCacheRecord) new_record = record->make_copy(); 00614 00615 if (_index->add_record(new_record)) { 00616 mark_index_stale(); 00617 check_cache_size(); 00618 } 00619 } 00620 00621 //////////////////////////////////////////////////////////////////// 00622 // Function: BamCache::remove_from_index 00623 // Access: Private 00624 // Description: Removes the index entry for the indicated record, if 00625 // there is one. 00626 //////////////////////////////////////////////////////////////////// 00627 void BamCache:: 00628 remove_from_index(const Filename &source_pathname) { 00629 if (_index->remove_record(source_pathname)) { 00630 mark_index_stale(); 00631 } 00632 } 00633 00634 //////////////////////////////////////////////////////////////////// 00635 // Function: BamCache::check_cache_size 00636 // Access: Private 00637 // Description: If the cache size has exceeded its specified size 00638 // limit, removes an old file. 00639 //////////////////////////////////////////////////////////////////// 00640 void BamCache:: 00641 check_cache_size() { 00642 if (_index->_cache_size == 0) { 00643 // 0 means no limit. 00644 return; 00645 } 00646 00647 if (_index->_cache_size / 1024 > _max_kbytes) { 00648 while (_index->_cache_size / 1024 > _max_kbytes) { 00649 PT(BamCacheRecord) record = _index->evict_old_file(); 00650 if (record == NULL) { 00651 // Never mind; the cache is empty. 00652 break; 00653 } 00654 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00655 Filename cache_pathname(_root, record->get_cache_filename()); 00656 vfs->delete_file(cache_pathname); 00657 } 00658 mark_index_stale(); 00659 } 00660 } 00661 00662 //////////////////////////////////////////////////////////////////// 00663 // Function: BamCache::do_read_index 00664 // Access: Private, Static 00665 // Description: Reads the index data from the specified filename. 00666 // Returns a newly-allocated BamCacheIndex object on 00667 // success, or NULL on failure. 00668 //////////////////////////////////////////////////////////////////// 00669 BamCacheIndex *BamCache:: 00670 do_read_index(const Filename &index_pathname) { 00671 if (index_pathname.empty()) { 00672 return NULL; 00673 } 00674 00675 DatagramInputFile din; 00676 if (!din.open(index_pathname)) { 00677 util_cat.debug() 00678 << "Could not read index file: " << index_pathname << "\n"; 00679 return NULL; 00680 } 00681 00682 string head; 00683 if (!din.read_header(head, _bam_header.size())) { 00684 util_cat.debug() 00685 << index_pathname << " is not an index file.\n"; 00686 return NULL; 00687 } 00688 00689 if (head != _bam_header) { 00690 util_cat.debug() 00691 << index_pathname << " is not an index file.\n"; 00692 return NULL; 00693 } 00694 00695 BamReader reader(&din); 00696 if (!reader.init()) { 00697 return NULL; 00698 } 00699 00700 TypedWritable *object = reader.read_object(); 00701 00702 if (object == (TypedWritable *)NULL) { 00703 util_cat.error() 00704 << "Cache index " << index_pathname << " is empty.\n"; 00705 return NULL; 00706 00707 } else if (!object->is_of_type(BamCacheIndex::get_class_type())) { 00708 util_cat.error() 00709 << "Cache index " << index_pathname << " contains a " 00710 << object->get_type() << ", not a BamCacheIndex.\n"; 00711 return NULL; 00712 } 00713 00714 BamCacheIndex *index = DCAST(BamCacheIndex, object); 00715 if (!reader.resolve()) { 00716 util_cat.error() 00717 << "Unable to fully resolve cache index file.\n"; 00718 return NULL; 00719 } 00720 00721 return index; 00722 } 00723 00724 //////////////////////////////////////////////////////////////////// 00725 // Function: BamCache::do_write_index 00726 // Access: Private, Static 00727 // Description: Writes the given index data to the specified filename. 00728 //////////////////////////////////////////////////////////////////// 00729 bool BamCache:: 00730 do_write_index(const Filename &index_pathname, const BamCacheIndex *index) { 00731 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00732 DatagramOutputFile dout; 00733 if (!dout.open(index_pathname)) { 00734 util_cat.error() 00735 << "Could not write index file: " << index_pathname << "\n"; 00736 vfs->delete_file(index_pathname); 00737 return false; 00738 } 00739 00740 if (!dout.write_header(_bam_header)) { 00741 util_cat.error() 00742 << "Unable to write to " << index_pathname << "\n"; 00743 vfs->delete_file(index_pathname); 00744 return false; 00745 } 00746 00747 { 00748 BamWriter writer(&dout); 00749 if (!writer.init()) { 00750 vfs->delete_file(index_pathname); 00751 return false; 00752 } 00753 00754 if (!writer.write_object(index)) { 00755 vfs->delete_file(index_pathname); 00756 return false; 00757 } 00758 } 00759 00760 return true; 00761 } 00762 00763 //////////////////////////////////////////////////////////////////// 00764 // Function: BamCache::find_and_read_record 00765 // Access: Private 00766 // Description: Looks for the existing cache file that corresponds 00767 // to the indicated filename. Normally, this is the 00768 // specified cache filename exactly; but in the case of 00769 // a hash collision, it may be a variant of the cache 00770 // filename. 00771 //////////////////////////////////////////////////////////////////// 00772 PT(BamCacheRecord) BamCache:: 00773 find_and_read_record(const Filename &source_pathname, 00774 const Filename &cache_filename) { 00775 int pass = 0; 00776 while (true) { 00777 PT(BamCacheRecord) record = 00778 read_record(source_pathname, cache_filename, pass); 00779 if (record != (BamCacheRecord *)NULL) { 00780 add_to_index(record); 00781 return record; 00782 } 00783 ++pass; 00784 } 00785 } 00786 00787 //////////////////////////////////////////////////////////////////// 00788 // Function: BamCache::read_record 00789 // Access: Private 00790 // Description: Reads the indicated cache file and returns its 00791 // associated record if it can be read and it matches 00792 // the source filename. 00793 //////////////////////////////////////////////////////////////////// 00794 PT(BamCacheRecord) BamCache:: 00795 read_record(const Filename &source_pathname, 00796 const Filename &cache_filename, 00797 int pass) { 00798 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00799 Filename cache_pathname(_root, cache_filename); 00800 if (pass != 0) { 00801 ostringstream strm; 00802 strm << cache_pathname.get_basename_wo_extension() << "_" << pass; 00803 cache_pathname.set_basename_wo_extension(strm.str()); 00804 } 00805 00806 if (!cache_pathname.exists()) { 00807 // There is no such cache file already. Declare it. 00808 PT(BamCacheRecord) record = 00809 new BamCacheRecord(source_pathname, cache_filename); 00810 record->_cache_pathname = cache_pathname; 00811 return record; 00812 } 00813 00814 PT(BamCacheRecord) record = do_read_record(cache_pathname, true); 00815 if (record == (BamCacheRecord *)NULL) { 00816 // Well, it was invalid, so blow it away, and make a new one. 00817 vfs->delete_file(cache_pathname); 00818 remove_from_index(source_pathname); 00819 00820 PT(BamCacheRecord) record = 00821 new BamCacheRecord(source_pathname, cache_filename); 00822 record->_cache_pathname = cache_pathname; 00823 return record; 00824 } 00825 00826 if (record->get_source_pathname() != source_pathname) { 00827 // This might be just a hash conflict. 00828 util_cat.debug() 00829 << "Cache file " << cache_pathname << " references " 00830 << record->get_source_pathname() << ", not " 00831 << source_pathname << "\n"; 00832 return NULL; 00833 } 00834 00835 if (!record->has_data()) { 00836 // If we didn't find any data, the caller will have to reload it. 00837 record->clear_dependent_files(); 00838 } 00839 00840 record->_cache_pathname = cache_pathname; 00841 return record; 00842 } 00843 00844 //////////////////////////////////////////////////////////////////// 00845 // Function: BamCache::do_read_record 00846 // Access: Private, Static 00847 // Description: Actually reads a record from the file. 00848 //////////////////////////////////////////////////////////////////// 00849 PT(BamCacheRecord) BamCache:: 00850 do_read_record(const Filename &cache_pathname, bool read_data) { 00851 DatagramInputFile din; 00852 if (!din.open(cache_pathname)) { 00853 util_cat.debug() 00854 << "Could not read cache file: " << cache_pathname << "\n"; 00855 return NULL; 00856 } 00857 00858 string head; 00859 if (!din.read_header(head, _bam_header.size())) { 00860 util_cat.debug() 00861 << cache_pathname << " is not a cache file.\n"; 00862 return NULL; 00863 } 00864 00865 if (head != _bam_header) { 00866 util_cat.debug() 00867 << cache_pathname << " is not a cache file.\n"; 00868 return NULL; 00869 } 00870 00871 BamReader reader(&din); 00872 if (!reader.init()) { 00873 return NULL; 00874 } 00875 00876 TypedWritable *object = reader.read_object(); 00877 if (object == (TypedWritable *)NULL) { 00878 util_cat.debug() 00879 << cache_pathname << " is empty.\n"; 00880 return NULL; 00881 00882 } else if (!object->is_of_type(BamCacheRecord::get_class_type())) { 00883 util_cat.debug() 00884 << "Cache file " << cache_pathname << " contains a " 00885 << object->get_type() << ", not a BamCacheRecord.\n"; 00886 return NULL; 00887 } 00888 00889 PT(BamCacheRecord) record = DCAST(BamCacheRecord, object); 00890 if (!reader.resolve()) { 00891 util_cat.debug() 00892 << "Unable to fully resolve cache record in " << cache_pathname << "\n"; 00893 return NULL; 00894 } 00895 00896 // From this point below, we have validated that the selected 00897 // filename is indeed a cache record for the indicated source file, 00898 // and therefore the cache record will be returned. 00899 00900 // We still need to decide whether the cache record is stale. 00901 if (read_data && record->dependents_unchanged()) { 00902 // The cache record doesn't appear to be stale. Load the cached 00903 // object. 00904 TypedWritable *ptr; 00905 ReferenceCount *ref_ptr; 00906 00907 if (reader.read_object(ptr, ref_ptr)) { 00908 if (!reader.resolve()) { 00909 util_cat.debug() 00910 << "Unable to fully resolve cached object in " << cache_pathname << "\n"; 00911 delete object; 00912 } else { 00913 // The object is valid. Store it in the record. 00914 record->set_data(ptr, ref_ptr); 00915 } 00916 } 00917 } 00918 00919 // Also get the total file size. 00920 istream &in = din.get_stream(); 00921 in.clear(); 00922 in.seekg(0, ios::end); 00923 record->_record_size = in.tellg(); 00924 00925 // And the last access time is now, duh. 00926 record->_record_access_time = time(NULL); 00927 00928 return record; 00929 } 00930 00931 //////////////////////////////////////////////////////////////////// 00932 // Function: BamCache::hash_filename 00933 // Access: Private, Static 00934 // Description: Returns the appropriate filename to use for a cache 00935 // file, given the fullpath string to the source 00936 // filename. 00937 //////////////////////////////////////////////////////////////////// 00938 string BamCache:: 00939 hash_filename(const string &filename) { 00940 #ifdef HAVE_OPENSSL 00941 // With OpenSSl, use the MD5 hash of the filename. 00942 HashVal hv; 00943 hv.hash_string(filename); 00944 ostringstream strm; 00945 hv.output_hex(strm); 00946 return strm.str(); 00947 00948 #else // HAVE_OPENSSL 00949 // Without OpenSSL, don't get fancy; just build a simple hash. 00950 unsigned int hash = 0; 00951 for (string::const_iterator si = filename.begin(); 00952 si != filename.end(); 00953 ++si) { 00954 hash = (hash * 9109) + (unsigned int)(*si); 00955 } 00956 00957 ostringstream strm; 00958 strm << hex << setw(8) << setfill('0') << hash; 00959 return strm.str(); 00960 00961 #endif // HAVE_OPENSSL 00962 } 00963 00964 //////////////////////////////////////////////////////////////////// 00965 // Function: BamCache::make_global 00966 // Access: Private, Static 00967 // Description: Constructs the global BamCache object. 00968 //////////////////////////////////////////////////////////////////// 00969 void BamCache:: 00970 make_global() { 00971 _global_ptr = new BamCache; 00972 00973 if (_global_ptr->_root.empty()) { 00974 _global_ptr->set_active(false); 00975 } 00976 }