Panda3D
|
00001 // Filename: bamCache.cxx 00002 // Created by: drose (09Jun06) 00003 // 00004 //////////////////////////////////////////////////////////////////// 00005 // 00006 // PANDA 3D SOFTWARE 00007 // Copyright (c) Carnegie Mellon University. All rights reserved. 00008 // 00009 // All use of this software is subject to the terms of the revised BSD 00010 // license. You should have received a copy of this license along 00011 // with this source code in a file named "LICENSE." 00012 // 00013 //////////////////////////////////////////////////////////////////// 00014 00015 #include "bamCache.h" 00016 #include "bamCacheIndex.h" 00017 #include "hashVal.h" 00018 #include "datagramInputFile.h" 00019 #include "datagramOutputFile.h" 00020 #include "config_util.h" 00021 #include "bam.h" 00022 #include "typeRegistry.h" 00023 #include "string_utils.h" 00024 #include "configVariableInt.h" 00025 #include "configVariableString.h" 00026 #include "configVariableFilename.h" 00027 #include "virtualFileSystem.h" 00028 00029 BamCache *BamCache::_global_ptr = NULL; 00030 00031 //////////////////////////////////////////////////////////////////// 00032 // Function: BamCache::Constructor 00033 // Access: Published 00034 // Description: 00035 //////////////////////////////////////////////////////////////////// 00036 BamCache:: 00037 BamCache() : 00038 _active(true), 00039 _read_only(false), 00040 _index(new BamCacheIndex), 00041 _index_stale_since(0) 00042 { 00043 ConfigVariableFilename model_cache_dir 00044 ("model-cache-dir", Filename(), 00045 PRC_DESC("The full path to a directory, local to this computer, in which " 00046 "model and texture files will be cached on load. If a directory " 00047 "name is specified here, files may be loaded from the cache " 00048 "instead of from their actual pathnames, which may save load time, " 00049 "especially if you are loading egg files instead of bam files, " 00050 "or if you are loading models from a shared network drive. " 00051 "If this is the empty string, no cache will be used.")); 00052 00053 ConfigVariableInt model_cache_flush 00054 ("model-cache-flush", 30, 00055 PRC_DESC("This is the amount of time, in seconds, between automatic " 00056 "flushes of the model-cache index.")); 00057 00058 ConfigVariableBool model_cache_models 00059 ("model-cache-models", true, 00060 PRC_DESC("If this is set to true, models will be cached in the " 00061 "model cache, as bam files.")); 00062 00063 ConfigVariableBool model_cache_textures 00064 ("model-cache-textures", true, 00065 PRC_DESC("If this is set to true, textures will also be cached in the " 00066 "model cache, as txo files.")); 00067 00068 ConfigVariableBool model_cache_compressed_textures 00069 ("model-cache-compressed-textures", false, 00070 PRC_DESC("If this is set to true, compressed textures will be cached " 00071 "in the model cache, in their compressed form as downloaded " 00072 "by the GSG. This may be set in conjunction with " 00073 "model-cache-textures, or it may be independent.")); 00074 00075 ConfigVariableInt model_cache_max_kbytes 00076 ("model-cache-max-kbytes", 1048576, 00077 PRC_DESC("This is the maximum size of the model cache, in kilobytes.")); 00078 00079 _cache_models = model_cache_models; 00080 _cache_textures = model_cache_textures; 00081 _cache_compressed_textures = model_cache_compressed_textures; 00082 00083 _flush_time = model_cache_flush; 00084 _max_kbytes = model_cache_max_kbytes; 00085 00086 if (!model_cache_dir.empty()) { 00087 set_root(model_cache_dir); 00088 } 00089 } 00090 00091 //////////////////////////////////////////////////////////////////// 00092 // Function: BamCache::Destructor 00093 // Access: Published 00094 // Description: 00095 //////////////////////////////////////////////////////////////////// 00096 BamCache:: 00097 ~BamCache() { 00098 flush_index(); 00099 delete _index; 00100 _index = NULL; 00101 } 00102 00103 //////////////////////////////////////////////////////////////////// 00104 // Function: BamCache::set_root 00105 // Access: Published 00106 // Description: Changes the current root pathname of the cache. This 00107 // specifies where the cache files are stored on disk. 00108 // This should name a directory that is on a disk local 00109 // to the machine (not on a network-mounted disk), for 00110 // instance, /tmp/panda-cache or /c/panda-cache. 00111 // 00112 // If the directory does not already exist, it will be 00113 // created as a result of this call. 00114 //////////////////////////////////////////////////////////////////// 00115 void BamCache:: 00116 set_root(const Filename &root) { 00117 ReMutexHolder holder(_lock); 00118 flush_index(); 00119 _root = root; 00120 00121 // For now, the filename must be a directory. Maybe eventually we 00122 // will support writing caches to a Panda multifile (though maybe it 00123 // would be better to implement this kind of thing at a lower level, 00124 // via a writable VFS, in which case the specified root filename 00125 // will still be a "directory"). 00126 if (!root.is_directory()) { 00127 Filename dirname(_root, Filename(".")); 00128 dirname.make_dir(); 00129 } 00130 nassertv(root.is_directory()); 00131 00132 delete _index; 00133 _index = new BamCacheIndex; 00134 _index_stale_since = 0; 00135 read_index(); 00136 check_cache_size(); 00137 } 00138 00139 //////////////////////////////////////////////////////////////////// 00140 // Function: BamCache::lookup 00141 // Access: Published 00142 // Description: Looks up a file in the cache. 00143 // 00144 // If the file is cacheable, then regardless of whether 00145 // the file is found in the cache or not, this returns a 00146 // BamCacheRecord. On the other hand, if the file 00147 // cannot be cached, returns NULL. 00148 // 00149 // If record->has_data() returns true, then the file was 00150 // found in the cache, and you may call 00151 // record->extract_data() to get the object. If 00152 // record->has_data() returns false, then the file was 00153 // not found in the cache or the cache was stale; and 00154 // you should reload the source file (calling 00155 // record->add_dependent_file() for each file loaded, 00156 // including the original source file), and then call 00157 // record->set_data() to record the resulting loaded 00158 // object; and finally, you should call store() to write 00159 // the cached record to disk. 00160 //////////////////////////////////////////////////////////////////// 00161 PT(BamCacheRecord) BamCache:: 00162 lookup(const Filename &source_filename, const string &cache_extension) { 00163 ReMutexHolder holder(_lock); 00164 consider_flush_index(); 00165 00166 VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); 00167 00168 Filename source_pathname(source_filename); 00169 source_pathname.make_absolute(vfs->get_cwd()); 00170 00171 Filename rel_pathname(source_pathname); 00172 rel_pathname.make_relative_to(_root, false); 00173 if (rel_pathname.is_local()) { 00174 // If the source pathname is already within the cache directory, 00175 // don't cache it further. 00176 return NULL; 00177 } 00178 00179 Filename cache_filename = hash_filename(source_pathname.get_fullpath()); 00180 cache_filename.set_extension(cache_extension); 00181 00182 return find_and_read_record(source_pathname, cache_filename); 00183 } 00184 00185 //////////////////////////////////////////////////////////////////// 00186 // Function: BamCache::store 00187 // Access: Published 00188 // Description: Flushes a cache entry to disk. You must have 00189 // retrieved the cache record via a prior call to 00190 // lookup(), and then stored the data via 00191 // record->set_data(). Returns true on success, false 00192 // on failure. 00193 //////////////////////////////////////////////////////////////////// 00194 bool BamCache:: 00195 store(BamCacheRecord *record) { 00196 ReMutexHolder holder(_lock); 00197 nassertr(!record->_cache_pathname.empty(), false); 00198 nassertr(record->has_data(), false); 00199 00200 if (_read_only) { 00201 return false; 00202 } 00203 00204 consider_flush_index(); 00205 00206 #ifndef NDEBUG 00207 // Ensure that the cache_pathname is within the _root directory tree. 00208 Filename rel_pathname(record->_cache_pathname); 00209 rel_pathname.make_relative_to(_root, false); 00210 nassertr(rel_pathname.is_local(), false); 00211 #endif // NDEBUG 00212 00213 record->_recorded_time = time(NULL); 00214 00215 Filename cache_pathname = Filename::binary_filename(record->_cache_pathname); 00216 00217 // We actually do the write to a temporary filename first, and then 00218 // move it into place, so that no one attempts to read the file 00219 // while it is in the process of being written. 00220 Thread *current_thread = Thread::get_current_thread(); 00221 string extension = current_thread->get_unique_id() + string(".tmp"); 00222 Filename temp_pathname = cache_pathname; 00223 temp_pathname.set_extension(extension); 00224 temp_pathname.set_binary(); 00225 00226 pofstream temp_file; 00227 if (!temp_pathname.open_write(temp_file)) { 00228 util_cat.error() 00229 << "Could not open cache file: " << temp_pathname << "\n"; 00230 emergency_read_only(); 00231 return false; 00232 } 00233 00234 DatagramOutputFile dout; 00235 if (!dout.open(temp_file)) { 00236 util_cat.error() 00237 << "Could not write cache file: " << temp_pathname << "\n"; 00238 temp_pathname.unlink(); 00239 emergency_read_only(); 00240 return false; 00241 } 00242 00243 if (!dout.write_header(_bam_header)) { 00244 util_cat.error() 00245 << "Unable to write to " << temp_pathname << "\n"; 00246 temp_pathname.unlink(); 00247 return false; 00248 } 00249 00250 { 00251 BamWriter writer(&dout, temp_pathname); 00252 if (!writer.init()) { 00253 temp_pathname.unlink(); 00254 return false; 00255 } 00256 00257 TypeRegistry *type_registry = TypeRegistry::ptr(); 00258 TypeHandle texture_type = type_registry->find_type("Texture"); 00259 if (record->get_data()->is_of_type(texture_type)) { 00260 // Texture objects write the actual texture image. 00261 writer.set_file_texture_mode(BamWriter::BTM_rawdata); 00262 } else { 00263 // Any other kinds of objects write texture references. 00264 writer.set_file_texture_mode(BamWriter::BTM_fullpath); 00265 } 00266 00267 if (!writer.write_object(record)) { 00268 temp_pathname.unlink(); 00269 return false; 00270 } 00271 00272 if (!writer.write_object(record->get_data())) { 00273 temp_pathname.unlink(); 00274 return false; 00275 } 00276 00277 // Now that we are done with the BamWriter, it's important to let 00278 // it destruct now and clean itself up, or it might get mad if we 00279 // delete any TypedWritables below that haven't been written yet. 00280 } 00281 00282 record->_record_size = temp_file.tellp(); 00283 temp_file.close(); 00284 00285 // Now move the file into place. 00286 if (!temp_pathname.rename_to(cache_pathname) && temp_pathname.exists()) { 00287 cache_pathname.unlink(); 00288 if (!temp_pathname.rename_to(cache_pathname)) { 00289 util_cat.error() 00290 << "Unable to rename " << temp_pathname << " to " 00291 << cache_pathname << "\n"; 00292 temp_pathname.unlink(); 00293 return false; 00294 } 00295 } 00296 00297 add_to_index(record); 00298 00299 return true; 00300 } 00301 00302 //////////////////////////////////////////////////////////////////// 00303 // Function: BamCache::emergency_read_only 00304 // Access: Private 00305 // Description: Called when an attempt to write to the cache dir 00306 // has failed, usually for lack of disk space or 00307 // because of incorrect file permissions. Outputs 00308 // an error and puts the BamCache into read-only 00309 // mode. 00310 //////////////////////////////////////////////////////////////////// 00311 void BamCache:: 00312 emergency_read_only() { 00313 util_cat.error() << 00314 "Could not write to the Bam Cache. Disabling future attempts.\n"; 00315 _read_only = true; 00316 } 00317 00318 //////////////////////////////////////////////////////////////////// 00319 // Function: BamCache::consider_flush_index 00320 // Access: Published 00321 // Description: Flushes the index if enough time has elapsed since 00322 // the index was last flushed. 00323 //////////////////////////////////////////////////////////////////// 00324 void BamCache:: 00325 consider_flush_index() { 00326 ReMutexHolder holder(_lock); 00327 if (_index_stale_since != 0) { 00328 int elapsed = (int)time(NULL) - (int)_index_stale_since; 00329 if (elapsed > _flush_time) { 00330 flush_index(); 00331 } 00332 } 00333 } 00334 00335 //////////////////////////////////////////////////////////////////// 00336 // Function: BamCache::flush_index 00337 // Access: Published 00338 // Description: Ensures the index is written to disk. 00339 //////////////////////////////////////////////////////////////////// 00340 void BamCache:: 00341 flush_index() { 00342 ReMutexHolder holder(_lock); 00343 if (_index_stale_since == 0) { 00344 // Never mind. 00345 return; 00346 } 00347 00348 while (true) { 00349 if (_read_only) { 00350 return; 00351 } 00352 00353 Filename temp_pathname = Filename::temporary(_root, "index-", ".boo"); 00354 00355 if (!do_write_index(temp_pathname, _index)) { 00356 emergency_read_only(); 00357 return; 00358 } 00359 00360 // Now atomically write the name of this index file to the index 00361 // reference file. 00362 Filename index_ref_pathname(_root, Filename("index_name.txt")); 00363 string old_index = _index_ref_contents; 00364 string new_index = temp_pathname.get_basename() + "\n"; 00365 string orig_index; 00366 if (index_ref_pathname.atomic_compare_and_exchange_contents(orig_index, old_index, new_index)) { 00367 // We successfully wrote our version of the index, and no other 00368 // process beat us to it. Our index is now the official one. 00369 // Remove the old index. 00370 _index_pathname.unlink(); 00371 _index_pathname = temp_pathname; 00372 _index_ref_contents = new_index; 00373 _index_stale_since = 0; 00374 return; 00375 } 00376 00377 // Shoot, some other process updated the index while we were 00378 // trying to update it, and they beat us to it. We have to merge, 00379 // and try again. 00380 temp_pathname.unlink(); 00381 _index_pathname = Filename(_root, Filename(trim(orig_index))); 00382 _index_ref_contents = orig_index; 00383 read_index(); 00384 } 00385 check_cache_size(); 00386 } 00387 00388 //////////////////////////////////////////////////////////////////// 00389 // Function: BamCache::read_index 00390 // Access: Private 00391 // Description: Reads, or re-reads the index file from disk. If 00392 // _index_stale_since is nonzero, the index file is read 00393 // and then merged with our current index. 00394 //////////////////////////////////////////////////////////////////// 00395 void BamCache:: 00396 read_index() { 00397 if (!read_index_pathname(_index_pathname, _index_ref_contents)) { 00398 // Couldn't read the index ref; rebuild the index. 00399 rebuild_index(); 00400 return; 00401 } 00402 00403 while (true) { 00404 BamCacheIndex *new_index = do_read_index(_index_pathname); 00405 if (new_index != (BamCacheIndex *)NULL) { 00406 merge_index(new_index); 00407 return; 00408 } 00409 00410 // We couldn't read the index. Maybe it's been removed already. 00411 // See if the index_pathname has changed. 00412 Filename old_index_pathname = _index_pathname; 00413 if (!read_index_pathname(_index_pathname, _index_ref_contents)) { 00414 // Couldn't read the index ref; rebuild the index. 00415 rebuild_index(); 00416 return; 00417 } 00418 00419 if (old_index_pathname == _index_pathname) { 00420 // Nope, we just couldn't read it. Delete it and build a new 00421 // one. 00422 _index_pathname.unlink(); 00423 rebuild_index(); 00424 flush_index(); 00425 return; 00426 } 00427 } 00428 } 00429 00430 //////////////////////////////////////////////////////////////////// 00431 // Function: BamCache::read_index_pathname 00432 // Access: Private 00433 // Description: Atomically reads the current index filename from the 00434 // index reference file. The index filename moves 00435 // around as different processes update the index. 00436 //////////////////////////////////////////////////////////////////// 00437 bool BamCache:: 00438 read_index_pathname(Filename &index_pathname, string &index_ref_contents) const { 00439 index_ref_contents.clear(); 00440 Filename index_ref_pathname(_root, Filename("index_name.txt")); 00441 if (!index_ref_pathname.atomic_read_contents(index_ref_contents)) { 00442 return false; 00443 } 00444 00445 string trimmed = trim(index_ref_contents); 00446 if (trimmed.empty()) { 00447 index_pathname = Filename(); 00448 } else { 00449 index_pathname = Filename(_root, Filename(trimmed)); 00450 } 00451 return true; 00452 } 00453 00454 //////////////////////////////////////////////////////////////////// 00455 // Function: BamCache::merge_index 00456 // Access: Private 00457 // Description: The supplied index file has been updated by some other 00458 // process. Merge it with our current index. 00459 // 00460 // Ownership of the pointer is transferred with this 00461 // call. The caller should assume that new_index will 00462 // be deleted by this method. 00463 //////////////////////////////////////////////////////////////////// 00464 void BamCache:: 00465 merge_index(BamCacheIndex *new_index) { 00466 if (_index_stale_since == 0) { 00467 // If our index isn't stale, just replace it. 00468 delete _index; 00469 _index = new_index; 00470 return; 00471 } 00472 00473 BamCacheIndex *old_index = _index; 00474 old_index->release_records(); 00475 new_index->release_records(); 00476 _index = new BamCacheIndex; 00477 00478 BamCacheIndex::Records::const_iterator ai = old_index->_records.begin(); 00479 BamCacheIndex::Records::const_iterator bi = new_index->_records.begin(); 00480 00481 while (ai != old_index->_records.end() && 00482 bi != new_index->_records.end()) { 00483 if ((*ai).first < (*bi).first) { 00484 // Here is an entry we have in our index, not present in the new 00485 // index. 00486 PT(BamCacheRecord) record = (*ai).second; 00487 Filename cache_pathname(_root, record->get_cache_filename()); 00488 if (cache_pathname.exists()) { 00489 // The file exists; keep it. 00490 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); 00491 } 00492 ++ai; 00493 00494 } else if ((*bi).first < (*ai).first) { 00495 // Here is an entry in the new index, not present in our index. 00496 PT(BamCacheRecord) record = (*bi).second; 00497 Filename cache_pathname(_root, record->get_cache_filename()); 00498 if (cache_pathname.exists()) { 00499 // The file exists; keep it. 00500 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); 00501 } 00502 ++bi; 00503 00504 } else { 00505 // Here is an entry we have in both. 00506 PT(BamCacheRecord) a_record = (*ai).second; 00507 PT(BamCacheRecord) b_record = (*bi).second; 00508 if (*a_record == *b_record) { 00509 // They're the same entry. It doesn't really matter which one 00510 // we keep. 00511 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(a_record->get_source_pathname(), a_record)); 00512 00513 } else { 00514 // They're different. Just throw them both away, and re-read 00515 // the current data from the cache file. 00516 00517 Filename cache_pathname(_root, a_record->get_cache_filename()); 00518 00519 if (cache_pathname.exists()) { 00520 PT(BamCacheRecord) record = do_read_record(cache_pathname, false); 00521 if (record != (BamCacheRecord *)NULL) { 00522 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); 00523 } 00524 } 00525 } 00526 00527 ++ai; 00528 ++bi; 00529 } 00530 } 00531 00532 while (ai != old_index->_records.end()) { 00533 // Here is an entry we have in our index, not present in the new 00534 // index. 00535 PT(BamCacheRecord) record = (*ai).second; 00536 Filename cache_pathname(_root, record->get_cache_filename()); 00537 if (cache_pathname.exists()) { 00538 // The file exists; keep it. 00539 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); 00540 } 00541 ++ai; 00542 } 00543 00544 while (bi != new_index->_records.end()) { 00545 // Here is an entry in the new index, not present in our index. 00546 PT(BamCacheRecord) record = (*bi).second; 00547 Filename cache_pathname(_root, record->get_cache_filename()); 00548 if (cache_pathname.exists()) { 00549 // The file exists; keep it. 00550 _index->_records.insert(_index->_records.end(), BamCacheIndex::Records::value_type(record->get_source_pathname(), record)); 00551 } 00552 ++bi; 00553 } 00554 00555 _index->process_new_records(); 00556 } 00557 00558 //////////////////////////////////////////////////////////////////// 00559 // Function: BamCache::rebuild_index 00560 // Access: Private 00561 // Description: Regenerates the index from scratch by scanning the 00562 // directory. 00563 //////////////////////////////////////////////////////////////////// 00564 void BamCache:: 00565 rebuild_index() { 00566 vector_string contents; 00567 if (!_root.scan_directory(contents)) { 00568 util_cat.error() 00569 << "Unable to read directory " << _root << ", caching disabled.\n"; 00570 set_active(false); 00571 return; 00572 } 00573 00574 delete _index; 00575 _index = new BamCacheIndex; 00576 00577 vector_string::const_iterator ci; 00578 for (ci = contents.begin(); ci != contents.end(); ++ci) { 00579 Filename filename(*ci); 00580 if (filename.get_extension() == "bam" || 00581 filename.get_extension() == "txo") { 00582 Filename pathname(_root, filename); 00583 00584 PT(BamCacheRecord) record = do_read_record(pathname, false); 00585 if (record == (BamCacheRecord *)NULL) { 00586 // Well, it was invalid, so blow it away. 00587 pathname.unlink(); 00588 00589 } else { 00590 record->_record_access_time = record->_recorded_time; 00591 00592 bool inserted = _index->_records.insert(BamCacheIndex::Records::value_type(record->get_source_pathname(), record)).second; 00593 if (!inserted) { 00594 util_cat.info() 00595 << "Multiple cache files defining " << record->get_source_pathname() << "\n"; 00596 pathname.unlink(); 00597 } 00598 } 00599 } 00600 } 00601 _index->process_new_records(); 00602 00603 _index_stale_since = time(NULL); 00604 check_cache_size(); 00605 flush_index(); 00606 } 00607 00608 //////////////////////////////////////////////////////////////////// 00609 // Function: BamCache::add_to_index 00610 // Access: Private 00611 // Description: Updates the index entry for the indicated record. 00612 // Note that a copy of the record is made first. 00613 //////////////////////////////////////////////////////////////////// 00614 void BamCache:: 00615 add_to_index(const BamCacheRecord *record) { 00616 PT(BamCacheRecord) new_record = record->make_copy(); 00617 00618 if (_index->add_record(new_record)) { 00619 mark_index_stale(); 00620 check_cache_size(); 00621 } 00622 } 00623 00624 //////////////////////////////////////////////////////////////////// 00625 // Function: BamCache::remove_from_index 00626 // Access: Private 00627 // Description: Removes the index entry for the indicated record, if 00628 // there is one. 00629 //////////////////////////////////////////////////////////////////// 00630 void BamCache:: 00631 remove_from_index(const Filename &source_pathname) { 00632 if (_index->remove_record(source_pathname)) { 00633 mark_index_stale(); 00634 } 00635 } 00636 00637 //////////////////////////////////////////////////////////////////// 00638 // Function: BamCache::check_cache_size 00639 // Access: Private 00640 // Description: If the cache size has exceeded its specified size 00641 // limit, removes an old file. 00642 //////////////////////////////////////////////////////////////////// 00643 void BamCache:: 00644 check_cache_size() { 00645 if (_index->_cache_size == 0) { 00646 // 0 means no limit. 00647 return; 00648 } 00649 00650 if (_index->_cache_size / 1024 > _max_kbytes) { 00651 while (_index->_cache_size / 1024 > _max_kbytes) { 00652 PT(BamCacheRecord) record = _index->evict_old_file(); 00653 if (record == NULL) { 00654 // Never mind; the cache is empty. 00655 break; 00656 } 00657 Filename cache_pathname(_root, record->get_cache_filename()); 00658 cache_pathname.unlink(); 00659 } 00660 mark_index_stale(); 00661 } 00662 } 00663 00664 //////////////////////////////////////////////////////////////////// 00665 // Function: BamCache::do_read_index 00666 // Access: Private, Static 00667 // Description: Reads the index data from the specified filename. 00668 // Returns a newly-allocated BamCacheIndex object on 00669 // success, or NULL on failure. 00670 //////////////////////////////////////////////////////////////////// 00671 BamCacheIndex *BamCache:: 00672 do_read_index(Filename &index_pathname) { 00673 if (index_pathname.empty()) { 00674 return NULL; 00675 } 00676 00677 index_pathname.set_binary(); 00678 pifstream index_file; 00679 if (!index_pathname.open_read(index_file)) { 00680 util_cat.error() 00681 << "Could not open index file: " << index_pathname << "\n"; 00682 return NULL; 00683 } 00684 00685 DatagramInputFile din; 00686 00687 if (!din.open(index_file)) { 00688 util_cat.debug() 00689 << "Could not read index file: " << index_pathname << "\n"; 00690 return NULL; 00691 } 00692 00693 string head; 00694 if (!din.read_header(head, _bam_header.size())) { 00695 util_cat.debug() 00696 << index_pathname << " is not an index file.\n"; 00697 return NULL; 00698 } 00699 00700 if (head != _bam_header) { 00701 util_cat.debug() 00702 << index_pathname << " is not an index file.\n"; 00703 return NULL; 00704 } 00705 00706 BamReader reader(&din, index_pathname); 00707 if (!reader.init()) { 00708 return NULL; 00709 } 00710 00711 TypedWritable *object = reader.read_object(); 00712 00713 if (object == (TypedWritable *)NULL) { 00714 util_cat.error() 00715 << "Cache index " << index_pathname << " is empty.\n"; 00716 return NULL; 00717 00718 } else if (!object->is_of_type(BamCacheIndex::get_class_type())) { 00719 util_cat.error() 00720 << "Cache index " << index_pathname << " contains a " 00721 << object->get_type() << ", not a BamCacheIndex.\n"; 00722 return NULL; 00723 } 00724 00725 BamCacheIndex *index = DCAST(BamCacheIndex, object); 00726 if (!reader.resolve()) { 00727 util_cat.error() 00728 << "Unable to fully resolve cache index file.\n"; 00729 return NULL; 00730 } 00731 00732 return index; 00733 } 00734 00735 //////////////////////////////////////////////////////////////////// 00736 // Function: BamCache::do_write_index 00737 // Access: Private, Static 00738 // Description: Writes the given index data to the specified filename. 00739 //////////////////////////////////////////////////////////////////// 00740 bool BamCache:: 00741 do_write_index(Filename &index_pathname, const BamCacheIndex *index) { 00742 index_pathname.set_binary(); 00743 pofstream index_file; 00744 00745 if (!index_pathname.open_write(index_file)) { 00746 util_cat.error() 00747 << "Could not open index file: " << index_pathname << "\n"; 00748 return false; 00749 } 00750 00751 DatagramOutputFile dout; 00752 if (!dout.open(index_file)) { 00753 util_cat.error() 00754 << "Could not write index file: " << index_pathname << "\n"; 00755 index_pathname.unlink(); 00756 return false; 00757 } 00758 00759 if (!dout.write_header(_bam_header)) { 00760 util_cat.error() 00761 << "Unable to write to " << index_pathname << "\n"; 00762 index_pathname.unlink(); 00763 return false; 00764 } 00765 00766 { 00767 BamWriter writer(&dout, index_pathname); 00768 if (!writer.init()) { 00769 index_pathname.unlink(); 00770 return false; 00771 } 00772 00773 if (!writer.write_object(index)) { 00774 index_pathname.unlink(); 00775 return false; 00776 } 00777 } 00778 00779 index_file.close(); 00780 return true; 00781 } 00782 00783 //////////////////////////////////////////////////////////////////// 00784 // Function: BamCache::find_and_read_record 00785 // Access: Private 00786 // Description: Looks for the existing cache file that corresponds 00787 // to the indicated filename. Normally, this is the 00788 // specified cache filename exactly; but in the case of 00789 // a hash collision, it may be a variant of the cache 00790 // filename. 00791 //////////////////////////////////////////////////////////////////// 00792 PT(BamCacheRecord) BamCache:: 00793 find_and_read_record(const Filename &source_pathname, 00794 const Filename &cache_filename) { 00795 int pass = 0; 00796 while (true) { 00797 PT(BamCacheRecord) record = 00798 read_record(source_pathname, cache_filename, pass); 00799 if (record != (BamCacheRecord *)NULL) { 00800 add_to_index(record); 00801 return record; 00802 } 00803 ++pass; 00804 } 00805 } 00806 00807 //////////////////////////////////////////////////////////////////// 00808 // Function: BamCache::read_record 00809 // Access: Private 00810 // Description: Reads the indicated cache file and returns its 00811 // associated record if it can be read and it matches 00812 // the source filename. 00813 //////////////////////////////////////////////////////////////////// 00814 PT(BamCacheRecord) BamCache:: 00815 read_record(const Filename &source_pathname, 00816 const Filename &cache_filename, 00817 int pass) { 00818 Filename cache_pathname(_root, cache_filename); 00819 if (pass != 0) { 00820 ostringstream strm; 00821 strm << cache_pathname.get_basename_wo_extension() << "_" << pass; 00822 cache_pathname.set_basename_wo_extension(strm.str()); 00823 } 00824 00825 if (!cache_pathname.exists()) { 00826 // There is no such cache file already. Declare it. 00827 PT(BamCacheRecord) record = 00828 new BamCacheRecord(source_pathname, cache_filename); 00829 record->_cache_pathname = cache_pathname; 00830 return record; 00831 } 00832 00833 PT(BamCacheRecord) record = do_read_record(cache_pathname, true); 00834 if (record == (BamCacheRecord *)NULL) { 00835 // Well, it was invalid, so blow it away, and make a new one. 00836 cache_pathname.unlink(); 00837 remove_from_index(source_pathname); 00838 00839 PT(BamCacheRecord) record = 00840 new BamCacheRecord(source_pathname, cache_filename); 00841 record->_cache_pathname = cache_pathname; 00842 return record; 00843 } 00844 00845 if (record->get_source_pathname() != source_pathname) { 00846 // This might be just a hash conflict. 00847 util_cat.debug() 00848 << "Cache file " << cache_pathname << " references " 00849 << record->get_source_pathname() << ", not " 00850 << source_pathname << "\n"; 00851 return NULL; 00852 } 00853 00854 if (!record->has_data()) { 00855 // If we didn't find any data, the caller will have to reload it. 00856 record->clear_dependent_files(); 00857 } 00858 00859 record->_cache_pathname = cache_pathname; 00860 return record; 00861 } 00862 00863 //////////////////////////////////////////////////////////////////// 00864 // Function: BamCache::do_read_record 00865 // Access: Private, Static 00866 // Description: Actually reads a record from the file. 00867 //////////////////////////////////////////////////////////////////// 00868 PT(BamCacheRecord) BamCache:: 00869 do_read_record(Filename &cache_pathname, bool read_data) { 00870 cache_pathname.set_binary(); 00871 pifstream cache_file; 00872 if (!cache_pathname.open_read(cache_file)) { 00873 util_cat.debug() 00874 << "Could not open cache file: " << cache_pathname << "\n"; 00875 return NULL; 00876 } 00877 00878 DatagramInputFile din; 00879 00880 if (!din.open(cache_file)) { 00881 util_cat.debug() 00882 << "Could not read cache file: " << cache_pathname << "\n"; 00883 return NULL; 00884 } 00885 00886 string head; 00887 if (!din.read_header(head, _bam_header.size())) { 00888 util_cat.debug() 00889 << cache_pathname << " is not a cache file.\n"; 00890 return NULL; 00891 } 00892 00893 if (head != _bam_header) { 00894 util_cat.debug() 00895 << cache_pathname << " is not a cache file.\n"; 00896 return NULL; 00897 } 00898 00899 BamReader reader(&din, cache_pathname); 00900 if (!reader.init()) { 00901 return NULL; 00902 } 00903 00904 TypedWritable *object = reader.read_object(); 00905 if (object == (TypedWritable *)NULL) { 00906 util_cat.debug() 00907 << cache_pathname << " is empty.\n"; 00908 return NULL; 00909 00910 } else if (!object->is_of_type(BamCacheRecord::get_class_type())) { 00911 util_cat.debug() 00912 << "Cache file " << cache_pathname << " contains a " 00913 << object->get_type() << ", not a BamCacheRecord.\n"; 00914 return NULL; 00915 } 00916 00917 PT(BamCacheRecord) record = DCAST(BamCacheRecord, object); 00918 if (!reader.resolve()) { 00919 util_cat.debug() 00920 << "Unable to fully resolve cache record in " << cache_pathname << "\n"; 00921 return NULL; 00922 } 00923 00924 // From this point below, we have validated that the selected 00925 // filename is indeed a cache record for the indicated source file, 00926 // and therefore the cache record will be returned. 00927 00928 // We still need to decide whether the cache record is stale. 00929 if (read_data && record->dependents_unchanged()) { 00930 // The cache record doesn't appear to be stale. Load the cached 00931 // object. 00932 TypedWritable *ptr; 00933 ReferenceCount *ref_ptr; 00934 00935 if (reader.read_object(ptr, ref_ptr)) { 00936 if (!reader.resolve()) { 00937 util_cat.debug() 00938 << "Unable to fully resolve cached object in " << cache_pathname << "\n"; 00939 delete object; 00940 } else { 00941 // The object is valid. Store it in the record. 00942 record->set_data(ptr, ref_ptr); 00943 } 00944 } 00945 } 00946 00947 // Also get the file size. 00948 cache_file.clear(); 00949 cache_file.seekg(0, ios::end); 00950 record->_record_size = cache_file.tellg(); 00951 00952 // And the last access time is now, duh. 00953 record->_record_access_time = time(NULL); 00954 00955 return record; 00956 } 00957 00958 //////////////////////////////////////////////////////////////////// 00959 // Function: BamCache::hash_filename 00960 // Access: Private, Static 00961 // Description: Returns the appropriate filename to use for a cache 00962 // file, given the fullpath string to the source 00963 // filename. 00964 //////////////////////////////////////////////////////////////////// 00965 string BamCache:: 00966 hash_filename(const string &filename) { 00967 #ifdef HAVE_OPENSSL 00968 // With OpenSSl, use the MD5 hash of the filename. 00969 HashVal hv; 00970 hv.hash_string(filename); 00971 ostringstream strm; 00972 hv.output_hex(strm); 00973 return strm.str(); 00974 00975 #else // HAVE_OPENSSL 00976 // Without OpenSSL, don't get fancy; just build a simple hash. 00977 unsigned int hash = 0; 00978 for (string::const_iterator si = filename.begin(); 00979 si != filename.end(); 00980 ++si) { 00981 hash = (hash * 9109) + (unsigned int)(*si); 00982 } 00983 00984 ostringstream strm; 00985 strm << hex << setw(8) << setfill('0') << hash; 00986 return strm.str(); 00987 00988 #endif // HAVE_OPENSSL 00989 } 00990 00991 //////////////////////////////////////////////////////////////////// 00992 // Function: BamCache::make_global 00993 // Access: Private, Static 00994 // Description: Constructs the global BamCache object. 00995 //////////////////////////////////////////////////////////////////// 00996 void BamCache:: 00997 make_global() { 00998 _global_ptr = new BamCache; 00999 01000 if (_global_ptr->_root.empty()) { 01001 _global_ptr->set_active(false); 01002 } 01003 }