25using std::istringstream;
32uint32_t DownloadDb::_magic_number = 0xfeedfeed;
38uint32_t DownloadDb::_bogus_magic_number = 0x11111111;
42back_to_front_slash(
const string &str) {
45 for (si = result.begin(); si != result.end(); ++si) {
60 if (downloader_cat.is_debug())
61 downloader_cat.debug()
62 <<
"DownloadDb constructor called" << endl;
63 _client_db = read_db(client_file, 0);
64 _client_db._filename = client_file;
65 _server_db = read_db(server_file, 1);
73 if (downloader_cat.is_debug())
74 downloader_cat.debug()
75 <<
"DownloadDb constructor called" << endl;
76 _client_db = read_db(client_file, 0);
77 _client_db._filename = client_file;
78 _server_db = read_db(server_file, 1);
79 _server_db._filename = server_file;
96 if (downloader_cat.is_debug())
97 downloader_cat.debug()
98 <<
"DownloadDb destructor called" << endl;
106output(ostream &out)
const {
107 out <<
"[" << _server_db._filename <<
" " << _client_db._filename <<
"]";
114write(ostream &out)
const {
115 out <<
"DownloadDb" << endl;
116 out <<
"============================================================" << endl;
117 out <<
" Client DB file: " << _client_db._filename << endl;
118 out <<
"============================================================" << endl;
119 _client_db.write(out);
121 out <<
"============================================================" << endl;
122 out <<
" Server DB file: " << _server_db._filename << endl;
123 out <<
"============================================================" << endl;
124 _server_db.write(out);
125 write_version_map(out);
135 return write_db(file, _client_db, 0);
144 return write_db(file, _server_db, 1);
151client_multifile_exists(
string mfname)
const {
152 return (_client_db.multifile_exists(mfname));
161 int client_status = _client_db.get_multifile_record_named(mfname)->_status;
162 return (client_status >= Status_complete);
169client_multifile_decompressed(
string mfname)
const {
170 int client_status = _client_db.get_multifile_record_named(mfname)->_status;
171 return (client_status >= Status_decompressed);
178client_multifile_extracted(
string mfname)
const {
179 int client_status = _client_db.get_multifile_record_named(mfname)->_status;
180 return (client_status >= Status_extracted);
189 return _client_db.get_multifile_record_named(mfname)->_hash;
198 return _server_db.get_multifile_record_named(mfname)->_hash;
207 _client_db.get_multifile_record_named(mfname)->_hash = val;
208 write_client_db(_client_db._filename);
217 _server_db.get_multifile_record_named(mfname)->_hash = val;
226delete_client_multifile(
string mfname) {
233add_client_multifile(
string server_mfname) {
234 PT(MultifileRecord) server_mfr = _server_db.get_multifile_record_named(server_mfname);
235 PT(MultifileRecord) client_mfr =
new MultifileRecord;
236 client_mfr->_name = server_mfr->_name;
237 client_mfr->_phase = server_mfr->_phase;
238 _client_db.add_multifile_record(client_mfr);
246expand_client_multifile(
string mfname) {
254read_db(
Filename &file,
bool want_server_info) {
262 if (read_stream ==
nullptr) {
263 downloader_cat.error()
264 <<
"failed to open input file: "
270 if (!db.read(sr, want_server_info)) {
271 downloader_cat.error()
278 if (want_server_info) {
279 if (!read_version_map(sr)) {
280 downloader_cat.error()
281 <<
"read_version_map() failed: "
295read_db(
Ramfile &file,
bool want_server_info) {
297 istringstream read_stream(file._data);
302 if (!db.read(sr, want_server_info)) {
303 downloader_cat.error()
304 <<
"read failed" << endl;
307 if (want_server_info) {
308 if (!read_version_map(sr)) {
309 downloader_cat.error()
310 <<
"read_version_map() failed" << endl;
322write_db(
Filename &file, Db db,
bool want_server_info) {
323 pofstream write_stream;
326 downloader_cat.error()
327 <<
"DownloadDb::write_db() - Failed to open output file: "
332 downloader_cat.spam()
333 <<
"Writing to file: " << file << endl;
338 db.write_bogus_header(sw);
339 db.write(sw, want_server_info);
340 if (want_server_info) {
341 write_version_map(sw);
344 db.write_header(write_stream);
345 write_stream.close();
363server_add_multifile(
string mfname, Phase phase,
int size,
int status) {
364 PT(MultifileRecord) mfr =
new MultifileRecord(mfname, phase, size, status);
365 _server_db.add_multifile_record(mfr);
373server_add_file(
string mfname,
string fname) {
375 PT(FileRecord) fr =
new FileRecord(fname);
378 pvector< PT(MultifileRecord) >::iterator i = _server_db._mfile_records.begin();
379 for (; i != _server_db._mfile_records.end(); ++i) {
380 if (mfname == (*i)->_name) {
381 (*i)->add_file_record(fr);
387 downloader_cat.error() <<
"Could not find record named "
388 << mfname <<
" in database " << endl;
400DownloadDb::MultifileRecord::
405 _status = Status_incomplete;
412DownloadDb::MultifileRecord::
413MultifileRecord(
string name, Phase phase,
int size,
int status) {
424void DownloadDb::MultifileRecord::
425write(ostream &out)
const {
426 out <<
"==================================================" << endl;
427 out <<
"MultifileRecord: " << _name << endl
428 <<
" phase: " << _phase << endl
429 <<
" size: " << _size << endl
430 <<
" status: " << _status << endl
431 <<
" hash: " << _hash.as_dec() << endl;
432 out <<
"--------------------------------------------------" << endl;
433 pvector< PT(FileRecord) >::const_iterator i = _file_records.begin();
434 for(; i != _file_records.end(); ++i) {
444int DownloadDb::MultifileRecord::
445get_num_files()
const {
446 return _file_records.size();
452string DownloadDb::MultifileRecord::
453get_file_name(
int index)
const {
454 return _file_records[index]->_name;
461bool DownloadDb::MultifileRecord::
462file_exists(
string fname)
const {
463 pvector< PT(FileRecord) >::const_iterator i = _file_records.begin();
464 for(; i != _file_records.end(); ++i) {
465 if (fname == (*i)->_name) {
477get_file_record_named(
string fname)
const {
478 pvector< PT(FileRecord) >::const_iterator i = _file_records.begin();
479 for(; i != _file_records.end(); ++i) {
480 if (fname == (*i)->_name) {
485 downloader_cat.error() <<
"Could not find record named "
486 << fname <<
" in multifile " << _name << endl;
487 PT(FileRecord) foo =
new FileRecord;
488 nassertr(
false, foo);
496void DownloadDb::MultifileRecord::
497add_file_record(PT(FileRecord) fr) {
498 _file_records.push_back(fr);
514 _header_length =
sizeof(_magic_number) +
sizeof(int32_t);
522write(ostream &out)
const {
523 pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
524 for(; i != _mfile_records.end(); ++i) {
534get_num_multifiles()
const {
535 return _mfile_records.size();
541string DownloadDb::Db::
542get_multifile_name(
int index)
const {
543 return _mfile_records[index]->_name;
550multifile_exists(
string mfname)
const {
551 pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
552 for(; i != _mfile_records.end(); ++i) {
553 if (mfname == (*i)->_name) {
564get_multifile_record_named(
string mfname)
const {
565 pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
566 for(; i != _mfile_records.end(); ++i) {
567 if (mfname == (*i)->_name) {
572 downloader_cat.error() <<
"Could not find record named "
573 << mfname <<
" in database " << endl;
574 PT(MultifileRecord) foo =
new MultifileRecord;
575 nassertr(
false, foo);
583add_multifile_record(PT(MultifileRecord) mfr) {
584 _mfile_records.push_back(mfr);
596 downloader_cat.debug()
597 <<
"Parsed magic number: " << magic_number << endl;
600 if (magic_number == _bogus_magic_number) {
601 downloader_cat.error()
602 <<
"DownloadDb::parse_header() - "
603 <<
"Bogus magic number, previous write incomplete: "
604 << magic_number <<
" expected: " << _magic_number << endl;
608 else if (magic_number != _magic_number) {
609 downloader_cat.error()
610 <<
"DownloadDb::parse_header() - Invalid magic number: "
611 << magic_number <<
" expected: " << _magic_number << endl;
616 downloader_cat.debug()
617 <<
"Parsed number of multifiles: " << num_multifiles << endl;
620 return num_multifiles;
633 downloader_cat.spam()
634 <<
"Parsed record header length: " << record_length << endl;
637 return record_length;
650 mfr->_name = di.get_string32();
651 mfr->_phase = di.get_float64();
652 mfr->_size = di.get_int32();
653 mfr->_status = di.get_int32();
654 mfr->_num_files = di.get_int32();
659 mfr->_name = back_to_front_slash(mfr->_name);
662 mfr->_hash.read_datagram(di);
664 downloader_cat.debug()
665 <<
"Parsed multifile record: " << mfr->_name <<
" phase: " << mfr->_phase
666 <<
" size: " << mfr->_size
667 <<
" status: " << mfr->_status <<
" num_files: " << mfr->_num_files << endl;
685 fr->_name = di.get_string32();
690 fr->_name = back_to_front_slash(fr->_name);
692 downloader_cat.spam()
693 <<
"Parsed file record: " << fr->_name << endl;
709 if (header.size() != (
size_t)_header_length) {
710 downloader_cat.error() <<
"truncated db file" << endl;
715 int num_multifiles = parse_header(
Datagram(std::move(header)));
716 if (num_multifiles < 0) {
717 downloader_cat.error() <<
"invalid db header" << endl;
723 for (
int i = 0; i < num_multifiles; i++) {
726 int mfr_header_length =
sizeof(int32_t);
728 vector_uchar mfr_header = sr.
extract_bytes(mfr_header_length);
729 if (mfr_header.size() != (size_t)mfr_header_length) {
730 downloader_cat.error() <<
"invalid mfr header" << endl;
735 int mfr_length = parse_record_header(
Datagram(std::move(mfr_header)));
739 int read_length = (mfr_length - mfr_header_length);
741 if (mfr_record.size() != (size_t)read_length) {
742 downloader_cat.error() <<
"invalid mfr record" << endl;
750 if (want_server_info) {
753 for (
int j = 0; j < mfr->_num_files; j++) {
756 int fr_header_length =
sizeof(int32_t);
760 if (fr_header.size() != (size_t)fr_header_length) {
761 downloader_cat.error() <<
"invalid fr header" << endl;
766 int fr_length = parse_record_header(
Datagram(std::move(fr_header)));
770 int read_length = (fr_length - fr_header_length);
773 if (fr_record.size() != (size_t)read_length) {
774 downloader_cat.error() <<
"invalid fr record" << endl;
782 mfr->add_file_record(fr);
787 add_multifile_record(mfr);
807 int32_t header_length;
810 pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
811 for(; i != _mfile_records.end(); ++i) {
813 phase = (*i)->_phase;
815 status = (*i)->_status;
816 num_files = (*i)->get_num_files();
817 name_length = (*i)->_name.length();
821 sizeof(header_length) +
822 sizeof(name_length) +
823 (*i)->_name.length() +
824 sizeof(phase) +
sizeof(size) +
825 sizeof(status) +
sizeof(num_files) +
842 (*i)->_hash.write_stream(sw);
845 if (want_server_info) {
848 pvector< PT(FileRecord) >::const_iterator j = (*i)->_file_records.begin();
849 for(; j != (*i)->_file_records.end(); ++j) {
850 name_length = (*j)->_name.length();
854 sizeof(header_length) +
855 sizeof(name_length) +
856 (*j)->_name.length();
902 write_stream.seekp(0);
916DownloadDb::FileRecord::
925DownloadDb::FileRecord::
926FileRecord(
string name) {
933void DownloadDb::FileRecord::
934write(ostream &out)
const {
935 out <<
" FileRecord: " << _name << endl;
946 nassertv(version >= 1);
949 int size = vhash.size();
952 nassertv(version <= size+1);
954 if (version-1 < size) {
956 vhash[version-1] = hash;
960 vhash.push_back(hash);
971 vhash.insert(vhash.begin(), hash);
981 return (_versions.find(name) != _versions.end());
989 VersionMap::const_iterator vmi = _versions.find(name);
990 if (vmi == _versions.end()) {
994 return (
int)(*vmi).second.size();
1003 VersionMap::iterator vmi = _versions.find(name);
1004 if (vmi == _versions.end()) {
1005 nassertv(num_versions == 0);
1011 nassertv(num_versions <= (
int)vhash.size());
1012 vhash.erase(vhash.begin() + num_versions, vhash.end());
1022 VersionMap::const_iterator vmi = _versions.find(name);
1023 if (vmi == _versions.end()) {
1024 downloader_cat.debug()
1025 <<
"DownloadDb::get_version() - can't find: " << name << endl;
1029 VectorHash::const_iterator i = find(vhash.begin(), vhash.end(), hash);
1030 if (i != vhash.end())
1031 return (i - vhash.begin() + 1);
1032 downloader_cat.debug()
1033 <<
"DownloadDb::get_version() - can't find hash: " << hash << endl;
1045 VersionMap::const_iterator vmi = _versions.find(name);
1046 if (vmi == _versions.end()) {
1047 downloader_cat.error()
1048 <<
"DownloadDb::get_hash() - can't find: " << name << endl;
1053 if (version < 1 || version > (
int)vhash.size()) {
1054 downloader_cat.error()
1055 <<
"DownloadDb::get_hash() - no version " << version
1056 <<
" for " << name << endl;
1059 return vhash[version - 1];
1067 VersionMap::iterator vmi;
1068 VectorHash::iterator i;
1072 for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
1073 name = (*vmi).first;
1074 downloader_cat.spam()
1075 <<
"DownloadDb::write_version_map() - writing file: "
1076 << name <<
" of length: " << name.length() << endl;
1080 for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
1082 (*i).write_stream(sw);
1097 for (
int i = 0; i < num_entries; i++) {
1101 downloader_cat.spam()
1102 <<
"DownloadDb::read_version_map() - name: " << name << endl;
1109 downloader_cat.spam()
1110 <<
"DownloadDb::read_version_map() - number of values: " << length
1113 for (
int j = 0; j < length; j++) {
1115 hash.read_stream(sr);
1129write_version_map(ostream &out)
const {
1130 out <<
"Version Map: " << endl;
1131 VersionMap::const_iterator vmi;
1132 VectorHash::const_iterator i;
1133 for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
1134 out <<
" " << (*vmi).first << endl;
1135 for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
1137 out <<
" " << hash.
as_dec() << endl;
A class to retrieve the individual data elements previously stored in a Datagram.
uint32_t get_uint32()
Extracts an unsigned 32-bit integer.
int32_t get_int32()
Extracts a signed 32-bit integer.
An ordered list of data elements, formatted in memory for transmission over a socket or writing to a ...
void add_uint32(uint32_t value)
Adds an unsigned 32-bit integer to the datagram.
size_t get_length() const
Returns the number of bytes in the datagram.
void add_int32(int32_t value)
Adds a signed 32-bit integer to the datagram.
const void * get_data() const
Returns a pointer to the beginning of the datagram's data.
int parse_header(Datagram dg)
Verifies magic number, returns the number of multifiles or -1 if invalid.
bool write_bogus_header(StreamWriter &sw)
Writes the bogus header uncompressed with platform- independent byte ordering.
bool write_header(std::ostream &write_stream)
Writes the header uncompressed with platform- independent byte ordering.
int parse_record_header(Datagram dg)
Parses a file record (fr) header and returns the length of the next file record.
const HashVal & get_hash(const Filename &name, int version) const
Returns the MD5 hash associated with the indicated version of the indicated file.
void set_server_multifile_hash(std::string mfname, HashVal val)
Set the hash value of file we are working on.
void create_new_server_db()
Used on the server side makefiles to create a new clean server db.
HashVal get_client_multifile_hash(std::string mfname) const
Return the hash value of the file we are working on.
bool client_multifile_complete(std::string mfname) const
A multifile is complete when it is completely downloaded.
int get_version(const Filename &name, const HashVal &hash) const
Returns the version number of this particular file, determined by looking up the hash generated from ...
bool has_version(const Filename &name) const
Returns true if the indicated file has version information, false otherwise.
void insert_new_version(const Filename &name, const HashVal &hash)
Inserts a new version 1 copy of the file, sliding all the other versions up by one.
HashVal get_server_multifile_hash(std::string mfname) const
Return the hash value of the server file.
DownloadDb()
Primarily used for testing.
void set_client_multifile_hash(std::string mfname, HashVal val)
Set the hash value of file we are working on.
void set_num_versions(const Filename &name, int num_versions)
Reduces the number of versions of a particular file stored in the ddb by throwing away all versions h...
void add_version(const Filename &name, const HashVal &hash, int version)
Appends a new version of the file onto the end of the list, or changes the hash associated with a ver...
int get_num_versions(const Filename &name) const
Returns the number of versions stored for the indicated file.
The name of a file, such as a texture file or an Egg file.
void set_binary()
Indicates that the filename represents a binary file.
bool open_write(std::ofstream &stream, bool truncate=true) const
Opens the indicated ifstream for writing the file, if possible.
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer.
std::string as_dec() const
Returns the HashVal as a string with four decimal numbers.
An in-memory buffer specifically designed for downloading files to memory.
A class to read sequential binary data directly from an istream.
std::string get_string32()
Extracts a variable-length string with a 32-bit length field.
int32_t get_int32()
Extracts a signed 32-bit integer.
size_t extract_bytes(unsigned char *into, size_t size)
Extracts the indicated number of bytes in the stream into the given character buffer.
get_istream
Returns the stream in use.
A StreamWriter object is used to write sequential binary data directly to an ostream.
void append_data(const void *data, size_t size)
Appends some more raw data to the end of the streamWriter.
void add_float64(PN_float64 value)
Adds a 64-bit floating-point number to the stream.
void add_int32(int32_t value)
Adds a signed 32-bit integer to the stream.
void add_uint32(uint32_t value)
Adds an unsigned 32-bit integer to the stream.
A hierarchy of directories and files that appears to be one continuous file system,...
static void close_read_file(std::istream *stream)
Closes a file opened by a previous call to open_read_file().
std::istream * open_read_file(const Filename &filename, bool auto_unwrap) const
Convenience function; returns a newly allocated istream if the file exists and can be read,...
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
This is our own Panda specialization on the default STL vector.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.