25 using std::istringstream;
33 uint32_t DownloadDb::_magic_number = 0xfeedfeed;
39 uint32_t DownloadDb::_bogus_magic_number = 0x11111111;
43 back_to_front_slash(
const string &str) {
46 for (si = result.begin(); si != result.end(); ++si) {
61 if (downloader_cat.is_debug())
62 downloader_cat.debug()
63 <<
"DownloadDb constructor called" << endl;
64 _client_db = read_db(client_file, 0);
65 _client_db._filename = client_file;
66 _server_db = read_db(server_file, 1);
74 if (downloader_cat.is_debug())
75 downloader_cat.debug()
76 <<
"DownloadDb constructor called" << endl;
77 _client_db = read_db(client_file, 0);
78 _client_db._filename = client_file;
79 _server_db = read_db(server_file, 1);
80 _server_db._filename = server_file;
97 if (downloader_cat.is_debug())
98 downloader_cat.debug()
99 <<
"DownloadDb destructor called" << endl;
107 output(ostream &out)
const {
108 out <<
"[" << _server_db._filename <<
" " << _client_db._filename <<
"]";
115 write(ostream &out)
const {
116 out <<
"DownloadDb" << endl;
117 out <<
"============================================================" << endl;
118 out <<
" Client DB file: " << _client_db._filename << endl;
119 out <<
"============================================================" << endl;
120 _client_db.write(out);
122 out <<
"============================================================" << endl;
123 out <<
" Server DB file: " << _server_db._filename << endl;
124 out <<
"============================================================" << endl;
125 _server_db.write(out);
126 write_version_map(out);
136 return write_db(file, _client_db, 0);
145 return write_db(file, _server_db, 1);
152 client_multifile_exists(
string mfname)
const {
153 return (_client_db.multifile_exists(mfname));
162 int client_status = _client_db.get_multifile_record_named(mfname)->_status;
163 return (client_status >= Status_complete);
170 client_multifile_decompressed(
string mfname)
const {
171 int client_status = _client_db.get_multifile_record_named(mfname)->_status;
172 return (client_status >= Status_decompressed);
179 client_multifile_extracted(
string mfname)
const {
180 int client_status = _client_db.get_multifile_record_named(mfname)->_status;
181 return (client_status >= Status_extracted);
190 return _client_db.get_multifile_record_named(mfname)->_hash;
199 return _server_db.get_multifile_record_named(mfname)->_hash;
208 _client_db.get_multifile_record_named(mfname)->_hash = val;
209 write_client_db(_client_db._filename);
218 _server_db.get_multifile_record_named(mfname)->_hash = val;
227 delete_client_multifile(
string mfname) {
234 add_client_multifile(
string server_mfname) {
235 PT(MultifileRecord) server_mfr = _server_db.get_multifile_record_named(server_mfname);
236 PT(MultifileRecord) client_mfr =
new MultifileRecord;
237 client_mfr->_name = server_mfr->_name;
238 client_mfr->_phase = server_mfr->_phase;
239 _client_db.add_multifile_record(client_mfr);
247 expand_client_multifile(
string mfname) {
255 read_db(
Filename &file,
bool want_server_info) {
263 if (read_stream ==
nullptr) {
264 downloader_cat.error()
265 <<
"failed to open input file: "
271 if (!db.read(sr, want_server_info)) {
272 downloader_cat.error()
279 if (want_server_info) {
280 if (!read_version_map(sr)) {
281 downloader_cat.error()
282 <<
"read_version_map() failed: "
296 read_db(
Ramfile &file,
bool want_server_info) {
298 istringstream read_stream(file._data);
303 if (!db.read(sr, want_server_info)) {
304 downloader_cat.error()
305 <<
"read failed" << endl;
308 if (want_server_info) {
309 if (!read_version_map(sr)) {
310 downloader_cat.error()
311 <<
"read_version_map() failed" << endl;
323 write_db(
Filename &file, Db db,
bool want_server_info) {
324 pofstream write_stream;
327 downloader_cat.error()
328 <<
"DownloadDb::write_db() - Failed to open output file: "
333 downloader_cat.spam()
334 <<
"Writing to file: " << file << endl;
339 db.write_bogus_header(sw);
340 db.write(sw, want_server_info);
341 if (want_server_info) {
342 write_version_map(sw);
345 db.write_header(write_stream);
346 write_stream.close();
364 server_add_multifile(
string mfname, Phase phase,
int size,
int status) {
365 PT(MultifileRecord) mfr =
new MultifileRecord(mfname, phase, size, status);
366 _server_db.add_multifile_record(mfr);
374 server_add_file(
string mfname,
string fname) {
376 PT(FileRecord) fr =
new FileRecord(fname);
379 pvector<
PT(MultifileRecord) >::iterator i = _server_db._mfile_records.begin();
380 for (; i != _server_db._mfile_records.end(); ++i) {
381 if (mfname == (*i)->_name) {
382 (*i)->add_file_record(fr);
388 downloader_cat.error() <<
"Could not find record named "
389 << mfname <<
" in database " << endl;
401 DownloadDb::MultifileRecord::
406 _status = Status_incomplete;
413 DownloadDb::MultifileRecord::
414 MultifileRecord(
string name, Phase phase,
int size,
int status) {
425 void DownloadDb::MultifileRecord::
426 write(ostream &out)
const {
427 out <<
"==================================================" << endl;
428 out <<
"MultifileRecord: " << _name << endl
429 <<
" phase: " << _phase << endl
430 <<
" size: " << _size << endl
431 <<
" status: " << _status << endl
432 <<
" hash: " << _hash.as_dec() << endl;
433 out <<
"--------------------------------------------------" << endl;
434 pvector<
PT(FileRecord) >::const_iterator i = _file_records.begin();
435 for(; i != _file_records.end(); ++i) {
445 int DownloadDb::MultifileRecord::
446 get_num_files()
const {
447 return _file_records.size();
453 string DownloadDb::MultifileRecord::
454 get_file_name(
int index)
const {
455 return _file_records[index]->_name;
462 bool DownloadDb::MultifileRecord::
463 file_exists(
string fname)
const {
464 pvector<
PT(FileRecord) >::const_iterator i = _file_records.begin();
465 for(; i != _file_records.end(); ++i) {
466 if (fname == (*i)->_name) {
478 get_file_record_named(
string fname)
const {
479 pvector<
PT(FileRecord) >::const_iterator i = _file_records.begin();
480 for(; i != _file_records.end(); ++i) {
481 if (fname == (*i)->_name) {
486 downloader_cat.error() <<
"Could not find record named "
487 << fname <<
" in multifile " << _name << endl;
488 PT(FileRecord) foo =
new FileRecord;
489 nassertr(
false, foo);
497 void DownloadDb::MultifileRecord::
498 add_file_record(
PT(FileRecord) fr) {
499 _file_records.push_back(fr);
515 _header_length =
sizeof(_magic_number) +
sizeof(int32_t);
522 void DownloadDb::Db::
523 write(ostream &out)
const {
524 pvector<
PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
525 for(; i != _mfile_records.end(); ++i) {
535 get_num_multifiles()
const {
536 return _mfile_records.size();
542 string DownloadDb::Db::
543 get_multifile_name(
int index)
const {
544 return _mfile_records[index]->_name;
550 bool DownloadDb::Db::
551 multifile_exists(
string mfname)
const {
552 pvector<
PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
553 for(; i != _mfile_records.end(); ++i) {
554 if (mfname == (*i)->_name) {
565 get_multifile_record_named(
string mfname)
const {
566 pvector<
PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
567 for(; i != _mfile_records.end(); ++i) {
568 if (mfname == (*i)->_name) {
573 downloader_cat.error() <<
"Could not find record named "
574 << mfname <<
" in database " << endl;
575 PT(MultifileRecord) foo =
new MultifileRecord;
576 nassertr(
false, foo);
583 void DownloadDb::Db::
584 add_multifile_record(
PT(MultifileRecord) mfr) {
585 _mfile_records.push_back(mfr);
597 downloader_cat.debug()
598 <<
"Parsed magic number: " << magic_number << endl;
601 if (magic_number == _bogus_magic_number) {
602 downloader_cat.error()
603 <<
"DownloadDb::parse_header() - "
604 <<
"Bogus magic number, previous write incomplete: "
605 << magic_number <<
" expected: " << _magic_number << endl;
609 else if (magic_number != _magic_number) {
610 downloader_cat.error()
611 <<
"DownloadDb::parse_header() - Invalid magic number: "
612 << magic_number <<
" expected: " << _magic_number << endl;
617 downloader_cat.debug()
618 <<
"Parsed number of multifiles: " << num_multifiles << endl;
621 return num_multifiles;
634 downloader_cat.spam()
635 <<
"Parsed record header length: " << record_length << endl;
638 return record_length;
651 mfr->_name = di.get_string32();
652 mfr->_phase = di.get_float64();
653 mfr->_size = di.get_int32();
654 mfr->_status = di.get_int32();
655 mfr->_num_files = di.get_int32();
660 mfr->_name = back_to_front_slash(mfr->_name);
663 mfr->_hash.read_datagram(di);
665 downloader_cat.debug()
666 <<
"Parsed multifile record: " << mfr->_name <<
" phase: " << mfr->_phase
667 <<
" size: " << mfr->_size
668 <<
" status: " << mfr->_status <<
" num_files: " << mfr->_num_files << endl;
686 fr->_name = di.get_string32();
691 fr->_name = back_to_front_slash(fr->_name);
693 downloader_cat.spam()
694 <<
"Parsed file record: " << fr->_name << endl;
706 bool DownloadDb::Db::
710 if (header.size() != (
size_t)_header_length) {
711 downloader_cat.error() <<
"truncated db file" << endl;
716 int num_multifiles = parse_header(
Datagram(move(header)));
717 if (num_multifiles < 0) {
718 downloader_cat.error() <<
"invalid db header" << endl;
724 for (
int i = 0; i < num_multifiles; i++) {
727 int mfr_header_length =
sizeof(int32_t);
729 vector_uchar mfr_header = sr.
extract_bytes(mfr_header_length);
730 if (mfr_header.size() != (size_t)mfr_header_length) {
731 downloader_cat.error() <<
"invalid mfr header" << endl;
736 int mfr_length = parse_record_header(
Datagram(move(mfr_header)));
740 int read_length = (mfr_length - mfr_header_length);
742 if (mfr_record.size() != (size_t)read_length) {
743 downloader_cat.error() <<
"invalid mfr record" << endl;
751 if (want_server_info) {
754 for (
int j = 0; j < mfr->_num_files; j++) {
757 int fr_header_length =
sizeof(int32_t);
761 if (fr_header.size() != (size_t)fr_header_length) {
762 downloader_cat.error() <<
"invalid fr header" << endl;
767 int fr_length = parse_record_header(
Datagram(move(fr_header)));
771 int read_length = (fr_length - fr_header_length);
774 if (fr_record.size() != (size_t)read_length) {
775 downloader_cat.error() <<
"invalid fr record" << endl;
783 mfr->add_file_record(fr);
788 add_multifile_record(mfr);
799 bool DownloadDb::Db::
808 int32_t header_length;
811 pvector<
PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
812 for(; i != _mfile_records.end(); ++i) {
814 phase = (*i)->_phase;
816 status = (*i)->_status;
817 num_files = (*i)->get_num_files();
818 name_length = (*i)->_name.length();
822 sizeof(header_length) +
823 sizeof(name_length) +
824 (*i)->_name.length() +
825 sizeof(phase) +
sizeof(size) +
826 sizeof(status) +
sizeof(num_files) +
843 (*i)->_hash.write_stream(sw);
846 if (want_server_info) {
849 pvector<
PT(FileRecord) >::const_iterator j = (*i)->_file_records.begin();
850 for(; j != (*i)->_file_records.end(); ++j) {
851 name_length = (*j)->_name.length();
855 sizeof(header_length) +
856 sizeof(name_length) +
857 (*j)->_name.length();
903 write_stream.seekp(0);
917 DownloadDb::FileRecord::
926 DownloadDb::FileRecord::
927 FileRecord(
string name) {
934 void DownloadDb::FileRecord::
935 write(ostream &out)
const {
936 out <<
" FileRecord: " << _name << endl;
947 nassertv(version >= 1);
950 int size = vhash.size();
953 nassertv(version <= size+1);
955 if (version-1 < size) {
957 vhash[version-1] = hash;
961 vhash.push_back(hash);
972 vhash.insert(vhash.begin(), hash);
982 return (_versions.find(name) != _versions.end());
990 VersionMap::const_iterator vmi = _versions.find(name);
991 if (vmi == _versions.end()) {
995 return (
int)(*vmi).second.size();
1004 VersionMap::iterator vmi = _versions.find(name);
1005 if (vmi == _versions.end()) {
1006 nassertv(num_versions == 0);
1012 nassertv(num_versions <= (
int)vhash.size());
1013 vhash.erase(vhash.begin() + num_versions, vhash.end());
1023 VersionMap::const_iterator vmi = _versions.find(name);
1024 if (vmi == _versions.end()) {
1025 downloader_cat.debug()
1026 <<
"DownloadDb::get_version() - can't find: " << name << endl;
1030 VectorHash::const_iterator i = find(vhash.begin(), vhash.end(), hash);
1031 if (i != vhash.end())
1032 return (i - vhash.begin() + 1);
1033 downloader_cat.debug()
1034 <<
"DownloadDb::get_version() - can't find hash: " << hash << endl;
1046 VersionMap::const_iterator vmi = _versions.find(name);
1047 if (vmi == _versions.end()) {
1048 downloader_cat.error()
1049 <<
"DownloadDb::get_hash() - can't find: " << name << endl;
1054 if (version < 1 || version > (
int)vhash.size()) {
1055 downloader_cat.error()
1056 <<
"DownloadDb::get_hash() - no version " << version
1057 <<
" for " << name << endl;
1060 return vhash[version - 1];
1068 VersionMap::iterator vmi;
1069 VectorHash::iterator i;
1073 for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
1074 name = (*vmi).first;
1075 downloader_cat.spam()
1076 <<
"DownloadDb::write_version_map() - writing file: "
1077 << name <<
" of length: " << name.length() << endl;
1081 for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
1083 (*i).write_stream(sw);
1098 for (
int i = 0; i < num_entries; i++) {
1102 downloader_cat.spam()
1103 <<
"DownloadDb::read_version_map() - name: " << name << endl;
1110 downloader_cat.spam()
1111 <<
"DownloadDb::read_version_map() - number of values: " << length
1114 for (
int j = 0; j < length; j++) {
1116 hash.read_stream(sr);
1130 write_version_map(ostream &out)
const {
1131 out <<
"Version Map: " << endl;
1132 VersionMap::const_iterator vmi;
1133 VectorHash::const_iterator i;
1134 for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
1135 out <<
" " << (*vmi).first << endl;
1136 for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
1138 out <<
" " << hash.
as_dec() << endl;
A class to retrieve the individual data elements previously stored in a Datagram.
uint32_t get_uint32()
Extracts an unsigned 32-bit integer.
int32_t get_int32()
Extracts a signed 32-bit integer.
An ordered list of data elements, formatted in memory for transmission over a socket or writing to a ...
void add_uint32(uint32_t value)
Adds an unsigned 32-bit integer to the datagram.
size_t get_length() const
Returns the number of bytes in the datagram.
void add_int32(int32_t value)
Adds a signed 32-bit integer to the datagram.
const void * get_data() const
Returns a pointer to the beginning of the datagram's data.
int parse_header(Datagram dg)
Verifies magic number, returns the number of multifiles or -1 if invalid.
bool write_bogus_header(StreamWriter &sw)
Writes the bogus header uncompressed with platform- independent byte ordering.
bool write_header(std::ostream &write_stream)
Writes the header uncompressed with platform- independent byte ordering.
int parse_record_header(Datagram dg)
Parses a file record (fr) header and returns the length of the next file record.
const HashVal & get_hash(const Filename &name, int version) const
Returns the MD5 hash associated with the indicated version of the indicated file.
void set_server_multifile_hash(std::string mfname, HashVal val)
Set the hash value of file we are working on.
void create_new_server_db()
Used on the server side makefiles to create a new clean server db.
HashVal get_client_multifile_hash(std::string mfname) const
Return the hash value of the file we are working on.
bool client_multifile_complete(std::string mfname) const
A multifile is complete when it is completely downloaded.
int get_version(const Filename &name, const HashVal &hash) const
Returns the version number of this particular file, determined by looking up the hash generated from ...
bool has_version(const Filename &name) const
Returns true if the indicated file has version information, false otherwise.
void insert_new_version(const Filename &name, const HashVal &hash)
Inserts a new version 1 copy of the file, sliding all the other versions up by one.
HashVal get_server_multifile_hash(std::string mfname) const
Return the hash value of the server file.
DownloadDb()
Primarily used for testing.
void set_client_multifile_hash(std::string mfname, HashVal val)
Set the hash value of file we are working on.
void set_num_versions(const Filename &name, int num_versions)
Reduces the number of versions of a particular file stored in the ddb by throwing away all versions h...
void add_version(const Filename &name, const HashVal &hash, int version)
Appends a new version of the file onto the end of the list, or changes the hash associated with a ver...
int get_num_versions(const Filename &name) const
Returns the number of versions stored for the indicated file.
The name of a file, such as a texture file or an Egg file.
void set_binary()
Indicates that the filename represents a binary file.
bool open_write(std::ofstream &stream, bool truncate=true) const
Opens the indicated ifstream for writing the file, if possible.
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer.
std::string as_dec() const
Returns the HashVal as a string with four decimal numbers.
An in-memory buffer specifically designed for downloading files to memory.
A class to read sequential binary data directly from an istream.
std::string get_string32()
Extracts a variable-length string with a 32-bit length field.
int32_t get_int32()
Extracts a signed 32-bit integer.
size_t extract_bytes(unsigned char *into, size_t size)
Extracts the indicated number of bytes in the stream into the given character buffer.
get_istream
Returns the stream in use.
A StreamWriter object is used to write sequential binary data directly to an ostream.
void append_data(const void *data, size_t size)
Appends some more raw data to the end of the streamWriter.
void add_float64(PN_float64 value)
Adds a 64-bit floating-point number to the stream.
void add_int32(int32_t value)
Adds a signed 32-bit integer to the stream.
void add_uint32(uint32_t value)
Adds an unsigned 32-bit integer to the stream.
A hierarchy of directories and files that appears to be one continuous file system,...
static void close_read_file(std::istream *stream)
Closes a file opened by a previous call to open_read_file().
std::istream * open_read_file(const Filename &filename, bool auto_unwrap) const
Convenience function; returns a newly allocated istream if the file exists and can be read,...
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
This is our own Panda specialization on the default STL vector.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PT(DownloadDb::FileRecord) DownloadDb
Parses a file record (fr) and returns one.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.