15 #include "config_downloader.h" 16 #include "downloadDb.h" 17 #include "streamReader.h" 18 #include "streamWriter.h" 20 #include "virtualFileSystem.h" 29 PN_uint32 DownloadDb::_magic_number = 0xfeedfeed;
35 PN_uint32 DownloadDb::_bogus_magic_number = 0x11111111;
39 back_to_front_slash(
const string &str) {
42 for (si = result.begin(); si != result.end(); ++si) {
59 if (downloader_cat.is_debug())
60 downloader_cat.debug()
61 <<
"DownloadDb constructor called" << endl;
62 _client_db = read_db(client_file, 0);
63 _client_db._filename = client_file;
64 _server_db = read_db(server_file, 1);
74 if (downloader_cat.is_debug())
75 downloader_cat.debug()
76 <<
"DownloadDb constructor called" << endl;
77 _client_db = read_db(client_file, 0);
78 _client_db._filename = client_file;
79 _server_db = read_db(server_file, 1);
80 _server_db._filename = server_file;
101 if (downloader_cat.is_debug())
102 downloader_cat.debug()
103 <<
"DownloadDb destructor called" << endl;
113 output(ostream &out)
const {
114 out <<
"[" << _server_db._filename <<
" " << _client_db._filename <<
"]";
123 write(ostream &out)
const {
124 out <<
"DownloadDb" << endl;
125 out <<
"============================================================" << endl;
126 out <<
" Client DB file: " << _client_db._filename << endl;
127 out <<
"============================================================" << endl;
128 _client_db.write(out);
130 out <<
"============================================================" << endl;
131 out <<
" Server DB file: " << _server_db._filename << endl;
132 out <<
"============================================================" << endl;
133 _server_db.write(out);
134 write_version_map(out);
146 return write_db(file, _client_db, 0);
157 return write_db(file, _server_db, 1);
166 client_multifile_exists(
string mfname)
const {
167 return (_client_db.multifile_exists(mfname));
179 int client_status = _client_db.get_multifile_record_named(mfname)->_status;
180 return (client_status >= Status_complete);
189 client_multifile_decompressed(
string mfname)
const {
190 int client_status = _client_db.get_multifile_record_named(mfname)->_status;
191 return (client_status >= Status_decompressed);
200 client_multifile_extracted(
string mfname)
const {
201 int client_status = _client_db.get_multifile_record_named(mfname)->_status;
202 return (client_status >= Status_extracted);
213 return _client_db.get_multifile_record_named(mfname)->_hash;
224 return _server_db.get_multifile_record_named(mfname)->_hash;
235 _client_db.get_multifile_record_named(mfname)->_hash = val;
236 write_client_db(_client_db._filename);
247 _server_db.get_multifile_record_named(mfname)->_hash = val;
258 delete_client_multifile(
string mfname) {
267 add_client_multifile(
string server_mfname) {
268 PT(
MultifileRecord) server_mfr = _server_db.get_multifile_record_named(server_mfname);
270 client_mfr->_name = server_mfr->_name;
271 client_mfr->_phase = server_mfr->_phase;
272 _client_db.add_multifile_record(client_mfr);
282 expand_client_multifile(
string mfname) {
292 read_db(
Filename &file,
bool want_server_info) {
300 if (read_stream == (istream *)NULL) {
301 downloader_cat.error()
302 <<
"failed to open input file: " 308 if (!db.read(sr, want_server_info)) {
309 downloader_cat.error()
316 if (want_server_info) {
317 if (!read_version_map(sr)) {
318 downloader_cat.error()
319 <<
"read_version_map() failed: " 335 read_db(
Ramfile &file,
bool want_server_info) {
342 if (!db.read(sr, want_server_info)) {
343 downloader_cat.error()
344 <<
"read failed" << endl;
347 if (want_server_info) {
348 if (!read_version_map(sr)) {
349 downloader_cat.error()
350 <<
"read_version_map() failed" << endl;
364 write_db(
Filename &file,
Db db,
bool want_server_info) {
365 pofstream write_stream;
368 downloader_cat.error()
369 <<
"DownloadDb::write_db() - Failed to open output file: " 374 downloader_cat.spam()
375 <<
"Writing to file: " << file << endl;
381 db.write(sw, want_server_info);
382 if (want_server_info) {
383 write_version_map(sw);
387 write_stream.close();
410 server_add_multifile(
string mfname, Phase phase,
int size,
int status) {
412 _server_db.add_multifile_record(mfr);
422 server_add_file(
string mfname,
string fname) {
428 for (; i != _server_db._mfile_records.end(); ++i) {
429 if (mfname == (*i)->_name) {
430 (*i)->add_file_record(fr);
436 downloader_cat.error() <<
"Could not find record named " 437 << mfname <<
" in database " << endl;
453 DownloadDb::MultifileRecord::
458 _status = Status_incomplete;
467 DownloadDb::MultifileRecord::
468 MultifileRecord(
string name, Phase phase,
int size,
int status) {
481 void DownloadDb::MultifileRecord::
482 write(ostream &out)
const {
483 out <<
"==================================================" << endl;
484 out <<
"MultifileRecord: " << _name << endl
485 <<
" phase: " << _phase << endl
486 <<
" size: " << _size << endl
487 <<
" status: " << _status << endl
488 <<
" hash: " << _hash.as_dec() << endl;
489 out <<
"--------------------------------------------------" << endl;
491 for(; i != _file_records.end(); ++i) {
503 int DownloadDb::MultifileRecord::
504 get_num_files()
const {
505 return _file_records.size();
513 string DownloadDb::MultifileRecord::
514 get_file_name(
int index)
const {
515 return _file_records[index]->_name;
524 bool DownloadDb::MultifileRecord::
525 file_exists(
string fname)
const {
527 for(; i != _file_records.end(); ++i) {
528 if (fname == (*i)->_name) {
542 get_file_record_named(
string fname)
const {
544 for(; i != _file_records.end(); ++i) {
545 if (fname == (*i)->_name) {
550 downloader_cat.error() <<
"Could not find record named " 551 << fname <<
" in multifile " << _name << endl;
553 nassertr(
false, foo);
563 void DownloadDb::MultifileRecord::
565 _file_records.push_back(fr);
585 _header_length =
sizeof(_magic_number) +
sizeof(PN_int32);
594 void DownloadDb::Db::
595 write(ostream &out)
const {
597 for(; i != _mfile_records.end(); ++i) {
609 get_num_multifiles()
const {
610 return _mfile_records.size();
618 string DownloadDb::Db::
619 get_multifile_name(
int index)
const {
620 return _mfile_records[index]->_name;
628 bool DownloadDb::Db::
629 multifile_exists(
string mfname)
const {
631 for(; i != _mfile_records.end(); ++i) {
632 if (mfname == (*i)->_name) {
645 get_multifile_record_named(
string mfname)
const {
647 for(; i != _mfile_records.end(); ++i) {
648 if (mfname == (*i)->_name) {
653 downloader_cat.error() <<
"Could not find record named " 654 << mfname <<
" in database " << endl;
656 nassertr(
false, foo);
665 void DownloadDb::Db::
667 _mfile_records.push_back(mfr);
684 downloader_cat.debug()
685 <<
"Parsed magic number: " << magic_number << endl;
688 if (magic_number == _bogus_magic_number) {
689 downloader_cat.error()
690 <<
"DownloadDb::parse_header() - " 691 <<
"Bogus magic number, previous write incomplete: " 692 << magic_number <<
" expected: " << _magic_number << endl;
697 else if (magic_number != _magic_number) {
698 downloader_cat.error()
699 <<
"DownloadDb::parse_header() - Invalid magic number: " 700 << magic_number <<
" expected: " << _magic_number << endl;
704 PN_int32 num_multifiles = di.
get_int32();
705 downloader_cat.debug()
706 <<
"Parsed number of multifiles: " << num_multifiles << endl;
709 return num_multifiles;
725 downloader_cat.spam()
726 <<
"Parsed record header length: " << record_length << endl;
729 return record_length;
739 parse_mfr(
const string &data) {
745 PN_int32 mfr_name_length = di.
get_int32();
755 mfr->_name = back_to_front_slash(mfr->_name);
758 mfr->_hash.read_datagram(di);
760 downloader_cat.debug()
761 <<
"Parsed multifile record: " << mfr->_name <<
" phase: " << mfr->_phase
762 <<
" size: " << mfr->_size
763 <<
" status: " << mfr->_status <<
" num_files: " << mfr->_num_files << endl;
778 parse_fr(
const string &data) {
784 PN_int32 fr_name_length = di.
get_int32();
790 fr->_name = back_to_front_slash(fr->_name);
792 downloader_cat.spam()
793 <<
"Parsed file record: " << fr->_name << endl;
807 bool DownloadDb::Db::
812 if (header.size() != (size_t)_header_length) {
813 downloader_cat.error() <<
"truncated db file" << endl;
818 int num_multifiles = parse_header(header);
819 if (num_multifiles < 0) {
820 downloader_cat.error() <<
"invalid db header" << endl;
826 for (
int i = 0; i < num_multifiles; i++) {
829 int mfr_header_length =
sizeof(PN_int32);
832 if (mfr_header.size() != (size_t)mfr_header_length) {
833 downloader_cat.error() <<
"invalid mfr header" << endl;
838 int mfr_length = parse_record_header(mfr_header);
843 int read_length = (mfr_length - mfr_header_length);
845 if (mfr_record.size() != (size_t)read_length) {
846 downloader_cat.error() <<
"invalid mfr record" << endl;
854 if (want_server_info) {
857 for (
int j = 0; j < mfr->_num_files; j++) {
860 int fr_header_length =
sizeof(PN_int32);
864 if (fr_header.size() != (size_t)fr_header_length) {
865 downloader_cat.error() <<
"invalid fr header" << endl;
870 int fr_length = parse_record_header(fr_header);
874 int read_length = (fr_length - fr_header_length);
877 if (fr_record.size() != (size_t)read_length) {
878 downloader_cat.error() <<
"invalid fr record" << endl;
886 mfr->add_file_record(fr);
891 add_multifile_record(mfr);
904 bool DownloadDb::Db::
912 PN_int32 name_length;
913 PN_int32 header_length;
917 for(; i != _mfile_records.end(); ++i) {
919 phase = (*i)->_phase;
921 status = (*i)->_status;
922 num_files = (*i)->get_num_files();
923 name_length = (*i)->_name.length();
927 sizeof(header_length) +
928 sizeof(name_length) +
929 (*i)->_name.length() +
930 sizeof(phase) +
sizeof(size) +
931 sizeof(status) +
sizeof(num_files) +
948 (*i)->_hash.write_stream(sw);
951 if (want_server_info) {
955 for(; j != (*i)->_file_records.end(); ++j) {
956 name_length = (*j)->_name.length();
960 sizeof(header_length) +
961 sizeof(name_length) +
962 (*j)->_name.length();
1016 write_stream.seekp(0);
1018 write_stream.write(msg.data(), msg.length());
1034 DownloadDb::FileRecord::
1045 DownloadDb::FileRecord::
1046 FileRecord(
string name) {
1055 void DownloadDb::FileRecord::
1056 write(ostream &out)
const {
1057 out <<
" FileRecord: " << _name << endl;
1071 nassertv(version >= 1);
1074 int size = vhash.size();
1077 nassertv(version <= size+1);
1079 if (version-1 < size) {
1081 vhash[version-1] = hash;
1085 vhash.push_back(hash);
1098 vhash.insert(vhash.begin(), hash);
1110 return (_versions.find(name) != _versions.end());
1121 VersionMap::const_iterator vmi = _versions.find(name);
1122 if (vmi == _versions.end()) {
1126 return (
int)(*vmi).second.size();
1138 VersionMap::iterator vmi = _versions.find(name);
1139 if (vmi == _versions.end()) {
1140 nassertv(num_versions == 0);
1146 nassertv(num_versions <= (
int)vhash.size());
1147 vhash.erase(vhash.begin() + num_versions, vhash.end());
1160 VersionMap::const_iterator vmi = _versions.find(name);
1161 if (vmi == _versions.end()) {
1162 downloader_cat.debug()
1163 <<
"DownloadDb::get_version() - can't find: " << name << endl;
1167 VectorHash::const_iterator i = find(vhash.begin(), vhash.end(), hash);
1168 if (i != vhash.end())
1169 return (i - vhash.begin() + 1);
1170 downloader_cat.debug()
1171 <<
"DownloadDb::get_version() - can't find hash: " << hash << endl;
1185 VersionMap::const_iterator vmi = _versions.find(name);
1186 if (vmi == _versions.end()) {
1187 downloader_cat.error()
1188 <<
"DownloadDb::get_hash() - can't find: " << name << endl;
1193 if (version < 1 || version > (
int)vhash.size()) {
1194 downloader_cat.error()
1195 <<
"DownloadDb::get_hash() - no version " << version
1196 <<
" for " << name << endl;
1199 return vhash[version - 1];
1209 VersionMap::iterator vmi;
1210 VectorHash::iterator i;
1214 for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
1215 name = (*vmi).first;
1216 downloader_cat.spam()
1217 <<
"DownloadDb::write_version_map() - writing file: " 1218 << name <<
" of length: " << name.length() << endl;
1222 for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
1224 (*i).write_stream(sw);
1241 for (
int i = 0; i < num_entries; i++) {
1248 downloader_cat.spam()
1249 <<
"DownloadDb::read_version_map() - name length: " << name_length
1254 downloader_cat.spam()
1255 <<
"DownloadDb::read_version_map() - name: " << name << endl;
1262 downloader_cat.spam()
1263 <<
"DownloadDb::read_version_map() - number of values: " << length
1266 for (
int j = 0; j < length; j++) {
1268 hash.read_stream(sr);
1284 write_version_map(ostream &out)
const {
1285 out <<
"Version Map: " << endl;
1286 VersionMap::const_iterator vmi;
1287 VectorHash::const_iterator i;
1288 for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
1289 out <<
" " << (*vmi).first << endl;
1290 for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
1292 out <<
" " << hash.
as_dec() << endl;
A StreamWriter object is used to write sequential binary data directly to an ostream.
HashVal get_client_multifile_hash(string mfname) const
Return the hash value of the file we are working on.
void append_data(const void *data, size_t size)
Appends some more raw data to the end of the streamWriter.
void set_server_multifile_hash(string mfname, HashVal val)
Set the hash value of file we are working on.
bool open_write(ofstream &stream, bool truncate=true) const
Opens the indicated ifstream for writing the file, if possible.
bool client_multifile_complete(string mfname) const
A multifile is complete when it is completely downloaded.
void insert_new_version(const Filename &name, const HashVal &hash)
Inserts a new version 1 copy of the file, sliding all the other versions up by one.
int get_num_versions(const Filename &name) const
Returns the number of versions stored for the indicated file.
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS's file system.
istream * open_read_file(const Filename &filename, bool auto_unwrap) const
Convenience function; returns a newly allocated istream if the file exists and can be read...
string extract_bytes(size_t size)
Extracts the indicated number of bytes in the stream and returns them as a string.
DownloadDb()
Primarily used for testing.
void set_binary()
Indicates that the filename represents a binary file.
istream * get_istream() const
Returns the stream in use.
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer...
PN_int32 get_int32()
Extracts a signed 32-bit integer.
PN_uint32 get_uint32()
Extracts an unsigned 32-bit integer.
void create_new_server_db()
Used on the server side makefiles to create a new clean server db.
static void close_read_file(istream *stream)
Closes a file opened by a previous call to open_read_file().
This is our own Panda specialization on the default STL vector.
string extract_bytes(size_t size)
Extracts the indicated number of bytes in the datagram and returns them as a string.
The name of a file, such as a texture file or an Egg file.
void add_version(const Filename &name, const HashVal &hash, int version)
Appends a new version of the file onto the end of the list, or changes the hash associated with a ver...
An in-memory buffer specifically designed for downloading files to memory.
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
bool write_header(ostream &write_stream)
Writes the header uncompressed with platform- independent byte ordering.
void add_uint32(PN_uint32 value)
Adds an unsigned 32-bit integer to the stream.
const HashVal & get_hash(const Filename &name, int version) const
Returns the MD5 hash associated with the indicated version of the indicated file. ...
void set_num_versions(const Filename &name, int num_versions)
Reduces the number of versions of a particular file stored in the ddb by throwing away all versions h...
bool write_bogus_header(StreamWriter &sw)
Writes the bogus header uncompressed with platform- independent byte ordering.
PN_float64 get_float64()
Extracts a 64-bit floating-point number.
void add_uint32(PN_uint32 value)
Adds an unsigned 32-bit integer to the datagram.
int parse_header(const string &data)
Verifies magic number, returns the number of multifiles or -1 if invalid.
void add_float64(PN_float64 value)
Adds a 64-bit floating-point number to the stream.
void add_int32(PN_int32 value)
Adds a signed 32-bit integer to the datagram.
A class to retrieve the individual data elements previously stored in a Datagram. ...
PN_int32 get_int32()
Extracts a signed 32-bit integer.
int get_version(const Filename &name, const HashVal &hash) const
Returns the version number of this particular file, determined by looking up the hash generated from ...
void add_int32(PN_int32 value)
Adds a signed 32-bit integer to the stream.
string get_message() const
Returns the datagram's data as a string.
A class to read sequential binary data directly from an istream.
An ordered list of data elements, formatted in memory for transmission over a socket or writing to a ...
int parse_record_header(const string &data)
Parses a file record (fr) header and returns the length of the next file record.
HashVal get_server_multifile_hash(string mfname) const
Return the hash value of the server file.
void set_client_multifile_hash(string mfname, HashVal val)
Set the hash value of file we are working on.
string as_dec() const
Returns the HashVal as a string with four decimal numbers.
bool has_version(const Filename &name) const
Returns true if the indicated file has version information, false otherwise.