Panda3D
 All Classes Functions Variables Enumerations
downloadDb.cxx
1 // Filename: downloadDb.cxx
2 // Created by: shochet (08Sep00)
3 //
4 ////////////////////////////////////////////////////////////////////
5 //
6 // PANDA 3D SOFTWARE
7 // Copyright (c) Carnegie Mellon University. All rights reserved.
8 //
9 // All use of this software is subject to the terms of the revised BSD
10 // license. You should have received a copy of this license along
11 // with this source code in a file named "LICENSE."
12 //
13 ////////////////////////////////////////////////////////////////////
14 
15 #include "config_downloader.h"
16 #include "downloadDb.h"
17 #include "streamReader.h"
18 #include "streamWriter.h"
19 #include "ramfile.h"
20 #include "virtualFileSystem.h"
21 
22 #include <algorithm>
23 
24 ////////////////////////////////////////////////////////////////////
25 // Defines
26 ////////////////////////////////////////////////////////////////////
27 
28 // Written at the top of the file so we know this is a downloadDb
29 PN_uint32 DownloadDb::_magic_number = 0xfeedfeed;
30 
31 // Written at the top of the file to signify we are not done
32 // writing to the file yet. If you load a db with this magic
33 // number that means the previous time it got written out was
34 // probably interrupted in the middle of the write.
35 PN_uint32 DownloadDb::_bogus_magic_number = 0x11111111;
36 
37 
38 static string
39 back_to_front_slash(const string &str) {
40  string result = str;
41  string::iterator si;
42  for (si = result.begin(); si != result.end(); ++si) {
43  if ((*si) == '\\') {
44  (*si) = '/';
45  }
46  }
47 
48  return result;
49 }
50 
51 
52 ////////////////////////////////////////////////////////////////////
53 // Function: DownloadDb::Constructor
54 // Access: Public
55 // Description: Create a download db with these client and server dbs
56 ////////////////////////////////////////////////////////////////////
58 DownloadDb(Ramfile &server_file, Filename &client_file) {
59  if (downloader_cat.is_debug())
60  downloader_cat.debug()
61  << "DownloadDb constructor called" << endl;
62  _client_db = read_db(client_file, 0);
63  _client_db._filename = client_file;
64  _server_db = read_db(server_file, 1);
65 }
66 
67 ////////////////////////////////////////////////////////////////////
68 // Function: DownloadDb::Constructor
69 // Access: Public
70 // Description: Create a download db with these client and server dbs
71 ////////////////////////////////////////////////////////////////////
73 DownloadDb(Filename &server_file, Filename &client_file) {
74  if (downloader_cat.is_debug())
75  downloader_cat.debug()
76  << "DownloadDb constructor called" << endl;
77  _client_db = read_db(client_file, 0);
78  _client_db._filename = client_file;
79  _server_db = read_db(server_file, 1);
80  _server_db._filename = server_file;
81 }
82 
83 ////////////////////////////////////////////////////////////////////
84 // Function: DownloadDb::Constructor
85 // Access: Public
86 // Description: Primarily used for testing.
87 ////////////////////////////////////////////////////////////////////
90  _client_db = Db();
91  _server_db = Db();
92 }
93 
94 ////////////////////////////////////////////////////////////////////
95 // Function: DownloadDb::Destructor
96 // Access: Public
97 // Description:
98 ////////////////////////////////////////////////////////////////////
99 DownloadDb::
100 ~DownloadDb() {
101  if (downloader_cat.is_debug())
102  downloader_cat.debug()
103  << "DownloadDb destructor called" << endl;
104 }
105 
106 
107 ////////////////////////////////////////////////////////////////////
108 // Function: DownloadDb::output
109 // Access: Public
110 // Description:
111 ////////////////////////////////////////////////////////////////////
112 void DownloadDb::
113 output(ostream &out) const {
114  out << "[" << _server_db._filename << " " << _client_db._filename << "]";
115 }
116 
117 ////////////////////////////////////////////////////////////////////
118 // Function: DownloadDb::write
119 // Access: Public
120 // Description:
121 ////////////////////////////////////////////////////////////////////
122 void DownloadDb::
123 write(ostream &out) const {
124  out << "DownloadDb" << endl;
125  out << "============================================================" << endl;
126  out << " Client DB file: " << _client_db._filename << endl;
127  out << "============================================================" << endl;
128  _client_db.write(out);
129  out << endl;
130  out << "============================================================" << endl;
131  out << " Server DB file: " << _server_db._filename << endl;
132  out << "============================================================" << endl;
133  _server_db.write(out);
134  write_version_map(out);
135  out << endl;
136 }
137 
138 
139 ////////////////////////////////////////////////////////////////////
140 // Function: DownloadDb::
141 // Access: Public
142 // Description:
143 ////////////////////////////////////////////////////////////////////
144 bool DownloadDb::
145 write_client_db(Filename &file) {
146  return write_db(file, _client_db, 0);
147 }
148 
149 
150 ////////////////////////////////////////////////////////////////////
151 // Function: DownloadDb::
152 // Access: Public
153 // Description:
154 ////////////////////////////////////////////////////////////////////
155 bool DownloadDb::
156 write_server_db(Filename &file) {
157  return write_db(file, _server_db, 1);
158 }
159 
160 ////////////////////////////////////////////////////////////////////
161 // Function: DownloadDb::
162 // Access: Public
163 // Description:
164 ////////////////////////////////////////////////////////////////////
165 bool DownloadDb::
166 client_multifile_exists(string mfname) const {
167  return (_client_db.multifile_exists(mfname));
168 }
169 
170 ////////////////////////////////////////////////////////////////////
171 // Function: DownloadDb::
172 // Access: Public
173 // Description: A multifile is complete when it is completely
174 // downloaded. Note: it may already be decompressed
175 // or extracted and it is still complete
176 ////////////////////////////////////////////////////////////////////
177 bool DownloadDb::
178 client_multifile_complete(string mfname) const {
179  int client_status = _client_db.get_multifile_record_named(mfname)->_status;
180  return (client_status >= Status_complete);
181 }
182 
183 ////////////////////////////////////////////////////////////////////
184 // Function: DownloadDb::
185 // Access: Public
186 // Description:
187 ////////////////////////////////////////////////////////////////////
188 bool DownloadDb::
189 client_multifile_decompressed(string mfname) const {
190  int client_status = _client_db.get_multifile_record_named(mfname)->_status;
191  return (client_status >= Status_decompressed);
192 }
193 
194 ////////////////////////////////////////////////////////////////////
195 // Function: DownloadDb::
196 // Access: Public
197 // Description:
198 ////////////////////////////////////////////////////////////////////
199 bool DownloadDb::
200 client_multifile_extracted(string mfname) const {
201  int client_status = _client_db.get_multifile_record_named(mfname)->_status;
202  return (client_status >= Status_extracted);
203 }
204 
205 
206 ////////////////////////////////////////////////////////////////////
207 // Function: DownloadDb::
208 // Access: Public
209 // Description: Return the hash value of the file we are working on
210 ////////////////////////////////////////////////////////////////////
212 get_client_multifile_hash(string mfname) const {
213  return _client_db.get_multifile_record_named(mfname)->_hash;
214 }
215 
216 
217 ////////////////////////////////////////////////////////////////////
218 // Function: DownloadDb::
219 // Access: Public
220 // Description: Return the hash value of the server file
221 ////////////////////////////////////////////////////////////////////
223 get_server_multifile_hash(string mfname) const {
224  return _server_db.get_multifile_record_named(mfname)->_hash;
225 }
226 
227 
228 ////////////////////////////////////////////////////////////////////
229 // Function: DownloadDb::
230 // Access: Public
231 // Description: Set the hash value of file we are working on
232 ////////////////////////////////////////////////////////////////////
233 void DownloadDb::
234 set_client_multifile_hash(string mfname, HashVal val) {
235  _client_db.get_multifile_record_named(mfname)->_hash = val;
236  write_client_db(_client_db._filename);
237 }
238 
239 
240 ////////////////////////////////////////////////////////////////////
241 // Function: DownloadDb::
242 // Access: Public
243 // Description: Set the hash value of file we are working on
244 ////////////////////////////////////////////////////////////////////
245 void DownloadDb::
246 set_server_multifile_hash(string mfname, HashVal val) {
247  _server_db.get_multifile_record_named(mfname)->_hash = val;
248 }
249 
250 // Operations on multifiles
251 
252 ////////////////////////////////////////////////////////////////////
253 // Function: DownloadDb::
254 // Access: Public
255 // Description:
256 ////////////////////////////////////////////////////////////////////
257 void DownloadDb::
258 delete_client_multifile(string mfname) {
259 }
260 
261 ////////////////////////////////////////////////////////////////////
262 // Function: DownloadDb::
263 // Access: Public
264 // Description:
265 ////////////////////////////////////////////////////////////////////
266 void DownloadDb::
267 add_client_multifile(string server_mfname) {
268  PT(MultifileRecord) server_mfr = _server_db.get_multifile_record_named(server_mfname);
269  PT(MultifileRecord) client_mfr = new MultifileRecord;
270  client_mfr->_name = server_mfr->_name;
271  client_mfr->_phase = server_mfr->_phase;
272  _client_db.add_multifile_record(client_mfr);
273 }
274 
275 
276 ////////////////////////////////////////////////////////////////////
277 // Function: DownloadDb::
278 // Access: Public
279 // Description:
280 ////////////////////////////////////////////////////////////////////
281 void DownloadDb::
282 expand_client_multifile(string mfname) {
283 }
284 
285 
286 ////////////////////////////////////////////////////////////////////
287 // Function: DownloadDb::read_db
288 // Access: Published
289 // Description:
290 ////////////////////////////////////////////////////////////////////
291 DownloadDb::Db DownloadDb::
292 read_db(Filename &file, bool want_server_info) {
294  Db db;
295 
296  // Open the multifile for reading
297  file.set_binary();
298  istream *read_stream = vfs->open_read_file(file, true);
299 
300  if (read_stream == (istream *)NULL) {
301  downloader_cat.error()
302  << "failed to open input file: "
303  << file << endl;
304  return db;
305  }
306 
307  StreamReader sr(*read_stream);
308  if (!db.read(sr, want_server_info)) {
309  downloader_cat.error()
310  << "read failed: "
311  << file << endl;
312  vfs->close_read_file(read_stream);
313  return db;
314  }
315 
316  if (want_server_info) {
317  if (!read_version_map(sr)) {
318  downloader_cat.error()
319  << "read_version_map() failed: "
320  << file << endl;
321  }
322  }
323 
324  vfs->close_read_file(read_stream);
325 
326  return db;
327 }
328 
329 ////////////////////////////////////////////////////////////////////
330 // Function: DownloadDb::read_db
331 // Access: Published
332 // Description:
333 ////////////////////////////////////////////////////////////////////
334 DownloadDb::Db DownloadDb::
335 read_db(Ramfile &file, bool want_server_info) {
336  // Open the multifile for reading
337  istringstream read_stream(file._data);
338  Db db;
339 
340  StreamReader sr(read_stream);
341 
342  if (!db.read(sr, want_server_info)) {
343  downloader_cat.error()
344  << "read failed" << endl;
345  return db;
346  }
347  if (want_server_info) {
348  if (!read_version_map(sr)) {
349  downloader_cat.error()
350  << "read_version_map() failed" << endl;
351  }
352  }
353 
354  return db;
355 }
356 
357 
358 ////////////////////////////////////////////////////////////////////
359 // Function: DownloadDb::write_db
360 // Access: Published
361 // Description:
362 ////////////////////////////////////////////////////////////////////
363 bool DownloadDb::
364 write_db(Filename &file, Db db, bool want_server_info) {
365  pofstream write_stream;
366  file.set_binary();
367  if (!file.open_write(write_stream)) {
368  downloader_cat.error()
369  << "DownloadDb::write_db() - Failed to open output file: "
370  << file << endl;
371  return false;
372  }
373 
374  downloader_cat.spam()
375  << "Writing to file: " << file << endl;
376 
377  StreamWriter sw(write_stream);
378 
379  // Initialize the write stream with a bogus header
380  db.write_bogus_header(sw);
381  db.write(sw, want_server_info);
382  if (want_server_info) {
383  write_version_map(sw);
384  }
385  // Now write the real header
386  db.write_header(write_stream);
387  write_stream.close();
388  return true;
389 }
390 
391 
392 ////////////////////////////////////////////////////////////////////
393 // Function: DownloadDb::create_new_server_db
394 // Access: Public
395 // Description: Used on the server side makefiles to create a
396 // new clean server db
397 ////////////////////////////////////////////////////////////////////
398 void DownloadDb::
400  _server_db = Db();
401 }
402 
403 
404 ////////////////////////////////////////////////////////////////////
405 // Function: DownloadDb::
406 // Access: Public
407 // Description:
408 ////////////////////////////////////////////////////////////////////
409 void DownloadDb::
410 server_add_multifile(string mfname, Phase phase, int size, int status) {
411  PT(MultifileRecord) mfr = new MultifileRecord(mfname, phase, size, status);
412  _server_db.add_multifile_record(mfr);
413 }
414 
415 
416 ////////////////////////////////////////////////////////////////////
417 // Function: DownloadDb::
418 // Access: Public
419 // Description:
420 ////////////////////////////////////////////////////////////////////
421 void DownloadDb::
422 server_add_file(string mfname, string fname) {
423  // Make the new file record
424  PT(FileRecord) fr = new FileRecord(fname);
425 
426  // Find the multifile with mfname
427  pvector< PT(MultifileRecord) >::iterator i = _server_db._mfile_records.begin();
428  for (; i != _server_db._mfile_records.end(); ++i) {
429  if (mfname == (*i)->_name) {
430  (*i)->add_file_record(fr);
431  return;
432  }
433  }
434 
435  // Uh-oh, did not find it
436  downloader_cat.error() << "Could not find record named "
437  << mfname << " in database " << endl;
438  nassertv(false);
439  return;
440 }
441 
442 
443 ////////////////////////////////////////////////////////////////////
444 // Multifile methods
445 ////////////////////////////////////////////////////////////////////
446 
447 
448 ////////////////////////////////////////////////////////////////////
449 // Function: DownloadDb::MultifileRecord::Constructor
450 // Access: Public
451 // Description:
452 ////////////////////////////////////////////////////////////////////
453 DownloadDb::MultifileRecord::
454 MultifileRecord() {
455  _name = "";
456  _phase = 0;
457  _size = 0;
458  _status = Status_incomplete;
459 }
460 
461 
462 ////////////////////////////////////////////////////////////////////
463 // Function: DownloadDb::MultifileRecord::Constructor
464 // Access: Public
465 // Description:
466 ////////////////////////////////////////////////////////////////////
467 DownloadDb::MultifileRecord::
468 MultifileRecord(string name, Phase phase, int size, int status) {
469  _name = name;
470  _phase = phase;
471  _size = size;
472  _status = status;
473 }
474 
475 
476 ////////////////////////////////////////////////////////////////////
477 // Function: DownloadDb::MultifileRecord::write
478 // Access: Public
479 // Description:
480 ////////////////////////////////////////////////////////////////////
481 void DownloadDb::MultifileRecord::
482 write(ostream &out) const {
483  out << "==================================================" << endl;
484  out << "MultifileRecord: " << _name << endl
485  << " phase: " << _phase << endl
486  << " size: " << _size << endl
487  << " status: " << _status << endl
488  << " hash: " << _hash.as_dec() << endl;
489  out << "--------------------------------------------------" << endl;
490  pvector< PT(FileRecord) >::const_iterator i = _file_records.begin();
491  for(; i != _file_records.end(); ++i) {
492  (*i)->write(out);
493  }
494 }
495 
496 
497 
498 ////////////////////////////////////////////////////////////////////
499 // Function: DownloadDb::MultifileRecord::
500 // Access: Public
501 // Description:
502 ////////////////////////////////////////////////////////////////////
503 int DownloadDb::MultifileRecord::
504 get_num_files() const {
505  return _file_records.size();
506 }
507 
508 ////////////////////////////////////////////////////////////////////
509 // Function: DownloadDb::MultifileRecord::
510 // Access: Public
511 // Description:
512 ////////////////////////////////////////////////////////////////////
513 string DownloadDb::MultifileRecord::
514 get_file_name(int index) const {
515  return _file_records[index]->_name;
516 }
517 
518 
519 ////////////////////////////////////////////////////////////////////
520 // Function: DownloadDb::MultifileRecord::
521 // Access: Public
522 // Description:
523 ////////////////////////////////////////////////////////////////////
524 bool DownloadDb::MultifileRecord::
525 file_exists(string fname) const {
526  pvector< PT(FileRecord) >::const_iterator i = _file_records.begin();
527  for(; i != _file_records.end(); ++i) {
528  if (fname == (*i)->_name) {
529  return true;
530  }
531  }
532  return false;
533 }
534 
535 
536 ////////////////////////////////////////////////////////////////////
537 // Function: DownloadDb::MultifileRecord::
538 // Access: Public
539 // Description:
540 ////////////////////////////////////////////////////////////////////
541 PT(DownloadDb::FileRecord) DownloadDb::MultifileRecord::
542 get_file_record_named(string fname) const {
543  pvector< PT(FileRecord) >::const_iterator i = _file_records.begin();
544  for(; i != _file_records.end(); ++i) {
545  if (fname == (*i)->_name) {
546  return (*i);
547  }
548  }
549  // Did not find it, just return an empty version
550  downloader_cat.error() << "Could not find record named "
551  << fname << " in multifile " << _name << endl;
552  PT(FileRecord) foo = new FileRecord;
553  nassertr(false, foo);
554  return foo;
555 }
556 
557 
558 ////////////////////////////////////////////////////////////////////
559 // Function: DownloadDb::MultifileRecord::
560 // Access: Public
561 // Description:
562 ////////////////////////////////////////////////////////////////////
563 void DownloadDb::MultifileRecord::
564 add_file_record(PT(FileRecord) fr) {
565  _file_records.push_back(fr);
566 }
567 
568 
569 
570 ////////////////////////////////////////////////////////////////////
571 // Db methods
572 ////////////////////////////////////////////////////////////////////
573 
574 
575 
576 
577 ////////////////////////////////////////////////////////////////////
578 // Function: DownloadDb::Db::constructor
579 // Access: Public
580 // Description:
581 ////////////////////////////////////////////////////////////////////
582 DownloadDb::Db::
583 Db() {
584  // The head is a magic number and the number of multifiles in the db
585  _header_length = sizeof(_magic_number) + sizeof(PN_int32);
586 }
587 
588 
589 ////////////////////////////////////////////////////////////////////
590 // Function: DownloadDb::Db::output
591 // Access: Public
592 // Description:
593 ////////////////////////////////////////////////////////////////////
594 void DownloadDb::Db::
595 write(ostream &out) const {
596  pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
597  for(; i != _mfile_records.end(); ++i) {
598  (*i)->write(out);
599  }
600 }
601 
602 
603 ////////////////////////////////////////////////////////////////////
604 // Function: DownloadDb::Db::
605 // Access: Public
606 // Description:
607 ////////////////////////////////////////////////////////////////////
608 int DownloadDb::Db::
609 get_num_multifiles() const {
610  return _mfile_records.size();
611 }
612 
613 ////////////////////////////////////////////////////////////////////
614 // Function: DownloadDb::Db::
615 // Access: Public
616 // Description:
617 ////////////////////////////////////////////////////////////////////
618 string DownloadDb::Db::
619 get_multifile_name(int index) const {
620  return _mfile_records[index]->_name;
621 }
622 
623 ////////////////////////////////////////////////////////////////////
624 // Function: DownloadDb::Db::
625 // Access: Public
626 // Description:
627 ////////////////////////////////////////////////////////////////////
628 bool DownloadDb::Db::
629 multifile_exists(string mfname) const {
630  pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
631  for(; i != _mfile_records.end(); ++i) {
632  if (mfname == (*i)->_name) {
633  return true;
634  }
635  }
636  return false;
637 }
638 
639 ////////////////////////////////////////////////////////////////////
640 // Function: DownloadDb::Db::
641 // Access: Public
642 // Description:
643 ////////////////////////////////////////////////////////////////////
645 get_multifile_record_named(string mfname) const {
646  pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
647  for(; i != _mfile_records.end(); ++i) {
648  if (mfname == (*i)->_name) {
649  return (*i);
650  }
651  }
652  // Did not find it, just return an empty version
653  downloader_cat.error() << "Could not find record named "
654  << mfname << " in database " << endl;
655  PT(MultifileRecord) foo = new MultifileRecord;
656  nassertr(false, foo);
657  return foo;
658 }
659 
660 ////////////////////////////////////////////////////////////////////
661 // Function: DownloadDb::Db::
662 // Access: Public
663 // Description:
664 ////////////////////////////////////////////////////////////////////
665 void DownloadDb::Db::
666 add_multifile_record(PT(MultifileRecord) mfr) {
667  _mfile_records.push_back(mfr);
668 }
669 
670 
671 ////////////////////////////////////////////////////////////////////
672 // Function: DownloadDb::Db::parse_header
673 // Access: Private
674 // Description: Verifies magic number, returns the number of
675 // multifiles or -1 if invalid
676 ////////////////////////////////////////////////////////////////////
678 parse_header(const string &data) {
679  Datagram dg(data);
680 
681  // Make sure we have a good header
682  DatagramIterator di(dg);
683  PN_uint32 magic_number = di.get_uint32();
684  downloader_cat.debug()
685  << "Parsed magic number: " << magic_number << endl;
686  // If the magic number is equal to the bogus magic number
687  // it signifies that the previous write was interrupted
688  if (magic_number == _bogus_magic_number) {
689  downloader_cat.error()
690  << "DownloadDb::parse_header() - "
691  << "Bogus magic number, previous write incomplete: "
692  << magic_number << " expected: " << _magic_number << endl;
693  return -1;
694  }
695  // If the magic number does not match at all, something is
696  // really wrong
697  else if (magic_number != _magic_number) {
698  downloader_cat.error()
699  << "DownloadDb::parse_header() - Invalid magic number: "
700  << magic_number << " expected: " << _magic_number << endl;
701  return -1;
702  }
703 
704  PN_int32 num_multifiles = di.get_int32();
705  downloader_cat.debug()
706  << "Parsed number of multifiles: " << num_multifiles << endl;
707 
708  // If we got all the way here, must be done
709  return num_multifiles;
710 }
711 
712 
713 
714 ////////////////////////////////////////////////////////////////////
715 // Function: DownloadDb::Db::parse_fr_header
716 // Access: Private
717 // Description: Parses a file record (fr) header and returns
718 // the length of the next file record
719 ////////////////////////////////////////////////////////////////////
721 parse_record_header(const string &data) {
722  Datagram dg(data);
723  DatagramIterator di(dg);
724  PN_int32 record_length = di.get_int32();
725  downloader_cat.spam()
726  << "Parsed record header length: " << record_length << endl;
727 
728  // If we got all the way here, must be done
729  return record_length;
730 }
731 
732 
733 ////////////////////////////////////////////////////////////////////
734 // Function: DownloadDb::Db::parse_mfr
735 // Access: Private
736 // Description: Parses a multifile record (mfr) and returns one
737 ////////////////////////////////////////////////////////////////////
739 parse_mfr(const string &data) {
740 
741  PT(DownloadDb::MultifileRecord) mfr = new DownloadDb::MultifileRecord;
742 
743  Datagram dg(data);
744  DatagramIterator di(dg);
745  PN_int32 mfr_name_length = di.get_int32();
746  mfr->_name = di.extract_bytes(mfr_name_length);
747  mfr->_phase = di.get_float64();
748  mfr->_size = di.get_int32();
749  mfr->_status = di.get_int32();
750  mfr->_num_files = di.get_int32();
751 
752  // At one time, we stored files in the database with a backslash
753  // separator. Nowadays we use a forward slash, but we should make
754  // sure we properly convert any old records we might read.
755  mfr->_name = back_to_front_slash(mfr->_name);
756 
757  // Read the hash value
758  mfr->_hash.read_datagram(di);
759 
760  downloader_cat.debug()
761  << "Parsed multifile record: " << mfr->_name << " phase: " << mfr->_phase
762  << " size: " << mfr->_size
763  << " status: " << mfr->_status << " num_files: " << mfr->_num_files << endl;
764 
765  // Return the new MultifileRecord
766  return mfr;
767 }
768 
769 
770 
771 
772 ////////////////////////////////////////////////////////////////////
773 // Function: DownloadDb::Db::parse_fr
774 // Access: Private
775 // Description: Parses a file record (fr) and returns one
776 ////////////////////////////////////////////////////////////////////
777 PT(DownloadDb::FileRecord) DownloadDb::Db::
778 parse_fr(const string &data) {
779 
780  PT(DownloadDb::FileRecord) fr = new DownloadDb::FileRecord;
781 
782  Datagram dg(data);
783  DatagramIterator di(dg);
784  PN_int32 fr_name_length = di.get_int32();
785  fr->_name = di.extract_bytes(fr_name_length);
786 
787  // At one time, we stored files in the database with a backslash
788  // separator. Nowadays we use a forward slash, but we should make
789  // sure we properly convert any old records we might read.
790  fr->_name = back_to_front_slash(fr->_name);
791 
792  downloader_cat.spam()
793  << "Parsed file record: " << fr->_name << endl;
794 
795  // Return the new MultifileRecord
796  return fr;
797 }
798 
799 
800 
801 
802 ////////////////////////////////////////////////////////////////////
803 // Function: DownloadDb::Db::read
804 // Access: Private
805 // Description:
806 ////////////////////////////////////////////////////////////////////
807 bool DownloadDb::Db::
808 read(StreamReader &sr, bool want_server_info) {
809  // Read the header
810  string header;
811  header = sr.extract_bytes(_header_length);
812  if (header.size() != (size_t)_header_length) {
813  downloader_cat.error() << "truncated db file" << endl;
814  return false;
815  }
816 
817  // Parse the header
818  int num_multifiles = parse_header(header);
819  if (num_multifiles < 0) {
820  downloader_cat.error() << "invalid db header" << endl;
821  return false;
822  }
823 
824  // Now that we know how many multifiles this db has, we can iterate
825  // reading them off one by one
826  for (int i = 0; i < num_multifiles; i++) {
827  // The multifile record header is just one int which
828  // represents the size of the record
829  int mfr_header_length = sizeof(PN_int32);
830 
831  string mfr_header = sr.extract_bytes(mfr_header_length);
832  if (mfr_header.size() != (size_t)mfr_header_length) {
833  downloader_cat.error() << "invalid mfr header" << endl;
834  return false;
835  }
836 
837  // Parse the header
838  int mfr_length = parse_record_header(mfr_header);
839 
840  // Ok, now that we know the size of the mfr, read it in
841  // Make a buffer to read the multifile record into
842  // do not count the header length twice
843  int read_length = (mfr_length - mfr_header_length);
844  string mfr_record = sr.extract_bytes(read_length);
845  if (mfr_record.size() != (size_t)read_length) {
846  downloader_cat.error() << "invalid mfr record" << endl;
847  return false;
848  }
849 
850  // Parse the mfr
851  PT(DownloadDb::MultifileRecord) mfr = parse_mfr(mfr_record);
852 
853  // Only read in the individual file info if you are the server
854  if (want_server_info) {
855 
856  // Read off all the file records this multifile has
857  for (int j = 0; j < mfr->_num_files; j++) {
858  // The file record header is just one int which
859  // represents the size of the record
860  int fr_header_length = sizeof(PN_int32);
861 
862  // Read the header
863  string fr_header = sr.extract_bytes(fr_header_length);
864  if (fr_header.size() != (size_t)fr_header_length) {
865  downloader_cat.error() << "invalid fr header" << endl;
866  return false;
867  }
868 
869  // Parse the header
870  int fr_length = parse_record_header(fr_header);
871 
872  // Ok, now that we know the size of the mfr, read it in
873  // do not count the header length twice
874  int read_length = (fr_length - fr_header_length);
875 
876  string fr_record = sr.extract_bytes(read_length);
877  if (fr_record.size() != (size_t)read_length) {
878  downloader_cat.error() << "invalid fr record" << endl;
879  return false;
880  }
881 
882  // Parse the file record
883  PT(DownloadDb::FileRecord) fr = parse_fr(fr_record);
884 
885  // Add this file record to the current multifilerecord
886  mfr->add_file_record(fr);
887  }
888  }
889 
890  // Add the current multifilerecord to our database
891  add_multifile_record(mfr);
892  }
893 
894  return true;
895 }
896 
897 
898 
899 ////////////////////////////////////////////////////////////////////
900 // Function: DownloadDb::Db::write
901 // Access: Private
902 // Description:
903 ////////////////////////////////////////////////////////////////////
904 bool DownloadDb::Db::
905 write(StreamWriter &sw, bool want_server_info) {
906  // Declare these outside the loop so we do not keep creating
907  // and deleting them
908  PN_float64 phase;
909  PN_int32 size;
910  PN_int32 status;
911  PN_int32 num_files;
912  PN_int32 name_length;
913  PN_int32 header_length;
914 
915  // Iterate over the multifiles writing them to the stream
916  pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
917  for(; i != _mfile_records.end(); ++i) {
918  // Cache some properties so we do not have to keep asking for them
919  phase = (*i)->_phase;
920  size = (*i)->_size;
921  status = (*i)->_status;
922  num_files = (*i)->get_num_files();
923  name_length = (*i)->_name.length();
924 
925  // Compute the length of this datagram
926  header_length =
927  sizeof(header_length) + // Size of this header length
928  sizeof(name_length) + // Size of the size of the name string
929  (*i)->_name.length() + // Size of the name string
930  sizeof(phase) + sizeof(size) +
931  sizeof(status) + sizeof(num_files) +
932  sizeof(PN_uint32)*4; // Size of hash value
933 
934  // Add the length of this entire datagram
935  sw.add_int32(header_length);
936 
937  // Add the length of the name
938  sw.add_int32(name_length);
939  // Add the name
940  sw.append_data((*i)->_name);
941 
942  // Add all the properties
943  sw.add_float64(phase);
944  sw.add_int32(size);
945  sw.add_int32(status);
946  sw.add_int32(num_files);
947 
948  (*i)->_hash.write_stream(sw);
949 
950  // Only write out the file information if you are the server
951  if (want_server_info) {
952  // Now iterate over this multifile's files writing them to the stream
953  // Iterate over the multifiles writing them to the stream
954  pvector< PT(FileRecord) >::const_iterator j = (*i)->_file_records.begin();
955  for(; j != (*i)->_file_records.end(); ++j) {
956  name_length = (*j)->_name.length();
957 
958  // Compute the length of this datagram
959  header_length =
960  sizeof(header_length) + // Size of this header length
961  sizeof(name_length) + // Size of the size of the name string
962  (*j)->_name.length(); // Size of the name string
963 
964  // Add the length of this entire datagram
965  sw.add_int32(header_length);
966 
967  // Add the length of the name
968  sw.add_int32(name_length);
969  // Add the name
970  sw.append_data((*j)->_name);
971  }
972  }
973  }
974 
975  return true;
976 }
977 
978 ////////////////////////////////////////////////////////////////////
979 // Function: DownloadDb::Db::write_bogus_header
980 // Access: Private
981 // Description: Writes the bogus header uncompressed with platform-
982 // independent byte ordering. This header will get
983 // overwritten with the real magic number as the last
984 // step in the write
985 ////////////////////////////////////////////////////////////////////
986 bool DownloadDb::Db::
988  // Write the db magic number
989  sw.add_uint32(_bogus_magic_number);
990 
991  // Write the number of multifiles
992  sw.add_int32(get_num_multifiles());
993 
994  return true;
995 }
996 
997 ////////////////////////////////////////////////////////////////////
998 // Function: DownloadDb::Db::write_header
999 // Access: Private
1000 // Description: Writes the header uncompressed with platform-
1001 // independent byte ordering
1002 ////////////////////////////////////////////////////////////////////
1003 bool DownloadDb::Db::
1004 write_header(ostream &write_stream) {
1005  Datagram dg;
1006 
1007  // Write the db magic number
1008  dg.add_uint32(_magic_number);
1009 
1010  // Write the number of multifiles
1011  dg.add_int32(get_num_multifiles());
1012 
1013  string msg = dg.get_message();
1014 
1015  // Seek back to the beginning of the write stream
1016  write_stream.seekp(0);
1017  // Overwrite the old bogus header with the real header
1018  write_stream.write(msg.data(), msg.length());
1019  return true;
1020 }
1021 
1022 
1023 
1024 ////////////////////////////////////////////////////////////////////
1025 // FileRecord methods
1026 ////////////////////////////////////////////////////////////////////
1027 
1028 
1029 ////////////////////////////////////////////////////////////////////
1030 // Function: DownloadDb::FileRecord::Constructor
1031 // Access: Public
1032 // Description:
1033 ////////////////////////////////////////////////////////////////////
1034 DownloadDb::FileRecord::
1035 FileRecord() {
1036  _name = "";
1037 }
1038 
1039 
1040 ////////////////////////////////////////////////////////////////////
1041 // Function: DownloadDb::FileRecord::Constructor
1042 // Access: Public
1043 // Description:
1044 ////////////////////////////////////////////////////////////////////
1045 DownloadDb::FileRecord::
1046 FileRecord(string name) {
1047  _name = name;
1048 }
1049 
1050 ////////////////////////////////////////////////////////////////////
1051 // Function: DownloadDb::FileRecord::output
1052 // Access: Public
1053 // Description:
1054 ////////////////////////////////////////////////////////////////////
1055 void DownloadDb::FileRecord::
1056 write(ostream &out) const {
1057  out << " FileRecord: " << _name << endl;
1058 }
1059 
1060 ////////////////////////////////////////////////////////////////////
1061 // Function: DownloadDb::add_version
1062 // Access: Published
1063 // Description: Appends a new version of the file onto the end of the
1064 // list, or changes the hash associated with a version
1065 // previously added.
1066 //
1067 // Note: version numbers start at 1
1068 ////////////////////////////////////////////////////////////////////
1069 void DownloadDb::
1070 add_version(const Filename &name, const HashVal &hash, int version) {
1071  nassertv(version >= 1);
1072 
1073  VectorHash &vhash = _versions[name];
1074  int size = vhash.size();
1075 
1076  // We should not skip over versions as we add them.
1077  nassertv(version <= size+1);
1078 
1079  if (version-1 < size) {
1080  // If you are overwriting an old hash value, just rewrite the value
1081  vhash[version-1] = hash;
1082 
1083  } else {
1084  // Otherwise, extend the vector.
1085  vhash.push_back(hash);
1086  }
1087 }
1088 
1089 ////////////////////////////////////////////////////////////////////
1090 // Function: DownloadDb::insert_new_version
1091 // Access: Published
1092 // Description: Inserts a new version 1 copy of the file, sliding all
1093 // the other versions up by one.
1094 ////////////////////////////////////////////////////////////////////
1095 void DownloadDb::
1096 insert_new_version(const Filename &name, const HashVal &hash) {
1097  VectorHash &vhash = _versions[name];
1098  vhash.insert(vhash.begin(), hash);
1099 }
1100 
1101 ////////////////////////////////////////////////////////////////////
1102 // Function: DownloadDb::has_version
1103 // Access: Published
1104 // Description: Returns true if the indicated file has version
1105 // information, false otherwise. Some files recorded in
1106 // the database may not bother to track versions.
1107 ////////////////////////////////////////////////////////////////////
1108 bool DownloadDb::
1109 has_version(const Filename &name) const {
1110  return (_versions.find(name) != _versions.end());
1111 }
1112 
1113 ////////////////////////////////////////////////////////////////////
1114 // Function: DownloadDb::get_num_versions
1115 // Access: Published
1116 // Description: Returns the number of versions stored for the
1117 // indicated file.
1118 ////////////////////////////////////////////////////////////////////
1119 int DownloadDb::
1120 get_num_versions(const Filename &name) const {
1121  VersionMap::const_iterator vmi = _versions.find(name);
1122  if (vmi == _versions.end()) {
1123  return 0;
1124  }
1125 
1126  return (int)(*vmi).second.size();
1127 }
1128 
1129 ////////////////////////////////////////////////////////////////////
1130 // Function: DownloadDb::set_num_versions
1131 // Access: Published
1132 // Description: Reduces the number of versions of a particular file
1133 // stored in the ddb by throwing away all versions
1134 // higher than the indicated index.
1135 ////////////////////////////////////////////////////////////////////
1136 void DownloadDb::
1137 set_num_versions(const Filename &name, int num_versions) {
1138  VersionMap::iterator vmi = _versions.find(name);
1139  if (vmi == _versions.end()) {
1140  nassertv(num_versions == 0);
1141  return;
1142  }
1143 
1144  VectorHash &vhash = (*vmi).second;
1145 
1146  nassertv(num_versions <= (int)vhash.size());
1147  vhash.erase(vhash.begin() + num_versions, vhash.end());
1148 }
1149 
1150 ////////////////////////////////////////////////////////////////////
1151 // Function: DownloadDb::get_version
1152 // Access: Published
1153 // Description: Returns the version number of this particular file,
1154 // determined by looking up the hash generated from the
1155 // file. Returns -1 if the version number cannot be
1156 // determined.
1157 ////////////////////////////////////////////////////////////////////
1158 int DownloadDb::
1159 get_version(const Filename &name, const HashVal &hash) const {
1160  VersionMap::const_iterator vmi = _versions.find(name);
1161  if (vmi == _versions.end()) {
1162  downloader_cat.debug()
1163  << "DownloadDb::get_version() - can't find: " << name << endl;
1164  return -1;
1165  }
1166  const VectorHash &vhash = (*vmi).second;
1167  VectorHash::const_iterator i = find(vhash.begin(), vhash.end(), hash);
1168  if (i != vhash.end())
1169  return (i - vhash.begin() + 1);
1170  downloader_cat.debug()
1171  << "DownloadDb::get_version() - can't find hash: " << hash << endl;
1172  return -1;
1173 }
1174 
1175 ////////////////////////////////////////////////////////////////////
1176 // Function: DownloadDb::get_hash
1177 // Access: Published
1178 // Description: Returns the MD5 hash associated with the indicated
1179 // version of the indicated file.
1180 ////////////////////////////////////////////////////////////////////
1181 const HashVal &DownloadDb::
1182 get_hash(const Filename &name, int version) const {
1183  static HashVal bogus_hash;
1184 
1185  VersionMap::const_iterator vmi = _versions.find(name);
1186  if (vmi == _versions.end()) {
1187  downloader_cat.error()
1188  << "DownloadDb::get_hash() - can't find: " << name << endl;
1189  return bogus_hash;
1190  }
1191 
1192  const VectorHash &vhash = (*vmi).second;
1193  if (version < 1 || version > (int)vhash.size()) {
1194  downloader_cat.error()
1195  << "DownloadDb::get_hash() - no version " << version
1196  << " for " << name << endl;
1197  return bogus_hash;
1198  }
1199  return vhash[version - 1];
1200 }
1201 
1202 ////////////////////////////////////////////////////////////////////
1203 // Function: DownloadDb::write_version_map
1204 // Access: Protected
1205 // Description:
1206 ////////////////////////////////////////////////////////////////////
1207 void DownloadDb::
1208 write_version_map(StreamWriter &sw) {
1209  VersionMap::iterator vmi;
1210  VectorHash::iterator i;
1211  string name;
1212 
1213  sw.add_int32(_versions.size());
1214  for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
1215  name = (*vmi).first;
1216  downloader_cat.spam()
1217  << "DownloadDb::write_version_map() - writing file: "
1218  << name << " of length: " << name.length() << endl;
1219  sw.add_int32(name.length());
1220  sw.append_data(name);
1221  sw.add_int32((*vmi).second.size());
1222  for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
1223  // *i will point to a HashVal
1224  (*i).write_stream(sw);
1225  }
1226  }
1227 }
1228 
1229 ////////////////////////////////////////////////////////////////////
1230 // Function: DownloadDb::read_version_map
1231 // Access: Protected
1232 // Description:
1233 ////////////////////////////////////////////////////////////////////
1234 bool DownloadDb::
1235 read_version_map(StreamReader &sr) {
1236  int num_entries = sr.get_int32();
1237  if (sr.get_istream()->fail()) {
1238  return false;
1239  }
1240 
1241  for (int i = 0; i < num_entries; i++) {
1242 
1243  // Get the length of the file name
1244  int name_length = sr.get_int32();
1245  if (sr.get_istream()->fail()) {
1246  return false;
1247  }
1248  downloader_cat.spam()
1249  << "DownloadDb::read_version_map() - name length: " << name_length
1250  << endl;
1251 
1252  // Get the file name
1253  string name = sr.extract_bytes(name_length);
1254  downloader_cat.spam()
1255  << "DownloadDb::read_version_map() - name: " << name << endl;
1256 
1257  // Get number of hash values for name
1258  int length = sr.get_int32();
1259  if (sr.get_istream()->fail()) {
1260  return false;
1261  }
1262  downloader_cat.spam()
1263  << "DownloadDb::read_version_map() - number of values: " << length
1264  << endl;
1265 
1266  for (int j = 0; j < length; j++) {
1267  HashVal hash;
1268  hash.read_stream(sr);
1269  if (sr.get_istream()->fail()) {
1270  return false;
1271  }
1272  add_version(name, hash, j + 1);
1273  }
1274  }
1275  return true;
1276 }
1277 
1278 ////////////////////////////////////////////////////////////////////
1279 // Function: DownloadDb::write_version_map
1280 // Access: Public
1281 // Description:
1282 ////////////////////////////////////////////////////////////////////
1283 void DownloadDb::
1284 write_version_map(ostream &out) const {
1285  out << "Version Map: " << endl;
1286  VersionMap::const_iterator vmi;
1287  VectorHash::const_iterator i;
1288  for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
1289  out << " " << (*vmi).first << endl;
1290  for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
1291  HashVal hash = *i;
1292  out << " " << hash.as_dec() << endl;
1293  }
1294  }
1295  out << endl;
1296 }
A StreamWriter object is used to write sequential binary data directly to an ostream.
Definition: streamWriter.h:33
A listing of files within multifiles for management of client-side synchronization with a server-prov...
Definition: downloadDb.h:67
void append_data(const void *data, size_t size)
Appends some more raw data to the end of the streamWriter.
Definition: streamWriter.I:394
void set_server_multifile_hash(string mfname, HashVal val)
Set the hash value of file we are working on.
Definition: downloadDb.cxx:246
string get_message() const
Returns the datagram&#39;s data as a string.
Definition: datagram.I:431
bool has_version(const Filename &name) const
Returns true if the indicated file has version information, false otherwise.
void insert_new_version(const Filename &name, const HashVal &hash)
Inserts a new version 1 copy of the file, sliding all the other versions up by one.
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS&#39;s file system.
string as_dec() const
Returns the HashVal as a string with four decimal numbers.
Definition: hashVal.cxx:116
istream * get_istream() const
Returns the stream in use.
Definition: streamReader.I:87
string extract_bytes(size_t size)
Extracts the indicated number of bytes in the stream and returns them as a string.
DownloadDb()
Primarily used for testing.
Definition: downloadDb.cxx:89
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:494
HashVal get_client_multifile_hash(string mfname) const
Return the hash value of the file we are working on.
Definition: downloadDb.cxx:212
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer...
Definition: hashVal.h:32
PN_int32 get_int32()
Extracts a signed 32-bit integer.
PN_uint32 get_uint32()
Extracts an unsigned 32-bit integer.
void create_new_server_db()
Used on the server side makefiles to create a new clean server db.
Definition: downloadDb.cxx:399
static void close_read_file(istream *stream)
Closes a file opened by a previous call to open_read_file().
int get_num_versions(const Filename &name) const
Returns the number of versions stored for the indicated file.
This is our own Panda specialization on the default STL vector.
Definition: pvector.h:39
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:44
int get_version(const Filename &name, const HashVal &hash) const
Returns the version number of this particular file, determined by looking up the hash generated from ...
void add_version(const Filename &name, const HashVal &hash, int version)
Appends a new version of the file onto the end of the list, or changes the hash associated with a ver...
HashVal get_server_multifile_hash(string mfname) const
Return the hash value of the server file.
Definition: downloadDb.cxx:223
An in-memory buffer specifically designed for downloading files to memory.
Definition: ramfile.h:27
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
bool write_header(ostream &write_stream)
Writes the header uncompressed with platform- independent byte ordering.
void add_uint32(PN_uint32 value)
Adds an unsigned 32-bit integer to the stream.
Definition: streamWriter.I:170
istream * open_read_file(const Filename &filename, bool auto_unwrap) const
Convenience function; returns a newly allocated istream if the file exists and can be read...
bool client_multifile_complete(string mfname) const
A multifile is complete when it is completely downloaded.
Definition: downloadDb.cxx:178
void set_num_versions(const Filename &name, int num_versions)
Reduces the number of versions of a particular file stored in the ddb by throwing away all versions h...
bool write_bogus_header(StreamWriter &sw)
Writes the bogus header uncompressed with platform- independent byte ordering.
Definition: downloadDb.cxx:987
const HashVal & get_hash(const Filename &name, int version) const
Returns the MD5 hash associated with the indicated version of the indicated file. ...
void add_uint32(PN_uint32 value)
Adds an unsigned 32-bit integer to the datagram.
Definition: datagram.I:192
int parse_header(const string &data)
Verifies magic number, returns the number of multifiles or -1 if invalid.
Definition: downloadDb.cxx:678
void add_int32(PN_int32 value)
Adds a signed 32-bit integer to the datagram.
Definition: datagram.I:159
A class to retrieve the individual data elements previously stored in a Datagram. ...
PN_int32 get_int32()
Extracts a signed 32-bit integer.
Definition: streamReader.I:141
bool open_write(ofstream &stream, bool truncate=true) const
Opens the indicated ifstream for writing the file, if possible.
Definition: filename.cxx:2045
void add_int32(PN_int32 value)
Adds a signed 32-bit integer to the stream.
Definition: streamWriter.I:137
A class to read sequential binary data directly from an istream.
Definition: streamReader.h:30
An ordered list of data elements, formatted in memory for transmission over a socket or writing to a ...
Definition: datagram.h:43
int parse_record_header(const string &data)
Parses a file record (fr) header and returns the length of the next file record.
Definition: downloadDb.cxx:721
void set_client_multifile_hash(string mfname, HashVal val)
Set the hash value of file we are working on.
Definition: downloadDb.cxx:234