Panda3D
downloadDb.cxx
Go to the documentation of this file.
1 /**
2  * PANDA 3D SOFTWARE
3  * Copyright (c) Carnegie Mellon University. All rights reserved.
4  *
5  * All use of this software is subject to the terms of the revised BSD
6  * license. You should have received a copy of this license along
7  * with this source code in a file named "LICENSE."
8  *
9  * @file downloadDb.cxx
10  * @author shochet
11  * @date 2000-09-08
12  */
13 
14 #include "config_downloader.h"
15 #include "downloadDb.h"
16 #include "streamReader.h"
17 #include "streamWriter.h"
18 #include "ramfile.h"
19 #include "virtualFileSystem.h"
20 
21 #include <algorithm>
22 
23 using std::endl;
24 using std::istream;
25 using std::istringstream;
26 using std::move;
27 using std::ostream;
28 using std::string;
29 
30 // Defines
31 
32 // Written at the top of the file so we know this is a downloadDb
33 uint32_t DownloadDb::_magic_number = 0xfeedfeed;
34 
35 // Written at the top of the file to signify we are not done writing to the
36 // file yet. If you load a db with this magic number that means the previous
37 // time it got written out was probably interrupted in the middle of the
38 // write.
39 uint32_t DownloadDb::_bogus_magic_number = 0x11111111;
40 
41 
42 static string
43 back_to_front_slash(const string &str) {
44  string result = str;
45  string::iterator si;
46  for (si = result.begin(); si != result.end(); ++si) {
47  if ((*si) == '\\') {
48  (*si) = '/';
49  }
50  }
51 
52  return result;
53 }
54 
55 
56 /**
57  * Create a download db with these client and server dbs
58  */
60 DownloadDb(Ramfile &server_file, Filename &client_file) {
61  if (downloader_cat.is_debug())
62  downloader_cat.debug()
63  << "DownloadDb constructor called" << endl;
64  _client_db = read_db(client_file, 0);
65  _client_db._filename = client_file;
66  _server_db = read_db(server_file, 1);
67 }
68 
69 /**
70  * Create a download db with these client and server dbs
71  */
73 DownloadDb(Filename &server_file, Filename &client_file) {
74  if (downloader_cat.is_debug())
75  downloader_cat.debug()
76  << "DownloadDb constructor called" << endl;
77  _client_db = read_db(client_file, 0);
78  _client_db._filename = client_file;
79  _server_db = read_db(server_file, 1);
80  _server_db._filename = server_file;
81 }
82 
83 /**
84  * Primarily used for testing.
85  */
87 DownloadDb() {
88  _client_db = Db();
89  _server_db = Db();
90 }
91 
92 /**
93  *
94  */
95 DownloadDb::
96 ~DownloadDb() {
97  if (downloader_cat.is_debug())
98  downloader_cat.debug()
99  << "DownloadDb destructor called" << endl;
100 }
101 
102 
103 /**
104  *
105  */
106 void DownloadDb::
107 output(ostream &out) const {
108  out << "[" << _server_db._filename << " " << _client_db._filename << "]";
109 }
110 
111 /**
112  *
113  */
114 void DownloadDb::
115 write(ostream &out) const {
116  out << "DownloadDb" << endl;
117  out << "============================================================" << endl;
118  out << " Client DB file: " << _client_db._filename << endl;
119  out << "============================================================" << endl;
120  _client_db.write(out);
121  out << endl;
122  out << "============================================================" << endl;
123  out << " Server DB file: " << _server_db._filename << endl;
124  out << "============================================================" << endl;
125  _server_db.write(out);
126  write_version_map(out);
127  out << endl;
128 }
129 
130 
131 /**
132  *
133  */
134 bool DownloadDb::
135 write_client_db(Filename &file) {
136  return write_db(file, _client_db, 0);
137 }
138 
139 
140 /**
141  *
142  */
143 bool DownloadDb::
144 write_server_db(Filename &file) {
145  return write_db(file, _server_db, 1);
146 }
147 
148 /**
149  *
150  */
151 bool DownloadDb::
152 client_multifile_exists(string mfname) const {
153  return (_client_db.multifile_exists(mfname));
154 }
155 
156 /**
157  * A multifile is complete when it is completely downloaded. Note: it may
158  * already be decompressed or extracted and it is still complete
159  */
161 client_multifile_complete(string mfname) const {
162  int client_status = _client_db.get_multifile_record_named(mfname)->_status;
163  return (client_status >= Status_complete);
164 }
165 
166 /**
167  *
168  */
169 bool DownloadDb::
170 client_multifile_decompressed(string mfname) const {
171  int client_status = _client_db.get_multifile_record_named(mfname)->_status;
172  return (client_status >= Status_decompressed);
173 }
174 
175 /**
176  *
177  */
178 bool DownloadDb::
179 client_multifile_extracted(string mfname) const {
180  int client_status = _client_db.get_multifile_record_named(mfname)->_status;
181  return (client_status >= Status_extracted);
182 }
183 
184 
185 /**
186  * Return the hash value of the file we are working on
187  */
189 get_client_multifile_hash(string mfname) const {
190  return _client_db.get_multifile_record_named(mfname)->_hash;
191 }
192 
193 
194 /**
195  * Return the hash value of the server file
196  */
198 get_server_multifile_hash(string mfname) const {
199  return _server_db.get_multifile_record_named(mfname)->_hash;
200 }
201 
202 
203 /**
204  * Set the hash value of file we are working on
205  */
207 set_client_multifile_hash(string mfname, HashVal val) {
208  _client_db.get_multifile_record_named(mfname)->_hash = val;
209  write_client_db(_client_db._filename);
210 }
211 
212 
213 /**
214  * Set the hash value of file we are working on
215  */
217 set_server_multifile_hash(string mfname, HashVal val) {
218  _server_db.get_multifile_record_named(mfname)->_hash = val;
219 }
220 
221 // Operations on multifiles
222 
223 /**
224  *
225  */
226 void DownloadDb::
227 delete_client_multifile(string mfname) {
228 }
229 
230 /**
231  *
232  */
233 void DownloadDb::
234 add_client_multifile(string server_mfname) {
235  PT(MultifileRecord) server_mfr = _server_db.get_multifile_record_named(server_mfname);
236  PT(MultifileRecord) client_mfr = new MultifileRecord;
237  client_mfr->_name = server_mfr->_name;
238  client_mfr->_phase = server_mfr->_phase;
239  _client_db.add_multifile_record(client_mfr);
240 }
241 
242 
243 /**
244  *
245  */
246 void DownloadDb::
247 expand_client_multifile(string mfname) {
248 }
249 
250 
251 /**
252  *
253  */
254 DownloadDb::Db DownloadDb::
255 read_db(Filename &file, bool want_server_info) {
257  Db db;
258 
259  // Open the multifile for reading
260  file.set_binary();
261  istream *read_stream = vfs->open_read_file(file, true);
262 
263  if (read_stream == nullptr) {
264  downloader_cat.error()
265  << "failed to open input file: "
266  << file << endl;
267  return db;
268  }
269 
270  StreamReader sr(*read_stream);
271  if (!db.read(sr, want_server_info)) {
272  downloader_cat.error()
273  << "read failed: "
274  << file << endl;
275  vfs->close_read_file(read_stream);
276  return db;
277  }
278 
279  if (want_server_info) {
280  if (!read_version_map(sr)) {
281  downloader_cat.error()
282  << "read_version_map() failed: "
283  << file << endl;
284  }
285  }
286 
287  vfs->close_read_file(read_stream);
288 
289  return db;
290 }
291 
292 /**
293  *
294  */
295 DownloadDb::Db DownloadDb::
296 read_db(Ramfile &file, bool want_server_info) {
297  // Open the multifile for reading
298  istringstream read_stream(file._data);
299  Db db;
300 
301  StreamReader sr(read_stream);
302 
303  if (!db.read(sr, want_server_info)) {
304  downloader_cat.error()
305  << "read failed" << endl;
306  return db;
307  }
308  if (want_server_info) {
309  if (!read_version_map(sr)) {
310  downloader_cat.error()
311  << "read_version_map() failed" << endl;
312  }
313  }
314 
315  return db;
316 }
317 
318 
319 /**
320  *
321  */
322 bool DownloadDb::
323 write_db(Filename &file, Db db, bool want_server_info) {
324  pofstream write_stream;
325  file.set_binary();
326  if (!file.open_write(write_stream)) {
327  downloader_cat.error()
328  << "DownloadDb::write_db() - Failed to open output file: "
329  << file << endl;
330  return false;
331  }
332 
333  downloader_cat.spam()
334  << "Writing to file: " << file << endl;
335 
336  StreamWriter sw(write_stream);
337 
338  // Initialize the write stream with a bogus header
339  db.write_bogus_header(sw);
340  db.write(sw, want_server_info);
341  if (want_server_info) {
342  write_version_map(sw);
343  }
344  // Now write the real header
345  db.write_header(write_stream);
346  write_stream.close();
347  return true;
348 }
349 
350 
351 /**
352  * Used on the server side makefiles to create a new clean server db
353  */
356  _server_db = Db();
357 }
358 
359 
360 /**
361  *
362  */
363 void DownloadDb::
364 server_add_multifile(string mfname, Phase phase, int size, int status) {
365  PT(MultifileRecord) mfr = new MultifileRecord(mfname, phase, size, status);
366  _server_db.add_multifile_record(mfr);
367 }
368 
369 
370 /**
371  *
372  */
373 void DownloadDb::
374 server_add_file(string mfname, string fname) {
375  // Make the new file record
376  PT(FileRecord) fr = new FileRecord(fname);
377 
378  // Find the multifile with mfname
379  pvector< PT(MultifileRecord) >::iterator i = _server_db._mfile_records.begin();
380  for (; i != _server_db._mfile_records.end(); ++i) {
381  if (mfname == (*i)->_name) {
382  (*i)->add_file_record(fr);
383  return;
384  }
385  }
386 
387  // Uh-oh, did not find it
388  downloader_cat.error() << "Could not find record named "
389  << mfname << " in database " << endl;
390  nassertv(false);
391  return;
392 }
393 
394 
395 // Multifile methods
396 
397 
398 /**
399  *
400  */
401 DownloadDb::MultifileRecord::
402 MultifileRecord() {
403  _name = "";
404  _phase = 0;
405  _size = 0;
406  _status = Status_incomplete;
407 }
408 
409 
410 /**
411  *
412  */
413 DownloadDb::MultifileRecord::
414 MultifileRecord(string name, Phase phase, int size, int status) {
415  _name = name;
416  _phase = phase;
417  _size = size;
418  _status = status;
419 }
420 
421 
422 /**
423  *
424  */
425 void DownloadDb::MultifileRecord::
426 write(ostream &out) const {
427  out << "==================================================" << endl;
428  out << "MultifileRecord: " << _name << endl
429  << " phase: " << _phase << endl
430  << " size: " << _size << endl
431  << " status: " << _status << endl
432  << " hash: " << _hash.as_dec() << endl;
433  out << "--------------------------------------------------" << endl;
434  pvector< PT(FileRecord) >::const_iterator i = _file_records.begin();
435  for(; i != _file_records.end(); ++i) {
436  (*i)->write(out);
437  }
438 }
439 
440 
441 
442 /**
443  *
444  */
445 int DownloadDb::MultifileRecord::
446 get_num_files() const {
447  return _file_records.size();
448 }
449 
450 /**
451  *
452  */
453 string DownloadDb::MultifileRecord::
454 get_file_name(int index) const {
455  return _file_records[index]->_name;
456 }
457 
458 
459 /**
460  *
461  */
462 bool DownloadDb::MultifileRecord::
463 file_exists(string fname) const {
464  pvector< PT(FileRecord) >::const_iterator i = _file_records.begin();
465  for(; i != _file_records.end(); ++i) {
466  if (fname == (*i)->_name) {
467  return true;
468  }
469  }
470  return false;
471 }
472 
473 
474 /**
475  *
476  */
477 PT(DownloadDb::FileRecord) DownloadDb::MultifileRecord::
478 get_file_record_named(string fname) const {
479  pvector< PT(FileRecord) >::const_iterator i = _file_records.begin();
480  for(; i != _file_records.end(); ++i) {
481  if (fname == (*i)->_name) {
482  return (*i);
483  }
484  }
485  // Did not find it, just return an empty version
486  downloader_cat.error() << "Could not find record named "
487  << fname << " in multifile " << _name << endl;
488  PT(FileRecord) foo = new FileRecord;
489  nassertr(false, foo);
490  return foo;
491 }
492 
493 
494 /**
495  *
496  */
497 void DownloadDb::MultifileRecord::
498 add_file_record(PT(FileRecord) fr) {
499  _file_records.push_back(fr);
500 }
501 
502 
503 
504 // Db methods
505 
506 
507 
508 
509 /**
510  *
511  */
512 DownloadDb::Db::
513 Db() {
514  // The head is a magic number and the number of multifiles in the db
515  _header_length = sizeof(_magic_number) + sizeof(int32_t);
516 }
517 
518 
519 /**
520  *
521  */
522 void DownloadDb::Db::
523 write(ostream &out) const {
524  pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
525  for(; i != _mfile_records.end(); ++i) {
526  (*i)->write(out);
527  }
528 }
529 
530 
531 /**
532  *
533  */
534 int DownloadDb::Db::
535 get_num_multifiles() const {
536  return _mfile_records.size();
537 }
538 
539 /**
540  *
541  */
542 string DownloadDb::Db::
543 get_multifile_name(int index) const {
544  return _mfile_records[index]->_name;
545 }
546 
547 /**
548  *
549  */
550 bool DownloadDb::Db::
551 multifile_exists(string mfname) const {
552  pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
553  for(; i != _mfile_records.end(); ++i) {
554  if (mfname == (*i)->_name) {
555  return true;
556  }
557  }
558  return false;
559 }
560 
561 /**
562  *
563  */
565 get_multifile_record_named(string mfname) const {
566  pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
567  for(; i != _mfile_records.end(); ++i) {
568  if (mfname == (*i)->_name) {
569  return (*i);
570  }
571  }
572  // Did not find it, just return an empty version
573  downloader_cat.error() << "Could not find record named "
574  << mfname << " in database " << endl;
575  PT(MultifileRecord) foo = new MultifileRecord;
576  nassertr(false, foo);
577  return foo;
578 }
579 
580 /**
581  *
582  */
583 void DownloadDb::Db::
584 add_multifile_record(PT(MultifileRecord) mfr) {
585  _mfile_records.push_back(mfr);
586 }
587 
588 
589 /**
590  * Verifies magic number, returns the number of multifiles or -1 if invalid
591  */
594  // Make sure we have a good header
595  DatagramIterator di(dg);
596  uint32_t magic_number = di.get_uint32();
597  downloader_cat.debug()
598  << "Parsed magic number: " << magic_number << endl;
599  // If the magic number is equal to the bogus magic number it signifies that
600  // the previous write was interrupted
601  if (magic_number == _bogus_magic_number) {
602  downloader_cat.error()
603  << "DownloadDb::parse_header() - "
604  << "Bogus magic number, previous write incomplete: "
605  << magic_number << " expected: " << _magic_number << endl;
606  return -1;
607  }
608  // If the magic number does not match at all, something is really wrong
609  else if (magic_number != _magic_number) {
610  downloader_cat.error()
611  << "DownloadDb::parse_header() - Invalid magic number: "
612  << magic_number << " expected: " << _magic_number << endl;
613  return -1;
614  }
615 
616  int32_t num_multifiles = di.get_int32();
617  downloader_cat.debug()
618  << "Parsed number of multifiles: " << num_multifiles << endl;
619 
620  // If we got all the way here, must be done
621  return num_multifiles;
622 }
623 
624 
625 
626 /**
627  * Parses a file record (fr) header and returns the length of the next file
628  * record
629  */
632  DatagramIterator di(dg);
633  int32_t record_length = di.get_int32();
634  downloader_cat.spam()
635  << "Parsed record header length: " << record_length << endl;
636 
637  // If we got all the way here, must be done
638  return record_length;
639 }
640 
641 
642 /**
643  * Parses a multifile record (mfr) and returns one
644  */
645 PT(DownloadDb::MultifileRecord) DownloadDb::Db::
646 parse_mfr(Datagram dg) {
647 
649 
650  DatagramIterator di(dg);
651  mfr->_name = di.get_string32();
652  mfr->_phase = di.get_float64();
653  mfr->_size = di.get_int32();
654  mfr->_status = di.get_int32();
655  mfr->_num_files = di.get_int32();
656 
657  // At one time, we stored files in the database with a backslash separator.
658  // Nowadays we use a forward slash, but we should make sure we properly
659  // convert any old records we might read.
660  mfr->_name = back_to_front_slash(mfr->_name);
661 
662  // Read the hash value
663  mfr->_hash.read_datagram(di);
664 
665  downloader_cat.debug()
666  << "Parsed multifile record: " << mfr->_name << " phase: " << mfr->_phase
667  << " size: " << mfr->_size
668  << " status: " << mfr->_status << " num_files: " << mfr->_num_files << endl;
669 
670  // Return the new MultifileRecord
671  return mfr;
672 }
673 
674 
675 
676 
677 /**
678  * Parses a file record (fr) and returns one
679  */
680 PT(DownloadDb::FileRecord) DownloadDb::Db::
681 parse_fr(Datagram dg) {
682 
684 
685  DatagramIterator di(dg);
686  fr->_name = di.get_string32();
687 
688  // At one time, we stored files in the database with a backslash separator.
689  // Nowadays we use a forward slash, but we should make sure we properly
690  // convert any old records we might read.
691  fr->_name = back_to_front_slash(fr->_name);
692 
693  downloader_cat.spam()
694  << "Parsed file record: " << fr->_name << endl;
695 
696  // Return the new MultifileRecord
697  return fr;
698 }
699 
700 
701 
702 
703 /**
704  *
705  */
706 bool DownloadDb::Db::
707 read(StreamReader &sr, bool want_server_info) {
708  // Read the header
709  vector_uchar header = sr.extract_bytes(_header_length);
710  if (header.size() != (size_t)_header_length) {
711  downloader_cat.error() << "truncated db file" << endl;
712  return false;
713  }
714 
715  // Parse the header
716  int num_multifiles = parse_header(Datagram(move(header)));
717  if (num_multifiles < 0) {
718  downloader_cat.error() << "invalid db header" << endl;
719  return false;
720  }
721 
722  // Now that we know how many multifiles this db has, we can iterate reading
723  // them off one by one
724  for (int i = 0; i < num_multifiles; i++) {
725  // The multifile record header is just one int which represents the size
726  // of the record
727  int mfr_header_length = sizeof(int32_t);
728 
729  vector_uchar mfr_header = sr.extract_bytes(mfr_header_length);
730  if (mfr_header.size() != (size_t)mfr_header_length) {
731  downloader_cat.error() << "invalid mfr header" << endl;
732  return false;
733  }
734 
735  // Parse the header
736  int mfr_length = parse_record_header(Datagram(move(mfr_header)));
737 
738  // Ok, now that we know the size of the mfr, read it in Make a buffer to
739  // read the multifile record into do not count the header length twice
740  int read_length = (mfr_length - mfr_header_length);
741  vector_uchar mfr_record = sr.extract_bytes(read_length);
742  if (mfr_record.size() != (size_t)read_length) {
743  downloader_cat.error() << "invalid mfr record" << endl;
744  return false;
745  }
746 
747  // Parse the mfr
748  PT(DownloadDb::MultifileRecord) mfr = parse_mfr(Datagram(move(mfr_record)));
749 
750  // Only read in the individual file info if you are the server
751  if (want_server_info) {
752 
753  // Read off all the file records this multifile has
754  for (int j = 0; j < mfr->_num_files; j++) {
755  // The file record header is just one int which represents the size of
756  // the record
757  int fr_header_length = sizeof(int32_t);
758 
759  // Read the header
760  vector_uchar fr_header = sr.extract_bytes(fr_header_length);
761  if (fr_header.size() != (size_t)fr_header_length) {
762  downloader_cat.error() << "invalid fr header" << endl;
763  return false;
764  }
765 
766  // Parse the header
767  int fr_length = parse_record_header(Datagram(move(fr_header)));
768 
769  // Ok, now that we know the size of the mfr, read it in do not count
770  // the header length twice
771  int read_length = (fr_length - fr_header_length);
772 
773  vector_uchar fr_record = sr.extract_bytes(read_length);
774  if (fr_record.size() != (size_t)read_length) {
775  downloader_cat.error() << "invalid fr record" << endl;
776  return false;
777  }
778 
779  // Parse the file record
780  PT(DownloadDb::FileRecord) fr = parse_fr(Datagram(move(fr_record)));
781 
782  // Add this file record to the current multifilerecord
783  mfr->add_file_record(fr);
784  }
785  }
786 
787  // Add the current multifilerecord to our database
788  add_multifile_record(mfr);
789  }
790 
791  return true;
792 }
793 
794 
795 
796 /**
797  *
798  */
799 bool DownloadDb::Db::
800 write(StreamWriter &sw, bool want_server_info) {
801  // Declare these outside the loop so we do not keep creating and deleting
802  // them
803  PN_float64 phase;
804  int32_t size;
805  int32_t status;
806  int32_t num_files;
807  int32_t name_length;
808  int32_t header_length;
809 
810  // Iterate over the multifiles writing them to the stream
811  pvector< PT(MultifileRecord) >::const_iterator i = _mfile_records.begin();
812  for(; i != _mfile_records.end(); ++i) {
813  // Cache some properties so we do not have to keep asking for them
814  phase = (*i)->_phase;
815  size = (*i)->_size;
816  status = (*i)->_status;
817  num_files = (*i)->get_num_files();
818  name_length = (*i)->_name.length();
819 
820  // Compute the length of this datagram
821  header_length =
822  sizeof(header_length) + // Size of this header length
823  sizeof(name_length) + // Size of the size of the name string
824  (*i)->_name.length() + // Size of the name string
825  sizeof(phase) + sizeof(size) +
826  sizeof(status) + sizeof(num_files) +
827  sizeof(uint32_t)*4; // Size of hash value
828 
829  // Add the length of this entire datagram
830  sw.add_int32(header_length);
831 
832  // Add the length of the name
833  sw.add_int32(name_length);
834  // Add the name
835  sw.append_data((*i)->_name);
836 
837  // Add all the properties
838  sw.add_float64(phase);
839  sw.add_int32(size);
840  sw.add_int32(status);
841  sw.add_int32(num_files);
842 
843  (*i)->_hash.write_stream(sw);
844 
845  // Only write out the file information if you are the server
846  if (want_server_info) {
847  // Now iterate over this multifile's files writing them to the stream
848  // Iterate over the multifiles writing them to the stream
849  pvector< PT(FileRecord) >::const_iterator j = (*i)->_file_records.begin();
850  for(; j != (*i)->_file_records.end(); ++j) {
851  name_length = (*j)->_name.length();
852 
853  // Compute the length of this datagram
854  header_length =
855  sizeof(header_length) + // Size of this header length
856  sizeof(name_length) + // Size of the size of the name string
857  (*j)->_name.length(); // Size of the name string
858 
859  // Add the length of this entire datagram
860  sw.add_int32(header_length);
861 
862  // Add the length of the name
863  sw.add_int32(name_length);
864  // Add the name
865  sw.append_data((*j)->_name);
866  }
867  }
868  }
869 
870  return true;
871 }
872 
873 /**
874  * Writes the bogus header uncompressed with platform- independent byte
875  * ordering. This header will get overwritten with the real magic number as
876  * the last step in the write
877  */
880  // Write the db magic number
881  sw.add_uint32(_bogus_magic_number);
882 
883  // Write the number of multifiles
884  sw.add_int32(get_num_multifiles());
885 
886  return true;
887 }
888 
889 /**
890  * Writes the header uncompressed with platform- independent byte ordering
891  */
893 write_header(ostream &write_stream) {
894  Datagram dg;
895 
896  // Write the db magic number
897  dg.add_uint32(_magic_number);
898 
899  // Write the number of multifiles
900  dg.add_int32(get_num_multifiles());
901 
902  // Seek back to the beginning of the write stream
903  write_stream.seekp(0);
904  // Overwrite the old bogus header with the real header
905  write_stream.write((const char *)dg.get_data(), dg.get_length());
906  return true;
907 }
908 
909 
910 
911 // FileRecord methods
912 
913 
914 /**
915  *
916  */
917 DownloadDb::FileRecord::
918 FileRecord() {
919  _name = "";
920 }
921 
922 
923 /**
924  *
925  */
926 DownloadDb::FileRecord::
927 FileRecord(string name) {
928  _name = name;
929 }
930 
931 /**
932  *
933  */
934 void DownloadDb::FileRecord::
935 write(ostream &out) const {
936  out << " FileRecord: " << _name << endl;
937 }
938 
939 /**
940  * Appends a new version of the file onto the end of the list, or changes the
941  * hash associated with a version previously added.
942  *
943  * Note: version numbers start at 1
944  */
946 add_version(const Filename &name, const HashVal &hash, int version) {
947  nassertv(version >= 1);
948 
949  VectorHash &vhash = _versions[name];
950  int size = vhash.size();
951 
952  // We should not skip over versions as we add them.
953  nassertv(version <= size+1);
954 
955  if (version-1 < size) {
956  // If you are overwriting an old hash value, just rewrite the value
957  vhash[version-1] = hash;
958 
959  } else {
960  // Otherwise, extend the vector.
961  vhash.push_back(hash);
962  }
963 }
964 
965 /**
966  * Inserts a new version 1 copy of the file, sliding all the other versions up
967  * by one.
968  */
970 insert_new_version(const Filename &name, const HashVal &hash) {
971  VectorHash &vhash = _versions[name];
972  vhash.insert(vhash.begin(), hash);
973 }
974 
975 /**
976  * Returns true if the indicated file has version information, false
977  * otherwise. Some files recorded in the database may not bother to track
978  * versions.
979  */
981 has_version(const Filename &name) const {
982  return (_versions.find(name) != _versions.end());
983 }
984 
985 /**
986  * Returns the number of versions stored for the indicated file.
987  */
989 get_num_versions(const Filename &name) const {
990  VersionMap::const_iterator vmi = _versions.find(name);
991  if (vmi == _versions.end()) {
992  return 0;
993  }
994 
995  return (int)(*vmi).second.size();
996 }
997 
998 /**
999  * Reduces the number of versions of a particular file stored in the ddb by
1000  * throwing away all versions higher than the indicated index.
1001  */
1003 set_num_versions(const Filename &name, int num_versions) {
1004  VersionMap::iterator vmi = _versions.find(name);
1005  if (vmi == _versions.end()) {
1006  nassertv(num_versions == 0);
1007  return;
1008  }
1009 
1010  VectorHash &vhash = (*vmi).second;
1011 
1012  nassertv(num_versions <= (int)vhash.size());
1013  vhash.erase(vhash.begin() + num_versions, vhash.end());
1014 }
1015 
1016 /**
1017  * Returns the version number of this particular file, determined by looking
1018  * up the hash generated from the file. Returns -1 if the version number
1019  * cannot be determined.
1020  */
1022 get_version(const Filename &name, const HashVal &hash) const {
1023  VersionMap::const_iterator vmi = _versions.find(name);
1024  if (vmi == _versions.end()) {
1025  downloader_cat.debug()
1026  << "DownloadDb::get_version() - can't find: " << name << endl;
1027  return -1;
1028  }
1029  const VectorHash &vhash = (*vmi).second;
1030  VectorHash::const_iterator i = find(vhash.begin(), vhash.end(), hash);
1031  if (i != vhash.end())
1032  return (i - vhash.begin() + 1);
1033  downloader_cat.debug()
1034  << "DownloadDb::get_version() - can't find hash: " << hash << endl;
1035  return -1;
1036 }
1037 
1038 /**
1039  * Returns the MD5 hash associated with the indicated version of the indicated
1040  * file.
1041  */
1043 get_hash(const Filename &name, int version) const {
1044  static HashVal bogus_hash;
1045 
1046  VersionMap::const_iterator vmi = _versions.find(name);
1047  if (vmi == _versions.end()) {
1048  downloader_cat.error()
1049  << "DownloadDb::get_hash() - can't find: " << name << endl;
1050  return bogus_hash;
1051  }
1052 
1053  const VectorHash &vhash = (*vmi).second;
1054  if (version < 1 || version > (int)vhash.size()) {
1055  downloader_cat.error()
1056  << "DownloadDb::get_hash() - no version " << version
1057  << " for " << name << endl;
1058  return bogus_hash;
1059  }
1060  return vhash[version - 1];
1061 }
1062 
1063 /**
1064  *
1065  */
1066 void DownloadDb::
1067 write_version_map(StreamWriter &sw) {
1068  VersionMap::iterator vmi;
1069  VectorHash::iterator i;
1070  string name;
1071 
1072  sw.add_int32(_versions.size());
1073  for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
1074  name = (*vmi).first;
1075  downloader_cat.spam()
1076  << "DownloadDb::write_version_map() - writing file: "
1077  << name << " of length: " << name.length() << endl;
1078  sw.add_int32(name.length());
1079  sw.append_data(name);
1080  sw.add_int32((*vmi).second.size());
1081  for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
1082  // *i will point to a HashVal
1083  (*i).write_stream(sw);
1084  }
1085  }
1086 }
1087 
1088 /**
1089  *
1090  */
1091 bool DownloadDb::
1092 read_version_map(StreamReader &sr) {
1093  int num_entries = sr.get_int32();
1094  if (sr.get_istream()->fail()) {
1095  return false;
1096  }
1097 
1098  for (int i = 0; i < num_entries; i++) {
1099 
1100  // Get the file name
1101  string name = sr.get_string32();
1102  downloader_cat.spam()
1103  << "DownloadDb::read_version_map() - name: " << name << endl;
1104 
1105  // Get number of hash values for name
1106  int length = sr.get_int32();
1107  if (sr.get_istream()->fail()) {
1108  return false;
1109  }
1110  downloader_cat.spam()
1111  << "DownloadDb::read_version_map() - number of values: " << length
1112  << endl;
1113 
1114  for (int j = 0; j < length; j++) {
1115  HashVal hash;
1116  hash.read_stream(sr);
1117  if (sr.get_istream()->fail()) {
1118  return false;
1119  }
1120  add_version(name, hash, j + 1);
1121  }
1122  }
1123  return true;
1124 }
1125 
1126 /**
1127  *
1128  */
1129 void DownloadDb::
1130 write_version_map(ostream &out) const {
1131  out << "Version Map: " << endl;
1132  VersionMap::const_iterator vmi;
1133  VectorHash::const_iterator i;
1134  for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
1135  out << " " << (*vmi).first << endl;
1136  for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
1137  HashVal hash = *i;
1138  out << " " << hash.as_dec() << endl;
1139  }
1140  }
1141  out << endl;
1142 }
A class to retrieve the individual data elements previously stored in a Datagram.
uint32_t get_uint32()
Extracts an unsigned 32-bit integer.
int32_t get_int32()
Extracts a signed 32-bit integer.
An ordered list of data elements, formatted in memory for transmission over a socket or writing to a ...
Definition: datagram.h:38
void add_uint32(uint32_t value)
Adds an unsigned 32-bit integer to the datagram.
Definition: datagram.I:94
size_t get_length() const
Returns the number of bytes in the datagram.
Definition: datagram.I:335
void add_int32(int32_t value)
Adds a signed 32-bit integer to the datagram.
Definition: datagram.I:67
const void * get_data() const
Returns a pointer to the beginning of the datagram's data.
Definition: datagram.I:327
int parse_header(Datagram dg)
Verifies magic number, returns the number of multifiles or -1 if invalid.
Definition: downloadDb.cxx:593
bool write_bogus_header(StreamWriter &sw)
Writes the bogus header uncompressed with platform- independent byte ordering.
Definition: downloadDb.cxx:879
bool write_header(std::ostream &write_stream)
Writes the header uncompressed with platform- independent byte ordering.
Definition: downloadDb.cxx:893
int parse_record_header(Datagram dg)
Parses a file record (fr) header and returns the length of the next file record.
Definition: downloadDb.cxx:631
const HashVal & get_hash(const Filename &name, int version) const
Returns the MD5 hash associated with the indicated version of the indicated file.
void set_server_multifile_hash(std::string mfname, HashVal val)
Set the hash value of file we are working on.
Definition: downloadDb.cxx:217
void create_new_server_db()
Used on the server side makefiles to create a new clean server db.
Definition: downloadDb.cxx:355
HashVal get_client_multifile_hash(std::string mfname) const
Return the hash value of the file we are working on.
Definition: downloadDb.cxx:189
bool client_multifile_complete(std::string mfname) const
A multifile is complete when it is completely downloaded.
Definition: downloadDb.cxx:161
int get_version(const Filename &name, const HashVal &hash) const
Returns the version number of this particular file, determined by looking up the hash generated from ...
bool has_version(const Filename &name) const
Returns true if the indicated file has version information, false otherwise.
Definition: downloadDb.cxx:981
void insert_new_version(const Filename &name, const HashVal &hash)
Inserts a new version 1 copy of the file, sliding all the other versions up by one.
Definition: downloadDb.cxx:970
HashVal get_server_multifile_hash(std::string mfname) const
Return the hash value of the server file.
Definition: downloadDb.cxx:198
DownloadDb()
Primarily used for testing.
Definition: downloadDb.cxx:87
void set_client_multifile_hash(std::string mfname, HashVal val)
Set the hash value of file we are working on.
Definition: downloadDb.cxx:207
void set_num_versions(const Filename &name, int num_versions)
Reduces the number of versions of a particular file stored in the ddb by throwing away all versions h...
void add_version(const Filename &name, const HashVal &hash, int version)
Appends a new version of the file onto the end of the list, or changes the hash associated with a ver...
Definition: downloadDb.cxx:946
int get_num_versions(const Filename &name) const
Returns the number of versions stored for the indicated file.
Definition: downloadDb.cxx:989
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:39
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:414
bool open_write(std::ofstream &stream, bool truncate=true) const
Opens the indicated ifstream for writing the file, if possible.
Definition: filename.cxx:1899
Stores a 128-bit value that represents the hashed contents (typically MD5) of a file or buffer.
Definition: hashVal.h:31
std::string as_dec() const
Returns the HashVal as a string with four decimal numbers.
Definition: hashVal.cxx:108
An in-memory buffer specifically designed for downloading files to memory.
Definition: ramfile.h:25
A class to read sequential binary data directly from an istream.
Definition: streamReader.h:28
std::string get_string32()
Extracts a variable-length string with a 32-bit length field.
int32_t get_int32()
Extracts a signed 32-bit integer.
Definition: streamReader.I:115
size_t extract_bytes(unsigned char *into, size_t size)
Extracts the indicated number of bytes in the stream into the given character buffer.
get_istream
Returns the stream in use.
Definition: streamReader.h:38
A StreamWriter object is used to write sequential binary data directly to an ostream.
Definition: streamWriter.h:29
void append_data(const void *data, size_t size)
Appends some more raw data to the end of the streamWriter.
Definition: streamWriter.I:325
void add_float64(PN_float64 value)
Adds a 64-bit floating-point number to the stream.
Definition: streamWriter.I:180
void add_int32(int32_t value)
Adds a signed 32-bit integer to the stream.
Definition: streamWriter.I:120
void add_uint32(uint32_t value)
Adds an unsigned 32-bit integer to the stream.
Definition: streamWriter.I:147
A hierarchy of directories and files that appears to be one continuous file system,...
static void close_read_file(std::istream *stream)
Closes a file opened by a previous call to open_read_file().
std::istream * open_read_file(const Filename &filename, bool auto_unwrap) const
Convenience function; returns a newly allocated istream if the file exists and can be read,...
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
This is our own Panda specialization on the default STL vector.
Definition: pvector.h:42
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PT(DownloadDb::FileRecord) DownloadDb
Parses a file record (fr) and returns one.
Definition: downloadDb.cxx:477
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.