Panda3D
datagramInputFile.cxx
Go to the documentation of this file.
1 /**
2  * PANDA 3D SOFTWARE
3  * Copyright (c) Carnegie Mellon University. All rights reserved.
4  *
5  * All use of this software is subject to the terms of the revised BSD
6  * license. You should have received a copy of this license along
7  * with this source code in a file named "LICENSE."
8  *
9  * @file datagramInputFile.cxx
10  * @author drose
11  * @date 2000-10-30
12  */
13 
14 #include "datagramInputFile.h"
15 #include "temporaryFile.h"
16 #include "numeric_types.h"
17 #include "datagramIterator.h"
18 #include "profileTimer.h"
19 #include "config_putil.h"
20 #include "config_express.h"
21 #include "virtualFileSystem.h"
22 #include "streamReader.h"
23 #include "thread.h"
24 
25 using std::streampos;
26 using std::streamsize;
27 
28 /**
29  * Opens the indicated filename for reading. Returns true on success, false
30  * on failure.
31  */
33 open(const FileReference *file) {
34  close();
35 
36  _file = file;
37  _filename = _file->get_filename();
38 
39  // DatagramInputFiles are always binary.
40  _filename.set_binary();
41 
43  _vfile = vfs->get_file(_filename);
44  if (_vfile == nullptr) {
45  // No such file.
46  return false;
47  }
48  _timestamp = _vfile->get_timestamp();
49  _in = _vfile->open_read_file(true);
50  _owns_in = (_in != nullptr);
51  return _owns_in && !_in->fail();
52 }
53 
54 /**
55  * Starts reading from the indicated stream. Returns true on success, false
56  * on failure. The DatagramInputFile does not take ownership of the stream;
57  * you are responsible for closing or deleting it when you are done.
58  */
60 open(std::istream &in, const Filename &filename) {
61  close();
62 
63  _in = ∈
64  _owns_in = false;
65  _filename = filename;
66  _timestamp = 0;
67 
68  if (!filename.empty()) {
69  _file = new FileReference(filename);
70  }
71 
72  return !_in->fail();
73 }
74 
75 /**
76  * Closes the file. This is also implicitly done when the DatagramInputFile
77  * destructs.
78  */
80 close() {
81  _vfile.clear();
82  if (_owns_in) {
84  vfs->close_read_file(_in);
85  }
86  _in = nullptr;
87  _owns_in = false;
88 
89  _file.clear();
90  _filename = Filename();
91  _timestamp = 0;
92 
93  _read_first_datagram = false;
94  _error = false;
95 }
96 
97 /**
98  * Reads a sequence of bytes from the beginning of the datagram file. This
99  * may be called any number of times after the file has been opened and before
100  * the first datagram is read. It may not be called once the first datagram
101  * has been read.
102  */
104 read_header(std::string &header, size_t num_bytes) {
105  nassertr(!_read_first_datagram, false);
106  nassertr(_in != nullptr, false);
107 
108  char *buffer = (char *)alloca(num_bytes);
109  nassertr(buffer != nullptr, false);
110 
111  _in->read(buffer, num_bytes);
112  if (_in->fail() || _in->eof()) {
113  return false;
114  }
115 
116  header = std::string(buffer, num_bytes);
118  return true;
119 }
120 
121 /**
122  * Reads the next datagram from the file. Returns true on success, false if
123  * there is an error or end of file.
124  */
127  nassertr(_in != nullptr, false);
128  _read_first_datagram = true;
129 
130  // First, get the size of the upcoming datagram.
131  StreamReader reader(_in, false);
132  uint32_t num_bytes_32 = reader.get_uint32();
133  if (_in->fail() || _in->eof()) {
134  return false;
135  }
136 
137  if (num_bytes_32 == 0) {
138  // A special case for a zero-length datagram: no need to try to read any
139  // data.
140  data.clear();
141  return true;
142  }
143 
144  size_t num_bytes = (size_t)num_bytes_32;
145  if (num_bytes_32 == (uint32_t)-1) {
146  // Another special case for a value larger than 32 bits.
147  uint64_t num_bytes_64 = reader.get_uint64();
148 
149  if (_in->fail() || _in->eof()) {
150  _error = true;
151  return false;
152  }
153 
154  num_bytes = (size_t)num_bytes_64;
155 
156  // Make sure we have a reasonable datagram size for putting into memory.
157  if (num_bytes_64 != (uint64_t)num_bytes) {
158  _error = true;
159  return false;
160  }
161  }
162 
163  // Now, read the datagram itself. We construct an empty datagram, use
164  // pad_bytes to make it big enough, and read *directly* into the datagram's
165  // internal buffer. Doing this saves us a copy operation.
166  data = Datagram();
167 
168  size_t bytes_read = 0;
169  while (bytes_read < num_bytes) {
170  size_t bytes_left = num_bytes - bytes_read;
171 
172  // Hold up a second - datagrams >4MB are pretty large by bam/network
173  // standards. Let's take it 4MB at a time just in case the length is
174  // corrupt, so we don't allocate potentially a few GBs of RAM only to
175  // find a truncated file.
176  bytes_left = std::min(bytes_left, (size_t)4*1024*1024);
177 
178  PTA_uchar buffer = data.modify_array();
179  buffer.resize(buffer.size() + bytes_left);
180  unsigned char *ptr = &buffer.p()[bytes_read];
181 
182  _in->read((char *)ptr, (streamsize)bytes_left);
183  if (_in->fail() || _in->eof()) {
184  _error = true;
185  return false;
186  }
187 
188  bytes_read += bytes_left;
189  }
190 
192 
193  return true;
194 }
195 
196 /**
197  * Skips over the next datagram without extracting it, but saves the relevant
198  * file information in the SubfileInfo object so that its data may be read
199  * later. For non-file-based datagram generators, this may mean creating a
200  * temporary file and copying the contents of the datagram to disk.
201  *
202  * Returns true on success, false on failure or if this method is
203  * unimplemented.
204  */
207  nassertr(_in != nullptr, false);
208  _read_first_datagram = true;
209 
210  // First, get the size of the upcoming datagram.
211  StreamReader reader(_in, false);
212  size_t num_bytes_32 = reader.get_uint32();
213  if (_in->fail() || _in->eof()) {
214  return false;
215  }
216 
217  streamsize num_bytes = (streamsize)num_bytes_32;
218  if (num_bytes_32 == (uint32_t)-1) {
219  // Another special case for a value larger than 32 bits.
220  num_bytes = reader.get_uint64();
221  }
222 
223  // If this stream is file-based, we can just point the SubfileInfo directly
224  // into this file.
225  if (_file != nullptr) {
226  info = SubfileInfo(_file, _in->tellg(), num_bytes);
227  _in->seekg(num_bytes, std::ios::cur);
228  return true;
229  }
230 
231  // Otherwise, we have to dump the data into a temporary file.
232  PT(TemporaryFile) tfile = new TemporaryFile(Filename::temporary("", ""));
233  pofstream out;
234  Filename filename = tfile->get_filename();
235  filename.set_binary();
236  if (!filename.open_write(out)) {
237  util_cat.error()
238  << "Couldn't write to " << tfile->get_filename() << "\n";
239  return false;
240  }
241 
242  if (util_cat.is_debug()) {
243  util_cat.debug()
244  << "Copying " << num_bytes << " bytes to " << tfile->get_filename() << "\n";
245  }
246 
247  streamsize num_remaining = num_bytes;
248  static const size_t buffer_size = 4096;
249  char buffer[buffer_size];
250 
251  _in->read(buffer, std::min((streamsize)buffer_size, num_remaining));
252  streamsize count = _in->gcount();
253  while (count != 0) {
254  out.write(buffer, count);
255  if (out.fail()) {
256  util_cat.error()
257  << "Couldn't write " << num_bytes << " bytes to "
258  << tfile->get_filename() << "\n";
259  return false;
260  }
261  num_remaining -= count;
262  if (num_remaining == 0) {
263  break;
264  }
265  _in->read(buffer, std::min((streamsize)buffer_size, num_remaining));
266  count = _in->gcount();
267  }
268 
269  if (num_remaining != 0) {
270  util_cat.error()
271  << "Truncated data stream.\n";
272  return false;
273  }
274 
275  info = SubfileInfo(tfile, 0, num_bytes);
276  return true;
277 }
278 
279 /**
280  * Returns true if the file has reached the end-of-file. This test may only
281  * be made after a call to read_header() or get_datagram() has failed.
282  */
285  return _in != nullptr ? _in->eof() : true;
286 }
287 
288 /**
289  * Returns true if the file has reached an error condition.
290  */
293  if (_in == nullptr) {
294  return true;
295  }
296 
297  if (_in->fail()) {
298  _error = true;
299  }
300  return _error;
301 }
302 
303 /**
304  * Returns the filename that provides the source for these datagrams, if any,
305  * or empty string if the datagrams do not originate from a file on disk.
306  */
309  return _filename;
310 }
311 
312 /**
313  * Returns the on-disk timestamp of the file that was read, at the time it was
314  * opened, if that is available, or 0 if it is not.
315  */
316 time_t DatagramInputFile::
317 get_timestamp() const {
318  return _timestamp;
319 }
320 
321 /**
322  * Returns the FileReference that provides the source for these datagrams, if
323  * any, or NULL if the datagrams do not originate from a file on disk.
324  */
327  return _file;
328 }
329 
330 /**
331  * Returns the VirtualFile that provides the source for these datagrams, if
332  * any, or NULL if the datagrams do not originate from a VirtualFile.
333  */
336  return _vfile;
337 }
338 
339 /**
340  * Returns the current file position within the data stream, if any, or 0 if
341  * the file position is not meaningful or cannot be determined.
342  *
343  * For DatagramInputFiles that return a meaningful file position, this will be
344  * pointing to the first byte following the datagram returned after a call to
345  * get_datagram().
346  */
347 streampos DatagramInputFile::
349  if (_in == nullptr) {
350  return 0;
351  }
352  return _in->tellg();
353 }
bool open_write(std::ofstream &stream, bool truncate=true) const
Opens the indicated ifstream for writing the file, if possible.
Definition: filename.cxx:1899
Keeps a reference-counted pointer to a file on disk.
Definition: fileReference.h:26
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
virtual bool is_error()
Returns true if the file has reached an error condition.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
A hierarchy of directories and files that appears to be one continuous file system, even though the files may originate from several different sources that may not be related to the actual OS&#39;s file system.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
virtual std::streampos get_file_pos()
Returns the current file position within the data stream, if any, or 0 if the file position is not me...
void set_binary()
Indicates that the filename represents a binary file.
Definition: filename.I:414
bool read_header(std::string &header, size_t num_bytes)
Reads a sequence of bytes from the beginning of the datagram file.
virtual bool save_datagram(SubfileInfo &info)
Skips over the next datagram without extracting it, but saves the relevant file information in the Su...
uint64_t get_uint64()
Extracts an unsigned 64-bit integer.
Definition: streamReader.I:163
void close()
Closes the file.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
bool open(const FileReference *file)
Opens the indicated filename for reading.
static void consider_yield()
Possibly suspends the current thread for the rest of the current epoch, if it has run for enough this...
Definition: thread.I:212
The abstract base class for a file or directory within the VirtualFileSystem.
Definition: virtualFile.h:35
virtual bool get_datagram(Datagram &data)
Reads the next datagram from the file.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
virtual VirtualFile * get_vfile()
Returns the VirtualFile that provides the source for these datagrams, if any, or NULL if the datagram...
virtual bool is_eof()
Returns true if the file has reached the end-of-file.
static void close_read_file(std::istream *stream)
Closes a file opened by a previous call to open_read_file().
This is a special kind of FileReference class that automatically deletes the file in question when it...
Definition: temporaryFile.h:26
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:39
uint32_t get_uint32()
Extracts an unsigned 32-bit integer.
Definition: streamReader.I:151
static VirtualFileSystem * get_global_ptr()
Returns the default global VirtualFileSystem.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PointerTo< VirtualFile > get_file(const Filename &filename, bool status_only=false) const
Looks up the file by the indicated name in the file system.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
virtual const Filename & get_filename()
Returns the filename that provides the source for these datagrams, if any, or empty string if the dat...
This class records a particular byte sub-range within an existing file on disk.
Definition: subfileInfo.h:26
static Filename temporary(const std::string &dirname, const std::string &prefix, const std::string &suffix=std::string(), Type type=T_general)
Generates a temporary filename within the indicated directory, using the indicated prefix...
Definition: filename.cxx:424
virtual const FileReference * get_file()
Returns the FileReference that provides the source for these datagrams, if any, or NULL if the datagr...
A class to read sequential binary data directly from an istream.
Definition: streamReader.h:28
An ordered list of data elements, formatted in memory for transmission over a socket or writing to a ...
Definition: datagram.h:38
virtual time_t get_timestamp() const
Returns the on-disk timestamp of the file that was read, at the time it was opened, if that is available, or 0 if it is not.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.