Panda3D
openCVTexture.cxx
Go to the documentation of this file.
1 /**
2  * PANDA 3D SOFTWARE
3  * Copyright (c) Carnegie Mellon University. All rights reserved.
4  *
5  * All use of this software is subject to the terms of the revised BSD
6  * license. You should have received a copy of this license along
7  * with this source code in a file named "LICENSE."
8  *
9  * @file openCVTexture.cxx
10  * @author zacpavlov
11  * @date 2005-08-19
12  */
13 
14 #include "pandabase.h"
15 
16 #ifdef HAVE_OPENCV
17 #include "openCVTexture.h"
18 #include "clockObject.h"
19 #include "config_gobj.h"
20 #include "config_vision.h"
21 #include "bamReader.h"
22 #include "bamCacheRecord.h"
23 
24 // This symbol is predefined by the Panda3D build system to select whether we
25 // are using the OpenCV 2.3 or later interface, or if it is not defined, we
26 // are using the original interface.
27 #ifdef OPENCV_VER_23
28 
29 #include <opencv2/core/core.hpp>
30 // #include <opencv2videovideo.hpp>
31 #include <opencv2/highgui/highgui.hpp>
32 
33 #else
34 #include <cv.h>
35 #include <cxcore.h>
36 #include <highgui.h>
37 
38 #endif // OPENCV_VER_23
39 
40 TypeHandle OpenCVTexture::_type_handle;
41 
42 /**
43  * Sets up the texture to read frames from a camera
44  */
45 OpenCVTexture::
46 OpenCVTexture(const std::string &name) :
47  VideoTexture(name)
48 {
49 }
50 
51 /**
52  *
53  */
54 OpenCVTexture::
55 ~OpenCVTexture() {
56 }
57 
58 /**
59  * Calls update_frame() if the current frame has changed.
60  */
61 void OpenCVTexture::
62 consider_update() {
63  int this_frame = ClockObject::get_global_clock()->get_frame_count();
64  if (this_frame != _last_frame_update) {
65  int frame = get_frame();
66  if (_current_frame != frame) {
67  Texture::CDWriter cdata(Texture::_cycler, false);
68  do_update_frame(cdata, frame);
69  _current_frame = frame;
70  } else {
71  // Loop through the pages to see if there's any camera stream to update.
72  Texture::CDWriter cdata(Texture::_cycler, false);
73  int max_z = std::max(cdata->_z_size, (int)_pages.size());
74  for (int z = 0; z < max_z; ++z) {
75  VideoPage &page = _pages[z];
76  if (!page._color.is_from_file() || !page._alpha.is_from_file()) {
77  do_update_frame(cdata, frame, z);
78  }
79  }
80  }
81  _last_frame_update = this_frame;
82  }
83 }
84 
85 /**
86  * Returns a new copy of the same Texture. This copy, if applied to geometry,
87  * will be copied into texture as a separate texture from the original, so it
88  * will be duplicated in texture memory (and may be independently modified if
89  * desired).
90  *
91  * If the Texture is an OpenCVTexture, the resulting duplicate may be animated
92  * independently of the original.
93  */
94 PT(Texture) OpenCVTexture::
95 make_copy_impl() const {
96  Texture::CDReader cdata_tex(Texture::_cycler);
97  PT(OpenCVTexture) copy = new OpenCVTexture(get_name());
98  Texture::CDWriter cdata_copy_tex(copy->Texture::_cycler, true);
99  copy->do_assign(cdata_copy_tex, this, cdata_tex);
100 
101  return copy;
102 }
103 
104 /**
105  * Implements make_copy().
106  */
107 void OpenCVTexture::
108 do_assign(Texture::CData *cdata_tex, const OpenCVTexture *copy,
109  const Texture::CData *cdata_copy_tex) {
110  VideoTexture::do_assign(cdata_tex, copy, cdata_copy_tex);
111  _pages = copy->_pages;
112 }
113 
114 /**
115  * Sets up the OpenCVTexture (or the indicated page, if z is specified) to
116  * accept its input from the camera with the given index number, or the
117  * default camera if the index number is -1 or unspecified.
118  *
119  * If alpha_file_channel is 0, then the camera image becomes a normal RGB
120  * texture. If it is 1, 2, or 3, then the camera image becomes an alpha
121  * texture, using the indicated channel of the source.
122  */
123 bool OpenCVTexture::
124 from_camera(int camera_index, int z, int alpha_file_channel,
125  const LoaderOptions &options) {
126  Texture::CDWriter cdata(Texture::_cycler, true);
127  if (!do_reconsider_z_size(cdata, z, options)) {
128  return false;
129  }
130  nassertr(z >= 0 && z < cdata->_z_size, false);
131 
132  cdata->_alpha_file_channel = alpha_file_channel;
133 
134  VideoPage &page = do_modify_page(cdata, z);
135  if (alpha_file_channel == 0) {
136  // A normal RGB texture.
137  page._alpha.clear();
138  if (!page._color.from_camera(camera_index)) {
139  return false;
140  }
141 
142  if (!do_reconsider_video_properties(cdata, page._color, 3, z, options)) {
143  page._color.clear();
144  return false;
145  }
146  } else {
147  // An alpha texture.
148  page._color.clear();
149  if (!page._alpha.from_camera(camera_index)) {
150  return false;
151  }
152 
153  if (!do_reconsider_video_properties(cdata, page._alpha, 1, z, options)) {
154  page._alpha.clear();
155  return false;
156  }
157  do_set_format(cdata, F_alpha);
158  }
159 
160  cdata->_loaded_from_image = true;
161  clear_current_frame();
162  do_update_frame(cdata, 0);
163  return true;
164 }
165 
166 /**
167  * Returns a reference to the zth VideoPage (level) of the texture. In the
168  * case of a 2-d texture, there is only one page, level 0; but cube maps and
169  * 3-d textures have more.
170  */
171 OpenCVTexture::VideoPage &OpenCVTexture::
172 do_modify_page(const Texture::CData *cdata, int z) {
173  nassertr(z < cdata->_z_size, _pages[0]);
174  while (z >= (int)_pages.size()) {
175  _pages.push_back(VideoPage());
176  }
177  return _pages[z];
178 }
179 
180 /**
181  * Resets the internal Texture properties when a new video file is loaded.
182  * Returns true if the new image is valid, false otherwise.
183  */
184 bool OpenCVTexture::
185 do_reconsider_video_properties(Texture::CData *cdata,
186  const OpenCVTexture::VideoStream &stream,
187  int num_components, int z,
188  const LoaderOptions &options) {
189  double frame_rate = 0.0f;
190  int num_frames = 0;
191 
192  if (stream.is_from_file()) {
193  frame_rate = cvGetCaptureProperty(stream._capture, CV_CAP_PROP_FPS);
194  num_frames = (int)cvGetCaptureProperty(stream._capture, CV_CAP_PROP_FRAME_COUNT);
195  if (vision_cat.is_debug()) {
196  vision_cat.debug()
197  << "Loaded " << stream._filename << ", " << num_frames << " frames at "
198  << frame_rate << " fps\n";
199  }
200  } else {
201  // In this case, we don't have a specific frame rate or number of frames.
202  // Let both values remain at 0.
203  if (vision_cat.is_debug()) {
204  vision_cat.debug()
205  << "Loaded camera stream\n";
206  }
207  }
208 
209  int width = (int)cvGetCaptureProperty(stream._capture, CV_CAP_PROP_FRAME_WIDTH);
210  int height = (int)cvGetCaptureProperty(stream._capture, CV_CAP_PROP_FRAME_HEIGHT);
211 
212  int x_size = width;
213  int y_size = height;
214  do_adjust_this_size(cdata, x_size, y_size, get_name(), true);
215 
216  if (vision_cat.is_debug()) {
217  vision_cat.debug()
218  << "Video stream is " << width << " by " << height
219  << " pixels; fitting in texture " << x_size << " by "
220  << y_size << " texels.\n";
221  }
222 
223  if (!do_reconsider_image_properties(cdata, x_size, y_size, num_components,
224  T_unsigned_byte, z, options)) {
225  return false;
226  }
227 
228  if (cdata->_loaded_from_image &&
229  (get_video_width() != width || get_video_height() != height ||
230  get_num_frames() != num_frames || get_frame_rate() != frame_rate)) {
231  vision_cat.error()
232  << "Video properties have changed for texture " << get_name()
233  << " level " << z << ".\n";
234  return false;
235  }
236 
237  set_frame_rate(frame_rate);
238  set_num_frames(num_frames);
239  set_video_size(width, height);
240 
241  // By default, the newly-loaded video stream will immediately start looping.
242  loop(true);
243 
244  return true;
245 }
246 
247 /**
248  * A factory function to make a new OpenCVTexture, used to pass to the
249  * TexturePool.
250  */
251 PT(Texture) OpenCVTexture::
252 make_texture() {
253  return new OpenCVTexture;
254 }
255 
256 /**
257  * Called once per frame, as needed, to load the new image contents.
258  */
259 void OpenCVTexture::
260 do_update_frame(Texture::CData *cdata, int frame) {
261  int max_z = std::max(cdata->_z_size, (int)_pages.size());
262  for (int z = 0; z < max_z; ++z) {
263  do_update_frame(cdata, frame, z);
264  }
265 }
266 
267 /**
268  * This variant of update_frame updates the indicated page only.
269  */
270 void OpenCVTexture::
271 do_update_frame(Texture::CData *cdata, int frame, int z) {
272  if (vision_cat.is_spam()) {
273  vision_cat.spam()
274  << "Updating OpenCVTexture page " << z << "\n";
275  }
276 
277  VideoPage &page = _pages[z];
278  if (page._color.is_valid() || page._alpha.is_valid()) {
279  do_modify_ram_image(cdata);
280  ++(cdata->_image_modified);
281  }
282  int dest_x_pitch = cdata->_num_components * cdata->_component_width;
283  int dest_y_pitch = cdata->_x_size * dest_x_pitch;
284 
285  if (page._color.is_valid()) {
286  nassertv(get_num_components() >= 3 && get_component_width() == 1);
287 
288  const unsigned char *r, *g, *b;
289  int x_pitch, y_pitch;
290  if (page._color.get_frame_data(frame, r, g, b, x_pitch, y_pitch)) {
291  nassertv(get_video_width() <= cdata->_x_size && get_video_height() <= cdata->_y_size);
292  nassertv(!cdata->_ram_images.empty())
293  unsigned char *dest = cdata->_ram_images[0]._image.p() + do_get_expected_ram_page_size(cdata) * z;
294 
295  if (cdata->_num_components == 3 && x_pitch == 3) {
296  // The easy case--copy the whole thing in, row by row.
297  int copy_bytes = get_video_width() * dest_x_pitch;
298  nassertv(copy_bytes <= dest_y_pitch && copy_bytes <= abs(y_pitch));
299 
300  for (int y = 0; y < get_video_height(); ++y) {
301  memcpy(dest, r, copy_bytes);
302  dest += dest_y_pitch;
303  r += y_pitch;
304  }
305 
306  } else {
307  // The harder case--interleave in the color channels, pixel by pixel,
308  // possibly leaving room for alpha.
309 
310  for (int y = 0; y < get_video_height(); ++y) {
311  int dx = 0;
312  int sx = 0;
313  for (int x = 0; x < get_video_width(); ++x) {
314  dest[dx] = r[sx];
315  dest[dx + 1] = g[sx];
316  dest[dx + 2] = b[sx];
317  dx += dest_x_pitch;
318  sx += x_pitch;
319  }
320  dest += dest_y_pitch;
321  r += y_pitch;
322  g += y_pitch;
323  b += y_pitch;
324  }
325  }
326  }
327  }
328  if (page._alpha.is_valid()) {
329  nassertv(get_component_width() == 1);
330 
331  const unsigned char *source[3];
332  int x_pitch, y_pitch;
333  if (page._alpha.get_frame_data(frame, source[0], source[1], source[2],
334  x_pitch, y_pitch)) {
335  nassertv(get_video_width() <= cdata->_x_size && get_video_height() <= cdata->_y_size);
336  nassertv(!cdata->_ram_images.empty())
337  unsigned char *dest = cdata->_ram_images[0]._image.p() + do_get_expected_ram_page_size(cdata) * z;
338 
339  // Interleave the alpha in with the color, pixel by pixel. Even though
340  // the alpha will probably be a grayscale video, the OpenCV library
341  // presents it as RGB.
342  const unsigned char *sch = source[0];
343  if (cdata->_alpha_file_channel >= 1 && cdata->_alpha_file_channel <= 3) {
344  sch = source[cdata->_alpha_file_channel - 1];
345  }
346 
347  for (int y = 0; y < get_video_height(); ++y) {
348  // Start dx at _num_components - 1, which writes to the last channel,
349  // i.e. the alpha channel.
350  int dx = (cdata->_num_components - 1) * cdata->_component_width;
351  int sx = 0;
352  for (int x = 0; x < get_video_width(); ++x) {
353  dest[dx] = sch[sx];
354  dx += dest_x_pitch;
355  sx += x_pitch;
356  }
357  dest += dest_y_pitch;
358  sch += y_pitch;
359  }
360  }
361  }
362 }
363 
364 /**
365  * Combines a color and alpha video image from the two indicated filenames.
366  * Both must be the same kind of video with similar properties.
367  */
368 bool OpenCVTexture::
369 do_read_one(Texture::CData *cdata,
370  const Filename &fullpath, const Filename &alpha_fullpath,
371  int z, int n, int primary_file_num_channels, int alpha_file_channel,
372  const LoaderOptions &options,
373  bool header_only, BamCacheRecord *record) {
374  if (record != nullptr) {
375  record->add_dependent_file(fullpath);
376  }
377 
378  nassertr(n == 0, false);
379  nassertr(z >= 0 && z < cdata->_z_size, false);
380 
381  VideoPage &page = do_modify_page(cdata, z);
382  if (!page._color.read(fullpath)) {
383  vision_cat.error()
384  << "OpenCV couldn't read " << fullpath << " as video.\n";
385  return false;
386  }
387  if (!alpha_fullpath.empty()) {
388  if (!page._alpha.read(alpha_fullpath)) {
389  vision_cat.error()
390  << "OpenCV couldn't read " << alpha_fullpath << " as video.\n";
391  page._color.clear();
392  return false;
393  }
394  }
395 
396  if (z == 0) {
397  if (!has_name()) {
398  set_name(fullpath.get_basename_wo_extension());
399  }
400  // Don't use has_filename() here, it will cause a deadlock
401  if (cdata->_filename.empty()) {
402  cdata->_filename = fullpath;
403  cdata->_alpha_filename = alpha_fullpath;
404  }
405 
406  cdata->_fullpath = fullpath;
407  cdata->_alpha_fullpath = alpha_fullpath;
408  }
409 
410  cdata->_primary_file_num_channels = 3;
411  cdata->_alpha_file_channel = 0;
412 
413  if (alpha_fullpath.empty()) {
414  // Only one RGB movie.
415  if (!do_reconsider_video_properties(cdata, page._color, 3, z, options)) {
416  page._color.clear();
417  return false;
418  }
419 
420  } else {
421  // An RGB movie combined with an alpha movie.
422  cdata->_alpha_file_channel = alpha_file_channel;
423 
424  if (!do_reconsider_video_properties(cdata, page._color, 4, z, options)) {
425  page._color.clear();
426  page._alpha.clear();
427  return false;
428  }
429 
430  if (!do_reconsider_video_properties(cdata, page._alpha, 4, z, options)) {
431  page._color.clear();
432  page._alpha.clear();
433  return false;
434  }
435  }
436 
437  set_loaded_from_image();
438  clear_current_frame();
439  do_update_frame(cdata, 0);
440  return true;
441 }
442 
443 /**
444  * Resets the texture (or the particular level of the texture) to the
445  * indicated static image.
446  */
447 bool OpenCVTexture::
448 do_load_one(Texture::CData *cdata,
449  const PNMImage &pnmimage, const std::string &name,
450  int z, int n, const LoaderOptions &options) {
451  if (z <= (int)_pages.size()) {
452  VideoPage &page = do_modify_page(cdata, z);
453  page._color.clear();
454  page._alpha.clear();
455  }
456 
457  return Texture::do_load_one(cdata, pnmimage, name, z, n, options);
458 }
459 
460 /**
461  * Factory method to generate a Texture object
462  */
463 void OpenCVTexture::
464 register_with_read_factory() {
465  // Since Texture is such a funny object that is reloaded from the
466  // TexturePool each time, instead of actually being read fully from the bam
467  // file, and since the VideoTexture and OpenCVTexture classes don't really
468  // add any useful data to the bam record, we don't need to define
469  // make_from_bam(), fillin(), or write_datagram() in this class--we just
470  // inherit the same functions from Texture.
471 
472  // We do, however, have to register this class with the BamReader, to avoid
473  // warnings about creating the wrong kind of object from the bam file.
474  BamReader::get_factory()->register_factory(get_class_type(), make_from_bam);
475 }
476 
477 /**
478  *
479  */
480 OpenCVTexture::VideoStream::
481 VideoStream() :
482  _capture(nullptr),
483  _camera_index(-1),
484  _next_frame(0)
485 {
486 }
487 
488 /**
489  *
490  */
491 OpenCVTexture::VideoStream::
492 VideoStream(const OpenCVTexture::VideoStream &copy) :
493  _capture(nullptr),
494  _camera_index(-1)
495 {
496  // Rather than copying the _capture pointer, we must open a new stream that
497  // references the same file.
498  if (copy.is_valid()) {
499  if (copy.is_from_file()) {
500  read(copy._filename);
501  } else {
502  from_camera(copy._camera_index);
503  }
504  }
505 }
506 
507 /**
508  *
509  */
510 OpenCVTexture::VideoStream::
511 ~VideoStream() {
512  clear();
513 }
514 
515 /**
516  * Gets the data needed to traverse through the decompressed buffer for the
517  * indicated frame number. It is most efficient to call this in increasing
518  * order of frame number. Returns true on success, false on failure.
519  *
520  * In the case of a success indication (true return value), the three pointers
521  * r, g, b are loaded with the addresses of the three components of the
522  * bottom-left pixel of the image. (They will be adjacent in memory in the
523  * case of an interleaved image, and separated in the case of a separate-
524  * channel image.) The x_pitch value is filled with the amount to add to each
525  * pointer to advance to the pixel to the right; and the y_pitch value is
526  * filled with the amount to add to each pointer to advance to the pixel
527  * above. Note that these values may be negative (particularly in the case of
528  * a top-down image).
529  */
530 bool OpenCVTexture::VideoStream::
531 get_frame_data(int frame,
532  const unsigned char *&r,
533  const unsigned char *&g,
534  const unsigned char *&b,
535  int &x_pitch, int &y_pitch) {
536  nassertr(is_valid(), false);
537 
538  if (is_from_file() && _next_frame != frame) {
539  cvSetCaptureProperty(_capture, CV_CAP_PROP_POS_FRAMES, frame);
540  }
541 
542  _next_frame = frame + 1;
543  IplImage *image = cvQueryFrame(_capture);
544  if (image == nullptr) {
545  return false;
546  }
547 
548  r = (const unsigned char *)image->imageData;
549  g = r + 1;
550  b = g + 1;
551  x_pitch = 3;
552  y_pitch = image->widthStep;
553 
554  if (image->dataOrder == 1) {
555  // Separate channel images. That means a block of r, followed by a block
556  // of g, followed by a block of b.
557  x_pitch = 1;
558  g = r + image->height * y_pitch;
559  b = g + image->height * y_pitch;
560  }
561 
562  if (image->origin == 0) {
563  // The image data starts with the top row and ends with the bottom row--
564  // the opposite of Texture::_ram_data's storage convention. Therefore, we
565  // must increment the initial pointers to the last row, and count
566  // backwards.
567  r += (image->height - 1) * y_pitch;
568  g += (image->height - 1) * y_pitch;
569  b += (image->height - 1) * y_pitch;
570  y_pitch = -y_pitch;
571  }
572 
573  return true;
574 }
575 
576 /**
577  * Sets up the stream to read the indicated file. Returns true on success,
578  * false on failure.
579  */
580 bool OpenCVTexture::VideoStream::
581 read(const Filename &filename) {
582  clear();
583 
584  std::string os_specific = filename.to_os_specific();
585  _capture = cvCaptureFromFile(os_specific.c_str());
586  if (_capture == nullptr) {
587  return false;
588  }
589  _filename = filename;
590  return true;
591 }
592 
593 /**
594  * Sets up the stream to display the indicated camera. Returns true on
595  * success, false on failure.
596  */
597 bool OpenCVTexture::VideoStream::
598 from_camera(int camera_index) {
599  clear();
600 
601  _capture = cvCaptureFromCAM(camera_index);
602  if (_capture == nullptr) {
603  return false;
604  }
605  _camera_index = camera_index;
606  return true;
607 }
608 
609 /**
610  * Stops the video playback and frees the associated resources.
611  */
612 void OpenCVTexture::VideoStream::
613 clear() {
614  if (_capture != nullptr) {
615  cvReleaseCapture(&_capture);
616  _capture = nullptr;
617  }
618  _filename = Filename();
619  _camera_index = -1;
620  _next_frame = 0;
621 }
622 
623 #endif // HAVE_OPENCV
static ClockObject * get_global_clock()
Returns a pointer to the global ClockObject.
Definition: clockObject.I:215
The name of this class derives from the fact that we originally implemented it as a layer on top of t...
Definition: pnmImage.h:58
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
Specifies parameters that may be passed to the loader.
Definition: loaderOptions.h:23
std::string get_basename_wo_extension() const
Returns the basename part of the filename, without the file extension.
Definition: filename.I:386
Represents a texture object, which is typically a single 2-d image but may also represent a 1-d or 3-...
Definition: texture.h:71
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
The base class for a family of animated Textures that take their input from a video source...
Definition: videoTexture.h:28
This template class calls PipelineCycler::read_unlocked(), and then provides a transparent read-only ...
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
get_frame_count
Returns the number of times tick() has been called since the ClockObject was created, or since it was last reset.
Definition: clockObject.h:94
The name of a file, such as a texture file or an Egg file.
Definition: filename.h:39
An instance of this class is written to the front of a Bam or Txo file to make the file a cached inst...
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
void register_factory(TypeHandle handle, CreateFunc *func, void *user_data=nullptr)
Registers a new kind of thing the Factory will be able to create.
Definition: factory.I:73
This template class calls PipelineCycler::write() in the constructor and PipelineCycler::release_writ...
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
static WritableFactory * get_factory()
Returns the global WritableFactory for generating TypedWritable objects.
Definition: bamReader.I:177
void add_dependent_file(const Filename &pathname)
Adds the indicated file to the list of files that will be loaded to generate the data in this record...
TypeHandle is the identifier used to differentiate C++ class types.
Definition: typeHandle.h:81
std::string to_os_specific() const
Converts the filename from our generic Unix-like convention (forward slashes starting with the root a...
Definition: filename.cxx:1123