Panda3D
ffmpegVideoCursor.cxx
Go to the documentation of this file.
1 /**
2  * PANDA 3D SOFTWARE
3  * Copyright (c) Carnegie Mellon University. All rights reserved.
4  *
5  * All use of this software is subject to the terms of the revised BSD
6  * license. You should have received a copy of this license along
7  * with this source code in a file named "LICENSE."
8  *
9  * @file ffmpegVideoCursor.cxx
10  * @author jyelon
11  * @date 2007-08-01
12  */
13 
14 #include "ffmpegVideoCursor.h"
15 #include "config_ffmpeg.h"
16 #include "pStatCollector.h"
17 #include "pStatTimer.h"
18 #include "mutexHolder.h"
19 #include "reMutexHolder.h"
20 #include "ffmpegVideo.h"
21 #include "bamReader.h"
22 extern "C" {
23  #include <libavcodec/avcodec.h>
24  #include <libavformat/avformat.h>
25  #include <libavutil/pixdesc.h>
26 #ifdef HAVE_SWSCALE
27  #include <libswscale/swscale.h>
28 #endif
29 }
30 
31 ReMutex FfmpegVideoCursor::_av_lock;
32 TypeHandle FfmpegVideoCursor::_type_handle;
33 TypeHandle FfmpegVideoCursor::FfmpegBuffer::_type_handle;
34 
35 PStatCollector FfmpegVideoCursor::_fetch_buffer_pcollector("*:FFMPEG Video Decoding:Fetch");
36 PStatCollector FfmpegVideoCursor::_seek_pcollector("*:FFMPEG Video Decoding:Seek");
37 PStatCollector FfmpegVideoCursor::_export_frame_pcollector("*:FFMPEG Convert Video to BGR");
38 
39 #if LIBAVUTIL_VERSION_INT < AV_VERSION_INT(52, 32, 100)
40  #define AV_PIX_FMT_FLAG_ALPHA PIX_FMT_ALPHA
41 #endif
42 
43 /**
44  * This constructor is only used when reading from a bam file.
45  */
46 FfmpegVideoCursor::
47 FfmpegVideoCursor() :
48  _max_readahead_frames(0),
49  _thread_priority(ffmpeg_thread_priority),
50  _lock("FfmpegVideoCursor::_lock"),
51  _action_cvar(_lock),
52  _thread_status(TS_stopped),
53  _seek_frame(0),
54  _packet(nullptr),
55  _format_ctx(nullptr),
56  _video_ctx(nullptr),
57  _convert_ctx(nullptr),
58  _pixel_format((int)AV_PIX_FMT_NONE),
59  _video_index(-1),
60  _frame(nullptr),
61  _frame_out(nullptr),
62  _eof_known(false)
63 {
64 }
65 
66 /**
67  * Specifies the source of the video cursor. This is normally called only by
68  * the constructor or when reading from a bam file.
69  */
70 void FfmpegVideoCursor::
71 init_from(FfmpegVideo *source) {
72  nassertv(_thread == nullptr && _thread_status == TS_stopped);
73  nassertv(source != nullptr);
74  _source = source;
75  _filename = _source->get_filename();
76 
77  if (!open_stream()) {
78  cleanup();
79  return;
80  }
81 
82 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 45, 101)
83  _frame = av_frame_alloc();
84  _frame_out = av_frame_alloc();
85 #else
86  _frame = avcodec_alloc_frame();
87  _frame_out = avcodec_alloc_frame();
88 #endif
89 
90  if ((_frame == nullptr)||(_frame_out == nullptr)) {
91  cleanup();
92  return;
93  }
94 
95 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
96  _packet = av_packet_alloc();
97 #else
98  _packet = new AVPacket;
99  av_init_packet(_packet);
100 #endif
101 
102  fetch_packet(0);
103  fetch_frame(-1);
104  _initial_dts = _begin_frame;
105 
106  _current_frame = -1;
107  _eof_known = false;
108  _eof_frame = 0;
109 
110  ReMutexHolder av_holder(_av_lock);
111 
112  // Check if we got an alpha format. Please note that some video codecs
113  // (eg. libvpx) change the pix_fmt after decoding the first frame, which is
114  // why we didn't do this earlier.
115  switch (_video_ctx->pix_fmt) {
116  case AV_PIX_FMT_GRAY8:
117  _num_components = 1;
118  _pixel_format = (int)AV_PIX_FMT_GRAY8;
119  break;
120  case AV_PIX_FMT_Y400A: // aka AV_PIX_FMT_YA8
121  _num_components = 2;
122  _pixel_format = (int)AV_PIX_FMT_Y400A;
123  break;
124  default:
125  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(_video_ctx->pix_fmt);
126  if (desc && (desc->flags & AV_PIX_FMT_FLAG_ALPHA) != 0) {
127  _num_components = 4;
128  _pixel_format = (int)AV_PIX_FMT_BGRA;
129  } else {
130  _num_components = 3;
131  _pixel_format = (int)AV_PIX_FMT_BGR24;
132  }
133  break;
134  }
135 
136 #ifdef HAVE_SWSCALE
137  nassertv(_convert_ctx == nullptr);
138  _convert_ctx = sws_getContext(_size_x, _size_y, _video_ctx->pix_fmt,
139  _size_x, _size_y, (AVPixelFormat)_pixel_format,
140  SWS_BILINEAR | SWS_PRINT_INFO, nullptr, nullptr, nullptr);
141 #endif // HAVE_SWSCALE
142 
143 #ifdef HAVE_THREADS
144  set_max_readahead_frames(ffmpeg_max_readahead_frames);
145 #endif // HAVE_THREADS
146 }
147 
148 /**
149  *
150  */
151 FfmpegVideoCursor::
152 FfmpegVideoCursor(FfmpegVideo *src) :
153  _max_readahead_frames(0),
154  _thread_priority(ffmpeg_thread_priority),
155  _lock("FfmpegVideoCursor::_lock"),
156  _action_cvar(_lock),
157  _thread_status(TS_stopped),
158  _seek_frame(0),
159  _packet(nullptr),
160  _format_ctx(nullptr),
161  _video_ctx(nullptr),
162  _convert_ctx(nullptr),
163  _video_index(-1),
164  _frame(nullptr),
165  _frame_out(nullptr),
166  _eof_known(false)
167 {
168  init_from(src);
169 }
170 
171 /**
172  *
173  */
174 FfmpegVideoCursor::
175 ~FfmpegVideoCursor() {
176  cleanup();
177 }
178 
179 /**
180  * Specifies the maximum number of frames that a sub-thread will attempt to
181  * read ahead of the current frame. Setting this to a nonzero allows the
182  * video decoding to take place in a sub-thread, which smoothes out the video
183  * decoding time by spreading it evenly over several frames. Set this number
184  * larger to increase the buffer between the currently visible frame and the
185  * first undecoded frame; set it smaller to reduce memory consumption.
186  *
187  * Setting this to zero forces the video to be decoded in the main thread. If
188  * threading is not available in the Panda build, this value is always zero.
189  */
191 set_max_readahead_frames(int max_readahead_frames) {
192 #ifndef HAVE_THREADS
193  if (max_readahead_frames > 0) {
194  ffmpeg_cat.warning()
195  << "Couldn't set max_readahead_frames to " << max_readahead_frames
196  << ": threading not available.\n";
197  max_readahead_frames = 0;
198  }
199 #endif // HAVE_THREADS
200 
201  _max_readahead_frames = max_readahead_frames;
202  if (_max_readahead_frames > 0) {
203  if (_thread_status == TS_stopped) {
204  start_thread();
205  }
206  } else {
207  if (_thread_status != TS_stopped) {
208  stop_thread();
209  }
210  }
211 }
212 
213 /**
214  * Returns the maximum number of frames that a sub-thread will attempt to read
215  * ahead of the current frame. See set_max_readahead_frames().
216  */
218 get_max_readahead_frames() const {
219  return _max_readahead_frames;
220 }
221 
222 /**
223  * Changes the thread priority of the thread that decodes the ffmpeg video
224  * stream (if max_readahead_frames is nonzero). Normally you shouldn't mess
225  * with this, but there may be special cases where a precise balance of CPU
226  * utilization between the main thread and the various ffmpeg service threads
227  * may be needed.
228  */
230 set_thread_priority(ThreadPriority thread_priority) {
231  if (_thread_priority != thread_priority) {
232  _thread_priority = thread_priority;
233  if (is_thread_started()) {
234  stop_thread();
235  start_thread();
236  }
237  }
238 }
239 
240 /**
241  * Returns the current thread priority of the thread that decodes the ffmpeg
242  * video stream (if max_readahead_frames is nonzero). See
243  * set_thread_priority().
244  */
245 ThreadPriority FfmpegVideoCursor::
246 get_thread_priority() const {
247  return _thread_priority;
248 }
249 
250 /**
251  * Explicitly starts the ffmpeg decoding thread after it has been stopped by a
252  * call to stop_thread(). The thread is normally started automatically, so
253  * there is no need to call this method unless you have previously called
254  * stop_thread() for some reason.
255  */
257 start_thread() {
258  MutexHolder holder(_lock);
259 
260  if (_thread_status == TS_stopped && _max_readahead_frames > 0) {
261  // Get a unique name for the thread's sync name.
262  std::ostringstream strm;
263  strm << (void *)this;
264  _sync_name = strm.str();
265 
266  // Create and start the thread object.
267  _thread_status = TS_wait;
268  _thread = new GenericThread(_filename.get_basename(), _sync_name, st_thread_main, this);
269  if (!_thread->start(_thread_priority, true)) {
270  // Couldn't start the thread.
271  _thread = nullptr;
272  _thread_status = TS_stopped;
273  }
274  }
275 }
276 
277 /**
278  * Explicitly stops the ffmpeg decoding thread. There is normally no reason
279  * to do this unless you want to maintain precise control over what threads
280  * are consuming CPU resources. Calling this method will make the video
281  * update in the main thread, regardless of the setting of
282  * max_readahead_frames, until you call start_thread() again.
283  */
285 stop_thread() {
286  if (_thread_status != TS_stopped) {
287  PT(GenericThread) thread = _thread;
288  {
289  MutexHolder holder(_lock);
290  if (_thread_status != TS_stopped) {
291  _thread_status = TS_shutdown;
292  }
293  _action_cvar.notify();
294  _thread = nullptr;
295  }
296 
297  // Now that we've released the lock, we can join the thread.
298  thread->join();
299  }
300 
301  // This is a good time to clean up all of the allocated frame objects. It's
302  // not really necessary to be holding the lock, since the thread is gone,
303  // but we'll grab it anyway just in case someone else starts the thread up
304  // again.
305  MutexHolder holder(_lock);
306 
307  _readahead_frames.clear();
308 }
309 
310 /**
311  * Returns true if the thread has been started, false if not. This will
312  * always return false if max_readahead_frames is 0.
313  */
315 is_thread_started() const {
316  return (_thread_status != TS_stopped);
317 }
318 
319 /**
320  * See MovieVideoCursor::set_time().
321  */
323 set_time(double timestamp, int loop_count) {
324  int frame = (int)(timestamp / _video_timebase + 0.5);
325 
326  if (_eof_known) {
327  if (loop_count == 0) {
328  frame = frame % (_eof_frame + 1);
329  } else {
330  int last_frame = (_eof_frame + 1) * loop_count;
331  if (frame < last_frame) {
332  frame = frame % (_eof_frame + 1);
333  } else {
334  frame = _eof_frame;
335  }
336  }
337  }
338 
339  // No point in trying to position before the first frame.
340  frame = std::max(frame, _initial_dts);
341 
342  if (ffmpeg_cat.is_spam() && frame != _current_frame) {
343  ffmpeg_cat.spam()
344  << "set_time(" << timestamp << "): " << frame
345  << ", loop_count = " << loop_count << "\n";
346  }
347 
348  _current_frame = frame;
349  if (_current_frame_buffer != nullptr) {
350  // If we've previously returned a frame, don't bother asking for a next
351  // one if that frame is still valid.
352  return (_current_frame >= _current_frame_buffer->_end_frame ||
353  _current_frame < _current_frame_buffer->_begin_frame);
354  }
355 
356  // If our last request didn't return a frame, try again.
357  return true;
358 }
359 
360 /**
361  * See MovieVideoCursor::fetch_buffer.
362  */
363 PT(MovieVideoCursor::Buffer) FfmpegVideoCursor::
364 fetch_buffer() {
365  MutexHolder holder(_lock);
366 
367  // If there was an error at any point, just return NULL.
368  if (_format_ctx == nullptr) {
369  return nullptr;
370  }
371 
372  PT(FfmpegBuffer) frame;
373  if (_thread_status == TS_stopped) {
374  // Non-threaded case. Just get the next frame directly.
375  advance_to_frame(_current_frame);
376  if (_frame_ready) {
377  frame = do_alloc_frame();
378  export_frame(frame);
379  }
380 
381  } else {
382  // Threaded case. Wait for the thread to serve up the required frames.
383  if (!_readahead_frames.empty()) {
384  frame = _readahead_frames.front();
385  _readahead_frames.pop_front();
386  _action_cvar.notify();
387  while (frame->_end_frame < _current_frame && !_readahead_frames.empty()) {
388  // This frame is too old. Discard it.
389  if (ffmpeg_cat.is_debug()) {
390  ffmpeg_cat.debug()
391  << "ffmpeg for " << _filename.get_basename()
392  << " at frame " << _current_frame << ", discarding frame at "
393  << frame->_begin_frame << "\n";
394  }
395  frame = _readahead_frames.front();
396  _readahead_frames.pop_front();
397  }
398  if (frame->_begin_frame > _current_frame) {
399  // This frame is too new. Empty all remaining frames and seek
400  // backwards.
401  if (ffmpeg_cat.is_debug()) {
402  ffmpeg_cat.debug()
403  << "ffmpeg for " << _filename.get_basename()
404  << " at frame " << _current_frame << ", encountered too-new frame at "
405  << frame->_begin_frame << "\n";
406  }
407  do_clear_all_frames();
408  if (_thread_status == TS_wait || _thread_status == TS_seek || _thread_status == TS_readahead) {
409  _thread_status = TS_seek;
410  _seek_frame = _current_frame;
411  _action_cvar.notify();
412  }
413  }
414  }
415  if (frame == nullptr || frame->_end_frame < _current_frame) {
416  // No frame available, or the frame is too old. Seek.
417  if (_thread_status == TS_wait || _thread_status == TS_seek || _thread_status == TS_readahead) {
418  _thread_status = TS_seek;
419  _seek_frame = _current_frame;
420  _action_cvar.notify();
421  }
422  }
423  }
424 
425  if (frame != nullptr) {
426  bool too_old = (frame->_end_frame < _current_frame && !ffmpeg_show_seek_frames);
427  bool too_new = frame->_begin_frame > _current_frame;
428  if (too_old || too_new) {
429  // The frame is too old or too new. Just recycle it.
430  frame = nullptr;
431  }
432  }
433 
434  if (frame != nullptr) {
435  _current_frame_buffer = frame;
436  if (ffmpeg_cat.is_debug()) {
437  ffmpeg_cat.debug()
438  << "ffmpeg for " << _filename.get_basename()
439  << " at frame " << _current_frame << ", returning frame at "
440  << frame->_begin_frame << "\n";
441  }
442  } else {
443  if (ffmpeg_cat.is_debug()) {
444  ffmpeg_cat.debug()
445  << "ffmpeg for " << _filename.get_basename()
446  << " at frame " << _current_frame << ", returning NULL\n";
447  }
448  }
449  return frame;
450 }
451 
452 /**
453  * May be called by a derived class to allocate a new Buffer object.
454  */
455 PT(MovieVideoCursor::Buffer) FfmpegVideoCursor::
456 make_new_buffer() {
457  PT(FfmpegBuffer) frame = new FfmpegBuffer(size_x() * size_y() * get_num_components(), _video_timebase);
458  return frame;
459 }
460 
461 /**
462  * Opens the stream for the first time, or when needed internally.
463  */
464 bool FfmpegVideoCursor::
465 open_stream() {
466  nassertr(!_ffvfile.is_open(), false);
467 
468  // Hold the global lock while we open the file and create avcodec objects.
469  ReMutexHolder av_holder(_av_lock);
470 
471  if (!_source->get_subfile_info().is_empty()) {
472  // Read a subfile.
473  if (!_ffvfile.open_subfile(_source->get_subfile_info())) {
474  ffmpeg_cat.info()
475  << "Couldn't open " << _source->get_subfile_info() << "\n";
476  close_stream();
477  return false;
478  }
479 
480  } else {
481  // Read a filename.
482  if (!_ffvfile.open_vfs(_filename)) {
483  ffmpeg_cat.info()
484  << "Couldn't open " << _filename << "\n";
485  close_stream();
486  return false;
487  }
488  }
489 
490  nassertr(_format_ctx == nullptr, false);
491  _format_ctx = _ffvfile.get_format_context();
492  nassertr(_format_ctx != nullptr, false);
493 
494  if (avformat_find_stream_info(_format_ctx, nullptr) < 0) {
495  ffmpeg_cat.info()
496  << "Couldn't find stream info\n";
497  close_stream();
498  return false;
499  }
500 
501  nassertr(_video_ctx == nullptr, false);
502 
503  // As of libavformat version 57.41.100, AVStream.codec is deprecated in favor
504  // of AVStream.codecpar. Fortunately, the two structures have
505  // similarly-named members, so we can just switch out the declaration.
506 #if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
507  AVCodecParameters *codecpar;
508 #else
509  AVCodecContext *codecpar;
510 #endif
511 
512  // Find the video stream
513  AVStream *stream = nullptr;
514  for (int i = 0; i < (int)_format_ctx->nb_streams; ++i) {
515 #if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
516  codecpar = _format_ctx->streams[i]->codecpar;
517 #else
518  codecpar = _format_ctx->streams[i]->codec;
519 #endif
520  if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
521  _video_index = i;
522  stream = _format_ctx->streams[i];
523  break;
524  }
525  }
526 
527  if (stream == nullptr) {
528  ffmpeg_cat.info()
529  << "Couldn't find stream\n";
530  close_stream();
531  return false;
532  }
533 
534  _video_timebase = av_q2d(stream->time_base);
535  _min_fseek = (int)(3.0 / _video_timebase);
536 
537  const AVCodec *pVideoCodec = nullptr;
538  if (ffmpeg_prefer_libvpx) {
539 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 0, 0)
540  if (codecpar->codec_id == AV_CODEC_ID_VP9) {
541  pVideoCodec = avcodec_find_decoder_by_name("libvpx-vp9");
542  } else
543 #endif
544  if (codecpar->codec_id == AV_CODEC_ID_VP8) {
545  pVideoCodec = avcodec_find_decoder_by_name("libvpx");
546  }
547  }
548  if (pVideoCodec == nullptr) {
549  pVideoCodec = avcodec_find_decoder(codecpar->codec_id);
550  }
551  if (pVideoCodec == nullptr) {
552  ffmpeg_cat.info()
553  << "Couldn't find codec\n";
554  close_stream();
555  return false;
556  }
557 
558  _video_ctx = avcodec_alloc_context3(pVideoCodec);
559 
560  if (_video_ctx == nullptr) {
561  ffmpeg_cat.info()
562  << "Couldn't allocate _video_ctx\n";
563  close_stream();
564  return false;
565  }
566 
567 #if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
568  avcodec_parameters_to_context(_video_ctx, codecpar);
569 #else
570  avcodec_copy_context(_video_ctx, codecpar);
571 #endif
572 
573  if (avcodec_open2(_video_ctx, pVideoCodec, nullptr) < 0) {
574  ffmpeg_cat.info()
575  << "Couldn't open codec\n";
576  close_stream();
577  return false;
578  }
579 
580  _size_x = _video_ctx->width;
581  _size_y = _video_ctx->height;
582  _num_components = 3;
583  _length = (double)_format_ctx->duration / (double)AV_TIME_BASE;
584  _can_seek = true;
585  _can_seek_fast = true;
586 
587  return true;
588 }
589 
590 /**
591  * Closes the stream, during cleanup or when needed internally.
592  */
593 void FfmpegVideoCursor::
594 close_stream() {
595  // Hold the global lock while we free avcodec objects.
596  ReMutexHolder av_holder(_av_lock);
597 
598  if (_video_ctx && _video_ctx->codec) {
599 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100)
600  // We need to drain the codec to prevent a memory leak.
601  avcodec_send_packet(_video_ctx, nullptr);
602  while (avcodec_receive_frame(_video_ctx, _frame) == 0) {}
603  avcodec_flush_buffers(_video_ctx);
604 #endif
605 
606  avcodec_close(_video_ctx);
607 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 52, 0)
608  avcodec_free_context(&_video_ctx);
609 #else
610  av_free(_video_ctx);
611 #endif
612  }
613  _video_ctx = nullptr;
614 
615  _ffvfile.close();
616  _format_ctx = nullptr;
617 
618  _video_index = -1;
619 }
620 
621 /**
622  * Reset to a standard inactive state.
623  */
624 void FfmpegVideoCursor::
625 cleanup() {
626  stop_thread();
627  close_stream();
628 
629  ReMutexHolder av_holder(_av_lock);
630 
631 #ifdef HAVE_SWSCALE
632  if (_convert_ctx != nullptr) {
633  sws_freeContext(_convert_ctx);
634  }
635  _convert_ctx = nullptr;
636 #endif // HAVE_SWSCALE
637 
638  if (_frame) {
639  av_free(_frame);
640  _frame = nullptr;
641  }
642 
643  if (_frame_out) {
644  _frame_out->data[0] = nullptr;
645  av_free(_frame_out);
646  _frame_out = nullptr;
647  }
648 
649  if (_packet) {
650 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
651  av_packet_free(&_packet);
652 #else
653  if (_packet->data) {
654  av_free_packet(_packet);
655  }
656  delete _packet;
657  _packet = nullptr;
658 #endif
659  }
660 }
661 
662 /**
663  * The thread main function, static version (for passing to GenericThread).
664  */
665 void FfmpegVideoCursor::
666 st_thread_main(void *self) {
667  ((FfmpegVideoCursor *)self)->thread_main();
668 }
669 
670 /**
671  * The thread main function.
672  */
673 void FfmpegVideoCursor::
674 thread_main() {
675  if (ffmpeg_cat.is_spam()) {
676  ffmpeg_cat.spam()
677  << "ffmpeg thread for " << _filename.get_basename() << " starting.\n";
678  }
679 
680  // First, push the first frame onto the readahead queue.
681  if (_frame_ready) {
682  PT(FfmpegBuffer) frame = do_alloc_frame();
683  export_frame(frame);
684  MutexHolder holder(_lock);
685  _readahead_frames.push_back(frame);
686  }
687 
688  // Now repeatedly wait for something interesting to do, until we're told to
689  // shut down.
690  MutexHolder holder(_lock);
691  while (_thread_status != TS_shutdown) {
692  nassertv(_thread_status != TS_stopped);
693  _action_cvar.wait();
694 
695  while (do_poll()) {
696  // Keep doing stuff as long as there's something to do.
697  _lock.release();
698  PStatClient::thread_tick(_sync_name);
700  _lock.acquire();
701  }
702  }
703 
704  _thread_status = TS_stopped;
705  if (ffmpeg_cat.is_spam()) {
706  ffmpeg_cat.spam()
707  << "ffmpeg thread for " << _filename.get_basename() << " stopped.\n";
708  }
709 }
710 
711 /**
712  * Called within the sub-thread. Assumes the lock is already held. If there
713  * is something for the thread to do, does it and returns true. If there is
714  * nothing for the thread to do, returns false.
715  */
716 bool FfmpegVideoCursor::
717 do_poll() {
718  switch (_thread_status) {
719  case TS_stopped:
720  case TS_seeking:
721  // This shouldn't be possible while the thread is running.
722  nassertr(false, false);
723  return false;
724 
725  case TS_wait:
726  // The video hasn't started playing yet.
727  return false;
728 
729  case TS_readahead:
730  if ((int)_readahead_frames.size() < _max_readahead_frames) {
731  // Time to read the next frame.
732  PT(FfmpegBuffer) frame = do_alloc_frame();
733  nassertr(frame != nullptr, false);
734  _lock.release();
735  fetch_frame(-1);
736  if (_frame_ready) {
737  export_frame(frame);
738  _lock.acquire();
739  _readahead_frames.push_back(frame);
740  } else {
741  // No frame.
742  _lock.acquire();
743  }
744  return true;
745  }
746 
747  // No room for the next frame yet. Wait for more.
748  return false;
749 
750  case TS_seek:
751  // Seek to a specific frame.
752  {
753  int seek_frame = _seek_frame;
754  _thread_status = TS_seeking;
755  PT(FfmpegBuffer) frame = do_alloc_frame();
756  nassertr(frame != nullptr, false);
757  _lock.release();
758  if (seek_frame != _begin_frame) {
759  advance_to_frame(seek_frame);
760  }
761  if (_frame_ready) {
762  export_frame(frame);
763  _lock.acquire();
764  do_clear_all_frames();
765  _readahead_frames.push_back(frame);
766  } else {
767  _lock.acquire();
768  do_clear_all_frames();
769  }
770 
771  if (_thread_status == TS_seeking) {
772  // After seeking, we automatically transition to readahead.
773  _thread_status = TS_readahead;
774  }
775  }
776  return true;
777 
778  case TS_shutdown:
779  // Time to stop the thread.
780  return false;
781  }
782 
783  return false;
784 }
785 
786 /**
787  * Allocates a new Buffer object. Assumes the lock is held.
788  */
790 do_alloc_frame() {
791  PT(Buffer) buffer = make_new_buffer();
792  return (FfmpegBuffer *)buffer.p();
793 }
794 
795 /**
796  * Empties the entire readahead_frames queue. Assumes the lock is held.
797  */
798 void FfmpegVideoCursor::
799 do_clear_all_frames() {
800  _readahead_frames.clear();
801 }
802 
803 /**
804  * Called within the sub-thread. Fetches a video packet and stores it in the
805  * packet0 buffer. Sets packet_frame to the packet's timestamp. If a packet
806  * could not be read, the packet is cleared and the packet_frame is set to the
807  * specified default value. Returns true on failure (such as the end of the
808  * video), or false on success.
809  */
810 bool FfmpegVideoCursor::
811 fetch_packet(int default_frame) {
812  if (ffmpeg_global_lock) {
813  ReMutexHolder av_holder(_av_lock);
814  return do_fetch_packet(default_frame);
815  } else {
816  return do_fetch_packet(default_frame);
817  }
818 }
819 
820 /**
821  * As above, with the ffmpeg global lock held (if configured on).
822  */
823 bool FfmpegVideoCursor::
824 do_fetch_packet(int default_frame) {
825  if (_packet->data) {
826 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
827  av_packet_unref(_packet);
828 #else
829  av_free_packet(_packet);
830 #endif
831  }
832  while (av_read_frame(_format_ctx, _packet) >= 0) {
833  if (_packet->stream_index == _video_index) {
834  _packet_frame = _packet->dts;
835  return false;
836  }
837 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
838  av_packet_unref(_packet);
839 #else
840  av_free_packet(_packet);
841 #endif
842  }
843  _packet->data = nullptr;
844 
845  if (!_eof_known && default_frame != 0) {
846  _eof_frame = _packet_frame;
847  _eof_known = true;
848  }
849 
850  if (ffmpeg_cat.is_spam()) {
851  if (_eof_known) {
852  ffmpeg_cat.spam()
853  << "end of video at frame " << _eof_frame << "\n";
854  } else {
855  ffmpeg_cat.spam()
856  << "end of video\n";
857  }
858  }
859  _packet_frame = default_frame;
860  return true;
861 }
862 
863 /**
864  * Called within the sub-thread. Slides forward until the indicated frame,
865  * then fetches a frame from the stream and stores it in the frame buffer.
866  * Sets _begin_frame and _end_frame to indicate the extents of the frame.
867  * Sets _frame_ready true to indicate a frame is now available, or false if it
868  * is not (for instance, because the end of the video was reached).
869  */
870 void FfmpegVideoCursor::
871 fetch_frame(int frame) {
872  PStatTimer timer(_fetch_buffer_pcollector);
873 
874  int finished = 0;
875 
876  if (_packet_frame <= frame) {
877  finished = 0;
878 
879  // Get the next packet. The first packet beyond the frame we're looking
880  // for marks the point to stop.
881  while (_packet_frame <= frame) {
882  PStatTimer timer(_seek_pcollector);
883 
884  // Decode the previous packet, and get the next one.
885  decode_frame(finished);
886  _begin_frame = _packet_frame;
887  if (fetch_packet(frame)) {
888  _end_frame = _packet_frame;
889  _frame_ready = false;
890  return;
891  }
892  }
893 
894  } else {
895  // Just get the next frame.
896  finished = 0;
897  while (!finished && _packet->data) {
898  decode_frame(finished);
899  _begin_frame = _packet_frame;
900  fetch_packet(_begin_frame + 1);
901  }
902  }
903 
904  _end_frame = _packet_frame;
905  _frame_ready = true;
906 }
907 
908 /**
909  * Called within the sub-thread. Decodes the data in the specified packet
910  * into _frame.
911  */
912 void FfmpegVideoCursor::
913 decode_frame(int &finished) {
914  if (ffmpeg_global_lock) {
915  ReMutexHolder av_holder(_av_lock);
916  do_decode_frame(finished);
917  } else {
918  do_decode_frame(finished);
919  }
920 }
921 
922 /**
923  * As above, with the ffmpeg global lock held (if configured on).
924  */
925 void FfmpegVideoCursor::
926 do_decode_frame(int &finished) {
927 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100)
928  // While the audio cursor has a really nice async loop for decoding, we
929  // don't really do that much with video since we're already delegated to
930  // another thread here. This is just to silence the deprecation warning
931  // on avcodec_decode_video2.
932  avcodec_send_packet(_video_ctx, _packet);
933 
934  int ret = avcodec_receive_frame(_video_ctx, _frame);
935  finished = (ret == 0);
936 #else
937  avcodec_decode_video2(_video_ctx, _frame, &finished, _packet);
938 #endif
939 }
940 
941 /**
942  * Called within the sub-thread. Seeks to a target location. Afterward, the
943  * packet_frame is guaranteed to be less than or equal to the specified frame.
944  */
945 void FfmpegVideoCursor::
946 seek(int frame, bool backward) {
947  PStatTimer timer(_seek_pcollector);
948 
949  if (ffmpeg_support_seek) {
950  if (ffmpeg_global_lock) {
951  ReMutexHolder av_holder(_av_lock);
952  do_seek(frame, backward);
953  } else {
954  do_seek(frame, backward);
955  }
956 
957  } else {
958  // If seeking isn't supported, close-and-reopen.
959  if (backward) {
960  reset_stream();
961  }
962  }
963 }
964 
965 /**
966  * As above, with the ffmpeg global lock held (if configured on). Also only
967  * if ffmpeg-support-seek is on.
968  */
969 void FfmpegVideoCursor::
970 do_seek(int frame, bool backward) {
971  int64_t target_ts = (int64_t)frame;
972  if (target_ts < (int64_t)(_initial_dts)) {
973  // Attempts to seek before the first packet will fail.
974  target_ts = _initial_dts;
975  }
976  int flags = 0;
977  if (backward) {
978  flags = AVSEEK_FLAG_BACKWARD;
979  }
980 
981  if (av_seek_frame(_format_ctx, _video_index, target_ts, flags) < 0) {
982  if (ffmpeg_cat.is_spam()) {
983  ffmpeg_cat.spam()
984  << "Seek failure.\n";
985  }
986 
987  if (backward) {
988  // Now try to seek forward.
989  reset_stream();
990  seek(frame, false);
991  return;
992  }
993 
994  // Try a binary search to get a little closer.
995  if (binary_seek(_initial_dts, frame, frame, 1) < 0) {
996  if (ffmpeg_cat.is_spam()) {
997  ffmpeg_cat.spam()
998  << "Seek double failure.\n";
999  }
1000  reset_stream();
1001  return;
1002  }
1003  }
1004 
1005  fetch_packet(0);
1006  fetch_frame(-1);
1007 }
1008 
1009 /**
1010  * Casts about within the stream for a reasonably-close frame to seek to.
1011  * We're trying to get as close as possible to target_frame.
1012  */
1013 int FfmpegVideoCursor::
1014 binary_seek(int min_frame, int max_frame, int target_frame, int num_iterations) {
1015  int try_frame = (min_frame + max_frame) / 2;
1016  if (num_iterations > 5 || try_frame >= max_frame) {
1017  // Success.
1018  return 0;
1019  }
1020 
1021  if (av_seek_frame(_format_ctx, _video_index, try_frame, AVSEEK_FLAG_BACKWARD) < 0) {
1022  // Failure. Try lower.
1023  if (binary_seek(min_frame, try_frame - 1, target_frame, num_iterations + 1) < 0) {
1024  return -1;
1025  }
1026  } else {
1027  // Success. Try higher.
1028  if (binary_seek(try_frame + 1, max_frame, target_frame, num_iterations + 1) < 0) {
1029  return -1;
1030  }
1031  }
1032  return 0;
1033 }
1034 
1035 /**
1036  * Resets the stream to its initial, first-opened state by closing and re-
1037  * opening it.
1038  */
1039 void FfmpegVideoCursor::
1040 reset_stream() {
1041  if (ffmpeg_cat.is_spam()) {
1042  ffmpeg_cat.spam()
1043  << "Resetting ffmpeg stream.\n";
1044  }
1045 
1046  close_stream();
1047  if (!open_stream()) {
1048  ffmpeg_cat.error()
1049  << "Stream error, invalidating movie.\n";
1050  cleanup();
1051  return;
1052  }
1053 
1054  fetch_packet(0);
1055  fetch_frame(-1);
1056 }
1057 
1058 /**
1059  * Called within the sub-thread. Advance until the specified frame is in the
1060  * export buffer.
1061  */
1062 void FfmpegVideoCursor::
1063 advance_to_frame(int frame) {
1064  PStatTimer timer(_fetch_buffer_pcollector);
1065 
1066  if (frame < _begin_frame) {
1067  // Frame is in the past.
1068  if (ffmpeg_cat.is_spam()) {
1069  ffmpeg_cat.spam()
1070  << "Seeking backward to " << frame << " from " << _begin_frame << "\n";
1071  }
1072  seek(frame, true);
1073  if (_begin_frame > frame) {
1074  if (ffmpeg_cat.is_spam()) {
1075  ffmpeg_cat.spam()
1076  << "Ended up at " << _begin_frame << ", not far enough back!\n";
1077  }
1078  reset_stream();
1079  if (ffmpeg_cat.is_spam()) {
1080  ffmpeg_cat.spam()
1081  << "Reseek to 0, got " << _begin_frame << "\n";
1082  }
1083  }
1084  if (frame > _end_frame) {
1085  if (ffmpeg_cat.is_spam()) {
1086  ffmpeg_cat.spam()
1087  << "Now sliding forward to " << frame << " from " << _begin_frame << "\n";
1088  }
1089  fetch_frame(frame);
1090  }
1091 
1092  } else if (frame < _end_frame) {
1093  // Frame is in the present: already have the frame.
1094  if (ffmpeg_cat.is_spam()) {
1095  ffmpeg_cat.spam()
1096  << "Currently have " << frame << " within " << _begin_frame << " .. " << _end_frame << "\n";
1097  }
1098 
1099  } else if (frame < _end_frame + _min_fseek) {
1100  // Frame is in the near future.
1101  if (ffmpeg_cat.is_spam()) {
1102  ffmpeg_cat.spam()
1103  << "Sliding forward to " << frame << " from " << _begin_frame << "\n";
1104  }
1105  fetch_frame(frame);
1106 
1107  } else {
1108  // Frame is in the far future. Seek forward, then read. There's a danger
1109  // here: because keyframes are spaced unpredictably, trying to seek
1110  // forward could actually move us backward in the stream! This must be
1111  // avoided. So the rule is, try the seek. If it hurts us by moving us
1112  // backward, we increase the minimum threshold distance for forward-
1113  // seeking in the future.
1114 
1115  if (ffmpeg_cat.is_spam()) {
1116  ffmpeg_cat.spam()
1117  << "Jumping forward to " << frame << " from " << _begin_frame << "\n";
1118  }
1119  int base = _begin_frame;
1120  seek(frame, false);
1121  if (_begin_frame < base) {
1122  _min_fseek += (base - _begin_frame);
1123  if (ffmpeg_cat.is_spam()) {
1124  ffmpeg_cat.spam()
1125  << "Wrong way! Increasing _min_fseek to " << _min_fseek << "\n";
1126  }
1127  }
1128  if (frame > _end_frame) {
1129  if (ffmpeg_cat.is_spam()) {
1130  ffmpeg_cat.spam()
1131  << "Correcting, sliding forward to " << frame << " from " << _begin_frame << "\n";
1132  }
1133  fetch_frame(frame);
1134  }
1135  }
1136 
1137  if (ffmpeg_cat.is_spam()) {
1138  ffmpeg_cat.spam()
1139  << "Wanted " << frame << ", got " << _begin_frame << "\n";
1140  }
1141 }
1142 
1143 /**
1144  * Called within the sub-thread. Exports the contents of the frame buffer
1145  * into the indicated target buffer.
1146  */
1147 void FfmpegVideoCursor::
1148 export_frame(FfmpegBuffer *buffer) {
1149  PStatTimer timer(_export_frame_pcollector);
1150 
1151  if (!_frame_ready) {
1152  // No frame data ready, just fill with black.
1153  if (ffmpeg_cat.is_spam()) {
1154  ffmpeg_cat.spam()
1155  << "ffmpeg for " << _filename.get_basename()
1156  << ", no frame available.\n";
1157  }
1158  memset(buffer->_block, 0, buffer->_block_size);
1159  return;
1160  }
1161 
1162  _frame_out->data[0] = buffer->_block + ((_size_y - 1) * _size_x * _num_components);
1163  _frame_out->linesize[0] = _size_x * -_num_components;
1164  buffer->_begin_frame = _begin_frame;
1165  buffer->_end_frame = _end_frame;
1166 
1167  if (ffmpeg_global_lock) {
1168  ReMutexHolder av_holder(_av_lock);
1169 #ifdef HAVE_SWSCALE
1170  nassertv(_convert_ctx != nullptr && _frame != nullptr && _frame_out != nullptr);
1171  sws_scale(_convert_ctx, _frame->data, _frame->linesize, 0, _size_y, _frame_out->data, _frame_out->linesize);
1172 #else
1173  img_convert((AVPicture *)_frame_out, (AVPixelFormat)_pixel_format,
1174  (AVPicture *)_frame, _video_ctx->pix_fmt, _size_x, _size_y);
1175 #endif
1176  } else {
1177 #ifdef HAVE_SWSCALE
1178  nassertv(_convert_ctx != nullptr && _frame != nullptr && _frame_out != nullptr);
1179  sws_scale(_convert_ctx, _frame->data, _frame->linesize, 0, _size_y, _frame_out->data, _frame_out->linesize);
1180 #else
1181  img_convert((AVPicture *)_frame_out, (AVPixelFormat)_pixel_format,
1182  (AVPicture *)_frame, _video_ctx->pix_fmt, _size_x, _size_y);
1183 #endif
1184  }
1185 }
1186 
1187 /**
1188  * Tells the BamReader how to create objects of type FfmpegVideo.
1189  */
1192  BamReader::get_factory()->register_factory(get_class_type(), make_from_bam);
1193 }
1194 
1195 /**
1196  * Writes the contents of this object to the datagram for shipping out to a
1197  * Bam file.
1198  */
1200 write_datagram(BamWriter *manager, Datagram &dg) {
1201  MovieVideoCursor::write_datagram(manager, dg);
1202 
1203  // No need to write any additional data here--all of it comes implicitly
1204  // from the underlying MovieVideo, which we process in finalize().
1205 }
1206 
1207 /**
1208  * Called by the BamReader to perform any final actions needed for setting up
1209  * the object after all objects have been read and all pointers have been
1210  * completed.
1211  */
1213 finalize(BamReader *) {
1214  if (_source != nullptr) {
1215  FfmpegVideo *video;
1216  DCAST_INTO_V(video, _source);
1217  init_from(video);
1218  }
1219 }
1220 
1221 /**
1222  * This function is called by the BamReader's factory when a new object of
1223  * type FfmpegVideo is encountered in the Bam file. It should create the
1224  * FfmpegVideo and extract its information from the file.
1225  */
1226 TypedWritable *FfmpegVideoCursor::
1227 make_from_bam(const FactoryParams &params) {
1228  FfmpegVideoCursor *video = new FfmpegVideoCursor;
1229  DatagramIterator scan;
1230  BamReader *manager;
1231 
1232  parse_params(params, scan, manager);
1233  video->fillin(scan, manager);
1234 
1235  return video;
1236 }
1237 
1238 /**
1239  * This internal function is called by make_from_bam to read in all of the
1240  * relevant data from the BamFile for the new FfmpegVideo.
1241  */
1242 void FfmpegVideoCursor::
1243 fillin(DatagramIterator &scan, BamReader *manager) {
1244  MovieVideoCursor::fillin(scan, manager);
1245 
1246  // The MovieVideoCursor gets the underlying MovieVideo pointer. We need a
1247  // finalize callback so we can initialize ourselves once that has been read
1248  // completely.
1249  manager->register_finalize(this);
1250 }
1251 
1252 /**
1253  * Used to sort different buffers to ensure they correspond to the same source
1254  * frame, particularly important when synchronizing the different pages of a
1255  * multi-page texture.
1256  *
1257  * Returns 0 if the two buffers are of the same frame, <0 if this one comes
1258  * earlier than the other one, and >0 if the other one comes earlier.
1259  */
1261 compare_timestamp(const Buffer *other) const {
1262  const FfmpegBuffer *fother;
1263  DCAST_INTO_R(fother, other, 0);
1264  if (_end_frame * _video_timebase <= fother->_begin_frame * fother->_video_timebase) {
1265  return -1;
1266  } else if (_begin_frame * _video_timebase >= fother->_end_frame * fother->_video_timebase) {
1267  return 1;
1268  }
1269  return 0;
1270 }
1271 
1272 /**
1273  * Returns the nearest timestamp value of this particular buffer. Ideally,
1274  * MovieVideoCursor::set_time() for this timestamp would return this buffer
1275  * again. This need be defined only if compare_timestamp() is also defined.
1276  */
1278 get_timestamp() const {
1279  int mid_frame = (_begin_frame + _end_frame - 1) / 2;
1280  return mid_frame * _video_timebase;
1281 }
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
void parse_params(const FactoryParams &params, DatagramIterator &scan, BamReader *&manager)
Takes in a FactoryParams, passed from a WritableFactory into any TypedWritable's make function,...
Definition: bamReader.I:275
This is the fundamental interface for extracting binary objects from a Bam file, as generated by a Ba...
Definition: bamReader.h:110
void register_finalize(TypedWritable *whom)
Should be called by an object reading itself from the Bam file to indicate that this particular objec...
Definition: bamReader.cxx:808
static WritableFactory * get_factory()
Returns the global WritableFactory for generating TypedWritable objects.
Definition: bamReader.I:177
This is the fundamental interface for writing binary objects to a Bam file, to be extracted later by ...
Definition: bamWriter.h:63
Definition: buffer.h:24
void notify()
Informs one of the other threads who are currently blocked on wait() that the relevant condition has ...
void wait()
Waits on the condition.
A class to retrieve the individual data elements previously stored in a Datagram.
An ordered list of data elements, formatted in memory for transmission over a socket or writing to a ...
Definition: datagram.h:38
An instance of this class is passed to the Factory when requesting it to do its business and construc...
Definition: factoryParams.h:36
void register_factory(TypeHandle handle, CreateFunc *func, void *user_data=nullptr)
Registers a new kind of thing the Factory will be able to create.
Definition: factory.I:73
virtual int compare_timestamp(const Buffer *other) const
Used to sort different buffers to ensure they correspond to the same source frame,...
virtual double get_timestamp() const
Returns the nearest timestamp value of this particular buffer.
void set_max_readahead_frames(int max_readahead_frames)
Specifies the maximum number of frames that a sub-thread will attempt to read ahead of the current fr...
int get_max_readahead_frames() const
Returns the maximum number of frames that a sub-thread will attempt to read ahead of the current fram...
virtual bool set_time(double timestamp, int loop_count)
See MovieVideoCursor::set_time().
static void register_with_read_factory()
Tells the BamReader how to create objects of type FfmpegVideo.
bool is_thread_started() const
Returns true if the thread has been started, false if not.
void start_thread()
Explicitly starts the ffmpeg decoding thread after it has been stopped by a call to stop_thread().
void stop_thread()
Explicitly stops the ffmpeg decoding thread.
ThreadPriority get_thread_priority() const
Returns the current thread priority of the thread that decodes the ffmpeg video stream (if max_readah...
virtual void finalize(BamReader *manager)
Called by the BamReader to perform any final actions needed for setting up the object after all objec...
void set_thread_priority(ThreadPriority thread_priority)
Changes the thread priority of the thread that decodes the ffmpeg video stream (if max_readahead_fram...
virtual void write_datagram(BamWriter *manager, Datagram &dg)
Writes the contents of this object to the datagram for shipping out to a Bam file.
bool open_vfs(const Filename &filename)
Opens the movie file via Panda's VFS.
bool open_subfile(const SubfileInfo &info)
Opens the movie file directly from a file on disk (does not go through the VFS).
bool is_open() const
Returns true if the stream is successfully opened, false otherwise.
AVFormatContext * get_format_context() const
Returns a pointer to the opened ffmpeg context, or NULL if the file was not successfully opened.
void close()
Explicitly closes the opened file.
std::string get_basename() const
Returns the basename part of the filename.
Definition: filename.I:367
A generic thread type that allows calling a C-style thread function without having to subclass.
Definition: genericThread.h:24
virtual void write_datagram(BamWriter *manager, Datagram &dg)
Writes the contents of this object to the datagram for shipping out to a Bam file.
void release() const
Releases the mutex.
Definition: mutexDirect.I:78
void acquire() const
Grabs the mutex if it is available.
Definition: mutexDirect.I:55
A lightweight C++ object whose constructor calls acquire() and whose destructor calls release() on a ...
Definition: mutexHolder.h:25
A lightweight class that represents a single element that may be timed and/or counted via stats.
A lightweight class that can be used to automatically start and stop a PStatCollector around a sectio...
Definition: pStatTimer.h:30
Similar to MutexHolder, but for a reentrant mutex.
Definition: reMutexHolder.h:25
A reentrant mutex.
Definition: reMutex.h:34
static void consider_yield()
Possibly suspends the current thread for the rest of the current epoch, if it has run for enough this...
Definition: thread.I:212
TypeHandle is the identifier used to differentiate C++ class types.
Definition: typeHandle.h:81
Base class for objects that can be written to and read from Bam files.
Definition: typedWritable.h:35
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PT(MovieVideoCursor::Buffer) FfmpegVideoCursor
See MovieVideoCursor::fetch_buffer.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.
PANDA 3D SOFTWARE Copyright (c) Carnegie Mellon University.