23 #include <libavcodec/avcodec.h>
24 #include <libavformat/avformat.h>
25 #include <libavutil/pixdesc.h>
27 #include <libswscale/swscale.h>
31 ReMutex FfmpegVideoCursor::_av_lock;
33 TypeHandle FfmpegVideoCursor::FfmpegBuffer::_type_handle;
35 PStatCollector FfmpegVideoCursor::_fetch_buffer_pcollector(
"*:FFMPEG Video Decoding:Fetch");
36 PStatCollector FfmpegVideoCursor::_seek_pcollector(
"*:FFMPEG Video Decoding:Seek");
37 PStatCollector FfmpegVideoCursor::_export_frame_pcollector(
"*:FFMPEG Convert Video to BGR");
39 #if LIBAVUTIL_VERSION_INT < AV_VERSION_INT(52, 32, 100)
40 #define AV_PIX_FMT_FLAG_ALPHA PIX_FMT_ALPHA
48 _max_readahead_frames(0),
49 _thread_priority(ffmpeg_thread_priority),
50 _lock(
"FfmpegVideoCursor::_lock"),
52 _thread_status(TS_stopped),
57 _convert_ctx(nullptr),
58 _pixel_format((int)AV_PIX_FMT_NONE),
70 void FfmpegVideoCursor::
72 nassertv(_thread ==
nullptr && _thread_status == TS_stopped);
73 nassertv(source !=
nullptr);
75 _filename = _source->get_filename();
82 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 45, 101)
83 _frame = av_frame_alloc();
84 _frame_out = av_frame_alloc();
86 _frame = avcodec_alloc_frame();
87 _frame_out = avcodec_alloc_frame();
90 if ((_frame ==
nullptr)||(_frame_out ==
nullptr)) {
95 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
96 _packet = av_packet_alloc();
98 _packet =
new AVPacket;
99 av_init_packet(_packet);
104 _initial_dts = _begin_frame;
115 switch (_video_ctx->pix_fmt) {
116 case AV_PIX_FMT_GRAY8:
118 _pixel_format = (int)AV_PIX_FMT_GRAY8;
120 case AV_PIX_FMT_Y400A:
122 _pixel_format = (int)AV_PIX_FMT_Y400A;
125 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(_video_ctx->pix_fmt);
126 if (desc && (desc->flags & AV_PIX_FMT_FLAG_ALPHA) != 0) {
128 _pixel_format = (int)AV_PIX_FMT_BGRA;
131 _pixel_format = (int)AV_PIX_FMT_BGR24;
137 nassertv(_convert_ctx ==
nullptr);
138 _convert_ctx = sws_getContext(_size_x, _size_y, _video_ctx->pix_fmt,
139 _size_x, _size_y, (AVPixelFormat)_pixel_format,
140 SWS_BILINEAR | SWS_PRINT_INFO,
nullptr,
nullptr,
nullptr);
141 #endif // HAVE_SWSCALE
145 #endif // HAVE_THREADS
153 _max_readahead_frames(0),
154 _thread_priority(ffmpeg_thread_priority),
155 _lock(
"FfmpegVideoCursor::_lock"),
157 _thread_status(TS_stopped),
160 _format_ctx(nullptr),
162 _convert_ctx(nullptr),
175 ~FfmpegVideoCursor() {
193 if (max_readahead_frames > 0) {
195 <<
"Couldn't set max_readahead_frames to " << max_readahead_frames
196 <<
": threading not available.\n";
197 max_readahead_frames = 0;
199 #endif // HAVE_THREADS
201 _max_readahead_frames = max_readahead_frames;
202 if (_max_readahead_frames > 0) {
203 if (_thread_status == TS_stopped) {
207 if (_thread_status != TS_stopped) {
219 return _max_readahead_frames;
231 if (_thread_priority != thread_priority) {
232 _thread_priority = thread_priority;
247 return _thread_priority;
260 if (_thread_status == TS_stopped && _max_readahead_frames > 0) {
262 std::ostringstream strm;
263 strm << (
void *)
this;
264 _sync_name = strm.str();
267 _thread_status = TS_wait;
269 if (!_thread->start(_thread_priority,
true)) {
272 _thread_status = TS_stopped;
286 if (_thread_status != TS_stopped) {
290 if (_thread_status != TS_stopped) {
291 _thread_status = TS_shutdown;
307 _readahead_frames.clear();
316 return (_thread_status != TS_stopped);
323 set_time(
double timestamp,
int loop_count) {
324 int frame = (int)(timestamp / _video_timebase + 0.5);
327 if (loop_count == 0) {
328 frame = frame % (_eof_frame + 1);
330 int last_frame = (_eof_frame + 1) * loop_count;
331 if (frame < last_frame) {
332 frame = frame % (_eof_frame + 1);
340 frame = std::max(frame, _initial_dts);
342 if (ffmpeg_cat.is_spam() && frame != _current_frame) {
344 <<
"set_time(" << timestamp <<
"): " << frame
345 <<
", loop_count = " << loop_count <<
"\n";
348 _current_frame = frame;
349 if (_current_frame_buffer !=
nullptr) {
352 return (_current_frame >= _current_frame_buffer->_end_frame ||
353 _current_frame < _current_frame_buffer->_begin_frame);
368 if (_format_ctx ==
nullptr) {
372 PT(FfmpegBuffer) frame;
373 if (_thread_status == TS_stopped) {
375 advance_to_frame(_current_frame);
377 frame = do_alloc_frame();
383 if (!_readahead_frames.empty()) {
384 frame = _readahead_frames.front();
385 _readahead_frames.pop_front();
387 while (frame->_end_frame < _current_frame && !_readahead_frames.empty()) {
389 if (ffmpeg_cat.is_debug()) {
392 <<
" at frame " << _current_frame <<
", discarding frame at "
393 << frame->_begin_frame <<
"\n";
395 frame = _readahead_frames.front();
396 _readahead_frames.pop_front();
398 if (frame->_begin_frame > _current_frame) {
401 if (ffmpeg_cat.is_debug()) {
404 <<
" at frame " << _current_frame <<
", encountered too-new frame at "
405 << frame->_begin_frame <<
"\n";
407 do_clear_all_frames();
408 if (_thread_status == TS_wait || _thread_status == TS_seek || _thread_status == TS_readahead) {
409 _thread_status = TS_seek;
410 _seek_frame = _current_frame;
415 if (frame ==
nullptr || frame->_end_frame < _current_frame) {
417 if (_thread_status == TS_wait || _thread_status == TS_seek || _thread_status == TS_readahead) {
418 _thread_status = TS_seek;
419 _seek_frame = _current_frame;
425 if (frame !=
nullptr) {
426 bool too_old = (frame->_end_frame < _current_frame && !ffmpeg_show_seek_frames);
427 bool too_new = frame->_begin_frame > _current_frame;
428 if (too_old || too_new) {
434 if (frame !=
nullptr) {
435 _current_frame_buffer = frame;
436 if (ffmpeg_cat.is_debug()) {
439 <<
" at frame " << _current_frame <<
", returning frame at "
440 << frame->_begin_frame <<
"\n";
443 if (ffmpeg_cat.is_debug()) {
446 <<
" at frame " << _current_frame <<
", returning NULL\n";
457 PT(FfmpegBuffer) frame =
new FfmpegBuffer(size_x() * size_y() * get_num_components(), _video_timebase);
464 bool FfmpegVideoCursor::
466 nassertr(!_ffvfile.
is_open(),
false);
471 if (!_source->get_subfile_info().is_empty()) {
473 if (!_ffvfile.
open_subfile(_source->get_subfile_info())) {
475 <<
"Couldn't open " << _source->get_subfile_info() <<
"\n";
482 if (!_ffvfile.
open_vfs(_filename)) {
484 <<
"Couldn't open " << _filename <<
"\n";
490 nassertr(_format_ctx ==
nullptr,
false);
492 nassertr(_format_ctx !=
nullptr,
false);
494 if (avformat_find_stream_info(_format_ctx,
nullptr) < 0) {
496 <<
"Couldn't find stream info\n";
501 nassertr(_video_ctx ==
nullptr,
false);
506 #if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
507 AVCodecParameters *codecpar;
509 AVCodecContext *codecpar;
513 AVStream *stream =
nullptr;
514 for (
int i = 0; i < (int)_format_ctx->nb_streams; ++i) {
515 #if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
516 codecpar = _format_ctx->streams[i]->codecpar;
518 codecpar = _format_ctx->streams[i]->codec;
520 if (codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
522 stream = _format_ctx->streams[i];
527 if (stream ==
nullptr) {
529 <<
"Couldn't find stream\n";
534 _video_timebase = av_q2d(stream->time_base);
535 _min_fseek = (int)(3.0 / _video_timebase);
537 AVCodec *pVideoCodec =
nullptr;
538 if (ffmpeg_prefer_libvpx) {
539 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 0, 0)
540 if (codecpar->codec_id == AV_CODEC_ID_VP9) {
541 pVideoCodec = avcodec_find_decoder_by_name(
"libvpx-vp9");
544 if (codecpar->codec_id == AV_CODEC_ID_VP8) {
545 pVideoCodec = avcodec_find_decoder_by_name(
"libvpx");
548 if (pVideoCodec ==
nullptr) {
549 pVideoCodec = avcodec_find_decoder(codecpar->codec_id);
551 if (pVideoCodec ==
nullptr) {
553 <<
"Couldn't find codec\n";
558 _video_ctx = avcodec_alloc_context3(pVideoCodec);
560 if (_video_ctx ==
nullptr) {
562 <<
"Couldn't allocate _video_ctx\n";
567 #if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 41, 100)
568 avcodec_parameters_to_context(_video_ctx, codecpar);
570 avcodec_copy_context(_video_ctx, codecpar);
573 if (avcodec_open2(_video_ctx, pVideoCodec,
nullptr) < 0) {
575 <<
"Couldn't open codec\n";
580 _size_x = _video_ctx->width;
581 _size_y = _video_ctx->height;
583 _length = (double)_format_ctx->duration / (
double)AV_TIME_BASE;
585 _can_seek_fast =
true;
593 void FfmpegVideoCursor::
598 if (_video_ctx && _video_ctx->codec) {
599 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100)
601 avcodec_send_packet(_video_ctx,
nullptr);
602 while (avcodec_receive_frame(_video_ctx, _frame) == 0) {}
603 avcodec_flush_buffers(_video_ctx);
606 avcodec_close(_video_ctx);
607 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 52, 0)
608 avcodec_free_context(&_video_ctx);
613 _video_ctx =
nullptr;
616 _format_ctx =
nullptr;
624 void FfmpegVideoCursor::
632 if (_convert_ctx !=
nullptr) {
633 sws_freeContext(_convert_ctx);
635 _convert_ctx =
nullptr;
636 #endif // HAVE_SWSCALE
644 _frame_out->data[0] =
nullptr;
646 _frame_out =
nullptr;
650 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
651 av_packet_free(&_packet);
654 av_free_packet(_packet);
665 void FfmpegVideoCursor::
666 st_thread_main(
void *
self) {
673 void FfmpegVideoCursor::
675 if (ffmpeg_cat.is_spam()) {
677 <<
"ffmpeg thread for " << _filename.
get_basename() <<
" starting.\n";
682 PT(FfmpegBuffer) frame = do_alloc_frame();
685 _readahead_frames.push_back(frame);
691 while (_thread_status != TS_shutdown) {
692 nassertv(_thread_status != TS_stopped);
698 PStatClient::thread_tick(_sync_name);
704 _thread_status = TS_stopped;
705 if (ffmpeg_cat.is_spam()) {
707 <<
"ffmpeg thread for " << _filename.
get_basename() <<
" stopped.\n";
716 bool FfmpegVideoCursor::
718 switch (_thread_status) {
722 nassertr(
false,
false);
730 if ((
int)_readahead_frames.size() < _max_readahead_frames) {
732 PT(FfmpegBuffer) frame = do_alloc_frame();
733 nassertr(frame !=
nullptr,
false);
739 _readahead_frames.push_back(frame);
753 int seek_frame = _seek_frame;
754 _thread_status = TS_seeking;
755 PT(FfmpegBuffer) frame = do_alloc_frame();
756 nassertr(frame !=
nullptr,
false);
758 if (seek_frame != _begin_frame) {
759 advance_to_frame(seek_frame);
764 do_clear_all_frames();
765 _readahead_frames.push_back(frame);
768 do_clear_all_frames();
771 if (_thread_status == TS_seeking) {
773 _thread_status = TS_readahead;
791 PT(
Buffer) buffer = make_new_buffer();
792 return (FfmpegBuffer *)buffer.p();
798 void FfmpegVideoCursor::
799 do_clear_all_frames() {
800 _readahead_frames.clear();
810 bool FfmpegVideoCursor::
811 fetch_packet(
int default_frame) {
812 if (ffmpeg_global_lock) {
814 return do_fetch_packet(default_frame);
816 return do_fetch_packet(default_frame);
823 bool FfmpegVideoCursor::
824 do_fetch_packet(
int default_frame) {
826 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
827 av_packet_unref(_packet);
829 av_free_packet(_packet);
832 while (av_read_frame(_format_ctx, _packet) >= 0) {
833 if (_packet->stream_index == _video_index) {
834 _packet_frame = _packet->dts;
837 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 12, 100)
838 av_packet_unref(_packet);
840 av_free_packet(_packet);
843 _packet->data =
nullptr;
845 if (!_eof_known && default_frame != 0) {
846 _eof_frame = _packet_frame;
850 if (ffmpeg_cat.is_spam()) {
853 <<
"end of video at frame " << _eof_frame <<
"\n";
859 _packet_frame = default_frame;
870 void FfmpegVideoCursor::
871 fetch_frame(
int frame) {
876 if (_packet_frame <= frame) {
881 while (_packet_frame <= frame) {
885 decode_frame(finished);
886 _begin_frame = _packet_frame;
887 if (fetch_packet(frame)) {
888 _end_frame = _packet_frame;
889 _frame_ready =
false;
897 while (!finished && _packet->data) {
898 decode_frame(finished);
899 _begin_frame = _packet_frame;
900 fetch_packet(_begin_frame + 1);
904 _end_frame = _packet_frame;
912 void FfmpegVideoCursor::
913 decode_frame(
int &finished) {
914 if (ffmpeg_global_lock) {
916 do_decode_frame(finished);
918 do_decode_frame(finished);
925 void FfmpegVideoCursor::
926 do_decode_frame(
int &finished) {
927 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100)
932 avcodec_send_packet(_video_ctx, _packet);
934 int ret = avcodec_receive_frame(_video_ctx, _frame);
935 finished = (ret == 0);
937 avcodec_decode_video2(_video_ctx, _frame, &finished, _packet);
945 void FfmpegVideoCursor::
946 seek(
int frame,
bool backward) {
949 if (ffmpeg_support_seek) {
950 if (ffmpeg_global_lock) {
952 do_seek(frame, backward);
954 do_seek(frame, backward);
969 void FfmpegVideoCursor::
970 do_seek(
int frame,
bool backward) {
971 int64_t target_ts = (int64_t)frame;
972 if (target_ts < (int64_t)(_initial_dts)) {
974 target_ts = _initial_dts;
978 flags = AVSEEK_FLAG_BACKWARD;
981 if (av_seek_frame(_format_ctx, _video_index, target_ts, flags) < 0) {
982 if (ffmpeg_cat.is_spam()) {
984 <<
"Seek failure.\n";
995 if (binary_seek(_initial_dts, frame, frame, 1) < 0) {
996 if (ffmpeg_cat.is_spam()) {
998 <<
"Seek double failure.\n";
1013 int FfmpegVideoCursor::
1014 binary_seek(
int min_frame,
int max_frame,
int target_frame,
int num_iterations) {
1015 int try_frame = (min_frame + max_frame) / 2;
1016 if (num_iterations > 5 || try_frame >= max_frame) {
1021 if (av_seek_frame(_format_ctx, _video_index, try_frame, AVSEEK_FLAG_BACKWARD) < 0) {
1023 if (binary_seek(min_frame, try_frame - 1, target_frame, num_iterations + 1) < 0) {
1028 if (binary_seek(try_frame + 1, max_frame, target_frame, num_iterations + 1) < 0) {
1039 void FfmpegVideoCursor::
1041 if (ffmpeg_cat.is_spam()) {
1043 <<
"Resetting ffmpeg stream.\n";
1047 if (!open_stream()) {
1049 <<
"Stream error, invalidating movie.\n";
1062 void FfmpegVideoCursor::
1063 advance_to_frame(
int frame) {
1066 if (frame < _begin_frame) {
1068 if (ffmpeg_cat.is_spam()) {
1070 <<
"Seeking backward to " << frame <<
" from " << _begin_frame <<
"\n";
1073 if (_begin_frame > frame) {
1074 if (ffmpeg_cat.is_spam()) {
1076 <<
"Ended up at " << _begin_frame <<
", not far enough back!\n";
1079 if (ffmpeg_cat.is_spam()) {
1081 <<
"Reseek to 0, got " << _begin_frame <<
"\n";
1084 if (frame > _end_frame) {
1085 if (ffmpeg_cat.is_spam()) {
1087 <<
"Now sliding forward to " << frame <<
" from " << _begin_frame <<
"\n";
1092 }
else if (frame < _end_frame) {
1094 if (ffmpeg_cat.is_spam()) {
1096 <<
"Currently have " << frame <<
" within " << _begin_frame <<
" .. " << _end_frame <<
"\n";
1099 }
else if (frame < _end_frame + _min_fseek) {
1101 if (ffmpeg_cat.is_spam()) {
1103 <<
"Sliding forward to " << frame <<
" from " << _begin_frame <<
"\n";
1115 if (ffmpeg_cat.is_spam()) {
1117 <<
"Jumping forward to " << frame <<
" from " << _begin_frame <<
"\n";
1119 int base = _begin_frame;
1121 if (_begin_frame < base) {
1122 _min_fseek += (base - _begin_frame);
1123 if (ffmpeg_cat.is_spam()) {
1125 <<
"Wrong way! Increasing _min_fseek to " << _min_fseek <<
"\n";
1128 if (frame > _end_frame) {
1129 if (ffmpeg_cat.is_spam()) {
1131 <<
"Correcting, sliding forward to " << frame <<
" from " << _begin_frame <<
"\n";
1137 if (ffmpeg_cat.is_spam()) {
1139 <<
"Wanted " << frame <<
", got " << _begin_frame <<
"\n";
1147 void FfmpegVideoCursor::
1148 export_frame(FfmpegBuffer *buffer) {
1151 if (!_frame_ready) {
1153 if (ffmpeg_cat.is_spam()) {
1156 <<
", no frame available.\n";
1158 memset(buffer->_block, 0, buffer->_block_size);
1162 _frame_out->data[0] = buffer->_block + ((_size_y - 1) * _size_x * _num_components);
1163 _frame_out->linesize[0] = _size_x * -_num_components;
1164 buffer->_begin_frame = _begin_frame;
1165 buffer->_end_frame = _end_frame;
1167 if (ffmpeg_global_lock) {
1170 nassertv(_convert_ctx !=
nullptr && _frame !=
nullptr && _frame_out !=
nullptr);
1171 sws_scale(_convert_ctx, _frame->data, _frame->linesize, 0, _size_y, _frame_out->data, _frame_out->linesize);
1173 img_convert((AVPicture *)_frame_out, (AVPixelFormat)_pixel_format,
1174 (AVPicture *)_frame, _video_ctx->pix_fmt, _size_x, _size_y);
1178 nassertv(_convert_ctx !=
nullptr && _frame !=
nullptr && _frame_out !=
nullptr);
1179 sws_scale(_convert_ctx, _frame->data, _frame->linesize, 0, _size_y, _frame_out->data, _frame_out->linesize);
1181 img_convert((AVPicture *)_frame_out, (AVPixelFormat)_pixel_format,
1182 (AVPicture *)_frame, _video_ctx->pix_fmt, _size_x, _size_y);
1214 if (_source !=
nullptr) {
1216 DCAST_INTO_V(video, _source);
1233 video->fillin(scan, manager);
1242 void FfmpegVideoCursor::
1244 MovieVideoCursor::fillin(scan, manager);
1263 DCAST_INTO_R(fother, other, 0);
1264 if (_end_frame * _video_timebase <= fother->_begin_frame * fother->_video_timebase) {
1266 }
else if (_begin_frame * _video_timebase >= fother->_end_frame * fother->_video_timebase) {
1279 int mid_frame = (_begin_frame + _end_frame - 1) / 2;
1280 return mid_frame * _video_timebase;