Commit 61bce10b authored by Ken Martin's avatar Ken Martin
Browse files

improve performance of mpegvideosource fix cmake issues

Use two threads for decompression and fix a few cmake
issues.
parent 3d195c2d
......@@ -46,10 +46,11 @@ MACRO(FFMPEG_FIND varname shortname headername)
DOC "Location of FFMPEG Headers"
)
# on windows sometimes ffmpeg is built as lib<name>.a files
# on windows static ffmpeg is put in lib/lib<name>.a
# and shared it is put in bin/<name>.lib
if (WIN32)
list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".a")
list(APPEND CMAKE_FIND_LIBRARY_PREFIXES "lib")
list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".lib")
list(APPEND CMAKE_FIND_LIBRARY_PREFIXES "" "lib")
endif()
FIND_LIBRARY(FFMPEG_${varname}_LIBRARIES
......@@ -68,6 +69,7 @@ MACRO(FFMPEG_FIND varname shortname headername)
/opt/csw/lib
/opt/lib
/usr/freeware/lib64
${FFMPEG_ROOT}/bin # on windows shared they install the .lib in bin
DOC "Location of FFMPEG Libraries"
)
......
......@@ -65,7 +65,7 @@ int main()
#include <${FFMEG_CODEC_HEADER_PATH}/avcodec.h>
int main()
{
avcode_send_packet(0, 0);
avcodec_send_packet(0, 0);
return 0;
}\n")
check_c_source_compiles("${_source}" VTK_FFMPEG_AVCODEC_SEND_PACKET)
......
......@@ -14,6 +14,7 @@
=========================================================================*/
#include "vtkFFMPEGVideoSource.h"
#include "vtkConditionVariable.h"
#include "vtkCriticalSection.h"
#include "vtkMultiThreader.h"
#include "vtkMutexLock.h"
......@@ -36,6 +37,9 @@ extern "C" {
//
// inside it run msys_command.cmd
//
// if building libvpx make sure to add -enable-libvpx to the ffmpeg configure line
// and do the following
//
// export INCLUDE="C:\ffmpeg\include"\;$INCLUDE
// export LIB="C:\ffmpeg\lib;C:\ffmpeg\lib\x64"\;$LIB
//
......@@ -44,7 +48,9 @@ extern "C" {
// make install
// rename to vpx.lib
//
// ./configure --enable-asm --enable-x86asm --arch=amd64 --disable-avdevice --enable-swscale --disable-doc --disable-ffplay --disable-ffprobe --disable-ffmpeg --disable-shared --enable-static --disable-bzlib --disable-libopenjpeg --disable-iconv --disable-zlib --prefix=/c/ffmpeg --toolchain=msvc --enable-libvpx
// Then to build ffmpeg as follows
//
// ./configure --enable-asm --enable-x86asm --arch=amd64 --disable-avdevice --enable-swscale --disable-doc --disable-ffplay --disable-ffprobe --disable-ffmpeg --enable-shared --disable-static --disable-bzlib --disable-libopenjpeg --disable-iconv --disable-zlib --prefix=/c/ffmpeg --toolchain=msvc
// make
// make install
//
......@@ -53,14 +59,42 @@ extern "C" {
// Ws2_32.lib
// Bcrypt.lib
// Secur32.dll
// C:/ffmpeg/lib/x64/vpx.lib
//
/////////////////////////////////////////////////////////////////////////
class vtkFFMPEGVideoSourceInternal
{
public:
vtkFFMPEGVideoSourceInternal() {}
void ReleaseSystemResources()
{
if (this->Frame)
{
av_frame_free(&this->Frame);
this->Frame = nullptr;
}
if (this->VideoDecodeContext)
{
avcodec_close(this->VideoDecodeContext);
this->VideoDecodeContext = nullptr;
}
if (this->AudioDecodeContext)
{
avcodec_close(this->AudioDecodeContext);
this->AudioDecodeContext = nullptr;
}
if (this->FormatContext)
{
avformat_close_input(&this->FormatContext);
this->FormatContext = nullptr;
}
if (this->RGBContext)
{
sws_freeContext(this->RGBContext);
this->RGBContext = nullptr;
}
}
AVFormatContext *FormatContext = nullptr;
AVCodecContext *VideoDecodeContext = nullptr;
AVCodecContext *AudioDecodeContext = nullptr;
......@@ -99,54 +133,6 @@ vtkFFMPEGVideoSource::~vtkFFMPEGVideoSource()
delete this->Internal;
}
int vtkFFMPEGVideoSource::DecodePacket(int *got_frame)
{
int ret = 0;
AVPacket pkt = this->Internal->Packet;
*got_frame = 0;
if (pkt.stream_index == this->Internal->VideoStreamIndex)
{
ret = avcodec_send_packet(this->Internal->VideoDecodeContext, &pkt);
if (ret == AVERROR_EOF)
{
return ret;
}
if (ret < 0)
{
vtkErrorMacro("codec did not send packet");
return ret;
}
ret = avcodec_receive_frame(this->Internal->VideoDecodeContext,
this->Internal->Frame);
if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
{
vtkErrorMacro("codec did not receive video frame");
return ret;
}
*got_frame = 1;
}
else if (pkt.stream_index == this->Internal->AudioStreamIndex)
{
/* decode audio frame */
ret = avcodec_send_packet(this->Internal->AudioDecodeContext, &pkt);
if (ret < 0)
{
vtkErrorMacro("codec did not send packet");
return ret;
}
ret = avcodec_receive_frame(this->Internal->AudioDecodeContext,
this->Internal->Frame);
if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
{
vtkErrorMacro("codec did not receive audio frame");
return ret;
}
// do something with audio...
}
return pkt.size;
}
//----------------------------------------------------------------------------
void vtkFFMPEGVideoSource::Initialize()
{
......@@ -159,6 +145,10 @@ void vtkFFMPEGVideoSource::Initialize()
// though the initialization but need the framebuffer for Updates
this->UpdateFrameBuffer();
#ifdef NDEBUG
av_log_set_level(AV_LOG_ERROR);
#endif
/* open input file, and allocate format context */
if (avformat_open_input(&this->Internal->FormatContext, this->FileName, nullptr, nullptr) < 0)
{
......@@ -241,7 +231,7 @@ void vtkFFMPEGVideoSource::Initialize()
/* initialize packet, set data to NULL, let the demuxer fill it */
av_init_packet(&this->Internal->Packet);
this->Internal->Packet.data = NULL;
this->Internal->Packet.data = nullptr;
this->Internal->Packet.size = 0;
// update framebuffer again to reflect any changes which
......@@ -252,179 +242,332 @@ void vtkFFMPEGVideoSource::Initialize()
}
//----------------------------------------------------------------------------
// copy the frame into the
// vtkVideoSource framebuffer
void vtkFFMPEGVideoSource::InternalGrab()
// Feed frames to the decoder
void *vtkFFMPEGVideoSource::FeedThread(vtkMultiThreader::ThreadInfo *data)
{
vtkFFMPEGVideoSource *self = static_cast<vtkFFMPEGVideoSource *>(data->UserData);
return self->Feed(data);
}
int got_frame = 0;
int ret = 0;
if (this->Internal->Packet.size <= 0)
{
ret = av_read_frame(this->Internal->FormatContext, &this->Internal->Packet);
}
if (ret == 0)
//
// based off of https://blogs.gentoo.org/lu_zero/2016/03/29/new-avcodec-api/
//
void *vtkFFMPEGVideoSource::Feed(vtkMultiThreader::ThreadInfo *data)
{
bool done = false;
unsigned short count = 0;
bool retryPacket = false;
int fret = AVERROR_EOF;
while (!done)
{
ret = this->DecodePacket(&got_frame);
if (got_frame)
// read in the packet
if (!retryPacket)
{
av_packet_unref(&this->Internal->Packet);
fret = av_read_frame(this->Internal->FormatContext, &this->Internal->Packet);
}
retryPacket = false;
if (fret >= 0 &&
this->Internal->Packet.stream_index == this->Internal->VideoStreamIndex)
{
// get a thread lock on the frame buffer
this->FrameBufferMutex->Lock();
// lock the decoder
this->FeedMutex->Lock();
if (this->AutoAdvance)
int sret = avcodec_send_packet(
this->Internal->VideoDecodeContext,
&this->Internal->Packet);
if (sret == 0) // good decode
{
this->AdvanceFrameBuffer(1);
if (this->FrameIndex + 1 < this->FrameBufferSize)
{
this->FrameIndex++;
}
this->FeedCondition->Signal();
}
else if (sret == AVERROR(EAGAIN))
{
// Signal the draining loop
this->FeedCondition->Signal();
// Wait here
this->FeedCondition->Wait(this->FeedMutex);
retryPacket = true;
}
else if (sret < 0) // error
{
this->FeedMutex->Unlock();
return nullptr;
}
int index = this->FrameBufferIndex;
this->FrameCount++;
unsigned char *ptr = (unsigned char *)
((reinterpret_cast<vtkUnsignedCharArray*>(this->FrameBuffer[index])) \
->GetPointer(0));
// the DIB has rows which are multiples of 4 bytes
int outBytesPerRow = ((this->FrameBufferExtent[1]-
this->FrameBufferExtent[0]+1)
* this->FrameBufferBitsPerPixel + 7)/8;
outBytesPerRow += outBytesPerRow % this->FrameBufferRowAlignment;
outBytesPerRow += outBytesPerRow % 4;
int rows = this->FrameBufferExtent[3]-this->FrameBufferExtent[2]+1;
// update frame time
this->FrameBufferTimeStamps[index] =
this->StartTimeStamp + this->FrameCount / this->FrameRate;
uint8_t * dst[4];
int dstStride[4];
// we flip y axis here
dstStride[0] = -outBytesPerRow;
dst[0] = ptr + outBytesPerRow*(rows-1);
sws_scale(this->Internal->RGBContext,
this->Internal->Frame->data,
this->Internal->Frame->linesize,
0,
this->Internal->Frame->height,
dst, dstStride);
this->FrameBufferMutex->Unlock();
this->Modified();
this->FeedMutex->Unlock();
}
this->Internal->Packet.data += ret;
this->Internal->Packet.size -= ret;
}
else
{
this->EndOfFile = true;
}
}
//----------------------------------------------------------------------------
void vtkFFMPEGVideoSource::ReleaseSystemResources()
{
if (this->Initialized)
{
avcodec_close(this->Internal->VideoDecodeContext);
avcodec_close(this->Internal->AudioDecodeContext);
avformat_close_input(&this->Internal->FormatContext);
av_frame_free(&this->Internal->Frame);
sws_freeContext(this->Internal->RGBContext);
this->Initialized = 0;
this->Modified();
}
}
//----------------------------------------------------------------------------
void vtkFFMPEGVideoSource::Grab()
{
if (this->Recording)
{
return;
}
// are we out of data?
if (fret == AVERROR_EOF)
{
done = true;
}
// ensure that the frame buffer is properly initialized
this->Initialize();
if (!this->Initialized)
{
return;
// check to see if we are being told to quit every so often
if (count == 10)
{
data->ActiveFlagLock->Lock();
done = done || (*(data->ActiveFlag) == 0);
data->ActiveFlagLock->Unlock();
count = 0;
}
count++;
}
this->InternalGrab();
}
// flush remaning data
this->FeedMutex->Lock();
avcodec_send_packet(this->Internal->VideoDecodeContext, nullptr);
this->FeedCondition->Signal();
this->FeedMutex->Unlock();
//----------------------------------------------------------------------------
void vtkFFMPEGVideoSource::Play()
{
this->vtkVideoSource::Play();
return nullptr;
}
//----------------------------------------------------------------------------
// Sleep until the specified absolute time has arrived.
// You must pass a handle to the current thread.
// If '0' is returned, then the thread was aborted before or during the wait.
static int vtkThreadSleep(vtkMultiThreader::ThreadInfo *data, double time)
static void vtkThreadSleep(double time)
{
static size_t count = 0;
// loop either until the time has arrived or until the thread is ended
for (int i = 0;; i++)
{
count++;
double remaining = time - vtkTimerLog::GetUniversalTime();
// check to see if we are being told to quit
// data->ActiveFlagLock->Lock();
// int activeFlag = *(data->ActiveFlag);
// data->ActiveFlagLock->Unlock();
// if (activeFlag == 0)
// {
// break;
// }
// check to see if we have reached the specified time
if (remaining <= 0.0)
{
if (i == 0)
if (i == 0 && count % 100 == 0)
{
// vtkGenericWarningMacro("Dropped a video frame.");
cerr << "dropped frames, now beind by " << remaining << " seconds\n";
}
return 1;
break;
}
// check the ActiveFlag at least every 0.1 seconds
// do not sleep for more than 0.1 seconds
if (remaining > 0.1)
{
remaining = 0.1;
}
// check to see if we are being told to quit
data->ActiveFlagLock->Lock();
int activeFlag = *(data->ActiveFlag);
data->ActiveFlagLock->Unlock();
vtksys::SystemTools::Delay(static_cast<unsigned int>(remaining * 1000.0));
}
}
void *vtkFFMPEGVideoSource::DrainThread(vtkMultiThreader::ThreadInfo *data)
{
vtkFFMPEGVideoSource *self = static_cast<vtkFFMPEGVideoSource *>(data->UserData);
return self->Drain(data);
}
void *vtkFFMPEGVideoSource::Drain(vtkMultiThreader::ThreadInfo *data)
{
bool done = false;
unsigned short count = 0;
double startTime = vtkTimerLog::GetUniversalTime();
double rate = this->GetFrameRate();
int frame = 0;
while (!done)
{
this->FeedMutex->Lock();
if (activeFlag == 0)
int ret = avcodec_receive_frame(this->Internal->VideoDecodeContext,
this->Internal->Frame);
if (ret == 0)
{
break;
this->FeedCondition->Signal();
}
else if (ret == AVERROR(EAGAIN))
{
// Signal the feeding loop
this->FeedCondition->Signal();
// Wait here
this->FeedCondition->Wait(this->FeedMutex);
}
else if (ret == AVERROR_EOF)
{
done = true;
return nullptr;
}
else if (ret < 0) // error code
{
this->FeedMutex->Unlock();
return nullptr;
}
vtksys::SystemTools::Delay(static_cast<unsigned int>(remaining * 1000.0));
this->FeedMutex->Unlock();
if (ret == 0)
{
vtkThreadSleep(startTime + frame/rate);
this->InternalGrab();
frame++;
}
// check to see if we are being told to quit every so often
if (count == 10)
{
data->ActiveFlagLock->Lock();
done = done || (*(data->ActiveFlag) == 0);
data->ActiveFlagLock->Unlock();
count = 0;
}
count++;
}
return nullptr;
}
void vtkFFMPEGVideoSource::ReadFrame()
{
// first try to grab a frame from data we already have
bool gotFrame = false;
while (!gotFrame)
{
int ret = AVERROR(EAGAIN);
if (this->Internal->Packet.size > 0)
{
ret = avcodec_receive_frame(this->Internal->VideoDecodeContext,
this->Internal->Frame);
if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
{
vtkErrorMacro("codec did not receive video frame");
return;
}
if (ret == AVERROR_EOF)
{
this->EndOfFile = true;
return;
}
if (ret == 0)
{
gotFrame = true;
}
}
// if we are out of data then we must send more
if (ret == AVERROR(EAGAIN))
{
// if the packet is empty read more data from the file
av_packet_unref(&this->Internal->Packet);
int fret = av_read_frame(this->Internal->FormatContext, &this->Internal->Packet);
if (fret >= 0 &&
this->Internal->Packet.stream_index == this->Internal->VideoStreamIndex)
{
int sret = avcodec_send_packet(this->Internal->VideoDecodeContext, &this->Internal->Packet);
if (sret < 0 && sret != AVERROR(EAGAIN) && sret != AVERROR_EOF)
{
vtkErrorMacro("codec did not send packet");
return;
}
}
}
}
}
void vtkFFMPEGVideoSource::InternalGrab()
{
// get a thread lock on the frame buffer
this->FrameBufferMutex->Lock();
if (this->AutoAdvance)
{
this->AdvanceFrameBuffer(1);
if (this->FrameIndex + 1 < this->FrameBufferSize)
{
this->FrameIndex++;
}
}
int index = this->FrameBufferIndex;
this->FrameCount++;
unsigned char *ptr = (unsigned char *)
((reinterpret_cast<vtkUnsignedCharArray*>(this->FrameBuffer[index])) \
->GetPointer(0));
// the DIB has rows which are multiples of 4 bytes
int outBytesPerRow = ((this->FrameBufferExtent[1]-
this->FrameBufferExtent[0]+1)
* this->FrameBufferBitsPerPixel + 7)/8;
outBytesPerRow += outBytesPerRow % this->FrameBufferRowAlignment;
outBytesPerRow += outBytesPerRow % 4;
int rows = this->FrameBufferExtent[3]-this->FrameBufferExtent[2]+1;
return 0;
// update frame time
this->FrameBufferTimeStamps[index] =
this->StartTimeStamp + this->FrameCount / this->FrameRate;
uint8_t * dst[4];
int dstStride[4];
// we flip y axis here
dstStride[0] = -outBytesPerRow;
dst[0] = ptr + outBytesPerRow*(rows-1);
sws_scale(this->Internal->RGBContext,
this->Internal->Frame->data,
this->Internal->Frame->linesize,
0,
this->Internal->Frame->height,
dst, dstStride);
this->FrameBufferMutex->Unlock();
this->Modified();
}
//----------------------------------------------------------------------------
// this function runs in an alternate thread to asynchronously grab frames
void *vtkFFMPEGVideoSource::RecordThread(vtkMultiThreader::ThreadInfo *data)
void vtkFFMPEGVideoSource::ReleaseSystemResources()
{
vtkFFMPEGVideoSource *self = static_cast<vtkFFMPEGVideoSource *>(data->UserData);
if (this->Initialized)
{
this->Internal->ReleaseSystemResources();
this->Initialized = 0;
this->Modified();
}
}
double startTime = vtkTimerLog::GetUniversalTime();
double rate = self->GetFrameRate();
int frame = 0;
//----------------------------------------------------------------------------
void vtkFFMPEGVideoSource::Grab()
{
if (this->Recording)
{
return;
}
do
// ensure that the frame buffer is properly initialized
this->Initialize();
if (!this->Initialized)
{
self->InternalGrab();