QAbstractVideoSurface Strange behaviour

Question

I have implemented QAbstractVideoSurface as below and I am facing following issue:

1. Sometimes present method of QAbstractVideosurface is not called at all. There is no error as well. I can see video is playing from audio and timestamp getting updated.

What am I missing in my implementation ?

video_frame_grabber.cpp

#include "video_frame_grabber.h"
#include <QtWidgets>
#include <qabstractvideosurface.h>
#include <qvideosurfaceformat.h>

VideoFrameGrabber::VideoFrameGrabber(QGraphicsView *view, QGraphicsPixmapItem *pixmap, QObject *parent)
    : QAbstractVideoSurface(parent)
    , view(view)
    , pixmap(pixmap)
    , imageFormat(QImage::Format_Invalid)
{
}

QList<QVideoFrame::PixelFormat> VideoFrameGrabber::supportedPixelFormats(QAbstractVideoBuffer::HandleType handleType) const
{
    Q_UNUSED(handleType);
    return QList<QVideoFrame::PixelFormat>()
        << QVideoFrame::Format_ARGB32
        << QVideoFrame::Format_ARGB32_Premultiplied
        << QVideoFrame::Format_RGB32
        << QVideoFrame::Format_RGB24
        << QVideoFrame::Format_RGB565
        << QVideoFrame::Format_RGB555
        << QVideoFrame::Format_ARGB8565_Premultiplied
        << QVideoFrame::Format_BGRA32
        << QVideoFrame::Format_BGRA32_Premultiplied
        << QVideoFrame::Format_BGR32
        << QVideoFrame::Format_BGR24
        << QVideoFrame::Format_BGR565
        << QVideoFrame::Format_BGR555
        << QVideoFrame::Format_BGRA5658_Premultiplied
        << QVideoFrame::Format_AYUV444
        << QVideoFrame::Format_AYUV444_Premultiplied
        << QVideoFrame::Format_YUV444
        << QVideoFrame::Format_YUV420P
        << QVideoFrame::Format_YV12
        << QVideoFrame::Format_UYVY
        << QVideoFrame::Format_YUYV
        << QVideoFrame::Format_NV12
        << QVideoFrame::Format_NV21
        << QVideoFrame::Format_IMC1
        << QVideoFrame::Format_IMC2
        << QVideoFrame::Format_IMC3
        << QVideoFrame::Format_IMC4
        << QVideoFrame::Format_Y8
        << QVideoFrame::Format_Y16
        << QVideoFrame::Format_Jpeg
        << QVideoFrame::Format_CameraRaw
        << QVideoFrame::Format_AdobeDng;
}

bool VideoFrameGrabber::isFormatSupported(const QVideoSurfaceFormat &format) const
{
    const QImage::Format imageFormat = QVideoFrame::imageFormatFromPixelFormat(format.pixelFormat());
    const QSize size = format.frameSize();

    return imageFormat != QImage::Format_Invalid
            && !size.isEmpty()
            && format.handleType() == QAbstractVideoBuffer::NoHandle;
}

bool VideoFrameGrabber::start(const QVideoSurfaceFormat &format)
{
    const QImage::Format imageFormat = QVideoFrame::imageFormatFromPixelFormat(format.pixelFormat());
    const QSize size = format.frameSize();

    if (imageFormat != QImage::Format_Invalid && !size.isEmpty()) {
        this->imageFormat = imageFormat;
        QAbstractVideoSurface::start(format);
        return true;
    } 
    else{
        return false;
    }
}

void VideoFrameGrabber::stop()
{
    QAbstractVideoSurface::stop();

    view->update();
}

bool VideoFrameGrabber::present(const QVideoFrame &frame)
{
    if ((surfaceFormat().pixelFormat() != frame.pixelFormat()) || (surfaceFormat().frameSize() != frame.size())) {
        setError(IncorrectFormatError);
        stop();
        qDebug() << "error";
        return false;
    } 
    else {
        QImage img = VideoFrameToImage(frame);
        displayframe(img);
        emit frame_received();
        return true;
    }
}

void VideoFrameGrabber::displayframe(QImage img)
{
    pixmap->setPixmap(QPixmap::fromImage(img));
    view->fitInView(QRectF(0,0,img.width(),img.height()),Qt::KeepAspectRatio);
}

    QImage VideoFrameGrabber::VideoFrameToImage(QVideoFrame frame)
    {
        if (frame.map(QAbstractVideoBuffer::ReadOnly)) {
       return QImage(frame.bits(), frame.width(), frame.height(), frame.bytesPerLine(), imageFormat);
       }
    return QImage();
    }

video_frame_grabber.h

#include <QAbstractVideoSurface>
#include <QObject>
#include <QGraphicsView>
#include <QGraphicsScene>
#include <QPixmap>

class VideoFrameGrabber : public QAbstractVideoSurface
{
    Q_OBJECT
public:
    VideoFrameGrabber(QGraphicsView *view, QGraphicsPixmapItem *pixmap ,QObject *parent = nullptr);

    QList<QVideoFrame::PixelFormat> supportedPixelFormats(
            QAbstractVideoBuffer::HandleType handleType = QAbstractVideoBuffer::NoHandle) const;
    bool isFormatSupported(const QVideoSurfaceFormat &format) const;

    bool start(const QVideoSurfaceFormat &format);
    void stop();

    bool present(const QVideoFrame &frame);
    void displayframe(QImage img);
    QImage VideoFrameToImage(QVideoFrame frame);

private:
    QGraphicsView *view;
    QGraphicsPixmapItem *pixmap;
    QImage::Format imageFormat;
    int current_function;

signals:
    void frame_received();
};

And mainwindow.cpp

m_MediaPlayer = new QMediaPlayer(this);
m_GraphicsScene = new QGraphicsScene();
pixmapItem = new QGraphicsPixmapItem();
m_GraphicsScene->addItem(pixmapItem);
ui->m_GraphicsView->setScene(m_GraphicsScene);
grabber = new VideoFrameGrabber(ui->m_GraphicsView, pixmapItem);
m_MediaPlayer->setVideoOutput(grabber);

m_MediaPlayer->setMedia(QUrl::fromLocalFile("1.mp4"));

Thanks in advance

Answer 1

My best bet is that you are lagging, and this is due to the inefficient conversion

 QVideoFrame -> QImage -> QPixmap

specifically the QPixmap::fromImage(img) is the bottleneck in your pipeline.

The QMediaPlayer media player already supports output to QGraphicsVideoItem. Which means you should be able to do :

m_MediaPlayer = new QMediaPlayer(this);
m_GraphicsScene = new QGraphicsScene();
videoItem = new QGraphicsVideoItem();
m_GraphicsScene->addItem(videoItem); 
m_MediaPlayer->setVideoOutput(videoItem);