Fix QSGVideoNode rendering of frames with stride != width.
[qt:qtmultimedia.git] / src / qtmultimediaquicktools / qsgvideonode_i420.cpp
1 /****************************************************************************
2 **
3 ** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
4 ** Contact: http://www.qt-project.org/legal
5 **
6 ** This file is part of the Qt Toolkit.
7 **
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** Commercial License Usage
10 ** Licensees holding valid commercial Qt licenses may use this file in
11 ** accordance with the commercial license agreement provided with the
12 ** Software or, alternatively, in accordance with the terms contained in
13 ** a written agreement between you and Digia.  For licensing terms and
14 ** conditions see http://qt.digia.com/licensing.  For further information
15 ** use the contact form at http://qt.digia.com/contact-us.
16 **
17 ** GNU Lesser General Public License Usage
18 ** Alternatively, this file may be used under the terms of the GNU Lesser
19 ** General Public License version 2.1 as published by the Free Software
20 ** Foundation and appearing in the file LICENSE.LGPL included in the
21 ** packaging of this file.  Please review the following information to
22 ** ensure the GNU Lesser General Public License version 2.1 requirements
23 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
24 **
25 ** In addition, as a special exception, Digia gives you certain additional
26 ** rights.  These rights are described in the Digia Qt LGPL Exception
27 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
28 **
29 ** GNU General Public License Usage
30 ** Alternatively, this file may be used under the terms of the GNU
31 ** General Public License version 3.0 as published by the Free Software
32 ** Foundation and appearing in the file LICENSE.GPL included in the
33 ** packaging of this file.  Please review the following information to
34 ** ensure the GNU General Public License version 3.0 requirements will be
35 ** met: http://www.gnu.org/copyleft/gpl.html.
36 **
37 **
38 ** $QT_END_LICENSE$
39 **
40 ****************************************************************************/
41 #include "qsgvideonode_i420.h"
42 #include <QtCore/qmutex.h>
43 #include <QtQuick/qsgtexturematerial.h>
44 #include <QtQuick/qsgmaterial.h>
45 #include <QtGui/QOpenGLContext>
46 #include <QtGui/QOpenGLFunctions>
47 #include <QtGui/QOpenGLShaderProgram>
48
49 QT_BEGIN_NAMESPACE
50
51 QList<QVideoFrame::PixelFormat> QSGVideoNodeFactory_I420::supportedPixelFormats(
52                                         QAbstractVideoBuffer::HandleType handleType) const
53 {
54     QList<QVideoFrame::PixelFormat> formats;
55
56     if (handleType == QAbstractVideoBuffer::NoHandle)
57         formats << QVideoFrame::Format_YUV420P << QVideoFrame::Format_YV12;
58
59     return formats;
60 }
61
62 QSGVideoNode *QSGVideoNodeFactory_I420::createNode(const QVideoSurfaceFormat &format)
63 {
64     if (supportedPixelFormats(format.handleType()).contains(format.pixelFormat()))
65         return new QSGVideoNode_I420(format);
66
67     return 0;
68 }
69
70
71 class QSGVideoMaterialShader_YUV420 : public QSGMaterialShader
72 {
73 public:
74     void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial);
75
76     virtual char const *const *attributeNames() const {
77         static const char *names[] = {
78             "qt_VertexPosition",
79             "qt_VertexTexCoord",
80             0
81         };
82         return names;
83     }
84
85 protected:
86
87     virtual const char *vertexShader() const {
88         const char *shader =
89         "uniform highp mat4 qt_Matrix;                      \n"
90         "uniform highp float yWidth;                        \n"
91         "uniform highp float uvWidth;                       \n"
92         "attribute highp vec4 qt_VertexPosition;            \n"
93         "attribute highp vec2 qt_VertexTexCoord;            \n"
94         "varying highp vec2 yTexCoord;                      \n"
95         "varying highp vec2 uvTexCoord;                     \n"
96         "void main() {                                      \n"
97         "    yTexCoord   = qt_VertexTexCoord * vec2(yWidth, 1);\n"
98         "    uvTexCoord  = qt_VertexTexCoord * vec2(uvWidth, 1);\n"
99         "    gl_Position = qt_Matrix * qt_VertexPosition;   \n"
100         "}";
101         return shader;
102     }
103
104     virtual const char *fragmentShader() const {
105         static const char *shader =
106         "uniform sampler2D yTexture;"
107         "uniform sampler2D uTexture;"
108         "uniform sampler2D vTexture;"
109         "uniform mediump mat4 colorMatrix;"
110         "uniform lowp float opacity;"
111         ""
112         "varying highp vec2 yTexCoord;"
113         "varying highp vec2 uvTexCoord;"
114         ""
115         "void main()"
116         "{"
117         "    mediump float Y = texture2D(yTexture, yTexCoord).r;"
118         "    mediump float U = texture2D(uTexture, uvTexCoord).r;"
119         "    mediump float V = texture2D(vTexture, uvTexCoord).r;"
120         "    mediump vec4 color = vec4(Y, U, V, 1.);"
121         "    gl_FragColor = colorMatrix * color * opacity;"
122         "}";
123         return shader;
124     }
125
126     virtual void initialize() {
127         m_id_matrix = program()->uniformLocation("qt_Matrix");
128         m_id_yWidth = program()->uniformLocation("yWidth");
129         m_id_uvWidth = program()->uniformLocation("uvWidth");
130         m_id_yTexture = program()->uniformLocation("yTexture");
131         m_id_uTexture = program()->uniformLocation("uTexture");
132         m_id_vTexture = program()->uniformLocation("vTexture");
133         m_id_colorMatrix = program()->uniformLocation("colorMatrix");
134         m_id_opacity = program()->uniformLocation("opacity");
135     }
136
137     int m_id_matrix;
138     int m_id_yWidth;
139     int m_id_uvWidth;
140     int m_id_yTexture;
141     int m_id_uTexture;
142     int m_id_vTexture;
143     int m_id_colorMatrix;
144     int m_id_opacity;
145 };
146
147
148 class QSGVideoMaterial_YUV420 : public QSGMaterial
149 {
150 public:
151     QSGVideoMaterial_YUV420(const QVideoSurfaceFormat &format);
152     ~QSGVideoMaterial_YUV420();
153
154     virtual QSGMaterialType *type() const {
155         static QSGMaterialType theType;
156         return &theType;
157     }
158
159     virtual QSGMaterialShader *createShader() const {
160         return new QSGVideoMaterialShader_YUV420;
161     }
162
163     virtual int compare(const QSGMaterial *other) const {
164         const QSGVideoMaterial_YUV420 *m = static_cast<const QSGVideoMaterial_YUV420 *>(other);
165         int d = m_textureIds[0] - m->m_textureIds[0];
166         if (d)
167             return d;
168         else if ((d = m_textureIds[1] - m->m_textureIds[1]) != 0)
169             return d;
170         else
171             return m_textureIds[2] - m->m_textureIds[2];
172     }
173
174     void updateBlending() {
175         setFlag(Blending, qFuzzyCompare(m_opacity, qreal(1.0)) ? false : true);
176     }
177
178     void setCurrentFrame(const QVideoFrame &frame) {
179         QMutexLocker lock(&m_frameMutex);
180         m_frame = frame;
181     }
182
183     void bind();
184     void bindTexture(int id, int w, int h, const uchar *bits);
185
186     QVideoSurfaceFormat m_format;
187     QSize m_textureSize;
188
189     static const uint Num_Texture_IDs = 3;
190     GLuint m_textureIds[Num_Texture_IDs];
191
192     qreal m_opacity;
193     GLfloat m_yWidth;
194     GLfloat m_uvWidth;
195     QMatrix4x4 m_colorMatrix;
196
197     QVideoFrame m_frame;
198     QMutex m_frameMutex;
199 };
200
201 QSGVideoMaterial_YUV420::QSGVideoMaterial_YUV420(const QVideoSurfaceFormat &format) :
202     m_format(format),
203     m_opacity(1.0),
204     m_yWidth(1.0),
205     m_uvWidth(1.0)
206 {
207     memset(m_textureIds, 0, sizeof(m_textureIds));
208
209     switch (format.yCbCrColorSpace()) {
210     case QVideoSurfaceFormat::YCbCr_JPEG:
211         m_colorMatrix = QMatrix4x4(
212                     1.0f,  0.000f,  1.402f, -0.701f,
213                     1.0f, -0.344f, -0.714f,  0.529f,
214                     1.0f,  1.772f,  0.000f, -0.886f,
215                     0.0f,  0.000f,  0.000f,  1.0000f);
216         break;
217     case QVideoSurfaceFormat::YCbCr_BT709:
218     case QVideoSurfaceFormat::YCbCr_xvYCC709:
219         m_colorMatrix = QMatrix4x4(
220                     1.164f,  0.000f,  1.793f, -0.5727f,
221                     1.164f, -0.534f, -0.213f,  0.3007f,
222                     1.164f,  2.115f,  0.000f, -1.1302f,
223                     0.0f,    0.000f,  0.000f,  1.0000f);
224         break;
225     default: //BT 601:
226         m_colorMatrix = QMatrix4x4(
227                     1.164f,  0.000f,  1.596f, -0.8708f,
228                     1.164f, -0.392f, -0.813f,  0.5296f,
229                     1.164f,  2.017f,  0.000f, -1.081f,
230                     0.0f,    0.000f,  0.000f,  1.0000f);
231     }
232
233     setFlag(Blending, false);
234 }
235
236 QSGVideoMaterial_YUV420::~QSGVideoMaterial_YUV420()
237 {
238     if (!m_textureSize.isEmpty())
239         glDeleteTextures(Num_Texture_IDs, m_textureIds);
240 }
241
242 void QSGVideoMaterial_YUV420::bind()
243 {
244     QOpenGLFunctions *functions = QOpenGLContext::currentContext()->functions();
245
246     QMutexLocker lock(&m_frameMutex);
247     if (m_frame.isValid()) {
248         if (m_frame.map(QAbstractVideoBuffer::ReadOnly)) {
249             int fw = m_frame.width();
250             int fh = m_frame.height();
251
252             // Frame has changed size, recreate textures...
253             if (m_textureSize != m_frame.size()) {
254                 if (!m_textureSize.isEmpty())
255                     glDeleteTextures(Num_Texture_IDs, m_textureIds);
256                 glGenTextures(Num_Texture_IDs, m_textureIds);
257                 m_textureSize = m_frame.size();
258             }
259
260             const uchar *bits = m_frame.bits();
261             int yStride = m_frame.bytesPerLine();
262             // The UV stride is usually half the Y stride and is 32-bit aligned.
263             // However it's not always the case, at least on Windows where the
264             // UV planes are sometimes not aligned.
265             // We calculate the stride using the UV byte count to always
266             // have a correct stride.
267             int uvStride = (m_frame.mappedBytes() - yStride * fh) / fh;
268             int offsetU = yStride * fh;
269             int offsetV = yStride * fh + uvStride * fh / 2;
270
271             m_yWidth = qreal(fw) / yStride;
272             m_uvWidth = qreal(fw) /  (2 * uvStride);
273
274             if (m_frame.pixelFormat() == QVideoFrame::Format_YV12)
275                 qSwap(offsetU, offsetV);
276
277             GLint previousAlignment;
278             glGetIntegerv(GL_UNPACK_ALIGNMENT, &previousAlignment);
279             glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
280
281             functions->glActiveTexture(GL_TEXTURE1);
282             bindTexture(m_textureIds[1], uvStride, fh / 2, bits + offsetU);
283             functions->glActiveTexture(GL_TEXTURE2);
284             bindTexture(m_textureIds[2], uvStride, fh / 2, bits + offsetV);
285             functions->glActiveTexture(GL_TEXTURE0); // Finish with 0 as default texture unit
286             bindTexture(m_textureIds[0], yStride, fh, bits);
287
288             glPixelStorei(GL_UNPACK_ALIGNMENT, previousAlignment);
289
290             m_frame.unmap();
291         }
292
293         m_frame = QVideoFrame();
294     } else {
295         functions->glActiveTexture(GL_TEXTURE1);
296         glBindTexture(GL_TEXTURE_2D, m_textureIds[1]);
297         functions->glActiveTexture(GL_TEXTURE2);
298         glBindTexture(GL_TEXTURE_2D, m_textureIds[2]);
299         functions->glActiveTexture(GL_TEXTURE0); // Finish with 0 as default texture unit
300         glBindTexture(GL_TEXTURE_2D, m_textureIds[0]);
301     }
302 }
303
304 void QSGVideoMaterial_YUV420::bindTexture(int id, int w, int h, const uchar *bits)
305 {
306     glBindTexture(GL_TEXTURE_2D, id);
307     glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, w, h, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, bits);
308     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
309     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
310     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
311     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
312 }
313
314 QSGVideoNode_I420::QSGVideoNode_I420(const QVideoSurfaceFormat &format) :
315     m_format(format)
316 {
317     setFlag(QSGNode::OwnsMaterial);
318     m_material = new QSGVideoMaterial_YUV420(format);
319     setMaterial(m_material);
320 }
321
322 QSGVideoNode_I420::~QSGVideoNode_I420()
323 {
324 }
325
326 void QSGVideoNode_I420::setCurrentFrame(const QVideoFrame &frame)
327 {
328     m_material->setCurrentFrame(frame);
329     markDirty(DirtyMaterial);
330 }
331
332
333 void QSGVideoMaterialShader_YUV420::updateState(const RenderState &state,
334                                                 QSGMaterial *newMaterial,
335                                                 QSGMaterial *oldMaterial)
336 {
337     Q_UNUSED(oldMaterial);
338
339     QSGVideoMaterial_YUV420 *mat = static_cast<QSGVideoMaterial_YUV420 *>(newMaterial);
340     program()->setUniformValue(m_id_yTexture, 0);
341     program()->setUniformValue(m_id_uTexture, 1);
342     program()->setUniformValue(m_id_vTexture, 2);
343
344     mat->bind();
345
346     program()->setUniformValue(m_id_colorMatrix, mat->m_colorMatrix);
347     program()->setUniformValue(m_id_yWidth, mat->m_yWidth);
348     program()->setUniformValue(m_id_uvWidth, mat->m_uvWidth);
349     if (state.isOpacityDirty()) {
350         mat->m_opacity = state.opacity();
351         program()->setUniformValue(m_id_opacity, GLfloat(mat->m_opacity));
352     }
353
354     if (state.isMatrixDirty())
355         program()->setUniformValue(m_id_matrix, state.combinedMatrix());
356 }
357
358 QT_END_NAMESPACE