Merge lp:~jhodapp/qtvideo-node/rotate-camera-video into lp:qtvideo-node

[Jim Hodapp]

Rotate the video 90 degrees and do an orthographic projection when necessary, otherwise render same as always.

* Rotate the video 90 degrees and do an orthographic projection when necessary, otherwise render same as always.

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:53
http://jenkins.qa.ubuntu.com/job/qtvideo-node-ci/54/
Executed test runs:
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-amd64-ci/3/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-armhf-ci/2/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-i386-ci/2/console

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/qtvideo-node-ci/54/rebuild

review: Needs Fixing (continuous-integration)

[lp:~jhodapp/qtvideo-node/rotate-camera-video]

54. By Jim Hodapp on 2014-10-15

Do a manual tranpose of the matrix before uploading to the GPU since OpenGLES 2.0 doesn't support doing this in hardware.

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:54
http://jenkins.qa.ubuntu.com/job/qtvideo-node-ci/55/
Executed test runs:
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-amd64-ci/4/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-armhf-ci/3/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-i386-ci/3/console

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/qtvideo-node-ci/55/rebuild

review: Needs Fixing (continuous-integration)

[lp:~jhodapp/qtvideo-node/rotate-camera-video]

55. By Jim Hodapp on 2014-10-16

Properly rotate the matrix 90 degrees if necessary and undo the Android Y flip (which becomes an X flip after height/width flip and 90 degree rotation).

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:55
http://jenkins.qa.ubuntu.com/job/qtvideo-node-ci/56/
Executed test runs:
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-amd64-ci/5/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-armhf-ci/4/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-i386-ci/4/console

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/qtvideo-node-ci/56/rebuild

review: Needs Fixing (continuous-integration)

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:57
http://jenkins.qa.ubuntu.com/job/qtvideo-node-ci/57/
Executed test runs:
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-amd64-ci/6/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-armhf-ci/5/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/qtvideo-node-utopic-i386-ci/5/console

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/qtvideo-node-ci/57/rebuild

review: Needs Fixing (continuous-integration)

[Ricardo Salveti (rsalveti)]

LGTM, but this only fixes the orientation in one case. Recording at 180 and 270 degrees were still wrong for me.

[Ricardo Salveti (rsalveti)]

From IRC:
<jhodapp> rsalveti: go ahead and approve that qtvideo-node branch, and I'll do a separate MR for the 180/270 cases

So the extra use cases will be covered by another MR.

review: Approve

[lp:~jhodapp/qtvideo-node/rotate-camera-video]

56. By Jim Hodapp on 2014-10-20

Add 180 and 270 degree rotation/flipping of video playback.

57. By Jim Hodapp on 2014-10-21

Merged with trunk.

=== modified file 'src/shadervideomaterial.cpp'
--- src/shadervideomaterial.cpp	2014-04-03 15:46:09 +0000
+++ src/shadervideomaterial.cpp	2014-10-21 14:29:15 +0000
@@ -22,6 +22,8 @@
 #endif
 
 #include <QGLShaderProgram>
+#include <QtOpenGL>
+#include <QTransform>
 
 #include <QtCore/qdebug.h>
 
@@ -29,6 +31,7 @@
 #include "shadervideoshader.h"
 
 #include <camera_compatibility_layer.h>
+#include <qtubuntu_media_signals.h>
 #include <surface_texture_client_hybris.h>
 
 ShaderVideoShader *ShaderVideoMaterial::m_videoShader = 0;
@@ -39,8 +42,11 @@
       m_textureId(0),
       m_surfaceTextureClient(0),
       m_glConsumer(0),
-      m_readyToRender(false)
+      m_readyToRender(false),
+      m_orientation(SharedSignal::Orientation::rotate0)
 {
+    connect(SharedSignal::instance(), SIGNAL(setOrientation(const SharedSignal::Orientation&, const QSize&)),
+            this, SLOT(onSetOrientation(const SharedSignal::Orientation&, const QSize&)));
 }
 
 QSGMaterialShader *ShaderVideoMaterial::createShader() const
@@ -109,8 +115,19 @@
         textureDirty = true;
     }
 
-    undoAndroidYFlip(m_textureMatrix);
-    glUniformMatrix4fv(m_videoShader->m_tex_matrix, 1, GL_FALSE, m_textureMatrix);
+    // See if the video needs rotation
+    if (m_orientation == SharedSignal::Orientation::rotate90 ||
+        m_orientation == SharedSignal::Orientation::rotate180 ||
+        m_orientation == SharedSignal::Orientation::rotate270)
+    {
+        QMatrix4x4 qRotated = rotateAndFlip(m_textureMatrix, m_orientation);
+        glUniformMatrix4fv(m_videoShader->m_tex_matrix, 1, GL_FALSE, qRotated.data());
+    }
+    else
+    {
+        undoAndroidYFlip(m_textureMatrix);
+        glUniformMatrix4fv(m_videoShader->m_tex_matrix, 1, GL_FALSE, m_textureMatrix);
+    }
 
     glTexParameteri(TEXTURE_TARGET, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     glTexParameteri(TEXTURE_TARGET, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
@@ -120,6 +137,97 @@
     return textureDirty;
 }
 
+void ShaderVideoMaterial::onSetOrientation(const SharedSignal::Orientation& orientation,
+        const QSize &size)
+{
+    m_orientation = orientation;
+    m_frameSize = size;
+    qDebug() << "orientation: " << orientation;
+    qDebug() << "frameSize: " << size;
+}
+
+// Takes a GLfloat texture matrix and desired orientation, and outputs a rotated and
+// horizontally flipped matrix
+QMatrix4x4 ShaderVideoMaterial::rotateAndFlip(GLfloat *m, const SharedSignal::Orientation &orientation)
+{
+    QMatrix4x4 ret;
+
+    if (m == NULL)
+        return ret;
+
+    /* TODO: Because of the column-to-row major format change, horizontal flips
+     * are really vertical flips, and vice versa. An improvement that needs to happen
+     * to more generically handle any container transformation matrix is to not convert
+     * to row-major format.
+     */
+    QMatrix4x4 qRowMajorTextureMatrix(m[0], m[4], m[8], m[12],
+                                      m[1], m[5], m[9], m[13],
+                                      m[2], m[6], m[10], m[14],
+                                      m[3], m[7], m[11], m[15]);
+    const QMatrix4x4 qFlipH (-1,  0, 0, 1,
+                              0,  1, 0, 0,
+                              0,  0, 1, 0,
+                              0,  0, 0, 1);
+    const QMatrix4x4 qFlipV ( 1,  0, 0, 0,
+                              0, -1, 0, 1,
+                              0,  0, 1, 0,
+                              0,  0, 0, 1);
+
+    switch (orientation)
+    {
+        case SharedSignal::Orientation::rotate90:
+        {
+            // FIXME: This matrix comes from the file container, but could not
+            // get this actual matrix up from GStreamer to here, so hardcoded
+            // for now.
+            const QMatrix4x4 qRotate90 (0, 1, 0, 0,
+                                       -1, 0, 0, 0,
+                                        0, 0, 1, 0,
+                                        0, 0, 0, 1);
+
+            ret = qRowMajorTextureMatrix * qRotate90;
+            // This must be done manually since OpenGLES 2.0 does not support doing the transpose
+            // when uploading the matrix to the GPU. OpenGLES 3.0 supports this.
+            ret = ret.transposed();
+            // Undo the Android mirroring. Since we already flipped height/width and rotated,
+            // do this about the vertical axis.
+            ret = ret * qFlipH;
+        }
+        break;
+        case SharedSignal::Orientation::rotate180:
+        {
+            ret = qRowMajorTextureMatrix * qFlipH;
+        }
+        break;
+        case SharedSignal::Orientation::rotate270:
+        {
+            // FIXME: This matrix comes from the file container, but could not
+            // get this actual matrix up from GStreamer to here, so hardcoded
+            // for now.
+            const QMatrix4x4 qRotate270( 0, 1, 0, 0,
+                                        -1, 0, 0, 0,
+                                         0, 0, 1, 0,
+                                         0, 0, 0, 1);
+
+            ret = qRowMajorTextureMatrix * qRotate270;
+            // This must be done manually since OpenGLES 2.0 does not support doing the transpose
+            // when uploading the matrix to the GPU. OpenGLES 3.0 supports this.
+            ret = ret.transposed();
+            // Undo the Android mirroring. Since we already flipped height/width and rotated,
+            // do this about the horizontal axis.
+            ret = ret * qFlipV;
+        }
+        break;
+        case SharedSignal::Orientation::rotate0:
+            // No-op, no rotation needed
+        default:
+            qDebug() << "Not rotating";
+            break;
+    }
+
+    return ret;
+}
+
 void ShaderVideoMaterial::undoAndroidYFlip(GLfloat matrix[])
 {
     // The android matrix flips the y coordinate
@@ -133,7 +241,7 @@
 
 /*!
  * \brief ShaderVideoMaterial::printGLMaxtrix
- * Prints a OpenGL matrix (GLfloat m[16]) to std out.
+ * Prints an EGL matrix (GLfloat m[16]) to stdout.
  * This function stays here for convenience in case some more debugging is necessary for the android
  * transformation matrix.
  * \param matrix Matrix to be printed to std out
@@ -145,3 +253,15 @@
     qDebug() << matrix[2] << matrix[6] << matrix[10] << matrix[14];
     qDebug() << matrix[3] << matrix[7] << matrix[11] << matrix[15];
 }
+
+/*!
+ * \brief Prints a row-major matrix to stdout.
+ * \param matrix Matrix to be printed to std out
+ */
+void ShaderVideoMaterial::printMaxtrix(float matrix[])
+{
+    qDebug() << matrix[0] << matrix[1] << matrix[2] << matrix[3];
+    qDebug() << matrix[4] << matrix[5] << matrix[6] << matrix[7];
+    qDebug() << matrix[8] << matrix[9] << matrix[10] << matrix[11];
+    qDebug() << matrix[12] << matrix[13] << matrix[14] << matrix[15];
+}

=== modified file 'src/shadervideomaterial.h'
--- src/shadervideomaterial.h	2014-04-03 15:46:09 +0000
+++ src/shadervideomaterial.h	2014-10-21 14:29:15 +0000
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Canonical, Ltd.
+ * Copyright (C) 2013-2014 Canonical, Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -23,15 +23,19 @@
 #else
 #include <QtQuick/QSGMaterial>
 #endif
+#include <qgl.h>
+#include <QMatrix4x4>
+#include <QObject>
 #include <QVideoSurfaceFormat>
-#include <qgl.h>
+#include <qtubuntu_media_signals.h>
 
 struct CameraControl;
 struct MediaPlayerWrapper;
 class ShaderVideoShader;
 
-class ShaderVideoMaterial : public QSGMaterial
+class ShaderVideoMaterial : public QObject, public QSGMaterial
 {
+    Q_OBJECT
 public:
     typedef void* SurfaceTextureClientHybris;
     typedef void* GLConsumerWrapperHybris;
@@ -54,11 +58,16 @@
 
     bool updateTexture();
 
-    GLfloat m_textureMatrix[16];
+    GLfloat m_textureMatrix[16] {};
+
+private Q_SLOTS:
+    void onSetOrientation(const SharedSignal::Orientation& orientation, const QSize &size);
 
 private:
+    QMatrix4x4 rotateAndFlip(GLfloat *m, const SharedSignal::Orientation &orientation);
     void undoAndroidYFlip(GLfloat matrix[]);
     void printGLMaxtrix(GLfloat matrix[]);
+    void printMaxtrix(float matrix[]);
 
     QVideoSurfaceFormat m_format;
     CameraControl *m_camControl;
@@ -67,6 +76,8 @@
     GLConsumerWrapperHybris m_glConsumer;
     bool m_readyToRender;
     static ShaderVideoShader *m_videoShader; // the shader is cached in the Qt scene graph
+    SharedSignal::Orientation m_orientation;
+    QSize m_frameSize;
 };
 
 #endif // SHADERVIDEOMATERIAL_H

=== modified file 'src/shadervideoshader.cpp'
--- src/shadervideoshader.cpp	2014-04-01 19:31:38 +0000
+++ src/shadervideoshader.cpp	2014-10-21 14:29:15 +0000
@@ -75,7 +75,7 @@
         "varying highp vec2 qt_TexCoord;                     \n"
         "void main()                                         \n"
         "{                                                   \n"
-        "  gl_FragColor = texture2D( sTexture, qt_TexCoord );\n"
+        "  gl_FragColor = texture2D( sTexture, qt_TexCoord ) * vec4(opacity);\n"
         "}                                                   \n";
     return shader;
 }

=== modified file 'src/src.pro'
--- src/src.pro	2014-06-06 14:54:20 +0000
+++ src/src.pro	2014-10-21 14:29:15 +0000
@@ -1,6 +1,6 @@
 TARGET = shadervideonode
 TEMPLATE = lib
-CONFIG += plugin
+CONFIG += plugin c++11
 QT += multimedia opengl quick-private quick
 
 include(../coverage.pri)

=== modified file 'unittests/shadervideonode/shadervideonode.pro'
--- unittests/shadervideonode/shadervideonode.pro	2014-04-03 15:46:09 +0000
+++ unittests/shadervideonode/shadervideonode.pro	2014-10-21 14:29:15 +0000
@@ -1,5 +1,5 @@
 CONFIG += testcase
-CONFIG += no_private_qt_headers_warning
+CONFIG += no_private_qt_headers_warning c++11
 TARGET = tst_shadervideonode
 
 # We don't want to install unit tests anywhere as it doesn't make sense