Make direct hardware decoding work on Intel/Linux

First, we need to force Qt to use EGL instead of GLX. This is required
because the direct hw interop mpv uses requires EGL. On the other hand,
always using EGL would break direct hw interop on nVidia GPUs, because
those require GLX. Qt can't just change the backend on the fly. In fact,
it seems that once the underlying plugin is loaded, it's stuck with the
choice. Some elaborate detection is needed. It probably can't be
achieved with Qt in its current state.

Achieve this with a hack: create a small mpv instance, which probes
whether GLX or EGL is better. This is (probably) better than writing
code to create EGL and GLX contects using the raw APIs.

In addition, PMP needs to hand down the X11 Display handle to mpv, as
mpv can't grab it automatically (as it's possible on GLX). Fortunately,
the X11Extras Qt module provides a sane way to do this.

CMakeModules/QtConfiguration.cmake

@@ -33,6 +33,11 @@ set(REQUIRED_QT_VERSION "5.6.0")
 set(QTCONFIGROOT ${QTROOT}/lib/cmake/Qt5)
 set(components Core Network WebChannel Qml Quick Xml WebEngine Widgets)
 
+if(UNIX AND (NOT APPLE) AND ((NOT BUILD_TARGET STREQUAL "RPI")))
+  add_definitions(-DUSE_X11EXTRAS)
+  set(components ${components} X11Extras)
+endif()
+
 if(OPENELEC)
   set(components ${components} DBus)
 endif(OPENELEC)

src/player/OpenGLDetect.cpp

@@ -1,5 +1,9 @@
+#include <QtGlobal>
 #include <QSurfaceFormat>
 
+#include <mpv/client.h>
+#include <mpv/qthelper.hpp>
+
 #include "OpenGLDetect.h"
 
 #if defined(Q_OS_MAC)
@@ -17,6 +21,38 @@ void detectOpenGL()
     QSurfaceFormat::setDefaultFormat(format);
 }
 
+#elif defined(Q_OS_LINUX)
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Attempt to reuse mpv's code for detecting whether we want GLX or EGL (which
+// is tricky to do because of hardware decoding concerns). This is not pretty,
+// but quite effective and without having to duplicate too much GLX/EGL code.
+static QString probeHwdecInterop()
+{
+  auto mpv = mpv::qt::Handle::FromRawHandle(mpv_create());
+  if (!mpv)
+    return "";
+  mpv::qt::set_option_variant(mpv, "hwdec-preload", "auto");
+  // Actually creating a window is required. There is currently no way to keep
+  // this window hidden or invisible.
+  mpv::qt::set_option_variant(mpv, "force-window", true);
+  // As a mitigation, put the window in the top/right corner, and make it as
+  // small as possible by forcing 1x1 size and removing window borders.
+  mpv::qt::set_option_variant(mpv, "geometry", "1x1+0+0");
+  mpv::qt::set_option_variant(mpv, "border", false);
+  if (mpv_initialize(mpv) < 0)
+    return "";
+  return mpv::qt::get_property_variant(mpv, "hwdec-interop").toString();
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+void detectOpenGL()
+{
+  // The putenv call must happen before Qt initializes its platform stuff.
+  if (probeHwdecInterop() == "vaapi-egl")
+    qputenv("QT_XCB_GL_INTEGRATION", "xcb_egl");
+}
+
 #else
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////

src/player/PlayerQuickItem.cpp

@@ -14,6 +14,10 @@
 #include "QsLog.h"
 #include "utils/Utils.h"
 
+#ifdef USE_X11EXTRAS
+#include <QX11Info>
+#endif
+
 #if defined(Q_OS_WIN32)
 
 #include <windows.h>
@@ -93,6 +97,14 @@ static void* __stdcall MPGetNativeDisplay(const char* name)
   return NULL;
 }
 // defined(Q_OS_WIN32)
+#elif defined(USE_X11EXTRAS)
+// Linux
+static void* MPGetNativeDisplay(const char* name)
+{
+  if (strcmp(name, "x11") == 0)
+    return QX11Info::display();
+  return nullptr;
+}
 #else
 // Unsupported or not needed. Also, not using Windows-specific calling convention.
 static void* MPGetNativeDisplay(const char* name)