Bug 1665844 [Wayland] Fix mutter to start in Wayland mode in Mozilla automation, r=ahal

Differential Revision: https://phabricator.services.mozilla.com/D97161

taskcluster/scripts/tester/test-linux.sh

@@ -75,6 +75,7 @@ if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi
 
 if [ $MOZ_ENABLE_WAYLAND ]; then
   NEED_XVFB=true
+  NEED_WINDOW_MANAGER=true
 fi
 
 # make sure artifact directories exist
@@ -98,12 +99,12 @@ cleanup() {
       # To share X issues
       cp "$HOME/.xsession-errors" "$WORKING_DIR/artifacts/public/xsession-errors.log"
     fi
-    if $NEED_XVFB; then
-        cleanup_xvfb
-    fi
     if [ $MOZ_ENABLE_WAYLAND ]; then
         cleanup_mutter
     fi
+    if $NEED_XVFB; then
+        cleanup_xvfb
+    fi
     exit $rv
 }
 trap cleanup EXIT INT
@@ -155,10 +156,6 @@ if $NEED_XVFB; then
     # note that this file is not available when run under native-worker
     . $HOME/scripts/xvfb.sh
     start_xvfb '1600x1200x24' 0
-    if [ $MOZ_ENABLE_WAYLAND ]; then
-        mutter --wayland &
-        export WAYLAND_DISPLAY=wayland-0
-    fi
 fi
 
 if $START_VNC; then
@@ -175,7 +172,6 @@ if $NEED_WINDOW_MANAGER; then
         echo export XDG_CURRENT_DESKTOP=GNOME > $HOME/.xsessionrc
         if [ $MOZ_ENABLE_WAYLAND ]; then
             echo export XDG_SESSION_TYPE=wayland >> $HOME/.xsessionrc
-            echo export XDG_RUNTIME_DIR=/run/user/$(id -u) >> $HOME/.xsessionrc
         else
             echo export XDG_SESSION_TYPE=x11 >> $HOME/.xsessionrc
         fi
@@ -204,6 +200,26 @@ if $NEED_WINDOW_MANAGER; then
         eval `dbus-launch --sh-syntax`
     fi
     eval `echo '' | /usr/bin/gnome-keyring-daemon -r -d --unlock --components=secrets`
+
+    # Run mutter as nested wayland compositor to provide Wayland environment
+    # on top of XVfb.
+    if [ $MOZ_ENABLE_WAYLAND ]; then
+      env | grep "DISPLAY"
+      export XDG_RUNTIME_DIR=$WORKING_DIR
+      mutter --display=:0 --wayland --nested &
+      export WAYLAND_DISPLAY=wayland-0
+      retry_count=0
+      max_retries=5
+      until [ $retry_count -gt $max_retries ]; do
+        if [ -S "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" ]; then
+            retry_count=$(($max_retries + 1))
+        else
+            retry_count=$(($retry_count + 1))
+            echo "Waiting for Mutter, retry: $retry_count"
+            sleep 2
+        fi
+      done
+    fi
 fi
 
 if [[ $NEED_COMPIZ == true ]]  && [[ $RELEASE == 16.04 ]]; then