vm: fix endless loop in map_self if map returns nullptr

rpcs3/util/vm_native.cpp

@@ -1001,6 +1001,16 @@ namespace utils
 			if (!mapped)
 			{
 				mapped = this->map(nullptr, prot);
+
+				if (!mapped)
+				{
+					if ((ptr = m_ptr))
+					{
+						break;
+					}
+
+					return nullptr;
+				}
 			}
 
 			// Install mapped memory