AI Service overlay.

menu/menu_driver.c

@@ -2284,10 +2284,24 @@ void menu_display_draw(menu_display_ctx_draw_t *draw,
       return;
    if (draw->width <= 0)
       return;
-
    menu_disp->draw(draw, video_info);
 }
 
+void menu_display_draw_blend(menu_display_ctx_draw_t *draw,
+      video_frame_info_t *video_info)
+{
+   if (!menu_disp || !draw || !menu_disp->draw)
+      return;
+
+   if (draw->height <= 0)
+      return;
+   if (draw->width <= 0)
+      return;
+   menu_display_blend_begin(video_info);
+   menu_disp->draw(draw, video_info);
+   menu_display_blend_end(video_info);
+}
+
 void menu_display_draw_pipeline(menu_display_ctx_draw_t *draw,
       video_frame_info_t *video_info)
 {
@@ -3185,6 +3199,35 @@ bool menu_display_reset_textures_list(
 }
 
 
+bool menu_display_reset_textures_list_buffer(
+        uintptr_t *item, enum texture_filter_type filter_type,
+        void* buffer, unsigned buffer_len, enum image_type_enum image_type,
+        unsigned *width, unsigned *height)
+{
+   struct texture_image ti;
+
+   ti.width                      = 0;
+   ti.height                     = 0;
+   ti.pixels                     = NULL;
+   ti.supports_rgba              = video_driver_supports_rgba();
+
+   if (!image_texture_load_buffer(&ti, image_type, buffer, buffer_len))
+      return false;
+
+   if (width)
+      *width = ti.width;
+
+   if (height)
+      *height = ti.height;
+
+   /* if the poke interface doesn't support texture load then return false */  
+   if (!video_driver_texture_load(&ti, filter_type, item))
+       return false;
+   image_texture_free(&ti);
+   return true;
+}
+
+
 /**
  * menu_driver_find_handle:
  * @idx              : index of driver to get handle to.

menu/menu_driver.h

@@ -583,6 +583,8 @@ void menu_display_clear_color(menu_display_ctx_clearcolor_t *color,
       video_frame_info_t *video_info);
 void menu_display_draw(menu_display_ctx_draw_t *draw,
       video_frame_info_t *video_info);
+void menu_display_draw_blend(menu_display_ctx_draw_t *draw,
+      video_frame_info_t *video_info);
 void menu_display_draw_keyboard(
       uintptr_t hover_texture,
       const font_data_t *font,
@@ -680,6 +682,11 @@ bool menu_display_reset_textures_list(
       uintptr_t *item, enum texture_filter_type filter_type,
       unsigned *width, unsigned *height);
 
+bool menu_display_reset_textures_list_buffer(
+        uintptr_t *item, enum texture_filter_type filter_type,
+        void* buffer, unsigned buffer_len, enum image_type_enum image_type,
+        unsigned *width, unsigned *height);
+
 /* Returns the OSK key at a given position */
 int menu_display_osk_ptr_at_pos(void *data, int x, int y,
       unsigned width, unsigned height);

menu/widgets/menu_widgets.c

@@ -261,6 +261,12 @@ static menu_timer_t screenshot_timer;
 
 static unsigned screenshot_shotname_length;
 
+/* AI Service Overlay */
+static int ai_service_overlay_state                  = 0;
+static unsigned ai_service_overlay_width             = 0;
+static unsigned ai_service_overlay_height            = 0;
+static menu_texture_item ai_service_overlay_texture  = 0;
+
 /* Generic message */
 static menu_timer_t generic_message_timer;
 static float generic_message_alpha = 0.0f;
@@ -709,6 +715,55 @@ static void menu_widgets_draw_icon(
    menu_display_draw(&draw, video_info);
 }
 
+
+static void menu_widgets_draw_icon_blend(
+      video_frame_info_t *video_info,
+      unsigned icon_width,
+      unsigned icon_height,
+      uintptr_t texture,
+      float x, float y,
+      unsigned width, unsigned height,
+      float rotation, float scale_factor,
+      float *color)
+{
+   menu_display_ctx_rotate_draw_t rotate_draw;
+   menu_display_ctx_draw_t draw;
+   struct video_coords coords;
+   math_matrix_4x4 mymat;
+
+   if (!texture)
+      return;
+
+   rotate_draw.matrix       = &mymat;
+   rotate_draw.rotation     = rotation;
+   rotate_draw.scale_x      = scale_factor;
+   rotate_draw.scale_y      = scale_factor;
+   rotate_draw.scale_z      = 1;
+   rotate_draw.scale_enable = true;
+
+   menu_display_rotate_z(&rotate_draw, video_info);
+
+   coords.vertices      = 4;
+   coords.vertex        = NULL;
+   coords.tex_coord     = NULL;
+   coords.lut_tex_coord = NULL;
+   coords.color         = color;
+
+   draw.x               = x;
+   draw.y               = height - y - icon_height;
+   draw.width           = icon_width;
+   draw.height          = icon_height;
+   draw.scale_factor    = scale_factor;
+   draw.rotation        = rotation;
+   draw.coords          = &coords;
+   draw.matrix_data     = &mymat;
+   draw.texture         = texture;
+   draw.prim_type       = MENU_DISPLAY_PRIM_TRIANGLESTRIP;
+   draw.pipeline.id     = 0;
+
+   menu_display_draw_blend(&draw, video_info);
+}
+
 static float menu_widgets_get_thumbnail_scale_factor(const float dst_width, const float dst_height,
       const float image_width, const float image_height)
 {
@@ -1134,7 +1189,6 @@ static void menu_widgets_draw_regular_msg(menu_widget_msg_t *msg, video_frame_in
          (msg_queue_scissor_start_x + msg->width - simple_widget_padding*2) * msg->unfold, video_info->height);
    }
 
-
    if (msg_queue_has_icons)
    {
       menu_display_blend_begin(video_info);
@@ -1286,6 +1340,57 @@ void menu_widgets_frame(video_frame_info_t *video_info)
    font_raster_regular.carr.coords.vertices = 0;
    font_raster_bold.carr.coords.vertices    = 0;
 
+   /* AI Service overlay */
+   if (ai_service_overlay_state > 0)
+   {
+      float outline_color[16] = {
+      0.00, 1.00, 0.00, 1.00,
+      0.00, 1.00, 0.00, 1.00,
+      0.00, 1.00, 0.00, 1.00,
+      0.00, 1.00, 0.00, 1.00,
+      };
+
+      menu_widgets_draw_icon_blend(video_info,
+         video_info->width, video_info->height,
+         ai_service_overlay_texture,
+         0, 0,
+         video_info->width, video_info->height,
+         0, 1, menu_widgets_pure_white
+      );
+      /* top line */
+      menu_display_draw_quad(video_info,
+         0, 0,
+         video_info->width, 1,
+         video_info->width, video_info->height,
+         outline_color
+      );
+      /* bottom line */
+      menu_display_draw_quad(video_info,
+         0, video_info->height-1,
+         video_info->width, 1,
+         video_info->width, video_info->height,
+         outline_color
+      );
+      /* left line */
+      menu_display_draw_quad(video_info,
+         0, 0,
+         1, video_info->height,
+         video_info->width, video_info->height,
+         outline_color
+      );
+      /* right line */
+      menu_display_draw_quad(video_info,
+         video_info->width-1, 0,
+         1, video_info->height,
+         video_info->width, video_info->height,
+         outline_color
+      );
+
+      if (ai_service_overlay_state == 2)
+          ai_service_overlay_state = 3;
+   }
+
+
    /* Libretro message */
    if (libretro_message_alpha > 0.0f)
    {
@@ -1957,6 +2062,9 @@ void menu_widgets_free(void)
    menu_timer_kill(&libretro_message_timer);
    menu_animation_kill_by_tag(&libretro_tag);
 
+   /* AI Service overlay */
+   /* ... */
+
    /* Volume */
    volume_alpha = 0.0f;
 
@@ -2014,6 +2122,46 @@ bool menu_widgets_set_fps_text(const char *new_fps_text)
    return true;
 }
 
+int menu_widgets_ai_service_overlay_get_state()
+{
+   return ai_service_overlay_state;
+}
+
+bool menu_widgets_ai_service_overlay_set_state(int state)
+{
+   ai_service_overlay_state = state;
+   return true;
+}
+
+
+
+bool menu_widgets_ai_service_overlay_load(
+        char* buffer, unsigned buffer_len, enum image_type_enum image_type)
+{
+   if (ai_service_overlay_state == 0)
+   {
+      bool res;
+      res = menu_display_reset_textures_list_buffer(
+               &ai_service_overlay_texture, 
+               TEXTURE_FILTER_MIPMAP_LINEAR, 
+               (void *) buffer, buffer_len, image_type,
+               &ai_service_overlay_width, &ai_service_overlay_height);
+      if (res)
+         ai_service_overlay_state = 1;
+      return res;
+   }
+   return true;
+}
+
+void menu_widgets_ai_service_overlay_unload()
+{
+   if (ai_service_overlay_state == 1)
+   {
+      video_driver_texture_unload(&ai_service_overlay_texture);
+      ai_service_overlay_state = 0;
+   }
+}
+
 static void menu_widgets_screenshot_fadeout(void *userdata)
 {
    menu_animation_ctx_entry_t entry;

menu/widgets/menu_widgets.h

@@ -55,6 +55,15 @@ void menu_widgets_iterate(unsigned width, unsigned height);
 
 void menu_widgets_screenshot_taken(const char *shotname, const char *filename);
 
+/* AI Service functions */
+int menu_widgets_ai_service_overlay_get_state();
+bool menu_widgets_ai_service_overlay_set_state(int state);
+
+bool menu_widgets_ai_service_overlay_load(
+        char* buffer, unsigned buffer_len, enum image_type_enum image_type);
+void menu_widgets_ai_service_overlay_unload();
+
+
 void menu_widgets_start_load_content_animation(const char *content_name, bool remove_extension);
 
 void menu_widgets_cleanup_load_content_animation(void);

retroarch.c

@@ -2730,11 +2730,10 @@ static void handle_translation_cb(
    int i                             = 0;
    int start                         = -1;
    char* found_string                = NULL;
+   char* error_string                = NULL;
    int curr_state                    = 0;
 
-   if (!is_paused && settings->uints.ai_service_mode != 1)
+   RARCH_LOG("RESULT FROM AI SERVICE...\n");
-      goto finish;
-
    if (!data || error)
       goto finish;
 
@@ -2775,6 +2774,12 @@ static void handle_translation_cb(
                curr_state = 0;
             }
 #endif
+            else if (curr_state == 3)
+            {
+               error_string = (char*)malloc(i-start+1);
+               strlcpy(error_string, body_copy+start+1, i-start);
+               curr_state = 0;
+            }
             else if (string_is_equal(found_string, "image"))
             {
                curr_state = 1;
@@ -2785,15 +2790,34 @@ static void handle_translation_cb(
                curr_state = 2;
                free(found_string);
             }
+            else if (string_is_equal(found_string, "error"))
+            {
+               curr_state = 3;
+               free(found_string);
+            }
             else
+            {
               curr_state = 0;
+              free(found_string);
+            }
             start = -1;
          }
       }
       i++;
    }
-   if (found_string)
+   
-      free(found_string);
+   if (string_is_equal(error_string, "No text found."))
+   {
+      RARCH_LOG("No text found...\n");
+#ifdef HAVE_MENU_WIDGETS
+      if (menu_widgets_paused)
+      {
+         /* In this case we have to unpause and then repause for a frame */
+         menu_widgets_ai_service_overlay_set_state(2);
+         command_event(CMD_EVENT_UNPAUSE, NULL);        
+      }
+#endif
+   }
 
    if (!raw_image_file_data && !raw_sound_data)
    {
@@ -2806,6 +2830,60 @@ static void handle_translation_cb(
       /* Get the video frame dimensions reference */
       video_driver_cached_frame_get(&dummy_data, &width, &height, &pitch);
 
+      /* try two different modes for text display *
+       * In the first mode, we use menu widget overlays, but they require
+       * the video poke interface to be able to load image buffers.
+       *
+       * The other method is to draw to the video buffer directly, which needs
+       * a software core to be running. */
+#ifdef HAVE_MENU_WIDGETS
+      if (video_driver_poke
+          && video_driver_poke->load_texture && video_driver_poke->unload_texture)
+      {
+         bool ai_res;
+         enum image_type_enum image_type;
+         /* Write to overlay */
+         if (raw_image_file_data[0] == 'B' && raw_image_file_data[1] == 'M')
+         {
+             image_type = IMAGE_TYPE_BMP;
+         }
+         else if (raw_image_file_data[1] == 'P' && 
+                  raw_image_file_data[2] == 'N' &&
+                  raw_image_file_data[3] == 'G')
+         {
+            image_type = IMAGE_TYPE_PNG;
+         }
+         else
+         {
+            RARCH_LOG("Invalid image type returned from server.\n");
+            goto finish;
+         }
+         
+         ai_res = menu_widgets_ai_service_overlay_load(
+                     raw_image_file_data, (unsigned) new_image_size, 
+                     image_type);
+         
+         if (!ai_res)
+         {
+            RARCH_LOG("Video driver not supported for AI Service.");
+            runloop_msg_queue_push(
+               /* msg_hash_to_str(MSG_VIDEO_DRIVER_NOT_SUPPORTED), */
+               "Video driver not supported.",
+               1, 180, true,
+               NULL, MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO);
+         }
+         else if (menu_widgets_paused)
+         {
+            /* In this case we have to unpause and then repause for a frame */
+            menu_widgets_ai_service_overlay_set_state(2);/* Unpausing state */
+            command_event(CMD_EVENT_UNPAUSE, NULL);
+         }
+      }
+      else 
+#endif
+      /* Can't use menu widget overlays, so try writing to video buffer */
+      {
+         /* Write to video buffer directly (software cores only) */
          if (raw_image_file_data[0] == 'B' && raw_image_file_data[1] == 'M')
          {
             /* This is a BMP file coming back. */
@@ -2884,6 +2962,7 @@ static void handle_translation_cb(
             RARCH_LOG("Output from URL not a valid file type, or is not supported.\n");
             goto finish;
          }    
+
          scaler = (struct scaler_ctx*)calloc(1, sizeof(struct scaler_ctx));
          if (!scaler)
             goto finish;
@@ -2895,10 +2974,7 @@ static void handle_translation_cb(
                and translate it, and we have the translated image in
                the raw_image_data buffer.
             */
-
+            RARCH_LOG("Hardware frame buffer core, but selected video driver isn't supported.\n");
-         /* TODO: write to the viewport in this case */
-         RARCH_LOG("Hardware frame buffer... writing to viewport"
-               " not yet supported.\n");
             goto finish;
          }
 
@@ -2939,6 +3015,7 @@ static void handle_translation_cb(
                (uint8_t*)raw_image_data + (image_height - 1) * width * 3);
          video_driver_frame(raw_output_data, image_width, image_height, pitch);
       }
+   }
   
 #ifdef HAVE_AUDIOMIXER
    if (raw_sound_data)
@@ -2998,7 +3075,8 @@ finish:
       free(raw_image_data);
    if (scaler)
       free(scaler);
-
+   if (error_string)
+      free(error_string);
    if (raw_output_data)
       free(raw_output_data);
 }
@@ -3163,10 +3241,10 @@ static const char *ai_service_get_str(enum translation_lang id)
 
    To make your own server, it must listen for a POST request, which
    will consist of a JSON body, with the "image" field as a base64
-   encoded string of a 24bit-BMP that the will be translated.  The server
+   encoded string of a 24bit-BMP/PNG that the will be translated.  
-   must output the translated image in the form of a JSON body, with
+   The server must output the translated image in the form of a 
-   the "image" field also as a base64 encoded 24bit-BMP, or
+   JSON body, with the "image" field also as a base64 encoded 
-   as an alpha channel png.
+   24bit-BMP, or as an alpha channel png.
    */
 static bool run_translation_service(void)
 {
@@ -3194,11 +3272,37 @@ static bool run_translation_service(void)
    const char *rf2                       = "\"}\0";
    char *rf3                             = NULL;
    bool TRANSLATE_USE_BMP                = false;
+   bool use_overlay                      = true;
 
    const char *label                     = NULL;
    char* system_label                    = NULL;
    core_info_t *core_info                = NULL;
 
+#ifdef HAVE_MENU_WIDGETS   
+   if (menu_widgets_ai_service_overlay_get_state() != 0)
+   {
+      /* For the case when ai service pause is disabled. */
+      menu_widgets_ai_service_overlay_unload();
+      goto finish;
+   }
+#else
+   if (!settings->bools.ai_service_pause)
+   {
+      RARCH_LOG("Pause toggle not supported without menu widgets.\n");
+   }
+#endif
+
+#ifdef HAVE_MENU_WIDGETS
+   if (video_driver_poke
+       && video_driver_poke->load_texture && video_driver_poke->unload_texture)
+   {
+      use_overlay = true;
+   }
+   else
+#endif
+      use_overlay = false; 
+
+
    /* get the core info here so we can pass long the game name */
    core_info_get_current_core(&core_info);
 
@@ -3264,16 +3368,29 @@ static bool run_translation_service(void)
          goto finish;
 
       if (!video_driver_read_viewport(bit24_image_prev, false))
+      {
+         RARCH_LOG("Could not read viewport for translation service...\n");
          goto finish;
+      }
 
       /* TODO: Rescale down to regular resolution */
-      width = vp.width;
+      scaler->in_fmt = SCALER_FMT_BGR24;
-      height = vp.height;
+      scaler->out_fmt = SCALER_FMT_BGR24;
-      bit24_image      = bit24_image_prev;
+      scaler->scaler_type = SCALER_TYPE_POINT;
-      bit24_image_prev = NULL;
+      scaler->in_width = vp.width;
+      scaler->in_height = vp.height;
+      scaler->out_width = width;
+      scaler->out_height = height;
+      scaler_ctx_gen_filter(scaler);
+
+      scaler->in_stride = vp.width*3;
+      scaler->out_stride = width*3;
+      scaler_ctx_scale_direct(scaler, bit24_image, bit24_image_prev);
+      scaler_ctx_gen_reset(scaler);
    }
    else
    {
+      /* This is a software core, so just change the pixel format to 24-bit. */
       bit24_image = (uint8_t*)malloc(width * height * 3);
       if (!bit24_image)
           goto finish;
@@ -3298,13 +3415,14 @@ static bool run_translation_service(void)
       goto finish;
    }
 
+   if (TRANSLATE_USE_BMP)
+   {
       /*
         At this point, we should have a screenshot in the buffer, so allocate
         an array to contain the BMP image along with the BMP header as bytes,
         and then covert that to a b64 encoded array for transport in JSON.
       */
-   if (TRANSLATE_USE_BMP)
+
-   {
       form_bmp_header(header, width, height, false);
       bmp_buffer = (uint8_t*)malloc(width * height * 3+54);
       if (!bmp_buffer)
@@ -3355,7 +3473,7 @@ static bool run_translation_service(void)
       memcpy(json_buffer+11+out_length, (const void*)rf3, (16+strlen(system_label))*sizeof(uint8_t));   
    else
       memcpy(json_buffer+11+out_length, (const void*)rf2, 3*sizeof(uint8_t));
-
+   RARCH_LOG("Request size: %d\n", out_length);
    {
       char separator  = '?';
       char new_ai_service_url[PATH_MAX_LENGTH];
@@ -3409,9 +3527,23 @@ static bool run_translation_service(void)
 
          /*"image" is included for backwards compatability with
           * vgtranslate < 1.04 */
-         char* mode_chr = "image,png";
+         char* mode_chr;
-         if (settings->uints.ai_service_mode == 1)
+         if (settings->uints.ai_service_mode == 0)
+         {
+            if (use_overlay)
+               mode_chr = "image,png,png-a";
+            else
+               mode_chr = "image,png";
+         }
+         else if (settings->uints.ai_service_mode == 1)
             mode_chr = "sound,wav";
+         else if (settings->uints.ai_service_mode == 2)
+         {
+            if (use_overlay)
+               mode_chr = "image,png,png-a,sound,wav";
+            else
+               mode_chr = "image,png,sound,wav";         
+         }
 
          snprintf(temp_string,
                sizeof(temp_string),
@@ -3421,7 +3553,7 @@ static bool run_translation_service(void)
          strlcat(new_ai_service_url, temp_string,
                  sizeof(new_ai_service_url));
       }
-
+      RARCH_LOG("SENDING... %s\n", new_ai_service_url);
       task_push_http_post_transfer(new_ai_service_url,
             json_buffer, true, NULL, handle_translation_cb, NULL);
    }
@@ -4625,7 +4757,6 @@ static void retroarch_pause_checks(void)
    if (is_paused)
    {
       RARCH_LOG("%s\n", msg_hash_to_str(MSG_PAUSED));
-      command_event(CMD_EVENT_AUDIO_STOP, NULL);
 
 #if defined(HAVE_MENU) && defined(HAVE_MENU_WIDGETS)
       if (menu_widgets_inited)
@@ -4652,8 +4783,13 @@ static void retroarch_pause_checks(void)
          menu_widgets_paused = is_paused;
 #endif
       RARCH_LOG("%s\n", msg_hash_to_str(MSG_UNPAUSED));
-      command_event(CMD_EVENT_AUDIO_START, NULL);
+
    }
+
+#ifdef HAVE_MENU_WIDGETS
+   if (menu_widgets_ai_service_overlay_get_state() == 1)
+      menu_widgets_ai_service_overlay_unload();
+#endif
 }
 
 static void retroarch_frame_time_free(void)
@@ -4749,23 +4885,26 @@ bool command_event(enum event_command cmd, void *data)
       {
 #ifdef HAVE_TRANSLATE
          settings_t *settings      = configuration_settings;
-         if (settings->uints.ai_service_mode == 0)
+         if (settings->bools.ai_service_pause)
          {
-            /* Default mode - pause on call, unpause on second press. */
+            /* pause on call, unpause on second press. */
             if (!runloop_paused)
             {
                command_event(CMD_EVENT_PAUSE, NULL);
                command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
             }
             else
+            {
                command_event(CMD_EVENT_UNPAUSE, NULL);
             }
-         /* Text-to-Speech mode - don't pause */
+         }       
-         else if (settings->uints.ai_service_mode == 1)
-            command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
          else
          {
-            RARCH_LOG("Invalid AI Service Mode.\n");
+           /* Don't pause - useful for Text-To-Speech since
+            * the audio can't currently play while paused.
+            * Also useful for cases when users don't want the
+            * core's sound to stop while translating. */       
+            command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
          }
 #endif
          break;
@@ -24849,6 +24988,14 @@ static enum runloop_state runloop_check_state(void)
 #endif
 #endif
 
+#ifdef HAVE_MENU_WIDGETS
+   if (menu_widgets_ai_service_overlay_get_state() == 3)
+   {
+      command_event(CMD_EVENT_PAUSE, NULL);
+      menu_widgets_ai_service_overlay_set_state(1);
+   }
+#endif
+
 #ifdef HAVE_LIBNX
    /* Should be called once per frame */
    if (!appletMainLoop())