libretro/RetroArch
1
0
Fork 0
mirror of https://github.com/libretro/RetroArch.git synced 2024-11-24 16:39:43 +00:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

AI Service overlay.

Browse Source
This commit is contained in:
Barry Rowe 2019-10-23 14:15:14 -07:00
parent 14f8ee1e40
commit d641e67876
5 changed files with 512 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
menu/menu_driver.c
View File

@ -2284,10 +2284,24 @@ void menu_display_draw(menu_display_ctx_draw_t *draw,
return;
if (draw->width <= 0)
return;
menu_disp->draw(draw, video_info);
}
void menu_display_draw_blend(menu_display_ctx_draw_t *draw,
video_frame_info_t *video_info)
{
if (!menu_disp || !draw || !menu_disp->draw)
return;
if (draw->height <= 0)
return;
if (draw->width <= 0)
return;
menu_display_blend_begin(video_info);
menu_disp->draw(draw, video_info);
menu_display_blend_end(video_info);
}
void menu_display_draw_pipeline(menu_display_ctx_draw_t *draw,
video_frame_info_t *video_info)
{
@ -3185,6 +3199,35 @@ bool menu_display_reset_textures_list(
}
bool menu_display_reset_textures_list_buffer(
uintptr_t *item, enum texture_filter_type filter_type,
void* buffer, unsigned buffer_len, enum image_type_enum image_type,
unsigned *width, unsigned *height)
{
struct texture_image ti;
ti.width = 0;
ti.height = 0;
ti.pixels = NULL;
ti.supports_rgba = video_driver_supports_rgba();
if (!image_texture_load_buffer(&ti, image_type, buffer, buffer_len))
return false;
if (width)
*width = ti.width;
if (height)
*height = ti.height;
/* if the poke interface doesn't support texture load then return false */
if (!video_driver_texture_load(&ti, filter_type, item))
return false;
image_texture_free(&ti);
return true;
}
/**
* menu_driver_find_handle:
* @idx : index of driver to get handle to.

7
menu/menu_driver.h
View File

@ -583,6 +583,8 @@ void menu_display_clear_color(menu_display_ctx_clearcolor_t *color,
video_frame_info_t *video_info);
void menu_display_draw(menu_display_ctx_draw_t *draw,
video_frame_info_t *video_info);
void menu_display_draw_blend(menu_display_ctx_draw_t *draw,
video_frame_info_t *video_info);
void menu_display_draw_keyboard(
uintptr_t hover_texture,
const font_data_t *font,
@ -680,6 +682,11 @@ bool menu_display_reset_textures_list(
uintptr_t *item, enum texture_filter_type filter_type,
unsigned *width, unsigned *height);
bool menu_display_reset_textures_list_buffer(
uintptr_t *item, enum texture_filter_type filter_type,
void* buffer, unsigned buffer_len, enum image_type_enum image_type,
unsigned *width, unsigned *height);
/* Returns the OSK key at a given position */
int menu_display_osk_ptr_at_pos(void *data, int x, int y,
unsigned width, unsigned height);

150
menu/widgets/menu_widgets.c
View File

@ -261,6 +261,12 @@ static menu_timer_t screenshot_timer;
static unsigned screenshot_shotname_length;
/* AI Service Overlay */
static int ai_service_overlay_state = 0;
static unsigned ai_service_overlay_width = 0;
static unsigned ai_service_overlay_height = 0;
static menu_texture_item ai_service_overlay_texture = 0;
/* Generic message */
static menu_timer_t generic_message_timer;
static float generic_message_alpha = 0.0f;
@ -709,6 +715,55 @@ static void menu_widgets_draw_icon(
menu_display_draw(&draw, video_info);
}
static void menu_widgets_draw_icon_blend(
video_frame_info_t *video_info,
unsigned icon_width,
unsigned icon_height,
uintptr_t texture,
float x, float y,
unsigned width, unsigned height,
float rotation, float scale_factor,
float *color)
{
menu_display_ctx_rotate_draw_t rotate_draw;
menu_display_ctx_draw_t draw;
struct video_coords coords;
math_matrix_4x4 mymat;
if (!texture)
return;
rotate_draw.matrix = &mymat;
rotate_draw.rotation = rotation;
rotate_draw.scale_x = scale_factor;
rotate_draw.scale_y = scale_factor;
rotate_draw.scale_z = 1;
rotate_draw.scale_enable = true;
menu_display_rotate_z(&rotate_draw, video_info);
coords.vertices = 4;
coords.vertex = NULL;
coords.tex_coord = NULL;
coords.lut_tex_coord = NULL;
coords.color = color;
draw.x = x;
draw.y = height - y - icon_height;
draw.width = icon_width;
draw.height = icon_height;
draw.scale_factor = scale_factor;
draw.rotation = rotation;
draw.coords = &coords;
draw.matrix_data = &mymat;
draw.texture = texture;
draw.prim_type = MENU_DISPLAY_PRIM_TRIANGLESTRIP;
draw.pipeline.id = 0;
menu_display_draw_blend(&draw, video_info);
}
static float menu_widgets_get_thumbnail_scale_factor(const float dst_width, const float dst_height,
const float image_width, const float image_height)
{
@ -1134,7 +1189,6 @@ static void menu_widgets_draw_regular_msg(menu_widget_msg_t *msg, video_frame_in
(msg_queue_scissor_start_x + msg->width - simple_widget_padding*2) * msg->unfold, video_info->height);
}
if (msg_queue_has_icons)
{
menu_display_blend_begin(video_info);
@ -1286,6 +1340,57 @@ void menu_widgets_frame(video_frame_info_t *video_info)
font_raster_regular.carr.coords.vertices = 0;
font_raster_bold.carr.coords.vertices = 0;
/* AI Service overlay */
if (ai_service_overlay_state > 0)
{
float outline_color[16] = {
0.00, 1.00, 0.00, 1.00,
0.00, 1.00, 0.00, 1.00,
0.00, 1.00, 0.00, 1.00,
0.00, 1.00, 0.00, 1.00,
};
menu_widgets_draw_icon_blend(video_info,
video_info->width, video_info->height,
ai_service_overlay_texture,
0, 0,
video_info->width, video_info->height,
0, 1, menu_widgets_pure_white
);
/* top line */
menu_display_draw_quad(video_info,
0, 0,
video_info->width, 1,
video_info->width, video_info->height,
outline_color
);
/* bottom line */
menu_display_draw_quad(video_info,
0, video_info->height-1,
video_info->width, 1,
video_info->width, video_info->height,
outline_color
);
/* left line */
menu_display_draw_quad(video_info,
0, 0,
1, video_info->height,
video_info->width, video_info->height,
outline_color
);
/* right line */
menu_display_draw_quad(video_info,
video_info->width-1, 0,
1, video_info->height,
video_info->width, video_info->height,
outline_color
);
if (ai_service_overlay_state == 2)
ai_service_overlay_state = 3;
}
/* Libretro message */
if (libretro_message_alpha > 0.0f)
{
@ -1957,6 +2062,9 @@ void menu_widgets_free(void)
menu_timer_kill(&libretro_message_timer);
menu_animation_kill_by_tag(&libretro_tag);
/* AI Service overlay */
/* ... */
/* Volume */
volume_alpha = 0.0f;
@ -2014,6 +2122,46 @@ bool menu_widgets_set_fps_text(const char *new_fps_text)
return true;
}
int menu_widgets_ai_service_overlay_get_state()
{
return ai_service_overlay_state;
}
bool menu_widgets_ai_service_overlay_set_state(int state)
{
ai_service_overlay_state = state;
return true;
}
bool menu_widgets_ai_service_overlay_load(
char* buffer, unsigned buffer_len, enum image_type_enum image_type)
{
if (ai_service_overlay_state == 0)
{
bool res;
res = menu_display_reset_textures_list_buffer(
&ai_service_overlay_texture,
TEXTURE_FILTER_MIPMAP_LINEAR,
(void *) buffer, buffer_len, image_type,
&ai_service_overlay_width, &ai_service_overlay_height);
if (res)
ai_service_overlay_state = 1;
return res;
}
return true;
}
void menu_widgets_ai_service_overlay_unload()
{
if (ai_service_overlay_state == 1)
{
video_driver_texture_unload(&ai_service_overlay_texture);
ai_service_overlay_state = 0;
}
}
static void menu_widgets_screenshot_fadeout(void *userdata)
{
menu_animation_ctx_entry_t entry;

9
menu/widgets/menu_widgets.h
View File

@ -55,6 +55,15 @@ void menu_widgets_iterate(unsigned width, unsigned height);
void menu_widgets_screenshot_taken(const char *shotname, const char *filename);
/* AI Service functions */
int menu_widgets_ai_service_overlay_get_state();
bool menu_widgets_ai_service_overlay_set_state(int state);
bool menu_widgets_ai_service_overlay_load(
char* buffer, unsigned buffer_len, enum image_type_enum image_type);
void menu_widgets_ai_service_overlay_unload();
void menu_widgets_start_load_content_animation(const char *content_name, bool remove_extension);
void menu_widgets_cleanup_load_content_animation(void);

211
retroarch.c
View File

@ -2730,11 +2730,10 @@ static void handle_translation_cb(
int i = 0;
int start = -1;
char* found_string = NULL;
char* error_string = NULL;
int curr_state = 0;
if (!is_paused && settings->uints.ai_service_mode != 1)
goto finish;
RARCH_LOG("RESULT FROM AI SERVICE...\n");
if (!data || error)
goto finish;
@ -2775,6 +2774,12 @@ static void handle_translation_cb(
curr_state = 0;
}
#endif
else if (curr_state == 3)
{
error_string = (char*)malloc(i-start+1);
strlcpy(error_string, body_copy+start+1, i-start);
curr_state = 0;
}
else if (string_is_equal(found_string, "image"))
{
curr_state = 1;
@ -2785,15 +2790,34 @@ static void handle_translation_cb(
curr_state = 2;
free(found_string);
}
else if (string_is_equal(found_string, "error"))
{
curr_state = 3;
free(found_string);
}
else
{
curr_state = 0;
free(found_string);
}
start = -1;
}
}
i++;
}
if (found_string)
free(found_string);
if (string_is_equal(error_string, "No text found."))
{
RARCH_LOG("No text found...\n");
#ifdef HAVE_MENU_WIDGETS
if (menu_widgets_paused)
{
/* In this case we have to unpause and then repause for a frame */
menu_widgets_ai_service_overlay_set_state(2);
command_event(CMD_EVENT_UNPAUSE, NULL);
}
#endif
}
if (!raw_image_file_data && !raw_sound_data)
{
@ -2806,6 +2830,60 @@ static void handle_translation_cb(
/* Get the video frame dimensions reference */
video_driver_cached_frame_get(&dummy_data, &width, &height, &pitch);
/* try two different modes for text display *
* In the first mode, we use menu widget overlays, but they require
* the video poke interface to be able to load image buffers.
*
* The other method is to draw to the video buffer directly, which needs
* a software core to be running. */
#ifdef HAVE_MENU_WIDGETS
if (video_driver_poke
&& video_driver_poke->load_texture && video_driver_poke->unload_texture)
{
bool ai_res;
enum image_type_enum image_type;
/* Write to overlay */
if (raw_image_file_data[0] == 'B' && raw_image_file_data[1] == 'M')
{
image_type = IMAGE_TYPE_BMP;
}
else if (raw_image_file_data[1] == 'P' &&
raw_image_file_data[2] == 'N' &&
raw_image_file_data[3] == 'G')
{
image_type = IMAGE_TYPE_PNG;
}
else
{
RARCH_LOG("Invalid image type returned from server.\n");
goto finish;
}
ai_res = menu_widgets_ai_service_overlay_load(
raw_image_file_data, (unsigned) new_image_size,
image_type);
if (!ai_res)
{
RARCH_LOG("Video driver not supported for AI Service.");
runloop_msg_queue_push(
/* msg_hash_to_str(MSG_VIDEO_DRIVER_NOT_SUPPORTED), */
"Video driver not supported.",
1, 180, true,
NULL, MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO);
}
else if (menu_widgets_paused)
{
/* In this case we have to unpause and then repause for a frame */
menu_widgets_ai_service_overlay_set_state(2);/* Unpausing state */
command_event(CMD_EVENT_UNPAUSE, NULL);
}
}
else
#endif
/* Can't use menu widget overlays, so try writing to video buffer */
{
/* Write to video buffer directly (software cores only) */
if (raw_image_file_data[0] == 'B' && raw_image_file_data[1] == 'M')
{
/* This is a BMP file coming back. */
@ -2884,6 +2962,7 @@ static void handle_translation_cb(
RARCH_LOG("Output from URL not a valid file type, or is not supported.\n");
goto finish;
}
scaler = (struct scaler_ctx*)calloc(1, sizeof(struct scaler_ctx));
if (!scaler)
goto finish;
@ -2895,10 +2974,7 @@ static void handle_translation_cb(
and translate it, and we have the translated image in
the raw_image_data buffer.
*/
/* TODO: write to the viewport in this case */
RARCH_LOG("Hardware frame buffer... writing to viewport"
" not yet supported.\n");
RARCH_LOG("Hardware frame buffer core, but selected video driver isn't supported.\n");
goto finish;
}
@ -2939,6 +3015,7 @@ static void handle_translation_cb(
(uint8_t*)raw_image_data + (image_height - 1) * width * 3);
video_driver_frame(raw_output_data, image_width, image_height, pitch);
}
}
#ifdef HAVE_AUDIOMIXER
if (raw_sound_data)
@ -2998,7 +3075,8 @@ finish:
free(raw_image_data);
if (scaler)
free(scaler);
if (error_string)
free(error_string);
if (raw_output_data)
free(raw_output_data);
}
@ -3163,10 +3241,10 @@ static const char *ai_service_get_str(enum translation_lang id)
To make your own server, it must listen for a POST request, which
will consist of a JSON body, with the "image" field as a base64
encoded string of a 24bit-BMP that the will be translated. The server
must output the translated image in the form of a JSON body, with
the "image" field also as a base64 encoded 24bit-BMP, or
as an alpha channel png.
encoded string of a 24bit-BMP/PNG that the will be translated.
The server must output the translated image in the form of a
JSON body, with the "image" field also as a base64 encoded
24bit-BMP, or as an alpha channel png.
*/
static bool run_translation_service(void)
{
@ -3194,11 +3272,37 @@ static bool run_translation_service(void)
const char *rf2 = "\"}\0";
char *rf3 = NULL;
bool TRANSLATE_USE_BMP = false;
bool use_overlay = true;
const char *label = NULL;
char* system_label = NULL;
core_info_t *core_info = NULL;
#ifdef HAVE_MENU_WIDGETS
if (menu_widgets_ai_service_overlay_get_state() != 0)
{
/* For the case when ai service pause is disabled. */
menu_widgets_ai_service_overlay_unload();
goto finish;
}
#else
if (!settings->bools.ai_service_pause)
{
RARCH_LOG("Pause toggle not supported without menu widgets.\n");
}
#endif
#ifdef HAVE_MENU_WIDGETS
if (video_driver_poke
&& video_driver_poke->load_texture && video_driver_poke->unload_texture)
{
use_overlay = true;
}
else
#endif
use_overlay = false;
/* get the core info here so we can pass long the game name */
core_info_get_current_core(&core_info);
@ -3264,16 +3368,29 @@ static bool run_translation_service(void)
goto finish;
if (!video_driver_read_viewport(bit24_image_prev, false))
{
RARCH_LOG("Could not read viewport for translation service...\n");
goto finish;
}
/* TODO: Rescale down to regular resolution */
width = vp.width;
height = vp.height;
bit24_image = bit24_image_prev;
bit24_image_prev = NULL;
scaler->in_fmt = SCALER_FMT_BGR24;
scaler->out_fmt = SCALER_FMT_BGR24;
scaler->scaler_type = SCALER_TYPE_POINT;
scaler->in_width = vp.width;
scaler->in_height = vp.height;
scaler->out_width = width;
scaler->out_height = height;
scaler_ctx_gen_filter(scaler);
scaler->in_stride = vp.width*3;
scaler->out_stride = width*3;
scaler_ctx_scale_direct(scaler, bit24_image, bit24_image_prev);
scaler_ctx_gen_reset(scaler);
}
else
{
/* This is a software core, so just change the pixel format to 24-bit. */
bit24_image = (uint8_t*)malloc(width * height * 3);
if (!bit24_image)
goto finish;
@ -3298,13 +3415,14 @@ static bool run_translation_service(void)
goto finish;
}
if (TRANSLATE_USE_BMP)
{
/*
At this point, we should have a screenshot in the buffer, so allocate
an array to contain the BMP image along with the BMP header as bytes,
and then covert that to a b64 encoded array for transport in JSON.
*/
if (TRANSLATE_USE_BMP)
{
form_bmp_header(header, width, height, false);
bmp_buffer = (uint8_t*)malloc(width * height * 3+54);
if (!bmp_buffer)
@ -3355,7 +3473,7 @@ static bool run_translation_service(void)
memcpy(json_buffer+11+out_length, (const void*)rf3, (16+strlen(system_label))*sizeof(uint8_t));
else
memcpy(json_buffer+11+out_length, (const void*)rf2, 3*sizeof(uint8_t));
RARCH_LOG("Request size: %d\n", out_length);
{
char separator = '?';
char new_ai_service_url[PATH_MAX_LENGTH];
@ -3409,9 +3527,23 @@ static bool run_translation_service(void)
/*"image" is included for backwards compatability with
* vgtranslate < 1.04 */
char* mode_chr = "image,png";
if (settings->uints.ai_service_mode == 1)
char* mode_chr;
if (settings->uints.ai_service_mode == 0)
{
if (use_overlay)
mode_chr = "image,png,png-a";
else
mode_chr = "image,png";
}
else if (settings->uints.ai_service_mode == 1)
mode_chr = "sound,wav";
else if (settings->uints.ai_service_mode == 2)
{
if (use_overlay)
mode_chr = "image,png,png-a,sound,wav";
else
mode_chr = "image,png,sound,wav";
}
snprintf(temp_string,
sizeof(temp_string),
@ -3421,7 +3553,7 @@ static bool run_translation_service(void)
strlcat(new_ai_service_url, temp_string,
sizeof(new_ai_service_url));
}
RARCH_LOG("SENDING... %s\n", new_ai_service_url);
task_push_http_post_transfer(new_ai_service_url,
json_buffer, true, NULL, handle_translation_cb, NULL);
}
@ -4625,7 +4757,6 @@ static void retroarch_pause_checks(void)
if (is_paused)
{
RARCH_LOG("%s\n", msg_hash_to_str(MSG_PAUSED));
command_event(CMD_EVENT_AUDIO_STOP, NULL);
#if defined(HAVE_MENU) && defined(HAVE_MENU_WIDGETS)
if (menu_widgets_inited)
@ -4652,8 +4783,13 @@ static void retroarch_pause_checks(void)
menu_widgets_paused = is_paused;
#endif
RARCH_LOG("%s\n", msg_hash_to_str(MSG_UNPAUSED));
command_event(CMD_EVENT_AUDIO_START, NULL);
}
#ifdef HAVE_MENU_WIDGETS
if (menu_widgets_ai_service_overlay_get_state() == 1)
menu_widgets_ai_service_overlay_unload();
#endif
}
static void retroarch_frame_time_free(void)
@ -4749,23 +4885,26 @@ bool command_event(enum event_command cmd, void *data)
{
#ifdef HAVE_TRANSLATE
settings_t *settings = configuration_settings;
if (settings->uints.ai_service_mode == 0)
if (settings->bools.ai_service_pause)
{
/* Default mode - pause on call, unpause on second press. */
/* pause on call, unpause on second press. */
if (!runloop_paused)
{
command_event(CMD_EVENT_PAUSE, NULL);
command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
}
else
{
command_event(CMD_EVENT_UNPAUSE, NULL);
}
/* Text-to-Speech mode - don't pause */
else if (settings->uints.ai_service_mode == 1)
command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
}
else
{
RARCH_LOG("Invalid AI Service Mode.\n");
/* Don't pause - useful for Text-To-Speech since
* the audio can't currently play while paused.
* Also useful for cases when users don't want the
* core's sound to stop while translating. */
command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
}
#endif
break;
@ -24849,6 +24988,14 @@ static enum runloop_state runloop_check_state(void)
#endif
#endif
#ifdef HAVE_MENU_WIDGETS
if (menu_widgets_ai_service_overlay_get_state() == 3)
{
command_event(CMD_EVENT_PAUSE, NULL);
menu_widgets_ai_service_overlay_set_state(1);
}
#endif
#ifdef HAVE_LIBNX
/* Should be called once per frame */
if (!appletMainLoop())