libretro/RetroArch
1
0
Fork 0
mirror of https://github.com/libretro/RetroArch.git synced 2024-11-23 16:09:47 +00:00
Code Issues Actions 23 Packages Projects Releases Wiki Activity

AI Service feature.

Browse Source
This commit is contained in:
Barry Rowe 2019-05-30 09:06:16 -07:00
parent 6a72b19907
commit e9625b3ed9
9 changed files with 229 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
command.c
View File

@ -242,7 +242,7 @@ static const struct cmd_map map[] = {
{ "MENU_RIGHT", RETRO_DEVICE_ID_JOYPAD_RIGHT },
{ "MENU_A", RETRO_DEVICE_ID_JOYPAD_A },
{ "MENU_B", RETRO_DEVICE_ID_JOYPAD_B },
{ "MENU_B", RETRO_DEVICE_ID_JOYPAD_B },
{ "AI_SERVICE", RARCH_AI_SERVICE },
};
#endif
@ -1767,8 +1767,34 @@ bool command_event(enum event_command cmd, void *data)
bsv_movie_check();
break;
case CMD_EVENT_AI_SERVICE_TOGGLE:
/* TODO/FIXME - implement */
{
settings_t *settings = config_get_ptr();
#ifdef HAVE_TRANSLATE
if (settings->uints.ai_service_mode == 0)
{
/* Default mode - pause on call, unpause on second press. */
if (!rarch_ctl(RARCH_CTL_IS_PAUSED, NULL))
{
command_event(CMD_EVENT_PAUSE, NULL);
command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
}
else
{
command_event(CMD_EVENT_UNPAUSE, NULL);
}
}
else if (settings->uints.ai_service_mode == 1)
{
/* Text-to-Speech mode - don't pause */
command_event(CMD_EVENT_AI_SERVICE_CALL, NULL);
}
else
{
RARCH_LOG("Invalid AI Service Mode.\n");
}
#endif
break;
}
case CMD_EVENT_STREAMING_TOGGLE:
if (streaming_is_enabled())
command_event(CMD_EVENT_RECORD_DEINIT, NULL);
@ -2524,7 +2550,6 @@ TODO: Add a setting for these tweaks */
bool is_idle = false;
bool is_slowmotion = false;
bool is_perfcnt_enable = false;
settings_t *settings = config_get_ptr();
#ifdef HAVE_DISCORD
discord_userdata_t userdata;
@ -2548,22 +2573,6 @@ TODO: Add a setting for these tweaks */
if (!is_idle)
video_driver_cached_frame();
/* If OCR enabled, translate the screen while paused */
if (settings->bools.translation_service_enable)
{
#ifdef HAVE_TRANSLATE
if (!g_translation_service_status)
{
RARCH_LOG("OCR START\n");
run_translation_service();
g_translation_service_status = true;
}
#else
RARCH_LOG("OCR Translation not enabled in build. Include HAVE_TRANSLATE define.\n");
#endif
}
#ifdef HAVE_DISCORD
userdata.status = DISCORD_PRESENCE_GAME_PAUSED;
command_event(CMD_EVENT_DISCORD_UPDATE, &userdata);
@ -2573,9 +2582,6 @@ TODO: Add a setting for these tweaks */
{
#if defined(HAVE_MENU) && defined(HAVE_MENU_WIDGETS)
menu_widgets_set_paused(is_paused);
#endif
#ifdef HAVE_TRANSLATE
g_translation_service_status = false;
#endif
RARCH_LOG("%s\n", msg_hash_to_str(MSG_UNPAUSED));
command_event(CMD_EVENT_AUDIO_START, NULL);
@ -3094,6 +3100,14 @@ TODO: Add a setting for these tweaks */
}
#endif
break;
case CMD_EVENT_AI_SERVICE_CALL:
{
#ifdef HAVE_TRANSLATE
RARCH_LOG("AI Service Called...\n");
run_translation_service();
#endif
}
case CMD_EVENT_NONE:
return false;
}

3
command.h
View File

@ -255,7 +255,8 @@ enum event_command
CMD_EVENT_SHADER_PREV,
CMD_EVENT_CHEAT_INDEX_PLUS,
CMD_EVENT_CHEAT_INDEX_MINUS,
CMD_EVENT_CHEAT_TOGGLE
CMD_EVENT_CHEAT_TOGGLE,
CMD_EVENT_AI_SERVICE_CALL
};
bool command_set_shader(const char *arg);

4
config.def.h
View File

@ -457,8 +457,6 @@ static bool menu_swap_ok_cancel_buttons = false;
static bool quit_press_twice = false;
static bool default_translation_service_enable = false;
static bool default_log_to_file = false;
static bool log_to_file_timestamp = false;
@ -982,6 +980,6 @@ static char buildbot_assets_server_url[] = "http://buildbot.libretro.com/assets/
static char default_discord_app_id[] = "475456035851599874";
static char default_translation_service_url[] = "http://localhost:4404/";
static char default_ai_service_url[] = "http://localhost:4404/";
#endif

3
config.def.keybinds.h
View File

@ -104,6 +104,7 @@ static const struct retro_keybind retro_keybinds_1[] = {
{ true, RARCH_MENU_TOGGLE, MENU_ENUM_LABEL_VALUE_INPUT_META_MENU_TOGGLE, RETROK_SPACE, NO_BTN, NO_BTN, 0, AXIS_NONE },
{ true, RARCH_RECORDING_TOGGLE, MENU_ENUM_LABEL_VALUE_INPUT_META_RECORDING_TOGGLE, RETROK_UNKNOWN, NO_BTN, NO_BTN, 0, AXIS_NONE },
{ true, RARCH_STREAMING_TOGGLE, MENU_ENUM_LABEL_VALUE_INPUT_META_STREAMING_TOGGLE, RETROK_UNKNOWN, NO_BTN, NO_BTN, 0, AXIS_NONE },
{ true, RARCH_AI_SERVICE, MENU_ENUM_LABEL_VALUE_INPUT_META_AI_SERVICE, RETROK_UNKNOWN, NO_BTN, NO_BTN, 0, AXIS_NONE },
#else
{ true, RETRO_DEVICE_ID_JOYPAD_B, MENU_ENUM_LABEL_VALUE_INPUT_JOYPAD_B, RETROK_z, NO_BTN, NO_BTN, 0, AXIS_NONE },
{ true, RETRO_DEVICE_ID_JOYPAD_Y, MENU_ENUM_LABEL_VALUE_INPUT_JOYPAD_Y, RETROK_a, NO_BTN, NO_BTN, 0, AXIS_NONE },
@ -185,6 +186,8 @@ static const struct retro_keybind retro_keybinds_1[] = {
{ true, RARCH_MENU_TOGGLE, MENU_ENUM_LABEL_VALUE_INPUT_META_MENU_TOGGLE, RETROK_F1, NO_BTN, NO_BTN, 0, AXIS_NONE },
{ true, RARCH_RECORDING_TOGGLE, MENU_ENUM_LABEL_VALUE_INPUT_META_RECORDING_TOGGLE, RETROK_UNKNOWN, NO_BTN, NO_BTN, 0, AXIS_NONE },
{ true, RARCH_STREAMING_TOGGLE, MENU_ENUM_LABEL_VALUE_INPUT_META_STREAMING_TOGGLE, RETROK_UNKNOWN, NO_BTN, NO_BTN, 0, AXIS_NONE },
{ true, RARCH_AI_SERVICE, MENU_ENUM_LABEL_VALUE_INPUT_META_AI_SERVICE, RETROK_UNKNOWN, NO_BTN, NO_BTN, 0, AXIS_NONE },
#endif
};

9
configuration.c
View File

@ -1167,8 +1167,7 @@ static struct config_array_setting *populate_settings_array(settings_t *settings
SETTING_ARRAY("youtube_stream_key", settings->arrays.youtube_stream_key, true, NULL, true);
SETTING_ARRAY("twitch_stream_key", settings->arrays.twitch_stream_key, true, NULL, true);
SETTING_ARRAY("discord_app_id", settings->arrays.discord_app_id, true, default_discord_app_id, true);
SETTING_ARRAY("translation_service_url", settings->arrays.translation_service_url, true, default_translation_service_url, true);
SETTING_ARRAY("ai_service_url", settings->arrays.ai_service_url, true, "", true);
SETTING_ARRAY("ai_service_url", settings->arrays.ai_service_url, true, default_ai_service_url, true);
*size = count;
@ -1616,8 +1615,6 @@ static struct config_bool_setting *populate_settings_bool(settings_t *settings,
#ifdef HAVE_OZONE
SETTING_BOOL("ozone_collapse_sidebar", &settings->bools.ozone_collapse_sidebar, true, DEFAULT_OZONE_COLLAPSE_SIDEBAR, false);
#endif
SETTING_BOOL("translation_service_enable", &settings->bools.translation_service_enable, true, default_translation_service_enable, false);
SETTING_BOOL("log_to_file", &settings->bools.log_to_file, true, default_log_to_file, false);
SETTING_OVERRIDE(RARCH_OVERRIDE_SETTING_LOG_TO_FILE);
SETTING_BOOL("log_to_file_timestamp", &settings->bools.log_to_file_timestamp, true, log_to_file_timestamp, false);
@ -2022,8 +2019,8 @@ void config_set_defaults(void)
strlcpy(settings->arrays.discord_app_id,
default_discord_app_id, sizeof(settings->arrays.discord_app_id));
strlcpy(settings->arrays.translation_service_url,
default_translation_service_url, sizeof(settings->arrays.translation_service_url));
strlcpy(settings->arrays.ai_service_url,
default_ai_service_url, sizeof(settings->arrays.ai_service_url));
#ifdef HAVE_MATERIALUI

3
configuration.h
View File

@ -334,9 +334,6 @@ typedef struct settings
bool enable_device_vibration;
bool ozone_collapse_sidebar;
bool translation_service_enable;
bool log_to_file;
bool log_to_file_timestamp;

3
retroarch.c
View File

@ -2215,7 +2215,7 @@ const struct input_bind_map input_config_bind_map[RARCH_BIND_LIST_END_NULL] = {
#endif
DECLARE_META_BIND(2, recording_toggle, RARCH_RECORDING_TOGGLE, MENU_ENUM_LABEL_VALUE_INPUT_META_RECORDING_TOGGLE),
DECLARE_META_BIND(2, streaming_toggle, RARCH_STREAMING_TOGGLE, MENU_ENUM_LABEL_VALUE_INPUT_META_STREAMING_TOGGLE),
DECLARE_META_BIND(2, streaming_toggle, RARCH_AI_SERVICE, MENU_ENUM_LABEL_VALUE_INPUT_META_AI_SERVICE),
DECLARE_META_BIND(2, ai_service, RARCH_AI_SERVICE, MENU_ENUM_LABEL_VALUE_INPUT_META_AI_SERVICE),
};
typedef struct turbo_buttons turbo_buttons_t;
@ -16900,7 +16900,6 @@ void rarch_force_video_driver_fallback(const char *driver)
free(title);
}
exit(1);
}

276
translation/translation_service.c
View File

@ -7,12 +7,28 @@
#include "gfx/video_frame.h"
#include "gfx/scaler/scaler.h"
#include "tasks/tasks_internal.h"
#include <audio/audio_mixer.h>
#include "tasks/task_audio_mixer.h"
#include "configuration.h"
#include "retroarch.h"
#include "verbosity.h"
#ifdef HAVE_TRANSLATE
#include "translation/translation_service.h"
#endif
typedef struct nbio_buf
{
void *buf;
unsigned bufsize;
char *path;
} nbio_buf_t;
/*
bool g_translation_service_status = false;
*/
static void form_bmp_header(uint8_t *header, unsigned width, unsigned height,
bool is32bpp)
@ -103,9 +119,9 @@ bool run_translation_service(void)
image to the screen.
Supported client/services (thus far)
-Ztranslate client/service ( www.ztranslate.net/docs/service )
-VGTranslate client ( www.gitlab.com/spherebeaker/vg_translate )
-Ztranslate client/service ( www.ztranslate.net/docs/service )
To use a client, download the relevant code/release, configure
them, and run them on your local machine, or network. Set the
retroarch configuration to point to your local client (usually
@ -116,7 +132,6 @@ bool run_translation_service(void)
downside here is that your retroarch device will have to have
an internet connection, and you may have to sign up for it.
To make your own server, it must listen for a POST request, which
will consist of a json body, with the "image" field as a base64
encoded string of a 24bit-BMP that the will be translated. The server
@ -307,17 +322,20 @@ void handle_translation_cb(retro_task_t *task, void *task_data, void *user_data,
char* ret;
char* string = NULL;
int new_size = 0;
int new_image_size = 0;
int new_sound_size = 0;
unsigned width, height;
unsigned image_width, image_height;
size_t pitch;
const void* dummy_data;
void* raw_image_data;
void* raw_image_data = NULL;
void* raw_sound_data = NULL;
settings_t *settings = config_get_ptr();
runloop_get_status(&is_paused, &is_idle, &is_slowmotion,
&is_perfcnt_enable);
if (!is_paused)
if (!is_paused && settings->uints.ai_service_mode != 1)
goto finish;
if (!data || error)
@ -329,111 +347,179 @@ void handle_translation_cb(retro_task_t *task, void *task_data, void *user_data,
data->data[data->len] = '\0';
/* Parse JSON body for the image data */
body_copy = strdup(strchr(data->data, ch));
ret = body_copy;
if (!ret)
goto finish;
/* Parse JSON body for the image and sound data */
while (strncmp(ret, "\"image\":", strlen("\"image\":"))!=0)
body_copy = strdup(data->data);
int i = 0;
int start = -1;
char* found_string = NULL;
int curr_state = 0;
char curr;
while (true)
{
ret = strchr(ret, ch);
if (ret == NULL)
break;
else
ret = ret+sizeof(char);
curr = (char) *(body_copy+i);
if (curr == '\0')
break;
if (curr == '\"')
{
if (start == -1)
start = i;
else
{
found_string = malloc(i-start);
strncpy(found_string, body_copy+start+1, i-start-1);
*(found_string+i-start-1) = '\0';
if (curr_state == 1)/*image*/
{
raw_bmp_data = (void*) unbase64(found_string,
strlen(found_string),
&new_image_size);
curr_state = 0;
}
else if (curr_state == 2)
{
raw_sound_data = (void*) unbase64(found_string,
strlen(found_string),
&new_sound_size);
curr_state = 0;
}
else if (strcmp(found_string, "image")==0)
{
curr_state = 1;
free(found_string);
}
else if (strcmp(found_string, "sound")==0)
{
curr_state = 2;
free(found_string);
}
else
{
curr_state = 0;
}
start = -1;
}
}
i++;
}
if (ret != NULL)
{
ret = ret + sizeof(char);
ret = ret+strlen("\"image\":")*sizeof(char);
if (found_string)
free(found_string);
ret = strchr(ret, ch)+sizeof(char);
string = strtok(ret, s);
}
if (ret == NULL || string == NULL)
if (raw_bmp_data == NULL && raw_sound_data == NULL)
{
error = "Invalid JSON body.";
goto finish;
}
/* decode the image data from base64 */
raw_bmp_data = (void*) unbase64(string, strlen(string), &new_size);
if (!raw_bmp_data)
goto finish;
/* Get the video frame dimensions reference */
video_driver_cached_frame_get(&dummy_data, &width, &height, &pitch);
if (raw_bmp_data != NULL)
{
/* Get the video frame dimensions reference */
video_driver_cached_frame_get(&dummy_data, &width, &height, &pitch);
/* Get image data (24 bit), and conver to the emulated pixel format */
image_width = ((uint32_t) ((uint8_t) raw_bmp_data[21]) << 24) +
((uint32_t) ((uint8_t) raw_bmp_data[20]) << 16) +
((uint32_t) ((uint8_t) raw_bmp_data[19]) << 8) +
((uint32_t) ((uint8_t) raw_bmp_data[18]) << 0);
/* Get image data (24 bit), and conver to the emulated pixel format */
image_width = ((uint32_t) ((uint8_t) raw_bmp_data[21]) << 24) +
((uint32_t) ((uint8_t) raw_bmp_data[20]) << 16) +
((uint32_t) ((uint8_t) raw_bmp_data[19]) << 8) +
((uint32_t) ((uint8_t) raw_bmp_data[18]) << 0);
image_height = ((uint32_t) ((uint8_t) raw_bmp_data[25]) << 24) +
((uint32_t) ((uint8_t) raw_bmp_data[24]) << 16) +
((uint32_t) ((uint8_t) raw_bmp_data[23]) << 8) +
((uint32_t) ((uint8_t) raw_bmp_data[22]) << 0);
raw_image_data = raw_bmp_data+54*sizeof(uint8_t);
image_height = ((uint32_t) ((uint8_t) raw_bmp_data[25]) << 24) +
((uint32_t) ((uint8_t) raw_bmp_data[24]) << 16) +
((uint32_t) ((uint8_t) raw_bmp_data[23]) << 8) +
((uint32_t) ((uint8_t) raw_bmp_data[22]) << 0);
raw_image_data = raw_bmp_data+54*sizeof(uint8_t);
scaler = calloc(1, sizeof(struct scaler_ctx));
if (!scaler)
goto finish;
scaler = calloc(1, sizeof(struct scaler_ctx));
if (!scaler)
goto finish;
if (dummy_data == RETRO_HW_FRAME_BUFFER_VALID)
{
/*
In this case, we used the viewport to grab the image
and translate it, and we have the translated image in
the raw_image_data buffer.
if (dummy_data == RETRO_HW_FRAME_BUFFER_VALID)
{
/*
In this case, we used the viewport to grab the image
and translate it, and we have the translated image in
the raw_image_data buffer.
*/
/* TODO: write to the viewport in this case */
RARCH_LOG("Hardware frame buffer... writing to viewport not yet supported.\n");
goto finish;
}
/* The assigned pitch may not be reliable. The width of
the video frame can change during run-time, but the
pitch may not, so we just assign it as the width
times the byte depth.
*/
/* TODO: write to the viewport in this case */
RARCH_LOG("WRITING TO VIEWPORT...\n");
goto finish;
}
/* The assigned pitch may not be reliable. The width of
the video frame can change during run-time, but the
pitch may not, so we just assign it as the width
times the byte depth.
*/
if (video_driver_get_pixel_format() == RETRO_PIXEL_FORMAT_XRGB8888)
{
raw_output_data = (uint8_t*) malloc(width*height*4*sizeof(uint8_t));
scaler->out_fmt = SCALER_FMT_ARGB8888;
pitch = width*4;
scaler->out_stride = width*4;
}
else
{
raw_output_data = (uint8_t*) malloc(width*height*2*sizeof(uint8_t));
scaler->out_fmt = SCALER_FMT_RGB565;
pitch = width*2;
scaler->out_stride = width*1;
}
if (video_driver_get_pixel_format() == RETRO_PIXEL_FORMAT_XRGB8888)
{
raw_output_data = (uint8_t*) malloc(width*height*4*sizeof(uint8_t));
scaler->out_fmt = SCALER_FMT_ARGB8888;
pitch = width*4;
scaler->out_stride = width*4;
}
else
{
raw_output_data = (uint8_t*) malloc(width*height*2*sizeof(uint8_t));
scaler->out_fmt = SCALER_FMT_RGB565;
pitch = width*2;
scaler->out_stride = width*1;
}
if (!raw_output_data)
goto finish;
scaler->in_fmt = SCALER_FMT_BGR24;
scaler->in_width = image_width;
scaler->in_height = image_height;
scaler->out_width = width;
scaler->out_height = height;
scaler->scaler_type = SCALER_TYPE_POINT;
scaler_ctx_gen_filter(scaler);
scaler->in_stride = -1*width*3;
if (!raw_output_data)
goto finish;
scaler->in_fmt = SCALER_FMT_BGR24;
scaler->in_width = image_width;
scaler->in_height = image_height;
scaler->out_width = width;
scaler->out_height = height;
scaler->scaler_type = SCALER_TYPE_POINT;
scaler_ctx_gen_filter(scaler);
scaler->in_stride = -1*width*3;
scaler_ctx_scale_direct(scaler, raw_output_data, (uint8_t*)raw_image_data+(image_height-1)*width*3);
/*
video_driver_frame(raw_output_data+(height-1)*pitch, image_width, image_height, -pitch);
*/
video_driver_frame(raw_output_data, image_width, image_height, pitch);
scaler_ctx_scale_direct(scaler, raw_output_data,
(uint8_t*)raw_image_data+(image_height-1)*width*3);
video_driver_frame(raw_output_data, image_width, image_height, pitch);
}
if (raw_sound_data != NULL)
{
retro_task_t *t = task_init();
nbio_buf_t *task_data = (nbio_buf_t*)calloc(1, sizeof(nbio_buf_t));
task_data->buf = raw_sound_data;
task_data->bufsize = new_sound_size;
task_data->path=NULL;
audio_mixer_stream_params_t params;
nbio_buf_t *img = (nbio_buf_t*)task_data;
if (!img)
return;
params.volume = 1.0f;
params.slot_selection_type = AUDIO_MIXER_SLOT_SELECTION_AUTOMATIC;//user->slot_selection_type;
params.slot_selection_idx = 10;
params.stream_type = AUDIO_STREAM_TYPE_SYSTEM;//user->stream_type;
params.type = AUDIO_MIXER_TYPE_WAV;
params.state = AUDIO_STREAM_STATE_PLAYING;
params.buf = img->buf;
params.bufsize = img->bufsize;
params.cb = NULL;
params.basename = NULL;
audio_driver_mixer_add_stream(&params);
if (img->path)
free(img->path);
if (params.basename != NULL)
free(params.basename);
free(img);
RARCH_LOG("Loaded Sound Size: %i\n", new_sound_size);
}
RARCH_LOG("Translation done.\n");
finish:
if (error)
RARCH_ERR("%s: %s\n", msg_hash_to_str(MSG_DOWNLOAD_FAILED), error);
@ -465,7 +551,7 @@ void call_translation_server(const char* body)
{
settings_t *settings = config_get_ptr();
RARCH_LOG("Server url: %s\n", settings->arrays.translation_service_url);
task_push_http_post_transfer(settings->arrays.translation_service_url,
RARCH_LOG("Server url: %s\n", settings->arrays.ai_service_url);
task_push_http_post_transfer(settings->arrays.ai_service_url,
body, true, NULL, handle_translation_cb, NULL);
}

2
translation/translation_service.h
View File

@ -5,7 +5,9 @@
void call_translation_server(const char* body);
/*
bool g_translation_service_status;
*/
bool run_translation_service(void);