diff --git a/.github/workflows/crowdin_prep.yml b/.github/workflows/crowdin_prep.yml index e24c9b5..a63cca5 100644 --- a/.github/workflows/crowdin_prep.yml +++ b/.github/workflows/crowdin_prep.yml @@ -30,5 +30,4 @@ jobs: env: CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} run: | - python3 intl/crowdin_prep.py "libgambatte/libretro" - python3 intl/crowdin_source_upload.py "$CROWDIN_API_KEY" "gambatte-libretro" + python3 intl/upload_workflow.py $CROWDIN_API_KEY "gambatte-libretro" "libgambatte/libretro" diff --git a/.github/workflows/crowdin_translate.yml b/.github/workflows/crowdin_translate.yml index 61925b0..3140388 100644 --- a/.github/workflows/crowdin_translate.yml +++ b/.github/workflows/crowdin_translate.yml @@ -29,7 +29,7 @@ jobs: env: CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} run: | - python3 intl/translation_workflow.py "$CROWDIN_API_KEY" "gambatte-libretro" "libgambatte/libretro" + python3 intl/download_workflow.py $CROWDIN_API_KEY "gambatte-libretro" "libgambatte/libretro" - name: Commit files run: | diff --git a/intl/.gitignore b/intl/.gitignore index 811d745..458f3d3 100644 --- a/intl/.gitignore +++ b/intl/.gitignore @@ -1,3 +1,4 @@ __pycache__ crowdin-cli.jar -_*/* +*.h +*.json diff --git a/intl/core_option_translation.py b/intl/core_option_translation.py index 991f4ab..743ea13 100644 --- a/intl/core_option_translation.py +++ b/intl/core_option_translation.py @@ -9,7 +9,7 @@ Both v1 and v2 structs are supported. It is, however, recommended to convert v1 'v1_to_v2_converter.py'. Usage: -python3 path/to/core_option_translation.py "path/to/where/libretro_core_options.h & libretro_core_options_intl.h/are" +python3 path/to/core_option_translation.py "path/to/where/libretro_core_options.h & libretro_core_options_intl.h/are" "core_name" This script will: 1.) create key words for & extract the texts from libretro_core_options.h & save them into intl/_us/core_options.h @@ -270,14 +270,14 @@ def create_msg_hash(intl_dir_path: str, core_name: str, keyword_string_dict: dic """Creates '.h' files in 'intl/_/' containing the macro name & string combinations. :param intl_dir_path: Path to the intl directory. - :param core_name: Name of the core, used for naming the files. + :param core_name: Name of the core, used for the files' paths. :param keyword_string_dict: Dictionary of the form { '_': { 'macro': 'string', ... }, ... }. :return: Dictionary of the form { '_': 'path/to/file (./intl/_/.h)', ... }. """ files = {} for localisation in keyword_string_dict: - path = os.path.join(intl_dir_path, localisation) # intl/_ - files[localisation] = os.path.join(path, core_name + '.h') # intl/_/.h + path = os.path.join(intl_dir_path, core_name) # intl// + files[localisation] = os.path.join(path, localisation + '.h') # intl//_.h if not os.path.exists(path): os.makedirs(path) with open(files[localisation], 'w', encoding='utf-8') as crowdin_file: @@ -296,6 +296,9 @@ def h2json(file_paths: dict) -> dict: """ jsons = {} for file_lang in file_paths: + if not os.path.isfile(file_paths[file_lang]): + continue + jsons[file_lang] = file_paths[file_lang][:-2] + '.json' p = cor.p_masked @@ -319,25 +322,17 @@ def h2json(file_paths: dict) -> dict: return jsons -def json2h(intl_dir_path: str, json_file_path: str, core_name: str) -> None: +def json2h(intl_dir_path: str, file_list) -> None: """Converts .json file in json_file_path into an .h ready to be included in C code. - :param intl_dir_path: Path to the intl directory. - :param json_file_path: Base path of translation .json. - :param core_name: Name of the core, required for naming the files. + :param intl_dir_path: Path to the intl/ directory. + :param file_list: Iterator of os.DirEntry objects. Contains localisation files to convert. :return: None """ - h_filename = os.path.join(json_file_path, core_name + '.h') - json_filename = os.path.join(json_file_path, core_name + '.json') - file_lang = os.path.basename(json_file_path).upper() - - if os.path.basename(json_file_path).lower() == '_us': - print(' skipped') - return p = cor.p_masked - def update(s_messages, s_template, s_source_messages): + def update(s_messages, s_template, s_source_messages, file_name): translation = '' template_messages = p.finditer(s_template) for tp_msg in template_messages: @@ -345,23 +340,33 @@ def json2h(intl_dir_path: str, json_file_path: str, core_name: str) -> None: if old_key in s_messages and s_messages[old_key] != s_source_messages[old_key]: tl_msg_val = s_messages[old_key] tl_msg_val = tl_msg_val.replace('"', '\\\"').replace('\n', '') # escape - translation = ''.join((translation, '#define ', old_key, file_lang, f' "{tl_msg_val}"\n')) + translation = ''.join((translation, '#define ', old_key, file_name.upper(), f' "{tl_msg_val}"\n')) else: # Remove English duplicates and non-translatable strings - translation = ''.join((translation, '#define ', old_key, file_lang, ' NULL\n')) + translation = ''.join((translation, '#define ', old_key, file_name.upper(), ' NULL\n')) return translation - with open(os.path.join(intl_dir_path, '_us', core_name + '.h'), 'r', encoding='utf-8') as template_file: + us_h = os.path.join(intl_dir_path, '_us.h') + us_json = os.path.join(intl_dir_path, '_us.json') + + with open(us_h, 'r', encoding='utf-8') as template_file: template = template_file.read() - with open(os.path.join(intl_dir_path, '_us', core_name + '.json'), 'r+', encoding='utf-8') as source_json_file: + with open(us_json, 'r+', encoding='utf-8') as source_json_file: source_messages = json.load(source_json_file) - with open(json_filename, 'r+', encoding='utf-8') as json_file: - messages = json.load(json_file) - new_translation = update(messages, template, source_messages) - with open(h_filename, 'w', encoding='utf-8') as h_file: - h_file.seek(0) - h_file.write(new_translation) - h_file.truncate() + + for file in file_list: + if file.name.lower().startswith('_us')\ + or file.name.lower().endswith('.h')\ + or file.is_dir(): + continue + + with open(file.path, 'r+', encoding='utf-8') as json_file: + messages = json.load(json_file) + new_translation = update(messages, template, source_messages, os.path.splitext(file.name)[0]) + with open(os.path.splitext(file.path)[0] + '.h', 'w', encoding='utf-8') as h_file: + h_file.seek(0) + h_file.write(new_translation) + h_file.truncate() return @@ -392,14 +397,13 @@ def get_crowdin_client(dir_path: str) -> str: return jar_path -def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str, core_name: str, file_path: str) -> None: +def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str, file_path: str) -> None: """Creates 'libretro_core_options_intl.h' from Crowdin translations. :param localisation_file_path: Path to 'libretro_core_options_intl.h' - :param intl_dir_path: Path to the intl directory. + :param intl_dir_path: Path to the intl/ directory. :param text: Content of the 'libretro_core_options.h' being translated. - :param core_name: Name of the core. Needed to identify the files to pull the translations from. - :param file_path: Path to the '.h' file, containing the original English texts. + :param file_path: Path to the '_us.h' file, containing the original English texts. :return: None """ msg_dict = {} @@ -472,11 +476,15 @@ def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str, return res - with open(file_path, 'r+', encoding='utf-8') as template: # intl/_us/.h - masked_msgs = cor.p_masked.finditer(template.read()) - for msg in masked_msgs: - msg_dict[msg.group(2)] = msg.group(1) + # ------------------------------------------------------------------------------------ + with open(file_path, 'r+', encoding='utf-8') as template: # intl//_us.h + masked_msgs = cor.p_masked.finditer(template.read()) + + for msg in masked_msgs: + msg_dict[msg.group(2)] = msg.group(1) + + # top of the file - in case there is no file to copy it from out_txt = "#ifndef LIBRETRO_CORE_OPTIONS_INTL_H__\n" \ "#define LIBRETRO_CORE_OPTIONS_INTL_H__\n\n" \ "#if defined(_MSC_VER) && (_MSC_VER >= 1500 && _MSC_VER < 1900)\n" \ @@ -484,9 +492,13 @@ def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str, '#pragma execution_character_set("utf-8")\n' \ "#pragma warning(disable:4566)\n" \ "#endif\n\n" \ - "#include \n\n" + "#include \n\n" \ + '#ifdef __cplusplus\n' \ + 'extern "C" {\n' \ + '#endif\n' if os.path.isfile(localisation_file_path): + # copy top of the file for re-use with open(localisation_file_path, 'r', encoding='utf-8') as intl: # libretro_core_options_intl.h in_text = intl.read() intl_start = re.search(re.escape('/*\n' @@ -503,20 +515,26 @@ def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str, if intl_start: out_txt = in_text[:intl_start.end(0)] - for folder in os.scandir(intl_dir_path): # intl/_* - if folder.is_dir() and folder.name.startswith('_')\ - and folder.name != '_us' and folder.name != '__pycache__': - translation_path = os.path.join(folder.path, core_name + '.h') # _.h - if not os.path.isfile(translation_path): - continue + # only write to file, if there is anything worthwhile to write! + overwrite = False + + # iterate through localisation files + for file in os.scandir(intl_dir_path): # intl//_* + if file.is_file() \ + and file.name.startswith('_') \ + and file.name.endswith('.h') \ + and not file.name.startswith('_us'): + translation_path = file.path # _.h # all structs: group(0) full struct, group(1) beginning, group(2) content struct_groups = cor.p_struct.finditer(text) - lang_up = folder.name.upper() - lang_low = folder.name.lower() + lang_low = os.path.splitext(file.name)[0].lower() + lang_up = lang_low.upper() out_txt = out_txt + f'/* RETRO_LANGUAGE{lang_up} */\n\n' # /* RETRO_LANGUAGE_NM */ + # copy adjusted translations (makros) with open(translation_path, 'r+', encoding='utf-8') as f_in: # .h out_txt = out_txt + f_in.read() + '\n' + # replace English texts with makros for construct in struct_groups: declaration = construct.group(1) struct_type_name = get_struct_type_name(declaration) @@ -541,18 +559,22 @@ def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str, start = construct.end(2) - offset_construct out_txt = out_txt + new_content + construct.group(0)[start:] + '\n' + # for v2 if 'retro_core_option_v2_definition' == struct_type_name[0]: out_txt = out_txt + f'struct retro_core_options_v2 options{lang_low}' \ ' = {\n' \ f' option_cats{lang_low},\n' \ f' option_defs{lang_low}\n' \ '};\n\n' - # shutil.rmtree(JOINER.join((intl_dir_path, folder))) + # if it got this far, we've got something to write + overwrite = True - with open(localisation_file_path, 'w', encoding='utf-8') as intl: - intl.write(out_txt + '\n#ifdef __cplusplus\n' - '}\n#endif\n' - '\n#endif') + # only write to file, if there is anything worthwhile to write! + if overwrite: + with open(localisation_file_path, 'w', encoding='utf-8') as intl: + intl.write(out_txt + '\n#ifdef __cplusplus\n' + '}\n#endif\n' + '\n#endif') return @@ -571,7 +593,8 @@ if __name__ == '__main__': TARGET_DIR_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) print("No path provided, assuming parent directory:\n" + TARGET_DIR_PATH) - NAME = 'core_options' + CORE_NAME = clean_file_name(sys.argv[2]) + DIR_PATH = os.path.dirname(os.path.realpath(__file__)) H_FILE_PATH = os.path.join(TARGET_DIR_PATH, 'libretro_core_options.h') INTL_FILE_PATH = os.path.join(TARGET_DIR_PATH, 'libretro_core_options_intl.h') @@ -580,7 +603,7 @@ if __name__ == '__main__': with open(H_FILE_PATH, 'r+', encoding='utf-8') as _h_file: _main_text = _h_file.read() _hash_n_str = get_texts(_main_text) - _files = create_msg_hash(DIR_PATH, NAME, _hash_n_str) + _files = create_msg_hash(DIR_PATH, CORE_NAME, _hash_n_str) _source_jsons = h2json(_files) print('Getting texts from libretro_core_options_intl.h') @@ -588,7 +611,7 @@ if __name__ == '__main__': with open(INTL_FILE_PATH, 'r+', encoding='utf-8') as _intl_file: _intl_text = _intl_file.read() _hash_n_str_intl = get_texts(_intl_text) - _intl_files = create_msg_hash(DIR_PATH, NAME, _hash_n_str_intl) + _intl_files = create_msg_hash(DIR_PATH, CORE_NAME, _hash_n_str_intl) _intl_jsons = h2json(_intl_files) print('\nAll done!') diff --git a/intl/crowdin.yaml b/intl/crowdin.yaml index 4931f51..0a54633 100644 --- a/intl/crowdin.yaml +++ b/intl/crowdin.yaml @@ -6,8 +6,8 @@ "files": [ { - "source": "/intl/_us/*.json", - "dest": "/_core_name_/%original_file_name%", - "translation": "/intl/_%two_letters_code%/%original_file_name%", + "source": "/intl/_core_name_/_us.json", + "dest": "/_core_name_/core_options.json", + "translation": "/intl/_core_name_/_%two_letters_code%.json", }, ] diff --git a/intl/crowdin_prep.py b/intl/crowdin_prep.py index 9ef6189..8195af6 100644 --- a/intl/crowdin_prep.py +++ b/intl/crowdin_prep.py @@ -4,8 +4,6 @@ import core_option_translation as t if __name__ == '__main__': - NAME = 'core_options' - try: if t.os.path.isfile(t.sys.argv[1]): _temp = t.os.path.dirname(t.sys.argv[1]) @@ -18,6 +16,7 @@ if __name__ == '__main__': TARGET_DIR_PATH = t.os.path.dirname(t.os.path.dirname(t.os.path.realpath(__file__))) print("No path provided, assuming parent directory:\n" + TARGET_DIR_PATH) + CORE_NAME = t.clean_file_name(t.sys.argv[2]) DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__)) H_FILE_PATH = t.os.path.join(TARGET_DIR_PATH, 'libretro_core_options.h') @@ -25,7 +24,7 @@ if __name__ == '__main__': with open(H_FILE_PATH, 'r+', encoding='utf-8') as _h_file: _main_text = _h_file.read() _hash_n_str = t.get_texts(_main_text) - _files = t.create_msg_hash(DIR_PATH, NAME, _hash_n_str) + _files = t.create_msg_hash(DIR_PATH, CORE_NAME, _hash_n_str) _source_jsons = t.h2json(_files) diff --git a/intl/crowdin_source_upload.py b/intl/crowdin_source_upload.py index b84acd3..993fb55 100644 --- a/intl/crowdin_source_upload.py +++ b/intl/crowdin_source_upload.py @@ -9,76 +9,85 @@ import urllib.request import zipfile import core_option_translation as t -# Check Crowdin API Token and core name -try: - api_key = sys.argv[1] - core_name = t.clean_file_name(sys.argv[2]) -except IndexError as e: - print('Please provide Crowdin API Token and core name!') - raise e +# -------------------- MAIN -------------------- # -DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__)) -YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml') +if __name__ == '__main__': + # Check Crowdin API Token and core name + try: + API_KEY = sys.argv[1] + CORE_NAME = t.clean_file_name(sys.argv[2]) + except IndexError as e: + print('Please provide Crowdin API Token and core name!') + raise e -# Apply Crowdin API Key -with open(YAML_PATH, 'r') as crowdin_config_file: - crowdin_config = crowdin_config_file.read() -crowdin_config = re.sub(r'"api_token": "_secret_"', - f'"api_token": "{api_key}"', - crowdin_config, 1) -crowdin_config = re.sub(r'"dest": "/_core_name_/%original_file_name%",', - f'"dest": "/{core_name}/%original_file_name%",' - , crowdin_config, 1) -with open(YAML_PATH, 'w') as crowdin_config_file: - crowdin_config_file.write(crowdin_config) + DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__)) + YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml') -try: - # Download Crowdin CLI - jar_name = 'crowdin-cli.jar' - jar_path = t.os.path.join(DIR_PATH, jar_name) - crowdin_cli_file = 'crowdin-cli.zip' - crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file - crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file) - - if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)): - print('download crowdin-cli.jar') - urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path) - with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref: - jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0]) - for file in zip_ref.namelist(): - if file.endswith(jar_name): - jar_file = file - break - zip_ref.extract(jar_file, path=DIR_PATH) - os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path) - os.remove(crowdin_cli_path) - shutil.rmtree(jar_dir) - - print('upload source *.json') - subprocess.run(['java', '-jar', jar_path, 'upload', 'sources', '--config', YAML_PATH]) - - # Reset Crowdin API Key + # Apply Crowdin API Key with open(YAML_PATH, 'r') as crowdin_config_file: crowdin_config = crowdin_config_file.read() - crowdin_config = re.sub(r'"api_token": ".*?"', - '"api_token": "_secret_"', + crowdin_config = re.sub(r'"api_token": "_secret_"', + f'"api_token": "{API_KEY}"', crowdin_config, 1) - crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",', - '"dest": "/_core_name_/%original_file_name%",' - , crowdin_config, 1) + crowdin_config = re.sub(r'/_core_name_/', + f'/{CORE_NAME}/' + , crowdin_config) with open(YAML_PATH, 'w') as crowdin_config_file: crowdin_config_file.write(crowdin_config) -except Exception as e: - # Try really hard to reset Crowdin API Key - with open(YAML_PATH, 'r') as crowdin_config_file: - crowdin_config = crowdin_config_file.read() - crowdin_config = re.sub(r'"api_token": ".*?"', - '"api_token": "_secret_"', - crowdin_config, 1) - crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",', - '"dest": "/_core_name_/%original_file_name%",' - , crowdin_config, 1) - with open(YAML_PATH, 'w') as crowdin_config_file: - crowdin_config_file.write(crowdin_config) - raise e + try: + # Download Crowdin CLI + jar_name = 'crowdin-cli.jar' + jar_path = t.os.path.join(DIR_PATH, jar_name) + crowdin_cli_file = 'crowdin-cli.zip' + crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file + crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file) + + if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)): + print('download crowdin-cli.jar') + urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path) + with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref: + jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0]) + for file in zip_ref.namelist(): + if file.endswith(jar_name): + jar_file = file + break + zip_ref.extract(jar_file, path=DIR_PATH) + os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path) + os.remove(crowdin_cli_path) + shutil.rmtree(jar_dir) + + print('upload source *.json') + subprocess.run(['java', '-jar', jar_path, 'upload', 'sources', '--config', YAML_PATH]) + + # Reset Crowdin API Key + with open(YAML_PATH, 'r') as crowdin_config_file: + crowdin_config = crowdin_config_file.read() + crowdin_config = re.sub(r'"api_token": ".*?"', + '"api_token": "_secret_"', + crowdin_config, 1) + + # TODO this is NOT safe! + crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'), + '/_core_name_/', + crowdin_config) + + with open(YAML_PATH, 'w') as crowdin_config_file: + crowdin_config_file.write(crowdin_config) + + except Exception as e: + # Try really hard to reset Crowdin API Key + with open(YAML_PATH, 'r') as crowdin_config_file: + crowdin_config = crowdin_config_file.read() + crowdin_config = re.sub(r'"api_token": ".*?"', + '"api_token": "_secret_"', + crowdin_config, 1) + + # TODO this is NOT safe! + crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'), + '/_core_name_/', + crowdin_config) + + with open(YAML_PATH, 'w') as crowdin_config_file: + crowdin_config_file.write(crowdin_config) + raise e diff --git a/intl/crowdin_translate.py b/intl/crowdin_translate.py index f6be335..fc7055d 100644 --- a/intl/crowdin_translate.py +++ b/intl/crowdin_translate.py @@ -16,9 +16,10 @@ if __name__ == '__main__': TARGET_DIR_PATH = t.os.path.dirname(t.os.path.dirname(t.os.path.realpath(__file__))) print("No path provided, assuming parent directory:\n" + TARGET_DIR_PATH) - NAME = 'core_options' + CORE_NAME = t.clean_file_name(t.sys.argv[2]) DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__)) - US_FILE_PATH = t.os.path.join(DIR_PATH, '_us', f'{NAME}.h') + LOCALISATIONS_PATH = t.os.path.join(DIR_PATH, CORE_NAME) + US_FILE_PATH = t.os.path.join(LOCALISATIONS_PATH, '_us.h') H_FILE_PATH = t.os.path.join(TARGET_DIR_PATH, 'libretro_core_options.h') INTL_FILE_PATH = t.os.path.join(TARGET_DIR_PATH, 'libretro_core_options_intl.h') @@ -26,15 +27,14 @@ if __name__ == '__main__': with open(H_FILE_PATH, 'r+', encoding='utf-8') as _h_file: _main_text = _h_file.read() _hash_n_str = t.get_texts(_main_text) - _files = t.create_msg_hash(DIR_PATH, NAME, _hash_n_str) + _files = t.create_msg_hash(DIR_PATH, CORE_NAME, _hash_n_str) + _source_jsons = t.h2json(_files) print('Converting translations *.json to *.h:') - for _folder in t.os.scandir(DIR_PATH): - if _folder.is_dir() and _folder.name.startswith('_') and _folder.name != '__pycache__': - print(_folder.name) - t.json2h(DIR_PATH, _folder.path, NAME) + localisation_files = t.os.scandir(LOCALISATIONS_PATH) + t.json2h(LOCALISATIONS_PATH, localisation_files) print('Constructing libretro_core_options_intl.h') - t.create_intl_file(INTL_FILE_PATH, DIR_PATH, _main_text, NAME, _files["_us"]) + t.create_intl_file(INTL_FILE_PATH, LOCALISATIONS_PATH, _main_text, _files["_us"]) print('\nAll done!') diff --git a/intl/crowdin_translation_download.py b/intl/crowdin_translation_download.py index 273a61f..cd0c799 100644 --- a/intl/crowdin_translation_download.py +++ b/intl/crowdin_translation_download.py @@ -9,76 +9,85 @@ import urllib.request import zipfile import core_option_translation as t -# Check Crowdin API Token and core name -try: - api_key = sys.argv[1] - core_name = t.clean_file_name(sys.argv[2]) -except IndexError as e: - print('Please provide Crowdin API Token and core name!') - raise e +# -------------------- MAIN -------------------- # -DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__)) -YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml') +if __name__ == '__main__': + # Check Crowdin API Token and core name + try: + API_KEY = sys.argv[1] + CORE_NAME = t.clean_file_name(sys.argv[2]) + except IndexError as e: + print('Please provide Crowdin API Token and core name!') + raise e -# Apply Crowdin API Key -with open(YAML_PATH, 'r') as crowdin_config_file: - crowdin_config = crowdin_config_file.read() -crowdin_config = re.sub(r'"api_token": "_secret_"', - f'"api_token": "{api_key}"', - crowdin_config, 1) -crowdin_config = re.sub(r'"dest": "/_core_name_/%original_file_name%",', - f'"dest": "/{core_name}/%original_file_name%",' - , crowdin_config, 1) -with open(YAML_PATH, 'w') as crowdin_config_file: - crowdin_config_file.write(crowdin_config) + DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__)) + YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml') -try: - # Download Crowdin CLI - jar_name = 'crowdin-cli.jar' - jar_path = t.os.path.join(DIR_PATH, jar_name) - crowdin_cli_file = 'crowdin-cli.zip' - crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file - crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file) - - if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)): - print('download crowdin-cli.jar') - urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path) - with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref: - jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0]) - for file in zip_ref.namelist(): - if file.endswith(jar_name): - jar_file = file - break - zip_ref.extract(jar_file, path=DIR_PATH) - os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path) - os.remove(crowdin_cli_path) - shutil.rmtree(jar_dir) - - print('download translation *.json') - subprocess.run(['java', '-jar', jar_path, 'download', '--config', YAML_PATH]) - - # Reset Crowdin API Key + # Apply Crowdin API Key with open(YAML_PATH, 'r') as crowdin_config_file: crowdin_config = crowdin_config_file.read() - crowdin_config = re.sub(r'"api_token": ".*?"', - '"api_token": "_secret_"', + crowdin_config = re.sub(r'"api_token": "_secret_"', + f'"api_token": "{API_KEY}"', crowdin_config, 1) - crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",', - '"dest": "/_core_name_/%original_file_name%",' - , crowdin_config, 1) + crowdin_config = re.sub(r'/_core_name_/', + f'/{CORE_NAME}/' + , crowdin_config) with open(YAML_PATH, 'w') as crowdin_config_file: crowdin_config_file.write(crowdin_config) -except Exception as e: - # Try really hard to reset Crowdin API Key - with open(YAML_PATH, 'r') as crowdin_config_file: - crowdin_config = crowdin_config_file.read() - crowdin_config = re.sub(r'"api_token": ".*?"', - '"api_token": "_secret_"', - crowdin_config, 1) - crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",', - '"dest": "/_core_name_/%original_file_name%",' - , crowdin_config, 1) - with open(YAML_PATH, 'w') as crowdin_config_file: - crowdin_config_file.write(crowdin_config) - raise e + try: + # Download Crowdin CLI + jar_name = 'crowdin-cli.jar' + jar_path = t.os.path.join(DIR_PATH, jar_name) + crowdin_cli_file = 'crowdin-cli.zip' + crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file + crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file) + + if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)): + print('download crowdin-cli.jar') + urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path) + with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref: + jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0]) + for file in zip_ref.namelist(): + if file.endswith(jar_name): + jar_file = file + break + zip_ref.extract(jar_file, path=DIR_PATH) + os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path) + os.remove(crowdin_cli_path) + shutil.rmtree(jar_dir) + + print('download translation *.json') + subprocess.run(['java', '-jar', jar_path, 'download', '--config', YAML_PATH]) + + # Reset Crowdin API Key + with open(YAML_PATH, 'r') as crowdin_config_file: + crowdin_config = crowdin_config_file.read() + crowdin_config = re.sub(r'"api_token": ".*?"', + '"api_token": "_secret_"', + crowdin_config, 1) + + # TODO this is NOT safe! + crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'), + '/_core_name_/', + crowdin_config) + + with open(YAML_PATH, 'w') as crowdin_config_file: + crowdin_config_file.write(crowdin_config) + + except Exception as e: + # Try really hard to reset Crowdin API Key + with open(YAML_PATH, 'r') as crowdin_config_file: + crowdin_config = crowdin_config_file.read() + crowdin_config = re.sub(r'"api_token": ".*?"', + '"api_token": "_secret_"', + crowdin_config, 1) + + # TODO this is NOT safe! + crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'), + '/_core_name_/', + crowdin_config) + + with open(YAML_PATH, 'w') as crowdin_config_file: + crowdin_config_file.write(crowdin_config) + raise e diff --git a/intl/translation_workflow.py b/intl/download_workflow.py similarity index 96% rename from intl/translation_workflow.py rename to intl/download_workflow.py index 1aad269..478513d 100644 --- a/intl/translation_workflow.py +++ b/intl/download_workflow.py @@ -12,4 +12,4 @@ except IndexError as e: raise e subprocess.run(['python3', 'intl/crowdin_translation_download.py', api_key, core_name]) -subprocess.run(['python3', 'intl/crowdin_translate.py', dir_path]) +subprocess.run(['python3', 'intl/crowdin_translate.py', dir_path, core_name]) diff --git a/intl/initial_sync.py b/intl/initial_sync.py index 8930154..08620c9 100644 --- a/intl/initial_sync.py +++ b/intl/initial_sync.py @@ -10,94 +10,103 @@ import urllib.request import zipfile import core_option_translation as t -# Check Crowdin API Token and core name -try: - api_key = sys.argv[1] - core_name = t.clean_file_name(sys.argv[2]) -except IndexError as e: - print('Please provide Crowdin API Token and core name!') - raise e +# -------------------- MAIN -------------------- # -DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__)) -YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml') +if __name__ == '__main__': + # Check Crowdin API Token and core name + try: + API_KEY = sys.argv[1] + CORE_NAME = t.clean_file_name(sys.argv[2]) + except IndexError as e: + print('Please provide Crowdin API Token and core name!') + raise e -# Apply Crowdin API Key -with open(YAML_PATH, 'r') as crowdin_config_file: - crowdin_config = crowdin_config_file.read() -crowdin_config = re.sub(r'"api_token": "_secret_"', - f'"api_token": "{api_key}"', - crowdin_config, 1) -crowdin_config = re.sub(r'"dest": "/_core_name_/%original_file_name%",', - f'"dest": "/{core_name}/%original_file_name%",' - , crowdin_config, 1) -with open(YAML_PATH, 'w') as crowdin_config_file: - crowdin_config_file.write(crowdin_config) + DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__)) + YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml') -try: - # Download Crowdin CLI - jar_name = 'crowdin-cli.jar' - jar_path = t.os.path.join(DIR_PATH, jar_name) - crowdin_cli_file = 'crowdin-cli.zip' - crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file - crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file) - - if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)): - print('download crowdin-cli.jar') - urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path) - with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref: - jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0]) - for file in zip_ref.namelist(): - if file.endswith(jar_name): - jar_file = file - break - zip_ref.extract(jar_file, path=DIR_PATH) - os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path) - os.remove(crowdin_cli_path) - shutil.rmtree(jar_dir) - - print('upload source & translations *.json') - subprocess.run(['java', '-jar', jar_path, 'upload', 'sources', '--config', YAML_PATH]) - subprocess.run(['java', '-jar', jar_path, 'upload', 'translations', '--config', YAML_PATH]) - - print('wait for crowdin server to process data') - time.sleep(10) - - print('download translation *.json') - subprocess.run(['java', '-jar', jar_path, 'download', '--config', YAML_PATH]) - - # Reset Crowdin API Key + # Apply Crowdin API Key with open(YAML_PATH, 'r') as crowdin_config_file: crowdin_config = crowdin_config_file.read() - crowdin_config = re.sub(r'"api_token": ".*?"', '"api_token": "_secret_"', crowdin_config, 1) - crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",', - '"dest": "/_core_name_/%original_file_name%",' - , crowdin_config, 1) - with open(YAML_PATH, 'w') as crowdin_config_file: - crowdin_config_file.write(crowdin_config) - - with open('intl/translation_workflow.py', 'r') as workflow: - workflow_config = workflow.read() - workflow_config = workflow_config.replace( - "subprocess.run(['python3', 'intl/core_option_translation.py', dir_path])", - "subprocess.run(['python3', 'intl/crowdin_translation_download.py', api_key, core_name])" - ) - workflow_config = workflow_config.replace( - "subprocess.run(['python3', 'intl/initial_sync.py', api_key, core_name])\n", - "" - ) - with open('intl/translation_workflow.py', 'w') as workflow: - workflow.write(workflow_config) - -except Exception as e: - # Try really hard to reset Crowdin API Key - with open(YAML_PATH, 'r') as crowdin_config_file: - crowdin_config = crowdin_config_file.read() - crowdin_config = re.sub(r'"api_token": ".*?"', - '"api_token": "_secret_"', + crowdin_config = re.sub(r'"api_token": "_secret_"', + f'"api_token": "{API_KEY}"', crowdin_config, 1) - crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",', - '"dest": "/_core_name_/%original_file_name%",' - , crowdin_config, 1) + crowdin_config = re.sub(r'/_core_name_/', + f'/{CORE_NAME}/' + , crowdin_config) with open(YAML_PATH, 'w') as crowdin_config_file: crowdin_config_file.write(crowdin_config) - raise e + + try: + # Download Crowdin CLI + jar_name = 'crowdin-cli.jar' + jar_path = t.os.path.join(DIR_PATH, jar_name) + crowdin_cli_file = 'crowdin-cli.zip' + crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file + crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file) + + if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)): + print('download crowdin-cli.jar') + urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path) + with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref: + jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0]) + for file in zip_ref.namelist(): + if file.endswith(jar_name): + jar_file = file + break + zip_ref.extract(jar_file, path=DIR_PATH) + os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path) + os.remove(crowdin_cli_path) + shutil.rmtree(jar_dir) + + print('upload source & translations *.json') + subprocess.run(['java', '-jar', jar_path, 'upload', 'sources', '--config', YAML_PATH]) + subprocess.run(['java', '-jar', jar_path, 'upload', 'translations', '--config', YAML_PATH]) + + print('wait for crowdin server to process data') + time.sleep(10) + + print('download translation *.json') + subprocess.run(['java', '-jar', jar_path, 'download', '--config', YAML_PATH]) + + # Reset Crowdin API Key + with open(YAML_PATH, 'r') as crowdin_config_file: + crowdin_config = crowdin_config_file.read() + crowdin_config = re.sub(r'"api_token": ".*?"', '"api_token": "_secret_"', crowdin_config, 1) + + # TODO this is NOT safe! + crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'), + '/_core_name_/', + crowdin_config) + + with open(YAML_PATH, 'w') as crowdin_config_file: + crowdin_config_file.write(crowdin_config) + + with open('intl/translation_workflow.py', 'r') as workflow: + workflow_config = workflow.read() + workflow_config = workflow_config.replace( + "subprocess.run(['python3', 'intl/core_option_translation.py', dir_path, core_name])", + "subprocess.run(['python3', 'intl/crowdin_translation_download.py', api_key, core_name])" + ) + workflow_config = workflow_config.replace( + "subprocess.run(['python3', 'intl/initial_sync.py', api_key, core_name])\n", + "" + ) + with open('intl/translation_workflow.py', 'w') as workflow: + workflow.write(workflow_config) + + except Exception as e: + # Try really hard to reset Crowdin API Key + with open(YAML_PATH, 'r') as crowdin_config_file: + crowdin_config = crowdin_config_file.read() + crowdin_config = re.sub(r'"api_token": ".*?"', + '"api_token": "_secret_"', + crowdin_config, 1) + + # TODO this is NOT safe! + crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'), + '/_core_name_/', + crowdin_config) + + with open(YAML_PATH, 'w') as crowdin_config_file: + crowdin_config_file.write(crowdin_config) + raise e diff --git a/intl/upload_workflow.py b/intl/upload_workflow.py new file mode 100644 index 0000000..4439710 --- /dev/null +++ b/intl/upload_workflow.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +import sys +import subprocess + +try: + api_key = sys.argv[1] + core_name = sys.argv[2] + dir_path = sys.argv[3] +except IndexError as e: + print('Please provide path to libretro_core_options.h, Crowdin API Token and core name!') + raise e + +subprocess.run(['python3', 'intl/crowdin_prep.py', dir_path, core_name]) +subprocess.run(['python3', 'intl/crowdin_source_upload.py', api_key, core_name])