Update auto sync scripts (#211)

.github/workflows/crowdin_prep.yml

@@ -30,5 +30,4 @@ jobs:
         env:
           CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }}
         run: |
-          python3 intl/crowdin_prep.py "libgambatte/libretro"
+          python3 intl/upload_workflow.py $CROWDIN_API_KEY "gambatte-libretro" "libgambatte/libretro"
-          python3 intl/crowdin_source_upload.py "$CROWDIN_API_KEY" "gambatte-libretro"

.github/workflows/crowdin_translate.yml

@@ -29,7 +29,7 @@ jobs:
         env:
           CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }}
         run: |
-          python3 intl/translation_workflow.py "$CROWDIN_API_KEY" "gambatte-libretro" "libgambatte/libretro"
+          python3 intl/download_workflow.py $CROWDIN_API_KEY "gambatte-libretro" "libgambatte/libretro"
 
       - name: Commit files
         run: |

intl/.gitignore

@@ -1,3 +1,4 @@
 __pycache__
 crowdin-cli.jar
-_*/*
+*.h
+*.json

intl/core_option_translation.py

@@ -9,7 +9,7 @@ Both v1 and v2 structs are supported. It is, however, recommended to convert v1
 'v1_to_v2_converter.py'.
 
 Usage:
-python3 path/to/core_option_translation.py "path/to/where/libretro_core_options.h & libretro_core_options_intl.h/are"
+python3 path/to/core_option_translation.py "path/to/where/libretro_core_options.h & libretro_core_options_intl.h/are" "core_name"
 
 This script will:
 1.) create key words for & extract the texts from libretro_core_options.h & save them into intl/_us/core_options.h
@@ -270,14 +270,14 @@ def create_msg_hash(intl_dir_path: str, core_name: str, keyword_string_dict: dic
     """Creates '<core_name>.h' files in 'intl/_<lang>/' containing the macro name & string combinations.
 
     :param intl_dir_path: Path to the intl directory.
-    :param core_name: Name of the core, used for naming the files.
+    :param core_name: Name of the core, used for the files' paths.
     :param keyword_string_dict: Dictionary of the form { '_<lang>': { 'macro': 'string', ... }, ... }.
     :return: Dictionary of the form { '_<lang>': 'path/to/file (./intl/_<lang>/<core_name>.h)', ... }.
     """
     files = {}
     for localisation in keyword_string_dict:
-        path = os.path.join(intl_dir_path, localisation)  # intl/_<lang>
+        path = os.path.join(intl_dir_path, core_name)  # intl/<core_name>/
-        files[localisation] = os.path.join(path, core_name + '.h')  # intl/_<lang>/<core_name>.h
+        files[localisation] = os.path.join(path, localisation + '.h')  # intl/<core_name>/_<lang>.h
         if not os.path.exists(path):
             os.makedirs(path)
         with open(files[localisation], 'w', encoding='utf-8') as crowdin_file:
@@ -296,6 +296,9 @@ def h2json(file_paths: dict) -> dict:
     """
     jsons = {}
     for file_lang in file_paths:
+        if not os.path.isfile(file_paths[file_lang]):
+            continue
+
         jsons[file_lang] = file_paths[file_lang][:-2] + '.json'
 
         p = cor.p_masked
@@ -319,25 +322,17 @@ def h2json(file_paths: dict) -> dict:
     return jsons
 
 
-def json2h(intl_dir_path: str, json_file_path: str, core_name: str) -> None:
+def json2h(intl_dir_path: str, file_list) -> None:
     """Converts .json file in json_file_path into an .h ready to be included in C code.
 
-    :param intl_dir_path: Path to the intl directory.
+    :param intl_dir_path: Path to the intl/<core_name> directory.
-    :param json_file_path: Base path of translation .json.
+    :param file_list: Iterator of os.DirEntry objects. Contains localisation files to convert.
-    :param core_name: Name of the core, required for naming the files.
     :return: None
     """
-    h_filename = os.path.join(json_file_path, core_name + '.h')
-    json_filename = os.path.join(json_file_path, core_name + '.json')
-    file_lang = os.path.basename(json_file_path).upper()
-
-    if os.path.basename(json_file_path).lower() == '_us':
-        print('    skipped')
-        return
 
     p = cor.p_masked
 
-    def update(s_messages, s_template, s_source_messages):
+    def update(s_messages, s_template, s_source_messages, file_name):
         translation = ''
         template_messages = p.finditer(s_template)
         for tp_msg in template_messages:
@@ -345,23 +340,33 @@ def json2h(intl_dir_path: str, json_file_path: str, core_name: str) -> None:
             if old_key in s_messages and s_messages[old_key] != s_source_messages[old_key]:
                 tl_msg_val = s_messages[old_key]
                 tl_msg_val = tl_msg_val.replace('"', '\\\"').replace('\n', '')  # escape
-                translation = ''.join((translation, '#define ', old_key, file_lang, f' "{tl_msg_val}"\n'))
+                translation = ''.join((translation, '#define ', old_key, file_name.upper(), f' "{tl_msg_val}"\n'))
 
             else:  # Remove English duplicates and non-translatable strings
-                translation = ''.join((translation, '#define ', old_key, file_lang, ' NULL\n'))
+                translation = ''.join((translation, '#define ', old_key, file_name.upper(), ' NULL\n'))
         return translation
 
-    with open(os.path.join(intl_dir_path, '_us', core_name + '.h'), 'r', encoding='utf-8') as template_file:
+    us_h = os.path.join(intl_dir_path, '_us.h')
+    us_json = os.path.join(intl_dir_path, '_us.json')
+
+    with open(us_h, 'r', encoding='utf-8') as template_file:
         template = template_file.read()
-    with open(os.path.join(intl_dir_path, '_us', core_name + '.json'), 'r+', encoding='utf-8') as source_json_file:
+    with open(us_json, 'r+', encoding='utf-8') as source_json_file:
         source_messages = json.load(source_json_file)
-    with open(json_filename, 'r+', encoding='utf-8') as json_file:
+
-        messages = json.load(json_file)
+    for file in file_list:
-        new_translation = update(messages, template, source_messages)
+        if file.name.lower().startswith('_us')\
-    with open(h_filename, 'w', encoding='utf-8') as h_file:
+                or file.name.lower().endswith('.h')\
-        h_file.seek(0)
+                or file.is_dir():
-        h_file.write(new_translation)
+            continue
-        h_file.truncate()
+
+        with open(file.path, 'r+', encoding='utf-8') as json_file:
+            messages = json.load(json_file)
+            new_translation = update(messages, template, source_messages, os.path.splitext(file.name)[0])
+        with open(os.path.splitext(file.path)[0] + '.h', 'w', encoding='utf-8') as h_file:
+            h_file.seek(0)
+            h_file.write(new_translation)
+            h_file.truncate()
     return
 
 
@@ -392,14 +397,13 @@ def get_crowdin_client(dir_path: str) -> str:
     return jar_path
 
 
-def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str, core_name: str, file_path: str) -> None:
+def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str, file_path: str) -> None:
     """Creates 'libretro_core_options_intl.h' from Crowdin translations.
 
     :param localisation_file_path: Path to 'libretro_core_options_intl.h'
-    :param intl_dir_path: Path to the intl directory.
+    :param intl_dir_path: Path to the intl/<core_name> directory.
     :param text: Content of the 'libretro_core_options.h' being translated.
-    :param core_name: Name of the core. Needed to identify the files to pull the translations from.
+    :param file_path: Path to the '_us.h' file, containing the original English texts.
-    :param file_path: Path to the '<core name>.h' file, containing the original English texts.
     :return: None
     """
     msg_dict = {}
@@ -472,11 +476,15 @@ def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str,
 
         return res
 
-    with open(file_path, 'r+', encoding='utf-8') as template:  # intl/_us/<core_name>.h
+    # ------------------------------------------------------------------------------------
-        masked_msgs = cor.p_masked.finditer(template.read())
-        for msg in masked_msgs:
-            msg_dict[msg.group(2)] = msg.group(1)
 
+    with open(file_path, 'r+', encoding='utf-8') as template:  # intl/<core_name>/_us.h
+        masked_msgs = cor.p_masked.finditer(template.read())
+
+    for msg in masked_msgs:
+        msg_dict[msg.group(2)] = msg.group(1)
+
+    # top of the file - in case there is no file to copy it from
     out_txt = "#ifndef LIBRETRO_CORE_OPTIONS_INTL_H__\n" \
               "#define LIBRETRO_CORE_OPTIONS_INTL_H__\n\n" \
               "#if defined(_MSC_VER) && (_MSC_VER >= 1500 && _MSC_VER < 1900)\n" \
@@ -484,9 +492,13 @@ def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str,
               '#pragma execution_character_set("utf-8")\n' \
               "#pragma warning(disable:4566)\n" \
               "#endif\n\n" \
-              "#include <libretro.h>\n\n"
+              "#include <libretro.h>\n\n" \
+              '#ifdef __cplusplus\n' \
+              'extern "C" {\n' \
+              '#endif\n'
 
     if os.path.isfile(localisation_file_path):
+        # copy top of the file for re-use
         with open(localisation_file_path, 'r', encoding='utf-8') as intl:  # libretro_core_options_intl.h
             in_text = intl.read()
             intl_start = re.search(re.escape('/*\n'
@@ -503,20 +515,26 @@ def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str,
                 if intl_start:
                     out_txt = in_text[:intl_start.end(0)]
 
-    for folder in os.scandir(intl_dir_path):  # intl/_*
+    # only write to file, if there is anything worthwhile to write!
-        if folder.is_dir() and folder.name.startswith('_')\
+    overwrite = False
-                and folder.name != '_us' and folder.name != '__pycache__':
+
-            translation_path = os.path.join(folder.path, core_name + '.h')  # <core_name>_<lang>.h
+    # iterate through localisation files
-            if not os.path.isfile(translation_path):
+    for file in os.scandir(intl_dir_path):  # intl/<core_name>/_*
-                continue
+        if file.is_file() \
+                and file.name.startswith('_') \
+                and file.name.endswith('.h') \
+                and not file.name.startswith('_us'):
+            translation_path = file.path  # <core_name>_<lang>.h
             # all structs: group(0) full struct, group(1) beginning, group(2) content
             struct_groups = cor.p_struct.finditer(text)
-            lang_up = folder.name.upper()
+            lang_low = os.path.splitext(file.name)[0].lower()
-            lang_low = folder.name.lower()
+            lang_up = lang_low.upper()
             out_txt = out_txt + f'/* RETRO_LANGUAGE{lang_up} */\n\n'  # /* RETRO_LANGUAGE_NM */
 
+            # copy adjusted translations (makros)
             with open(translation_path, 'r+', encoding='utf-8') as f_in:  # <core name>.h
                 out_txt = out_txt + f_in.read() + '\n'
+            # replace English texts with makros
             for construct in struct_groups:
                 declaration = construct.group(1)
                 struct_type_name = get_struct_type_name(declaration)
@@ -541,18 +559,22 @@ def create_intl_file(localisation_file_path: str, intl_dir_path: str, text: str,
                 start = construct.end(2) - offset_construct
                 out_txt = out_txt + new_content + construct.group(0)[start:] + '\n'
 
+                # for v2
                 if 'retro_core_option_v2_definition' == struct_type_name[0]:
                     out_txt = out_txt + f'struct retro_core_options_v2 options{lang_low}' \
                                         ' = {\n' \
                                         f'   option_cats{lang_low},\n' \
                                         f'   option_defs{lang_low}\n' \
                                         '};\n\n'
-        #    shutil.rmtree(JOINER.join((intl_dir_path, folder)))
+            # if it got this far, we've got something to write
+            overwrite = True
 
-    with open(localisation_file_path, 'w', encoding='utf-8') as intl:
+    # only write to file, if there is anything worthwhile to write!
-        intl.write(out_txt + '\n#ifdef __cplusplus\n'
+    if overwrite:
-                             '}\n#endif\n'
+        with open(localisation_file_path, 'w', encoding='utf-8') as intl:
-                             '\n#endif')
+            intl.write(out_txt + '\n#ifdef __cplusplus\n'
+                                 '}\n#endif\n'
+                                 '\n#endif')
     return
 
 
@@ -571,7 +593,8 @@ if __name__ == '__main__':
         TARGET_DIR_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
         print("No path provided, assuming parent directory:\n" + TARGET_DIR_PATH)
 
-    NAME = 'core_options'
+    CORE_NAME = clean_file_name(sys.argv[2])
+
     DIR_PATH = os.path.dirname(os.path.realpath(__file__))
     H_FILE_PATH = os.path.join(TARGET_DIR_PATH, 'libretro_core_options.h')
     INTL_FILE_PATH = os.path.join(TARGET_DIR_PATH, 'libretro_core_options_intl.h')
@@ -580,7 +603,7 @@ if __name__ == '__main__':
     with open(H_FILE_PATH, 'r+', encoding='utf-8') as _h_file:
         _main_text = _h_file.read()
     _hash_n_str = get_texts(_main_text)
-    _files = create_msg_hash(DIR_PATH, NAME, _hash_n_str)
+    _files = create_msg_hash(DIR_PATH, CORE_NAME, _hash_n_str)
     _source_jsons = h2json(_files)
 
     print('Getting texts from libretro_core_options_intl.h')
@@ -588,7 +611,7 @@ if __name__ == '__main__':
         with open(INTL_FILE_PATH, 'r+', encoding='utf-8') as _intl_file:
             _intl_text = _intl_file.read()
             _hash_n_str_intl = get_texts(_intl_text)
-            _intl_files = create_msg_hash(DIR_PATH, NAME, _hash_n_str_intl)
+            _intl_files = create_msg_hash(DIR_PATH, CORE_NAME, _hash_n_str_intl)
             _intl_jsons = h2json(_intl_files)
 
     print('\nAll done!')

intl/crowdin.yaml

@@ -6,8 +6,8 @@
 "files":
   [
     {
-      "source": "/intl/_us/*.json",
+      "source": "/intl/_core_name_/_us.json",
-      "dest": "/_core_name_/%original_file_name%",
+      "dest": "/_core_name_/core_options.json",
-      "translation": "/intl/_%two_letters_code%/%original_file_name%",
+      "translation": "/intl/_core_name_/_%two_letters_code%.json",
     },
   ]

intl/crowdin_prep.py

@@ -4,8 +4,6 @@ import core_option_translation as t
 
 
 if __name__ == '__main__':
-    NAME = 'core_options'
-
     try:
         if t.os.path.isfile(t.sys.argv[1]):
             _temp = t.os.path.dirname(t.sys.argv[1])
@@ -18,6 +16,7 @@ if __name__ == '__main__':
         TARGET_DIR_PATH = t.os.path.dirname(t.os.path.dirname(t.os.path.realpath(__file__)))
         print("No path provided, assuming parent directory:\n" + TARGET_DIR_PATH)
 
+    CORE_NAME = t.clean_file_name(t.sys.argv[2])
     DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
     H_FILE_PATH = t.os.path.join(TARGET_DIR_PATH, 'libretro_core_options.h')
 
@@ -25,7 +24,7 @@ if __name__ == '__main__':
     with open(H_FILE_PATH, 'r+', encoding='utf-8') as _h_file:
         _main_text = _h_file.read()
     _hash_n_str = t.get_texts(_main_text)
-    _files = t.create_msg_hash(DIR_PATH, NAME, _hash_n_str)
+    _files = t.create_msg_hash(DIR_PATH, CORE_NAME, _hash_n_str)
 
     _source_jsons = t.h2json(_files)
 

intl/crowdin_source_upload.py

@@ -9,76 +9,85 @@ import urllib.request
 import zipfile
 import core_option_translation as t
 
-# Check Crowdin API Token and core name
+# --------------------          MAIN          -------------------- #
-try:
-    api_key = sys.argv[1]
-    core_name = t.clean_file_name(sys.argv[2])
-except IndexError as e:
-    print('Please provide Crowdin API Token and core name!')
-    raise e
 
-DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
+if __name__ == '__main__':
-YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml')
+    # Check Crowdin API Token and core name
+    try:
+        API_KEY = sys.argv[1]
+        CORE_NAME = t.clean_file_name(sys.argv[2])
+    except IndexError as e:
+        print('Please provide Crowdin API Token and core name!')
+        raise e
 
-# Apply Crowdin API Key
+    DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
-with open(YAML_PATH, 'r') as crowdin_config_file:
+    YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml')
-    crowdin_config = crowdin_config_file.read()
-crowdin_config = re.sub(r'"api_token": "_secret_"',
-                        f'"api_token": "{api_key}"',
-                        crowdin_config, 1)
-crowdin_config = re.sub(r'"dest": "/_core_name_/%original_file_name%",',
-                        f'"dest": "/{core_name}/%original_file_name%",'
-                        , crowdin_config, 1)
-with open(YAML_PATH, 'w') as crowdin_config_file:
-    crowdin_config_file.write(crowdin_config)
 
-try:
+    # Apply Crowdin API Key
-    # Download Crowdin CLI
-    jar_name = 'crowdin-cli.jar'
-    jar_path = t.os.path.join(DIR_PATH, jar_name)
-    crowdin_cli_file = 'crowdin-cli.zip'
-    crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file
-    crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file)
-
-    if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)):
-        print('download crowdin-cli.jar')
-        urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path)
-        with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref:
-            jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0])
-            for file in zip_ref.namelist():
-                if file.endswith(jar_name):
-                    jar_file = file
-                    break
-            zip_ref.extract(jar_file, path=DIR_PATH)
-            os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path)
-            os.remove(crowdin_cli_path)
-            shutil.rmtree(jar_dir)
-
-    print('upload source *.json')
-    subprocess.run(['java', '-jar', jar_path, 'upload', 'sources', '--config', YAML_PATH])
-
-    # Reset Crowdin API Key
     with open(YAML_PATH, 'r') as crowdin_config_file:
         crowdin_config = crowdin_config_file.read()
-    crowdin_config = re.sub(r'"api_token": ".*?"',
+    crowdin_config = re.sub(r'"api_token": "_secret_"',
-                            '"api_token": "_secret_"',
+                            f'"api_token": "{API_KEY}"',
                             crowdin_config, 1)
-    crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",',
+    crowdin_config = re.sub(r'/_core_name_/',
-                            '"dest": "/_core_name_/%original_file_name%",'
+                            f'/{CORE_NAME}/'
-                            , crowdin_config, 1)
+                            , crowdin_config)
     with open(YAML_PATH, 'w') as crowdin_config_file:
         crowdin_config_file.write(crowdin_config)
 
-except Exception as e:
+    try:
-    # Try really hard to reset Crowdin API Key
+        # Download Crowdin CLI
-    with open(YAML_PATH, 'r') as crowdin_config_file:
+        jar_name = 'crowdin-cli.jar'
-        crowdin_config = crowdin_config_file.read()
+        jar_path = t.os.path.join(DIR_PATH, jar_name)
-    crowdin_config = re.sub(r'"api_token": ".*?"',
+        crowdin_cli_file = 'crowdin-cli.zip'
-                            '"api_token": "_secret_"',
+        crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file
-                            crowdin_config, 1)
+        crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file)
-    crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",',
+
-                            '"dest": "/_core_name_/%original_file_name%",'
+        if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)):
-                            , crowdin_config, 1)
+            print('download crowdin-cli.jar')
-    with open(YAML_PATH, 'w') as crowdin_config_file:
+            urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path)
-        crowdin_config_file.write(crowdin_config)
+            with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref:
-    raise e
+                jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0])
+                for file in zip_ref.namelist():
+                    if file.endswith(jar_name):
+                        jar_file = file
+                        break
+                zip_ref.extract(jar_file, path=DIR_PATH)
+                os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path)
+                os.remove(crowdin_cli_path)
+                shutil.rmtree(jar_dir)
+
+        print('upload source *.json')
+        subprocess.run(['java', '-jar', jar_path, 'upload', 'sources', '--config', YAML_PATH])
+
+        # Reset Crowdin API Key
+        with open(YAML_PATH, 'r') as crowdin_config_file:
+            crowdin_config = crowdin_config_file.read()
+        crowdin_config = re.sub(r'"api_token": ".*?"',
+                                '"api_token": "_secret_"',
+                                crowdin_config, 1)
+
+        # TODO this is NOT safe!
+        crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'),
+                                '/_core_name_/',
+                                crowdin_config)
+
+        with open(YAML_PATH, 'w') as crowdin_config_file:
+            crowdin_config_file.write(crowdin_config)
+
+    except Exception as e:
+        # Try really hard to reset Crowdin API Key
+        with open(YAML_PATH, 'r') as crowdin_config_file:
+            crowdin_config = crowdin_config_file.read()
+        crowdin_config = re.sub(r'"api_token": ".*?"',
+                                '"api_token": "_secret_"',
+                                crowdin_config, 1)
+
+        # TODO this is NOT safe!
+        crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'),
+                                '/_core_name_/',
+                                crowdin_config)
+
+        with open(YAML_PATH, 'w') as crowdin_config_file:
+            crowdin_config_file.write(crowdin_config)
+        raise e

intl/crowdin_translate.py

@@ -16,9 +16,10 @@ if __name__ == '__main__':
         TARGET_DIR_PATH = t.os.path.dirname(t.os.path.dirname(t.os.path.realpath(__file__)))
         print("No path provided, assuming parent directory:\n" + TARGET_DIR_PATH)
 
-    NAME = 'core_options'
+    CORE_NAME = t.clean_file_name(t.sys.argv[2])
     DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
-    US_FILE_PATH = t.os.path.join(DIR_PATH, '_us', f'{NAME}.h')
+    LOCALISATIONS_PATH = t.os.path.join(DIR_PATH, CORE_NAME)
+    US_FILE_PATH = t.os.path.join(LOCALISATIONS_PATH, '_us.h')
     H_FILE_PATH = t.os.path.join(TARGET_DIR_PATH, 'libretro_core_options.h')
     INTL_FILE_PATH = t.os.path.join(TARGET_DIR_PATH, 'libretro_core_options_intl.h')
 
@@ -26,15 +27,14 @@ if __name__ == '__main__':
     with open(H_FILE_PATH, 'r+', encoding='utf-8') as _h_file:
         _main_text = _h_file.read()
     _hash_n_str = t.get_texts(_main_text)
-    _files = t.create_msg_hash(DIR_PATH, NAME, _hash_n_str)
+    _files = t.create_msg_hash(DIR_PATH, CORE_NAME, _hash_n_str)
+    _source_jsons = t.h2json(_files)
 
     print('Converting translations *.json to *.h:')
-    for _folder in t.os.scandir(DIR_PATH):
+    localisation_files = t.os.scandir(LOCALISATIONS_PATH)
-        if _folder.is_dir() and _folder.name.startswith('_') and _folder.name != '__pycache__':
+    t.json2h(LOCALISATIONS_PATH, localisation_files)
-            print(_folder.name)
-            t.json2h(DIR_PATH, _folder.path, NAME)
 
     print('Constructing libretro_core_options_intl.h')
-    t.create_intl_file(INTL_FILE_PATH, DIR_PATH, _main_text, NAME, _files["_us"])
+    t.create_intl_file(INTL_FILE_PATH, LOCALISATIONS_PATH, _main_text, _files["_us"])
 
     print('\nAll done!')

intl/crowdin_translation_download.py

@@ -9,76 +9,85 @@ import urllib.request
 import zipfile
 import core_option_translation as t
 
-# Check Crowdin API Token and core name
+# --------------------          MAIN          -------------------- #
-try:
-    api_key = sys.argv[1]
-    core_name = t.clean_file_name(sys.argv[2])
-except IndexError as e:
-    print('Please provide Crowdin API Token and core name!')
-    raise e
 
-DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
+if __name__ == '__main__':
-YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml')
+    # Check Crowdin API Token and core name
+    try:
+        API_KEY = sys.argv[1]
+        CORE_NAME = t.clean_file_name(sys.argv[2])
+    except IndexError as e:
+        print('Please provide Crowdin API Token and core name!')
+        raise e
 
-# Apply Crowdin API Key
+    DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
-with open(YAML_PATH, 'r') as crowdin_config_file:
+    YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml')
-    crowdin_config = crowdin_config_file.read()
-crowdin_config = re.sub(r'"api_token": "_secret_"',
-                        f'"api_token": "{api_key}"',
-                        crowdin_config, 1)
-crowdin_config = re.sub(r'"dest": "/_core_name_/%original_file_name%",',
-                        f'"dest": "/{core_name}/%original_file_name%",'
-                        , crowdin_config, 1)
-with open(YAML_PATH, 'w') as crowdin_config_file:
-    crowdin_config_file.write(crowdin_config)
 
-try:
+    # Apply Crowdin API Key
-    # Download Crowdin CLI
-    jar_name = 'crowdin-cli.jar'
-    jar_path = t.os.path.join(DIR_PATH, jar_name)
-    crowdin_cli_file = 'crowdin-cli.zip'
-    crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file
-    crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file)
-
-    if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)):
-        print('download crowdin-cli.jar')
-        urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path)
-        with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref:
-            jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0])
-            for file in zip_ref.namelist():
-                if file.endswith(jar_name):
-                    jar_file = file
-                    break
-            zip_ref.extract(jar_file, path=DIR_PATH)
-            os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path)
-            os.remove(crowdin_cli_path)
-            shutil.rmtree(jar_dir)
-
-    print('download translation *.json')
-    subprocess.run(['java', '-jar', jar_path, 'download', '--config', YAML_PATH])
-
-    # Reset Crowdin API Key
     with open(YAML_PATH, 'r') as crowdin_config_file:
         crowdin_config = crowdin_config_file.read()
-    crowdin_config = re.sub(r'"api_token": ".*?"',
+    crowdin_config = re.sub(r'"api_token": "_secret_"',
-                            '"api_token": "_secret_"',
+                            f'"api_token": "{API_KEY}"',
                             crowdin_config, 1)
-    crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",',
+    crowdin_config = re.sub(r'/_core_name_/',
-                            '"dest": "/_core_name_/%original_file_name%",'
+                            f'/{CORE_NAME}/'
-                            , crowdin_config, 1)
+                            , crowdin_config)
     with open(YAML_PATH, 'w') as crowdin_config_file:
         crowdin_config_file.write(crowdin_config)
 
-except Exception as e:
+    try:
-    # Try really hard to reset Crowdin API Key
+        # Download Crowdin CLI
-    with open(YAML_PATH, 'r') as crowdin_config_file:
+        jar_name = 'crowdin-cli.jar'
-        crowdin_config = crowdin_config_file.read()
+        jar_path = t.os.path.join(DIR_PATH, jar_name)
-    crowdin_config = re.sub(r'"api_token": ".*?"',
+        crowdin_cli_file = 'crowdin-cli.zip'
-                            '"api_token": "_secret_"',
+        crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file
-                            crowdin_config, 1)
+        crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file)
-    crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",',
+
-                            '"dest": "/_core_name_/%original_file_name%",'
+        if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)):
-                            , crowdin_config, 1)
+            print('download crowdin-cli.jar')
-    with open(YAML_PATH, 'w') as crowdin_config_file:
+            urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path)
-        crowdin_config_file.write(crowdin_config)
+            with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref:
-    raise e
+                jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0])
+                for file in zip_ref.namelist():
+                    if file.endswith(jar_name):
+                        jar_file = file
+                        break
+                zip_ref.extract(jar_file, path=DIR_PATH)
+                os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path)
+                os.remove(crowdin_cli_path)
+                shutil.rmtree(jar_dir)
+
+        print('download translation *.json')
+        subprocess.run(['java', '-jar', jar_path, 'download', '--config', YAML_PATH])
+
+        # Reset Crowdin API Key
+        with open(YAML_PATH, 'r') as crowdin_config_file:
+            crowdin_config = crowdin_config_file.read()
+        crowdin_config = re.sub(r'"api_token": ".*?"',
+                                '"api_token": "_secret_"',
+                                crowdin_config, 1)
+
+        # TODO this is NOT safe!
+        crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'),
+                                '/_core_name_/',
+                                crowdin_config)
+
+        with open(YAML_PATH, 'w') as crowdin_config_file:
+            crowdin_config_file.write(crowdin_config)
+
+    except Exception as e:
+        # Try really hard to reset Crowdin API Key
+        with open(YAML_PATH, 'r') as crowdin_config_file:
+            crowdin_config = crowdin_config_file.read()
+        crowdin_config = re.sub(r'"api_token": ".*?"',
+                                '"api_token": "_secret_"',
+                                crowdin_config, 1)
+
+        # TODO this is NOT safe!
+        crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'),
+                                '/_core_name_/',
+                                crowdin_config)
+        
+        with open(YAML_PATH, 'w') as crowdin_config_file:
+            crowdin_config_file.write(crowdin_config)
+        raise e

intl/download_workflow.py

@@ -12,4 +12,4 @@ except IndexError as e:
     raise e
 
 subprocess.run(['python3', 'intl/crowdin_translation_download.py', api_key, core_name])
-subprocess.run(['python3', 'intl/crowdin_translate.py', dir_path])
+subprocess.run(['python3', 'intl/crowdin_translate.py', dir_path, core_name])

intl/initial_sync.py

@@ -10,94 +10,103 @@ import urllib.request
 import zipfile
 import core_option_translation as t
 
-# Check Crowdin API Token and core name
+# --------------------          MAIN          -------------------- #
-try:
-    api_key = sys.argv[1]
-    core_name = t.clean_file_name(sys.argv[2])
-except IndexError as e:
-    print('Please provide Crowdin API Token and core name!')
-    raise e
 
-DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
+if __name__ == '__main__':
-YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml')
+    # Check Crowdin API Token and core name
+    try:
+        API_KEY = sys.argv[1]
+        CORE_NAME = t.clean_file_name(sys.argv[2])
+    except IndexError as e:
+        print('Please provide Crowdin API Token and core name!')
+        raise e
 
-# Apply Crowdin API Key
+    DIR_PATH = t.os.path.dirname(t.os.path.realpath(__file__))
-with open(YAML_PATH, 'r') as crowdin_config_file:
+    YAML_PATH = t.os.path.join(DIR_PATH, 'crowdin.yaml')
-    crowdin_config = crowdin_config_file.read()
-crowdin_config = re.sub(r'"api_token": "_secret_"',
-                        f'"api_token": "{api_key}"',
-                        crowdin_config, 1)
-crowdin_config = re.sub(r'"dest": "/_core_name_/%original_file_name%",',
-                        f'"dest": "/{core_name}/%original_file_name%",'
-                        , crowdin_config, 1)
-with open(YAML_PATH, 'w') as crowdin_config_file:
-    crowdin_config_file.write(crowdin_config)
 
-try:
+    # Apply Crowdin API Key
-    # Download Crowdin CLI
-    jar_name = 'crowdin-cli.jar'
-    jar_path = t.os.path.join(DIR_PATH, jar_name)
-    crowdin_cli_file = 'crowdin-cli.zip'
-    crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file
-    crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file)
-
-    if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)):
-        print('download crowdin-cli.jar')
-        urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path)
-        with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref:
-            jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0])
-            for file in zip_ref.namelist():
-                if file.endswith(jar_name):
-                    jar_file = file
-                    break
-            zip_ref.extract(jar_file, path=DIR_PATH)
-            os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path)
-            os.remove(crowdin_cli_path)
-            shutil.rmtree(jar_dir)
-
-    print('upload source & translations *.json')
-    subprocess.run(['java', '-jar', jar_path, 'upload', 'sources', '--config', YAML_PATH])
-    subprocess.run(['java', '-jar', jar_path, 'upload', 'translations', '--config', YAML_PATH])
-
-    print('wait for crowdin server to process data')
-    time.sleep(10)
-
-    print('download translation *.json')
-    subprocess.run(['java', '-jar', jar_path, 'download', '--config', YAML_PATH])
-
-    # Reset Crowdin API Key
     with open(YAML_PATH, 'r') as crowdin_config_file:
         crowdin_config = crowdin_config_file.read()
-    crowdin_config = re.sub(r'"api_token": ".*?"', '"api_token": "_secret_"', crowdin_config, 1)
+    crowdin_config = re.sub(r'"api_token": "_secret_"',
-    crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",',
+                            f'"api_token": "{API_KEY}"',
-                            '"dest": "/_core_name_/%original_file_name%",'
-                            , crowdin_config, 1)
-    with open(YAML_PATH, 'w') as crowdin_config_file:
-        crowdin_config_file.write(crowdin_config)
-
-    with open('intl/translation_workflow.py', 'r') as workflow:
-        workflow_config = workflow.read()
-    workflow_config = workflow_config.replace(
-        "subprocess.run(['python3', 'intl/core_option_translation.py', dir_path])",
-        "subprocess.run(['python3', 'intl/crowdin_translation_download.py', api_key, core_name])"
-    )
-    workflow_config = workflow_config.replace(
-        "subprocess.run(['python3', 'intl/initial_sync.py', api_key, core_name])\n",
-        ""
-    )
-    with open('intl/translation_workflow.py', 'w') as workflow:
-        workflow.write(workflow_config)
-
-except Exception as e:
-    # Try really hard to reset Crowdin API Key
-    with open(YAML_PATH, 'r') as crowdin_config_file:
-        crowdin_config = crowdin_config_file.read()
-    crowdin_config = re.sub(r'"api_token": ".*?"',
-                            '"api_token": "_secret_"',
                             crowdin_config, 1)
-    crowdin_config = re.sub(r'"dest": "/.*?/%original_file_name%",',
+    crowdin_config = re.sub(r'/_core_name_/',
-                            '"dest": "/_core_name_/%original_file_name%",'
+                            f'/{CORE_NAME}/'
-                            , crowdin_config, 1)
+                            , crowdin_config)
     with open(YAML_PATH, 'w') as crowdin_config_file:
         crowdin_config_file.write(crowdin_config)
-    raise e
+
+    try:
+        # Download Crowdin CLI
+        jar_name = 'crowdin-cli.jar'
+        jar_path = t.os.path.join(DIR_PATH, jar_name)
+        crowdin_cli_file = 'crowdin-cli.zip'
+        crowdin_cli_url = 'https://downloads.crowdin.com/cli/v3/' + crowdin_cli_file
+        crowdin_cli_path = t.os.path.join(DIR_PATH, crowdin_cli_file)
+
+        if not os.path.isfile(t.os.path.join(DIR_PATH, jar_name)):
+            print('download crowdin-cli.jar')
+            urllib.request.urlretrieve(crowdin_cli_url, crowdin_cli_path)
+            with zipfile.ZipFile(crowdin_cli_path, 'r') as zip_ref:
+                jar_dir = t.os.path.join(DIR_PATH, zip_ref.namelist()[0])
+                for file in zip_ref.namelist():
+                    if file.endswith(jar_name):
+                        jar_file = file
+                        break
+                zip_ref.extract(jar_file, path=DIR_PATH)
+                os.rename(t.os.path.join(DIR_PATH, jar_file), jar_path)
+                os.remove(crowdin_cli_path)
+                shutil.rmtree(jar_dir)
+
+        print('upload source & translations *.json')
+        subprocess.run(['java', '-jar', jar_path, 'upload', 'sources', '--config', YAML_PATH])
+        subprocess.run(['java', '-jar', jar_path, 'upload', 'translations', '--config', YAML_PATH])
+
+        print('wait for crowdin server to process data')
+        time.sleep(10)
+
+        print('download translation *.json')
+        subprocess.run(['java', '-jar', jar_path, 'download', '--config', YAML_PATH])
+
+        # Reset Crowdin API Key
+        with open(YAML_PATH, 'r') as crowdin_config_file:
+            crowdin_config = crowdin_config_file.read()
+        crowdin_config = re.sub(r'"api_token": ".*?"', '"api_token": "_secret_"', crowdin_config, 1)
+
+        # TODO this is NOT safe!
+        crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'),
+                                '/_core_name_/',
+                                crowdin_config)
+
+        with open(YAML_PATH, 'w') as crowdin_config_file:
+            crowdin_config_file.write(crowdin_config)
+
+        with open('intl/translation_workflow.py', 'r') as workflow:
+            workflow_config = workflow.read()
+        workflow_config = workflow_config.replace(
+            "subprocess.run(['python3', 'intl/core_option_translation.py', dir_path, core_name])",
+            "subprocess.run(['python3', 'intl/crowdin_translation_download.py', api_key, core_name])"
+        )
+        workflow_config = workflow_config.replace(
+            "subprocess.run(['python3', 'intl/initial_sync.py', api_key, core_name])\n",
+            ""
+        )
+        with open('intl/translation_workflow.py', 'w') as workflow:
+            workflow.write(workflow_config)
+
+    except Exception as e:
+        # Try really hard to reset Crowdin API Key
+        with open(YAML_PATH, 'r') as crowdin_config_file:
+            crowdin_config = crowdin_config_file.read()
+        crowdin_config = re.sub(r'"api_token": ".*?"',
+                                '"api_token": "_secret_"',
+                                crowdin_config, 1)
+
+        # TODO this is NOT safe!
+        crowdin_config = re.sub(re.escape(f'/{CORE_NAME}/'),
+                                '/_core_name_/',
+                                crowdin_config)
+
+        with open(YAML_PATH, 'w') as crowdin_config_file:
+            crowdin_config_file.write(crowdin_config)
+        raise e

intl/upload_workflow.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+import sys
+import subprocess
+
+try:
+    api_key = sys.argv[1]
+    core_name = sys.argv[2]
+    dir_path = sys.argv[3]
+except IndexError as e:
+    print('Please provide path to libretro_core_options.h, Crowdin API Token and core name!')
+    raise e
+
+subprocess.run(['python3', 'intl/crowdin_prep.py', dir_path, core_name])
+subprocess.run(['python3', 'intl/crowdin_source_upload.py', api_key, core_name])