adjust h2json and json2h scripts to be comment agnostic (#12532)

Co-authored-by: Michael Burgardt <michael.burgardt@rwth-aachen.de>

intl/h2json.py

@@ -18,20 +18,39 @@ if h_filename == 'msg_hash_lbl.h':
     print("Skip")
     sys.exit(0)
 
-p = re.compile(r'MSG_HASH\(\s*\/?\*?.*\*?\/?\s*[a-zA-Z0-9_]+\s*,\s*\".*\"\s*\)')
+p = re.compile(
+    r'MSG_HASH\s*(?:\/\*(?:.|[\r\n])*?\*\/\s*)*\(\s*(?:\/\*(?:.|[\r\n])*?\*\/\s*)*[a-zA-Z0-9_]+\s*(?:\/\*(?:.|[\r\n])*?\*\/\s*)*,\s*(?:\/\*(?:.|[\r\n])*?\*\/\s*)*\".*\"\s*(?:\/\*(?:.|[\r\n])*?\*\/\s*)*\)')
+
 
 def parse_message(message):
-   key_start = max(message.find('(') + 1, message.find('*/') + 2)
+    a = message.find('/*')
-   key_end = message.find(',', key_start)
+    b = message.find('*/')
-   key = message[key_start:key_end].strip()
+    c = message.find('"')
-   value_start = message.find('"') + 1
+    new_msg = message
-   value_end = message.rfind('"')
+    while (a >= 0 and b >= 0) and (a < c < b or b < c):
-   value = message[value_start:value_end]
+        new_msg = new_msg[:a] + new_msg[b + 2:]
-   return key, value
+        c = new_msg.find('"', a)
+        b = new_msg.find('*/', a)
+        a = new_msg.find('/*', a)
+    # get key word
+    word = new_msg[new_msg.find('(') + 1:new_msg.find(',')].strip()
+
+    a = new_msg.rfind('/*')
+    b = new_msg.rfind('*/')
+    d = new_msg.rfind('"')
+    while (a >= 0 and b >= 0) and (a < d < b or a > d):
+        new_msg = new_msg[:a]
+        a = new_msg.rfind('/*')
+        b = new_msg.rfind('*/')
+        d = new_msg.rfind('"')
+    # get value
+    value = new_msg[c + 1:d]
+
+    return word, value
 
 
 try:
-   with open(h_filename, 'r+') as h_file:
+    with open(h_filename, 'r+', encoding='utf-8') as h_file:
         text = h_file.read()
         result = p.findall(text)
         seen = set()
@@ -44,7 +63,7 @@ try:
                     seen.add(key)
                 else:
                     print("Duplicate key: " + key)
-      with open(json_filename, 'w') as json_file:
+        with open(json_filename, 'w', encoding='utf-8') as json_file:
             json.dump(messages, json_file, indent=2)
 except EnvironmentError:
     print('Cannot read/write ' + h_filename)

intl/json2h.py

@@ -18,7 +18,9 @@ if json_filename == 'msg_hash_us.json' or json_filename == 'msg_hash_lbl.json':
     print("Skip")
     sys.exit(0)
 
-p = re.compile(r'MSG_HASH\(\s*\/?\*?.*\*?\/?\s*[a-zA-Z0-9_]+\s*,\s*\".*\"\s*\)')
+p = re.compile(
+    r'MSG_HASH\s*(?:\/\*(?:.|[\r\n])*?\*\/\s*)*\(\s*(?:\/\*(?:.|[\r\n])*?\*\/\s*)*[a-zA-Z0-9_]+\s*(?:\/\*(?:.|[\r\n])*?\*\/\s*)*,\s*(?:\/\*(?:.|[\r\n])*?\*\/\s*)*\".*\"\s*(?:\/\*(?:.|[\r\n])*?\*\/\s*)*\)')
+
 
 def c89_cut(old_str):
     new_str = ''
@@ -30,14 +32,34 @@ def c89_cut(old_str):
         new_str += c
     return new_str
 
+
 def parse_message(message):
-   key_start = max(message.find('(') + 1, message.find('*/') + 2)
+    # remove all comments before the value (= the string)
-   key_end = message.find(',')
+    a = message.find('/*')
-   key = message[key_start:key_end].strip()
+    b = message.find('*/')
-   value_start = message.find('"') + 1
+    c = message.find('"')
-   value_end = message.rfind('"')
+    new_msg = message
-   value = message[value_start:value_end]
+    while (a >= 0 and b >= 0) and (a < c < b or b < c):
-   return key, value
+        new_msg = new_msg[:a] + new_msg[b + 2:]
+        c = new_msg.find('"', a)
+        b = new_msg.find('*/', a)
+        a = new_msg.find('/*', a)
+    # get key word
+    word = new_msg[new_msg.find('(') + 1:new_msg.find(',')].strip()
+
+    # remove all comments after the value (= the string)
+    a = new_msg.rfind('/*')
+    b = new_msg.rfind('*/')
+    d = new_msg.rfind('"')
+    while (a >= 0 and b >= 0) and (a < d < b or a > d):
+        new_msg = new_msg[:a]
+        a = new_msg.rfind('/*')
+        b = new_msg.rfind('*/')
+        d = new_msg.rfind('"')
+    # get value
+    value = new_msg[c + 1:d]
+
+    return word, value
 
 
 def parse_messages(text):
@@ -52,11 +74,12 @@ def parse_messages(text):
             seen.add(key)
         else:
             print("Duplicate key: " + key)
+
     return msg_list
 
 
 def update(messages, template, source_messages):
-   new_translation = template
+    translation = template
     template_messages = parse_messages(template)
     for tp_msg in template_messages:
         old_msg = tp_msg['msg']
@@ -68,14 +91,14 @@ def update(messages, template, source_messages):
                 tl_msg_val = c89_cut(tl_msg_val)
             # Replace last match, in case the key contains the value string
             new_msg = old_msg[::-1].replace(tp_msg_val[::-1], tl_msg_val[::-1], 1)[::-1]
-         new_translation = new_translation.replace(old_msg, new_msg)
+            translation = translation.replace(old_msg, new_msg)
-      # Remove English duplicates and non-translateable strings
+        # Remove English duplicates and non-translatable strings
         else:
-         new_translation = new_translation.replace(old_msg + '\n', '')
+            translation = translation.replace(old_msg + '\n', '')
-   return new_translation
+    return translation
 
 
-with open('msg_hash_us.h', 'r') as template_file:
+with open('msg_hash_us.h', 'r', encoding='utf-8') as template_file:
     template = template_file.read()
     with open('msg_hash_us.json', 'r+', encoding='utf-8') as source_json_file:
         source_messages = json.load(source_json_file)