From 8fe695a8be3acb8472225b37199f367621ca67b8 Mon Sep 17 00:00:00 2001 From: alastrim Date: Wed, 31 May 2023 08:20:47 -0300 Subject: Adapt code to python 3 --- lang/updatelang.py | 639 +++++++++++++++++++++++++++++------------------------ 1 file changed, 354 insertions(+), 285 deletions(-) diff --git a/lang/updatelang.py b/lang/updatelang.py index 3398e20..8baa575 100755 --- a/lang/updatelang.py +++ b/lang/updatelang.py @@ -1,17 +1,30 @@ -#! /usr/bin/env python2.7 +#! /usr/bin/env python3 # -*- coding: utf8 -*- # # Copyright (C) 2010-2015 Evol Online # Author: Andrei Karas (4144) +import sys import os import re -defaultLang = "en" -filt = re.compile(".+[.]txt", re.IGNORECASE) +server_data_path = os.path.join("..", "..", "server-data") +db_path = os.path.join(server_data_path, "db", "re") +item_db_path = os.path.join(server_data_path, "db", "re", "item_db.conf") +script_path = os.path.join(server_data_path, "npc") +lang_path = os.path.join(server_data_path, "langs") + +default_language = "en" + +all_strings = set() +string_comments = dict() +lang_files = dict() +old_lang_files = dict() +langs = set() +item_names_pairs = dict() -allStrings = set() -strComments = dict() +# Regex patterns +filt = re.compile(".+[.]txt", re.IGNORECASE) strre1 = re.compile("[\t +(]l[(][\"](?P[^\"]+)[\"]") strre3 = re.compile("[\t +(]getitemlink[(][\"](?P[^\"]+)[\"][)]") strre2 = re.compile("^[^/](.+)([^\t]+)[\t](script|shop|trader|cashshop|monster)[\t](?P[^\t]+)[\t]([\w\d]+),") @@ -21,82 +34,134 @@ strre6 = re.compile("[\t ]mesn[ ][\"](?P[^\"]+)[\"]") strre7 = re.compile("[\t +(]lg[(][\"](?P[^\"]+)[\"],([ ]*)[\"](?P[^\"]+)[\"]") itemsplit = re.compile(",") -langFiles = dict() -oldLangFiles = dict() -langs = set() -itemNamesByName = dict() -def addStr(text, comment, fileName, lineNum, addNoC): - text = text.replace("\a", "\""); - allStrings.add(text) +def add_string(text, comment, file_name, line_number, add_no_c_format): + + # Add to all_strings list + + text = text.replace("\a", "\"") + all_strings.add(text) + if comment[-1:] == "\n": comment = comment[:-1] - if text not in strComments: - strComments[text] = set() - strComments[text].add("#. code: " + comment.strip() + "\n") - strComments[text].add("#: " + fileName + ":" + str(lineNum) + "\n") - if addNoC == True: - strComments[text].add("#, no-c-format\n") + if text not in string_comments: + string_comments[text] = set() -def collectScriptFileStrings(codeFile, file2): - global itemNamesByName - with open(file2, "r") as f: - cnt = -1 - for line in f: - cnt = cnt + 1 - line = line.replace("\\\"", "\a"); + string_comments[text].add(f"#. code: {comment.strip()}\n") + string_comments[text].add(f"#: file_name:{str(line_number)}\n") + + if add_no_c_format: + string_comments[text].add("#, no-c-format\n") + + +def load_item_db(dir): + + # Collect item names from item_db.conf and add to all_strings + + with open(f"{dir}/item_db.conf", "r", encoding="UTF-8") as f: + for index, line in enumerate(f): + line_number = index + 1 + line = line.strip() + index_in_line = line.find("Name: \"") + if index_in_line == 0 and line[-1] == "\"": + item_name = line[index_in_line + 7: -1] + add_string(item_name, line, "item_db.conf", line_number, True) + + if line == "{": + # current_item_id = "" + current_item_name = "" + for line in f: + line = line.strip() + if len(line) < 1: + continue + + if line.startswith("}") and current_item_name: + item_names_pairs[current_item_name.lower()] = current_item_name + break + + if 'Name:' in line and 'AegisName:' not in line and line.endswith("\""): + current_item_name = line[5:].strip() + current_item_name = current_item_name.strip('"') + + +def load_mob_names(dir): + + # Collect mob names from mob_db.conf and add to all_strings + + with open(f"{dir}/mob_db.conf", "r", encoding="UTF-8") as f: + for index, line in enumerate(f): + line_number = index + 1 + if len(line) < 1 or line[0:2] == "//": + continue + line = line.strip() + name_in_line = line.find("Name: \"") + if name_in_line != 0 or line[-1] != "\"": + continue + mob_name = line[name_in_line + 7:-1] + add_string(mob_name, line, "mob_db.conf", line_number, True) + + +def collect_script_file_strings(file): + + # Collect strings from a given file and add to all_strings + + with open(file, "r", encoding="UTF-8") as f: + for index, line in enumerate(f): + line_number = index + 1 + line = line.replace("\\\"", "\a") m = strre1.findall(line) if len(m) > 0: for str in m: - addStr(str, line, codeFile, cnt, True) + add_string(str, line, file, line_number, True) m = strre4.findall(line) if len(m) > 0: for str in m: - addStr(str + "#0", line, codeFile, cnt, True) - addStr(str + "#1", line, codeFile, cnt, True) + add_string(str + "#0", line, file, line_number, True) + add_string(str + "#1", line, file, line_number, True) m = strre2.search(line) if m is not None and m.group("str")[0] != "#": - addStr(m.group("str"), line, codeFile, cnt, True) + add_string(m.group("str"), line, file, line_number, True) m = strre3.findall(line) if len(m) > 0: for str in m: - if str.lower() in itemNamesByName: - addStr(itemNamesByName[str.lower()], line, codeFile, cnt, True) + if str.lower() in item_names_pairs: + add_string(item_names_pairs[str.lower()], line, file, line_number, True) m = strre5.findall(line) if len(m) > 0: for str in m: - if str.lower() in itemNamesByName: - addStr(itemNamesByName[str.lower()], line, codeFile, cnt, True) + if str.lower() in item_names_pairs: + add_string(item_names_pairs[str.lower()], line, file, line_number, True) m = strre6.findall(line) if len(m) > 0: for str in m: - addStr(str, line, codeFile, cnt, True) + add_string(str, line, file, line_number, True) m = strre7.findall(line) if len(m) > 0: for str in m: - addStr(str[0] + "#0", line, codeFile, cnt, True) - addStr(str[2] + "#1", line, codeFile, cnt, True) - - -def collectScriptStrings(parentDir, relativeDir): - files = os.listdir(parentDir) - for file1 in files: - if file1[0] == ".": - continue - if file1[len(file1)-1] == "~": - continue - file2 = os.path.abspath(parentDir + os.path.sep + file1) - relativeDir2 = relativeDir + os.path.sep + file1 - if not os.path.isfile(file2): - collectScriptStrings(file2, relativeDir2) - elif filt.search(file1): - codeFile = relativeDir + os.path.sep + file1 - collectScriptFileStrings(codeFile, file2) - - -def collectMessages(messagesDir): - with open(messagesDir, "r") as r: + add_string(str[0] + "#0", line, file, line_number, True) + add_string(str[2] + "#1", line, file, line_number, True) + + +def collect_script_strings(dir): + + # Iterate over script files in a given directory and collect its strings + + for dirpath, dirs, files in os.walk(dir): + for file in files: + file_path = os.path.join(dirpath, file) + + if file.startswith(".") or file.endswith("~"): + continue + + if not file.lower().endswith(".txt"): + continue + + collect_script_file_strings(file_path) + + +def collect_messages(messages_dir): + with open(messages_dir, "r") as r: cnt = -1 for line in r: cnt = cnt + 1 @@ -109,285 +174,289 @@ def collectMessages(messagesDir): msgStr = line[idx + 2:] if msgStr[-1:] == "\n": msgStr = msgStr[:-1] - addStr(msgStr, line, "conf/messages.conf", cnt, False) + add_string(msgStr, line, "conf/messages.conf", cnt, False) + + +def load_lang_files(): -def loadFiles(dir): - with open(dir + "/langs.txt", "r") as f: + # Load lang files (.txt and .old) from server-data/langs + + langs_file_path = os.path.join(lang_path, "langs.txt") + + with open(langs_file_path, "r") as f: for line in f: langs.add(line[:-1]) - for file in langs: - langFiles[file] = parseFile(dir + "/lang_" + file + ".txt", True) - oldLangFiles[file] = parseFile(dir + "/lang_" + file + ".old", False) - -def parseFile(name, readFirstLine): - trans = dict() - firstLine = None - if os.path.exists(name): - with open(name, "r") as f: - line1 = ""; - line2 = ""; - for line in f: - if readFirstLine is True and firstLine is None: - firstLine = line - continue - - if (line == ""): - line1 = "" - line2 = "" - continue - elif (line1 == ""): - line1 = line[:-1] - continue - - line2 = line[:-1] - trans[line1] = line2 - line1 = "" - line2 = "" - elif readFirstLine: - firstLine = "Copyright (C) 2010-2015 Evol Online\n" - return (trans, firstLine) - - -def loadPoFiles(podir): - files = os.listdir(podir) - for name in files: - if name[-3:] == ".po": - parsePoFile(name[:-3], podir + "/" + name) - - -def parsePoFile(name, path): - langFile = langFiles[name][0] - with open(path, "r") as f: + for language in langs: + file_path = os.path.join(lang_path, f"lang_{language}.txt") + old_file_path = os.path.join(lang_path, f"lang_{language}.old") + + lang_files[language] = parse_file(file_path, True) + old_lang_files[language] = parse_file(old_file_path, False) + + +def parse_file(file_path, read_first_line): + + # Return a tuple containing a dictionary {String: Translated String} and first_line + + translation_pair = dict() + first_line = None + + print(f"Reading {file_path}") + + if os.path.exists(file_path): + try: + with open(file_path, "r", encoding="UTF-8") as f: + original_line = "" + translated_line = "" + for index, line in enumerate(f): + if read_first_line and first_line is None: + first_line = line + continue + + if (line == ""): + original_line = "" + translated_line = "" + continue + + elif (original_line == ""): + original_line = line[:-1] + continue + + elif (translated_line == ""): + translated_line = line[:-1] + translation_pair[original_line] = translated_line + + original_line = "" + translated_line = "" + + except Exception as e: + print(f"Error: {e}") + + elif read_first_line: + first_line = "Copyright (C) 2010-2015 Evol Online\n" + + return (translation_pair, first_line) + + +def add_missing_lines(): + for lang in lang_files: + new_file = lang_files[lang][0] + old_file = old_lang_files[lang][0] + + # Catch strings from the lang files that are not in all_strings and move them to old_file + + for string in new_file: + if string not in all_strings: + old_file[string] = new_file[string] + + # Catch strings that are both in old_file and new_file and remove them from new_file + + for string in old_file: + if string in new_file: + del new_file[string] + + # Deal with collected strings that are not in new_file + + for string in all_strings: + for lang in lang_files: + new_file = lang_files[lang][0] + old_file = old_lang_files[lang][0] + if string not in new_file: + if lang == default_language: + new_file[string] = string + + elif string in old_file: + new_file[string] = old_file[string] + + else: + new_file[string] = "" + + +def load_po_files(po_dir): + + # Iterate over po_dir and parse its content + + for dirpath, dirs, files in os.walk(po_dir): + for file in files: + file_path = os.path.join(dirpath, file) + if file.lower().endswith(".po"): + lang = file[:-3] + parse_po_file(lang, file_path) + + +def parse_po_file(lang, file_path): + lang_file = lang_files[lang][0] + with open(file_path, "r", encoding="UTF-8") as po_file: flag = 0 - line1 = "" - line2 = "" - for line in f: + original_line = "" + translated_line = "" + for line in po_file: if flag == 0: - idx = line.find ("msgid ") - if idx == 0: - line2 = "" - line1 = line[len("msgid "):] - line1 = line1[1:len(line1) - 2] + msgid_in_line = line.find("msgid ") + if msgid_in_line == 0: + translated_line = "" + original_line = line[len("msgid "):] + original_line = original_line[1:len(original_line) - 2] flag = 1 elif flag == 1: - idx = line.find ("msgstr ") - if idx == 0: - line2 = line[len("msgstr "):] - line2 = line2[1:len(line2) - 2] + msgstr_in_line = line.find("msgstr ") + if msgstr_in_line == 0: + translated_line = line[len("msgstr "):] + translated_line = translated_line[1:len(translated_line) - 2] flag = 2 if line == "\n": if flag == 2: - if line1 != "": - if line1 in langFile and line2 != "": - langFile[line1] = line2.replace("\\\"", "\""); + if original_line != "": + if original_line in lang_file and translated_line != "": + lang_file[original_line] = translated_line.replace("\\\"", "\"") flag = 0 - idx = line.find ("\"") - if idx == 0: + quotation_in_line = line.find("\"") + if quotation_in_line == 0: line = line[1:len(line) - 2] if flag == 1: - line1 = line1 + line + original_line = original_line + line elif flag == 2: - line2 = line2 + line - - - -def addMissingLines(): - for trans in langFiles: - newFile = langFiles[trans][0] - oldFile = oldLangFiles[trans][0] - for str in newFile: - if str not in allStrings: -# if newFile[str] != "": - oldFile[str] = newFile[str] - for str in oldFile: - if str in newFile: - del newFile[str] -# print trans + ":moved to old: " + str - - for str in allStrings: - for trans in langFiles: - newFile = langFiles[trans][0] - oldFile = oldLangFiles[trans][0] - if str not in newFile: -# print "lang: " + trans + ", str: " + str - if trans == defaultLang: -# print "newFile[str] = str" - newFile[str] = str - elif str in oldFile: - newFile[str] = oldFile[str] -# print "newFile[str] = oldFile[str]: " + newFile[str] - else: -# print "newFile[str] = """ - newFile[str] = "" -# print trans + ":new string: " + str + translated_line = translated_line + line def sorting(): - for trans in langFiles: - newFile = langFiles[trans] - newFile = sortDict (newFile) - langFiles[trans] = newFile - oldFile = oldLangFiles[trans] - oldFile = sortDict (oldFile) - oldLangFiles[trans] = oldFile - -def sortDict(adict): - d2 = [] - keys = adict[0].keys() - keys.sort() + for translation in lang_files: + new_file = lang_files[translation] + new_file = sort_dict(new_file) + lang_files[translation] = new_file + old_file = old_lang_files[translation] + old_file = sort_dict(old_file) + old_lang_files[translation] = old_file + + +def sort_dict(dictionary): + sorted_dictionary = [] + keys = dictionary[0].keys() + sorted(keys) for key in keys: - d2.append ((key, adict[0][key])) - return (d2, adict[1]) + sorted_dictionary.append((key, dictionary[0][key])) + return (sorted_dictionary, dictionary[1]) + -def saveFiles(langDir, poDir): - for trans in langFiles: - writeLangFile (langDir + "/lang_" + trans + ".txt", langFiles[trans], trans, False) - writeLangFile (langDir + "/lang_" + trans + ".old", oldLangFiles[trans], trans, True) - writePoFile (poDir, langFiles[trans], trans) +def save_files(lang_dir, po_dir): + for trans in lang_files: + write_lang_file(lang_dir + "/lang_" + trans + ".txt", lang_files[trans], trans, False) + write_lang_file(lang_dir + "/lang_" + trans + ".old", old_lang_files[trans], trans, True) + write_po_file(po_dir, lang_files[trans], trans) -def writeLangFile(langDir, texts, trans, isold): - with open (langDir, "w") as f: + +def write_lang_file(lang_dir, texts, trans, isold): + with open(lang_dir, "w", encoding="UTF-8") as f: if texts[1] is not None: f.write(texts[1]) for line in texts[0]: if not isold or (line[1] is not None and len(line[1]) > 0): - f.write (line[0] + "\n") + f.write(line[0] + "\n") trLine = line[1] if trans == "en": if len(trLine) > 2 and (trLine[-2:] == "#0" or trLine[-2:] == "#1"): trLine = trLine[:-2] - f.write (trLine + "\n\n") + f.write(trLine + "\n\n") + -def writePoComments(w, comments): +def write_po_comments(w, comments): for line in comments: if line[:3] == "#. ": - w.write (line) + w.write(line) for line in comments: if line[:3] == "#: ": - w.write (line) + w.write(line) for line in comments: if line[:3] == "#, ": - w.write (line) + w.write(line) break -def writePoFile(poDir, texts, trans): + +def write_po_file(po_dir, texts, trans): if trans == "en": - langDir = poDir + "/" + trans + ".pot" + lang_dir = po_dir + "/" + trans + ".pot" else: - langDir = poDir + "/" + trans + ".po" - - print langDir - with open (langDir, "w") as w: - w.write ("# " + texts[1] + "") - w.write ("#\n\n") - w.write ("msgid \"\"\n") - w.write ("msgstr \"\"\n") - w.write ("\"Project-Id-Version: EvolOnline\\n\"\n") - w.write ("\"MIME-Version: 1.0\\n\"\n") - w.write ("\"Content-Type: text/plain; charset=UTF-8\\n\"\n") - w.write ("\"Content-Transfer-Encoding: 8bit\\n\"\n") - w.write ("\n") + lang_dir = po_dir + "/" + trans + ".po" + + print(f'Writing {lang_dir}') + with open(lang_dir, "w", encoding="UTF-8") as w: + w.write("# " + texts[1] + "") + w.write("#\n\n") + w.write("msgid \"\"\n") + w.write("msgstr \"\"\n") + w.write("\"Project-Id-Version: EvolOnline\\n\"\n") + w.write("\"MIME-Version: 1.0\\n\"\n") + w.write("\"Content-Type: text/plain; charset=UTF-8\\n\"\n") + w.write("\"Content-Transfer-Encoding: 8bit\\n\"\n") + w.write("\n") for line in texts[0]: - if line[0] in strComments.keys(): - writePoComments(w, strComments[line[0]]) + if line[0] in string_comments.keys(): + write_po_comments(w, string_comments[line[0]]) srcLine = line[0] - srcLine = srcLine.replace("\\", "\\\\"); + srcLine = srcLine.replace("\\", "\\\\") srcLine = srcLine.replace("\"", "\\\"") - w.write ("msgid \"" + srcLine + "\"\n") + w.write("msgid \"" + srcLine + "\"\n") trLine = line[1] if trans == "en": trLine = "" - trLine = trLine.replace("\\", "\\\\"); + trLine = trLine.replace("\\", "\\\\") trLine = trLine.replace("\"", "\\\"") - w.write ("msgstr \"" + trLine + "\"\n\n") - -def stripQuotes(data): - if len(data) == 0: - return data - if data[-1] == "\"": - data = data[:-1] - if data[0] == "\"": - data = data[1:] - if data[-1] == "'": - data = data[:-1] - if data[0] == "'": - data = data[1:] - return data - -def loadItemDb(dir): - global itemNamesByName - with open(dir + "/item_db.conf", "r") as f: - cnt = -1 - for line in f: - line = line.strip() + w.write("msgstr \"" + trLine + "\"\n\n") - idx = line.find("Name: \"") - if idx != 0 or line[len(line) - 1] != "\"": - continue - addStr(line[idx + 7: len(line) - 1], line, "item_db.conf", cnt, True) - if line == "{": - itemId = "" - itemName = "" - for line in f: - line = line.strip() - if len(line) < 1: - continue - if line[0] == "}": - if itemId != "" and itemName != "": - itemNamesByName[itemName.lower()] = itemName - break - if len(line) > 6 and line[:5] == "Name:": - itemName = stripQuotes(line[5:].strip()) - if len(line) > 4 and line[:3] == "Id:": - itemId = stripQuotes(line[3:].strip()) - -def dumpTranslations(): - for trans in oldLangFiles: - print "old lang: " + trans - newFile = oldLangFiles[trans][0] - for line in newFile: - print line - if line in newFile: - print newFile[line] - print "\n" - for trans in langFiles: - print "new lang: " + trans - newFile = langFiles[trans][0] - for line in newFile: - print line - if line in newFile: - print newFile[line] - print "\n" - - -def loadMobNames(dir): - with open(dir + "/mob_db.conf", "r") as r: - cnt = -1 - for line in r: - cnt = cnt + 1 - if len(line) < 1 or line[0:2] == "//": - continue - line = line.strip() - idx = line.find("Name: \"") - if idx != 0 or line[len(line) - 1] != "\"": - continue - addStr(line[idx + 7: len(line) - 1], line, "mob_db.conf", cnt, True) - -rootPath = "../../server-data/" - -loadItemDb(rootPath + "db/re") -loadMobNames(rootPath + "db/re") -collectScriptStrings(rootPath + "/npc", "npc") -collectScriptFileStrings("db/re/item_db.conf", rootPath + "/db/re/item_db.conf") -#collectMessages(rootPath + "/conf/messages.conf") -loadFiles(rootPath + "/langs") -addMissingLines() -loadPoFiles("in"); -#dumpTranslations(); + +def dump_translations(): + for trans in old_lang_files: + print("old lang: " + trans) + new_file = old_lang_files[trans][0] + for line in new_file: + print(line) + if line in new_file: + print(new_file[line]) + print("\n") + for trans in lang_files: + print("new lang: " + trans) + new_file = lang_files[trans][0] + for line in new_file: + print(line) + if line in new_file: + print(new_file[line]) + print("\n") + + +print("Collecting item names") +load_item_db(db_path) + +print("Collecting mob names") +load_mob_names(db_path) + +print("Collecting script strings") +collect_script_strings(script_path) + +print(f"Collecting item strings from {item_db_path}") +collect_script_file_strings(item_db_path) + +# print("Collecting messages") +# collect_messages(server_data_path + "/conf/messages.conf") + +print("Loading lang files") +load_lang_files() + +print("Adding missing lines") +add_missing_lines() + +print("Loading .po files") +load_po_files("in") + +# print("Dumping translations") +# dump_translations() + +print("Sorting strings") sorting() -saveFiles(rootPath + "/langs", "out") + +print("Saving files") +save_files(server_data_path + "/langs", "out") -- cgit v1.2.3-70-g09d2