New website translation system for @LawnCable

It should also automatically update Transifex.

Sorry, i18n (except english) will become purely machine-readable %%R

3 files changed, 159 insertions, 360 deletions

diff --git a/web/langs.txt b/web/langs.txt
index 4bb0883..cf68c14 100644
--- a/web/langs.txt
+++ b/web/langs.txt
@@ -1,3 +1,4 @@
 pt-BR
 fr
-pl
+de
+es
diff --git a/web/po/en.po b/web/po/en.po
index 71531bf..d3d6500 100644
--- a/web/po/en.po
+++ b/web/po/en.po
@@ -1,294 +1,82 @@
-# Copyright (C) 2018 TMW2
 #
-
 msgid ""
 msgstr ""
-"Project-Id-Version: TMW2\n"
+"Project-Id-Version: 1.0\n"
+"Report-Msgid-Bugs-To: dev@tmw2.org\n"
+"POT-Creation-Date: 2020-02-28 05:55-0300\n"
+"PO-Revision-Date: 2020-02-28 05:55-0300\n"
+"Last-Translator: TMW2 Team <dev@tmw2.org>\n"
 "MIME-Version: 1.0\n"
-"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#, no-c-format
-msgid "TMW-2: Monster World | Not Found"
-msgstr ""
-
-#, no-c-format
-msgid "        <h2>404 Not Found</h2>"
-msgstr ""
-
-#, no-c-format
-msgid "        The page you were looking for was not found."
-msgstr ""
-
-#, no-c-format
-msgid "TMW-2: Monster World | Download Page"
-msgstr ""
-
-#, no-c-format
-msgid "        <h2>Downloading the Game</h2>"
-msgstr ""
-
-#, no-c-format
-msgid "        <h3>Downloading the Client</h3>"
-msgstr ""
-
-#, no-c-format
-msgid "        The first step is to download <a href=https://manaplus.org rel=\"nofollow\">ManaPlus</a>,"
+msgid "a string for testing"
 msgstr ""
 
-#, no-c-format
-msgid "        our game client. If you download from a repository, ensure the version is"
+msgid "$s Online Players"
 msgstr ""
 
-#, no-c-format
-msgid "        1.8.4.14 or greater. Older versions may present bugs."
+msgid "Home"
 msgstr ""
 
-#, no-c-format
-msgid "        <h3>Adding the Server</h3>"
+msgid "Community"
 msgstr ""
 
-#, no-c-format
-msgid "        Once the game opens, a server list will show. Select “Add”. Insert a server"
+msgid "News"
 msgstr ""
 
-#, no-c-format
-msgid "        name and description. The important data is as follow:"
+msgid "Legal Notice"
 msgstr ""
 
-#, no-c-format
-msgid "        <b>Name:</b> TMW-2: The Monster World"
+msgid "Wiki & Info"
 msgstr ""
 
-#, no-c-format
-msgid "        <b>Address:</b> server.tmw2.org"
+msgid "Get involved"
 msgstr ""
 
-#, no-c-format
-msgid "        <b>Port:</b> 6901"
+msgid "Making Art"
 msgstr ""
 
-#, no-c-format
-msgid "        <b>Type:</b> Evol2"
+msgid "MapMaking"
 msgstr ""
 
-#, no-c-format
-msgid "        You'll need to register before playing the game. Please use a valid email"
+msgid "Programming"
 msgstr ""
 
-#, no-c-format
-msgid "        address as if we need to email your password back to you, we will do using"
+msgid "Game Content"
 msgstr ""
 
-#, no-c-format
-msgid "        that email."
+msgid "Music and SFX"
 msgstr ""
 
-#, no-c-format
-msgid "        <h3>Troubleshooting</h3>"
+msgid "Item Viewer"
 msgstr ""
 
-#, no-c-format
-msgid "        If you cannot connect, certify data entered is right. Server may also be at"
+msgid "Page not found"
 msgstr ""
 
-#, no-c-format
-msgid "        maintenance, but if you keep getting a “servers unavailable” message, or"
-msgstr ""
-
-#, no-c-format
-msgid "        is otherwise unable to connect, please contact us so we may fix this issue."
-msgstr ""
-
-#, no-c-format
-msgid "TMW-2: Monster World | Community"
-msgstr ""
-
-#, no-c-format
 msgid "Get in touch with the community!"
 msgstr ""
 
-#, no-c-format
-msgid "<p>"
-msgstr ""
-
-#, no-c-format
-msgid "  We have a great community that expands over many platforms with people from all over the world."
-msgstr ""
-
-#, no-c-format
-msgid "  If you have a question or you need help with something don't hesitate to ask us."
-msgstr ""
-
-#, no-c-format
-msgid "</p>"
-msgstr ""
-
-#, no-c-format
-msgid "  Come visit us, and bring (or make) friends!"
-msgstr ""
-
-#, no-c-format
-msgid "<h3>"
-msgstr ""
-
-#, no-c-format
-msgid "  Community:"
-msgstr ""
-
-#, no-c-format
-msgid "</h3>"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://discord.gg/J4gcaqM\" class=\"socialbutton\" rel=\"nofollow\"><i class=\"fab fa-discord fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  Discord"
-msgstr ""
-
-#, no-c-format
-msgid "</a>"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://tmw2.org/forums\" class=\"socialbutton\"><i class=\"fas fa-users fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  Forum"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"http://webchat.freenode.net/?channels=#tmw2\" class=\"socialbutton\" rel=\"nofollow\"><i class=\"fas fa-comments  fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  IRC Chat"
-msgstr ""
-
-#, no-c-format
-msgid "<br>"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://www.youtube.com/channel/UCp-aVxXfcQWFvQde6kGvYcg\" class=\"socialbutton\"><i class=\"fab fa-youtube fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  Current YouTube Channel"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://www.youtube.com/channel/UCiwW5dTmi2xMV_vJfj5Mq5Q\" class=\"socialbutton\"><i class=\"fab fa-youtube fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  New YouTube Channel"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://www.facebook.com/TheManaWorld2/\" class=\"socialbutton\"><i class=\"fab fa-facebook-square fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  Facebook"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"http://steamcommunity.com/groups/TheManaWorld2\" class=\"socialbutton\"><i class=\"fab fa-steam fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  Steam Group"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://twitter.com/TheManaWorld2\" class=\"socialbutton\"><i class=\"fab fa-twitter fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  Twitter"
-msgstr ""
-
-#, no-c-format
-msgid "<h3>Development:</h3>"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://gitlab.com/TMW2\" class=\"socialbutton\"><i class=\"fab fa-gitlab fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  Gitlab"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://trello.com/devtmw2\" class=\"socialbutton\"><i class=\"fab fa-trello fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  Trello"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://www.transifex.com/akaras/saulc-tmw-fork/\" class=\"socialbutton\"><i class=\"fas fa-globe fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  Translate TMW2"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://discord.gg/BQNTe68\" class=\"socialbutton\" rel=\"nofollow\"><i class=\"fab fa-discord fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "  Discord (dev channel)"
-msgstr ""
-
-#, no-c-format
-msgid "Support us:"
-msgstr ""
-
-#, no-c-format
-msgid "<a href=\"https://www.patreon.com/TMW2\" class=\"socialbutton\"><i class=\"fab fa-patreon fa-lg\"></i> "
-msgstr ""
-
-#, no-c-format
-msgid "Patreon"
-msgstr ""
-
-#, no-c-format
-msgid "  PS: You may also <a href=\"mailto:admin@tmw2.org\">email us</a> at any time!"
-msgstr ""
-
-#, no-c-format
-msgid "TMW-2: Monster World | Index Page"
+msgid ""
+"We have a great community that expands over many platforms with people from all over the world. If you have a question or you need help with something don't hesitate to ask us.\n"
+"Come visit us, and bring (or make) friends!"
 msgstr ""
 
-#, no-c-format
-msgid "        <h2>TMW-2: Monster Wars</h2>"
+msgid "Social Platforms"
 msgstr ""
 
-#, no-c-format
-msgid "        The Monster World: Monster Wars is an open source MMORPG project developed by the community and for the community, where contributors can fully express their creativity."
+msgid "Development"
 msgstr ""
 
-#, no-c-format
-msgid "        We believe that players should be rewarded for merit, and not for game time or luck."
+msgid "Support us"
 msgstr ""
 
-#, no-c-format
-msgid "        TMW2 runs in the ManaPlus client, freesoftware, and our game have its code database licensed under GPL 3 and resources (GFX,SFX,etc) typically under CC-BY-SA, although GPL content is also present."
+msgid "You want to contribute in a way not listed above?"
 msgstr ""
 
-#, no-c-format
-msgid "        In this game, the player is given the opportunity to permanently change the world and its story, affecting all players forever, mainly by events and quests."
+msgid "No problem!"
 msgstr ""
 
-#, no-c-format
-msgid "        The players are free to decide how they want the world to develop, and their actions change the storyline, besides always being able to join the Development Team."
+msgid "Reach out to us and tell us about it."
 msgstr ""
-
diff --git a/web/updatelang.py b/web/updatelang.py
index e7a367a..c2162c3 100755
--- a/web/updatelang.py
+++ b/web/updatelang.py
@@ -4,161 +4,171 @@
 # Copyright (C) 2018  TMW-2
 # Author: Jesusalva
 
-import os
+import os, copy, datetime
+import polib, yaml
 
 defaultLang = "en"
-rootPath = "../../web/"
+rootPath = "../../site/i18n/"
 
 langs=[]
 files={}
 originals={}
 tm=["","\n"]
 
-def invalidline(line, filx="none"):
-    return (line.startswith("include ") or
-            line.startswith("<?php") or
-            line.startswith("?>") or
-            "<div " in line or
-            "</div>" in line or
-            "iframe" in line or
-            "header" in filx or
-            (line.startswith("    <p>") and line.endswith("<p>\n")) or
-            (line.startswith("    </p>") and line.endswith("</p>\n")) or
-            (line.startswith("    <pre><code>") and line.endswith("<code>\n")) or
-            (line.startswith("    </code></pre>") and line.endswith("</pre>\n")) or
-            line.replace('\n','') == "")
-
-# Populates the basic arrays
-def populate():
+# Search for array[?]==search in an array of dicts
+# Returns the dictionary, or returns "ERROR"
+def dl_search(array, search):
+    for key in array.keys():
+        if array[key] == search:
+            return key
+    return "ERROR"
+
+# For headers
+def currentdate():
+    dt=datetime.datetime.now()
+
+    day=dt.timetuple()[2]
+    month=dt.timetuple()[1]
+    year=dt.timetuple()[0]
+
+    hour=dt.timetuple()[3]
+    minute=dt.timetuple()[4]
+    #second=5
+    #weekday=6
+    #yearday=7
+
+    return "%04d-%02d-%02d %02d:%02d-0300" % (year, month, day, hour, minute)
+
+
+def init():
+    global defaultLang, rootPath, langs, files, originals, tm
     # Populate langs
     o=open("langs.txt", "r")
     for i in o:
         langs.append(i.replace('\n',''))
     o.close()
 
-    # Populate files
-    for i in os.listdir(rootPath):
-        if i.endswith(".php.en"):
-            print("Valid file: "+str(i))
-            files[i]={}
-            for x in langs:
-                files[i][x]={}
-
-    #print(str(files))
-
+    # Create the original file
+    f=open(rootPath+defaultLang+".yml", "r")
+    originals=yaml.load(f, Loader=yaml.FullLoader) # Dictionary
+    f.close()
+
+    # We don't care with metadata
+    del originals["language_meta"]
+
+    # Create each language's template
+    for i in langs:
+        files[str(i)]=copy.copy(originals)
+
+    """
+    # Do the reading for each language
+    for i in langs:
+        f=open(rootpath+i+".yml", "r")
+        files[str(i)]=yaml.load(f, Loader=yaml.FullLoader) # Dictionary
+        f.close()
+    """
+
+# [OK] Returns name from language code
+def lgname(lg):
+    if lg == "en":
+        return "English"
+    elif lg == "pt-BR":
+        return "Português Brasileiro"
+    elif lg == "de":
+        return "Deustche"
+    elif lg == "fr":
+        return "François"
+    elif lg == "es":
+        return "Español"
+    else:
+        return "Unknown %s" % lg
+
+# [OK] Entry
+def poentry(org, ttl):
+    return polib.POEntry(msgid=org, msgstr=ttl)
 
 # Creates/Loads stuff
-def genreadPoFile():
-    for a in langs:
-        print("Reading translations for "+a)
-        for i in files:
-            readPoFile(i,a)
-
 def generatePoFiles():
     global tm
-    context=langs
+    context=copy.copy(langs)
     context.append('en')
-    for a in langs:
-        print("Updating po file for "+a)
-        xrv=open("po/"+a+".po", "w")
-        # Prints header
-        xrv.write('\
-# Copyright (C) 2018 TMW2\n\
-#\n\
-\n\
-msgid ""\n\
-msgstr ""\n\
-"Project-Id-Version: TMW2\\n"\n\
-"MIME-Version: 1.0\\n"\n\
-"Content-Type: text/plain; charset=UTF-8\\n"\n\
-"Content-Transfer-Encoding: 8bit\\n"\n\
-\n\
-')
-        xrv.close()
-
-        tm=[""]
-        for i in files:
-            writePoFile(i,a)
+    for lg in context:
+        print("Updating po file for "+lg)
+        po=polib.POFile()
+        po.metadata = {
+            'Project-Id-Version': '1.0',
+            'Report-Msgid-Bugs-To': 'dev@tmw2.org',
+            'POT-Creation-Date': currentdate(),
+            'PO-Revision-Date': currentdate(),
+            'Last-Translator': 'TMW2 Team <dev@tmw2.org>',
+            'MIME-Version': '1.0',
+            'Content-Type': 'text/plain; charset=utf-8',
+            'Content-Transfer-Encoding': '8bit',
+        }
+
+        for key, speech in originals.items():
+            if lg == "en":
+                po.append(poentry(speech, ""))
+            else:
+                po.append(poentry(speech, files[lg][key]))
+
+        po.save("po/%s.po" % lg)
     context.remove('en')
 
-def generateLocal():
-    for a in langs:
-        print("generating local file for "+a)
-        for i in files:
-            writeLocal(i,a)
 
 
 
-# Write po files. TODO: WARNING: Some translations are getting overriden!
-def writePoFile(nf,lg):
-    # Translation Memory, to prevent duplicates
-    global tm
 
-    a=open("po/"+lg+".po", "a")
-    f=open(rootPath+nf, "r")
+# [OK] Reads Po Files
+def readPoFile(lg):
+    try:
+        po=polib.pofile("po/"+lg+".po")
+    except:
+        a=open("po/"+lg+".po", "w")
+        a.close()
+        print("%s.po does not exist, not reading" % lg)
+        return
+
+    print("%s Progress: %d%%" % (lgname(lg), po.percent_translated()))
+
+    for entry in po:
+        print("%s = %s" % (entry.msgid, entry.msgstr))
 
-    for line in f:
-        if line in tm or invalidline(line, nf):
-            continue
+        if entry.msgstr != "":
+            dest=dl_search(files[lg], entry.msgid)
+            print("[+] %s" % dest)
+            files[lg][dest]=entry.msgstr
         else:
-            tm.append(line)
+            dest=dl_search(files[lg], entry.msgid)
+            print("IS EMPTY")
+            files[lg][dest]=originals[dest]
+
+# [OK] Save the new YML file
+def writeLocal(lg):
+    f=open(rootPath+lg+".yml", 'w')
+    f.write("# THIS FILE WAS GENERATED AUTOMATICALLY\n#EDITING IT WILL HAVE NO EFFECT\n\n")
+    files[lg]["language_meta"]={"code": lg, "name": lgname(lg)}
+    yaml.dump(files[lg], f)
+    f.close()
+
+
 
-        a.write("#, no-c-format\n")
-        a.write('msgid "%s"\n' % line.replace('\n', '').replace('"','\\"'))
-        try:
-            if files[nf][lg][line.replace('\n', '')] != line.replace('\n', '') and files[nf][lg][line.replace('\n', '')] != "":
-                a.write('msgstr "%s"\n' % files[nf][lg][line.replace('\n', '')].replace('\n','').replace('"','\\"'))
-            else:
-                a.write('msgstr "%s"\n' % "")
-        except KeyError:
-                a.write('msgstr "%s"\n' % "")
 
-        a.write('\n')
 
-    a.close()
 
 
-# Reads Po Files
-def readPoFile(nf, lg):
-    try:
-        a=open("po/"+lg+".po", "r")
-    except:
-        a=open("po/"+lg+".po", "w")
-        a.close()
-        a=open("po/"+lg+".po", "r")
-
-    ctx=""
-    for line in a:
-        if "msgid " in line:
-            ctx=line.replace('"\n','').replace('\\"','"').replace('msgid "', "")
-        if "msgstr " in line and ctx != "":
-            if line != 'msgstr ""\n':
-                files[nf][lg][ctx]=line.replace('"\n', '').replace('\\"','"').replace('msgstr "', "")
-            else:
-                files[nf][lg][ctx]=ctx
-            ctx=""
 
-    a.close()
 
-def writeLocal(nf, lg):
-    a=open(rootPath+nf[:-2]+lg, 'w')
-    b=open(rootPath+nf, 'r')
 
-    for line in b:
-        try:
-            a.write(files[nf][lg][line.replace('\n', '')])
-            a.write('\n')
-        except KeyError:
-            a.write(line)
 
-    b.close()
-    a.close()
+# Mainframe: setup
+init()
 
-# Mainframe: populate arrays
-populate()
+# Save translations to their YML files
+for lg in langs:
+    readPoFile(lg)
+    writeLocal(lg)
 
 # Mainframe: handle PO files
-genreadPoFile()
 generatePoFiles()
-generateLocal()