Include the logmaster, which improves performance and reduces lag

1 files changed, 208 insertions, 0 deletions

diff --git a/logmaster.py b/logmaster.py
new file mode 100755
index 000000000..97941a8a4
--- /dev/null
+++ b/logmaster.py
@@ -0,0 +1,208 @@
+#!/usr/bin/python3
+########################################################################################
+#     This file is part of Moubootaur Legends API.
+#     Copyright (C) 2019-2022  Jesusalva
+
+#     This library is free software; you can redistribute it and/or
+#     modify it under the terms of the GNU Lesser General Public
+#     License as published by the Free Software Foundation; either
+#     version 2.1 of the License, or (at your option) any later version.
+
+#     This library is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#     Lesser General Public License for more details.
+
+#     You should have received a copy of the GNU Lesser General Public
+#     License along with this library; if not, write to the Free Software
+#     Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+########################################################################################
+# This is the log master, to improve performance
+
+import mysql.connector, signal, sys, threading, time, traceback
+
+## Default values
+HOST="127.0.0.1"; PORT=0; USER=""; PASS=""; DBXT=""; db=None;
+sqli = []; running=True
+SQL_PINGTIME=300.0; SQL_FLUSH=3.0
+
+## Warnings
+ERR=0
+WRN=1
+DBG=2
+def stdout(mes, code=DBG):
+    if code == ERR:
+        color="31;1"
+        title="ERROR"
+    elif code == WRN:
+        color="33;1"
+        title="WARNING"
+    else:
+        color="32;1"
+        title="INFO"
+    print("\033[%sm[%s]\033[0m %s" % (color, title, mes))
+    return
+
+## Rudimentary parser
+with open("conf/import/sql_connection.conf", "r") as f:
+    for l in f:
+        r=l.replace("\"", "").replace(" ", "").replace("\t", "").replace("\n", "").replace("\r", "").split(":")
+        try:
+            if r[0] == "db_hostname":
+                HOST=str(r[1])
+            elif r[0] == "db_port":
+                PORT=int(r[1])
+            elif r[0] == "db_username":
+                USER=str(r[1])
+            elif r[0] == "db_password":
+                PASS=str(r[1])
+            elif r[0] == "db_database":
+                DBXT=str(r[1])
+            else:
+                pass
+        except:
+            traceback.print_exc()
+
+## Check for fails
+if USER == "":
+    stdout("Lacking user! Check conf/import/sql_connection.conf", ERR)
+    exit(1)
+if PASS == "":
+    stdout("Lacking password! Check conf/import/sql_connection.conf", ERR)
+    exit(1)
+if DBXT == "":
+    stdout("Lacking database! Check conf/import/sql_connection.conf", ERR)
+    exit(1)
+
+## Init the database
+def connect():
+    global db, HOST, USER, PASS, DBXT
+    stdout("Connecting to %s:%d (%s @ %s)" % (HOST, PORT, USER, DBXT))
+    db = mysql.connector.connect(
+          host=HOST,
+          port=str(PORT),
+          user=USER,
+          passwd=PASS,
+          database=DBXT
+        )
+    return
+
+## Function to keep a database alive
+def keep_alive():
+    global db
+    try:
+        db.ping(reconnect=True, attempts=10, delay=1)
+    except:
+        # SQL error
+        stdout("keep_alive: INTERNAL ERROR (ping timeout!)", ERR)
+        db.reconnect(attempts=12, delay=10)
+    return
+
+## Start keep_alive in a thread and keep it running
+def keep_alive_runner():
+    global db
+    sqlt_db1=threading.Thread(target=keep_alive, daemon=True)
+    sqlt_db1.start()
+    ####################################################
+    ## Run forever
+    time.sleep(0.05)
+    stall=0.05
+    while sqlt_db1.is_alive():
+        stdout("keep_alive: Waiting for ping")
+        time.sleep(2.0)
+        stall+=2.0
+        ## Rebuild connection if the stall time exceeds the ping time
+        if stall > SQL_PINGTIME:
+            if sqlt_db1.is_alive():
+                stdout("keep_alive: DBXT Connection Restarted", WRN)
+                connect()
+            break
+    sql_keep_alive=threading.Timer(max(1.0, SQL_PINGTIME-stall), keep_alive_runner)
+    sql_keep_alive.daemon=True
+    sql_keep_alive.start()
+    return
+
+## Read stdin for as long as possible
+def run_forever():
+    global sqli, running
+    while running:
+        bf = sys.stdin.readline()
+        if bf is None or bf == "":
+            continue
+        #stdout("Buffer set: %s" % str(bf))
+        sqli.append(str(bf).replace("\n","").replace("\r",""))
+    return
+
+## Handle term signals
+def EXIT_NOW(sn, frame):
+    global running, db
+    stdout("Exit Signal received!", ERR)
+    running = False
+    time.sleep(SQL_FLUSH)
+    db.close()
+    stdout("Terminated", WRN)
+    return
+
+## Try to close Database and finish safely
+#signal.signal(signal.SIGTERM, EXIT_NOW)
+signal.signal(signal.SIGABRT, EXIT_NOW)
+
+## Create the SQL connection and keep it alive
+time.sleep(1.0)
+connect()
+keep_alive_runner()
+
+## Watch for stdin
+runner=threading.Thread(target=run_forever, daemon=True)
+runner.start()
+
+## Handle the input
+stdout("Logmaster started", WRN)
+bf=""
+while running:
+    try:
+        ## We have stuff to push
+        if len(sqli) > 0:
+            w = db.cursor()
+            for com in list(sqli):
+                try:
+                    cmd=com.split("→")[0]
+                    args=com.replace("%s→" % cmd, "")
+
+                    ## Command: SQL
+                    ## Description: Prepares a SQL statement. No escapping.
+                    ## Supports "?1", "?2" etc. for use with SAD
+                    if cmd == "SQL":
+                        bf=str(args)
+                    ## Command: SAD
+                    ## Description: Replaces "?" with escaped data.
+                    elif cmd.startswith("SAD"):
+                        bf=bf.replace("?%s" % cmd.replace("SAD", ""), args.replace("\\", "\\\\").replace('"','\\"').replace("'", "\\'"))
+                    ## Command: SQLRUN
+                    ## Description: Executes the prepared SQL statement.
+                    ## Sanitization must be done using SAD commands.
+                    elif cmd == "SQLRUN":
+                        w.execute(bf)
+                        #stdout("Query OK: %s" % bf)
+                        bf=""
+                    ## TODO: Integration with the API
+                    else:
+                        stdout("Unrecognized command: %s" % cmd, ERR)
+                except:
+                    stdout("Statement failed: %s" % cmd, ERR)
+                    traceback.print_exc()
+                sqli.remove(com)
+            db.commit()
+            w.close()
+        ## No need to flush ALL the time
+        time.sleep(SQL_FLUSH)
+    except KeyboardInterrupt:
+        running=False
+        stdout("Shutdown in progress!")
+        break
+    except:
+        traceback.print_exc()
+
+db.close()
+stdout("Logmaster finished.")
+