From be66e922f7fb48f32ce28d53137e8c9d6373992a Mon Sep 17 00:00:00 2001 From: Jesusaves Date: Fri, 6 May 2022 09:59:12 -0300 Subject: Include the logmaster, which improves performance and reduces lag --- logmaster.py | 208 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100755 logmaster.py (limited to 'logmaster.py') diff --git a/logmaster.py b/logmaster.py new file mode 100755 index 000000000..97941a8a4 --- /dev/null +++ b/logmaster.py @@ -0,0 +1,208 @@ +#!/usr/bin/python3 +######################################################################################## +# This file is part of Moubootaur Legends API. +# Copyright (C) 2019-2022 Jesusalva + +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. + +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. + +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +######################################################################################## +# This is the log master, to improve performance + +import mysql.connector, signal, sys, threading, time, traceback + +## Default values +HOST="127.0.0.1"; PORT=0; USER=""; PASS=""; DBXT=""; db=None; +sqli = []; running=True +SQL_PINGTIME=300.0; SQL_FLUSH=3.0 + +## Warnings +ERR=0 +WRN=1 +DBG=2 +def stdout(mes, code=DBG): + if code == ERR: + color="31;1" + title="ERROR" + elif code == WRN: + color="33;1" + title="WARNING" + else: + color="32;1" + title="INFO" + print("\033[%sm[%s]\033[0m %s" % (color, title, mes)) + return + +## Rudimentary parser +with open("conf/import/sql_connection.conf", "r") as f: + for l in f: + r=l.replace("\"", "").replace(" ", "").replace("\t", "").replace("\n", "").replace("\r", "").split(":") + try: + if r[0] == "db_hostname": + HOST=str(r[1]) + elif r[0] == "db_port": + PORT=int(r[1]) + elif r[0] == "db_username": + USER=str(r[1]) + elif r[0] == "db_password": + PASS=str(r[1]) + elif r[0] == "db_database": + DBXT=str(r[1]) + else: + pass + except: + traceback.print_exc() + +## Check for fails +if USER == "": + stdout("Lacking user! Check conf/import/sql_connection.conf", ERR) + exit(1) +if PASS == "": + stdout("Lacking password! Check conf/import/sql_connection.conf", ERR) + exit(1) +if DBXT == "": + stdout("Lacking database! Check conf/import/sql_connection.conf", ERR) + exit(1) + +## Init the database +def connect(): + global db, HOST, USER, PASS, DBXT + stdout("Connecting to %s:%d (%s @ %s)" % (HOST, PORT, USER, DBXT)) + db = mysql.connector.connect( + host=HOST, + port=str(PORT), + user=USER, + passwd=PASS, + database=DBXT + ) + return + +## Function to keep a database alive +def keep_alive(): + global db + try: + db.ping(reconnect=True, attempts=10, delay=1) + except: + # SQL error + stdout("keep_alive: INTERNAL ERROR (ping timeout!)", ERR) + db.reconnect(attempts=12, delay=10) + return + +## Start keep_alive in a thread and keep it running +def keep_alive_runner(): + global db + sqlt_db1=threading.Thread(target=keep_alive, daemon=True) + sqlt_db1.start() + #################################################### + ## Run forever + time.sleep(0.05) + stall=0.05 + while sqlt_db1.is_alive(): + stdout("keep_alive: Waiting for ping") + time.sleep(2.0) + stall+=2.0 + ## Rebuild connection if the stall time exceeds the ping time + if stall > SQL_PINGTIME: + if sqlt_db1.is_alive(): + stdout("keep_alive: DBXT Connection Restarted", WRN) + connect() + break + sql_keep_alive=threading.Timer(max(1.0, SQL_PINGTIME-stall), keep_alive_runner) + sql_keep_alive.daemon=True + sql_keep_alive.start() + return + +## Read stdin for as long as possible +def run_forever(): + global sqli, running + while running: + bf = sys.stdin.readline() + if bf is None or bf == "": + continue + #stdout("Buffer set: %s" % str(bf)) + sqli.append(str(bf).replace("\n","").replace("\r","")) + return + +## Handle term signals +def EXIT_NOW(sn, frame): + global running, db + stdout("Exit Signal received!", ERR) + running = False + time.sleep(SQL_FLUSH) + db.close() + stdout("Terminated", WRN) + return + +## Try to close Database and finish safely +#signal.signal(signal.SIGTERM, EXIT_NOW) +signal.signal(signal.SIGABRT, EXIT_NOW) + +## Create the SQL connection and keep it alive +time.sleep(1.0) +connect() +keep_alive_runner() + +## Watch for stdin +runner=threading.Thread(target=run_forever, daemon=True) +runner.start() + +## Handle the input +stdout("Logmaster started", WRN) +bf="" +while running: + try: + ## We have stuff to push + if len(sqli) > 0: + w = db.cursor() + for com in list(sqli): + try: + cmd=com.split("ā†’")[0] + args=com.replace("%sā†’" % cmd, "") + + ## Command: SQL + ## Description: Prepares a SQL statement. No escapping. + ## Supports "?1", "?2" etc. for use with SAD + if cmd == "SQL": + bf=str(args) + ## Command: SAD + ## Description: Replaces "?" with escaped data. + elif cmd.startswith("SAD"): + bf=bf.replace("?%s" % cmd.replace("SAD", ""), args.replace("\\", "\\\\").replace('"','\\"').replace("'", "\\'")) + ## Command: SQLRUN + ## Description: Executes the prepared SQL statement. + ## Sanitization must be done using SAD commands. + elif cmd == "SQLRUN": + w.execute(bf) + #stdout("Query OK: %s" % bf) + bf="" + ## TODO: Integration with the API + else: + stdout("Unrecognized command: %s" % cmd, ERR) + except: + stdout("Statement failed: %s" % cmd, ERR) + traceback.print_exc() + sqli.remove(com) + db.commit() + w.close() + ## No need to flush ALL the time + time.sleep(SQL_FLUSH) + except KeyboardInterrupt: + running=False + stdout("Shutdown in progress!") + break + except: + traceback.print_exc() + +db.close() +stdout("Logmaster finished.") + -- cgit v1.2.3-60-g2f50