diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/HPMHookGen/HPMDataCheckGen.pl | 2 | ||||
| -rwxr-xr-x | tools/HPMHookGen/HPMHookGen.pl | 24 | ||||
| -rw-r--r-- | tools/HPMHookGen/doxygen.conf | 4 | ||||
| -rwxr-xr-x | tools/ci/retry.sh | 2 | ||||
| -rwxr-xr-x | tools/ci/travis.sh | 62 | ||||
| -rwxr-xr-x | tools/configconverter.pl | 1 | ||||
| -rwxr-xr-x | tools/mobdbconvall.sh | 2 | ||||
| -rwxr-xr-x | tools/mobdbconverter.py | 4 | ||||
| -rw-r--r-- | tools/mobskilldbconverter.py | 264 | ||||
| -rw-r--r-- | tools/petdbconverter.py | 214 | ||||
| -rw-r--r-- | tools/petevolutionconverter.py | 248 | ||||
| -rw-r--r-- | tools/skilldbconverter.php | 2 | ||||
| -rwxr-xr-x | tools/stackdump | 2 | ||||
| -rw-r--r-- | tools/utils/__init__.py | 0 | ||||
| -rw-r--r-- | tools/utils/common.py | 64 | ||||
| -rw-r--r-- | tools/utils/libconf.py | 693 |
16 files changed, 1555 insertions, 33 deletions
diff --git a/tools/HPMHookGen/HPMDataCheckGen.pl b/tools/HPMHookGen/HPMDataCheckGen.pl index e78a7bd93..f6e4dac24 100644 --- a/tools/HPMHookGen/HPMDataCheckGen.pl +++ b/tools/HPMHookGen/HPMDataCheckGen.pl @@ -3,7 +3,7 @@ # This file is part of Hercules. # http://herc.ws - http://github.com/HerculesWS/Hercules # -# Copyright (C) 2014-2016 Hercules Dev Team +# Copyright (C) 2014-2018 Hercules Dev Team # # Hercules is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/tools/HPMHookGen/HPMHookGen.pl b/tools/HPMHookGen/HPMHookGen.pl index b8835b376..e5a5c1914 100755 --- a/tools/HPMHookGen/HPMHookGen.pl +++ b/tools/HPMHookGen/HPMHookGen.pl @@ -3,7 +3,7 @@ # This file is part of Hercules. # http://herc.ws - http://github.com/HerculesWS/Hercules # -# Copyright (C) 2013-2016 Hercules Dev Team +# Copyright (C) 2013-2018 Hercules Dev Team # # Hercules is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -240,9 +240,9 @@ sub parse($$) { $rtinit = ' = BGQT_INVALID'; } elsif ($x =~ /^enum\s+parsefunc_rcode$/) { # Known enum parsefunc_rcode $rtinit = ' = PACKET_UNKNOWN'; - } elsif ($x =~ /^(?:enum\s+)?DBOptions$/) { # Known enum DBOptions + } elsif ($x =~ /^enum\s+DBOptions$/) { # Known enum DBOptions $rtinit = ' = DB_OPT_BASE'; - } elsif ($x =~ /^enum\s+thread_priority$/) { # Known enum DBOptions + } elsif ($x =~ /^enum\s+thread_priority$/) { # Known enum thread_priority $rtinit = ' = THREADPRIO_NORMAL'; } elsif ($x eq 'DBComparator' or $x eq 'DBHasher' or $x eq 'DBReleaser') { # DB function pointers $rtinit = ' = NULL'; @@ -255,6 +255,7 @@ sub parse($$) { or $x =~ /^u?int(?:8|16|32|64)$/ or $x eq 'defType' or $x eq 'size_t' + or $x eq 'time_t' ) { # Numeric variables $rtinit = ' = 0'; } else { # Anything else @@ -538,7 +539,8 @@ EOF next if $fileguards{$key}->{private}; print FH <<"EOF"; #ifdef $fileguards{$key}->{guard} /* $key */ -if ((server_type&($fileguards{$key}->{type})) && !HPM_SYMBOL("$exportsymbols{$key}", $key)) return "$exportsymbols{$key}"; + if ((server_type&($fileguards{$key}->{type})) != 0 && !HPM_SYMBOL("$exportsymbols{$key}", $key)) + return "$exportsymbols{$key}"; #endif // $fileguards{$key}->{guard} EOF } @@ -599,7 +601,7 @@ EOF EOF $idx += 2; - $maxlen = length($key."->".$if->{name}) if( length($key."->".$if->{name}) > $maxlen ); + $maxlen = length($key."->".$if->{name}) if (length($key."->".$if->{name}) > $maxlen); } } print FH <<"EOF"; @@ -619,7 +621,7 @@ EOF foreach my $key (@$keysref) { print FH <<"EOF"; -memcpy(&HPMHooks.source.$key, $key2pointer{$key}, sizeof(struct $key2original{$key})); +HPMHooks.source.$key = *$key2pointer{$key}; EOF } close FH; @@ -704,14 +706,14 @@ EOF print FH <<"EOF"; $if->{handlerdef} {$if->{notes} int hIndex = 0;${initialization} - if( HPMHooks.count.$if->{hname}_pre ) { + if (HPMHooks.count.$if->{hname}_pre > 0) { $if->{predef} *HPMforce_return = false; - for(hIndex = 0; hIndex < HPMHooks.count.$if->{hname}_pre; hIndex++ ) {$beforeblock3 + for (hIndex = 0; hIndex < HPMHooks.count.$if->{hname}_pre; hIndex++) {$beforeblock3 preHookFunc = HPMHooks.list.$if->{hname}_pre[hIndex].func; $if->{precall}$afterblock3 } - if( *HPMforce_return ) { + if (*HPMforce_return) { *HPMforce_return = false; return$retval; } @@ -719,9 +721,9 @@ $if->{handlerdef} {$if->{notes} {$beforeblock2 $if->{origcall}$afterblock2 } - if( HPMHooks.count.$if->{hname}_post ) { + if (HPMHooks.count.$if->{hname}_post > 0) { $if->{postdef} - for(hIndex = 0; hIndex < HPMHooks.count.$if->{hname}_post; hIndex++ ) {$beforeblock3 + for (hIndex = 0; hIndex < HPMHooks.count.$if->{hname}_post; hIndex++) {$beforeblock3 postHookFunc = HPMHooks.list.$if->{hname}_post[hIndex].func; $if->{postcall}$afterblock3 } diff --git a/tools/HPMHookGen/doxygen.conf b/tools/HPMHookGen/doxygen.conf index ec55967b1..c302f7f2f 100644 --- a/tools/HPMHookGen/doxygen.conf +++ b/tools/HPMHookGen/doxygen.conf @@ -269,7 +269,9 @@ INCLUDE_PATH = ../../src \ ../../3rdparty INCLUDE_FILE_PATTERNS = PREDEFINED = __attribute__(x)= \ - HPMHOOKGEN + HPMHOOKGEN \ + PACKETVER=20031028 \ + PACKETVER_MAIN_NUM=20031028 EXPAND_AS_DEFINED = SKIP_FUNCTION_MACROS = NO #--------------------------------------------------------------------------- diff --git a/tools/ci/retry.sh b/tools/ci/retry.sh index 6e79af1d5..688f02d9a 100755 --- a/tools/ci/retry.sh +++ b/tools/ci/retry.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This file is part of Hercules. # http://herc.ws - http://github.com/HerculesWS/Hercules diff --git a/tools/ci/travis.sh b/tools/ci/travis.sh index 9a6322df6..fa7d5be93 100755 --- a/tools/ci/travis.sh +++ b/tools/ci/travis.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This file is part of Hercules. # http://herc.ws - http://github.com/HerculesWS/Hercules @@ -33,6 +33,7 @@ function usage { echo "usage:" echo " $0 createdb <dbname> [dbuser] [dbpassword] [dbhost]" echo " $0 importdb <dbname> [dbuser] [dbpassword] [dbhost]" + echo " $0 adduser <dbname> <new_user> <new_user_password> [dbuser] [dbpassword] [dbhost]" echo " $0 build [configure args]" echo " $0 test <dbname> [dbuser] [dbpassword] [dbhost]" echo " $0 getplugins" @@ -48,10 +49,10 @@ function run_server { echo "Running: $1 --run-once $2" $1 --run-once $2 2>runlog.txt export errcode=$? - export teststr=$(cat runlog.txt) + export teststr=$(head -c 10000 runlog.txt) if [[ -n "${teststr}" ]]; then echo "Errors found in running server $1." - cat runlog.txt + head -c 10000 runlog.txt aborterror "Errors found in running server $1." else echo "No errors found for server $1." @@ -66,10 +67,10 @@ function run_test { echo "Running: test_$1" ./test_$1 2>runlog.txt export errcode=$? - export teststr=$(cat runlog.txt) + export teststr=$(head -c 10000 runlog.txt) if [[ -n "${teststr}" ]]; then echo "Errors found in running test $1." - cat runlog.txt + head -c 10000 runlog.txt aborterror "Errors found in running test $1." else echo "No errors found for test $1." @@ -93,29 +94,53 @@ case "$MODE" in fi DBNAME="$1" if [ -n "$2" ]; then - DBUSER_ARG="-u $2" + DBUSER_ARG="--user=$2" DBUSER="$2" fi if [ -n "$3" ]; then - DBPASS_ARG="-p$3" + DBPASS_ARG="--password=$3" DBPASS="$3" fi if [ -n "$4" ]; then - DBHOST_ARG="-h $4" + DBHOST_ARG="--host=$4" DBHOST="$4" fi ;; + adduser) + if [ -z "$3" ]; then + usage + fi + DBNAME="$1" + NEWUSER="$2" + NEWPASS="$3" + if [ -n "$4" ]; then + DBUSER_ARG="--user=$4" + DBUSER="$4" + fi + if [ -n "$5" ]; then + DBPASS_ARG="--password=$5" + DBPASS="$5" + fi + if [ -n "$6" ]; then + DBHOST_ARG="--host=$6" + DBHOST="$6" + fi + ;; esac case "$MODE" in createdb) - echo "Creating database $DBNAME..." - mysql $DBUSER_ARG $DBPASS_ARG $DBHOST_ARG -e "create database $DBNAME;" || aborterror "Unable to create database." + echo "Creating database $DBNAME as $DBUSER..." + mysql $DBUSER_ARG $DBPASS_ARG $DBHOST_ARG --execute="CREATE DATABASE $DBNAME;" || aborterror "Unable to create database." ;; importdb) - echo "Importing tables into $DBNAME..." - mysql $DBUSER_ARG $DBPASS_ARG $DBHOST_ARG $DBNAME < sql-files/main.sql || aborterror "Unable to import main database." - mysql $DBUSER_ARG $DBPASS_ARG $DBHOST_ARG $DBNAME < sql-files/logs.sql || aborterror "Unable to import logs database." + echo "Importing tables into $DBNAME as $DBUSER..." + mysql $DBUSER_ARG $DBPASS_ARG $DBHOST_ARG --database=$DBNAME < sql-files/main.sql || aborterror "Unable to import main database." + mysql $DBUSER_ARG $DBPASS_ARG $DBHOST_ARG --database=$DBNAME < sql-files/logs.sql || aborterror "Unable to import logs database." + ;; + adduser) + echo "Adding user $NEWUSER as $DBUSER, with access to database $DBNAME..." + mysql $DBUSER_ARG $DBPASS_ARG $DBHOST_ARG --execute="GRANT SELECT,INSERT,UPDATE,DELETE ON $DBNAME.* TO '$NEWUSER'@'$DBHOST' IDENTIFIED BY '$NEWPASS';" ;; build) (cd tools && ./validateinterfaces.py silent) || aborterror "Interface validation error." @@ -125,6 +150,11 @@ case "$MODE" in make plugin.script_mapquit -j3 || aborterror "Build failed." make test || aborterror "Build failed." ;; + buildhpm) + ./configure $@ || (cat config.log && aborterror "Configure error, aborting build.") + cd tools/HPMHookGen + make + ;; test) cat > conf/travis_sql_connection.conf << EOF sql_connection: { @@ -173,6 +203,12 @@ EOF ARGS="--load-plugin script_mapquit $ARGS --load-script npc/dev/ci_test.txt" PLUGINS="--load-plugin HPMHooking --load-plugin sample" echo "run tests" + if [[ $DBUSER == "travis" ]]; then + echo "Disable leak dection on travis" + export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=true:strict_init_order=true + else + export ASAN_OPTIONS=detect_stack_use_after_return=true:strict_init_order=true + fi # run_test spinlock # Not running the spinlock test for the time being (too time consuming) run_test libconfig echo "run all servers without HPM" diff --git a/tools/configconverter.pl b/tools/configconverter.pl index 4fafd1f64..dc511aaef 100755 --- a/tools/configconverter.pl +++ b/tools/configconverter.pl @@ -677,7 +677,6 @@ my @defaults = ( drops_by_luk => {parse => \&parsecfg_int, print => \&printcfg_int, path => "drops:", default => 0}, drops_by_luk2 => {parse => \&parsecfg_int, print => \&printcfg_int, path => "drops:", default => 0}, alchemist_summon_reward => {parse => \&parsecfg_int, print => \&printcfg_int, path => "drops:", default => 1}, - rare_drop_announce => {parse => \&parsecfg_int, print => \&printcfg_int, path => "drops:", default => 0}, base_exp_rate => {parse => \&parsecfg_int, print => \&printcfg_int, path => "exp:", default => 100}, job_exp_rate => {parse => \&parsecfg_int, print => \&printcfg_int, path => "exp:", default => 100}, multi_level_up => {parse => \&parsecfg_bool, print => \&printcfg_bool, path => "exp:", default => "false"}, diff --git a/tools/mobdbconvall.sh b/tools/mobdbconvall.sh index a6f421329..45eb8c38f 100755 --- a/tools/mobdbconvall.sh +++ b/tools/mobdbconvall.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # This file is part of Hercules. # http://herc.ws - http://github.com/HerculesWS/Hercules diff --git a/tools/mobdbconverter.py b/tools/mobdbconverter.py index 78047aed9..683e28274 100755 --- a/tools/mobdbconverter.py +++ b/tools/mobdbconverter.py @@ -188,7 +188,7 @@ def convertFile(inFile, itemDb): printField("ChaseRange", fields[21]) printField("Size", fields[22]) printField("Race", fields[23]) - print("\tElement: ({0}, {1})".format(int(fields[24]) % 10, int(fields[24]) / 20)); + print("\tElement: ({0}, {1})".format(int(fields[24]) % 10, int(int(fields[24]) / 20))); mode = int(fields[25], 0) if mode != 0: startGroup("Mode") @@ -260,7 +260,7 @@ def readItemDB(inFile, itemDb): elif line[:3] == "Id:": try: itemId = int(line[4:]) - except: + except ValueError: started = False if itemId != 0 and itemName != "": # was need for remove wrong characters diff --git a/tools/mobskilldbconverter.py b/tools/mobskilldbconverter.py new file mode 100644 index 000000000..4ba042062 --- /dev/null +++ b/tools/mobskilldbconverter.py @@ -0,0 +1,264 @@ +#!/usr/bin/python +# -*- coding: utf8 -*- +# +# This file is part of Hercules. +# http://herc.ws - http://github.com/HerculesWS/Hercules +# +# Copyright (C) 2018 Hercules Dev Team +# Copyright (C) 2018 Asheraf +# +# Hercules is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import re +import sys +import utils.common as Tools + +SKILL_STATES = { + "any": "MSS_ANY", + "idle": "MSS_IDLE", + "walk": "MSS_WALK", + "loot": "MSS_LOOT", + "dead": "MSS_DEAD", + "attack": "MSS_BERSERK", + "angry": "MSS_ANGRY", + "chase": "MSS_RUSH", + "follow": "MSS_FOLLOW", + "anytarget": "MSS_ANYTARGET" +} +SKILL_COND1 = { + "always": "MSC_ALWAYS", + "myhpltmaxrate": "MSC_MYHPLTMAXRATE", + "myhpinrate": "MSC_MYHPINRATE", + "friendhpltmaxrate": "MSC_FRIENDHPLTMAXRATE", + "friendhpinrate": "MSC_FRIENDHPINRATE", + "mystatuson": "MSC_MYSTATUSON", + "mystatusoff": "MSC_MYSTATUSOFF", + "friendstatuson": "MSC_FRIENDSTATUSON", + "friendstatusoff": "MSC_FRIENDSTATUSOFF", + "attackpcgt": "MSC_ATTACKPCGT", + "attackpcge": "MSC_ATTACKPCGE", + "slavelt": "MSC_SLAVELT", + "slavele": "MSC_SLAVELE", + "closedattacked": "MSC_CLOSEDATTACKED", + "longrangeattacked": "MSC_LONGRANGEATTACKED", + "skillused": "MSC_SKILLUSED", + "afterskill": "MSC_AFTERSKILL", + "casttargeted": "MSC_CASTTARGETED", + "rudeattacked": "MSC_RUDEATTACKED", + "masterhpltmaxrate": "MSC_MASTERHPLTMAXRATE", + "masterattacked": "MSC_MASTERATTACKED", + "alchemist": "MSC_ALCHEMIST", + "onspawn": "MSC_SPAWN" +} +SKILL_COND2 = { + "anybad": "MSC_ANY", + "stone": "SC_STONE", + "freeze": "SC_FREEZE", + "stun": "SC_STUN", + "sleep": "SC_SLEEP", + "poison": "SC_POISON", + "curse": "SC_CURSE", + "silence": "SC_SILENCE", + "confusion": "SC_CONFUSION", + "blind": "SC_BLIND", + "hiding": "SC_HIDING", + "sight": "SC_SIGHT" +} +SKILL_TARGET = { + "target": "MST_TARGET", + "randomtarget": "MST_RANDOM", + "self": "MST_SELF", + "friend": "MST_FRIEND", + "master": "MST_MASTER", + "around5": "MST_AROUND5", + "around6": "MST_AROUND6", + "around7": "MST_AROUND7", + "around8": "MST_AROUND8", + "around1": "MST_AROUND1", + "around2": "MST_AROUND2", + "around3": "MST_AROUND3", + "around4": "MST_AROUND4", + "around": "MST_AROUND" +} + +def printHeader(): + print(""" +mob_skill_db:( +{ +/************************************************************************** + ************* Entry structure ******************************************** + ************************************************************************** + <Monster_Constant>: { + <Skill_Constant>: { + ClearSkills: (boolean, defaults to false) allows cleaning all previous defined skills for the mob. + SkillLevel: (int, defaults to 1) + SkillState: (int, defaults to 0) + SkillTarget: (int, defaults to 0) + Rate: (int, defaults to 1) + CastTime: (int, defaults to 0) + Delay: (int, defaults to 0) + Cancelable: (boolean, defaults to false) + CastCondition: (int, defaults to 0) + ConditionData: (int, defaults to 0) + val0: (int, defaults to 0) + val1: (int, defaults to 0) + val2: (int, defaults to 0) + val3: (int, defaults to 0) + val4: (int, defaults to 0) + Emotion: (int, defaults to 0) + ChatMsgID: (int, defaults to 0) + } + } +**************************************************************************/""") + +def printFooter(): + print('}\n)\n') + +def isValidEntry(line): + if re.match('^[0-9]+,.*', line): + return True + return False + +def commaSplit(line): + return line.split(',') + +def stripLinebreak(line): + return line.replace('\r', '').replace('\n', '') + +def printInt(key, value): + if key in value: + if int(value[key]) is not 0: + print('\t\t\t{}: {}'.format(key, value[key])) + +def printStrToInt(key, value): + if value[key] is not '': + if int(value[key]) is not 0: + print('\t\t\t{}: {}'.format(key, value[key])) + +def printBool(key, value): + if value[key] == 'yes': + print('\t\t\t{}: true'.format(key)) + +def printClearSkills(key, value): + if value[key] == 'clear': + print('\t\t\t{}: true'.format(key)) + +def printSkillState(key, value): + if value[key]: + print('\t\t\t{}: "{}"'.format(key, SKILL_STATES[value[key]])) + +def printSkillTarget(key, value): + if value[key]: + print('\t\t\t{}: "{}"'.format(key, SKILL_TARGET[value[key]])) + +def printCastCondition(key, value): + if value[key]: + print('\t\t\t{}: "{}"'.format(key, SKILL_COND1[value[key]])) + +def printConditionData(key, value): + if value[key] in SKILL_COND2: + print('\t\t\t{}: "{}"'.format(key, SKILL_COND2[value[key]])) + elif value[key] is not '': + if int(value[key]) is not 0: + print('\t\t\t{}: {}'.format(key, value[key])) + +def printEmotion(key, value): + if value[key] is not '': + print('\t\t\t{}: {}'.format(key, value[key])) + +def LoadOldDB(mode, serverpath): + + r = open('{}db/{}/mob_skill_db.txt'.format(serverpath, mode), "r") + + Db = dict() + for line in r: + if isValidEntry(line) == True: + entry = commaSplit(stripLinebreak(line)) + MonsterId = entry[0] + if MonsterId not in Db: + Db[MonsterId] = dict() + skillidx = len(Db[MonsterId]) + Db[MonsterId][skillidx] = dict() + Db[MonsterId][skillidx]['ClearSkills'] = entry[1] + Db[MonsterId][skillidx]['SkillState'] = entry[2] + Db[MonsterId][skillidx]['SkillId'] = entry[3] + Db[MonsterId][skillidx]['SkillLevel'] = entry[4] + Db[MonsterId][skillidx]['Rate'] = entry[5] + Db[MonsterId][skillidx]['CastTime'] = entry[6] + Db[MonsterId][skillidx]['Delay'] = entry[7] + Db[MonsterId][skillidx]['Cancelable'] = entry[8] + Db[MonsterId][skillidx]['SkillTarget'] = entry[9] + Db[MonsterId][skillidx]['CastCondition'] = entry[10] + Db[MonsterId][skillidx]['ConditionData'] = entry[11] + for i in range(5): + if entry[12 + i] is '': + continue + try: + Db[MonsterId][skillidx]['val{}'.format(i)] = int(entry[12 + i]) + except ValueError: + Db[MonsterId][skillidx]['val{}'.format(i)] = int(entry[12 + i], 16) + Db[MonsterId][skillidx]['Emotion'] = entry[17] + Db[MonsterId][skillidx]['ChatMsgID'] = entry[18] + return Db + +def ConvertDB(mode, serverpath): + db = LoadOldDB(mode, serverpath) + MobDB = Tools.LoadDBConsts('mob_db', mode, serverpath) + SkillDB = Tools.LoadDBConsts('skill_db', mode, serverpath) + + printHeader() + for mobid in sorted(db.iterkeys()): + print('\t{}: {{'.format(MobDB[int(mobid)])) + for skillidx in sorted(db[mobid].iterkeys()): + valid = True + if int(db[mobid][skillidx]['SkillId']) not in SkillDB: + valid = False + print('/*') + print('// Can\'t find skill with id {} in skill_db'.format(db[mobid][skillidx]['SkillId'])) + print('\t\t{}: {{'.format(db[mobid][skillidx]['SkillId'])) + else: + print('\t\t{}: {{'.format(SkillDB[int(db[mobid][skillidx]['SkillId'])])) + printClearSkills('ClearSkills', db[mobid][skillidx]) + printSkillState('SkillState', db[mobid][skillidx]) + printStrToInt('SkillLevel', db[mobid][skillidx]) + printStrToInt('Rate', db[mobid][skillidx]) + printStrToInt('CastTime', db[mobid][skillidx]) + printStrToInt('Delay', db[mobid][skillidx]) + printBool('Cancelable', db[mobid][skillidx]) + printSkillTarget('SkillTarget', db[mobid][skillidx]) + printCastCondition('CastCondition', db[mobid][skillidx]) + printConditionData('ConditionData', db[mobid][skillidx]) + for i in range(5): + printInt('val{}'.format(i), db[mobid][skillidx]) + printEmotion('Emotion', db[mobid][skillidx]) + printStrToInt('ChatMsgID', db[mobid][skillidx]) + print('\t\t}') + if valid is False: + print('*/') + print('\t}') + printFooter() + +if len(sys.argv) != 3: + print('Monster Skill db converter from txt to conf format') + print('Usage:') + print(' mobskilldbconverter.py mode serverpath') + print("example:") + print(' mobskilldbconverter.py pre-re ../') + exit(1) + +if sys.argv[1] != 're' and sys.argv[1] != 'pre-re': + print('you have entred an invalid server mode') + exit(1) + +ConvertDB(sys.argv[1], sys.argv[2]) diff --git a/tools/petdbconverter.py b/tools/petdbconverter.py new file mode 100644 index 000000000..1b7d2e4d6 --- /dev/null +++ b/tools/petdbconverter.py @@ -0,0 +1,214 @@ +#! /usr/bin/env python +# -*- coding: utf8 -*- +# +# This file is part of Hercules. +# http://herc.ws - http://github.com/HerculesWS/Hercules +# +# Copyright (C) 2018 Hercules Dev Team +# Copyright (C) 2018 Asheraf +# Copyright (C) 2015 Andrei Karas (4144) +# +# Hercules is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import re +import sys + +def isValidEntry(line): + if re.match('^[0-9]+,.*', line): + return True + return False + +def curlSplit(line): + return re.split('[{}]|},', line) + +def commaSplit(line): + return line.split(',') + +def printIntField(name, value): + if int(value) != 0: + print('\t{0}: {1}'.format(name, value)) + +def printIntField2(name, value): + if int(value) != 0: + print('\t\t{0}: {1}'.format(name, value)) + +def printStrField(name, value): + if value != '': + print('\t{0}: \"{1}\"'.format(name, value)) + +def printBool(name, value): + if int(value) != 0: + print('\t{0}: true'.format(name)) + +def printIntimacy(arr): + if int(arr[9]) == 0 or int(arr[10]) == 0 or int(arr[11]) == 0 or int(arr[12]) == 0: + return + print('\tIntimacy: {') + printIntField2('Initial', arr[11]) + printIntField2('FeedIncrement', arr[9]) + printIntField2('OverFeedDecrement', arr[10]) + printIntField2('OwnerDeathDecrement', arr[12]) + print('\t}') + +def printScript(name, value): + if re.match('.*[a-zA-Z0-9,]+.*', value): + print('\t{0}: <\"{1}\">'.format(name, value)) + +def printItemName(fieldname, itemid, itemDb): + value = int(itemid) + if value != 0: + if value not in itemDb: + print("// Error: pet item with id {0} not found in item_db.conf".format(value)) + else: + printStrField(fieldname, itemDb[value]) + + +def printHeader(): + print(""" +pet_db:( +/************************************************************************** + ************* Entry structure ******************************************** + ************************************************************************** +{ + // ================ Mandatory fields ============================== + Id: ID (int) + SpriteName: "Sprite_Name" (string) + Name: "Pet Name" (string) + // ================ Optional fields =============================== + TamingItem: Taming Item (string, defaults to 0) + EggItem: Egg Id (string, defaults to 0) + AccessoryItem: Equipment Id (string, defaults to 0) + FoodItem: Food Id (string, defaults to 0) + FoodEffectiveness: hunger points (int, defaults to 0) + HungerDelay: hunger time (int, defaults to 0) + Intimacy: { + Initial: start intimacy (int, defaults to 0) + FeedIncrement: feeding intimacy (int, defaults to 0) + OverFeedDecrement: overfeeding intimacy (int, defaults to 0) + OwnerDeathDecrement: owner die intimacy (int, defaults to 0) + } + CaptureRate: capture rate (int, defaults to 0) + Speed: speed (int, defaults to 0) + SpecialPerformance: true/false (boolean, defaults to false) + TalkWithEmotes: convert talk (boolean, defaults to false) + AttackRate: attack rate (int, defaults to 0) + DefendRate: Defence attack (int, defaults to 0) + ChangeTargetRate: change target (int, defaults to 0) + PetScript: <" Pet Script (can also be multi-line) "> + EquipScript: <" Equip Script (can also be multi-line) "> +}, +**************************************************************************/ + """) + +def printFooter(): + print(')\n') + +def convertFile(inFile, itemDb): + if inFile != "" and not os.path.exists(inFile): + return + + if inFile == "": + r = sys.stdin + else: + r = open(inFile, "r") + + printHeader() + for line in r: + if isValidEntry(line) == True: + print('{') + firstsplit = curlSplit(line) + secondsplit = commaSplit(firstsplit[0]) + printIntField('Id', secondsplit[0]) + printStrField('SpriteName', secondsplit[1]) + printStrField('Name', secondsplit[2]) + printItemName('TamingItem', secondsplit[3], itemDb) + printItemName('EggItem', secondsplit[4], itemDb) + printItemName('AccessoryItem', secondsplit[5], itemDb) + printItemName('FoodItem', secondsplit[6], itemDb) + printIntField('FoodEffectiveness', secondsplit[7]) + printIntField('HungerDelay', secondsplit[8]) + printIntimacy(secondsplit) + printIntField('CaptureRate', secondsplit[13]) + printIntField('Speed', secondsplit[14]) + printBool('SpecialPerformance', secondsplit[15]) + printBool('TalkWithEmotes', secondsplit[16]) + printIntField('AttackRate', secondsplit[17]) + printIntField('DefendRate', secondsplit[18]) + printIntField('ChangeTargetRate', secondsplit[19]) + printScript('PetScript', firstsplit[1]) + printScript('EquipScript', firstsplit[3]) + print('},') + printFooter() + +def printHelp(): + print("PetDB converter from txt to conf format") + print("Usage:") + print(" petdbconverter.py re serverpath dbfilepath") + print(" petdbconverter.py pre-re serverpath dbfilepath") + print("Usage for read from stdin:") + print(" petdbconverter.py re dbfilepath") + +def readItemDB(inFile, itemDb): + itemId = 0 + itemName = "" + started = False + with open(inFile, "r") as r: + for line in r: + line = line.strip() + if started == True: + if line == "},": + started = False + elif line[:10] == "AegisName:": + itemName = line[12:-1] + elif line[:3] == "Id:": + try: + itemId = int(line[4:]) + except ValueError: + started = False + if itemId != 0 and itemName != "": +# was need for remove wrong characters +# itemName = itemName.replace(".", "") +# if itemName[0] >= "0" and itemName[0] <= "9": +# itemName = "Num" + itemName + itemDb[itemId] = itemName + started = False + else: + if line == "{": + started = True + itemId = 0 + itemName = "" + return itemDb + +if len(sys.argv) != 4 and len(sys.argv) != 3: + printHelp(); + exit(1) +startPath = sys.argv[2] +if len(sys.argv) == 4: + sourceFile = sys.argv[3] +else: + sourceFile = ""; + +itemDb = dict() +if sys.argv[1] == "re": + itemDb = readItemDB(startPath + "/db/re/item_db.conf", itemDb) + itemDb = readItemDB(startPath + "/db/item_db2.conf", itemDb) +elif sys.argv[1] == "pre-re": + itemDb = readItemDB(startPath + "/db/pre-re/item_db.conf", itemDb) + itemDb = readItemDB(startPath + "/db/item_db2.conf", itemDb) +else: + printHelp(); + exit(1) + +convertFile(sourceFile, itemDb)
\ No newline at end of file diff --git a/tools/petevolutionconverter.py b/tools/petevolutionconverter.py new file mode 100644 index 000000000..0ccc71314 --- /dev/null +++ b/tools/petevolutionconverter.py @@ -0,0 +1,248 @@ +#!/usr/bin/python +# -*- coding: utf8 -*- +# +# This file is part of Hercules. +# http://herc.ws - http://github.com/HerculesWS/Hercules +# +# Copyright (C) 2018 Hercules Dev Team +# Copyright (C) 2018 Dastgir +# +# Hercules is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# Usage: +# python petevolutionconverter.py PetEvolutionCln.lub re ../ > pet_evolve_db.conf + +import re +import sys +import utils.common as Tools + +def printHeader(): + print('''//================= Hercules Database ===================================== +//= _ _ _ +//= | | | | | | +//= | |_| | ___ _ __ ___ _ _| | ___ ___ +//= | _ |/ _ \ '__/ __| | | | |/ _ \/ __| +//= | | | | __/ | | (__| |_| | | __/\__ \ +//= \_| |_/\___|_| \___|\__,_|_|\___||___/ +//================= License =============================================== +//= This file is part of Hercules. +//= http://herc.ws - http://github.com/HerculesWS/Hercules +//= +//= Copyright (C) 2018 Hercules Dev Team +//= +//= Hercules is free software: you can redistribute it and/or modify +//= it under the terms of the GNU General Public License as published by +//= the Free Software Foundation, either version 3 of the License, or +//= (at your option) any later version. +//= +//= This program is distributed in the hope that it will be useful, +//= but WITHOUT ANY WARRANTY; without even the implied warranty of +//= MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//= GNU General Public License for more details. +//= +//= You should have received a copy of the GNU General Public License +//= along with this program. If not, see <http://www.gnu.org/licenses/>. +//========================================================================= +//= Pets Database +//========================================================================= + +pet_db:( +/************************************************************************** + ************* Entry structure ******************************************** + ************************************************************************** +{ + // ================ Mandatory fields ============================== + Id: ID (int) + SpriteName: "Sprite_Name" (string) + Name: "Pet Name" (string) + // ================ Optional fields =============================== + TamingItem: Taming Item (string, defaults to 0) + EggItem: Egg Id (string, defaults to 0) + AccessoryItem: Equipment Id (string, defaults to 0) + FoodItem: Food Id (string, defaults to 0) + FoodEffectiveness: hunger points (int, defaults to 0) + HungerDelay: hunger time (int, defaults to 0) + Intimacy: { + Initial: start intimacy (int, defaults to 0) + FeedIncrement: feeding intimacy (int, defaults to 0) + OverFeedDecrement: overfeeding intimacy (int, defaults to 0) + OwnerDeathDecrement: owner die intimacy (int, defaults to 0) + } + CaptureRate: capture rate (int, defaults to 0) + Speed: speed (int, defaults to 0) + SpecialPerformance: true/false (boolean, defaults to false) + TalkWithEmotes: convert talk (boolean, defaults to false) + AttackRate: attack rate (int, defaults to 0) + DefendRate: Defence attack (int, defaults to 0) + ChangeTargetRate: change target (int, defaults to 0) + Evolve: { + EggID: { (string, Evolved Pet EggID) + Name: Amount (items required to perform evolution) + ... + } + } + AutoFeed: true/false (boolean, defaults to false) + PetScript: <" Pet Script (can also be multi-line) "> + EquipScript: <" Equip Script (can also be multi-line) "> +}, +**************************************************************************/''') + +def printID(db, name, tabSize = 1): + if (name not in db or int(db[name]) == 0): + return + print('{}{}: {}'.format('\t'*tabSize, name, db[name])) + +def printString(db, name, tabSize = 1): + if (name not in db or db[name].strip() == ""): + return + print('{}{}: "{}"'.format('\t'*tabSize, name, db[name])) + +def printBool(db, name): + if (name not in db or db[name] == '0'): + return + print('\t{}: true'.format(name)) + +def printScript(db, name): + if (name not in db or db[name].strip() == ""): + return + print('\t{}: <{}>'.format(name, db[name])) + +def printEntry(ItemDB, EvolveDB, autoFeedDB, entry, mode, serverpath): + PetDB = Tools.LoadDB('pet_db', mode, serverpath) + + for i, db in enumerate(PetDB): + print('{') + printID(db, 'Id') + printString(db, 'SpriteName') + printString(db, 'Name') + + printString(db, 'TamingItem') + printString(db, 'EggItem') + printString(db, 'AccessoryItem') + printString(db, 'FoodItem') + printID(db, 'FoodEffectiveness') + printID(db, 'HungerDelay') + + if ('Intimacy' in db and (db['Intimacy']['Initial'] != 0 or db['Intimacy']['FeedIncrement'] != 0 or + db['Intimacy']['OverFeedDecrement'] != 0 or db['Intimacy']['OwnerDeathDecrement'] != 0)): + print('\tIntimacy: {') + printID(db['Intimacy'], 'Initial', 2) + printID(db['Intimacy'], 'FeedIncrement', 2) + printID(db['Intimacy'], 'OverFeedDecrement', 2) + printID(db['Intimacy'], 'OwnerDeathDecrement', 2) + print('\t}') + # + printID(db, 'CaptureRate') + printID(db, 'Speed') + printBool(db, 'SpecialPerformance') + printBool(db, 'TalkWithEmotes') + printID(db, 'AttackRate') + printID(db, 'DefendRate') + printID(db, 'ChangeTargetRate') + if (str(db['Id']) in autoFeedDB): + print('\tAutoFeed: true') + else: + print('\tAutoFeed: false') + printScript(db, 'PetScript') + printScript(db, 'EquipScript') + + if (db['EggItem'] in EvolveDB): + entry = EvolveDB[db['EggItem']] + print('\tEvolve: {') + + for evolve in entry: + if ('comment' in evolve): + print('/*') + print('\t\t{}: {'.format(evolve['Id'])) + + for items in evolve['items']: + print('\t\t\t{}: {}'.format(items[0], items[1])) + + print('\t\t}') + if ('comment' in evolve): + print('*/') + + print('\t}') + print('},') + +def saveEntry(EvolveDB, entry): + if (entry['from'] not in EvolveDB): + EvolveDB[entry['from']] = list() + EvolveDB[entry['from']].append(entry) + return EvolveDB + +def getItemConstant(entry, ItemDB, itemID): + if (itemID in ItemDB): + return ItemDB[itemID] + print(itemID, "not found", entry) + entry['comment'] = 1 + return itemID + +def ConvertDB(luaName, mode, serverpath): + ItemDB = Tools.LoadDBConsts('item_db', mode, serverpath) + f = open(luaName) + content = f.read() + f.close() + + recipeDB = re.findall(r'InsertEvolutionRecipeLGU\((\d+),\s*(\d+),\s*(\d+),\s*(\d+)\)', content) + autoFeedDB = re.findall(r'InsertPetAutoFeeding\((\d+)\)', content) + + current = 0 + + entry = dict() + EvolveDB = dict() + + printHeader() + for recipe in recipeDB: + fromEgg = getItemConstant(entry, ItemDB, int(recipe[0])) + petEgg = getItemConstant(entry, ItemDB, int(recipe[1])) + + if (current == 0): + entry = { + 'Id': petEgg, + 'from': fromEgg, + 'items': list() + } + current = petEgg + + if (current != petEgg): + EvolveDB = saveEntry(EvolveDB, entry) + entry = { + 'Id': petEgg, + 'from': fromEgg, + 'items': list() + } + entry['id'] = petEgg + entry['items'] = list() + current = petEgg + + itemConst = getItemConstant(entry, ItemDB, int(recipe[2])) + quantity = int(recipe[3]) + + entry['items'].append((itemConst, quantity)) + saveEntry(EvolveDB, entry) + + printEntry(ItemDB, EvolveDB, autoFeedDB, entry, mode, serverpath) + print(')') + + +if len(sys.argv) != 4: + print('Pet Evolution Lua to DB') + print('Usage:') + print(' petevolutionconverter.py lua mode serverpath') + print("example:") + print(' petevolutionconverter.py PetEvolutionCln.lua pre-re ../') + exit(1) + +ConvertDB(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/tools/skilldbconverter.php b/tools/skilldbconverter.php index d926e4474..8e241ff6f 100644 --- a/tools/skilldbconverter.php +++ b/tools/skilldbconverter.php @@ -955,7 +955,7 @@ function getcomments($re) //= This file is part of Hercules. //= http://herc.ws - http://github.com/HerculesWS/Hercules //= -//= Copyright (C) 2014-2016 Hercules Dev Team +//= Copyright (C) 2014-2018 Hercules Dev Team //= //= Hercules is free software: you can redistribute it and/or modify //= it under the terms of the GNU General Public License as published by diff --git a/tools/stackdump b/tools/stackdump index 25b1fa46a..47cb172ed 100755 --- a/tools/stackdump +++ b/tools/stackdump @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash case "$1" in map|char|login) diff --git a/tools/utils/__init__.py b/tools/utils/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tools/utils/__init__.py diff --git a/tools/utils/common.py b/tools/utils/common.py new file mode 100644 index 000000000..acceb9b30 --- /dev/null +++ b/tools/utils/common.py @@ -0,0 +1,64 @@ +#!/usr/bin/python +# -*- coding: utf8 -*- +# +# This file is part of Hercules. +# http://herc.ws - http://github.com/HerculesWS/Hercules +# +# Copyright (C) 2018 Hercules Dev Team +# Copyright (C) 2018 Asheraf +# +# Hercules is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import io +import libconf as libconf +import os.path + +def LoadDBConsts(DBname, mode, serverpath): + filenames = [serverpath + 'db/{}/{}.conf'.format(mode, DBname)] + + if os.path.isfile(serverpath + 'db/{}2.conf'.format(DBname)): + filenames.append(serverpath + 'db/{}2.conf'.format(DBname)) + + consts = dict() + for filename in filenames: + with io.open(filename) as f: + config = libconf.load(f) + db = config[DBname] + if DBname == 'item_db': + for i, v in enumerate(db): + consts[db[i].Id] = db[i].AegisName + elif DBname == 'mob_db': + for i, v in enumerate(db): + consts[db[i].Id] = db[i].SpriteName + elif DBname == 'skill_db': + for i, v in enumerate(db): + consts[db[i].Id] = db[i].Name + else: + print('LoadDBConsts: invalid database name {}'.format(DBname)) + exit(1) + return consts + +def LoadDB(DBname, mode, serverpath): + filenames = [serverpath + 'db/{}/{}.conf'.format(mode, DBname)] + + if os.path.isfile(serverpath + 'db/{}2.conf'.format(DBname)): + filenames.append(serverpath + 'db/{}2.conf'.format(DBname)) + + for filename in filenames: + with io.open(filename) as f: + config = libconf.load(f) + db = config[DBname] + return db + print('LoadDB: invalid database name {}'.format(DBname)) + exit(1) diff --git a/tools/utils/libconf.py b/tools/utils/libconf.py new file mode 100644 index 000000000..635efd07d --- /dev/null +++ b/tools/utils/libconf.py @@ -0,0 +1,693 @@ +#!/usr/bin/python +# -*- coding: utf8 -*- +# +# Copyright (C) 2018 Hercules Dev Team +# +# This library is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# This file originally licensed under the MIT License +# +# Copyright (c) 2016 Christian Aichinger <Greek0@gmx.net> +# https://github.com/Grk0/python-libconf +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import, division, print_function + +import sys +import os +import codecs +import collections +import io +import re + +# Define an isstr() and isint() that work on both Python2 and Python3. +# See http://stackoverflow.com/questions/11301138 +try: + basestring # attempt to evaluate basestring + + def isstr(s): + return isinstance(s, basestring) + + def isint(i): + return isinstance(i, (int, long)) +except NameError: + + def isstr(s): + return isinstance(s, str) + + def isint(i): + return isinstance(i, int) + +# Bounds to determine when an "L" suffix should be used during dump(). +SMALL_INT_MIN = -2**31 +SMALL_INT_MAX = 2**31 - 1 + +ESCAPE_SEQUENCE_RE = re.compile(r''' + ( \\x.. # 2-digit hex escapes + | \\[\\'"abfnrtv] # Single-character escapes + )''', re.UNICODE | re.VERBOSE) + +SKIP_RE = re.compile(r'\s+|#.*$|//.*$|/\*(.|\n)*?\*/', re.MULTILINE) +UNPRINTABLE_CHARACTER_RE = re.compile(r'[\x00-\x1F\x7F]') + + +# load() logic +############## + +def decode_escapes(s): + '''Unescape libconfig string literals''' + def decode_match(match): + return codecs.decode(match.group(0), 'unicode-escape') + + return ESCAPE_SEQUENCE_RE.sub(decode_match, s) + + +class AttrDict(collections.OrderedDict): + '''OrderedDict subclass giving access to string keys via attribute access + + This class derives from collections.OrderedDict. Thus, the original + order of the config entries in the input stream is maintained. + ''' + + def __getattr__(self, attr): + if attr == '_OrderedDict__root': + # Work around Python2's OrderedDict weirdness. + raise AttributeError("AttrDict has no attribute %r" % attr) + return self.__getitem__(attr) + + +class ConfigParseError(RuntimeError): + '''Exception class raised on errors reading the libconfig input''' + pass + + +class ConfigSerializeError(TypeError): + '''Exception class raised on errors serializing a config object''' + pass + + +class Token(object): + '''Base class for all tokens produced by the libconf tokenizer''' + def __init__(self, type, text, filename, row, column): + self.type = type + self.text = text + self.filename = filename + self.row = row + self.column = column + + def __str__(self): + return "%r in %r, row %d, column %d" % ( + self.text, self.filename, self.row, self.column) + + +class FltToken(Token): + '''Token subclass for floating point values''' + def __init__(self, *args, **kwargs): + super(FltToken, self).__init__(*args, **kwargs) + self.value = float(self.text) + + +class IntToken(Token): + '''Token subclass for integral values''' + def __init__(self, *args, **kwargs): + super(IntToken, self).__init__(*args, **kwargs) + self.is_long = self.text.endswith('L') + self.is_hex = (self.text[1:2].lower() == 'x') + self.value = int(self.text.rstrip('L'), 0) + + +class BoolToken(Token): + '''Token subclass for booleans''' + def __init__(self, *args, **kwargs): + super(BoolToken, self).__init__(*args, **kwargs) + self.value = (self.text[0].lower() == 't') + + +class StrToken(Token): + '''Token subclass for strings''' + def __init__(self, *args, **kwargs): + super(StrToken, self).__init__(*args, **kwargs) + self.value = decode_escapes(self.text[1:-1]) + + +def compile_regexes(token_map): + return [(cls, type, re.compile(regex)) + for cls, type, regex in token_map] + + +class Tokenizer: + '''Tokenize an input string + + Typical usage: + + tokens = list(Tokenizer("<memory>").tokenize("""a = 7; b = ();""")) + + The filename argument to the constructor is used only in error messages, no + data is loaded from the file. The input data is received as argument to the + tokenize function, which yields tokens or throws a ConfigParseError on + invalid input. + + Include directives are not supported, they must be handled at a higher + level (cf. the TokenStream class). + ''' + + token_map = compile_regexes([ + (FltToken, 'float', r'([-+]?(\d+)?\.\d*([eE][-+]?\d+)?)|' + r'([-+]?(\d+)(\.\d*)?[eE][-+]?\d+)'), + (IntToken, 'hex64', r'0[Xx][0-9A-Fa-f]+(L(L)?)'), + (IntToken, 'hex', r'0[Xx][0-9A-Fa-f]+'), + (BoolToken, 'boolean', r'(?i)(true|false)\b'), + (StrToken, 'string', r'"([^"\\]|\\.)*"'), + (StrToken, 'string', r'<"(?<=<")([\S\s]*?)(?=">)">'), + (Token, 'name', r'[0-9]*[A-Za-z\*][-A-Za-z0-9_\*]*'), + (IntToken, 'integer64', r'[-+]?[0-9]+L(L)?'), + (IntToken, 'integer', r'[-+]?[0-9]+'), + (Token, '}', r'\}'), + (Token, '{', r'\{'), + (Token, ')', r'\)'), + (Token, '(', r'\('), + (Token, ']', r'\]'), + (Token, '[', r'\['), + (Token, ',', r','), + (Token, ';', r';'), + (Token, '=', r'='), + (Token, ':', r':'), + ]) + + def __init__(self, filename): + self.filename = filename + self.row = 1 + self.column = 1 + + def tokenize(self, string): + '''Yield tokens from the input string or throw ConfigParseError''' + pos = 0 + while pos < len(string): + m = SKIP_RE.match(string, pos=pos) + if m: + skip_lines = m.group(0).split('\n') + if len(skip_lines) > 1: + self.row += len(skip_lines) - 1 + self.column = 1 + len(skip_lines[-1]) + else: + self.column += len(skip_lines[0]) + + pos = m.end() + continue + + for cls, type, regex in self.token_map: + m = regex.match(string, pos=pos) + if m: + yield cls(type, m.group(0), + self.filename, self.row, self.column) + self.column += len(m.group(0)) + pos = m.end() + break + else: + raise ConfigParseError( + "Couldn't load config in %r row %d, column %d: %r" % + (self.filename, self.row, self.column, + string[pos:pos+20])) + + +class TokenStream: + '''Offer a parsing-oriented view on tokens + + Provide several methods that are useful to parsers, like ``accept()``, + ``expect()``, ... + + The ``from_file()`` method is the preferred way to read input files, as + it handles include directives, which the ``Tokenizer`` class does not do. + ''' + + def __init__(self, tokens): + self.position = 0 + self.tokens = list(tokens) + + @classmethod + def from_file(cls, f, filename=None, includedir='', seenfiles=None): + '''Create a token stream by reading an input file + + Read tokens from `f`. If an include directive ('@include "file.cfg"') + is found, read its contents as well. + + The `filename` argument is used for error messages and to detect + circular imports. ``includedir`` sets the lookup directory for included + files. ``seenfiles`` is used internally to detect circular includes, + and should normally not be supplied by users of is function. + ''' + + if filename is None: + filename = getattr(f, 'name', '<unknown>') + if seenfiles is None: + seenfiles = set() + + if filename in seenfiles: + raise ConfigParseError("Circular include: %r" % (filename,)) + seenfiles = seenfiles | {filename} # Copy seenfiles, don't alter it. + + tokenizer = Tokenizer(filename=filename) + lines = [] + tokens = [] + for line in f: + m = re.match(r'@include "(.*)"$', line.strip()) + if m: + tokens.extend(tokenizer.tokenize(''.join(lines))) + lines = [re.sub(r'\S', ' ', line)] + + includefilename = decode_escapes(m.group(1)) + includefilename = os.path.join(includedir, includefilename) + try: + includefile = open(includefilename, "r") + except IOError: + raise ConfigParseError("Could not open include file %r" % + (includefilename,)) + + with includefile: + includestream = cls.from_file(includefile, + filename=includefilename, + includedir=includedir, + seenfiles=seenfiles) + tokens.extend(includestream.tokens) + + else: + lines.append(line) + + tokens.extend(tokenizer.tokenize(''.join(lines))) + return cls(tokens) + + def peek(self): + '''Return (but do not consume) the next token + + At the end of input, ``None`` is returned. + ''' + + if self.position >= len(self.tokens): + return None + + return self.tokens[self.position] + + def accept(self, *args): + '''Consume and return the next token if it has the correct type + + Multiple token types (as strings, e.g. 'integer64') can be given + as arguments. If the next token is one of them, consume and return it. + + If the token type doesn't match, return None. + ''' + + token = self.peek() + if token is None: + return None + + for arg in args: + if token.type == arg: + self.position += 1 + return token + + return None + + def expect(self, *args): + '''Consume and return the next token if it has the correct type + + Multiple token types (as strings, e.g. 'integer64') can be given + as arguments. If the next token is one of them, consume and return it. + + If the token type doesn't match, raise a ConfigParseError. + ''' + + t = self.accept(*args) + if t is not None: + return t + + self.error("expected: %r" % (args,)) + + def error(self, msg): + '''Raise a ConfigParseError at the current input position''' + if self.finished(): + raise ConfigParseError("Unexpected end of input; %s" % (msg,)) + else: + t = self.peek() + raise ConfigParseError("Unexpected token %s; %s" % (t, msg)) + + def finished(self): + '''Return ``True`` if the end of the token stream is reached.''' + return self.position >= len(self.tokens) + + +class Parser: + '''Recursive descent parser for libconfig files + + Takes a ``TokenStream`` as input, the ``parse()`` method then returns + the config file data in a ``json``-module-style format. + ''' + + def __init__(self, tokenstream): + self.tokens = tokenstream + + def parse(self): + return self.configuration() + + def configuration(self): + result = self.setting_list_or_empty() + if not self.tokens.finished(): + raise ConfigParseError("Expected end of input but found %s" % + (self.tokens.peek(),)) + + return result + + def setting_list_or_empty(self): + result = AttrDict() + while True: + s = self.setting() + if s is None: + return result + + result[s[0]] = s[1] + + def setting(self): + name = self.tokens.accept('name') + if name is None: + return None + + self.tokens.expect(':', '=') + + value = self.value() + if value is None: + self.tokens.error("expected a value") + + self.tokens.accept(';', ',') + + return (name.text, value) + + def value(self): + acceptable = [self.scalar_value, self.array, self.list, self.group] + return self._parse_any_of(acceptable) + + def scalar_value(self): + # This list is ordered so that more common tokens are checked first. + acceptable = [self.string, self.boolean, self.integer, self.float, + self.hex, self.integer64, self.hex64] + return self._parse_any_of(acceptable) + + def value_list_or_empty(self): + return tuple(self._comma_separated_list_or_empty(self.value)) + + def scalar_value_list_or_empty(self): + return self._comma_separated_list_or_empty(self.scalar_value) + + def array(self): + return self._enclosed_block('[', self.scalar_value_list_or_empty, ']') + + def list(self): + return self._enclosed_block('(', self.value_list_or_empty, ')') + + def group(self): + return self._enclosed_block('{', self.setting_list_or_empty, '}') + + def boolean(self): + return self._create_value_node('boolean') + + def integer(self): + return self._create_value_node('integer') + + def integer64(self): + return self._create_value_node('integer64') + + def hex(self): + return self._create_value_node('hex') + + def hex64(self): + return self._create_value_node('hex64') + + def float(self): + return self._create_value_node('float') + + def string(self): + t_first = self.tokens.accept('string') + if t_first is None: + return None + + values = [t_first.value] + while True: + t = self.tokens.accept('string') + if t is None: + break + values.append(t.value) + + return ''.join(values) + + def _create_value_node(self, tokentype): + t = self.tokens.accept(tokentype) + if t is None: + return None + + return t.value + + def _parse_any_of(self, nonterminals): + for fun in nonterminals: + result = fun() + if result is not None: + return result + + return None + + def _comma_separated_list_or_empty(self, nonterminal): + values = [] + first = True + while True: + v = nonterminal() + if v is None: + if first: + return [] + else: + # This is disabled to enable the last member in a list to have a comma at the end + # self.tokens.error("expected value after ','") + return values + + values.append(v) + if not self.tokens.accept(','): + return values + + first = False + + def _enclosed_block(self, start, nonterminal, end): + if not self.tokens.accept(start): + return None + result = nonterminal() + self.tokens.expect(end) + return result + + +def load(f, filename=None, includedir=''): + '''Load the contents of ``f`` (a file-like object) to a Python object + + The returned object is a subclass of ``dict`` that exposes string keys as + attributes as well. + + Example: + + >>> with open('test/example.cfg') as f: + ... config = libconf.load(f) + >>> config['window']['title'] + 'libconfig example' + >>> config.window.title + 'libconfig example' + ''' + + if isinstance(f.read(0), bytes): + raise TypeError("libconf.load() input file must by unicode") + + tokenstream = TokenStream.from_file(f, + filename=filename, + includedir=includedir) + return Parser(tokenstream).parse() + + +def loads(string, filename=None, includedir=''): + '''Load the contents of ``string`` to a Python object + + The returned object is a subclass of ``dict`` that exposes string keys as + attributes as well. + + Example: + + >>> config = libconf.loads('window: { title: "libconfig example"; };') + >>> config['window']['title'] + 'libconfig example' + >>> config.window.title + 'libconfig example' + ''' + + try: + f = io.StringIO(string) + except TypeError: + raise TypeError("libconf.loads() input string must by unicode") + + return load(f, filename=filename, includedir=includedir) + + +# dump() logic +############## + +def dump_int(i): + '''Stringize ``i``, append 'L' if ``i`` is exceeds the 32-bit int range''' + return str(i) + ('' if SMALL_INT_MIN <= i <= SMALL_INT_MAX else 'L') + + +def dump_string(s): + '''Stringize ``s``, adding double quotes and escaping as necessary + + Backslash escape backslashes, double quotes, ``\f``, ``\n``, ``\r``, and + ``\t``. Escape all remaining unprintable characters in ``\xFF``-style. + The returned string will be surrounded by double quotes. + ''' + + s = (s.replace('\\', '\\\\') + .replace('"', '\\"') + .replace('\f', r'\f') + .replace('\n', r'\n') + .replace('\r', r'\r') + .replace('\t', r'\t')) + s = UNPRINTABLE_CHARACTER_RE.sub( + lambda m: r'\x{:02x}'.format(ord(m.group(0))), + s) + return '"' + s + '"' + + +def dump_value(key, value, f, indent=0): + '''Save a value of any libconfig type + + This function serializes takes ``key`` and ``value`` and serializes them + into ``f``. If ``key`` is ``None``, a list-style output is produced. + Otherwise, output has ``key = value`` format. + ''' + + spaces = ' ' * indent + + if key is None: + key_prefix = '' + key_prefix_nl = '' + else: + key_prefix = key + ' = ' + key_prefix_nl = key + ' =\n' + spaces + + if isinstance(value, dict): + f.write(u'{}{}{{\n'.format(spaces, key_prefix_nl)) + dump_dict(value, f, indent + 4) + f.write(u'{}}}'.format(spaces)) + elif isinstance(value, tuple): + f.write(u'{}{}(\n'.format(spaces, key_prefix_nl)) + dump_collection(value, f, indent + 4) + f.write(u'\n{})'.format(spaces)) + elif isinstance(value, list): + f.write(u'{}{}[\n'.format(spaces, key_prefix_nl)) + dump_collection(value, f, indent + 4) + f.write(u'\n{}]'.format(spaces)) + elif isstr(value): + f.write(u'{}{}{}'.format(spaces, key_prefix, dump_string(value))) + elif isint(value): + f.write(u'{}{}{}'.format(spaces, key_prefix, dump_int(value))) + elif isinstance(value, float): + f.write(u'{}{}{}'.format(spaces, key_prefix, value)) + else: + raise ConfigSerializeError("Can not serialize object %r of type %s" % + (value, type(value))) + + +def dump_collection(cfg, f, indent=0): + '''Save a collection of attributes''' + + for i, value in enumerate(cfg): + dump_value(None, value, f, indent) + if i < len(cfg) - 1: + f.write(u',\n') + + +def dump_dict(cfg, f, indent=0): + '''Save a dictionary of attributes''' + + for key in cfg: + if not isstr(key): + raise ConfigSerializeError("Dict keys must be strings: %r" % + (key,)) + dump_value(key, cfg[key], f, indent) + f.write(u';\n') + + +def dumps(cfg): + '''Serialize ``cfg`` into a libconfig-formatted ``str`` + + ``cfg`` must be a ``dict`` with ``str`` keys and libconf-supported values + (numbers, strings, booleans, possibly nested dicts, lists, and tuples). + + Returns the formatted string. + ''' + + str_file = io.StringIO() + dump(cfg, str_file) + return str_file.getvalue() + + +def dump(cfg, f): + '''Serialize ``cfg`` as a libconfig-formatted stream into ``f`` + + ``cfg`` must be a ``dict`` with ``str`` keys and libconf-supported values + (numbers, strings, booleans, possibly nested dicts, lists, and tuples). + + ``f`` must be a ``file``-like object with a ``write()`` method. + ''' + + if not isinstance(cfg, dict): + raise ConfigSerializeError( + 'dump() requires a dict as input, not %r of type %r' % + (cfg, type(cfg))) + + dump_dict(cfg, f, 0) + + +# main(): small example of how to use libconf +############################################# + +def main(): + '''Open the libconfig file specified by sys.argv[1] and pretty-print it''' + global output + if len(sys.argv[1:]) == 1: + with io.open(sys.argv[1], 'r', encoding='utf-8') as f: + output = load(f) + else: + output = load(sys.stdin) + + dump(output, sys.stdout) + + +if __name__ == '__main__': + main() |