add a new item frob tool to replace the old one

removes the specified items, non-existant items and item stubs (0 qty)

5 files changed, 539 insertions, 0 deletions

diff --git a/server/frob/README.md b/server/frob/README.md
new file mode 100644
index 0000000..19fbe13
--- /dev/null
+++ b/server/frob/README.md
@@ -0,0 +1,19 @@
+# tmwAthena item frobber
+## Prerequisites
+- [Deno]
+
+## Compatibility
+- works with tmwAthena `v16.2.9 - v19.4.15`
+    - newer versions may modify the flatfile structure and break things
+
+## Usage
+- from `tmwa-server-data`:
+```
+# choose whatever syntax you prefer:
+make frob items="item[,item[,item...]]"     # item list (csv)
+make frob items="item[ item[ item...]]"     # item list
+make frob items="item-item"                 # item range
+make frob items="item..item"                # item range
+```
+
+[Deno]: https://deno.land
diff --git a/server/frob/char.ts b/server/frob/char.ts
new file mode 100644
index 0000000..e0cb5f3
--- /dev/null
+++ b/server/frob/char.ts
@@ -0,0 +1,152 @@
+class CharParser {
+    private char_line =
+    "^" +
+    "(?<char_id>[0-9]+)\t" +
+    "(?<account_id>[0-9]+),(?<char_num>[0-9]+)\t" +
+    "(?<name>[^\t]+)\t" +
+    "(?<species>[0-9]+),(?<base_level>[0-9]+),(?<job_level>[0-9]+)\t" +
+    "(?<base_exp>[0-9]+),(?<job_exp>[0-9]+),(?<zeny>[0-9]+)\t" +
+    "(?<hp>[0-9]+),(?<max_hp>[0-9]+),(?<sp>[0-9]+),(?<max_sp>[0-9]+)\t" +
+    "(?<str>[0-9]+),(?<agi>[0-9]+),(?<vit>[0-9]+),(?<int>[0-9]+),(?<dex>[0-9]+),(?<luk>[0-9]+)\t" +
+    "(?<status_point>[0-9]+),(?<skill_point>[0-9]+)\t" +
+    "(?<option>[0-9]+),(?<karma>[0-9]+),(?<manner>[0-9]+)\t" +
+    "(?<party_id>[0-9]+),[0-9]+,[0-9]+\t" +
+    "(?<hair>[0-9]+),(?<hair_color>[0-9]+),(?<clothes_color>[0-9]+)\t" +
+    "(?<weapon>[0-9]+),(?<shield>[0-9]+),(?<head_top>[0-9]+),(?<head_mid>[0-9]+),(?<head_bottom>[0-9]+)\t" +
+    "(?<last_map>[^,]+),(?<last_x>[0-9]+),(?<last_y>[0-9]+)\t" +
+    "(?<save_map>[^,]+),(?<save_x>[0-9]+),(?<save_y>[0-9]+),(?<partner_id>[0-9]+)\t" +
+    "(?<sex>[FMNS])\t" + // <= ignore S to ignore server accounts
+    "(?<items>([0-9]+,(?<nameid>[0-9]+),(?<amount>[0-9]+),(?<equip>[0-9]+),[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+ )*)\t" + // inventory
+    "\t" + // cart
+    "(?<skills>((?<skill_id>[0-9]+),(?<skill_lv>[0-9]+) )*)\t" +
+    "(?<variables>((?<var_name>[^,]+),(?<value>[-0-9]+) )*)\t" + // some chars have negative variables (overflows)
+    "$";
+    private char_items_line = "[0-9]+,(?<nameid>[0-9]+),(?<amount>[0-9]+),(?<equip>[0-9]+),[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+ ";
+    private char_regex: RegExp;
+    private char_regex_items: RegExp;
+
+    constructor () {
+        this.char_regex = new RegExp(this.char_line);
+        this.char_regex_items = new RegExp(this.char_items_line, "g");
+    }
+
+    private parseLine (line: string) {
+        const match = this.char_regex.exec(line);
+
+        if (!(match instanceof Object) || !Reflect.has(match, "groups")) {
+            console.error("line does not match the char regex:", line);
+            throw new SyntaxError();
+        }
+
+        const groups = (match as any).groups;
+        let items = [];
+
+        if (groups.items.length > 1) {
+            let match_items = this.char_regex_items.exec(groups.items);
+
+            while (match_items !== null) {
+                items.push((match_items as any).groups);
+                match_items = this.char_regex_items.exec(groups.items);
+            }
+        }
+
+        groups.items = items;
+        return groups;
+    }
+
+    public async * readDB () {
+        const decoder = new TextDecoder("utf-8");
+        console.info("\nwalking through athena.txt...");
+        const file = await Deno.open("world/save/athena.txt");
+        const buf = new Uint8Array(1024);
+        let accumulator = "";
+
+        while (true) {
+            const { nread, eof } = await Deno.read(file.rid, buf);
+
+            if (eof) {
+                break;
+            }
+
+            const str = decoder.decode(buf);
+
+            for (let c of str) {
+                if (c === "\n") {
+                    if (accumulator.length === 14) {
+                        // this is the newid line
+                        break;
+                    }
+                    yield this.parseLine(accumulator);
+                    accumulator = "";
+                } else {
+                    accumulator += c;
+                }
+            }
+        }
+    }
+}
+
+class CharWriter {
+    private file;
+    private highest: number = 0;
+    private encoder;
+
+    constructor () {
+        try {
+            Deno.removeSync("world/save/athena.txt.tmp");
+        } catch {
+            // ignore this
+        }
+        this.file = Deno.openSync("world/save/athena.txt.tmp", "a+");
+        this.encoder = new TextEncoder();
+    }
+
+    async write (char: any) {
+        let line =
+        `${char.char_id}\t` +
+        `${char.account_id},${char.char_num}\t` +
+        `${char.name}\t` +
+        `${char.species},${char.base_level},${char.job_level}\t` +
+        `${char.base_exp},${char.job_exp},${char.zeny}\t` +
+        `${char.hp},${char.max_hp},${char.sp},${char.max_sp}\t` +
+        `${char.str},${char.agi},${char.vit},${char.int},${char.dex},${char.luk}\t` +
+        `${char.status_point},${char.skill_point}\t` +
+        `${char.option},${char.karma},${char.manner}\t` +
+        `${char.party_id},0,0\t` +
+        `${char.hair},${char.hair_color},${char.clothes_color}\t` +
+        `${char.weapon},${char.shield},${char.head_top},${char.head_mid},${char.head_bottom}\t` +
+        `${char.last_map},${char.last_x},${char.last_y}\t` +
+        `${char.save_map},${char.save_x},${char.save_y},${char.partner_id}\t` +
+        `${char.sex}\t`;
+
+        for (let item of char.items) {
+            line += `0,${item.nameid},${item.amount},${item.equip},1,0,0,0,0,0,0,0 `;
+        }
+
+        line += `\t`; // end of items
+        line += `\t`; // cart
+        line += `${char.skills}\t`;
+        line += `${char.variables}\t`;
+        line += `\n`;
+
+        await Deno.write(this.file.rid, this.encoder.encode(line));
+
+        if (+char.char_id > this.highest) {
+            this.highest = +char.char_id;
+        }
+    }
+
+    async finalize() {
+        console.info("appending %newid%...");
+        await Deno.write(this.file.rid, this.encoder.encode(`${this.highest + 1}\t%newid%\n`));
+        this.file.close();
+        console.info("overwriting athena.txt...");
+        await Deno.rename("world/save/athena.txt", "world/save/athena.txt_pre-frob");
+        await Deno.rename("world/save/athena.txt.tmp", "world/save/athena.txt");
+    }
+}
+
+export {
+    CharParser,
+    CharWriter,
+}
diff --git a/server/frob/index.ts b/server/frob/index.ts
new file mode 100644
index 0000000..332989a
--- /dev/null
+++ b/server/frob/index.ts
@@ -0,0 +1,183 @@
+// this script removes specified items from inventories and storage
+import { CharParser, CharWriter } from "./char.ts";
+import { StorageParser, StorageWriter } from "./storage.ts";
+import { ItemDB } from "./itemdb.ts";
+
+const args: string[] = Deno.args.slice(1);
+const to_remove: Set<number> = new Set();
+const is_alpha: boolean = isNaN(args.join() as unknown as number);
+const char_parser = new CharParser();
+const storage_parser = new StorageParser();
+const char_writer = new CharWriter();
+const storage_writer = new StorageWriter();
+const item_db = new ItemDB();
+
+const stats = {
+    inventory: {
+        removed: 0,
+        pruned: 0,
+        stub: 0,
+        chars: 0,
+    },
+    storage: {
+        removed: 0,
+        pruned: 0,
+        stub: 0,
+        wiped: 0,
+        synced: 0,
+        accounts: 0,
+    },
+};
+
+
+(async () => {
+    const items_by_id = new Map(); // holds full item info
+    const items_by_name = new Map(); // holds references to the former
+
+    for await (let item of item_db.readDB()) {
+        items_by_id.set(+item.id, item);
+        items_by_name.set(item.name, +item.id);
+    }
+
+    const itemToNumber = (name: string|number): number => {
+        if (isNaN(name as number)) {
+            const item_id = items_by_name.get(name);
+
+            if (item_id) {
+                return +item_id;
+            } else {
+                console.error(`cannot find item '${name}' in the item db`);
+                throw new Error("item not found");
+            }
+        } else {
+            return +name;
+        }
+    }
+
+    const itemToString = (id: string|number): string => {
+        if (isNaN(id as number)) {
+            return `${id}`;
+        } else {
+            const item = items_by_id.get(+id);
+
+            if (item) {
+                return item.name;
+            } else {
+                console.error(`cannot find item ID ${id} in the item db`);
+                throw new Error("item not found");
+            }
+        }
+    }
+
+    if (args.length < 1) {
+        throw new RangeError("no items given!");
+    }
+
+    for (let arg of args.join(",").split(",")) {
+        if (arg.includes("-") || arg.includes("..")) {
+            const range = arg.split("-").join("..").split("..");
+            let from = itemToNumber(range[0]), to = itemToNumber(range[1]);
+
+            if (from > to) {
+                [from, to] = [to, from];
+            }
+
+            for (let i = from; i <= to; ++i) {
+                to_remove.add(i);
+            }
+        } else {
+            to_remove.add(itemToNumber(arg));
+        }
+    }
+
+    console.info("\nThe following items will be removed:");
+    for (let item of to_remove) {
+        console.info(`[${item}]: ${itemToString(item)}`);
+    }
+
+    // inventory:
+    for await (let char of char_parser.readDB()) {
+        let items_filtered = []; // this is not a Set because some items don't stack
+        let mod = false;
+
+        for (let item of char.items) {
+            if (!items_by_id.has(+item.nameid)) {
+                console.log(`removing ${+item.amount || 1}x non-existant item ID ${item.nameid} from inventory of character ${char.name} [${char.account_id}:${char.char_id}]`);
+                stats.inventory.pruned += +item.amount;
+                mod = true;
+            } else if (+item.amount < 1) {
+                console.log(`removing stub of item ${itemToString(item.nameid)} [${item.nameid}] from inventory of character ${char.name} [${char.account_id}:${char.char_id}]`);
+                stats.inventory.stub++;
+                mod = true;
+            } else if (to_remove.has(+item.nameid)) {
+                console.log(`removing ${item.amount}x ${itemToString(item.nameid)} [${item.nameid}] from inventory of character ${char.name} [${char.account_id}:${char.char_id}]`);
+                stats.inventory.removed += +item.amount;
+                mod = true;
+            } else {
+                items_filtered.push(item);
+            }
+        }
+
+        if (mod)
+            stats.inventory.chars++;
+
+        char.items = items_filtered;
+        await char_writer.write(char);
+    }
+
+    await char_writer.finalize();
+
+    // storage:
+    for await (let storage of storage_parser.readDB()) {
+        let items_filtered = []; // this is not a Set because some items don't stack
+        let mod = false;
+
+        for (let item of storage.items) {
+            if (!items_by_id.has(+item.nameid)) {
+                console.log(`removing ${+item.amount || 1}x non-existant item ID ${item.nameid} from storage of account ${storage.account_id}`);
+                stats.storage.pruned += +item.amount;
+                storage.storage_amount--;
+                mod = true;
+            } else if (+item.amount < 1) {
+                console.log(`removing stub of item ${itemToString(item.nameid)} [${item.nameid}] from storage of account ${storage.account_id}`);
+                stats.storage.stub++;
+                storage.storage_amount--;
+                mod = true;
+            } else if (to_remove.has(+item.nameid)) {
+                console.log(`removing ${item.amount}x ${itemToString(item.nameid)} [${item.nameid}] from storage of account ${storage.account_id}`);
+                stats.storage.removed += +item.amount;
+                storage.storage_amount--;
+                mod = true;
+            } else {
+                items_filtered.push(item);
+            }
+        }
+
+        if (mod)
+            stats.storage.accounts++;
+
+        storage.items = items_filtered;
+
+        if (+storage.storage_amount !== storage.items.length) {
+            const old_sync = +storage.storage_amount;
+            storage.storage_amount = storage.items.length;
+            console.log(`fixing sync of storage for account ${storage.account_id}: ${old_sync} => ${storage.storage_amount}`);
+            stats.storage.synced++;
+        }
+
+        if (storage.storage_amount >= 1) {
+            await storage_writer.write(storage);
+        } else {
+            console.log(`storage of account ${storage.account_id} is now empty: removing it from the storage db`);
+            stats.storage.wiped++;
+        }
+    }
+
+    await storage_writer.finalize();
+
+    console.info("\n=== all done ===");
+    console.info(`removed ${stats.inventory.removed} existant, ${stats.inventory.pruned} non-existant and ${stats.inventory.stub} stub items from the inventory of ${stats.inventory.chars} characters`);
+    console.info(`removed ${stats.storage.removed} existant, ${stats.storage.pruned} non-existant and ${stats.storage.stub} stub items from the storage of ${stats.storage.accounts} accounts`);
+    console.info(`removed ${stats.storage.wiped} empty storage entries from the storage db`);
+    console.info(`fixed storage sync of ${stats.storage.synced} accounts`);
+})()
diff --git a/server/frob/itemdb.ts b/server/frob/itemdb.ts
new file mode 100644
index 0000000..5ff943d
--- /dev/null
+++ b/server/frob/itemdb.ts
@@ -0,0 +1,68 @@
+class ItemDB {
+    private item_line =
+    "^" +
+    "(?<id>[0-9]+),[ \t]*" +
+    "(?<name>[^ \t,]+),[ \t]*" +
+    "(?<type>[0-9]+),[ \t]*" +
+    "(?<price>[0-9]+),[ \t]*" +
+    "(?<sell>[0-9]+),[ \t]*" +
+    "(?<weight>[0-9]+),[ \t]*" +
+    "(?<atk>[0-9]+),[ \t]*" +
+    "(?<def>[0-9]+),[ \t]*" +
+    "(?<range>[0-9]+),[ \t]*" +
+    "(?<mbonus>[0-9-]+),[ \t]*" +
+    "(?<slot>[0-9]+),[ \t]*" +
+    "(?<gender>[0-9]+),[ \t]*" +
+    "(?<loc>[0-9]+),[ \t]*" +
+    "(?<wlvl>[0-9]+),[ \t]*" +
+    "(?<elvl>[0-9]+),[ \t]*" +
+    "(?<view>[0-9]+),[ \t]*" +
+    "\{(?<usescript>[^\}]*)\},[ \t]*" +
+    "\{(?<equipscript>[^\}]*)\}[ \t]*" +
+    "$";
+    private item_regex: RegExp;
+
+    constructor () {
+        this.item_regex = new RegExp(this.item_line);
+    }
+
+    private parseLine (line) {
+        const match = this.item_regex.exec(line);
+
+        if (!(match instanceof Object) || !Reflect.has(match, "groups")) {
+            console.error("line does not match the item db regex:", line);
+            throw new SyntaxError();
+        }
+
+        return (match as any).groups;
+    }
+
+    public async * readDB () {
+        const decoder = new TextDecoder("utf-8");
+        console.info("reading tmwa-map.conf...");
+        const file = await Deno.readFile("world/map/conf/tmwa-map.conf");
+        const data = decoder.decode(file).split("\n");
+        const db_regex = new RegExp("^item_db: *(?<path>[A-Za-z0-9_\./]+)$");
+
+        for (const line of data) {
+            const match = db_regex.exec(line);
+            if (!(match instanceof Object)) continue;
+            const path = (match as any).groups.path;
+
+            console.info(`reading world/map/${path}...`)
+            const db = await Deno.readFile(`world/map/${path}`);
+
+            for (const item of decoder.decode(db).split("\n")) {
+                if (item.startsWith("//") || item.length < 2) {
+                    continue;
+                }
+
+                yield this.parseLine(item);
+            }
+        }
+    }
+}
+
+export {
+    ItemDB
+}
diff --git a/server/frob/storage.ts b/server/frob/storage.ts
new file mode 100644
index 0000000..720cd4b
--- /dev/null
+++ b/server/frob/storage.ts
@@ -0,0 +1,117 @@
+class StorageParser {
+    private storage_line =
+    "^" +
+    "(?<account_id>[0-9]+),(?<storage_amount>[0-9]+)\t" +
+    "(?<items>([0-9]+,(?<nameid>[0-9]+),(?<amount>[0-9]+),(?<equip>[0-9]+),[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+ )+)\t$";
+    private storage_items_line = "[0-9]+,(?<nameid>[0-9]+),(?<amount>[0-9]+),(?<equip>[0-9]+),[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+,[0-9]+ ";
+    private storage_regex: RegExp;
+    private storage_regex_items: RegExp;
+
+    constructor () {
+        this.storage_regex = new RegExp(this.storage_line);
+        this.storage_regex_items = new RegExp(this.storage_items_line, "g");
+    }
+
+    private parseLine (line: string) {
+        const match = this.storage_regex.exec(line);
+
+        if (!(match instanceof Object) || !Reflect.has(match, "groups")) {
+            console.error("line does not match the storage regex:", line);
+            throw new SyntaxError();
+        }
+
+        const groups = (match as any).groups;
+        let items = [];
+
+        if (groups.items.length > 1) {
+            let match_items = this.storage_regex_items.exec(groups.items);
+
+            while (match_items !== null) {
+                items.push((match_items as any).groups);
+                match_items = this.storage_regex_items.exec(groups.items);
+            }
+        }
+
+        groups.items = items;
+        return groups;
+    }
+
+    public async * readDB () {
+        const decoder = new TextDecoder("utf-8");
+        console.info("\nwalking through storage.txt...");
+        const file = await Deno.open("world/save/storage.txt");
+        const buf = new Uint8Array(1024);
+        let accumulator = "";
+
+        while (true) {
+            const { nread, eof } = await Deno.read(file.rid, buf);
+
+            if (eof) {
+                break;
+            }
+
+            const str = decoder.decode(buf);
+
+            if (nread < 1024) {
+                for (let c of str) {
+                    if (c === "\n") {
+                        yield this.parseLine(accumulator);
+                        break;
+                    } else {
+                        accumulator += c;
+                    }
+                }
+                break;
+            }
+
+            for (let c of str) {
+                if (c === "\n") {
+                    yield this.parseLine(accumulator);
+                    accumulator = "";
+                } else {
+                    accumulator += c;
+                }
+            }
+        }
+    }
+}
+
+class StorageWriter {
+    private file;
+    private encoder;
+
+    constructor () {
+        try {
+            Deno.removeSync("world/save/storage.txt.tmp");
+        } catch {
+            // ignore this
+        }
+        this.file = Deno.openSync("world/save/storage.txt.tmp", "a+");
+        this.encoder = new TextEncoder();
+    }
+
+    async write (storage: any) {
+        let line = `${storage.account_id},${storage.storage_amount}\t`;
+
+        for (let item of storage.items) {
+            line += `0,${item.nameid},${item.amount},${item.equip},0,0,0,0,0,0,0 `;
+        }
+
+        line += `\t`; // end of items
+        line += `\n`;
+
+        await Deno.write(this.file.rid, this.encoder.encode(line));
+    }
+
+    async finalize() {
+        this.file.close();
+        console.info("overwriting storage.txt...");
+        await Deno.rename("world/save/storage.txt", "world/save/storage.txt_pre-frob");
+        await Deno.rename("world/save/storage.txt.tmp", "world/save/storage.txt");
+    }
+}
+
+export {
+    StorageParser,
+    StorageWriter,
+}