[frob] add a mode to list item hoardersgithub/fork/Helianthella/hoarders

4 files changed, 273 insertions, 2 deletions

diff --git a/server/frob/README.md b/server/frob/README.md
index 88c6880..0d64f7d 100644
--- a/server/frob/README.md
+++ b/server/frob/README.md
@@ -18,4 +18,9 @@ make frob items="item..item"                # item range
 make frob items="item,item item-item item"  # mixed
 ```
 
+### Special flags
+- `--dry`: performs a dry run (does not modify databases)
+- `--sql`: converts the flatfile databases to MySQL tables in the `legacy` database
+- `--stats`: compiles statistics about general item distribution
+
 [Deno]: https://deno.land
diff --git a/server/frob/index.ts b/server/frob/index.ts
index df06975..7f92a59 100644
--- a/server/frob/index.ts
+++ b/server/frob/index.ts
@@ -1,5 +1,6 @@
 // this script removes specified items from inventories and storage
-import { SQLHandler } from "./sql.ts"
+import { SQLHandler } from "./sql.ts";
+import { ManaMarketHandler } from "./manamarket.ts";
 import { LoginParser, LoginSQL } from "./login.ts";
 import { CharParser, CharWriter, CharSQL } from "./char.ts";
 import { AccregParser, AccregSQL } from "./accreg.ts";
@@ -10,6 +11,7 @@ import { ItemDB } from "./itemdb.ts";
 const args: string[] = Deno.args.slice(1);
 const to_remove: Set<number> = new Set();
 const sql = new SQLHandler();
+const manamarket = new ManaMarketHandler();
 const login_parser = new LoginParser();
 const char_parser = new CharParser();
 const accreg_parser = new AccregParser();
@@ -24,6 +26,10 @@ const storage_writer = new StorageWriter();
 const storage_SQL = new StorageSQL(sql);
 const item_db = new ItemDB();
 
+// at which value an item is considered SSR
+const rare_threshold = 3000000;
+const hoarders: Map<number, any> = new Map();
+
 const stats = {
     inventory: {
         removed: 0,
@@ -44,6 +50,7 @@ const stats = {
 const flags = {
     dry_run: false,
     sql: false,
+    stats: false,
 };
 
 
@@ -51,6 +58,54 @@ const flags = {
     const items_by_id = new Map(); // holds full item info
     const items_by_name = new Map(); // holds references to the former
 
+    // TODO: split the hoarder stuff into hoarders.ts
+    const getHoarder = (account_id: number) => {
+        let hoarder = hoarders.get(account_id);
+
+        if (!hoarder) {
+            hoarder = hoarders.set(account_id, {
+                value: {
+                    items: 0,
+                    //gp: 0,
+                },
+                storage: {},
+                inventories: {},
+            }).get(account_id);
+        }
+
+        return hoarder;
+    };
+
+    const maybeAddToHoarders = (account_id: number, char_name: string, char_id: number, item: number, qty: number = 1) => {
+        // make sure all values are the expected format (Deno is weird sometimes)
+        [account_id, char_id, item, qty] = [+account_id, +char_id, +item, +qty];
+
+        const data = manamarket.items.get(item);
+        let value = data ? data.averageValue.overall : 0;
+
+        if (value < rare_threshold) {
+            return;
+        }
+
+        const hoarder = getHoarder(account_id);
+
+        item = (items_by_id.get(item)).name;
+
+        if (char_id && char_name) {
+            if (!Reflect.has(hoarder.inventories, char_name)) {
+                hoarder.inventories[char_name] = new Map();
+            }
+
+            const had = hoarder.inventories[char_name][item] || 0;
+            hoarder.inventories[char_name][item] = had + qty;
+        } else {
+            const had = hoarder.storage[item] || 0;
+            hoarder.storage[item] = had + qty;
+        }
+
+        hoarder.value.items += value;
+    };
+
     for await (let item of item_db.readDB()) {
         items_by_id.set(+item.id, item);
         items_by_name.set(item.name, +item.id);
@@ -102,6 +157,9 @@ const flags = {
             case "sql":
                 flags.sql = true;
                 break;
+            case "stats":
+                flags.stats = true;
+                break;
             case "clean":
             case "clean-only":
                 break;
@@ -148,6 +206,11 @@ const flags = {
         await sql.init();
     }
 
+    if (flags.stats) {
+        console.log("");
+        await manamarket.init();
+    }
+
     console.log("");
 
     // account:
@@ -174,7 +237,7 @@ const flags = {
 
         for (let item of char.items) {
             if (!items_by_id.has(+item.nameid)) {
-                console.log(`\rremoving ${+item.    amount || 1}x non-existant item ID ${item.nameid} from inventory of character ${char.name} [${char.account_id}:${char.char_id}]`);
+                console.log(`\rremoving ${+item.amount || 1}x non-existant item ID ${item.nameid} from inventory of character ${char.name} [${char.account_id}:${char.char_id}]`);
                 stats.inventory.pruned += +item.amount;
                 mod = true;
             } else if (+item.amount < 1) {
@@ -188,6 +251,10 @@ const flags = {
             } else {
                 items_filtered.push(item);
             }
+
+            if (flags.stats) {
+                maybeAddToHoarders(+char.account_id, char.name, +char.char_id, +item.nameid, +item.amount || 1);
+            }
         }
 
         if (mod)
@@ -244,6 +311,10 @@ const flags = {
             } else {
                 items_filtered.push(item);
             }
+
+            if (flags.stats) {
+                maybeAddToHoarders(+storage.account_id, "", 0, +item.nameid, +item.amount || 1);
+            }
         }
 
         if (mod)
@@ -273,6 +344,51 @@ const flags = {
     await storage_writer.finalize();
     await sql.close();
 
+    if (flags.stats) {
+        function partition(all: any[], left: number, right: number) {
+            var pivot   = all[Math.floor((right + left) / 2)], //middle element
+                i       = left, //left pointer
+                j       = right; //right pointer
+            while (i <= j) {
+                while (all[i][1].value.items < pivot[1].value.items) {
+                    i++;
+                }
+                while (all[j][1].value.items > pivot[1].value.items) {
+                    j--;
+                }
+                if (i <= j) {
+                    [all[i], all[j]] = [all[j], all[i]]; // swap them
+                    i++;
+                    j--;
+                }
+            }
+            return i;
+        }
+
+        function quickSort(all: any[], left: number, right: number) {
+            if (all.length > 1) {
+                const index = partition(all, left, right); //index returned from partition
+                if (left < index - 1) { //more elements on the left side of the pivot
+                    quickSort(all, left, index - 1);
+                }
+                if (index < right) { //more elements on the right side of the pivot
+                    quickSort(all, index, right);
+                }
+            }
+            return all;
+        }
+
+        console.log("Sorting hoarders...");
+        const entries = Array.from(hoarders.entries());
+        const sorted = quickSort(entries, 0, entries.length - 1).reverse();
+        const json = JSON.stringify(sorted, null, "\t");
+        const encoder = new TextEncoder();
+
+        console.log("Writing hoarders.json...");
+        await Deno.mkdir("log", {recursive: true});
+        await Deno.writeTextFile("log/hoarders.json", json);
+    }
+
     console.info("\r                                                            \n=== all done ===");
     console.info(`removed ${stats.inventory.removed} existant, ${stats.inventory.pruned} non-existant and ${stats.inventory.stub} stub items from the inventory of ${stats.inventory.chars} characters`);
     console.info(`removed ${stats.storage.removed} existant, ${stats.storage.pruned} non-existant and ${stats.storage.stub} stub items from the storage of ${stats.storage.accounts} accounts`);
diff --git a/server/frob/itemsXML.ts b/server/frob/itemsXML.ts
new file mode 100644
index 0000000..0a8e117
--- /dev/null
+++ b/server/frob/itemsXML.ts
@@ -0,0 +1,87 @@
+import { DOMParser, Element } from "https://deno.land/x/deno_dom@v0.1.2-alpha3/deno-dom-wasm.ts";
+
+class itemsXML {
+    private repo: string = "";
+    private encoder: TextEncoder;
+    private decoder: TextDecoder;
+    private items_by_name: Map<string, number> = new Map();
+    private items_by_id: Map<number, any> = new Map();
+
+    constructor (repo: string = "client-data") {
+        this.repo = repo;
+        this.encoder = new TextEncoder();
+        this.decoder = new TextDecoder("utf-8");
+    }
+
+    async init () {
+        console.log("Fetching items xml files from client-data...");
+
+        await this.fetchFile("items.xml");
+    }
+
+    private async fetchFile (path: string) {
+        const raw = await Deno.readFile(`${this.repo}/${path}`);
+        const xml = this.decoder.decode(raw);
+
+        await this.parseXML(path, xml);
+    }
+
+    private async parseXML (path: string, xml: string) {
+        Deno.write(Deno.stdout.rid, this.encoder.encode(`                                                                            \r⌛ parsing ${path}...`));
+
+        const domparser = new DOMParser();
+
+        if (xml.startsWith("<?")) {
+            // remove the xml doctype
+            xml = xml.split("\n").slice(1).join("\n");
+        }
+
+        if (xml.startsWith("<items>\n  <its:")) {
+            // translation stuff
+            xml = "<items>\n" + xml.split("\n").slice(7).join("\n");
+        }
+
+        const root = domparser.parseFromString(xml, "text/html")!;
+
+        if (root) {
+            const items = root.querySelectorAll("item");
+            const includes = root.querySelectorAll("include");
+
+            for (const el of includes) {
+                const tag: Element = el as Element;
+                await this.fetchFile(tag.attributes.name);
+            }
+
+            for (const el of items) {
+                const tag: Element = el as Element;
+
+                const item = {
+                    id: +tag.attributes.id,
+                    name: tag.attributes.name,
+                    type: tag.attributes.type,
+                };
+
+                this.items_by_id.set(item.id, item);
+                this.items_by_name.set(item.name, item.id);
+            }
+        }
+    }
+
+    getItem (item: string | number) {
+        if (typeof item === "string") {
+            const id = this.items_by_name.get(item);
+
+            if (id) {
+                return this.items_by_id.get(id) || null;
+            }
+        } else if (typeof item === "number") {
+            return this.items_by_id.get(item) || null;
+        }
+
+        return null;
+    }
+}
+
+export {
+    itemsXML,
+}
diff --git a/server/frob/manamarket.ts b/server/frob/manamarket.ts
new file mode 100644
index 0000000..1ea3918
--- /dev/null
+++ b/server/frob/manamarket.ts
@@ -0,0 +1,63 @@
+import { DOMParser, Element } from "https://deno.land/x/deno_dom@v0.1.2-alpha3/deno-dom-wasm.ts";
+import { itemsXML } from "./itemsXML.ts";
+
+class ManaMarketHandler {
+    private stats_html: string = "";
+    items: Map<number, any> = new Map();
+    private itemsXML: itemsXML;
+
+    constructor (server: string = "https://server.themanaworld.org") {
+        this.stats_html = `${server}/manamarket_stats.html`;
+        this.itemsXML = new itemsXML();
+    }
+
+    async init () {
+        console.log("Fetching ManaMarket stats...");
+
+        const res = await fetch(this.stats_html);
+        const html = await res.text();
+
+        await this.itemsXML.init();
+
+        this.parseHTML(html);
+    }
+
+    private parseHTML (html: string) {
+        console.log("\r                                                                                                  \rParsing ManaMarket stats...                ");
+
+        const domparser = new DOMParser();
+        const root = domparser.parseFromString(html, "text/html")!;
+        const rows = root.querySelectorAll("tr:nth-of-type(n+3)");
+
+        for (const row of rows) {
+            const el = row as Element;
+            const name = el.children[0].innerHTML as string;
+            const xml = this.itemsXML.getItem(name);
+
+            if (!xml) {
+                console.warn(`Cannot find item \`${name}\` in the item xml files`);
+                continue;
+            }
+
+            const item = {
+                name: name,
+                id: xml.id,
+                totalSold: +el.children[1].innerHTML.split(",").join(""),
+                minValue: +el.children[2].innerHTML.split(",").join(""),
+                maxValue: +el.children[3].innerHTML.split(",").join(""),
+                averageValue: {
+                    week: +el.children[4].innerHTML.split(",").join(""),
+                    month: +el.children[5].innerHTML.split(",").join(""),
+                    overall: +el.children[6].innerHTML.split(",").join(""),
+                },
+                lastSold: new Date(el.children[7].innerHTML),
+            };
+
+            this.items.set(item.id, item);
+        }
+    }
+}
+
+export {
+    ManaMarketHandler,
+}