diff options
| -rw-r--r-- | server/frob/README.md | 5 | ||||
| -rw-r--r-- | server/frob/index.ts | 120 | ||||
| -rw-r--r-- | server/frob/itemsXML.ts | 87 | ||||
| -rw-r--r-- | server/frob/manamarket.ts | 63 |
4 files changed, 273 insertions, 2 deletions
diff --git a/server/frob/README.md b/server/frob/README.md index 88c6880..0d64f7d 100644 --- a/server/frob/README.md +++ b/server/frob/README.md @@ -18,4 +18,9 @@ make frob items="item..item" # item range make frob items="item,item item-item item" # mixed ``` +### Special flags +- `--dry`: performs a dry run (does not modify databases) +- `--sql`: converts the flatfile databases to MySQL tables in the `legacy` database +- `--stats`: compiles statistics about general item distribution + [Deno]: https://deno.land diff --git a/server/frob/index.ts b/server/frob/index.ts index df06975..7f92a59 100644 --- a/server/frob/index.ts +++ b/server/frob/index.ts @@ -1,5 +1,6 @@ // this script removes specified items from inventories and storage -import { SQLHandler } from "./sql.ts" +import { SQLHandler } from "./sql.ts"; +import { ManaMarketHandler } from "./manamarket.ts"; import { LoginParser, LoginSQL } from "./login.ts"; import { CharParser, CharWriter, CharSQL } from "./char.ts"; import { AccregParser, AccregSQL } from "./accreg.ts"; @@ -10,6 +11,7 @@ import { ItemDB } from "./itemdb.ts"; const args: string[] = Deno.args.slice(1); const to_remove: Set<number> = new Set(); const sql = new SQLHandler(); +const manamarket = new ManaMarketHandler(); const login_parser = new LoginParser(); const char_parser = new CharParser(); const accreg_parser = new AccregParser(); @@ -24,6 +26,10 @@ const storage_writer = new StorageWriter(); const storage_SQL = new StorageSQL(sql); const item_db = new ItemDB(); +// at which value an item is considered SSR +const rare_threshold = 3000000; +const hoarders: Map<number, any> = new Map(); + const stats = { inventory: { removed: 0, @@ -44,6 +50,7 @@ const stats = { const flags = { dry_run: false, sql: false, + stats: false, }; @@ -51,6 +58,54 @@ const flags = { const items_by_id = new Map(); // holds full item info const items_by_name = new Map(); // holds references to the former + // TODO: split the hoarder stuff into hoarders.ts + const getHoarder = (account_id: number) => { + let hoarder = hoarders.get(account_id); + + if (!hoarder) { + hoarder = hoarders.set(account_id, { + value: { + items: 0, + //gp: 0, + }, + storage: {}, + inventories: {}, + }).get(account_id); + } + + return hoarder; + }; + + const maybeAddToHoarders = (account_id: number, char_name: string, char_id: number, item: number, qty: number = 1) => { + // make sure all values are the expected format (Deno is weird sometimes) + [account_id, char_id, item, qty] = [+account_id, +char_id, +item, +qty]; + + const data = manamarket.items.get(item); + let value = data ? data.averageValue.overall : 0; + + if (value < rare_threshold) { + return; + } + + const hoarder = getHoarder(account_id); + + item = (items_by_id.get(item)).name; + + if (char_id && char_name) { + if (!Reflect.has(hoarder.inventories, char_name)) { + hoarder.inventories[char_name] = new Map(); + } + + const had = hoarder.inventories[char_name][item] || 0; + hoarder.inventories[char_name][item] = had + qty; + } else { + const had = hoarder.storage[item] || 0; + hoarder.storage[item] = had + qty; + } + + hoarder.value.items += value; + }; + for await (let item of item_db.readDB()) { items_by_id.set(+item.id, item); items_by_name.set(item.name, +item.id); @@ -102,6 +157,9 @@ const flags = { case "sql": flags.sql = true; break; + case "stats": + flags.stats = true; + break; case "clean": case "clean-only": break; @@ -148,6 +206,11 @@ const flags = { await sql.init(); } + if (flags.stats) { + console.log(""); + await manamarket.init(); + } + console.log(""); // account: @@ -174,7 +237,7 @@ const flags = { for (let item of char.items) { if (!items_by_id.has(+item.nameid)) { - console.log(`\rremoving ${+item. amount || 1}x non-existant item ID ${item.nameid} from inventory of character ${char.name} [${char.account_id}:${char.char_id}]`); + console.log(`\rremoving ${+item.amount || 1}x non-existant item ID ${item.nameid} from inventory of character ${char.name} [${char.account_id}:${char.char_id}]`); stats.inventory.pruned += +item.amount; mod = true; } else if (+item.amount < 1) { @@ -188,6 +251,10 @@ const flags = { } else { items_filtered.push(item); } + + if (flags.stats) { + maybeAddToHoarders(+char.account_id, char.name, +char.char_id, +item.nameid, +item.amount || 1); + } } if (mod) @@ -244,6 +311,10 @@ const flags = { } else { items_filtered.push(item); } + + if (flags.stats) { + maybeAddToHoarders(+storage.account_id, "", 0, +item.nameid, +item.amount || 1); + } } if (mod) @@ -273,6 +344,51 @@ const flags = { await storage_writer.finalize(); await sql.close(); + if (flags.stats) { + function partition(all: any[], left: number, right: number) { + var pivot = all[Math.floor((right + left) / 2)], //middle element + i = left, //left pointer + j = right; //right pointer + while (i <= j) { + while (all[i][1].value.items < pivot[1].value.items) { + i++; + } + while (all[j][1].value.items > pivot[1].value.items) { + j--; + } + if (i <= j) { + [all[i], all[j]] = [all[j], all[i]]; // swap them + i++; + j--; + } + } + return i; + } + + function quickSort(all: any[], left: number, right: number) { + if (all.length > 1) { + const index = partition(all, left, right); //index returned from partition + if (left < index - 1) { //more elements on the left side of the pivot + quickSort(all, left, index - 1); + } + if (index < right) { //more elements on the right side of the pivot + quickSort(all, index, right); + } + } + return all; + } + + console.log("Sorting hoarders..."); + const entries = Array.from(hoarders.entries()); + const sorted = quickSort(entries, 0, entries.length - 1).reverse(); + const json = JSON.stringify(sorted, null, "\t"); + const encoder = new TextEncoder(); + + console.log("Writing hoarders.json..."); + await Deno.mkdir("log", {recursive: true}); + await Deno.writeTextFile("log/hoarders.json", json); + } + console.info("\r \n=== all done ==="); console.info(`removed ${stats.inventory.removed} existant, ${stats.inventory.pruned} non-existant and ${stats.inventory.stub} stub items from the inventory of ${stats.inventory.chars} characters`); console.info(`removed ${stats.storage.removed} existant, ${stats.storage.pruned} non-existant and ${stats.storage.stub} stub items from the storage of ${stats.storage.accounts} accounts`); diff --git a/server/frob/itemsXML.ts b/server/frob/itemsXML.ts new file mode 100644 index 0000000..0a8e117 --- /dev/null +++ b/server/frob/itemsXML.ts @@ -0,0 +1,87 @@ +import { DOMParser, Element } from "https://deno.land/x/deno_dom@v0.1.2-alpha3/deno-dom-wasm.ts"; + +class itemsXML { + private repo: string = ""; + private encoder: TextEncoder; + private decoder: TextDecoder; + private items_by_name: Map<string, number> = new Map(); + private items_by_id: Map<number, any> = new Map(); + + constructor (repo: string = "client-data") { + this.repo = repo; + this.encoder = new TextEncoder(); + this.decoder = new TextDecoder("utf-8"); + } + + async init () { + console.log("Fetching items xml files from client-data..."); + + await this.fetchFile("items.xml"); + } + + private async fetchFile (path: string) { + const raw = await Deno.readFile(`${this.repo}/${path}`); + const xml = this.decoder.decode(raw); + + await this.parseXML(path, xml); + } + + private async parseXML (path: string, xml: string) { + Deno.write(Deno.stdout.rid, this.encoder.encode(` \r⌛ parsing ${path}...`)); + + const domparser = new DOMParser(); + + if (xml.startsWith("<?")) { + // remove the xml doctype + xml = xml.split("\n").slice(1).join("\n"); + } + + if (xml.startsWith("<items>\n <its:")) { + // translation stuff + xml = "<items>\n" + xml.split("\n").slice(7).join("\n"); + } + + const root = domparser.parseFromString(xml, "text/html")!; + + if (root) { + const items = root.querySelectorAll("item"); + const includes = root.querySelectorAll("include"); + + for (const el of includes) { + const tag: Element = el as Element; + await this.fetchFile(tag.attributes.name); + } + + for (const el of items) { + const tag: Element = el as Element; + + const item = { + id: +tag.attributes.id, + name: tag.attributes.name, + type: tag.attributes.type, + }; + + this.items_by_id.set(item.id, item); + this.items_by_name.set(item.name, item.id); + } + } + } + + getItem (item: string | number) { + if (typeof item === "string") { + const id = this.items_by_name.get(item); + + if (id) { + return this.items_by_id.get(id) || null; + } + } else if (typeof item === "number") { + return this.items_by_id.get(item) || null; + } + + return null; + } +} + +export { + itemsXML, +} diff --git a/server/frob/manamarket.ts b/server/frob/manamarket.ts new file mode 100644 index 0000000..1ea3918 --- /dev/null +++ b/server/frob/manamarket.ts @@ -0,0 +1,63 @@ +import { DOMParser, Element } from "https://deno.land/x/deno_dom@v0.1.2-alpha3/deno-dom-wasm.ts"; +import { itemsXML } from "./itemsXML.ts"; + +class ManaMarketHandler { + private stats_html: string = ""; + items: Map<number, any> = new Map(); + private itemsXML: itemsXML; + + constructor (server: string = "https://server.themanaworld.org") { + this.stats_html = `${server}/manamarket_stats.html`; + this.itemsXML = new itemsXML(); + } + + async init () { + console.log("Fetching ManaMarket stats..."); + + const res = await fetch(this.stats_html); + const html = await res.text(); + + await this.itemsXML.init(); + + this.parseHTML(html); + } + + private parseHTML (html: string) { + console.log("\r \rParsing ManaMarket stats... "); + + const domparser = new DOMParser(); + const root = domparser.parseFromString(html, "text/html")!; + const rows = root.querySelectorAll("tr:nth-of-type(n+3)"); + + for (const row of rows) { + const el = row as Element; + const name = el.children[0].innerHTML as string; + const xml = this.itemsXML.getItem(name); + + if (!xml) { + console.warn(`Cannot find item \`${name}\` in the item xml files`); + continue; + } + + const item = { + name: name, + id: xml.id, + totalSold: +el.children[1].innerHTML.split(",").join(""), + minValue: +el.children[2].innerHTML.split(",").join(""), + maxValue: +el.children[3].innerHTML.split(",").join(""), + averageValue: { + week: +el.children[4].innerHTML.split(",").join(""), + month: +el.children[5].innerHTML.split(",").join(""), + overall: +el.children[6].innerHTML.split(",").join(""), + }, + lastSold: new Date(el.children[7].innerHTML), + }; + + this.items.set(item.id, item); + } + } +} + +export { + ManaMarketHandler, +} |