summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorgumi <git@gumi.ca>2020-10-04 22:08:09 -0400
committergumi <git@gumi.ca>2020-10-04 22:08:09 -0400
commit5a87ab2dd7fea89b35e3408481575c51f980ae3b (patch)
tree405b966884f5e43556781e69b486afb01ad96573
parent25a3c3fefae87d55da172f2aaef6a27987f439fd (diff)
downloadtools-5a87ab2dd7fea89b35e3408481575c51f980ae3b.tar.gz
tools-5a87ab2dd7fea89b35e3408481575c51f980ae3b.tar.bz2
tools-5a87ab2dd7fea89b35e3408481575c51f980ae3b.tar.xz
tools-5a87ab2dd7fea89b35e3408481575c51f980ae3b.zip
[frob] add a mode to list item hoardersgithub/fork/Helianthella/hoarders
-rw-r--r--server/frob/README.md5
-rw-r--r--server/frob/index.ts120
-rw-r--r--server/frob/itemsXML.ts87
-rw-r--r--server/frob/manamarket.ts63
4 files changed, 273 insertions, 2 deletions
diff --git a/server/frob/README.md b/server/frob/README.md
index 88c6880..0d64f7d 100644
--- a/server/frob/README.md
+++ b/server/frob/README.md
@@ -18,4 +18,9 @@ make frob items="item..item" # item range
make frob items="item,item item-item item" # mixed
```
+### Special flags
+- `--dry`: performs a dry run (does not modify databases)
+- `--sql`: converts the flatfile databases to MySQL tables in the `legacy` database
+- `--stats`: compiles statistics about general item distribution
+
[Deno]: https://deno.land
diff --git a/server/frob/index.ts b/server/frob/index.ts
index df06975..7f92a59 100644
--- a/server/frob/index.ts
+++ b/server/frob/index.ts
@@ -1,5 +1,6 @@
// this script removes specified items from inventories and storage
-import { SQLHandler } from "./sql.ts"
+import { SQLHandler } from "./sql.ts";
+import { ManaMarketHandler } from "./manamarket.ts";
import { LoginParser, LoginSQL } from "./login.ts";
import { CharParser, CharWriter, CharSQL } from "./char.ts";
import { AccregParser, AccregSQL } from "./accreg.ts";
@@ -10,6 +11,7 @@ import { ItemDB } from "./itemdb.ts";
const args: string[] = Deno.args.slice(1);
const to_remove: Set<number> = new Set();
const sql = new SQLHandler();
+const manamarket = new ManaMarketHandler();
const login_parser = new LoginParser();
const char_parser = new CharParser();
const accreg_parser = new AccregParser();
@@ -24,6 +26,10 @@ const storage_writer = new StorageWriter();
const storage_SQL = new StorageSQL(sql);
const item_db = new ItemDB();
+// at which value an item is considered SSR
+const rare_threshold = 3000000;
+const hoarders: Map<number, any> = new Map();
+
const stats = {
inventory: {
removed: 0,
@@ -44,6 +50,7 @@ const stats = {
const flags = {
dry_run: false,
sql: false,
+ stats: false,
};
@@ -51,6 +58,54 @@ const flags = {
const items_by_id = new Map(); // holds full item info
const items_by_name = new Map(); // holds references to the former
+ // TODO: split the hoarder stuff into hoarders.ts
+ const getHoarder = (account_id: number) => {
+ let hoarder = hoarders.get(account_id);
+
+ if (!hoarder) {
+ hoarder = hoarders.set(account_id, {
+ value: {
+ items: 0,
+ //gp: 0,
+ },
+ storage: {},
+ inventories: {},
+ }).get(account_id);
+ }
+
+ return hoarder;
+ };
+
+ const maybeAddToHoarders = (account_id: number, char_name: string, char_id: number, item: number, qty: number = 1) => {
+ // make sure all values are the expected format (Deno is weird sometimes)
+ [account_id, char_id, item, qty] = [+account_id, +char_id, +item, +qty];
+
+ const data = manamarket.items.get(item);
+ let value = data ? data.averageValue.overall : 0;
+
+ if (value < rare_threshold) {
+ return;
+ }
+
+ const hoarder = getHoarder(account_id);
+
+ item = (items_by_id.get(item)).name;
+
+ if (char_id && char_name) {
+ if (!Reflect.has(hoarder.inventories, char_name)) {
+ hoarder.inventories[char_name] = new Map();
+ }
+
+ const had = hoarder.inventories[char_name][item] || 0;
+ hoarder.inventories[char_name][item] = had + qty;
+ } else {
+ const had = hoarder.storage[item] || 0;
+ hoarder.storage[item] = had + qty;
+ }
+
+ hoarder.value.items += value;
+ };
+
for await (let item of item_db.readDB()) {
items_by_id.set(+item.id, item);
items_by_name.set(item.name, +item.id);
@@ -102,6 +157,9 @@ const flags = {
case "sql":
flags.sql = true;
break;
+ case "stats":
+ flags.stats = true;
+ break;
case "clean":
case "clean-only":
break;
@@ -148,6 +206,11 @@ const flags = {
await sql.init();
}
+ if (flags.stats) {
+ console.log("");
+ await manamarket.init();
+ }
+
console.log("");
// account:
@@ -174,7 +237,7 @@ const flags = {
for (let item of char.items) {
if (!items_by_id.has(+item.nameid)) {
- console.log(`\rremoving ${+item. amount || 1}x non-existant item ID ${item.nameid} from inventory of character ${char.name} [${char.account_id}:${char.char_id}]`);
+ console.log(`\rremoving ${+item.amount || 1}x non-existant item ID ${item.nameid} from inventory of character ${char.name} [${char.account_id}:${char.char_id}]`);
stats.inventory.pruned += +item.amount;
mod = true;
} else if (+item.amount < 1) {
@@ -188,6 +251,10 @@ const flags = {
} else {
items_filtered.push(item);
}
+
+ if (flags.stats) {
+ maybeAddToHoarders(+char.account_id, char.name, +char.char_id, +item.nameid, +item.amount || 1);
+ }
}
if (mod)
@@ -244,6 +311,10 @@ const flags = {
} else {
items_filtered.push(item);
}
+
+ if (flags.stats) {
+ maybeAddToHoarders(+storage.account_id, "", 0, +item.nameid, +item.amount || 1);
+ }
}
if (mod)
@@ -273,6 +344,51 @@ const flags = {
await storage_writer.finalize();
await sql.close();
+ if (flags.stats) {
+ function partition(all: any[], left: number, right: number) {
+ var pivot = all[Math.floor((right + left) / 2)], //middle element
+ i = left, //left pointer
+ j = right; //right pointer
+ while (i <= j) {
+ while (all[i][1].value.items < pivot[1].value.items) {
+ i++;
+ }
+ while (all[j][1].value.items > pivot[1].value.items) {
+ j--;
+ }
+ if (i <= j) {
+ [all[i], all[j]] = [all[j], all[i]]; // swap them
+ i++;
+ j--;
+ }
+ }
+ return i;
+ }
+
+ function quickSort(all: any[], left: number, right: number) {
+ if (all.length > 1) {
+ const index = partition(all, left, right); //index returned from partition
+ if (left < index - 1) { //more elements on the left side of the pivot
+ quickSort(all, left, index - 1);
+ }
+ if (index < right) { //more elements on the right side of the pivot
+ quickSort(all, index, right);
+ }
+ }
+ return all;
+ }
+
+ console.log("Sorting hoarders...");
+ const entries = Array.from(hoarders.entries());
+ const sorted = quickSort(entries, 0, entries.length - 1).reverse();
+ const json = JSON.stringify(sorted, null, "\t");
+ const encoder = new TextEncoder();
+
+ console.log("Writing hoarders.json...");
+ await Deno.mkdir("log", {recursive: true});
+ await Deno.writeTextFile("log/hoarders.json", json);
+ }
+
console.info("\r \n=== all done ===");
console.info(`removed ${stats.inventory.removed} existant, ${stats.inventory.pruned} non-existant and ${stats.inventory.stub} stub items from the inventory of ${stats.inventory.chars} characters`);
console.info(`removed ${stats.storage.removed} existant, ${stats.storage.pruned} non-existant and ${stats.storage.stub} stub items from the storage of ${stats.storage.accounts} accounts`);
diff --git a/server/frob/itemsXML.ts b/server/frob/itemsXML.ts
new file mode 100644
index 0000000..0a8e117
--- /dev/null
+++ b/server/frob/itemsXML.ts
@@ -0,0 +1,87 @@
+import { DOMParser, Element } from "https://deno.land/x/deno_dom@v0.1.2-alpha3/deno-dom-wasm.ts";
+
+class itemsXML {
+ private repo: string = "";
+ private encoder: TextEncoder;
+ private decoder: TextDecoder;
+ private items_by_name: Map<string, number> = new Map();
+ private items_by_id: Map<number, any> = new Map();
+
+ constructor (repo: string = "client-data") {
+ this.repo = repo;
+ this.encoder = new TextEncoder();
+ this.decoder = new TextDecoder("utf-8");
+ }
+
+ async init () {
+ console.log("Fetching items xml files from client-data...");
+
+ await this.fetchFile("items.xml");
+ }
+
+ private async fetchFile (path: string) {
+ const raw = await Deno.readFile(`${this.repo}/${path}`);
+ const xml = this.decoder.decode(raw);
+
+ await this.parseXML(path, xml);
+ }
+
+ private async parseXML (path: string, xml: string) {
+ Deno.write(Deno.stdout.rid, this.encoder.encode(` \r⌛ parsing ${path}...`));
+
+ const domparser = new DOMParser();
+
+ if (xml.startsWith("<?")) {
+ // remove the xml doctype
+ xml = xml.split("\n").slice(1).join("\n");
+ }
+
+ if (xml.startsWith("<items>\n <its:")) {
+ // translation stuff
+ xml = "<items>\n" + xml.split("\n").slice(7).join("\n");
+ }
+
+ const root = domparser.parseFromString(xml, "text/html")!;
+
+ if (root) {
+ const items = root.querySelectorAll("item");
+ const includes = root.querySelectorAll("include");
+
+ for (const el of includes) {
+ const tag: Element = el as Element;
+ await this.fetchFile(tag.attributes.name);
+ }
+
+ for (const el of items) {
+ const tag: Element = el as Element;
+
+ const item = {
+ id: +tag.attributes.id,
+ name: tag.attributes.name,
+ type: tag.attributes.type,
+ };
+
+ this.items_by_id.set(item.id, item);
+ this.items_by_name.set(item.name, item.id);
+ }
+ }
+ }
+
+ getItem (item: string | number) {
+ if (typeof item === "string") {
+ const id = this.items_by_name.get(item);
+
+ if (id) {
+ return this.items_by_id.get(id) || null;
+ }
+ } else if (typeof item === "number") {
+ return this.items_by_id.get(item) || null;
+ }
+
+ return null;
+ }
+}
+
+export {
+ itemsXML,
+}
diff --git a/server/frob/manamarket.ts b/server/frob/manamarket.ts
new file mode 100644
index 0000000..1ea3918
--- /dev/null
+++ b/server/frob/manamarket.ts
@@ -0,0 +1,63 @@
+import { DOMParser, Element } from "https://deno.land/x/deno_dom@v0.1.2-alpha3/deno-dom-wasm.ts";
+import { itemsXML } from "./itemsXML.ts";
+
+class ManaMarketHandler {
+ private stats_html: string = "";
+ items: Map<number, any> = new Map();
+ private itemsXML: itemsXML;
+
+ constructor (server: string = "https://server.themanaworld.org") {
+ this.stats_html = `${server}/manamarket_stats.html`;
+ this.itemsXML = new itemsXML();
+ }
+
+ async init () {
+ console.log("Fetching ManaMarket stats...");
+
+ const res = await fetch(this.stats_html);
+ const html = await res.text();
+
+ await this.itemsXML.init();
+
+ this.parseHTML(html);
+ }
+
+ private parseHTML (html: string) {
+ console.log("\r \rParsing ManaMarket stats... ");
+
+ const domparser = new DOMParser();
+ const root = domparser.parseFromString(html, "text/html")!;
+ const rows = root.querySelectorAll("tr:nth-of-type(n+3)");
+
+ for (const row of rows) {
+ const el = row as Element;
+ const name = el.children[0].innerHTML as string;
+ const xml = this.itemsXML.getItem(name);
+
+ if (!xml) {
+ console.warn(`Cannot find item \`${name}\` in the item xml files`);
+ continue;
+ }
+
+ const item = {
+ name: name,
+ id: xml.id,
+ totalSold: +el.children[1].innerHTML.split(",").join(""),
+ minValue: +el.children[2].innerHTML.split(",").join(""),
+ maxValue: +el.children[3].innerHTML.split(",").join(""),
+ averageValue: {
+ week: +el.children[4].innerHTML.split(",").join(""),
+ month: +el.children[5].innerHTML.split(",").join(""),
+ overall: +el.children[6].innerHTML.split(",").join(""),
+ },
+ lastSold: new Date(el.children[7].innerHTML),
+ };
+
+ this.items.set(item.id, item);
+ }
+ }
+}
+
+export {
+ ManaMarketHandler,
+}