Rewrite parser

This completely removes any need to look behind for already parsed records,
and adds a second pass that assigns stats to records that did not have that
information known at the time of parsing.

With the testing data, this changed the number of records with no stat
information from approximately 1/6 to none (!).

The parser is also much cleaner now, at least as much as a parser built around
regular expressions is clean. Apparently the late night coding me really likes
nesting closures.

This closes #7.
This closes #13.

1 files changed, 206 insertions, 162 deletions

diff --git a/public/js/mv/parse.js b/public/js/mv/parse.js
index f8c1fd7..396d88e 100644
--- a/public/js/mv/parse.js
+++ b/public/js/mv/parse.js
@@ -1,175 +1,219 @@
 var mv = function(mv) {
-  mv.parser = function() {
-    var parser = {};
-    /* The most recent information of a pc's stat */
-    var pcstat = {};
-    /*
-     * The first recorded state of a pc's stat.
-     * This is saved for a second pass, in which instances unknown at the time can have the pc's stat applied.
-     */
-    var firstpcstat = {};
-    /*
-     * The time stamp of the last unknown instance.
-     */
-    var fullyDefinedCutoff = 0;
-    parser.records = [];
-    parser.fullyDefinedCutoff = function() { return fullyDefinedCutoff; };
-    parser.parseRecords = function(data) {
-      var spl = data.split(/\r?\n/);
-      spl.forEach(function(e, i) {
-        var d;
-        d = e.match(/^(\d+\.\d+) PC(\d+) (\d+):(\d+),(\d+) GAINXP (\d+) (\d+) (\w+)/);
-        if (d) {
-          var mapSID = parseInt(d[3]);
-          var ts = new Date(0);
-          ts.setUTCSeconds(d[1]);
-          var rec = {
-            date: ts,
-            pc:   parseInt(d[2]),
-            map:  map.nameByServerID(parseInt(d[3]), ts),
-            x:    parseInt(d[4]),
-            y:    parseInt(d[5]),
-            e:    parseInt(d[6]),
-            j:    parseInt(d[7]),
-            type: d[8],
-            pcstat: pcstat[d[2]],
-            target: "UNKNOWN",
-            dmg: -1010,
-            wpn: "UNKNOWN",
-            atktype: "UNKNOWN"
-          };
-          if (pcstat[d[2]] == undefined && (!fullyDefinedCutoff || ts > fullyDefinedCutoff)) {
-            fullyDefinedCutoff = ts;
-          }
-          /* XXX: Fragile horrible and unstructured, this whole thing needs a rewrite really */
-          if (i >= 2 && rec.type == "KILLXP") {
-            d = spl[i - 1].match(/^(\d+\.\d+) MOB(\d+) DEAD/);
-            if (d) {
-              var mID = parseInt(d[2]);
-              /* There's a massive wealth of data that can be collected from this. Number of assailants, weapons used, the relationships with the assailants... this can't be done with a simple lookbehind. For now, just extract what mob it was, and what the killing weapon used was. */
-              d = spl[i - 2].match(/^(\d+\.\d+) PC(\d+) (\d+):(\d+),(\d+) WPNDMG MOB(\d+) (\d+) FOR (\d+) WPN (\d+)/);
-              if (d) {
-                softAssert(mID == parseInt(d[6]), "Integrity error: MOB ID mismatch!");
-  //               softAssert(rec.pc == parseInt(d[2]), "Integrity error: PC ID mismatch!");
-                rec.target = mob.nameByServerID(d[7]);
-                softAssert(rec.target, "Unknown target!")
-                rec.dmg = parseInt(d[8]);
-                rec.wpn = item.nameByServerID(d[9]);
-                rec.atktype = "Physical";
-              } else {
-                /* Not weapon damage, perhaps it was spell damage? */
-                d = spl[i - 2].match(/^(\d+\.\d+) PC(\d+) (\d+):(\d+),(\d+) SPELLDMG MOB(\d+) (\d+) FOR (\d+) BY ([^ ]+)/);
-                if (d) {
-                  rec.target = mob.nameByServerID(d[7]);
-                  rec.dmg = parseInt(d[8]);
-                  rec.wpn = d[9];
-                  rec.atktype = "Magical";
-                }
-//                 console.error("No match (deathblow):", spl[i - 2]);
-              }
-            } else {
-              d = spl[i - 1].match(/^(\d+\.\d+) PC(\d+) (\d+):(\d+),(\d+) GAINXP (\d+) (\d+) (\w+)/);
-              if (d) {
-                var clone = parser.records[parser.records.length - 1];
-                softAssert(rec.map == clone.map, "Integrity error: MAP ID mismatch!");
-                rec.target = clone.target;
-                softAssert(rec.target, "Unknown (cloned) target!");
-                rec.dmg = clone.dmg; /* FIXME: Take into account actual assist damage */
-                rec.wpn = clone.wpn;
-                rec.atktype = clone.atktype; /* FIXME: Take into account what the assists used */
-              } else {
-//                 console.error("No match (clone):", spl[i - 1]);
-              }
-            }
+  mv.parser = {
+    records: [],
+    fullyDefinedCutoff: function() { return fullyDefinedCutoff; },
+    parseRecords: parseRecords,
+    parseScrubbed: parseScrubbed,
+    postProcessing: postProcessing,
+    createBlobLink: createBlobLink
+  };
+  /* The most recent information of a pc's stat */
+  var pcstat = {};
+  /*
+    * The first recorded state of a pc's stat.
+    * This is saved for a second pass, in which instances unknown at the time can have the pc's stat applied.
+    */
+  var firstpcstat = {};
+  /*
+    * The time stamp of the last unknown instance.
+    */
+  var fullyDefinedCutoff = 0;
+  /*
+   * 0: No mob just died
+   * Positive: A mob just died, this is its ID.
+   */
+  var killedMobID = 0;
+  /*
+   * mob ID -> { mobClass, player IDs -> { total, weapon names -> { sum damage } } }
+   */
+  var combat = {};
+  function freeMob() {
+    /* We no longer need detailed information on the mob's combat. */
+    if (!killedMobID) {
+      return;
+    }
+    delete combat[killedMobID];
+    killedMobID = 0;
+  }
+  function parseRecords(data) {
+    var spl = data.split(/\r?\n/);
+    spl.forEach(function(e) {
+      /* Check for each of the record types we're looking for. */
+      if (checkDmg(e) || checkStat(e)) {
+        /* We have a record that has nothing to do with killed mobs, so no mob just died. */
+        freeMob();
+      } else {
+        checkXP(e) || checkMobDeath(e);
+        /* These functions deal directly with killed mobs and will handle setting or clearing of the ID internally. */
+      }
+    });
+  };
+  function checkXP(e) {
+    /* Try to parse an XP record. */
+    var d = e.match(/^(\d+\.\d+) PC(\d+) (\d+):(\d+),(\d+) GAINXP (\d+) (\d+) (\w+)/);;
+    if (!d) {
+      return false;
+    }
+    /* We have an XP record. */
+    /* Map's server ID. */
+    var mapSID = parseInt(d[3]);
+    /* Record timestamp. */
+    var ts = new Date(0);
+    ts.setUTCSeconds(d[1]);
+    var rec = {
+      date: ts,
+      pc:   parseInt(d[2]),
+      map:  map.nameByServerID(parseInt(d[3]), ts),
+      x:    parseInt(d[4]),
+      y:    parseInt(d[5]),
+      e:    parseInt(d[6]),
+      j:    parseInt(d[7]),
+      type: d[8],
+      pcstat: pcstat[d[2]],
+      target: "UNKNOWN",
+      dmg: 0,
+      wpn: "UNKNOWN"
+    };
+    if (pcstat[d[2]] == undefined && (!fullyDefinedCutoff || ts > fullyDefinedCutoff)) {
+      /* Undefined, and newer than any existing definedness cutoff */
+      fullyDefinedCutoff = ts;
+    }
+    if (rec.type == "KILLXP") {
+      if (killedMobID && killedMobID in combat && rec.pc in combat[killedMobID]) {
+        var mob = combat[killedMobID];
+        rec.target = mob.mobClass;
+        var weapons = mob[rec.pc];
+        /* We have the needed information. */
+        rec.dmg = weapons.total;
+        var maxDamage = 0;
+        for (var key in weapons) {
+          if (key == "total") {
+            continue;
           }
-          parser.records.push(rec);
-          return;
-        }
-        d = e.match(/^(?:\d+\.\d+) PC(\d+) (?:\d+):(?:\d+),(?:\d+) STAT (\d+) (\d+) (\d+) (\d+) (\d+) (\d+) /);
-        if (d) {
-          var s = {
-            str: parseInt(d[2]),
-            agi: parseInt(d[3]),
-            vit: parseInt(d[4]),
-            int: parseInt(d[5]),
-            dex: parseInt(d[6]),
-            luk: parseInt(d[7])
-          };
-          s.blvl = stat.minLevelForStats(s.str, s.agi, s.vit, s.int, s.dex, s.luk);
-          s.str = Math.floor(s.str / 10);
-          s.agi = Math.floor(s.agi / 10);
-          s.vit = Math.floor(s.vit / 10);
-          s.int = Math.floor(s.int / 10);
-          s.dex = Math.floor(s.dex / 10);
-          s.luk = Math.floor(s.luk / 10);
-          if (!(d[1] in firstpcstat)) {
-            firstpcstat = s;
+          if (weapons[key] > maxDamage) {
+            maxDamage = weapons[key];
+            rec.wpn = key;
           }
-          pcstat[d[1]] = s;
-          return;
-        }
-      });
-    };
-    parser.postProcessing = function() {
-      /* Scrub reference to pc id, and scan up until the fully defined cutoff line, assigning the pcstat from those that logged off */
-      var i = 0;
-      /* This name has way too many warts; suggestions for a replacement welcome! */
-      var postProcessedfullyDefinedCutoff = 0;
-      for (; i != parser.records.length && parser.records[i].date <= fullyDefinedCutoff; ++i) {
-        /* See if we've found out what the stats were from information logged after the record. */
-        if (parser.records[i].pc in firstpcstat) {
-          parser.records[i].pcstat = firstpcstat[parser.records[i].pc];
-        } else {
-          /* If not, adjust the fully defined cutoff. */
-          postProcessedfullyDefinedCutoff = parser.records[i].date;
         }
-        /* Remove references to pc from these records. */
-        delete parser.records[i].pc;
       }
-      /* Remove references to pc from the remaining records. */
-      for (; i != parser.records.length; ++i) {
-        delete parser.records[i].pc;
-      }
-      fullyDefinedCutoff = postProcessedfullyDefinedCutoff;
+    } else {
+      freeMob();
+    }
+    mv.parser.records.push(rec);
+    return true;
+  }
+  function checkStat(e) {
+    var d = e.match(/^(?:\d+\.\d+) PC(\d+) (?:\d+):(?:\d+),(?:\d+) STAT (\d+) (\d+) (\d+) (\d+) (\d+) (\d+) /);
+    if (!d) {
+      return false;
+    }
+    var s = {
+      str: parseInt(d[2]),
+      agi: parseInt(d[3]),
+      vit: parseInt(d[4]),
+      int: parseInt(d[5]),
+      dex: parseInt(d[6]),
+      luk: parseInt(d[7])
+    };
+    /* The base level is not logged. Derive the minimum needed base level for one to have the stats that they have. */
+    s.blvl = stat.minLevelForStats(s.str, s.agi, s.vit, s.int, s.dex, s.luk);
+    /* Round the stats down to groups of 10. Accurate enough to identify trends and hot spots, fuzzy enough to make unique identification hard. */
+    s.str = Math.floor(s.str / 10);
+    s.agi = Math.floor(s.agi / 10);
+    s.vit = Math.floor(s.vit / 10);
+    s.int = Math.floor(s.int / 10);
+    s.dex = Math.floor(s.dex / 10);
+    s.luk = Math.floor(s.luk / 10);
+    /* Record these. */
+    if (!(d[1] in firstpcstat)) {
+      firstpcstat[d[1]] = s;
+    }
+    pcstat[d[1]] = s;
+    return true;
+  }
+  function checkDmg(e) {
+    var d = e.match(/^(\d+\.\d+) PC(\d+) (\d+):(\d+),(\d+) ([A-Z]+)DMG MOB(\d+) (\d+) FOR (\d+) WPN (\d+)/);
+    if (!d) {
+      return false;
+    }
+    /* Parse out values */
+    var mobClass = mob.nameByServerID(d[8]);
+    var target = parseInt(d[7]);
+    var pc = parseInt(d[2]);
+    var wpn = item.nameByServerID(d[10]);
+    var damage = parseInt(d[9]);
+    /* Update combat state */
+    var mobData = combat[target] || (combat[target] = { mobClass: mobClass });
+    var pcData = mobData[pc] || (mobData[pc] = {});
+    (pcData[wpn] += damage) || (pcData[wpn] = damage);
+    (pcData.total += damage) || (pcData.total = damage);
+  }
+  function checkMobDeath(e) {
+    var d = e.match(/^(\d+\.\d+) MOB(\d+) DEAD/);
+    if (!d) {
+      return false;
     }
-    function softAssert(expr, msg) {
-      if (!expr) {
-        console.error("SOFTASSERT FAILURE: " + msg);
+    killedMobID = parseInt(d[2]);
+    return true;
+  }
+  function postProcessing() {
+    /* Scrub reference to pc id, and scan up until the fully defined cutoff line, assigning the pcstat from those that logged off */
+    var i = 0;
+    /* This name has way too many warts; suggestions for a replacement welcome! */
+    var postProcessedfullyDefinedCutoff = 0;
+    for (; i != mv.parser.records.length && mv.parser.records[i].date <= fullyDefinedCutoff; ++i) {
+      /* See if we've found out what the stats were from information logged after the record. */
+      if (mv.parser.records[i].pc in firstpcstat) {
+        mv.parser.records[i].pcstat = firstpcstat[mv.parser.records[i].pc];
+      } else {
+        /* If not, adjust the fully defined cutoff. */
+        postProcessedfullyDefinedCutoff = mv.parser.records[i].date;
       }
+      /* Remove references to pc from these records. */
+      delete mv.parser.records[i].pc;
     }
-    parser.createBlobLink = function() {
-      /* Make the scrubbed data available for download as a blob. */
-      var blob = new Blob(JSON.stringify(parser.records));
-      var a = d3.select('body').append('a');
-      a
-        .text("Scrubbed records")
-        .attr("download", "map.scrubbed")
-        .attr("href", window.URL.createObjectURL(blob))
-      ;
+    /* Remove references to pc from the remaining records. */
+    for (; i != mv.parser.records.length; ++i) {
+      delete mv.parser.records[i].pc;
     }
-    parser.parseScrubbed = function(scrubbedRecords) {
-      scrubbedRecords = JSON.parse(scrubbedRecords);
-      console.log(scrubbedRecords, scrubbedRecords.length);
-      /*
-       * The work is mostly all done for us. Just scan through to see if there
-       * are any undefined records, and update the pointer if so.
-       */
-      /*
-       * Note that because we do not have the IDs, we cannot do a second pass
-       * to see if there is any information outside of the file that would
-       * tell us what the stats are, because we do not have that information.
-       * We can only get as good as what we were given!
-       */
-      for (var i = 0; i != scrubbedRecords.length; ++i) {
-        scrubbedRecords[i].date = new Date(scrubbedRecords[i].date);
-        if (scrubbedRecords[i].pcstat == undefined && (!fullyDefinedCutoff || scrubbedRecords[i].date > fullyDefinedCutoff)) {
-          fullyDefinedCutoff = scrubbedRecords[i].date;
-        }
+    fullyDefinedCutoff = postProcessedfullyDefinedCutoff;
+  }
+  function softAssert(expr, msg) {
+    if (!expr) {
+      console.error("SOFTASSERT FAILURE: " + msg);
+    }
+  }
+  function createBlobLink() {
+    /* Make the scrubbed data available for download as a blob. */
+    var blob = new Blob(JSON.stringify(parser.records));
+    var a = d3.select('body').append('a');
+    a
+      .text("Scrubbed records")
+      .attr("download", "map.scrubbed")
+      .attr("href", window.URL.createObjectURL(blob))
+    ;
+  }
+  function parseScrubbed(scrubbedRecords) {
+    scrubbedRecords = JSON.parse(scrubbedRecords);
+    console.log(scrubbedRecords, scrubbedRecords.length);
+    /*
+     * The work is mostly all done for us. Just scan through to see if there
+     * are any undefined records, and update the pointer if so.
+     */
+    /*
+     * Note that because we do not have the IDs, we cannot do a second pass
+     * to see if there is any information outside of the file that would
+     * tell us what the stats are, because we do not have that information.
+     * We can only get as good as what we were given!
+     */
+    for (var i = 0; i != scrubbedRecords.length; ++i) {
+      scrubbedRecords[i].date = new Date(scrubbedRecords[i].date);
+      if (scrubbedRecords[i].pcstat == undefined && (!fullyDefinedCutoff || scrubbedRecords[i].date > fullyDefinedCutoff)) {
+        fullyDefinedCutoff = scrubbedRecords[i].date;
       }
-      /* It's simple when everything's already been done. */
-      parser.records = parser.records.concat(scrubbedRecords);
     }
-    return parser;
-  }();
+    /* It's simple when everything's already been done. */
+    parser.records = parser.records.concat(scrubbedRecords);
+  }
   return mv;
 }(mv || {});