Allow loading of zip files

.zip must be at the end of the file name.

Progress bars are not fully descriptive in this mode yet

This also alters the mv.loader interface.
The onprogress callback now takes current and total.
The each callback is now passed an after parameter, to be called when it's
done.

5 files changed, 99 insertions, 58 deletions

diff --git a/.gitmodules b/.gitmodules
index d719d48..a7d2a2d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -7,3 +7,6 @@
 [submodule "js/dc"]
 	path = public/js/dc
 	url = git://github.com/Freeyorp/dc.js.git
+[submodule "public/js/zip"]
+	path = public/js/zip
+	url = https://github.com/gildas-lormeau/zip.js.git
diff --git a/public/index.html b/public/index.html
index ac24d80..67e83a4 100644
--- a/public/index.html
+++ b/public/index.html
@@ -90,6 +90,7 @@
 <!-- Libs -->
 <script src="js/util/memoize.js"></script>
 <script src="js/util/progress.js"></script>
+<script src="js/zip/WebContent/zip.js"></script>
 <script src="js/d3/d3.js"></script>
 <script src="js/crossfilter/crossfilter.js"></script>
 <script src="js/dc/dc.js"></script>
diff --git a/public/js/mv/load.js b/public/js/mv/load.js
index 8e6ff49..0805214 100644
--- a/public/js/mv/load.js
+++ b/public/js/mv/load.js
@@ -29,6 +29,7 @@ var mv = function(mv) {
     loader.numfiles = function() { return numfiles; };
     loader.filenames = function() { return filenames; };
     loader.curfile = function() { return curfile; };
+    loader.setname = function(n) {};
     loader.init = function(each, after) {
       document.getElementById('input').addEventListener('change', function(fevt) {
         numfiles = fevt.target.files.length;
@@ -37,20 +38,38 @@ var mv = function(mv) {
         var reader = new FileReader();
         loader.onbulkstart(fevt);
         reader.onerror = function() { loader.onerror.apply(null, arguments) };
-        reader.onprogress = function() { loader.onprogress.apply(null, arguments) };
+        reader.onprogress = function(evt) { if (evt.lengthComputable) { loader.onprogress(evt.loaded, evt.total) } };
         reader.onabort = function() { loader.onabort.apply(null, arguments) };
         reader.onloadstart = function() { loader.onloadstart.apply(null, arguments) };
         reader.onload = function(evt) {
-          each(reader.result, curfile, numfiles);
-          ++curfile;
-          if (curfile == numfiles) {
-            after();
-          } else {
-            nextFile();
-          }
+          each(reader.result, filenames[curfile], function() {
+            ++curfile;
+            if (curfile == numfiles) {
+              after();
+            } else {
+              nextFile();
+            }
+          });
         };
         function nextFile() {
-          reader.readAsBinaryString(fevt.target.files[curfile]);
+          var file = fevt.target.files[curfile];
+          loader.onloadstart();
+          if (file.name.indexOf(".zip", name.length - 4) != -1) {
+            zip.createReader(new zip.BlobReader(file), function(zipReader) {
+              zipReader.getEntries(function(entries) {
+                entries.forEach(function(d, i) {
+                  loader.setname(file.name + "'; unzipping '" + d.filename + " (" + (i + 1) + "/" + entries.length + ")");
+                  d.getData(new zip.BlobWriter(), function(blob) {
+                    loader.setname(d.filename);
+                    reader.readAsBinaryString(blob);
+                  }, loader.onprogress);
+                });
+              }, loader.onerror);
+            }, loader.onerror);
+          } else {
+            loader.setname(file);
+            reader.readAsBinaryString(file);
+          }
         }
         nextFile();
       }, false);
diff --git a/public/js/mv/main.js b/public/js/mv/main.js
index 34a6c00..e5b74bf 100644
--- a/public/js/mv/main.js
+++ b/public/js/mv/main.js
@@ -1,16 +1,22 @@
 "use strict";
 var mv = function(mv) {
-  mv.init = function() {
-    var loadbar = progress('loadbar');
-    var filesbar = progress('filesbar');
-    var lbase = loadbar.label;
-    loadbar.label = function() {
-      return lbase() == '100%' ? "Loaded '" + mv.loader.filenames()[mv.loader.curfile()]+ "' - Done!" : "Loading '" + mv.loader.filenames()[mv.loader.curfile()] + "' - " + lbase();
-    };
-    var fbase = filesbar.label;
-    filesbar.label = function () {
-      return fbase() == '100%' ? "Loaded " + mv.loader.numfiles() + " file(s) - Done!" : "Loading file " + mv.loader.curfile() + " of " + mv.loader.numfiles() + " - " + fbase();
+  /* Set up common variables for the loading bars */
+  var loadbar = progress('loadbar');
+  var filesbar = progress('filesbar');
+  var lbase = loadbar.label;
+  var fbase = filesbar.label;
+  var name = "";
+  loadbar.label = function() {
+    return lbase() == '100%' ? "Loaded '" + name + "' - Done!" : "Loading '" + name + "' - " + lbase();
+  };
+  filesbar.label = function () {
+    return fbase() == '100%' ? "Loaded " + mv.loader.numfiles() + " file(s) - Done!" : "Loading file " + mv.loader.curfile() + " of " + mv.loader.numfiles() + " - " + fbase();
+  };
     }
+  /* Initialise modules that require initialisation */
+  mv.init = function() {
+    /* Loader module */
+    /* Callbacks for loading files from the filesystem */
     mv.loader.onbulkstart = function(fevt) {
       loadbar.show();
       filesbar.show();
@@ -19,47 +25,59 @@ var mv = function(mv) {
       filesbar.update(mv.loader.curfile(), mv.loader.numfiles());
       loadbar.reset();
     };
-    mv.loader.onprogress = function(evt) {
-      if (evt.lengthComputable) {
-        loadbar.update(evt.loaded, evt.total);
-      }
+    mv.loader.onprogress = function(current, total) {
+      loadbar.update(current, total);
     };
     mv.loader.init(handleFile, postLoading);
-    function handleFile(data, curFileNum, numFiles) {
-      loadbar.complete();
-      if (mv.loader.filenames()[curFileNum].indexOf("scrubbed") != -1) {
-        /* Scrubbed data! */
-        mv.parser.parseScrubbed(data);
-      } else {
-        /* Raw logs. */
-        mv.parser.parseRecords(data);
-      }
-    }
-    function postLoading() {
-      filesbar.complete();
-      /* TODO: This is still a total mess that doesn't really transition properly */
-      setTimeout(function() {
-        loadbar.hide();
-      }, 2000);
-      mv.parser.postProcessing();
-      mv.heap.init();
-      setTimeout(function() {
-        filesbar.hide();
-        d3.select("#mask")
-          .transition()
-          .style("opacity", 0)
-          .remove();
-        d3.selectAll(".vis-hide")
-          .style("display", "inline")
-          .transition()
-          .style("opacity", 1)
-          ;
-        mv.charter.init();
-        if (document.getElementById("connect-option").checked) {
-          mv.connect.connect();
-        }
-      }, 2000);
-    }
+    mv.loader.setname = function(n) { name = n; };
+    /* Set zip.js worker path */
+    zip.workerScriptsPath = "/js/zip/WebContent/";
+    status.text("Ready");
   };
+  /* When a file has finished loading, handle it. */
+  function handleFile(data, name, after) {
+    loadbar.complete();
+    if (name.indexOf("scrubbed") != -1) {
+      /* Scrubbed data! */
+      mv.parser.parseScrubbed(data);
+    } else {
+      /* Raw logs. */
+      mv.parser.parseRecords(data);
+    }
+    after();
+  }
+  /* When everything has finished loading, handle that, then show everything. */
+  function postLoading() {
+    filesbar.complete();
+    /* TODO: This is still a total mess that doesn't really transition properly */
+    setTimeout(function() {
+      loadbar.hide();
+    }, 2000);
+    /* Second pass to apply future information to previously unascertainable records */
+    mv.parser.postProcessing();
+    /* Start up crossfilter */
+    mv.heap.init();
+    /* Give the UI space to catch up for a bit. May need a few more of these breaks. */
+    setTimeout(function() {
+      filesbar.hide();
+      /* Remove the mask hiding the chart view */
+      d3.select("#mask")
+        .transition()
+        .style("opacity", 0)
+        .remove();
+      /* Show the chart view */
+      d3.selectAll(".vis-hide")
+        .style("display", "inline")
+        .transition()
+        .style("opacity", 1)
+        ;
+      /* Render the charts */
+      mv.charter.init();
+      /* If the user wishes to connect, then connect them. */
+      if (document.getElementById("connect-option").checked) {
+        mv.connect.connect();
+      }
+    }, 2000);
+  }
   return mv;
 }(mv || {});
diff --git a/public/js/zip b/public/js/zip
new file mode 160000
+Subproject 4c93974e90e8bf9405bc72e139436325f2caf95