From 9d5f56cff3e8f6b7f4a425414815b4de49244d00 Mon Sep 17 00:00:00 2001 From: Freeyorp Date: Sat, 25 May 2013 16:56:23 +1200 Subject: Allow loading of zip files .zip must be at the end of the file name. Progress bars are not fully descriptive in this mode yet This also alters the mv.loader interface. The onprogress callback now takes current and total. The each callback is now passed an after parameter, to be called when it's done. --- .gitmodules | 3 ++ public/index.html | 1 + public/js/mv/load.js | 37 ++++++++++++---- public/js/mv/main.js | 116 +++++++++++++++++++++++++++++---------------------- public/js/zip | 1 + 5 files changed, 100 insertions(+), 58 deletions(-) create mode 160000 public/js/zip diff --git a/.gitmodules b/.gitmodules index d719d48..a7d2a2d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "js/dc"] path = public/js/dc url = git://github.com/Freeyorp/dc.js.git +[submodule "public/js/zip"] + path = public/js/zip + url = https://github.com/gildas-lormeau/zip.js.git diff --git a/public/index.html b/public/index.html index ac24d80..67e83a4 100644 --- a/public/index.html +++ b/public/index.html @@ -90,6 +90,7 @@ + diff --git a/public/js/mv/load.js b/public/js/mv/load.js index 8e6ff49..0805214 100644 --- a/public/js/mv/load.js +++ b/public/js/mv/load.js @@ -29,6 +29,7 @@ var mv = function(mv) { loader.numfiles = function() { return numfiles; }; loader.filenames = function() { return filenames; }; loader.curfile = function() { return curfile; }; + loader.setname = function(n) {}; loader.init = function(each, after) { document.getElementById('input').addEventListener('change', function(fevt) { numfiles = fevt.target.files.length; @@ -37,20 +38,38 @@ var mv = function(mv) { var reader = new FileReader(); loader.onbulkstart(fevt); reader.onerror = function() { loader.onerror.apply(null, arguments) }; - reader.onprogress = function() { loader.onprogress.apply(null, arguments) }; + reader.onprogress = function(evt) { if (evt.lengthComputable) { loader.onprogress(evt.loaded, evt.total) } }; reader.onabort = function() { loader.onabort.apply(null, arguments) }; reader.onloadstart = function() { loader.onloadstart.apply(null, arguments) }; reader.onload = function(evt) { - each(reader.result, curfile, numfiles); - ++curfile; - if (curfile == numfiles) { - after(); - } else { - nextFile(); - } + each(reader.result, filenames[curfile], function() { + ++curfile; + if (curfile == numfiles) { + after(); + } else { + nextFile(); + } + }); }; function nextFile() { - reader.readAsBinaryString(fevt.target.files[curfile]); + var file = fevt.target.files[curfile]; + loader.onloadstart(); + if (file.name.indexOf(".zip", name.length - 4) != -1) { + zip.createReader(new zip.BlobReader(file), function(zipReader) { + zipReader.getEntries(function(entries) { + entries.forEach(function(d, i) { + loader.setname(file.name + "'; unzipping '" + d.filename + " (" + (i + 1) + "/" + entries.length + ")"); + d.getData(new zip.BlobWriter(), function(blob) { + loader.setname(d.filename); + reader.readAsBinaryString(blob); + }, loader.onprogress); + }); + }, loader.onerror); + }, loader.onerror); + } else { + loader.setname(file); + reader.readAsBinaryString(file); + } } nextFile(); }, false); diff --git a/public/js/mv/main.js b/public/js/mv/main.js index 34a6c00..e5b74bf 100644 --- a/public/js/mv/main.js +++ b/public/js/mv/main.js @@ -1,16 +1,22 @@ "use strict"; var mv = function(mv) { - mv.init = function() { - var loadbar = progress('loadbar'); - var filesbar = progress('filesbar'); - var lbase = loadbar.label; - loadbar.label = function() { - return lbase() == '100%' ? "Loaded '" + mv.loader.filenames()[mv.loader.curfile()]+ "' - Done!" : "Loading '" + mv.loader.filenames()[mv.loader.curfile()] + "' - " + lbase(); - }; - var fbase = filesbar.label; - filesbar.label = function () { - return fbase() == '100%' ? "Loaded " + mv.loader.numfiles() + " file(s) - Done!" : "Loading file " + mv.loader.curfile() + " of " + mv.loader.numfiles() + " - " + fbase(); + /* Set up common variables for the loading bars */ + var loadbar = progress('loadbar'); + var filesbar = progress('filesbar'); + var lbase = loadbar.label; + var fbase = filesbar.label; + var name = ""; + loadbar.label = function() { + return lbase() == '100%' ? "Loaded '" + name + "' - Done!" : "Loading '" + name + "' - " + lbase(); + }; + filesbar.label = function () { + return fbase() == '100%' ? "Loaded " + mv.loader.numfiles() + " file(s) - Done!" : "Loading file " + mv.loader.curfile() + " of " + mv.loader.numfiles() + " - " + fbase(); + }; } + /* Initialise modules that require initialisation */ + mv.init = function() { + /* Loader module */ + /* Callbacks for loading files from the filesystem */ mv.loader.onbulkstart = function(fevt) { loadbar.show(); filesbar.show(); @@ -19,47 +25,59 @@ var mv = function(mv) { filesbar.update(mv.loader.curfile(), mv.loader.numfiles()); loadbar.reset(); }; - mv.loader.onprogress = function(evt) { - if (evt.lengthComputable) { - loadbar.update(evt.loaded, evt.total); - } + mv.loader.onprogress = function(current, total) { + loadbar.update(current, total); }; mv.loader.init(handleFile, postLoading); - function handleFile(data, curFileNum, numFiles) { - loadbar.complete(); - if (mv.loader.filenames()[curFileNum].indexOf("scrubbed") != -1) { - /* Scrubbed data! */ - mv.parser.parseScrubbed(data); - } else { - /* Raw logs. */ - mv.parser.parseRecords(data); - } - } - function postLoading() { - filesbar.complete(); - /* TODO: This is still a total mess that doesn't really transition properly */ - setTimeout(function() { - loadbar.hide(); - }, 2000); - mv.parser.postProcessing(); - mv.heap.init(); - setTimeout(function() { - filesbar.hide(); - d3.select("#mask") - .transition() - .style("opacity", 0) - .remove(); - d3.selectAll(".vis-hide") - .style("display", "inline") - .transition() - .style("opacity", 1) - ; - mv.charter.init(); - if (document.getElementById("connect-option").checked) { - mv.connect.connect(); - } - }, 2000); - } + mv.loader.setname = function(n) { name = n; }; + /* Set zip.js worker path */ + zip.workerScriptsPath = "/js/zip/WebContent/"; + status.text("Ready"); }; + /* When a file has finished loading, handle it. */ + function handleFile(data, name, after) { + loadbar.complete(); + if (name.indexOf("scrubbed") != -1) { + /* Scrubbed data! */ + mv.parser.parseScrubbed(data); + } else { + /* Raw logs. */ + mv.parser.parseRecords(data); + } + after(); + } + /* When everything has finished loading, handle that, then show everything. */ + function postLoading() { + filesbar.complete(); + /* TODO: This is still a total mess that doesn't really transition properly */ + setTimeout(function() { + loadbar.hide(); + }, 2000); + /* Second pass to apply future information to previously unascertainable records */ + mv.parser.postProcessing(); + /* Start up crossfilter */ + mv.heap.init(); + /* Give the UI space to catch up for a bit. May need a few more of these breaks. */ + setTimeout(function() { + filesbar.hide(); + /* Remove the mask hiding the chart view */ + d3.select("#mask") + .transition() + .style("opacity", 0) + .remove(); + /* Show the chart view */ + d3.selectAll(".vis-hide") + .style("display", "inline") + .transition() + .style("opacity", 1) + ; + /* Render the charts */ + mv.charter.init(); + /* If the user wishes to connect, then connect them. */ + if (document.getElementById("connect-option").checked) { + mv.connect.connect(); + } + }, 2000); + } return mv; }(mv || {}); diff --git a/public/js/zip b/public/js/zip new file mode 160000 index 0000000..4c93974 --- /dev/null +++ b/public/js/zip @@ -0,0 +1 @@ +Subproject commit 4c93974e90e8bf9405bc72e139436325f2caf954 -- cgit v1.2.3-60-g2f50