Add on-demand range parsing

README.md

@@ -28,8 +28,23 @@ Worker — nothing is uploaded. It shows:
 * budget-group and top-node breakdowns; and
 * a draggable frame range so statistics can be scoped to part of a capture.
 
-For very large captures the per-frame samples are not retained; the viewer
-falls back to whole-recording aggregates and says so.
+### Large captures and frame ranges
+
+The first parse records a tiny index: the byte offset of every frame plus the
+frame in which each tree node first appeared. Nothing per-sample is kept in
+memory for large files. When a frame range is selected, the viewer asks the
+worker to re-parse **only that byte span** from the source file and aggregates
+it, so scoping works on arbitrarily large recordings without a memory blow-up.
+(The tree order at any frame is the final order filtered by each node's
+add-frame, because VProf only ever prepends children — so no per-frame tree
+snapshots are needed.)
+
+When the browser supports the [File System Access
+API](https://developer.mozilla.org/en-US/docs/Web/API/File_System_Access_API),
+the source file handle is remembered in IndexedDB and a *Reopen last recording*
+button is offered next time, so on-demand range reads work without re-picking
+the file. Plain `<input type=file>` and drag &amp; drop continue to work
+(within the session) when the API is unavailable.
 
 ## Python usage
 

index.html

@@ -53,6 +53,8 @@
   <h1>Source VProf Viewer</h1>
   <div class="row">
     <label>Choose a .vprof file: <input type="file" id="file"></label>
+    <button id="open-picker" class="hidden">Open with file picker&hellip;</button>
+    <button id="reopen-last" class="hidden">Reopen last recording</button>
     <span id="drop-hint" class="muted">or drag &amp; drop it onto this page</span>
     <div id="progress-wrap" class="hidden">
       <div class="progress"><div id="progress-bar"></div></div>
@@ -80,6 +82,7 @@
       <button id="frame-prev">&#9664; prev</button>
       <button id="frame-next">next &#9654;</button>
       <span id="frame-label" class="muted"></span>
+      <span id="range-status" class="muted"></span>
     </div>
     <div id="timeline-wrap"><canvas id="timeline"></canvas></div>
     <div id="timeline-caption" class="muted"></div>
@@ -178,14 +181,29 @@ var TOKEN_TIMINGS = 3;
 var TOKEN_END_OF_FRAME = 4;
 var TOKEN_FILE_FINISHED = 5;
 
+// Retained after a parse so range queries can re-parse only a byte range
+// without keeping every sample in memory.
+var g_recording = null;
+
 self.onmessage = function (e) {
   var msg = e.data;
-  if (!msg || msg.cmd !== "parse") return;
-  try {
-    var out = parseVprof(msg.buffer, msg.opts || {});
-    self.postMessage({ type: "done", result: out.result }, out.transfer);
-  } catch (err) {
-    self.postMessage({ type: "error", message: (err && err.message) || String(err) });
+  if (!msg) return;
+  if (msg.cmd === "parse") {
+    try {
+      var out = parseVprof(msg.buffer, msg.opts || {});
+      self.postMessage({ type: "done", result: out.result }, out.transfer);
+    } catch (err) {
+      self.postMessage({ type: "error", message: (err && err.message) || String(err) });
+    }
+  } else if (msg.cmd === "query") {
+    var t0 = (self.performance && self.performance.now) ? self.performance.now() : Date.now();
+    try {
+      var agg = parseRange(msg.buffer, msg.startFrame, msg.endFrame);
+      var t1 = (self.performance && self.performance.now) ? self.performance.now() : Date.now();
+      self.postMessage({ type: "range", requestId: msg.requestId, startFrame: msg.startFrame, endFrame: msg.endFrame, aggregates: agg, ms: t1 - t0 });
+    } catch (err) {
+      self.postMessage({ type: "rangeError", requestId: msg.requestId, message: (err && err.message) || String(err) });
+    }
   }
 };
 
@@ -232,9 +250,16 @@ function parseVprof(buffer, opts) {
 
   // ---- Nodes (sparse array indexed by unique node id) ----
   var nodesById = [];
+  // Frame index in which each node was first declared. Because
+  // CVProfNode::GetSubNode() only ever *prepends*, the children order at any
+  // frame is the final order restricted to nodes whose addedFrame <= frame.
+  // That lets a range be parsed without replaying structure from the start.
+  var nodesAddedFrame = [];
+  var frameCount = 0;
   function makeNode(id, name, parentId, budgetGroupId) {
     var n = { id: id, name: name, parentId: parentId, budgetGroupId: budgetGroupId, children: [] };
     nodesById[id] = n;
+    nodesAddedFrame[id] = frameCount;
     return n;
   }
   makeNode(rootNodeId, "Root", null, 0);
@@ -259,6 +284,11 @@ function parseVprof(buffer, opts) {
   var frameTotals = [];
   var frameCalls = [];
   var addedNodesByFrame = [];
+  // Byte offset of each frame's Token_StartFrame and of the byte just past its
+  // Token_EndOfFrame. A [start,end] frame range maps to one contiguous
+  // file.slice(frameStartOffsets[start], frameEndOffsets[end]).
+  var frameStartOffsets = [];
+  var frameEndOffsets = [];
 
   var anyTime = false;
   var anyCalls = false;
@@ -307,11 +337,11 @@ function parseVprof(buffer, opts) {
     for (var k = 0; k < kids.length; k++) readTimings(kids[k]);
   }
 
-  var frameCount = 0;
   var progressAt = 0;
   var totalFramesApprox = -1;
 
   while (pos < totalBytes) {
+    var frameStart = pos;
     var token = u8();
     if (token === TOKEN_FILE_FINISHED) break;
     if (token !== TOKEN_START_FRAME) {
@@ -354,6 +384,8 @@ function parseVprof(buffer, opts) {
     frameTotals.push(frameTotalUs);
     frameCalls.push(frameTotalCalls);
     addedNodesByFrame.push(added);
+    frameStartOffsets.push(frameStart);
+    frameEndOffsets.push(pos);
     frameCount++;
 
     if (frameCount - progressAt >= 500) {
@@ -367,7 +399,10 @@ function parseVprof(buffer, opts) {
   var frameTicksOut = Int32Array.from(frameTicks);
   var frameTotalsOut = Float64Array.from(frameTotals);
   var frameCallsOut = Uint32Array.from(frameCalls);
-  transfer.push(frameTicksOut.buffer, frameTotalsOut.buffer, frameCallsOut.buffer);
+  var frameStartOffsetsOut = Float64Array.from(frameStartOffsets);
+  var frameEndOffsetsOut = Float64Array.from(frameEndOffsets);
+  transfer.push(frameTicksOut.buffer, frameTotalsOut.buffer, frameCallsOut.buffer,
+                frameStartOffsetsOut.buffer, frameEndOffsetsOut.buffer);
 
   var addedOut = [];
   for (var f = 0; f < addedNodesByFrame.length; f++) {
@@ -391,6 +426,13 @@ function parseVprof(buffer, opts) {
 
   var t1 = (self.performance && self.performance.now) ? self.performance.now() : Date.now();
 
+  // Keep the parsed tree around so later range queries can walk it.
+  g_recording = {
+    nodes: nodesById,
+    nodesAddedFrame: nodesAddedFrame,
+    rootNodeId: rootNodeId
+  };
+
   return {
     result: {
       version: version,
@@ -402,6 +444,8 @@ function parseVprof(buffer, opts) {
       frameTicks: frameTicksOut,
       frameTotals: frameTotalsOut,
       frameCalls: frameCallsOut,
+      frameStartOffsets: frameStartOffsetsOut,
+      frameEndOffsets: frameEndOffsetsOut,
       addedNodesByFrame: addedOut,
       samples: samplesOut,
       samplesStored: storing,
@@ -414,6 +458,88 @@ function parseVprof(buffer, opts) {
     transfer: transfer
   };
 }
+
+// Re-parse only the frames in a contiguous byte range and aggregate per node.
+// `buffer` must start on the Token_StartFrame of `startFrame` and end just past
+// the Token_EndOfFrame of `endFrame` (i.e. file.slice(frameStartOffsets[start],
+// frameEndOffsets[end])).
+function parseRange(buffer, startFrame, endFrame) {
+  if (!g_recording) throw new Error("No recording loaded for a range query.");
+  var rec = g_recording;
+  var dv = new DataView(buffer);
+  var totalBytes = buffer.byteLength;
+  var pos = 0;
+  var decoder = new TextDecoder("utf-8");
+
+  function u8() { return dv.getUint8(pos++); }
+  function u16() { var v = dv.getUint16(pos, true); pos += 2; return v; }
+  function u32() { var v = dv.getUint32(pos, true); pos += 4; return v; }
+  function i32() { var v = dv.getInt32(pos, true); pos += 4; return v; }
+  function cstr() {
+    var start = pos;
+    while (dv.getUint8(pos) !== 0) pos++;
+    var s = decoder.decode(new Uint8Array(buffer, start, pos - start));
+    pos++;
+    return s;
+  }
+
+  var agg = [];
+  function aggFor(id) {
+    var a = agg[id];
+    if (!a) { a = { calls: 0, us: 0, maxUs: 0, frames: 0 }; agg[id] = a; }
+    return a;
+  }
+
+  function readTimings(nodeId, frameIndex) {
+    var callsToken = u8();
+    var calls;
+    if (callsToken === 255) calls = u16(); else calls = callsToken;
+
+    var timeUnits = u16();
+    var us;
+    if (timeUnits === 0xFFFF) us = u32() * 4; else us = timeUnits * 4;
+
+    var a = aggFor(nodeId);
+    a.calls += calls;
+    a.us += us;
+    if (us > a.maxUs) a.maxUs = us;
+    a.frames++;
+
+    // Only descend into children that existed at this frame; the final child
+    // order already reflects the prepend-only evolution of the tree.
+    var kids = rec.nodes[nodeId].children;
+    for (var k = 0; k < kids.length; k++) {
+      if (rec.nodesAddedFrame[kids[k]] <= frameIndex) readTimings(kids[k], frameIndex);
+    }
+  }
+
+  var frameIndex = startFrame;
+  while (pos < totalBytes) {
+    var token = u8();
+    if (token === TOKEN_FILE_FINISHED) break;
+    if (token !== TOKEN_START_FRAME) {
+      throw new Error("Range parse: expected start-of-frame at offset " + (pos - 1) + ", got " + token + ".");
+    }
+    i32(); // tick
+    while (true) {
+      var inner = u8();
+      if (inner === TOKEN_END_OF_FRAME) break;
+      if (inner === TOKEN_ADD_BUDGET_GROUP) {
+        cstr(); i32();
+      } else if (inner === TOKEN_ADD_NODE) {
+        i32(); cstr(); i32(); i32();
+      } else if (inner === TOKEN_TIMINGS) {
+        readTimings(rec.rootNodeId, frameIndex);
+      } else {
+        throw new Error("Range parse: unexpected token " + inner + " at offset " + (pos - 1) + ".");
+      }
+    }
+    frameIndex++;
+    if (frameIndex > endFrame + 1) break;
+  }
+
+  return agg;
+}
 </script>
 
 <!-- ===================== UI ===================== -->
@@ -466,8 +592,9 @@ function setError(msg) {
   box.textContent = msg;
 }
 
-function startParse(file) {
+function startParse(file, handle) {
   lastFile = file;
+  if (handle) rememberHandle(handle);
   setError("");
   $("progress-wrap").classList.remove("hidden");
   $("progress-bar").style.width = "0%";
@@ -498,6 +625,21 @@ function onWorkerMessage(e) {
   } else if (msg.type === "error") {
     setError("Parse error: " + msg.message);
     $("status").textContent = "Failed.";
+  } else if (msg.type === "range") {
+    var s = state;
+    if (!s || msg.requestId !== s.requestId) return;
+    s.agg = msg.aggregates;
+    s.rangeCache.set(msg.startFrame + ":" + msg.endFrame, msg.aggregates);
+    trimRangeCache(s.rangeCache);
+    s.rangePending = false;
+    renderAll();
+    updateRangeStatus("Frames " + msg.startFrame + "\u2013" + msg.endFrame + " parsed in " + Math.round(msg.ms) + " ms");
+  } else if (msg.type === "rangeError") {
+    var s2 = state;
+    if (!s2 || msg.requestId !== s2.requestId) return;
+    s2.rangePending = false;
+    renderAll();
+    updateRangeStatus("Range parse failed: " + msg.message);
   } else if (msg.type === "done") {
     $("progress-bar").style.width = "100%";
     $("status").textContent = "Done.";
@@ -507,6 +649,75 @@ function onWorkerMessage(e) {
   }
 }
 
+/* --------------------------- file system access --------------------------- */
+// Persist the FileSystemFileHandle (when the browser supports the File System
+// Access API) so a large recording can be reopened next time and its byte range
+// re-read on demand for granular frame queries. Falls back silently to the
+// plain <input type=file> File object, which is re-readable within a session.
+var HANDLE_DB = "vprof-viewer";
+var HANDLE_STORE = "handles";
+var HANDLE_KEY = "last";
+
+function idbOpen() {
+  return new Promise(function (resolve, reject) {
+    if (!window.indexedDB) return reject(new Error("IndexedDB unavailable"));
+    var req = indexedDB.open(HANDLE_DB, 1);
+    req.onupgradeneeded = function () {
+      if (!req.result.objectStoreNames.contains(HANDLE_STORE)) req.result.createObjectStore(HANDLE_STORE);
+    };
+    req.onsuccess = function () { resolve(req.result); };
+    req.onerror = function () { reject(req.error); };
+  });
+}
+function idbSet(key, value) {
+  return idbOpen().then(function (db) {
+    return new Promise(function (resolve, reject) {
+      var tx = db.transaction(HANDLE_STORE, "readwrite");
+      tx.objectStore(HANDLE_STORE).put(value, key);
+      tx.oncomplete = function () { resolve(); };
+      tx.onerror = function () { reject(tx.error); };
+    });
+  });
+}
+function idbGet(key) {
+  return idbOpen().then(function (db) {
+    return new Promise(function (resolve, reject) {
+      var req = db.transaction(HANDLE_STORE, "readonly").objectStore(HANDLE_STORE).get(key);
+      req.onsuccess = function () { resolve(req.result); };
+      req.onerror = function () { reject(req.error); };
+    });
+  });
+}
+function rememberHandle(handle) {
+  if (!handle || !handle.getFile) return;
+  handle.getFile().then(function (file) {
+    idbSet(HANDLE_KEY, { handle: handle, name: file.name, size: file.size, lastModified: file.lastModified });
+  }).catch(function () {});
+}
+function showReopenButton(entry) {
+  var btn = $("reopen-last");
+  btn.textContent = "Reopen " + entry.name;
+  btn.classList.remove("hidden");
+  btn.onclick = function () {
+    var permission = entry.handle.queryPermission
+      ? entry.handle.queryPermission({ mode: "read" })
+      : Promise.resolve("granted");
+    permission.then(function (status) {
+      if (status !== "granted" && entry.handle.requestPermission) {
+        return entry.handle.requestPermission({ mode: "read" });
+      }
+      return status;
+    }).then(function (status) {
+      if (status !== "granted") throw new Error("read permission denied");
+      return entry.handle.getFile();
+    }).then(function (file) {
+      startParse(file, entry.handle);
+    }).catch(function (err) {
+      setError("Could not reopen recording: " + err);
+    });
+  };
+}
+
 /* ------------------------------- state build ------------------------------- */
 function buildState(result, file) {
   var nodeById = result.nodes;
@@ -543,16 +754,19 @@ function buildState(result, file) {
     agg: result.aggregates,
     self: new Float64Array(nodeById.length),
     rootTotal: 0,
-    percentBase: 1
+    percentBase: 1,
+    rangeCache: new Map(),   // "start:end" -> aggregate (bounded)
+    requestId: 0,            // newest outstanding range request
+    rangePending: false
   };
   updateRangeInputs();
 }
 
 /* ------------------------------- aggregation ------------------------------- */
-function getAggregate(start, end) {
+// Aggregate per-node timings from the per-frame samples retained for small
+// captures.
+function computeFromSamples(start, end) {
   var s = state;
-  if (!s.r.samples || (start <= 0 && end >= s.frameCount - 1)) return s.r.aggregates;
-
   var agg = [];
   function get(id) {
     var a = agg[id];
@@ -576,9 +790,85 @@ function getAggregate(start, end) {
   return agg;
 }
 
+// Ranges can be served from retained samples or by re-parsing the original
+// file's byte range on demand (the frame offset index makes that seekable).
+function canQueryRanges() {
+  var s = state;
+  return !!(s && (s.r.samples || (s.file && s.r.frameStartOffsets && worker)));
+}
+
+function trimRangeCache(cache) {
+  while (cache.size > 32) {
+    var first = cache.keys().next().value;
+    cache.delete(first);
+  }
+}
+
+function updateRangeStatus(msg) {
+  var el = $("range-status");
+  if (el) el.textContent = msg || "";
+}
+
+// Compute state.agg for the current range: synchronously when cheap, otherwise
+// ask the worker to parse just the selected byte span.
+function refreshAggregate() {
+  var s = state;
+  var a = s.rangeStart, b = s.rangeEnd;
+
+  if (a <= 0 && b >= s.frameCount - 1) {
+    s.agg = s.r.aggregates;
+    s.rangePending = false;
+    renderAll();
+    return;
+  }
+
+  var key = a + ":" + b;
+  var cached = s.rangeCache.get(key);
+  if (cached) {
+    s.agg = cached;
+    s.rangePending = false;
+    renderAll();
+    return;
+  }
+
+  if (s.r.samples) {
+    s.agg = computeFromSamples(a, b);
+    s.rangeCache.set(key, s.agg);
+    trimRangeCache(s.rangeCache);
+    s.rangePending = false;
+    renderAll();
+    return;
+  }
+
+  if (!s.file || !s.r.frameStartOffsets || !worker) {
+    // No way to re-read the source; fall back to the whole-recording aggregate.
+    s.agg = s.r.aggregates;
+    s.rangePending = false;
+    renderAll();
+    return;
+  }
+
+  var requestId = ++s.requestId;
+  s.rangePending = true;
+  updateRangeStatus("Loading frames " + a + "\u2013" + b + "\u2026");
+  drawTimeline();
+
+  var startOff = s.r.frameStartOffsets[a];
+  var endOff = s.r.frameEndOffsets[b];
+  s.file.slice(startOff, endOff).arrayBuffer().then(function (buf) {
+    if (!state || state.requestId !== requestId) return;
+    worker.postMessage({ cmd: "query", buffer: buf, startFrame: a, endFrame: b, requestId: requestId }, [buf]);
+  }).catch(function (err) {
+    if (!state || state.requestId !== requestId) return;
+    state.rangePending = false;
+    updateRangeStatus("Range read failed: " + err);
+    renderAll();
+  });
+}
+
 function recompute() {
   var s = state;
-  var agg = getAggregate(s.rangeStart, s.rangeEnd);
+  var agg = s.agg || s.r.aggregates;
   s.agg = agg;
 
   var self = new Float64Array(s.maxId);
@@ -612,11 +902,13 @@ function renderAll() {
   ["summary-section", "timeline-section", "tree-section", "budget-section", "top-section"]
     .forEach(function (id) { $(id).classList.remove("hidden"); });
 
-  // Range scoping needs per-frame samples; disable it when they were dropped.
-  var rangeDisabled = !state.r.samples;
+  // Range scoping works with retained samples or by re-parsing the source file
+  // on demand, so only disable it when neither is available.
+  var rangeDisabled = !canQueryRanges();
   ["range-start", "range-end", "range-apply", "frame-prev", "frame-next"].forEach(function (id) {
     $(id).disabled = rangeDisabled;
   });
+  if (!state.rangePending) updateRangeStatus("");
 }
 
 function renderSummary() {
@@ -638,8 +930,9 @@ function renderSummary() {
     var d2 = document.createElement("div");
     d2.className = "notice";
     d2.textContent = "This recording is very large (" + fmtInt(r.totalSamples) +
-      " samples), so per-frame samples were not retained. Statistics cover the whole " +
-      "recording and frame-range selection is disabled.";
+      " samples), so per-frame samples were not kept in memory. Frame ranges are " +
+      "still available: the viewer re-parses only the selected byte range from the " +
+      "source file on demand.";
     notices.appendChild(d2);
   }
 
@@ -648,6 +941,7 @@ function renderSummary() {
     ["Size", s.file ? fmtBytes(s.file.size) : "-"],
     ["Format version", r.version],
     ["Frames", fmtInt(r.frameCount)],
+    ["Range", s.rangeStart + "\u2013" + s.rangeEnd + " (" + fmtInt(s.rangeEnd - s.rangeStart + 1) + " frames)"],
     ["Nodes", fmtInt(s.nodes.length)],
     ["Budget groups", fmtInt(r.budgetGroups.length)],
     ["Samples", fmtInt(r.totalSamples)],
@@ -741,7 +1035,7 @@ function applyRange(a, b) {
   if (a > b) { var t = a; a = b; b = t; }
   s.rangeStart = a; s.rangeEnd = b;
   updateRangeInputs();
-  renderAll();
+  refreshAggregate();
 }
 function updateRangeInputs() {
   $("range-start").value = state.rangeStart;
@@ -942,6 +1236,29 @@ document.addEventListener("drop", function (e) {
   if (f) startParse(f);
 });
 
+// File System Access API: use it when available so the source handle can be
+// persisted and reused for on-demand range reads.
+(function initFileSystemAccess() {
+  if (window.showOpenFilePicker) {
+    var picker = $("open-picker");
+    picker.classList.remove("hidden");
+    picker.onclick = function () {
+      window.showOpenFilePicker({
+        multiple: false,
+        types: [{ description: "VProf recording", accept: { "application/octet-stream": [".vprof"] } }]
+      }).then(function (handles) {
+        return handles[0].getFile().then(function (file) { startParse(file, handles[0]); });
+      }).catch(function (err) {
+        if (err && err.name === "AbortError") return;
+        setError("Could not open file: " + err);
+      });
+    };
+  }
+  idbGet(HANDLE_KEY).then(function (entry) {
+    if (entry && entry.handle) showReopenButton(entry);
+  }).catch(function () {});
+})();
+
 $("range-all").onclick = function () { applyRange(0, state.frameCount - 1); };
 $("range-apply").onclick = function () {
   applyRange(Number($("range-start").value), Number($("range-end").value));
@@ -969,7 +1286,7 @@ $("filter").addEventListener("input", function (e) {
 (function () {
   var c = $("timeline");
   c.addEventListener("mousedown", function (e) {
-    if (!state || !state.r.samples) return;
+    if (!state || !canQueryRanges()) return;
     e.preventDefault();
     drag = { active: true, start: frameAt(e.clientX), end: frameAt(e.clientX) };
     drawTimeline();