Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-24 13:21:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1570089 Part 1 - Recover from replaying process crashes, r=loganfsmyth.

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D39925

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Brian Hackett 2019-08-04 16:55:00 +00:00
parent 51548f5398
commit bc685f2db6
26 changed files with 377 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
devtools/client/webreplay/mochitest/browser_dbg_rr_breakpoints-01.js
View File

@ -11,7 +11,7 @@ add_task(async function() {
const dbg = await attachRecordingDebugger("doc_rr_basic.html", {
waitForRecording: true,
});
const { threadFront, tab, toolbox, target } = dbg;
const { threadFront, target } = dbg;
const bp = await setBreakpoint(threadFront, "doc_rr_basic.html", 21);
@ -37,6 +37,5 @@ add_task(async function() {
await checkEvaluateInTopFrame(target, "number", 10);
await threadFront.removeBreakpoint(bp);
await toolbox.closeToolbox();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
});

5
devtools/client/webreplay/mochitest/browser_dbg_rr_breakpoints-02.js
View File

@ -11,7 +11,7 @@ add_task(async function() {
const dbg = await attachRecordingDebugger("doc_rr_basic.html", {
waitForRecording: true,
});
const { threadFront, tab, toolbox, target } = dbg;
const { threadFront, target } = dbg;
const bp = await setBreakpoint(threadFront, "doc_rr_basic.html", 21);
await rewindToLine(threadFront, 21);
@ -23,6 +23,5 @@ add_task(async function() {
await checkEvaluateInTopFrame(target, "testStepping2()", undefined);
await threadFront.removeBreakpoint(bp);
await toolbox.destroy();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
});

5
devtools/client/webreplay/mochitest/browser_dbg_rr_breakpoints-03.js
View File

@ -9,7 +9,7 @@
// Test some issues when stepping around after hitting a breakpoint while recording.
add_task(async function() {
const dbg = await attachRecordingDebugger("doc_rr_continuous.html");
const { threadFront, tab, toolbox } = dbg;
const { threadFront } = dbg;
await threadFront.interrupt();
const bp1 = await setBreakpoint(threadFront, "doc_rr_continuous.html", 19);
@ -24,6 +24,5 @@ add_task(async function() {
await threadFront.removeBreakpoint(bp1);
await threadFront.removeBreakpoint(bp2);
await threadFront.removeBreakpoint(bp3);
await toolbox.destroy();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
});

5
devtools/client/webreplay/mochitest/browser_dbg_rr_breakpoints-04.js
View File

@ -10,7 +10,7 @@
// recording.
add_task(async function() {
const dbg = await attachRecordingDebugger("doc_rr_continuous.html");
const { threadFront, tab, toolbox, target } = dbg;
const { threadFront, target } = dbg;
const bp = await setBreakpoint(threadFront, "doc_rr_continuous.html", 14);
await resumeToLine(threadFront, 14);
@ -29,6 +29,5 @@ add_task(async function() {
await checkEvaluateInTopFrame(target, "number", value + 2);
await threadFront.removeBreakpoint(bp);
await toolbox.destroy();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
});

5
devtools/client/webreplay/mochitest/browser_dbg_rr_breakpoints-05.js
View File

@ -13,7 +13,7 @@ add_task(async function() {
waitForRecording: true,
});
const { threadFront, tab, toolbox, target } = dbg;
const { threadFront, target } = dbg;
await threadFront.interrupt();
@ -27,6 +27,5 @@ add_task(async function() {
await checkEvaluateInTopFrame(target, "number", 2);
await threadFront.removeBreakpoint(bp);
await toolbox.destroy();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
});

6
devtools/client/webreplay/mochitest/browser_dbg_rr_breakpoints-06.js
View File

@ -12,8 +12,7 @@ add_task(async function() {
const dbg = await attachRecordingDebugger("doc_control_flow.html", {
waitForRecording: true,
});
const { threadFront, tab, toolbox } = dbg;
const { threadFront } = dbg;
const breakpoints = [];
await rewindToBreakpoint(10);
@ -32,8 +31,7 @@ add_task(async function() {
for (const bp of breakpoints) {
await threadFront.removeBreakpoint(bp);
}
await toolbox.closeToolbox();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
async function rewindToBreakpoint(line) {
const bp = await setBreakpoint(threadFront, "doc_control_flow.html", line);

5
devtools/client/webreplay/mochitest/browser_dbg_rr_console_warp-01.js
View File

@ -12,7 +12,7 @@ add_task(async function() {
waitForRecording: true,
});
const { tab, toolbox, threadFront, target } = dbg;
const { threadFront, target } = dbg;
const console = await getDebuggerSplitConsole(dbg);
const hud = console.hud;
@ -32,6 +32,5 @@ add_task(async function() {
await checkEvaluateInTopFrame(target, "number", 5);
await threadFront.removeBreakpoint(bp);
await toolbox.destroy();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
});

5
devtools/client/webreplay/mochitest/browser_dbg_rr_console_warp-02.js
View File

@ -12,7 +12,7 @@ add_task(async function() {
waitForRecording: true,
});
const { tab, toolbox, threadFront } = dbg;
const { threadFront } = dbg;
const console = await getDebuggerSplitConsole(dbg);
const hud = console.hud;
@ -25,6 +25,5 @@ add_task(async function() {
message = findMessage(hud, "number: 1");
// ok(message.classList.contains("paused-before"), "paused before message is shown");
await toolbox.destroy();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
});

5
devtools/client/webreplay/mochitest/browser_dbg_rr_logpoint-01.js
View File

@ -13,7 +13,7 @@ add_task(async function() {
waitForRecording: true,
});
const { tab, toolbox, threadFront, target } = dbg;
const { threadFront, target } = dbg;
const console = await getDebuggerSplitConsole(dbg);
const hud = console.hud;
@ -43,6 +43,5 @@ add_task(async function() {
await threadFront.removeBreakpoint(bp1);
await threadFront.removeBreakpoint(bp2);
await threadFront.removeBreakpoint(bp3);
await toolbox.destroy();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
});

4
devtools/client/webreplay/mochitest/browser_dbg_rr_logpoint-02.js
View File

@ -13,7 +13,6 @@ add_task(async function() {
waitForRecording: true,
});
const { tab, toolbox } = dbg;
const console = await getDebuggerSplitConsole(dbg);
const hud = console.hud;
@ -40,6 +39,5 @@ add_task(async function() {
await dbg.actions.removeAllBreakpoints(getContext(dbg));
await toolbox.destroy();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
});

20
devtools/client/webreplay/mochitest/browser_dbg_rr_replay-01.js
View File

@ -27,19 +27,19 @@ add_task(async function() {
gBrowser.selectedTab = replayingTab;
await once(Services.ppmm, "HitRecordingEndpoint");
const { target, toolbox } = await attachDebugger(replayingTab);
const client = toolbox.threadFront;
await client.interrupt();
const bp = await setBreakpoint(client, "doc_rr_basic.html", 21);
await rewindToLine(client, 21);
const dbg = await attachDebugger(replayingTab);
const { threadFront } = dbg.toolbox;
const { target } = dbg;
await threadFront.interrupt();
const bp = await setBreakpoint(threadFront, "doc_rr_basic.html", 21);
await rewindToLine(threadFront, 21);
await checkEvaluateInTopFrame(target, "number", 10);
await rewindToLine(client, 21);
await rewindToLine(threadFront, 21);
await checkEvaluateInTopFrame(target, "number", 9);
await resumeToLine(client, 21);
await resumeToLine(threadFront, 21);
await checkEvaluateInTopFrame(target, "number", 10);
await client.removeBreakpoint(bp);
await toolbox.destroy();
await threadFront.removeBreakpoint(bp);
await gBrowser.removeTab(recordingTab);
await gBrowser.removeTab(replayingTab);
await shutdownDebugger(dbg);
});

39
devtools/client/webreplay/mochitest/browser_dbg_rr_replay-02.js
View File

@ -20,12 +20,12 @@ add_task(async function() {
const firstTab = await attachDebugger(recordingTab);
let toolbox = firstTab.toolbox;
let target = firstTab.target;
let client = toolbox.threadFront;
await client.interrupt();
let bp = await setBreakpoint(client, "doc_rr_continuous.html", 14);
await resumeToLine(client, 14);
await resumeToLine(client, 14);
await reverseStepOverToLine(client, 13);
let threadFront = toolbox.threadFront;
await threadFront.interrupt();
let bp = await setBreakpoint(threadFront, "doc_rr_continuous.html", 14);
await resumeToLine(threadFront, 14);
await resumeToLine(threadFront, 14);
await reverseStepOverToLine(threadFront, 13);
const lastNumberValue = await evaluateInTopFrame(target, "number");
const remoteTab = recordingTab.linkedBrowser.frameLoader.remoteTab;
@ -33,7 +33,7 @@ add_task(async function() {
ok(remoteTab.saveRecording(recordingFile), "Saved recording");
await once(Services.ppmm, "SaveRecordingFinished");
await client.removeBreakpoint(bp);
await threadFront.removeBreakpoint(bp);
await toolbox.destroy();
await gBrowser.removeTab(recordingTab);
@ -43,25 +43,24 @@ add_task(async function() {
gBrowser.selectedTab = replayingTab;
await once(Services.ppmm, "HitRecordingEndpoint");
const rplyTab = await attachDebugger(replayingTab);
toolbox = rplyTab.toolbox;
target = rplyTab.target;
client = toolbox.threadFront;
await client.interrupt();
const dbg = await attachDebugger(replayingTab);
toolbox = dbg.toolbox;
target = dbg.target;
threadFront = toolbox.threadFront;
await threadFront.interrupt();
// The recording does not actually end at the point where we saved it, but
// will do at the next checkpoint. Rewind to the point we are interested in.
bp = await setBreakpoint(client, "doc_rr_continuous.html", 14);
await rewindToLine(client, 14);
bp = await setBreakpoint(threadFront, "doc_rr_continuous.html", 14);
await rewindToLine(threadFront, 14);
await checkEvaluateInTopFrame(target, "number", lastNumberValue);
await reverseStepOverToLine(client, 13);
await rewindToLine(client, 14);
await reverseStepOverToLine(threadFront, 13);
await rewindToLine(threadFront, 14);
await checkEvaluateInTopFrame(target, "number", lastNumberValue - 1);
await resumeToLine(client, 14);
await resumeToLine(threadFront, 14);
await checkEvaluateInTopFrame(target, "number", lastNumberValue);
await client.removeBreakpoint(bp);
await toolbox.destroy();
await gBrowser.removeTab(replayingTab);
await threadFront.removeBreakpoint(bp);
await shutdownDebugger(dbg);
});

25
devtools/client/webreplay/mochitest/browser_dbg_rr_stepping-01.js
View File

@ -8,24 +8,21 @@
// Test basic step-over/back functionality in web replay.
add_task(async function() {
const tab = BrowserTestUtils.addTab(gBrowser, null, { recordExecution: "*" });
gBrowser.selectedTab = tab;
openTrustedLinkIn(EXAMPLE_URL + "doc_rr_basic.html", "current");
await once(Services.ppmm, "RecordingFinished");
const dbg = await attachRecordingDebugger("doc_rr_basic.html", {
waitForRecording: true,
});
const { threadFront, target } = dbg;
const { target, toolbox } = await attachDebugger(tab);
const client = toolbox.threadFront;
await client.interrupt();
const bp = await setBreakpoint(client, "doc_rr_basic.html", 21);
await rewindToLine(client, 21);
await threadFront.interrupt();
const bp = await setBreakpoint(threadFront, "doc_rr_basic.html", 21);
await rewindToLine(threadFront, 21);
await checkEvaluateInTopFrame(target, "number", 10);
await reverseStepOverToLine(client, 20);
await reverseStepOverToLine(threadFront, 20);
await checkEvaluateInTopFrame(target, "number", 9);
await checkEvaluateInTopFrameThrows(target, "window.alert(3)");
await stepOverToLine(client, 21);
await stepOverToLine(threadFront, 21);
await checkEvaluateInTopFrame(target, "number", 10);
await client.removeBreakpoint(bp);
await toolbox.destroy();
await gBrowser.removeTab(tab);
await threadFront.removeBreakpoint(bp);
await shutdownDebugger(dbg);
});

45
devtools/client/webreplay/mochitest/browser_dbg_rr_stepping-02.js
View File

@ -8,30 +8,27 @@
// Test fixes for some simple stepping bugs.
add_task(async function() {
const tab = BrowserTestUtils.addTab(gBrowser, null, { recordExecution: "*" });
gBrowser.selectedTab = tab;
openTrustedLinkIn(EXAMPLE_URL + "doc_rr_basic.html", "current");
await once(Services.ppmm, "RecordingFinished");
const dbg = await attachRecordingDebugger("doc_rr_basic.html", {
waitForRecording: true,
});
const { threadFront } = dbg;
const { toolbox } = await attachDebugger(tab);
const front = toolbox.threadFront;
await front.interrupt();
const bp = await setBreakpoint(front, "doc_rr_basic.html", 22);
await rewindToLine(front, 22);
await stepInToLine(front, 25);
await stepOverToLine(front, 26);
await stepOverToLine(front, 27);
await reverseStepOverToLine(front, 26);
await stepInToLine(front, 30);
await stepOverToLine(front, 31);
await stepOverToLine(front, 32);
await stepOverToLine(front, 33);
await reverseStepOverToLine(front, 32);
await stepOutToLine(front, 27);
await reverseStepOverToLine(front, 26);
await reverseStepOverToLine(front, 25);
await threadFront.interrupt();
const bp = await setBreakpoint(threadFront, "doc_rr_basic.html", 22);
await rewindToLine(threadFront, 22);
await stepInToLine(threadFront, 25);
await stepOverToLine(threadFront, 26);
await stepOverToLine(threadFront, 27);
await reverseStepOverToLine(threadFront, 26);
await stepInToLine(threadFront, 30);
await stepOverToLine(threadFront, 31);
await stepOverToLine(threadFront, 32);
await stepOverToLine(threadFront, 33);
await reverseStepOverToLine(threadFront, 32);
await stepOutToLine(threadFront, 27);
await reverseStepOverToLine(threadFront, 26);
await reverseStepOverToLine(threadFront, 25);
await front.removeBreakpoint(bp);
await toolbox.destroy();
await gBrowser.removeTab(tab);
await threadFront.removeBreakpoint(bp);
await shutdownDebugger(dbg);
});

24
devtools/client/webreplay/mochitest/browser_dbg_rr_stepping-03.js
View File

@ -8,23 +8,19 @@
// Test stepping back while recording, then resuming recording.
add_task(async function() {
const tab = BrowserTestUtils.addTab(gBrowser, null, { recordExecution: "*" });
gBrowser.selectedTab = tab;
openTrustedLinkIn(EXAMPLE_URL + "doc_rr_continuous.html", "current");
const dbg = await attachRecordingDebugger("doc_rr_continuous.html");
const { threadFront, target } = dbg;
const { toolbox, target } = await attachDebugger(tab);
const client = toolbox.threadFront;
await client.interrupt();
const bp = await setBreakpoint(client, "doc_rr_continuous.html", 13);
await resumeToLine(client, 13);
await threadFront.interrupt();
const bp = await setBreakpoint(threadFront, "doc_rr_continuous.html", 13);
await resumeToLine(threadFront, 13);
const value = await evaluateInTopFrame(target, "number");
await reverseStepOverToLine(client, 12);
await reverseStepOverToLine(threadFront, 12);
await checkEvaluateInTopFrame(target, "number", value - 1);
await resumeToLine(client, 13);
await resumeToLine(client, 13);
await resumeToLine(threadFront, 13);
await resumeToLine(threadFront, 13);
await checkEvaluateInTopFrame(target, "number", value + 1);
await client.removeBreakpoint(bp);
await toolbox.destroy();
await gBrowser.removeTab(tab);
await threadFront.removeBreakpoint(bp);
await shutdownDebugger(dbg);
});

40
devtools/client/webreplay/mochitest/browser_dbg_rr_stepping-04.js
View File

@ -8,37 +8,35 @@
// Stepping past the beginning or end of a frame should act like a step-out.
add_task(async function() {
const tab = BrowserTestUtils.addTab(gBrowser, null, { recordExecution: "*" });
gBrowser.selectedTab = tab;
openTrustedLinkIn(EXAMPLE_URL + "doc_rr_basic.html", "current");
await once(Services.ppmm, "RecordingFinished");
const dbg = await attachRecordingDebugger("doc_rr_basic.html", {
waitForRecording: true,
});
const { threadFront, target } = dbg;
const { target, toolbox } = await attachDebugger(tab);
const client = toolbox.threadFront;
await client.interrupt();
const bp = await setBreakpoint(client, "doc_rr_basic.html", 21);
await rewindToLine(client, 21);
await threadFront.interrupt();
const bp = await setBreakpoint(threadFront, "doc_rr_basic.html", 21);
await rewindToLine(threadFront, 21);
await checkEvaluateInTopFrame(target, "number", 10);
await reverseStepOverToLine(client, 20);
await reverseStepOverToLine(client, 12);
await reverseStepOverToLine(threadFront, 20);
await reverseStepOverToLine(threadFront, 12);
await reverseStepOverToLine(threadFront, 12);
// After reverse-stepping out of the topmost frame we should rewind to the
// last breakpoint hit.
await reverseStepOverToLine(client, 21);
await reverseStepOverToLine(threadFront, 21);
await checkEvaluateInTopFrame(target, "number", 9);
await stepOverToLine(client, 22);
await stepOverToLine(client, 23);
await stepOverToLine(client, 13);
await stepOverToLine(client, 17);
await stepOverToLine(client, 18);
await stepOverToLine(threadFront, 22);
await stepOverToLine(threadFront, 23);
await stepOverToLine(threadFront, 13);
await stepOverToLine(threadFront, 17);
await stepOverToLine(threadFront, 18);
// After forward-stepping out of the topmost frame we should run forward to
// the next breakpoint hit.
await stepOverToLine(client, 21);
await stepOverToLine(threadFront, 21);
await checkEvaluateInTopFrame(target, "number", 10);
await client.removeBreakpoint(bp);
await toolbox.destroy();
await gBrowser.removeTab(tab);
await threadFront.removeBreakpoint(bp);
await shutdownDebugger(dbg);
});

5
devtools/client/webreplay/mochitest/browser_rr_inspector-01.js
View File

@ -21,7 +21,7 @@ add_task(async function() {
const dbg = await attachRecordingDebugger("doc_inspector_basic.html", {
waitForRecording: true,
});
const { threadFront, tab, toolbox } = dbg;
const { threadFront } = dbg;
await threadFront.interrupt();
await threadFront.resume();
@ -59,6 +59,5 @@ add_task(async function() {
);
await threadFront.removeBreakpoint(bp);
await toolbox.closeToolbox();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
});

5
devtools/client/webreplay/mochitest/browser_rr_inspector-02.js
View File

@ -15,7 +15,7 @@ add_task(async function() {
const dbg = await attachRecordingDebugger("doc_inspector_basic.html", {
waitForRecording: true,
});
const { threadFront, tab, toolbox } = dbg;
const { threadFront, toolbox } = dbg;
await threadFront.interrupt();
await threadFront.resume();
@ -39,8 +39,7 @@ add_task(async function() {
await testActor.isNodeCorrectlyHighlighted("#maindiv", is);
await threadFront.removeBreakpoint(bp);
await toolbox.closeToolbox();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
function moveMouseOver(selector, x, y) {
info("Waiting for element " + selector + " to be highlighted");

5
devtools/client/webreplay/mochitest/browser_rr_inspector-03.js
View File

@ -32,7 +32,7 @@ add_task(async function() {
const dbg = await attachRecordingDebugger("doc_inspector_styles.html", {
waitForRecording: true,
});
const { threadFront, tab, toolbox } = dbg;
const { threadFront } = dbg;
await threadFront.interrupt();
await threadFront.resume();
@ -49,8 +49,7 @@ add_task(async function() {
await checkBackgroundColor("#maindiv", "rgb(255, 0, 0)");
await threadFront.removeBreakpoint(bp);
await toolbox.closeToolbox();
await gBrowser.removeTab(tab);
await shutdownDebugger(dbg);
async function checkBackgroundColor(node, color) {
await selectNode(node, inspector);

10
devtools/client/webreplay/mochitest/head.js
View File

@ -24,7 +24,7 @@ const EXAMPLE_URL =
async function attachDebugger(tab) {
const target = await TargetFactory.forTab(tab);
const toolbox = await gDevTools.showToolbox(target, "jsdebugger");
return { toolbox, target };
return { toolbox, tab, target };
}
async function attachRecordingDebugger(
@ -46,6 +46,12 @@ async function attachRecordingDebugger(
return { ...dbg, tab, threadFront, target };
}
async function shutdownDebugger(dbg) {
await waitForRequestsToSettle(dbg);
await dbg.toolbox.destroy();
await gBrowser.removeTab(dbg.tab);
}
// Return a promise that resolves when a breakpoint has been set.
async function setBreakpoint(threadFront, expectedFile, lineno, options = {}) {
const { sources } = await threadFront.getSources();
@ -170,4 +176,4 @@ PromiseTestUtils.whitelistRejectionsGlobally(
// When running the full test suite, long delays can occur early on in tests,
// before child processes have even been spawned. Allow a longer timeout to
// avoid failures from this.
requestLongerTimeout(120);
requestLongerTimeout(4);

268
devtools/server/actors/replay/control.js
View File

@ -37,6 +37,7 @@ const {
positionEquals,
positionSubsumes,
setInterval,
setTimeout,
} = sandbox;
const InvalidCheckpointId = 0;
@ -132,6 +133,9 @@ function ChildProcess(id, recording) {
// Whether this child has diverged from the recording and cannot run forward.
this.divergedFromRecording = false;
// Whether this child has crashed and is unusable.
this.crashed = false;
// Any manifest which is currently being executed. Child processes initially
// have a manifest to run forward to the first checkpoint.
this.manifest = {
@ -148,8 +152,8 @@ function ChildProcess(id, recording) {
// The time the manifest was sent to the child.
this.manifestSendTime = Date.now();
// Per-child state about asynchronous manifests.
this.asyncManifestInfo = new AsyncManifestChildInfo();
// Any async manifest which this child has partially processed.
this.asyncManifest = null;
}
ChildProcess.prototype = {
@ -176,6 +180,7 @@ ChildProcess.prototype = {
// destination: An optional destination where the child will end up.
// expectedDuration: Optional estimate of the time needed to run the manifest.
sendManifest(manifest) {
assert(!this.crashed);
assert(this.paused);
this.paused = false;
this.manifest = manifest;
@ -183,6 +188,24 @@ ChildProcess.prototype = {
dumpv(`SendManifest #${this.id} ${stringify(manifest.contents)}`);
RecordReplayControl.sendManifest(this.id, manifest.contents);
// When sending a manifest to a replaying process, make sure we can detect
// hangs in the process. A hanged replaying process will only be terminated
// if we wait on it, so use a sufficiently long timeout and enter the wait
// if the child hasn't completed its manifest by the deadline.
if (this != gMainChild) {
// Use the same 30 second wait which RecordReplayControl uses, so that the
// hang will be noticed immediately if we start waiting.
let waitDuration = 30 * 1000;
if (manifest.expectedDuration) {
waitDuration += manifest.expectedDuration;
}
setTimeout(() => {
if (!this.crashed && this.manifest == manifest) {
this.waitUntilPaused();
}
}, waitDuration);
}
},
// Called when the child's current manifest finishes.
@ -200,6 +223,10 @@ ChildProcess.prototype = {
this.manifest.onFinished(response);
this.manifest = null;
maybeDumpStatistics();
if (this != gMainChild) {
pokeChildSoon(this);
}
},
// Block until this child is paused. If maybeCreateCheckpoint is specified
@ -348,11 +375,6 @@ function sendAsyncManifest(manifest) {
});
}
function AsyncManifestChildInfo() {
// Any async manifest this child has partially processed.
this.inProgressManifest = null;
}
// Pick the best async manifest for a child to process.
function pickAsyncManifest(child, lowPriority) {
const worklist = asyncManifestWorklist(lowPriority);
@ -403,8 +425,8 @@ function pickAsyncManifest(child, lowPriority) {
function processAsyncManifest(child) {
// If the child has partially processed a manifest, continue with it.
let manifest = child.asyncManifestInfo.inProgressManifest;
child.asyncManifestInfo.inProgressManifest = null;
let manifest = child.asyncManifest;
child.asyncManifest = null;
if (manifest && child == gActiveChild) {
// After a child becomes the active child, it gives up on any in progress
@ -423,18 +445,19 @@ function processAsyncManifest(child) {
}
}
child.asyncManifest = manifest;
if (manifest.point && maybeReachPoint(child, manifest.point)) {
// The manifest has been partially processed.
child.asyncManifestInfo.inProgressManifest = manifest;
return true;
}
child.sendManifest({
contents: manifest.contents(child),
onFinished: data => {
child.asyncManifest = null;
manifest.onFinished(child, data);
manifest.resolve();
pokeChildSoon(child);
},
destination: manifest.destination,
expectedDuration: manifest.expectedDuration,
@ -530,21 +553,6 @@ function addCheckpoint(checkpoint, duration) {
getCheckpointInfo(checkpoint).duration = duration;
}
// Unpause a child and restore it to its most recent saved checkpoint at or
// before target.
function restoreCheckpoint(child, target) {
assert(child.savedCheckpoints.has(target));
child.sendManifest({
contents: { kind: "restoreCheckpoint", target },
onFinished({ restoredCheckpoint }) {
assert(restoredCheckpoint);
child.divergedFromRecording = false;
pokeChildSoon(child);
},
destination: checkpointExecutionPoint(target),
});
}
// Bring a child to the specified execution point, sending it one or more
// manifests if necessary. Returns true if the child has not reached the point
// yet but some progress was made, or false if the child is at the point.
@ -555,33 +563,45 @@ function maybeReachPoint(child, endpoint) {
) {
return false;
}
if (child.divergedFromRecording || child.pausePoint().position) {
restoreCheckpoint(
child,
child.lastSavedCheckpoint(child.pausePoint().checkpoint)
);
restoreCheckpointPriorTo(child.pausePoint().checkpoint);
return true;
}
if (endpoint.checkpoint < child.pauseCheckpoint()) {
restoreCheckpoint(child, child.lastSavedCheckpoint(endpoint.checkpoint));
restoreCheckpointPriorTo(endpoint.checkpoint);
return true;
}
child.sendManifest({
contents: {
kind: "runToPoint",
endpoint,
needSaveCheckpoints: child.flushNeedSaveCheckpoints(),
},
onFinished() {
pokeChildSoon(child);
},
onFinished() {},
destination: endpoint,
expectedDuration: checkpointRangeDuration(
child.pausePoint().checkpoint,
endpoint.checkpoint
),
});
return true;
// Send the child to its most recent saved checkpoint at or before target.
function restoreCheckpointPriorTo(target) {
target = child.lastSavedCheckpoint(target);
child.sendManifest({
contents: { kind: "restoreCheckpoint", target },
onFinished({ restoredCheckpoint }) {
assert(restoredCheckpoint);
child.divergedFromRecording = false;
},
destination: checkpointExecutionPoint(target),
});
}
}
function nextSavedCheckpoint(checkpoint) {
@ -619,7 +639,7 @@ function checkpointExecutionPoint(checkpoint) {
// Check to see if an idle replaying child can make any progress.
function pokeChild(child) {
assert(!child.recording);
assert(child != gMainChild);
if (!child.paused) {
return;
@ -630,12 +650,14 @@ function pokeChild(child) {
}
if (child == gActiveChild) {
sendChildToPausePoint(child);
sendActiveChildToPausePoint();
return;
}
// If there is nothing to do, run forward to the end of the recording.
maybeReachPoint(child, checkpointExecutionPoint(gLastFlushCheckpoint));
if (gLastFlushCheckpoint) {
maybeReachPoint(child, checkpointExecutionPoint(gLastFlushCheckpoint));
}
}
function pokeChildSoon(child) {
@ -977,15 +999,16 @@ function cachedPoints() {
// The pause mode classifies the current state of the debugger.
const PauseModes = {
// Process is actively recording. gPausePoint is the last point the main child
// reached.
// The main child is the active child, and is either paused or actively
// recording. gPausePoint is the last point the main child reached.
RUNNING: "RUNNING",
// gActiveChild is paused at gPausePoint.
// gActiveChild is a replaying child paused at gPausePoint.
PAUSED: "PAUSED",
// gActiveChild is being taken to gPausePoint. The debugger is considered to
// be paused, but interacting with the child must wait until it arrives.
// gActiveChild is a replaying child being taken to gPausePoint. The debugger
// is considered to be paused, but interacting with the child must wait until
// it arrives.
ARRIVING: "ARRIVING",
// gActiveChild is null, and we are looking for the last breakpoint hit prior
@ -997,16 +1020,14 @@ const PauseModes = {
// Current pause mode.
let gPauseMode = PauseModes.RUNNING;
// In PAUSED or ARRIVING mode, the point we are paused at or sending the active child to.
// In PAUSED or ARRIVING modes, the point we are paused at or sending the active
// child to.
let gPausePoint = null;
// In PAUSED mode, any debugger requests that have been sent to the child.
// In ARRIVING mode, the requests must be sent once the child arrives.
const gDebuggerRequests = [];
// In PAUSED mode, whether gDebuggerRequests contains artificial requests that
// need to be synced with the child before new requests can be sent to it.
let gSyncDebuggerRequests = false;
function setPauseState(mode, point, child) {
assert(mode);
const idString = child ? ` #${child.id}` : "";
@ -1016,11 +1037,6 @@ function setPauseState(mode, point, child) {
gPausePoint = point;
gActiveChild = child;
if (mode != PauseModes.PAUSED) {
gDebuggerRequests.length = 0;
gSyncDebuggerRequests = false;
}
if (mode == PauseModes.ARRIVING) {
updateNearbyPoints();
}
@ -1031,6 +1047,7 @@ function setPauseState(mode, point, child) {
// Mark the debugger as paused, and asynchronously send a child to the pause
// point.
function setReplayingPauseTarget(point) {
assert(!gDebuggerRequests.length);
setPauseState(PauseModes.ARRIVING, point, closestChild(point.checkpoint));
gDebugger._onPause();
@ -1038,33 +1055,37 @@ function setReplayingPauseTarget(point) {
findFrameSteps(point);
}
// Synchronously send a child to the specific point and pause.
function pauseReplayingChild(child, point) {
do {
child.waitUntilPaused();
} while (maybeReachPoint(child, point));
setPauseState(PauseModes.PAUSED, point, child);
}
function sendChildToPausePoint(child) {
assert(child.paused && child == gActiveChild);
function sendActiveChildToPausePoint() {
assert(gActiveChild.paused);
switch (gPauseMode) {
case PauseModes.PAUSED:
assert(pointEquals(child.pausePoint(), gPausePoint));
assert(pointEquals(gActiveChild.pausePoint(), gPausePoint));
return;
case PauseModes.ARRIVING:
if (pointEquals(child.pausePoint(), gPausePoint)) {
if (pointEquals(gActiveChild.pausePoint(), gPausePoint)) {
setPauseState(PauseModes.PAUSED, gPausePoint, gActiveChild);
// Send any debugger requests the child is considered to have received.
if (gDebuggerRequests.length) {
gActiveChild.sendManifest({
contents: {
kind: "batchDebuggerRequest",
requests: gDebuggerRequests,
},
onFinished(finishData) {
assert(!finishData || !finishData.restoredCheckpoint);
},
});
}
} else {
maybeReachPoint(child, gPausePoint);
maybeReachPoint(gActiveChild, gPausePoint);
}
return;
default:
throw new Error(`Unexpected pause mode: ${gPauseMode}`);
ThrowError(`Unexpected pause mode: ${gPauseMode}`);
}
}
@ -1085,6 +1106,12 @@ function waitUntilPauseFinishes() {
}
}
// Synchronously send a child to the specific point and pause.
function pauseReplayingChild(child, point) {
setPauseState(PauseModes.ARRIVING, point, child);
waitUntilPauseFinishes();
}
// After the debugger resumes, find the point where it should pause next.
async function finishResume() {
assert(
@ -1162,6 +1189,7 @@ async function finishResume() {
// Unpause the active child and asynchronously pause at the next or previous
// breakpoint hit.
function resume(forward) {
gDebuggerRequests.length = 0;
if (gActiveChild.recording) {
if (forward) {
maybeResumeRecording();
@ -1187,10 +1215,79 @@ function resume(forward) {
// Synchronously bring the active child to the specified execution point.
function timeWarp(point) {
gDebuggerRequests.length = 0;
setReplayingPauseTarget(point);
Services.cpmm.sendAsyncMessage("TimeWarpFinished");
}
////////////////////////////////////////////////////////////////////////////////
// Crash Recovery
////////////////////////////////////////////////////////////////////////////////
// The maximum number of crashes which we can recover from.
const MaxCrashes = 4;
// How many child processes have crashed.
let gNumCrashes = 0;
// eslint-disable-next-line no-unused-vars
function ChildCrashed(id) {
dumpv(`Child Crashed: ${id}`);
// In principle we can recover when any replaying child process crashes.
// For simplicity, there are some cases where we don't yet try to recover if
// a replaying process crashes.
//
// - It is the main child, and running forward through the recording. While it
// could crash here just as easily as any other replaying process, any crash
// will happen early on and won't interrupt a long-running debugger session.
//
// - It is the active child, and is paused at gPausePoint. It must have
// crashed while processing a debugger request, which is unlikely.
const child = gReplayingChildren[id];
if (
!child ||
!child.manifest ||
(child == gActiveChild && gPauseMode == PauseModes.PAUSED)
) {
ThrowError("Cannot recover from crashed child");
}
if (++gNumCrashes > MaxCrashes) {
ThrowError("Too many crashes");
}
delete gReplayingChildren[id];
child.crashed = true;
// Spawn a new child to replace the one which just crashed.
const newChild = spawnReplayingChild();
pokeChildSoon(newChild);
// The new child should save the same checkpoints as the old one.
for (const checkpoint of child.savedCheckpoints) {
newChild.addSavedCheckpoint(checkpoint);
}
// Any regions the old child scanned need to be rescanned.
for (const checkpoint of child.scannedCheckpoints) {
scanRecording(checkpoint);
}
// Requeue any async manifest the child was processing.
if (child.asyncManifest) {
sendAsyncManifest(child.asyncManifest);
}
// If the active child crashed while heading to the pause point, pick another
// child to head to the pause point.
if (child == gActiveChild) {
assert(gPauseMode == PauseModes.ARRIVING);
gActiveChild = closestChild(gPausePoint.checkpoint);
pokeChildSoon(gActiveChild);
}
}
////////////////////////////////////////////////////////////////////////////////
// Nearby Points
////////////////////////////////////////////////////////////////////////////////
@ -1522,13 +1619,19 @@ function setMainChild() {
// Child Management
////////////////////////////////////////////////////////////////////////////////
function spawnReplayingChild() {
const id = RecordReplayControl.spawnReplayingChild();
const child = new ChildProcess(id, false);
gReplayingChildren[id] = child;
return child;
}
// How many replaying children to spawn. This should be a pref instead...
const NumReplayingChildren = 4;
function spawnReplayingChildren() {
for (let i = 0; i < NumReplayingChildren; i++) {
const id = RecordReplayControl.spawnReplayingChild();
gReplayingChildren[id] = new ChildProcess(id, false);
spawnReplayingChild();
}
addSavedCheckpoint(FirstCheckpointId);
}
@ -1669,17 +1772,6 @@ const gControl = {
sendRequest(request) {
waitUntilPauseFinishes();
if (gSyncDebuggerRequests) {
gActiveChild.sendManifest({
contents: { kind: "batchDebuggerRequest", requests: gDebuggerRequests },
onFinished(finishData) {
assert(!finishData || !finishData.restoredCheckpoint);
},
});
gActiveChild.waitUntilPaused();
gSyncDebuggerRequests = false;
}
let data;
gActiveChild.sendManifest({
contents: { kind: "debuggerRequest", request },
@ -1694,13 +1786,6 @@ const gControl = {
// checkpoint. Restore the child to the point it should be paused at and
// fill its paused state back in by resending earlier debugger requests.
pauseReplayingChild(gActiveChild, gPausePoint);
gActiveChild.sendManifest({
contents: { kind: "batchDebuggerRequest", requests: gDebuggerRequests },
onFinished(finishData) {
assert(!finishData || !finishData.restoredCheckpoint);
},
});
gActiveChild.waitUntilPaused();
return { unhandledDivergence: true };
}
@ -1747,11 +1832,13 @@ const gControl = {
cachedPoints,
getPauseData() {
if (!gDebuggerRequests.length) {
assert(!gSyncDebuggerRequests);
// If the child has not arrived at the pause point yet, see if there is
// cached pause data for this point already which we can immediately return.
if (gPauseMode == PauseModes.ARRIVING && !gDebuggerRequests.length) {
const data = maybeGetPauseData(gPausePoint);
if (data) {
gSyncDebuggerRequests = true;
// After the child pauses, it will need to generate the pause data so
// that any referenced objects will be instantiated.
gDebuggerRequests.push({ type: "pauseData" });
return data;
}
@ -1907,4 +1994,5 @@ var EXPORTED_SYMBOLS = [
"ManifestFinished",
"BeforeSaveRecording",
"AfterSaveRecording",
"ChildCrashed",
];

1
devtools/server/actors/replay/rrIControl.idl
View File

@ -14,4 +14,5 @@ interface rrIControl : nsISupports {
void ManifestFinished(in long childId, in jsval response);
void BeforeSaveRecording();
void AfterSaveRecording();
void ChildCrashed(in long childId);
};

74
toolkit/recordreplay/ipc/ChildProcess.cpp
View File

@ -30,11 +30,7 @@ void ChildProcessInfo::SetIntroductionMessage(IntroductionMessage* aMessage) {
ChildProcessInfo::ChildProcessInfo(
const Maybe<RecordingProcessData>& aRecordingProcessData)
: mChannel(nullptr),
mRecording(aRecordingProcessData.isSome()),
mPaused(false),
mHasBegunFatalError(false),
mHasFatalError(false) {
: mRecording(aRecordingProcessData.isSome()) {
MOZ_RELEASE_ASSERT(NS_IsMainThread());
static bool gFirst = false;
@ -48,7 +44,7 @@ ChildProcessInfo::ChildProcessInfo(
ChildProcessInfo::~ChildProcessInfo() {
MOZ_RELEASE_ASSERT(NS_IsMainThread());
if (IsRecording()) {
if (IsRecording() && !HasCrashed()) {
SendMessage(TerminateMessage());
}
}
@ -96,6 +92,7 @@ void ChildProcessInfo::OnIncomingMessage(const Message& aMsg) {
void ChildProcessInfo::SendMessage(Message&& aMsg) {
MOZ_RELEASE_ASSERT(NS_IsMainThread());
MOZ_RELEASE_ASSERT(!HasCrashed());
// Update paused state.
MOZ_RELEASE_ASSERT(IsPaused() || aMsg.CanBeSentWhileUnpaused());
@ -190,6 +187,20 @@ void ChildProcessInfo::OnCrash(const char* aWhy) {
CrashReporter::AnnotateCrashReport(
CrashReporter::Annotation::RecordReplayError, nsAutoCString(aWhy));
if (!IsRecording()) {
// Notify the parent when a replaying process crashes so that a report can
// be generated.
dom::ContentChild::GetSingleton()->SendGenerateReplayCrashReport(GetId());
// Continue execution if we were able to recover from the crash.
if (js::RecoverFromCrash(this)) {
// Mark this child as crashed so it can't be used again, even if it didn't
// generate a minidump.
mHasFatalError = true;
return;
}
}
// If we received a FatalError message then the child generated a minidump.
// Shut down cleanly so that we don't mask the report with our own crash.
if (mHasFatalError) {
@ -279,33 +290,34 @@ void ChildProcessInfo::WaitUntilPaused() {
if (IsPaused()) {
return;
}
} else if (HasCrashed()) {
// If the child crashed but we recovered, we don't have to keep waiting.
return;
} else if (gChildrenAreDebugging || IsRecording()) {
// Don't watch for hangs when children are being debugged. Recording
// children are never treated as hanged both because we can't recover if
// they crash and because they may just be idling.
gMonitor->Wait();
} else {
if (gChildrenAreDebugging || IsRecording()) {
// Don't watch for hangs when children are being debugged. Recording
// children are never treated as hanged both because they cannot be
// restarted and because they may just be idling.
gMonitor->Wait();
} else {
TimeStamp deadline =
mLastMessageTime + TimeDuration::FromSeconds(HangSeconds);
if (TimeStamp::Now() >= deadline) {
MonitorAutoUnlock unlock(*gMonitor);
if (!sentTerminateMessage) {
// Try to get the child to crash, so that we can get a minidump.
// Sending the message will reset mLastMessageTime so we get to
// wait another HangSeconds before hitting the restart case below.
// Use SendMessageRaw to avoid problems if we are recovering.
CrashReporter::AnnotateCrashReport(
CrashReporter::Annotation::RecordReplayHang, true);
SendMessage(TerminateMessage());
sentTerminateMessage = true;
} else {
// The child is still non-responsive after sending the terminate
// message.
OnCrash("Child process non-responsive");
}
}
TimeStamp deadline =
mLastMessageTime + TimeDuration::FromSeconds(HangSeconds);
if (TimeStamp::Now() < deadline) {
gMonitor->WaitUntil(deadline);
} else {
MonitorAutoUnlock unlock(*gMonitor);
if (!sentTerminateMessage) {
// Try to get the child to crash, so that we can get a minidump.
// Sending the message will reset mLastMessageTime so we get to
// wait another HangSeconds before hitting the restart case below.
CrashReporter::AnnotateCrashReport(
CrashReporter::Annotation::RecordReplayHang, true);
SendMessage(TerminateMessage());
sentTerminateMessage = true;
} else {
// The child is still non-responsive after sending the terminate
// message.
OnCrash("Child process non-responsive");
}
}
}
}

25
toolkit/recordreplay/ipc/JSControl.cpp
View File

@ -50,8 +50,16 @@ static parent::ChildProcessInfo* GetChildById(JSContext* aCx,
return nullptr;
}
parent::ChildProcessInfo* child = parent::GetChildProcess(aValue.toNumber());
if (!child || (!aAllowUnpaused && !child->IsPaused())) {
JS_ReportErrorASCII(aCx, "Unpaused or bad child ID");
if (!child) {
JS_ReportErrorASCII(aCx, "Bad child ID");
return nullptr;
}
if (child->HasCrashed()) {
JS_ReportErrorASCII(aCx, "Child has crashed");
return nullptr;
}
if (!aAllowUnpaused && !child->IsPaused()) {
JS_ReportErrorASCII(aCx, "Child is unpaused");
return nullptr;
}
return child;
@ -126,6 +134,19 @@ void AfterSaveRecording() {
}
}
bool RecoverFromCrash(parent::ChildProcessInfo* aChild) {
MOZ_RELEASE_ASSERT(gControl);
AutoSafeJSContext cx;
JSAutoRealm ar(cx, xpc::PrivilegedJunkScope());
if (NS_FAILED(gControl->ChildCrashed(aChild->GetId()))) {
return false;
}
return true;
}
///////////////////////////////////////////////////////////////////////////////
// Middleman Methods
///////////////////////////////////////////////////////////////////////////////

3
toolkit/recordreplay/ipc/JSControl.h
View File

@ -77,6 +77,9 @@ void BeforeCheckpoint();
// true if we just rewound.
void AfterCheckpoint(size_t aCheckpoint, bool aRestoredCheckpoint);
// Called when a child crashes, returning whether the crash was recovered from.
bool RecoverFromCrash(parent::ChildProcessInfo* aChild);
// Accessors for state which can be accessed from JS.
// Mark a time span when the main thread is idle.

11
toolkit/recordreplay/ipc/ParentInternal.h
View File

@ -151,21 +151,21 @@ struct RecordingProcessData {
// Information about a recording or replaying child process.
class ChildProcessInfo {
// Channel for communicating with the process.
Channel* mChannel;
Channel* mChannel = nullptr;
// The last time we sent or received a message from this process.
TimeStamp mLastMessageTime;
// Whether this process is recording.
bool mRecording;
bool mRecording = false;
// Whether the process is currently paused.
bool mPaused;
bool mPaused = false;
// Flags for whether we have received messages from the child indicating it
// is crashing.
bool mHasBegunFatalError;
bool mHasFatalError;
bool mHasBegunFatalError = false;
bool mHasFatalError = false;
void OnIncomingMessage(const Message& aMsg);
@ -184,6 +184,7 @@ class ChildProcessInfo {
size_t GetId() { return mChannel->GetId(); }
bool IsRecording() { return mRecording; }
bool IsPaused() { return mPaused; }
bool HasCrashed() { return mHasFatalError; }
// Send a message over the underlying channel.
void SendMessage(Message&& aMessage);