mirror of
https://github.com/expo/explore.git
synced 2026-11-14 10:08:17 -10:00
b10fba99e6
Co-authored-by: Brent Vatne <[email protected]> Co-authored-by: Szymon Dziedzic <[email protected]>
492 lines
16 KiB
JavaScript
492 lines
16 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
|
|
import {
|
|
RUNTIME_EVIDENCE_COMPARISON_CONTRACT,
|
|
RUNTIME_EVIDENCE_CONTRACT,
|
|
RUNTIME_EVIDENCE_GROUP_ORDER,
|
|
projectRuntimeEvidence,
|
|
projectRuntimeEvidenceComparison,
|
|
serializeRuntimeEvidence,
|
|
} from "../../dist/review/runtime-evidence.js";
|
|
|
|
function evidence(side = "base", overrides = {}) {
|
|
const observationId = overrides.observationId ?? `${side}-observation`;
|
|
const variantId = overrides.variantId ?? "signed-in";
|
|
return {
|
|
context: {
|
|
kind: "configured-capture-context",
|
|
side,
|
|
logicalBuild: `${side}-sha`,
|
|
profile: "iphone-17-review",
|
|
surfaceId: "settings-profile",
|
|
route: "/settings/profile",
|
|
state: "ready",
|
|
variantId,
|
|
observationId,
|
|
expectedSampleCount: 3,
|
|
fixturePersona: "local-pro",
|
|
},
|
|
runtime: {
|
|
kind: "eas-agent-device-raw",
|
|
observation: {
|
|
id: observationId,
|
|
variantId,
|
|
capturedAt: "2026-09-01T01:00:00Z",
|
|
surfaceScope: "app",
|
|
accessibility: {
|
|
truncated: false,
|
|
elements: [
|
|
{ id: "z-heading", role: "text", label: "Profile", actionable: false, landmark: true },
|
|
{ id: "a-save", role: "button", label: "Save", actionable: true, landmark: false },
|
|
{ id: "b-cancel", role: "button", label: "Cancel", actionable: true, landmark: false },
|
|
],
|
|
},
|
|
},
|
|
interactions: [
|
|
{
|
|
id: "second",
|
|
observationId,
|
|
action: "back",
|
|
label: "Back",
|
|
outcome: "same-screen",
|
|
startedAt: "2026-09-01T01:00:03Z",
|
|
completedAt: "2026-09-01T01:00:03.100Z",
|
|
durationMs: 100,
|
|
},
|
|
{
|
|
id: "first",
|
|
observationId,
|
|
action: "press",
|
|
label: "Save",
|
|
outcome: "transition",
|
|
startedAt: "2026-09-01T01:00:01Z",
|
|
completedAt: "2026-09-01T01:00:01.250Z",
|
|
durationMs: 250,
|
|
target: { surfaceId: "settings-root", route: "/settings" },
|
|
},
|
|
],
|
|
captures: [
|
|
{
|
|
id: "sample-2",
|
|
observationId,
|
|
variantId,
|
|
capturedAt: "2026-09-01T01:00:02Z",
|
|
widthPixels: 1179,
|
|
heightPixels: 2556,
|
|
pixelDensity: 3,
|
|
fingerprints: { combined: "same", visual: "visual", accessibility: "a11y" },
|
|
},
|
|
{
|
|
id: "sample-1",
|
|
observationId,
|
|
variantId,
|
|
capturedAt: "2026-09-01T01:00:01Z",
|
|
widthPixels: 1179,
|
|
heightPixels: 2556,
|
|
pixelDensity: 3,
|
|
fingerprints: { combined: "same", visual: "visual", accessibility: "a11y" },
|
|
},
|
|
],
|
|
environment: {
|
|
provider: "eas-simulator",
|
|
controller: "agent-device",
|
|
platform: "ios",
|
|
deviceModel: side === "base" ? "iPhone 16" : "iPhone 17",
|
|
osVersion: side === "base" ? "26.4" : "26.5",
|
|
locale: "en-US",
|
|
orientation: "portrait",
|
|
colorScheme: "light",
|
|
sessionLifecycle: "stopped",
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
test("projects deterministic, origin-tagged Screen/UI/Interaction/Capture/Environment groups", () => {
|
|
const input = evidence();
|
|
const projected = projectRuntimeEvidence(input);
|
|
|
|
assert.equal(projected.contract, RUNTIME_EVIDENCE_CONTRACT);
|
|
assert.deepEqual(Object.keys(projected.groups), RUNTIME_EVIDENCE_GROUP_ORDER);
|
|
assert.deepEqual(projected.groups.screen.side, { origin: "capture-context", value: "base" });
|
|
assert.deepEqual(projected.groups.screen.observationId, {
|
|
origin: "eas-agent-device",
|
|
value: "base-observation",
|
|
});
|
|
assert.deepEqual(projected.groups.ui.roleCounts, {
|
|
origin: "derived",
|
|
value: [
|
|
{ role: "button", count: 2 },
|
|
{ role: "text", count: 1 },
|
|
],
|
|
});
|
|
assert.deepEqual(projected.groups.ui.landmarks.value, [
|
|
{ id: "z-heading", role: "text", label: "Profile" },
|
|
]);
|
|
assert.equal(projected.groups.ui.snapshotQuality.value, "complete");
|
|
assert.deepEqual(
|
|
projected.groups.interaction.attempts.value.map((attempt) => attempt.id),
|
|
["first", "second"],
|
|
);
|
|
assert.equal(projected.groups.interaction.transitionCount.value, 1);
|
|
assert.deepEqual(projected.groups.interaction.outcomeCounts.value, {
|
|
transition: 1,
|
|
"same-screen": 1,
|
|
"no-change": 0,
|
|
failed: 0,
|
|
blocked: 0,
|
|
external: 0,
|
|
});
|
|
assert.deepEqual(
|
|
projected.groups.capture.samples.value.map((sample) => sample.id),
|
|
["sample-1", "sample-2"],
|
|
);
|
|
assert.equal(projected.groups.capture.stability.value, "stable");
|
|
assert.deepEqual(projected.groups.capture.coverage.value, {
|
|
expected: 3,
|
|
captured: 2,
|
|
ratio: 0.666667,
|
|
complete: false,
|
|
});
|
|
assert.deepEqual(projected.groups.environment.profile, {
|
|
origin: "capture-context",
|
|
value: "iphone-17-review",
|
|
});
|
|
assert.deepEqual(projected.groups.environment.deviceModel, {
|
|
origin: "eas-agent-device",
|
|
value: "iPhone 16",
|
|
});
|
|
|
|
assert.deepEqual(projectRuntimeEvidence(structuredClone(input)), projected);
|
|
assert.equal(serializeRuntimeEvidence(input), `${JSON.stringify(projected)}\n`);
|
|
assert.deepEqual(JSON.parse(serializeRuntimeEvidence(input)), projected);
|
|
});
|
|
|
|
test("judges sample stability on the accessibility tree, not on pixel-identical screenshots", () => {
|
|
const input = evidence();
|
|
input.runtime.captures[1].fingerprints.combined = "different-pixels";
|
|
input.runtime.captures[1].fingerprints.visual = "different-pixels";
|
|
assert.equal(projectRuntimeEvidence(input).groups.capture.stability.value, "stable");
|
|
input.runtime.captures[1].fingerprints.accessibility = "different-tree";
|
|
assert.equal(projectRuntimeEvidence(input).groups.capture.stability.value, "unstable");
|
|
});
|
|
|
|
test("projects a deterministic non-empty allowlisted Diagnostics group", () => {
|
|
const input = evidence();
|
|
input.runtime.captures[1].fingerprints.combined = "different";
|
|
input.runtime.captures[1].fingerprints.accessibility = "different-tree";
|
|
input.diagnostics = {
|
|
kind: "diagnostics-extension",
|
|
entries: [
|
|
{
|
|
code: "capture.stability-unstable",
|
|
title: "Capture stability anomaly",
|
|
value: "unstable",
|
|
origin: "derived",
|
|
},
|
|
{
|
|
code: "capture.sample-shortfall",
|
|
title: "Capture sample anomaly",
|
|
value: "shortfall",
|
|
origin: "derived",
|
|
},
|
|
{
|
|
code: "capture.sample-plan",
|
|
title: "Capture sample plan",
|
|
value: "triple",
|
|
origin: "capture-context",
|
|
},
|
|
],
|
|
};
|
|
|
|
const projected = projectRuntimeEvidence(input);
|
|
assert.deepEqual(Object.keys(projected.groups), [
|
|
...RUNTIME_EVIDENCE_GROUP_ORDER,
|
|
"diagnostics",
|
|
]);
|
|
assert.deepEqual(projected.groups.diagnostics, {
|
|
id: "diagnostics",
|
|
entries: [
|
|
{
|
|
code: "capture.sample-plan",
|
|
title: "Capture sample plan",
|
|
value: "triple",
|
|
origin: "capture-context",
|
|
},
|
|
{
|
|
code: "capture.sample-shortfall",
|
|
title: "Capture sample anomaly",
|
|
value: "shortfall",
|
|
origin: "derived",
|
|
},
|
|
{
|
|
code: "capture.stability-unstable",
|
|
title: "Capture stability anomaly",
|
|
value: "unstable",
|
|
origin: "derived",
|
|
},
|
|
],
|
|
});
|
|
assert.deepEqual(projectRuntimeEvidence(structuredClone(input)), projected);
|
|
|
|
const controllerDiagnostic = evidence();
|
|
controllerDiagnostic.runtime.observation.accessibility.truncated = true;
|
|
controllerDiagnostic.diagnostics = {
|
|
kind: "diagnostics-extension",
|
|
entries: [{
|
|
code: "ui.accessibility-truncated",
|
|
title: "Accessibility snapshot truncated",
|
|
value: "truncated",
|
|
origin: "eas-agent-device",
|
|
}],
|
|
};
|
|
assert.deepEqual(
|
|
projectRuntimeEvidence(controllerDiagnostic).groups.diagnostics.entries[0],
|
|
controllerDiagnostic.diagnostics.entries[0],
|
|
);
|
|
});
|
|
|
|
test("does not fabricate pixel density when the runtime controller omits it", () => {
|
|
const input = evidence();
|
|
delete input.runtime.captures[0].pixelDensity;
|
|
delete input.runtime.captures[1].pixelDensity;
|
|
const projected = projectRuntimeEvidence(input);
|
|
assert.deepEqual(projected.groups.capture.pixelDensities.value, []);
|
|
assert.equal("pixelDensity" in projected.groups.capture.samples.value[0], false);
|
|
});
|
|
|
|
test("keeps base and head evidence independently scoped", () => {
|
|
const result = projectRuntimeEvidenceComparison({
|
|
status: "modified",
|
|
base: evidence("base"),
|
|
head: evidence("head"),
|
|
});
|
|
|
|
assert.equal(result.contract, RUNTIME_EVIDENCE_COMPARISON_CONTRACT);
|
|
assert.equal(result.status, "modified");
|
|
assert.equal(result.base.groups.screen.side.value, "base");
|
|
assert.equal(result.head.groups.screen.side.value, "head");
|
|
assert.equal(result.base.groups.screen.observationId.value, "base-observation");
|
|
assert.equal(result.head.groups.screen.observationId.value, "head-observation");
|
|
assert.equal(result.base.groups.environment.deviceModel.value, "iPhone 16");
|
|
assert.equal(result.head.groups.environment.deviceModel.value, "iPhone 17");
|
|
});
|
|
|
|
test("models added as head-only and removed as base-only without environment borrowing", () => {
|
|
const added = projectRuntimeEvidenceComparison({ status: "added", head: evidence("head") });
|
|
assert.equal(added.status, "added");
|
|
assert.equal("base" in added, false);
|
|
|
|
const removed = projectRuntimeEvidenceComparison({ status: "removed", base: evidence("base") });
|
|
assert.equal(removed.status, "removed");
|
|
assert.equal("head" in removed, false);
|
|
assert.equal(removed.base.groups.environment.deviceModel.value, "iPhone 16");
|
|
|
|
assert.throws(
|
|
() => projectRuntimeEvidenceComparison({
|
|
status: "removed",
|
|
base: evidence("base"),
|
|
head: evidence("head"),
|
|
}),
|
|
/unsupported fields/,
|
|
);
|
|
assert.throws(
|
|
() => projectRuntimeEvidenceComparison({ status: "removed", base: evidence("head") }),
|
|
/wrong side scope/,
|
|
);
|
|
assert.throws(
|
|
() => projectRuntimeEvidenceComparison({ status: "modified", base: evidence("base") }),
|
|
/incomplete/,
|
|
);
|
|
});
|
|
|
|
test("enforces representative observation and variant scope while allowing repeated observation samples", () => {
|
|
const wrongObservation = evidence();
|
|
wrongObservation.context.observationId = "another-observation";
|
|
assert.throws(() => projectRuntimeEvidence(wrongObservation), /observation scope/);
|
|
|
|
const wrongVariant = evidence();
|
|
wrongVariant.runtime.captures[0].variantId = "logged-out";
|
|
assert.throws(() => projectRuntimeEvidence(wrongVariant), /variant scope/);
|
|
|
|
const wrongCaptureObservation = evidence();
|
|
wrongCaptureObservation.runtime.captures[0].observationId = "another-observation";
|
|
assert.deepEqual(
|
|
projectRuntimeEvidence(wrongCaptureObservation).groups.capture.samples.value
|
|
.map((sample) => sample.observationId),
|
|
["base-observation", "another-observation"],
|
|
);
|
|
|
|
const wrongInteractionObservation = evidence();
|
|
wrongInteractionObservation.runtime.interactions[0].observationId = "another-observation";
|
|
assert.throws(() => projectRuntimeEvidence(wrongInteractionObservation), /observation scope/);
|
|
|
|
const duplicate = evidence();
|
|
duplicate.runtime.interactions.push(structuredClone(duplicate.runtime.interactions[0]));
|
|
assert.throws(() => projectRuntimeEvidence(duplicate), /duplicate ids/);
|
|
});
|
|
|
|
test("rejects fabricated mock fields and private diagnostics", () => {
|
|
const forbiddenFields = [
|
|
"server",
|
|
"hydration",
|
|
"tokenExpiry",
|
|
"formValidity",
|
|
"apiKey",
|
|
"rawLogs",
|
|
"networkTrace",
|
|
"performanceTrace",
|
|
"video",
|
|
"sourcePath",
|
|
];
|
|
for (const field of forbiddenFields) {
|
|
const input = evidence();
|
|
input.runtime.environment[field] = "do-not-publish-this-value";
|
|
assert.throws(
|
|
() => projectRuntimeEvidence(input),
|
|
(error) => {
|
|
assert.match(error.message, /forbidden/);
|
|
assert.doesNotMatch(error.message, /do-not-publish/);
|
|
return true;
|
|
},
|
|
field,
|
|
);
|
|
}
|
|
|
|
const unknown = evidence();
|
|
unknown.runtime.environment.permissionState = "granted";
|
|
assert.throws(() => projectRuntimeEvidence(unknown), /unsupported fields/);
|
|
|
|
for (const entry of [
|
|
{ kind: "log", value: "private" },
|
|
{ rawLogs: ["private"] },
|
|
{ networkTrace: "private" },
|
|
{ sourcePath: "/private/source.tsx" },
|
|
{ apiKey: "private" },
|
|
{ url: "https://example.test/private" },
|
|
]) {
|
|
const diagnostics = evidence();
|
|
diagnostics.diagnostics = { kind: "diagnostics-extension", entries: [entry] };
|
|
assert.throws(
|
|
() => projectRuntimeEvidence(diagnostics),
|
|
/forbidden|unsupported fields|Diagnostics code/,
|
|
);
|
|
}
|
|
});
|
|
|
|
test("rejects empty, fabricated, duplicate, and runtime-unsupported diagnostics", () => {
|
|
const diagnostics = evidence();
|
|
diagnostics.diagnostics = { kind: "diagnostics-extension", entries: [] };
|
|
assert.throws(() => projectRuntimeEvidence(diagnostics), /at least one safe entry/);
|
|
|
|
const unsupportedFact = evidence();
|
|
unsupportedFact.diagnostics = {
|
|
kind: "diagnostics-extension",
|
|
entries: [{
|
|
code: "capture.stability-unstable",
|
|
title: "Capture stability anomaly",
|
|
value: "unstable",
|
|
origin: "derived",
|
|
}],
|
|
};
|
|
assert.throws(() => projectRuntimeEvidence(unsupportedFact), /not supported by runtime evidence/);
|
|
|
|
for (const override of [
|
|
{ title: "https://example.test/private" },
|
|
{ value: "arbitrary reviewer prose" },
|
|
{ origin: "capture-context" },
|
|
]) {
|
|
const fabricated = evidence();
|
|
fabricated.diagnostics = {
|
|
kind: "diagnostics-extension",
|
|
entries: [{
|
|
code: "capture.sample-shortfall",
|
|
title: "Capture sample anomaly",
|
|
value: "shortfall",
|
|
origin: "derived",
|
|
...override,
|
|
}],
|
|
};
|
|
assert.throws(() => projectRuntimeEvidence(fabricated), /Diagnostics .*invalid/);
|
|
}
|
|
|
|
const duplicate = evidence();
|
|
const entry = {
|
|
code: "capture.sample-shortfall",
|
|
title: "Capture sample anomaly",
|
|
value: "shortfall",
|
|
origin: "derived",
|
|
};
|
|
duplicate.diagnostics = {
|
|
kind: "diagnostics-extension",
|
|
entries: [entry, structuredClone(entry)],
|
|
};
|
|
assert.throws(() => projectRuntimeEvidence(duplicate), /duplicate codes/);
|
|
});
|
|
|
|
test("rejects non-JSON numbers, unsafe routes, and inconsistent capture coverage", () => {
|
|
const nonFinite = evidence();
|
|
nonFinite.runtime.captures[0].pixelDensity = Number.NaN;
|
|
assert.throws(() => projectRuntimeEvidence(nonFinite), /pixel density.*invalid/i);
|
|
|
|
const unsafeRoute = evidence();
|
|
unsafeRoute.context.route = "/settings?access_token=private";
|
|
assert.throws(() => projectRuntimeEvidence(unsafeRoute), /Route is invalid/);
|
|
|
|
const invalidCalendarDate = evidence();
|
|
invalidCalendarDate.runtime.observation.capturedAt = "2026-02-30T01:00:00Z";
|
|
assert.throws(() => projectRuntimeEvidence(invalidCalendarDate), /capture time.*invalid/i);
|
|
|
|
const overflow = evidence();
|
|
overflow.context.expectedSampleCount = 1;
|
|
assert.throws(() => projectRuntimeEvidence(overflow), /exceed.*sample scope/);
|
|
});
|
|
|
|
test("preserves safe canonical app-route query variants", () => {
|
|
const vocabulary = evidence();
|
|
vocabulary.context.route = "/vocabulary?tab=human";
|
|
assert.equal(
|
|
projectRuntimeEvidence(vocabulary).groups.screen.route.value,
|
|
"/vocabulary?tab=human",
|
|
);
|
|
|
|
const translation = evidence();
|
|
translation.context.route = "/translation/record?mode=practice&locale=en-US";
|
|
assert.equal(
|
|
projectRuntimeEvidence(translation).groups.screen.route.value,
|
|
"/translation/record?mode=practice&locale=en-US",
|
|
);
|
|
|
|
for (const route of [
|
|
"https://example.test/profile",
|
|
"//example.test/profile",
|
|
"/profile#private",
|
|
"/profile/../private",
|
|
"/profile/%2e%2e/private",
|
|
"/profile?api_key=private",
|
|
"/profile?authorization=private",
|
|
"/profile?tab=%00private",
|
|
]) {
|
|
const unsafe = evidence();
|
|
unsafe.context.route = route;
|
|
assert.throws(() => projectRuntimeEvidence(unsafe), /Route is invalid/, route);
|
|
}
|
|
});
|
|
|
|
test("does not serialize mock, credential, raw telemetry, or source-path fields", () => {
|
|
const serialized = serializeRuntimeEvidence(evidence()).toLowerCase();
|
|
for (const forbidden of [
|
|
"hydration",
|
|
"token",
|
|
"formvalidity",
|
|
"rawlogs",
|
|
"networktrace",
|
|
"performancetrace",
|
|
"video",
|
|
"sourcepath",
|
|
]) {
|
|
assert.equal(serialized.includes(forbidden), false, forbidden);
|
|
}
|
|
});
|