mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-07 22:58:42 -10:00
fix(daemon): batch slice-membership index — kill the O(slices) queue scans stalling the Project mission index
PM flagship finding: the Project page sat on "Reading the mission index" for 10-30s (28 missions / 353 slices on the host; cold /api/slices 9.8-10.3s, concurrent healthz held 13s) because SliceIndexer.matchQitems ran per-slice full-table queue_items scans — a leading-wildcard tags LIKE with a full ORDER BY sort for every slice, plus up to three body-LIKE scans for every zero-typed slice (~1100 scans per rebuild), and the UI forces that cold rebuild on every mount/focus via refresh=1. The rebuild is synchronous on the daemon event loop, so every endpoint starved for the duration. Replace the per-slice tiers with a per-generation batch MembershipIndex built in TWO streamed scans: (1) typed membership — tags LIKE '%slice:%' prefilter in ts_created DESC, qitem_id DESC order, parseScopeTags as the authoritative confirm, a row crediting every slice it tags, capped after 500 confirmed per slice; (2) fallback buckets for zero-typed slices' term universe — combined regex prefilter, then EVERY term confirmed per row (overlap-safe), with SQL-LIKE byte-equivalence: ASCII-only case folding, %/_ translated to .*/. compiled 'su' so wildcards span newlines and '_' matches one Unicode code point (a😀b matches %a_b%). Term-first assembly in the original deduped order keeps equal-timestamp output identical. matchQitems answers from the index; fresh folders force a rebuild via the knownSlices set; invalidate()/TTL discipline unchanged; tags-column and table-absent degradations preserved. One production file. Load contract pinned by test: one cold list() now executes a constant number of queue_items LIKE statements (was 96 at 40 slices). Deterministic bench at host scale (28 missions/353 slices, 8000 qitems x 3KB): cold 11442.9ms -> 98.7ms (~116x); the per-rebuild event-loop blocking window drops from ~10s to <0.1s. qitem-20260720033934-ccf87c0d Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
v-openrig-buildandClaude Fable 5 committed
1 parent
7b19b73e5a
commit
c8f8580205
2 files changed
+473
-59
No files matched your search
@@ -137,6 +137,59 @@ interface SliceLocation {
|
||||
slicePath: string;
|
||||
}
|
||||
|
||||
/** qitem-ccf87c0d — the per-generation batch membership index. Replaces the
|
||||
* per-slice queue_items scans (353 tier-1 wildcard+ORDER-BY scans + up to 3
|
||||
* body-LIKE scans per zero-typed slice = ~10s cold at host scale,
|
||||
* synchronously blocking the event loop) with TWO streamed scans per
|
||||
* generation. Semantics are byte-equivalent to the per-slice path and are
|
||||
* pinned by test: typed-authoritative gating (VM-004), per-slice 500
|
||||
* confirmed cap in ts_created DESC/qitem_id DESC order, term-first fallback
|
||||
* assembly, SQL-LIKE ASCII-only case folding, %/_ wildcards spanning
|
||||
* newline, per-term 500 cap, and the tags-column/table-absent degradations. */
|
||||
interface MembershipIndex {
|
||||
/** slice name -> CONFIRMED typed qitem ids (scan order = ts DESC, id DESC; capped 500/slice). */
|
||||
typedBySlice: Map<string, string[]>;
|
||||
/** fallback term -> qitem ids in row (rowid) order, capped 500/term. */
|
||||
fallbackByTerm: Map<string, string[]>;
|
||||
/** location names the index was built from — a get() on a folder created
|
||||
* mid-generation (not in this set) forces a rebuild so fresh folders
|
||||
* resolve membership without waiting for invalidate/TTL, exactly like
|
||||
* the per-slice path did. */
|
||||
knownSlices: Set<string>;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
// --- SQL-LIKE-equivalent matching (qitem-ccf87c0d) -------------------------
|
||||
// SQLite LIKE semantics, replicated byte-for-byte (test-pinned):
|
||||
// - ASCII-ONLY case folding (A-Z <-> a-z; æ vs Æ do NOT fold),
|
||||
// - `%` matches any run and `_` exactly ONE CHARACTER (code point) — both
|
||||
// spanning newlines and astral chars (a surrogate pair is one `_`), so
|
||||
// the translated regexes MUST compile with the `su` flags
|
||||
// (LIKE_REGEXP_FLAGS below): dotAll for newline, `u` for code-point
|
||||
// stepping. A [\s\S] translation would count UTF-16 code units and
|
||||
// miss e.g. `a😀b` against `%a_b%` (guard blocker pin).
|
||||
// - every other byte is literal (regex specials escaped).
|
||||
|
||||
/** The one flag set every translated LIKE regex compiles with. */
|
||||
const LIKE_REGEXP_FLAGS = "su";
|
||||
|
||||
/** Lowercase ASCII A-Z only — never Unicode (SQLite LIKE's fold). */
|
||||
function asciiFold(s: string): string {
|
||||
return s.replace(/[A-Z]/g, (c) => String.fromCharCode(c.charCodeAt(0) + 32));
|
||||
}
|
||||
|
||||
/** Translate one LIKE term (already ascii-folded) to a RegExp source.
|
||||
* Compile the result with LIKE_REGEXP_FLAGS. */
|
||||
function likeTermToRegExpSource(term: string): string {
|
||||
let out = "";
|
||||
for (const ch of term) {
|
||||
if (ch === "%") out += ".*";
|
||||
else if (ch === "_") out += ".";
|
||||
else out += ch.replace(/[.*+?^${}()|[\]\\]/, "\\$&");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const FRONTMATTER_DELIM = "---";
|
||||
const DEFAULT_CACHE_TTL_MS = 60_000;
|
||||
|
||||
@@ -173,6 +226,8 @@ export class SliceIndexer {
|
||||
private readonly cacheTtlMs: number;
|
||||
private listingCache: CachedListing | null = null;
|
||||
private detailCache: Map<string, CachedSlice> = new Map();
|
||||
// qitem-ccf87c0d — batch membership index (see MembershipIndex doc).
|
||||
private membershipIndex: MembershipIndex | null = null;
|
||||
// VM-005: authored mission-status sidecar cache (same TTL as the listing).
|
||||
private missionStatusCache: {
|
||||
statuses: Record<string, { authoredStatus: string | null }>;
|
||||
@@ -197,6 +252,7 @@ export class SliceIndexer {
|
||||
this.listingCache = null;
|
||||
this.detailCache.clear();
|
||||
this.missionStatusCache = null;
|
||||
this.membershipIndex = null;
|
||||
}
|
||||
|
||||
/** VM-005 B1 (the narrow C-vii exception; arch ruling b8d91aee…) — the
|
||||
@@ -524,72 +580,39 @@ export class SliceIndexer {
|
||||
// Substring fallback (including the missionId term) is preserved for
|
||||
// slices whose qitem corpus pre-dates the typed-tag convention so
|
||||
// legacy mission-aware workspaces don't regress.
|
||||
// qitem-ccf87c0d — both tiers now answer from the per-generation batch
|
||||
// membership index (TWO scans per generation instead of O(slices) scans;
|
||||
// see ensureMembershipIndex). TWO-TIER DOCTRINE unchanged: the SIGNAL
|
||||
// tier answers from canonical typed membership ONLY; this DISPLAY tier
|
||||
// carries the gated legacy substring fallback. Never promote a
|
||||
// display-tier match into a signal. (P3)
|
||||
const ids = new Set<string>();
|
||||
try {
|
||||
// 1. Typed-tag matches: tags JSON contains `slice:<sliceName>` literal.
|
||||
// Both the `"slice:..."` JSON-array form and the comma-separated
|
||||
// form (legacy CLI tags) are caught by a wildcard on the slice id.
|
||||
let typedTagMatchCount = 0;
|
||||
try {
|
||||
// TWO-TIER DOCTRINE: the SIGNAL tier (phase / band / attention)
|
||||
// answers from canonical membership ONLY; this DISPLAY tier (the
|
||||
// queue-tab's qitemIds) may carry the gated legacy substring fallback
|
||||
// below. Never promote a display-tier match into a signal. (P3)
|
||||
//
|
||||
// B2 fix: cap on CONFIRMED matches, never the prefilter window. The
|
||||
// unquoted LIKE stays a PREFILTER (catches comma-legacy AND the
|
||||
// suffix/sibling over-matches); parseScopeTags is the authoritative
|
||||
// row-level confirm. We iterate() ORDER BY ts_created DESC, qitem_id
|
||||
// DESC (P2: the id tiebreak makes the confirmed set fully deterministic
|
||||
// under equal timestamps — byte-consistent with the id-lex tail
|
||||
// fallback below, since the id encodes the ts prefix), confirm per
|
||||
// row, and break at 500 CONFIRMED. There is NO pre-confirmation SQL
|
||||
// LIMIT: with one, >500 suffix/sibling over-match rows could hide a
|
||||
// true typed member and un-gate the fallback (VM-004 returns).
|
||||
// Streaming keeps memory bounded, and the leading-wildcard LIKE never
|
||||
// used an index anyway (the old LIMIT saved materialization only,
|
||||
// which iterate+break also saves). typedTagMatchCount counts CONFIRMED
|
||||
// rows only, so the substring gate reflects real typed membership.
|
||||
const stmt = this.db.prepare(
|
||||
`SELECT qitem_id, tags FROM queue_items WHERE tags LIKE ? ORDER BY ts_created DESC, qitem_id DESC`,
|
||||
);
|
||||
for (const r of stmt.iterate(`%slice:${sliceName}%`) as Iterable<{ qitem_id: string; tags: string | null }>) {
|
||||
if (parseScopeTags(r.tags).slices.has(sliceName)) {
|
||||
ids.add(r.qitem_id);
|
||||
typedTagMatchCount++;
|
||||
if (typedTagMatchCount >= 500) break;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// queue_items present but `tags` column missing (older test
|
||||
// harness): fall through to substring path with full term set.
|
||||
let index = this.ensureMembershipIndex();
|
||||
if (!index.knownSlices.has(sliceName)) {
|
||||
// Folder created mid-generation: rebuild so the fresh slice resolves
|
||||
// membership immediately (parity with the old per-slice queries).
|
||||
this.membershipIndex = null;
|
||||
index = this.ensureMembershipIndex();
|
||||
}
|
||||
|
||||
// 2. Substring fallback. Now that typed membership is canonical and
|
||||
// authoritative, the substring tier executes ONLY when zero CONFIRMED
|
||||
// typed-tag rows exist — the deliberate VM-004 behavior change for
|
||||
// typed corpora (it used to ALWAYS run, leaking sibling-name and
|
||||
// body-mention matches into typed slices). Legacy zero-typed corpora
|
||||
// keep today's full term set, including the missionId term (mission-
|
||||
// folder railItem defaults to missionId when no rail item is authored,
|
||||
// so it must be included for legacy mission-aware workspaces). LIMIT
|
||||
// 500 preserved.
|
||||
if (typedTagMatchCount === 0) {
|
||||
// 1. Typed-tag matches (VM-004 authoritative): confirmed rows for this
|
||||
// slice, in ts_created DESC/qitem_id DESC scan order, capped at 500
|
||||
// confirmed per slice by the builder.
|
||||
const typed = index.typedBySlice.get(sliceName);
|
||||
if (typed && typed.length > 0) {
|
||||
for (const id of typed) ids.add(id);
|
||||
} else {
|
||||
// 2. Substring fallback — executes ONLY when zero CONFIRMED typed
|
||||
// rows exist (the deliberate VM-004 gating). Term-first assembly in
|
||||
// the original deduped term order [sliceName, railItem, missionId]
|
||||
// keeps equal-timestamp output byte-identical to the per-slice path
|
||||
// (the final sort below is stable on equals).
|
||||
const substringTerms = [sliceName, railItem, missionId].filter(
|
||||
(v): v is string => !!v,
|
||||
);
|
||||
for (const term of Array.from(new Set(substringTerms))) {
|
||||
try {
|
||||
const rows = this.db.prepare(
|
||||
`SELECT qitem_id FROM queue_items WHERE body LIKE ? OR tags LIKE ? LIMIT 500`,
|
||||
).all(`%${term}%`, `%${term}%`) as Array<{ qitem_id: string }>;
|
||||
for (const r of rows) ids.add(r.qitem_id);
|
||||
} catch {
|
||||
const rows = this.db.prepare(
|
||||
`SELECT qitem_id FROM queue_items WHERE body LIKE ? LIMIT 500`,
|
||||
).all(`%${term}%`) as Array<{ qitem_id: string }>;
|
||||
for (const r of rows) ids.add(r.qitem_id);
|
||||
}
|
||||
const bucket = index.fallbackByTerm.get(term);
|
||||
if (bucket) for (const id of bucket) ids.add(id);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
@@ -627,6 +650,107 @@ export class SliceIndexer {
|
||||
return idsArr;
|
||||
}
|
||||
|
||||
/** qitem-ccf87c0d — build (or reuse) the per-generation membership index.
|
||||
* TWO queue_items scans per generation, replacing O(slices) per-slice
|
||||
* scans:
|
||||
* Scan 1 (typed): `tags LIKE '%slice:%'` prefilter, ORDER BY ts_created
|
||||
* DESC, qitem_id DESC (P2 determinism preserved), parseScopeTags as the
|
||||
* authoritative row-level confirm; a row credits EVERY slice it tags;
|
||||
* cap after 500 CONFIRMED per slice (never a pre-confirmation LIMIT —
|
||||
* the B2/VM-004 rule).
|
||||
* Scan 2 (fallback): one streamed pass over (body, tags) evaluated
|
||||
* against the term universe of all ZERO-TYPED locations. A combined
|
||||
* regex is a PREFILTER only; every term is then confirmed per row so
|
||||
* overlapping terms all credit (no consuming-alternation loss). Terms
|
||||
* use SQL-LIKE-equivalent matching (asciiFold + %/_ translation);
|
||||
* bucket order is row (rowid) order — the same order the un-ORDERed
|
||||
* per-term `LIMIT 500` queries returned; cap 500 per term.
|
||||
* Degradations preserved: tags column missing -> typed tier empty + body-
|
||||
* only fallback scan; queue_items absent -> throws to matchQitems' catch. */
|
||||
private ensureMembershipIndex(): MembershipIndex {
|
||||
const now = Date.now();
|
||||
if (this.membershipIndex && this.membershipIndex.expiresAt > now) {
|
||||
return this.membershipIndex;
|
||||
}
|
||||
|
||||
const typedBySlice = new Map<string, string[]>();
|
||||
const fallbackByTerm = new Map<string, string[]>();
|
||||
const knownSlices = new Set<string>();
|
||||
|
||||
// Term universe: every location's deduped [sliceName, railItem,
|
||||
// missionId], derived with the SAME helpers toListEntry/buildRecord use.
|
||||
const locations = this.readSliceLocations();
|
||||
const termsBySlice = new Map<string, string[]>();
|
||||
for (const location of locations) {
|
||||
knownSlices.add(location.name);
|
||||
const frontmatter = this.readPrimaryFrontmatter(location.slicePath);
|
||||
const railItem = this.extractRailItem(frontmatter, location.missionId);
|
||||
const terms = Array.from(new Set(
|
||||
[location.name, railItem, location.missionId].filter((v): v is string => !!v),
|
||||
));
|
||||
termsBySlice.set(location.name, terms);
|
||||
}
|
||||
|
||||
// Scan 1 — typed membership. A thrown error here (tags column missing)
|
||||
// leaves the typed map empty; a missing queue_items table throws from
|
||||
// scan 2's body-only retry too, surfacing to matchQitems' catch.
|
||||
let tagsColumnAvailable = true;
|
||||
try {
|
||||
const stmt = this.db.prepare(
|
||||
`SELECT qitem_id, tags FROM queue_items WHERE tags LIKE '%slice:%' ORDER BY ts_created DESC, qitem_id DESC`,
|
||||
);
|
||||
for (const r of stmt.iterate() as Iterable<{ qitem_id: string; tags: string | null }>) {
|
||||
for (const tagged of parseScopeTags(r.tags).slices) {
|
||||
const bucket = typedBySlice.get(tagged);
|
||||
if (!bucket) typedBySlice.set(tagged, [r.qitem_id]);
|
||||
else if (bucket.length < 500) bucket.push(r.qitem_id);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
tagsColumnAvailable = false;
|
||||
}
|
||||
|
||||
// Scan 2 — fallback buckets for the zero-typed slices' term universe.
|
||||
const fallbackTerms = new Set<string>();
|
||||
for (const [slice, terms] of termsBySlice) {
|
||||
const typed = typedBySlice.get(slice);
|
||||
if (typed && typed.length > 0) continue;
|
||||
for (const term of terms) fallbackTerms.add(term);
|
||||
}
|
||||
if (fallbackTerms.size > 0) {
|
||||
try {
|
||||
const termList = Array.from(fallbackTerms);
|
||||
const matchers = termList.map((term) => ({
|
||||
term,
|
||||
re: new RegExp(likeTermToRegExpSource(asciiFold(term)), LIKE_REGEXP_FLAGS),
|
||||
}));
|
||||
const prefilter = new RegExp(matchers.map((m) => m.re.source).join("|"), LIKE_REGEXP_FLAGS);
|
||||
for (const term of termList) fallbackByTerm.set(term, []);
|
||||
const scan = tagsColumnAvailable
|
||||
? this.db.prepare(`SELECT qitem_id, body, tags FROM queue_items`)
|
||||
: this.db.prepare(`SELECT qitem_id, body FROM queue_items`);
|
||||
for (const r of scan.iterate() as Iterable<{ qitem_id: string; body: string; tags?: string | null }>) {
|
||||
const hayBody = asciiFold(r.body);
|
||||
const hayTags = r.tags != null ? asciiFold(r.tags) : null;
|
||||
if (!prefilter.test(hayBody) && !(hayTags !== null && prefilter.test(hayTags))) continue;
|
||||
for (const m of matchers) {
|
||||
if (m.re.test(hayBody) || (hayTags !== null && m.re.test(hayTags))) {
|
||||
const bucket = fallbackByTerm.get(m.term)!;
|
||||
if (bucket.length < 500) bucket.push(r.qitem_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// queue_items table absent entirely: empty buckets — matchQitems
|
||||
// yields [] for every slice, byte-matching the old catch path,
|
||||
// while the built index still caches (no per-call rebuild churn).
|
||||
}
|
||||
}
|
||||
|
||||
this.membershipIndex = { typedBySlice, fallbackByTerm, knownSlices, expiresAt: now + this.cacheTtlMs };
|
||||
return this.membershipIndex;
|
||||
}
|
||||
|
||||
private findProofPacket(sliceName: string): SliceProofPacket | null {
|
||||
if (!this.dogfoodEvidenceRoot) return null;
|
||||
let entries: fs.Dirent[];
|
||||
|
||||
@@ -574,3 +574,293 @@ describe("PL-slice-story-view-v0 SliceIndexer", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// qitem-ccf87c0d — Project mission-index LOAD CONTRACT (guard-amended RED).
|
||||
// Forensic root cause: matchQitems runs O(slices) full-table queue_items LIKE
|
||||
// scans per cold rebuild (tier-1 wildcard+ORDER BY always; up to 3 more
|
||||
// body-LIKE scans per zero-typed slice) — 353 slices at host scale = ~10s
|
||||
// cold, synchronously blocking the daemon event loop. The contract below
|
||||
// counts EXECUTIONS of queue_items LIKE statements (not prepare calls):
|
||||
// a cold rebuild must run a CONSTANT number of scan-shaped statements,
|
||||
// independent of slice count. Behavior pins ride alongside so the batch
|
||||
// rewrite cannot drift membership semantics.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("qitem-ccf87c0d — mission-index load contract + membership pins", () => {
|
||||
let db: Database.Database;
|
||||
let cleanup: string;
|
||||
let missionsRoot: string;
|
||||
|
||||
beforeEach(() => {
|
||||
db = createDb();
|
||||
migrate(db, [coreSchema, eventsSchema, streamItemsSchema, queueItemsSchema]);
|
||||
db.prepare(`INSERT INTO rigs (id, name) VALUES ('r-1', 'rig')`).run();
|
||||
cleanup = fs.mkdtempSync(path.join(os.tmpdir(), "slice-indexer-load-"));
|
||||
missionsRoot = path.join(cleanup, "missions");
|
||||
fs.mkdirSync(missionsRoot, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(cleanup, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** Count EXECUTIONS (.all/.iterate/.get calls) of statements that both
|
||||
* name queue_items and carry a LIKE — the scan-shaped statements. run()
|
||||
* passes through uncounted (INSERT seeding). Non-LIKE statements are
|
||||
* returned unwrapped. */
|
||||
function instrumentLikeExecutions(target: Database.Database): () => number {
|
||||
let n = 0;
|
||||
const origPrepare = target.prepare.bind(target);
|
||||
(target as unknown as { prepare: (sql: string) => unknown }).prepare = (sql: string) => {
|
||||
const stmt = origPrepare(sql) as unknown as Record<string, (...a: unknown[]) => unknown>;
|
||||
if (!/queue_items/i.test(sql) || !/\bLIKE\b/i.test(sql)) return stmt;
|
||||
return {
|
||||
all: (...a: unknown[]) => { n++; return stmt.all!(...a); },
|
||||
iterate: (...a: unknown[]) => { n++; return stmt.iterate!(...a); },
|
||||
get: (...a: unknown[]) => { n++; return stmt.get!(...a); },
|
||||
run: (...a: unknown[]) => stmt.run!(...a),
|
||||
};
|
||||
};
|
||||
return () => n;
|
||||
}
|
||||
|
||||
/** Exact seed: 2 missions x 20 slices = 40 slices; the first 12 (30%) are
|
||||
* typed (one confirmed `slice:` row each); 60 qitems total (12 typed rows
|
||||
* + 48 zero-typed rows with 200-byte bodies). */
|
||||
function seedFixture(): { typedSlices: string[]; untypedSlices: string[] } {
|
||||
const typedSlices: string[] = [];
|
||||
const untypedSlices: string[] = [];
|
||||
const filler = "b".repeat(200);
|
||||
let idx = 0;
|
||||
for (let m = 0; m < 2; m++) {
|
||||
const mission = `load-mission-${m}`;
|
||||
for (let s = 0; s < 20; s++, idx++) {
|
||||
const slice = `ld-${String(idx).padStart(2, "0")}-topic`;
|
||||
writeSlice(path.join(missionsRoot, mission, "slices"), slice, {
|
||||
"README.md": `---\nstatus: active\n---\n# ${slice}\n`,
|
||||
});
|
||||
if (idx < 12) typedSlices.push(slice);
|
||||
else untypedSlices.push(slice);
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < 12; i++) {
|
||||
insertQitem(db, {
|
||||
qitemId: `q-typed-${String(i).padStart(2, "0")}`,
|
||||
body: `typed packet ${i} ${filler}`,
|
||||
tags: [`slice:${typedSlices[i]!}`],
|
||||
tsCreated: `2026-07-01T00:${String(i).padStart(2, "0")}:00.000Z`,
|
||||
});
|
||||
}
|
||||
for (let i = 0; i < 48; i++) {
|
||||
insertQitem(db, {
|
||||
qitemId: `q-plain-${String(i).padStart(2, "0")}`,
|
||||
body: `generic packet ${i} ${filler}`,
|
||||
tags: ["release:0.4.7"],
|
||||
tsCreated: `2026-07-02T00:${String(i % 60).padStart(2, "0")}:00.000Z`,
|
||||
});
|
||||
}
|
||||
return { typedSlices, untypedSlices };
|
||||
}
|
||||
|
||||
it("LOAD CONTRACT (RED): one cold list() over 40 slices executes a CONSTANT number of queue_items LIKE statements (<= 4), independent of slice count", () => {
|
||||
const { typedSlices } = seedFixture();
|
||||
const likeCount = instrumentLikeExecutions(db);
|
||||
const indexer = new SliceIndexer({ slicesRoot: missionsRoot, dogfoodEvidenceRoot: null, db });
|
||||
const entries = indexer.list();
|
||||
expect(entries).toHaveLength(40);
|
||||
// Membership sanity inside the same rebuild: a typed slice carries its row.
|
||||
const typed0 = entries.find((e) => e.name === typedSlices[0])!;
|
||||
expect(typed0.qitemCount).toBe(1);
|
||||
// The contract: constant scan count. Pre-fix this executes
|
||||
// 40 tier-1 LIKE iterations + 2 dedup'd fallback LIKEs for each of the
|
||||
// 28 zero-typed slices (= 96 total). Post-fix: bounded constant.
|
||||
expect(likeCount()).toBeLessThanOrEqual(4);
|
||||
});
|
||||
|
||||
it("PIN: invalidate() picks up a fresh slice folder AND its typed membership (Explorer auto-show read-after-write)", () => {
|
||||
seedFixture();
|
||||
const indexer = new SliceIndexer({ slicesRoot: missionsRoot, dogfoodEvidenceRoot: null, db });
|
||||
expect(indexer.list()).toHaveLength(40);
|
||||
writeSlice(path.join(missionsRoot, "load-mission-0", "slices"), "ld-fresh-folder", {
|
||||
"README.md": "---\nstatus: active\n---\n# Fresh\n",
|
||||
});
|
||||
insertQitem(db, {
|
||||
qitemId: "q-fresh-typed",
|
||||
body: "fresh dispatch",
|
||||
tags: ["slice:ld-fresh-folder"],
|
||||
});
|
||||
indexer.invalidate();
|
||||
const after = indexer.list();
|
||||
expect(after).toHaveLength(41);
|
||||
expect(after.find((e) => e.name === "ld-fresh-folder")!.qitemCount).toBe(1);
|
||||
expect(indexer.get("ld-fresh-folder")!.qitemIds).toEqual(["q-fresh-typed"]);
|
||||
});
|
||||
|
||||
it("PIN: cold direct get() BEFORE any list() resolves correct typed membership", () => {
|
||||
const { typedSlices } = seedFixture();
|
||||
const indexer = new SliceIndexer({ slicesRoot: missionsRoot, dogfoodEvidenceRoot: null, db });
|
||||
// No list() call first — the detail path must build whatever index it
|
||||
// needs lazily.
|
||||
const record = indexer.get(typedSlices[3]!)!;
|
||||
expect(record.qitemIds).toEqual(["q-typed-03"]);
|
||||
});
|
||||
|
||||
it("PIN: TTL expiry refreshes membership WITHOUT an explicit invalidate", async () => {
|
||||
const { typedSlices } = seedFixture();
|
||||
const indexer = new SliceIndexer({ slicesRoot: missionsRoot, dogfoodEvidenceRoot: null, db, cacheTtlMs: 40 });
|
||||
expect(indexer.get(typedSlices[0]!)!.qitemIds).toEqual(["q-typed-00"]);
|
||||
insertQitem(db, {
|
||||
qitemId: "q-typed-00-later",
|
||||
body: "late dispatch",
|
||||
tags: [`slice:${typedSlices[0]!}`],
|
||||
tsCreated: "2026-07-03T00:00:00.000Z",
|
||||
});
|
||||
await new Promise((r) => setTimeout(r, 60));
|
||||
expect(indexer.get(typedSlices[0]!)!.qitemIds).toEqual(["q-typed-00-later", "q-typed-00"]);
|
||||
});
|
||||
|
||||
it("PIN: overlapping zero-typed fallback terms credit ONE row to EVERY matching slice (no consuming-alternation loss)", () => {
|
||||
// Flat root: railItem/missionId are null, so each slice's only fallback
|
||||
// term is its name. The two names OVERLAP inside one body string —
|
||||
// "alpha-beta-gamma" contains both "alpha-beta" and "beta-gamma" with a
|
||||
// shared "beta" segment. A naive single consuming alternation would
|
||||
// credit only the first.
|
||||
const flatRoot = path.join(cleanup, "flat-slices");
|
||||
writeSlice(flatRoot, "alpha-beta", { "README.md": "---\nstatus: active\n---\n" });
|
||||
writeSlice(flatRoot, "beta-gamma", { "README.md": "---\nstatus: active\n---\n" });
|
||||
insertQitem(db, { qitemId: "q-overlap", body: "work on alpha-beta-gamma today", tags: [] });
|
||||
const indexer = new SliceIndexer({ slicesRoot: flatRoot, dogfoodEvidenceRoot: null, db });
|
||||
expect(indexer.get("alpha-beta")!.qitemIds).toEqual(["q-overlap"]);
|
||||
expect(indexer.get("beta-gamma")!.qitemIds).toEqual(["q-overlap"]);
|
||||
});
|
||||
|
||||
it("PIN: SQL-LIKE ASCII case-insensitivity is preserved in the fallback tier", () => {
|
||||
const flatRoot = path.join(cleanup, "flat-ci");
|
||||
writeSlice(flatRoot, "case-probe", { "README.md": "---\nstatus: active\n---\n" });
|
||||
insertQitem(db, { qitemId: "q-upper", body: "Ref: CASE-PROBE follow-up", tags: [] });
|
||||
const indexer = new SliceIndexer({ slicesRoot: flatRoot, dogfoodEvidenceRoot: null, db });
|
||||
expect(indexer.get("case-probe")!.qitemIds).toEqual(["q-upper"]);
|
||||
});
|
||||
|
||||
it("PIN: SQL-LIKE wildcard semantics preserved — an underscore in a slice name matches any single body character (current behavior, byte-for-byte)", () => {
|
||||
// Fallback terms come from folder names / frontmatter, which CAN carry
|
||||
// `_` (and in principle `%`). Today those bytes are LIKE wildcards:
|
||||
// body LIKE '%under_score-slice%' matches "underXscore-slice". The
|
||||
// batch rewrite must translate, not literalize, these bytes.
|
||||
const flatRoot = path.join(cleanup, "flat-wildcard");
|
||||
writeSlice(flatRoot, "under_score-slice", { "README.md": "---\nstatus: active\n---\n" });
|
||||
insertQitem(db, { qitemId: "q-wild-x", body: "see underXscore-slice notes", tags: [] });
|
||||
insertQitem(db, { qitemId: "q-wild-lit", body: "see under_score-slice notes", tags: [] });
|
||||
const indexer = new SliceIndexer({ slicesRoot: flatRoot, dogfoodEvidenceRoot: null, db });
|
||||
expect(indexer.get("under_score-slice")!.qitemIds.sort()).toEqual(["q-wild-lit", "q-wild-x"]);
|
||||
});
|
||||
|
||||
it("PIN: tags-column-missing degradation — fallback still matches by body (tier-1 failure is caught)", () => {
|
||||
const bareDb = createDb();
|
||||
migrate(bareDb, [coreSchema]);
|
||||
bareDb.exec(`CREATE TABLE queue_items (
|
||||
qitem_id TEXT PRIMARY KEY, ts_created TEXT NOT NULL, ts_updated TEXT NOT NULL,
|
||||
source_session TEXT NOT NULL, destination_session TEXT NOT NULL, state TEXT NOT NULL,
|
||||
priority TEXT NOT NULL DEFAULT 'routine', body TEXT NOT NULL)`);
|
||||
bareDb.prepare(
|
||||
`INSERT INTO queue_items (qitem_id, ts_created, ts_updated, source_session, destination_session, state, body)
|
||||
VALUES ('q-no-tags-col', '2026-07-01T00:00:00.000Z', '2026-07-01T00:00:00.000Z', 'a@r', 'b@r', 'done', 'notagcol-slice work')`,
|
||||
).run();
|
||||
const flatRoot = path.join(cleanup, "flat-notags");
|
||||
writeSlice(flatRoot, "notagcol-slice", { "README.md": "---\nstatus: active\n---\n" });
|
||||
const indexer = new SliceIndexer({ slicesRoot: flatRoot, dogfoodEvidenceRoot: null, db: bareDb });
|
||||
expect(indexer.get("notagcol-slice")!.qitemIds).toEqual(["q-no-tags-col"]);
|
||||
bareDb.close();
|
||||
});
|
||||
});
|
||||
|
||||
// qitem-ccf87c0d guard RED-verdict delta — two byte-equivalence blockers
|
||||
// pinned against the CURRENT engine before any batch matcher lands.
|
||||
describe("qitem-ccf87c0d — LIKE byte-equivalence + fallback-order pins (guard delta)", () => {
|
||||
let db: Database.Database;
|
||||
let cleanup: string;
|
||||
|
||||
beforeEach(() => {
|
||||
db = createDb();
|
||||
migrate(db, [coreSchema, eventsSchema, streamItemsSchema, queueItemsSchema]);
|
||||
db.prepare(`INSERT INTO rigs (id, name) VALUES ('r-1', 'rig')`).run();
|
||||
cleanup = fs.mkdtempSync(path.join(os.tmpdir(), "slice-indexer-eqv-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(cleanup, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("PIN: SQLite LIKE folds ASCII ONLY — non-ASCII case variants (æ vs Æ) do NOT match; ASCII letters still fold", () => {
|
||||
const flatRoot = path.join(cleanup, "flat-ascii-fold");
|
||||
writeSlice(flatRoot, "graey-æ-slice", { "README.md": "---\nstatus: active\n---\n" });
|
||||
// Æ (U+00C6) is the Unicode uppercase of æ (U+00E6): a JS /i regex would
|
||||
// match it; SQLite LIKE must NOT (ASCII-only fold).
|
||||
insertQitem(db, { qitemId: "q-unicode-upper", body: "ref GRAEY-Æ-SLICE here", tags: [] });
|
||||
// Same æ byte, ASCII letters case-varied: LIKE matches.
|
||||
insertQitem(db, { qitemId: "q-ascii-fold", body: "ref GRAEY-æ-SLICE here", tags: [] });
|
||||
const indexer = new SliceIndexer({ slicesRoot: flatRoot, dogfoodEvidenceRoot: null, db });
|
||||
expect(indexer.get("graey-æ-slice")!.qitemIds).toEqual(["q-ascii-fold"]);
|
||||
});
|
||||
|
||||
it("PIN: LIKE wildcards match across NEWLINE — an underscore in a slice name matches '\\n' in the body (dotAll-equivalent behavior required)", () => {
|
||||
const flatRoot = path.join(cleanup, "flat-nl-wild");
|
||||
writeSlice(flatRoot, "nl_probe", { "README.md": "---\nstatus: active\n---\n" });
|
||||
insertQitem(db, { qitemId: "q-newline-wild", body: "prefix nl\nprobe suffix", tags: [] });
|
||||
const indexer = new SliceIndexer({ slicesRoot: flatRoot, dogfoodEvidenceRoot: null, db });
|
||||
expect(indexer.get("nl_probe")!.qitemIds).toEqual(["q-newline-wild"]);
|
||||
});
|
||||
|
||||
it("PIN: equal-ts fallback rows keep TERM-FIRST order ([sliceName, railItem]), not DB row order", () => {
|
||||
// Row matching ONLY the railItem term is inserted FIRST in the DB; the
|
||||
// row matching ONLY the sliceName term second; identical ts_created.
|
||||
// Current engine: terms iterate [sliceName, railItem], ids enter the Set
|
||||
// in term order, and the final ts-DESC sort is stable on equals — so the
|
||||
// sliceName-matched row renders first. A row-first batch scan would
|
||||
// emit DB order and flip them.
|
||||
const flatRoot = path.join(cleanup, "flat-term-order");
|
||||
writeSlice(flatRoot, "term-order-slice", {
|
||||
"README.md": "---\nstatus: active\nrail-item: TORD-RAIL\n---\n",
|
||||
});
|
||||
const ts = "2026-07-05T12:00:00.000Z";
|
||||
insertQitem(db, { qitemId: "q-a-railitem", body: "TORD-RAIL work", tags: [], tsCreated: ts });
|
||||
insertQitem(db, { qitemId: "q-b-slicename", body: "term-order-slice work", tags: [], tsCreated: ts });
|
||||
const indexer = new SliceIndexer({ slicesRoot: flatRoot, dogfoodEvidenceRoot: null, db });
|
||||
expect(indexer.get("term-order-slice")!.qitemIds).toEqual(["q-b-slicename", "q-a-railitem"]);
|
||||
});
|
||||
});
|
||||
|
||||
// qitem-ccf87c0d guard POST-EDIT blocker — code-point parity for LIKE '_'.
|
||||
describe("qitem-ccf87c0d — LIKE '_' Unicode code-point parity (guard blocker pin)", () => {
|
||||
let db: Database.Database;
|
||||
let cleanup: string;
|
||||
|
||||
beforeEach(() => {
|
||||
db = createDb();
|
||||
migrate(db, [coreSchema, eventsSchema, streamItemsSchema, queueItemsSchema]);
|
||||
db.prepare(`INSERT INTO rigs (id, name) VALUES ('r-1', 'rig')`).run();
|
||||
cleanup = fs.mkdtempSync(path.join(os.tmpdir(), "slice-indexer-cp-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(cleanup, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("PIN: '_' matches ONE CHARACTER (code point) — an astral emoji (surrogate pair in UTF-16) satisfies a single '_', exactly like SQLite", () => {
|
||||
// SQLite LIKE '%emoji_probe%' matches body 'emoji😀probe' — 😀 (U+1F600)
|
||||
// is one character. A UTF-16 code-unit matcher ([\s\S]) consumes only
|
||||
// half the surrogate pair and misses; the translation must be
|
||||
// code-point-correct (dotAll + u).
|
||||
const flatRoot = path.join(cleanup, "flat-cp");
|
||||
writeSlice(flatRoot, "emoji_probe", { "README.md": "---\nstatus: active\n---\n" });
|
||||
insertQitem(db, { qitemId: "q-astral", body: "see emoji\u{1F600}probe notes", tags: [] });
|
||||
// Two-code-unit ASCII sequence must still NOT match a single '_':
|
||||
insertQitem(db, { qitemId: "q-two-chars", body: "see emojiXYprobe notes", tags: [] });
|
||||
const indexer = new SliceIndexer({ slicesRoot: flatRoot, dogfoodEvidenceRoot: null, db });
|
||||
expect(indexer.get("emoji_probe")!.qitemIds).toEqual(["q-astral"]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user