mirror of
https://github.com/anomalyco/opencode.git
synced 2026-08-13 23:09:50 -04:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1c583dae3c |
@@ -1038,6 +1038,7 @@
|
||||
"minimatch": "10.2.5",
|
||||
"npm-package-arg": "13.0.2",
|
||||
"resolve.exports": "catalog:",
|
||||
"xdg-basedir": "5.1.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tsconfig/bun": "catalog:",
|
||||
@@ -1103,6 +1104,7 @@
|
||||
"unenv": "2.0.0-rc.24",
|
||||
"vitest": "3.2.7",
|
||||
"wrangler": "4.28.0",
|
||||
"xdg-basedir": "5.1.0",
|
||||
},
|
||||
},
|
||||
"packages/www": {
|
||||
@@ -5997,6 +5999,8 @@
|
||||
|
||||
"xdg-app-paths": ["xdg-app-paths@5.5.1", "", { "dependencies": { "os-paths": "^4.0.1", "xdg-portable": "^7.2.0" } }, "sha512-hI3flOB4PLZIy5prbtTpirobtPE2ZtZ52szO+2mM9Efp6ErM398La+C1lIpNWDfNoQk+6Lsi6nMcCwVB7pxeMQ=="],
|
||||
|
||||
"xdg-basedir": ["xdg-basedir@5.1.0", "", {}, "sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ=="],
|
||||
|
||||
"xdg-portable": ["xdg-portable@7.3.0", "", { "dependencies": { "os-paths": "^4.0.1" } }, "sha512-sqMMuL1rc0FmMBOzCpd0yuy9trqF2yTTVe+E9ogwCSWQCdDEtQUwrZPT6AxqtsFGRNxycgncbP/xmOOSPw5ZUw=="],
|
||||
|
||||
"xml-naming": ["xml-naming@0.3.0", "", {}, "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ=="],
|
||||
|
||||
@@ -480,23 +480,23 @@ describe("server session", () => {
|
||||
|
||||
test("projects committed revert before server reconciliation", () => {
|
||||
const ctx = setup({ child: session("child") })
|
||||
ctx.store.remember({ ...session("child"), revert: { messageID: "msg_2", partID: "prt_1" } })
|
||||
ctx.store.set("input", "child", ["msg_1", "msg_2"])
|
||||
ctx.store.remember({ ...session("child"), revert: { messageID: "msg_000", partID: "prt_1" } })
|
||||
ctx.store.set("input", "child", ["msg_fff", "msg_000"])
|
||||
ctx.store.set("session_message", "child", [
|
||||
{ id: "msg_1", type: "user", text: "keep", time: { created: 1 } },
|
||||
{ id: "msg_2", type: "user", text: "remove", time: { created: 2 } },
|
||||
{ id: "msg_fff", type: "user", text: "keep", time: { created: 1 } },
|
||||
{ id: "msg_000", type: "user", text: "remove", time: { created: 2 } },
|
||||
])
|
||||
|
||||
ctx.store.applyV2({
|
||||
id: "evt_revert",
|
||||
created: 3,
|
||||
type: "session.revert.committed",
|
||||
data: { sessionID: "child", to: "msg_2" },
|
||||
data: { sessionID: "child", to: "msg_000" },
|
||||
} as OpenCodeEvent)
|
||||
|
||||
expect(ctx.store.data.info.child?.revert).toBeUndefined()
|
||||
expect(ctx.store.data.input.child).toEqual(["msg_1"])
|
||||
expect(ctx.store.data.session_message.child?.map((message) => message.id)).toEqual(["msg_1"])
|
||||
expect(ctx.store.data.input.child).toEqual(["msg_fff"])
|
||||
expect(ctx.store.data.session_message.child?.map((message) => message.id)).toEqual(["msg_fff"])
|
||||
})
|
||||
|
||||
test("does not restore a message hydrated before a committed revert", async () => {
|
||||
|
||||
@@ -1041,13 +1041,17 @@ export function createServerSession(
|
||||
if (event.type === "session.revert.committed") {
|
||||
messageHydrationRevision.set(sessionID, (messageHydrationRevision.get(sessionID) ?? 0) + 1)
|
||||
if (info) remember({ ...info, revert: undefined })
|
||||
setData("input", sessionID, (items) => items?.filter((id) => id < event.data.to))
|
||||
setData("input", sessionID, (items) => {
|
||||
const boundary = items?.findIndex((id) => id === event.data.to) ?? -1
|
||||
return boundary < 0 ? items : items?.slice(0, boundary)
|
||||
})
|
||||
const source = data.session_message[sessionID] ?? []
|
||||
const removed = source.filter((message) => message.id >= event.data.to).map((message) => message.id)
|
||||
const boundary = source.findIndex((message) => message.id === event.data.to)
|
||||
const removed = boundary < 0 ? [] : source.slice(boundary).map((message) => message.id)
|
||||
removedMessages.set(sessionID, new Set([...(removedMessages.get(sessionID) ?? []), ...removed]))
|
||||
projectV2({
|
||||
sessionID,
|
||||
messages: source.filter((message) => message.id < event.data.to),
|
||||
messages: boundary < 0 ? source : source.slice(0, boundary),
|
||||
touched: [],
|
||||
removed,
|
||||
})
|
||||
|
||||
@@ -239,5 +239,58 @@ export class SQLiteEffectDatabase<
|
||||
) => Effect.Effect<A, E | SqlError, R> = (tx, config) => this.session.transaction(tx, config)
|
||||
}
|
||||
|
||||
export type SQLiteEffectWithReplicas<Q> = Q & { $primary: Q; $replicas: Q[] }
|
||||
|
||||
export const withReplicas = <
|
||||
TEffectHKT extends QueryEffectHKTBase,
|
||||
TRunResult,
|
||||
TRelations extends AnyRelations,
|
||||
Q extends SQLiteEffectDatabase<TEffectHKT, TRunResult, TRelations>,
|
||||
>(
|
||||
primary: Q,
|
||||
replicas: [Q, ...Q[]],
|
||||
getReplica: (replicas: Q[]) => Q = () => replicas[Math.floor(Math.random() * replicas.length)]!,
|
||||
): SQLiteEffectWithReplicas<Q> => {
|
||||
const select: Q["select"] = (...args: []) => getReplica(replicas).select(...args)
|
||||
const selectDistinct: Q["selectDistinct"] = (...args: []) => getReplica(replicas).selectDistinct(...args)
|
||||
const $count: Q["$count"] = (...args: [any]) => getReplica(replicas).$count(...args)
|
||||
const _with: Q["with"] = (...args: []) => getReplica(replicas).with(...args)
|
||||
const $with = ((...args: [string] | [string, ColumnsSelection]) =>
|
||||
args.length === 1
|
||||
? getReplica(replicas).$with(args[0])
|
||||
: getReplica(replicas).$with(args[0], args[1])) as Q["$with"]
|
||||
|
||||
const update: Q["update"] = (...args: [any]) => primary.update(...args)
|
||||
const insert: Q["insert"] = (...args: [any]) => primary.insert(...args)
|
||||
const $delete: Q["delete"] = (...args: [any]) => primary.delete(...args)
|
||||
const run: Q["run"] = (...args: [any]) => primary.run(...args)
|
||||
const all: Q["all"] = (...args: [any]) => primary.all(...args)
|
||||
const get: Q["get"] = (...args: [any]) => primary.get(...args)
|
||||
const values: Q["values"] = (...args: [any]) => primary.values(...args)
|
||||
const transaction: Q["transaction"] = (...args: [any]) => primary.transaction(...args)
|
||||
|
||||
return {
|
||||
...primary,
|
||||
update,
|
||||
insert,
|
||||
delete: $delete,
|
||||
run,
|
||||
all,
|
||||
get,
|
||||
values,
|
||||
transaction,
|
||||
$primary: primary,
|
||||
$replicas: replicas,
|
||||
select,
|
||||
selectDistinct,
|
||||
$count,
|
||||
$with,
|
||||
with: _with,
|
||||
get query() {
|
||||
return getReplica(replicas).query
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export type AnySQLiteEffectDatabase = SQLiteEffectDatabase<any, any, any>
|
||||
export type AnySQLiteEffectSelectBase = SQLiteEffectSelectBase<any, any, any, any, any, any, any, any, any, any>
|
||||
|
||||
@@ -1,10 +1,22 @@
|
||||
/* oxlint-disable */
|
||||
import type { TablesRelationalConfig } from "drizzle-orm/_relations"
|
||||
import type { MigrationMeta } from "drizzle-orm/migrator"
|
||||
import type { AnyRelations } from "drizzle-orm/relations"
|
||||
import { type SQL, sql } from "drizzle-orm/sql/sql"
|
||||
import type { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core"
|
||||
import type { SQLiteSession } from "drizzle-orm/sqlite-core/session"
|
||||
import { GET_VERSION_FOR, MIGRATIONS_TABLE_VERSIONS, type UpgradeResult } from "./utils.js"
|
||||
|
||||
/** @internal */
|
||||
export type SQLiteMigrationTableRow = { id: number | null; hash: string; created_at: number }
|
||||
|
||||
type AsyncSQLiteDatabaseWithSession = BaseSQLiteDatabase<"async", unknown, Record<string, unknown>> & {
|
||||
session: {
|
||||
all<T>(query: SQL): Promise<T[]>
|
||||
}
|
||||
transaction<T>(transaction: (tx: { run(query: SQL): Promise<unknown> }) => Promise<T>): Promise<T>
|
||||
}
|
||||
|
||||
type SQLiteMigrationBackfillEntry = {
|
||||
name: string
|
||||
selector:
|
||||
@@ -103,3 +115,139 @@ export function buildSQLiteMigrationBackfillStatements(
|
||||
|
||||
return statements
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects the current version of the migrations table schema and upgrades it if needed.
|
||||
*
|
||||
* Version 0: Original schema (id, hash, created_at)
|
||||
* Version 1: Extended schema (id, hash, created_at, name, applied_at)
|
||||
*/
|
||||
export function upgradeSyncIfNeeded(
|
||||
migrationsTable: string,
|
||||
session: SQLiteSession<"sync", unknown, Record<string, unknown>, AnyRelations, TablesRelationalConfig>,
|
||||
localMigrations: MigrationMeta[],
|
||||
): UpgradeResult {
|
||||
const tableExists = session.all(sql`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ${migrationsTable}`)
|
||||
|
||||
if (tableExists.length === 0) {
|
||||
return { newDb: true }
|
||||
}
|
||||
|
||||
// Table exists, check table shape
|
||||
const rows = session.all<{ column_name: string }>(
|
||||
sql`SELECT name as column_name FROM pragma_table_info(${migrationsTable})`,
|
||||
)
|
||||
|
||||
const version = GET_VERSION_FOR.sqlite(rows.map((r) => r.column_name))
|
||||
|
||||
for (let v = version; v < MIGRATIONS_TABLE_VERSIONS.sqlite; v++) {
|
||||
const upgradeFn = upgradeSyncFunctions[v]
|
||||
if (!upgradeFn) {
|
||||
throw new Error(`No upgrade path from migration table version ${v} to ${v + 1}`)
|
||||
}
|
||||
upgradeFn(migrationsTable, session, localMigrations)
|
||||
}
|
||||
|
||||
return { newDb: false }
|
||||
}
|
||||
|
||||
const upgradeSyncFunctions: Record<
|
||||
number,
|
||||
(
|
||||
migrationsTable: string,
|
||||
session: SQLiteSession<"sync", unknown, Record<string, unknown>, AnyRelations, TablesRelationalConfig>,
|
||||
localMigrations: MigrationMeta[],
|
||||
) => void
|
||||
> = {
|
||||
/**
|
||||
* Upgrade from version 0 to version 1:
|
||||
* 1. Read all existing DB migrations
|
||||
* 2. Sort localMigrations ASC by millis and if the same - sort by name
|
||||
* 3. Match each DB row to a local migration
|
||||
* If multiple migrations share the same second, use hash matching as a tiebreaker
|
||||
* Not implemented for now -> If hash matching fails, fall back to serial id ordering
|
||||
* 5. Create extra column and backfill names for matched migrations
|
||||
*/
|
||||
0: (migrationsTable, session, localMigrations) => {
|
||||
const table = sql`${sql.identifier(migrationsTable)}`
|
||||
const dbRows = session.all<SQLiteMigrationTableRow>(sql`SELECT id, hash, created_at FROM ${table} ORDER BY id ASC`)
|
||||
const statements = buildSQLiteMigrationBackfillStatements(
|
||||
migrationsTable,
|
||||
prepareSQLiteMigrationBackfill(dbRows, localMigrations),
|
||||
)
|
||||
|
||||
session.transaction((tx) => {
|
||||
for (const statement of statements) {
|
||||
tx.run(statement)
|
||||
}
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects the current version of the migrations table schema and upgrades it if needed.
|
||||
*
|
||||
* Version 0: Original schema (id, hash, created_at)
|
||||
* Version 1: Extended schema (id, hash, created_at, name, applied_at)
|
||||
*/
|
||||
export async function upgradeAsyncIfNeeded(
|
||||
migrationsTable: string,
|
||||
db: AsyncSQLiteDatabaseWithSession,
|
||||
localMigrations: MigrationMeta[],
|
||||
): Promise<UpgradeResult> {
|
||||
// Check if the table exists at all
|
||||
const tableExists = await db.session.all(
|
||||
sql`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ${migrationsTable}`,
|
||||
)
|
||||
|
||||
if (tableExists.length === 0) {
|
||||
return { newDb: true }
|
||||
}
|
||||
|
||||
const rows = await db.session.all<{ column_name: string }>(
|
||||
sql`SELECT name as column_name FROM pragma_table_info(${migrationsTable})`,
|
||||
)
|
||||
|
||||
const version = GET_VERSION_FOR.sqlite(rows.map((r) => r.column_name))
|
||||
|
||||
for (let v = version; v < MIGRATIONS_TABLE_VERSIONS.sqlite; v++) {
|
||||
const upgradeFn = upgradeAsyncFunctions[v]
|
||||
if (!upgradeFn) {
|
||||
throw new Error(`No upgrade path from migration table version ${v} to ${v + 1}`)
|
||||
}
|
||||
await upgradeFn(migrationsTable, db, localMigrations)
|
||||
}
|
||||
|
||||
return { newDb: false }
|
||||
}
|
||||
|
||||
const upgradeAsyncFunctions: Record<
|
||||
number,
|
||||
(migrationsTable: string, db: AsyncSQLiteDatabaseWithSession, localMigrations: MigrationMeta[]) => Promise<void>
|
||||
> = {
|
||||
/**
|
||||
* Upgrade from version 0 to version 1:
|
||||
* 1. Read all existing DB migrations
|
||||
* 2. Sort localMigrations ASC by millis and if the same - sort by name
|
||||
* 3. Match each DB row to a local migration
|
||||
* If multiple migrations share the same second, use hash matching as a tiebreaker
|
||||
* Not implemented for now -> If hash matching fails, fall back to serial id ordering
|
||||
* 5. Create extra column and backfill names for matched migrations
|
||||
*/
|
||||
0: async (migrationsTable, db, localMigrations) => {
|
||||
const table = sql`${sql.identifier(migrationsTable)}`
|
||||
const dbRows = await db.session.all<SQLiteMigrationTableRow>(
|
||||
sql`SELECT id, hash, created_at FROM ${table} ORDER BY id ASC`,
|
||||
)
|
||||
const statements = buildSQLiteMigrationBackfillStatements(
|
||||
migrationsTable,
|
||||
prepareSQLiteMigrationBackfill(dbRows, localMigrations),
|
||||
)
|
||||
|
||||
await db.transaction(async (tx) => {
|
||||
for (const statement of statements) {
|
||||
await tx.run(statement)
|
||||
}
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
@@ -195,6 +195,18 @@ export function hash(value: Schema.Json) {
|
||||
return Hash.make(createHash("sha256").update(canonical(value)).digest("hex"))
|
||||
}
|
||||
|
||||
export function applyDelta(
|
||||
values: Readonly<Record<string, Schema.Json>>,
|
||||
delta: Readonly<Record<string, Option.Option<Schema.Json>>>,
|
||||
): Readonly<Record<string, Schema.Json>> {
|
||||
const result: Record<string, Schema.Json> = { ...values }
|
||||
for (const [key, value] of Object.entries(delta)) {
|
||||
if (Option.isNone(value)) delete result[key]
|
||||
else result[key] = value.value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function applyHashDelta(values: Values, delta: Delta): Values {
|
||||
const result: Record<string, Hash> = { ...values }
|
||||
for (const [key, value] of Object.entries(delta)) {
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
export const MAX_MARKDOWN_BYTES = 5 * 1024 * 1024
|
||||
@@ -1,7 +1,4 @@
|
||||
import { Parser } from "htmlparser2"
|
||||
import { MAX_MARKDOWN_BYTES } from "./html-markdown-limit.js"
|
||||
|
||||
export { MAX_MARKDOWN_BYTES } from "./html-markdown-limit.js"
|
||||
|
||||
const omitted = new Set(["script", "style", "noscript", "iframe", "object", "embed", "meta", "link", "template"])
|
||||
const blocks = new Set([
|
||||
@@ -50,6 +47,7 @@ type Frame = {
|
||||
|
||||
type Chunk = string | { raw: string }
|
||||
|
||||
export const MAX_MARKDOWN_BYTES = 5 * 1024 * 1024
|
||||
const CONTENT_BYTES = MAX_MARKDOWN_BYTES - 64 * 1024
|
||||
|
||||
export function convertHTMLToMarkdown(html: string) {
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
import { Parser } from "htmlparser2"
|
||||
import { convertHTMLToMarkdown } from "../html-markdown.js"
|
||||
|
||||
export { convertHTMLToMarkdown }
|
||||
|
||||
export function extractTextFromHTML(html: string) {
|
||||
let text = ""
|
||||
let skipDepth = 0
|
||||
const parser = new Parser({
|
||||
onopentag(name) {
|
||||
if (skipDepth > 0 || ["script", "style", "noscript", "iframe", "object", "embed"].includes(name)) skipDepth++
|
||||
},
|
||||
ontext(input) {
|
||||
if (skipDepth === 0) text += input
|
||||
},
|
||||
onclosetag() {
|
||||
if (skipDepth > 0) skipDepth--
|
||||
},
|
||||
})
|
||||
parser.write(html)
|
||||
parser.end()
|
||||
return text.trim()
|
||||
}
|
||||
@@ -4,8 +4,9 @@ import type { Context as PluginContext } from "@opencode-ai/plugin/effect/plugin
|
||||
import { ToolFailure } from "@opencode-ai/ai"
|
||||
import { Duration, Effect, Schema } from "effect"
|
||||
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import { Parser } from "htmlparser2"
|
||||
import { Permission } from "../../permission.js"
|
||||
import { MAX_MARKDOWN_BYTES } from "../html-markdown-limit.js"
|
||||
import { convertHTMLToMarkdown, MAX_MARKDOWN_BYTES } from "../html-markdown.js"
|
||||
import { collectBoundedResponseBody } from "../http-body.js"
|
||||
|
||||
export const name = "webfetch"
|
||||
@@ -102,12 +103,11 @@ const isTextualMime = (mime: string) =>
|
||||
mime.endsWith("+xml") ||
|
||||
mime === "application/javascript" ||
|
||||
mime === "application/x-javascript"
|
||||
const convert = async (content: string, contentType: string, format: Format) => {
|
||||
const convert = (content: string, contentType: string, format: Format) => {
|
||||
if (!contentType.includes("text/html")) return content
|
||||
if (format === "html") return content
|
||||
const { convertHTMLToMarkdown, extractTextFromHTML } = await import("./webfetch-convert.js")
|
||||
if (format === "markdown") return convertHTMLToMarkdown(content)
|
||||
return extractTextFromHTML(content)
|
||||
if (format === "text") return extractTextFromHTML(content)
|
||||
return content
|
||||
}
|
||||
|
||||
export const Plugin = {
|
||||
@@ -159,7 +159,7 @@ export const Plugin = {
|
||||
}),
|
||||
)
|
||||
const content = new TextDecoder().decode(body)
|
||||
const output = yield* Effect.tryPromise({
|
||||
const output = yield* Effect.try({
|
||||
try: () => convert(content, contentType, input.format),
|
||||
catch: (error) => error,
|
||||
})
|
||||
@@ -176,3 +176,24 @@ export const Plugin = {
|
||||
.pipe(Effect.orDie)
|
||||
}),
|
||||
}
|
||||
|
||||
export function extractTextFromHTML(html: string) {
|
||||
let text = ""
|
||||
let skipDepth = 0
|
||||
const parser = new Parser({
|
||||
onopentag(name) {
|
||||
if (skipDepth > 0 || ["script", "style", "noscript", "iframe", "object", "embed"].includes(name)) skipDepth++
|
||||
},
|
||||
ontext(input) {
|
||||
if (skipDepth === 0) text += input
|
||||
},
|
||||
onclosetag() {
|
||||
if (skipDepth > 0) skipDepth--
|
||||
},
|
||||
})
|
||||
parser.write(html)
|
||||
parser.end()
|
||||
return text.trim()
|
||||
}
|
||||
|
||||
export { convertHTMLToMarkdown }
|
||||
|
||||
@@ -55,6 +55,7 @@ const integrations = Layer.mock(Integration.Service, {
|
||||
})
|
||||
const npm = Layer.mock(Npm.Service, {
|
||||
add: () => Effect.die("unused"),
|
||||
install: () => Effect.die("unused"),
|
||||
which: () => Effect.die("unused"),
|
||||
})
|
||||
const aisdk = Layer.mock(AISDK.Service, {
|
||||
|
||||
@@ -125,6 +125,9 @@ describe("Instructions", () => {
|
||||
expect(
|
||||
Instructions.renderUpdate(instructions, { "api/value": "previous" }, { "api/value": Option.some(null) }),
|
||||
).toBe("null")
|
||||
expect(Instructions.applyDelta({ "api/value": "previous" }, { "api/value": Option.some(null) })).toEqual({
|
||||
"api/value": null,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
|
||||
@@ -34,11 +34,7 @@ export const readUpdate = (instructions: Instructions.List, previous: State) =>
|
||||
hash === "removed" ? Option.none() : Option.some(admission.blobs[hash]),
|
||||
]),
|
||||
) as Readonly<Record<string, Option.Option<Schema.Json>>>
|
||||
const values: Record<string, Schema.Json> = { ...previous.values }
|
||||
for (const [key, value] of Object.entries(delta)) {
|
||||
if (Option.isNone(value)) delete values[key]
|
||||
else values[key] = value.value
|
||||
}
|
||||
const values = Instructions.applyDelta(previous.values, delta)
|
||||
return {
|
||||
values,
|
||||
text: Instructions.renderUpdate(instructions, previous.values, delta),
|
||||
|
||||
@@ -29,6 +29,7 @@ const npmLayer = Layer.succeed(
|
||||
Npm.Service,
|
||||
Npm.Service.of({
|
||||
add: () => Effect.succeed({ directory: "", entrypoint: undefined }),
|
||||
install: () => Effect.void,
|
||||
which: () => Effect.succeed(undefined),
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -23,6 +23,7 @@ const itWithAISDK = testEffect(Layer.mergeAll(PluginTestLayer, AppNodeBuilder.bu
|
||||
function npmEntrypoint(entrypoint?: string) {
|
||||
return Npm.Service.of({
|
||||
add: () => Effect.succeed({ directory: "", entrypoint }),
|
||||
install: () => Effect.void,
|
||||
which: () => Effect.succeed(undefined),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ const fixtureProvider = new URL("./fixtures/provider-factory.ts", import.meta.ur
|
||||
const it = testEffect(PluginTestLayer)
|
||||
const npm = Npm.Service.of({
|
||||
add: () => Effect.succeed({ directory: "", entrypoint: undefined }),
|
||||
install: () => Effect.void,
|
||||
which: () => Effect.succeed(undefined),
|
||||
})
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ import { Permission } from "@opencode-ai/core/permission"
|
||||
import { Session } from "@opencode-ai/core/session"
|
||||
import { Tool } from "@opencode-ai/core/tool"
|
||||
import { WebFetchTool } from "@opencode-ai/core/tool/plugin/webfetch"
|
||||
import { convertHTMLToMarkdown, extractTextFromHTML } from "@opencode-ai/core/tool/plugin/webfetch-convert"
|
||||
import { makeLocationNode } from "@opencode-ai/util/effect/app-node"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { testEffect } from "./lib/effect"
|
||||
@@ -71,50 +70,52 @@ describe("WebFetchTool helpers", () => {
|
||||
test("ports HTML text and markdown conversions without active content", () => {
|
||||
const html =
|
||||
"<h1>Hello</h1><script>bad()</script><p>world <strong>wide</strong> <product-name>today</product-name></p><style>.bad {}</style>"
|
||||
expect(extractTextFromHTML(html)).toBe("Helloworld wide today")
|
||||
expect(convertHTMLToMarkdown(html)).toBe("# Hello\n\nworld **wide** today")
|
||||
expect(WebFetchTool.extractTextFromHTML(html)).toBe("Helloworld wide today")
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("# Hello\n\nworld **wide** today")
|
||||
})
|
||||
|
||||
test("renders headings, inline semantics, links, images, breaks, and thematic breaks", () => {
|
||||
const html = `<h2>Read <em>this</em></h2><p><a href="https://example.com/a (b)" title="Example">docs</a><br><img src="diagram.png" alt="a ] b"></p><hr><p><del>old</del></p>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`## Read *this*\n\n[docs](https://example.com/a%20\\(b\\) "Example") \n![a \\] b](diagram.png)\n\n---\n\n~~old~~`,
|
||||
)
|
||||
})
|
||||
|
||||
test("preserves inline and preformatted code verbatim with safe fences", () => {
|
||||
const html = `<p>Use <code>say(\`hello\`)</code> now.</p><pre><code class="language-ts">const fence = \`\`\`\n& stays decoded</code></pre>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`Use \`\`say(\`hello\`)\`\` now.\n\n~~~ts\nconst fence = \`\`\`\n& stays decoded\n~~~`,
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps nested ordered and unordered lists structurally readable", () => {
|
||||
const html = `<ol start="3"><li>alpha<ul><li>nested <strong>item</strong></li></ul></li><li><p>beta first</p><p>beta second</p></li></ol>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(`3. alpha\n\n - nested **item**\n\n4. beta first\n\n beta second`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`3. alpha\n\n - nested **item**\n\n4. beta first\n\n beta second`,
|
||||
)
|
||||
})
|
||||
|
||||
test("renders blockquotes and tables as readable Markdown", () => {
|
||||
const html = `<blockquote><p>quoted <em>text</em></p><ul><li>point</li></ul></blockquote><table><thead><tr><th>Name</th><th>Value</th></tr></thead><tbody><tr><td>one</td><td><code>1</code></td></tr></tbody></table>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`> quoted *text*\n\n> - point\n\n| Name | Value |\n| --- | --- |\n| one | \`1\` |`,
|
||||
)
|
||||
})
|
||||
|
||||
test("decodes entities and normalizes prose whitespace without joining words", () => {
|
||||
const html = `<p>alpha\n <span>& beta</span> <unknown>café</unknown> gamma 😀</p><p>delta</p>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(`alpha & beta café gamma 😀\n\ndelta`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`alpha & beta café gamma 😀\n\ndelta`)
|
||||
})
|
||||
|
||||
test("omits active and fallback content while retaining surrounding prose", () => {
|
||||
const html = `<p>before <script><b>bad</b></script><style>bad</style><noscript>bad</noscript><iframe>bad</iframe><object>bad</object><embed src="bad"><meta content="bad"><link href="bad"><template>bad</template> after</p>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe("before after")
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("before after")
|
||||
})
|
||||
|
||||
test("is deterministic and bounded for malformed maximum-size input", () => {
|
||||
const html = `<main><p>${"visible & text ".repeat(250_000)}</main></p></unknown>`
|
||||
const first = convertHTMLToMarkdown(html)
|
||||
expect(convertHTMLToMarkdown(html)).toBe(first)
|
||||
const first = WebFetchTool.convertHTMLToMarkdown(html)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(first)
|
||||
expect(first.startsWith("visible & text visible & text")).toBe(true)
|
||||
expect(first.length).toBeLessThanOrEqual(html.length)
|
||||
})
|
||||
@@ -123,62 +124,64 @@ describe("WebFetchTool helpers", () => {
|
||||
const lists = `${"<ul><li>item".repeat(2_000)}${"</li></ul>".repeat(2_000)}`
|
||||
const quotes = `${"<blockquote><p>item".repeat(2_000)}${"</p></blockquote>".repeat(2_000)}`
|
||||
const code = `<pre>${"` x ".repeat(250_000)}</pre>`
|
||||
expect(convertHTMLToMarkdown(lists).length).toBeLessThan(lists.length * 4)
|
||||
expect(convertHTMLToMarkdown(quotes).length).toBeLessThan(quotes.length * 4)
|
||||
expect(() => convertHTMLToMarkdown(code)).not.toThrow()
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(lists).length).toBeLessThan(lists.length * 4)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(quotes).length).toBeLessThan(quotes.length * 4)
|
||||
expect(() => WebFetchTool.convertHTMLToMarkdown(code)).not.toThrow()
|
||||
expect(
|
||||
convertHTMLToMarkdown("<div>".repeat(20_000) + "safe<script><b>bad</b>&</script><p>tail &</p>"),
|
||||
WebFetchTool.convertHTMLToMarkdown(
|
||||
"<div>".repeat(20_000) + "safe<script><b>bad</b>&</script><p>tail &</p>",
|
||||
),
|
||||
).toBe("safe tail &")
|
||||
})
|
||||
|
||||
test("escapes prose that would otherwise become Markdown structure", () => {
|
||||
expect(convertHTMLToMarkdown(`<p># heading</p><p>1. item</p><p>---</p><p>a | b</p>`)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<p># heading</p><p>1. item</p><p>---</p><p>a | b</p>`)).toBe(
|
||||
`\\# heading\n\n1\\. item\n\n\\---\n\na \\| b`,
|
||||
)
|
||||
})
|
||||
|
||||
test("preserves code whitespace and quotes every line of multiline blocks", () => {
|
||||
const html = `<blockquote><pre>line \n\n\nnext</pre><table><tr><td>a|b</td><td>c</td></tr></table></blockquote>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`> \`\`\`\n> line \n> \n> \n> next\n> \`\`\`\n\n> | a\\|b | c |\n> | --- | --- |`,
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps nested blockquotes inside their outer quote", () => {
|
||||
const html = `<blockquote><p>outer</p><blockquote><p>inner</p></blockquote><p>end</p></blockquote>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(`> outer\n>\n> > inner\n>\n> end`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`> outer\n>\n> > inner\n>\n> end`)
|
||||
})
|
||||
|
||||
test("keeps visible whitespace around inline emphasis", () => {
|
||||
expect(convertHTMLToMarkdown(`<p>a<strong> b</strong> c a <em>b </em>c</p>`)).toBe(`a **b** c a *b* c`)
|
||||
expect(convertHTMLToMarkdown(`a<strong> </strong>b a<em> </em>b`)).toBe(`a b a b`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<p>a<strong> b</strong> c a <em>b </em>c</p>`)).toBe(`a **b** c a *b* c`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`a<strong> </strong>b a<em> </em>b`)).toBe(`a b a b`)
|
||||
})
|
||||
|
||||
test("captures formatting elements inside preformatted content as code only", () => {
|
||||
expect(convertHTMLToMarkdown(`<pre><b>x</b><i>y</i><del>z</del></pre>`)).toBe(`\`\`\`\nxyz\n\`\`\``)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<pre><b>x</b><i>y</i><del>z</del></pre>`)).toBe(`\`\`\`\nxyz\n\`\`\``)
|
||||
})
|
||||
|
||||
test("normalizes multiline table cells without changing their columns", () => {
|
||||
const html = `<table><tr><td>x<br>y</td><td><code>a|b</code></td><td><p>first</p><p>second</p></td></tr></table>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(`| x y | \`a\\|b\` | first second |\n| --- | --- | --- |`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`| x y | \`a\\|b\` | first second |\n| --- | --- | --- |`)
|
||||
})
|
||||
|
||||
test("flattens nested tables without corrupting the outer table", () => {
|
||||
const html = `<table><tr><th>Parent</th><th>Sibling</th></tr><tr><td>Before<table><tr><th>Key</th><th>Value</th></tr><tr><td>A</td><td>1</td></tr></table>After</td><td>Tail</td></tr></table>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`| Parent | Sibling |\n| --- | --- |\n| Before Key Value A 1 After | Tail |`,
|
||||
)
|
||||
})
|
||||
|
||||
test("preserves loose text around malformed table rows", () => {
|
||||
expect(convertHTMLToMarkdown(`<table>before<tr><td>cell</td></tr>after</table>`)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<table>before<tr><td>cell</td></tr>after</table>`)).toBe(
|
||||
`before after\n\n| cell |\n| --- |`,
|
||||
)
|
||||
expect(convertHTMLToMarkdown(`<table>alpha</table>`)).toBe(`alpha`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<table>alpha</table>`)).toBe(`alpha`)
|
||||
})
|
||||
|
||||
test("escapes tilde fences and removes empty emphasis markers", () => {
|
||||
expect(convertHTMLToMarkdown(`<p>~~~</p><p><strong></strong>content</p><p>~~~</p>`)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<p>~~~</p><p><strong></strong>content</p><p>~~~</p>`)).toBe(
|
||||
`\\~\\~\\~\n\ncontent\n\n\\~\\~\\~`,
|
||||
)
|
||||
})
|
||||
@@ -187,10 +190,10 @@ describe("WebFetchTool helpers", () => {
|
||||
const small = "<a".repeat(250_000)
|
||||
const large = "<a".repeat(1_000_000)
|
||||
const start = Bun.nanoseconds()
|
||||
convertHTMLToMarkdown(small)
|
||||
WebFetchTool.convertHTMLToMarkdown(small)
|
||||
const smallDuration = Bun.nanoseconds() - start
|
||||
const next = Bun.nanoseconds()
|
||||
convertHTMLToMarkdown(large)
|
||||
WebFetchTool.convertHTMLToMarkdown(large)
|
||||
const largeDuration = Bun.nanoseconds() - next
|
||||
expect(largeDuration).toBeLessThan(smallDuration * 10)
|
||||
})
|
||||
@@ -198,69 +201,73 @@ describe("WebFetchTool helpers", () => {
|
||||
test("caps escaped prose and backtick-heavy pre output at the webfetch response ceiling", () => {
|
||||
const prose = `<p>${"*".repeat(WebFetchTool.MAX_RESPONSE_BYTES)}</p>`
|
||||
const code = `<pre>${"`".repeat(WebFetchTool.MAX_RESPONSE_BYTES - 11)}</pre>`
|
||||
const proseOutput = convertHTMLToMarkdown(prose)
|
||||
const codeOutput = convertHTMLToMarkdown(code)
|
||||
const proseOutput = WebFetchTool.convertHTMLToMarkdown(prose)
|
||||
const codeOutput = WebFetchTool.convertHTMLToMarkdown(code)
|
||||
expect(Buffer.byteLength(proseOutput)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
|
||||
expect(Buffer.byteLength(codeOutput)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
|
||||
expect(codeOutput.startsWith("~~~\n")).toBe(true)
|
||||
})
|
||||
|
||||
test("does not confuse source NUL text with buffered code", () => {
|
||||
expect(convertHTMLToMarkdown(`<p>before \u00000\u0000 after</p><pre>code</pre>`)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<p>before \u00000\u0000 after</p><pre>code</pre>`)).toBe(
|
||||
`before \u00000\u0000 after\n\n\`\`\`\ncode\n\`\`\``,
|
||||
)
|
||||
})
|
||||
|
||||
test("preserves multiline inline code verbatim", () => {
|
||||
expect(convertHTMLToMarkdown(`<p><code>first\n\n\nsecond </code></p>`)).toBe("` first\n\n\nsecond `")
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<p><code>first\n\n\nsecond </code></p>`)).toBe(
|
||||
"` first\n\n\nsecond `",
|
||||
)
|
||||
})
|
||||
|
||||
test("prefixes inline code at the start of a blockquote line", () => {
|
||||
expect(convertHTMLToMarkdown(`<blockquote><code>x</code> y</blockquote>`)).toBe(`> \`x\` y`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<blockquote><code>x</code> y</blockquote>`)).toBe(`> \`x\` y`)
|
||||
})
|
||||
|
||||
test("keeps links nested in inline code associated with their text", () => {
|
||||
const html = `<dl><dt><code>socket = new <a href="#constructor">WebSocket</a>(url)</code><dd>Creates one.</dl>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`**\` socket = new \`[\`WebSocket\`](#constructor)\`(url)\`**\n: Creates one.`,
|
||||
)
|
||||
expect(convertHTMLToMarkdown(`<code><a href="#x">x</a></code> after`)).toBe(`[\`x\`](#x) after`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<code><a href="#x">x</a></code> after`)).toBe(`[\`x\`](#x) after`)
|
||||
expect(
|
||||
convertHTMLToMarkdown(
|
||||
WebFetchTool.convertHTMLToMarkdown(
|
||||
`<dl><dt><code><var>socket</var> = new <code><a href="#constructor">WebSocket</a></code>(<var>url</var>)</code><dd>Creates one.</dl>`,
|
||||
),
|
||||
).toBe(`**\` socket = new \`[\`WebSocket\`](#constructor)\`(url)\`**\n: Creates one.`)
|
||||
expect(convertHTMLToMarkdown(`<code>a<a href="/x">b<a href="/y">c</a>d</a>e</code>`)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<code>a<a href="/x">b<a href="/y">c</a>d</a>e</code>`)).toBe(
|
||||
`\`a\`[\`b\`](\/x)[\`c\`](\/y)\`de\``,
|
||||
)
|
||||
expect(convertHTMLToMarkdown(`<code>a<a href="/x">b</code>c`)).toBe(`\`a\`[\`b\`](\/x)c`)
|
||||
expect(convertHTMLToMarkdown(`<code>a<a href="/x"><div>b</div>c</a>d</code>`)).toBe(`\`a\`[](\/x)\n\n\`bcd\``)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<code>a<a href="/x">b</code>c`)).toBe(`\`a\`[\`b\`](\/x)c`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<code>a<a href="/x"><div>b</div>c</a>d</code>`)).toBe(
|
||||
`\`a\`[](\/x)\n\n\`bcd\``,
|
||||
)
|
||||
})
|
||||
|
||||
test("indents nested list continuations and preserves ordered numbering", () => {
|
||||
const html = `<ol start="0"><li value="4"><p>first</p><p>continued</p><ul><li><p>nested</p><p>continued nested</p></li></ul></li><li>next</li></ol>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`4. first\n\n continued\n\n - nested\n\n continued nested\n\n5. next`,
|
||||
)
|
||||
})
|
||||
|
||||
test("renders block content outside link syntax", () => {
|
||||
expect(convertHTMLToMarkdown(`<a href="/docs">before<div>block</div>after</a>`)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<a href="/docs">before<div>block</div>after</a>`)).toBe(
|
||||
`[before](/docs)\n\nblock\n\n[after](/docs)`,
|
||||
)
|
||||
})
|
||||
|
||||
test("recovers nested anchors without unmatched Markdown syntax", () => {
|
||||
expect(convertHTMLToMarkdown(`<a href="/a">x<a href="/b">y</a>z</a>`)).toBe(`[x](/a)[y](/b)z`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<a href="/a">x<a href="/b">y</a>z</a>`)).toBe(`[x](/a)[y](/b)z`)
|
||||
})
|
||||
|
||||
test("keeps emphasis whitespace through neutral wrappers", () => {
|
||||
expect(convertHTMLToMarkdown(`<p>a<strong><span> bold</span></strong>c</p>`)).toBe(`a **bold** c`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<p>a<strong><span> bold</span></strong>c</p>`)).toBe(`a **bold** c`)
|
||||
})
|
||||
|
||||
test("flattens preformatted content inside table cells", () => {
|
||||
const html = `<table><tr><td><pre>a|b\nnext</pre></td><td><code>x|y</code></td></tr></table>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(`| a\\|b next | \`x\\|y\` |\n| --- | --- |`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`| a\\|b next | \`x\\|y\` |\n| --- | --- |`)
|
||||
})
|
||||
|
||||
test("keeps each near-boundary inline construct closed and UTF-8-safe", () => {
|
||||
@@ -272,7 +279,7 @@ describe("WebFetchTool helpers", () => {
|
||||
[`<code>${payload}</code>`, /^`[\s\S]*`$/],
|
||||
] as const
|
||||
for (const [html, pattern] of cases) {
|
||||
const output = convertHTMLToMarkdown(html)
|
||||
const output = WebFetchTool.convertHTMLToMarkdown(html)
|
||||
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
|
||||
expect(output).not.toContain("�")
|
||||
expect(output).toMatch(pattern)
|
||||
@@ -281,9 +288,11 @@ describe("WebFetchTool helpers", () => {
|
||||
|
||||
test("keeps near-boundary block constructs syntactically complete", () => {
|
||||
const payload = "x".repeat(WebFetchTool.MAX_RESPONSE_BYTES)
|
||||
const table = convertHTMLToMarkdown(`<table><tr><th>Name</th></tr><tr><td>${payload}</td></tr></table>`)
|
||||
const list = convertHTMLToMarkdown(`<ul><li>${payload}</li></ul><ul><li>nested</li></ul>`)
|
||||
const code = convertHTMLToMarkdown(`<pre>${payload}</pre>`)
|
||||
const table = WebFetchTool.convertHTMLToMarkdown(
|
||||
`<table><tr><th>Name</th></tr><tr><td>${payload}</td></tr></table>`,
|
||||
)
|
||||
const list = WebFetchTool.convertHTMLToMarkdown(`<ul><li>${payload}</li></ul><ul><li>nested</li></ul>`)
|
||||
const code = WebFetchTool.convertHTMLToMarkdown(`<pre>${payload}</pre>`)
|
||||
for (const output of [table, list, code]) {
|
||||
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
|
||||
expect(output).not.toContain("�")
|
||||
@@ -296,7 +305,7 @@ describe("WebFetchTool helpers", () => {
|
||||
|
||||
test("keeps quoted code within budget with a safe closed fence", () => {
|
||||
const html = `<blockquote><pre>${"`".repeat(32)}${"~".repeat(32)}${"x".repeat(WebFetchTool.MAX_RESPONSE_BYTES)}</pre></blockquote>`
|
||||
const output = convertHTMLToMarkdown(html)
|
||||
const output = WebFetchTool.convertHTMLToMarkdown(html)
|
||||
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
|
||||
const lines = output.split("\n")
|
||||
expect(lines[0]).toMatch(/^> (`{33}|~{33})$/)
|
||||
@@ -305,14 +314,14 @@ describe("WebFetchTool helpers", () => {
|
||||
|
||||
test("separates reconstructed tables from adjacent inline and quoted content", () => {
|
||||
const html = `intro<table><tr><td>x</td></tr></table>outro<blockquote>quote<table><tr><td>cell</td></tr></table></blockquote><ul><li>item<table><tr><td>cell</td></tr></table></li></ul>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`intro\n\n| x |\n| --- |\n\noutro\n\n> quote\n\n> | cell |\n> | --- |\n\n- item\n\n| cell |\n| --- |`,
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps multiline quoted code closed at the content budget", () => {
|
||||
const html = `<blockquote><pre>${"x\n".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 2)}</pre></blockquote><p>tail</p>`
|
||||
const output = convertHTMLToMarkdown(html)
|
||||
const output = WebFetchTool.convertHTMLToMarkdown(html)
|
||||
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
|
||||
expect((output.match(/(`{3}|~{3})/g) ?? []).length).toBe(2)
|
||||
expect(output.includes("\uFFFD")).toBe(false)
|
||||
@@ -321,52 +330,54 @@ describe("WebFetchTool helpers", () => {
|
||||
|
||||
test("keeps active content suppressed when depth fallback begins", () => {
|
||||
const html = `<object>${"<div>".repeat(10_001)}LEAK${"</div>".repeat(10_001)}</object><p>visible</p>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe("visible")
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("visible")
|
||||
})
|
||||
|
||||
test("keeps visible text after depth fallback begins inside preformatted content", () => {
|
||||
const html = `<pre>${"<i>".repeat(10_001)}visible${"</i>".repeat(10_001)}</pre><p>after</p>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe("visible after")
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("visible after")
|
||||
})
|
||||
|
||||
test("resumes links around every block structure", () => {
|
||||
const html = `<a href="/x">before<blockquote><p>quote</p></blockquote><ul><li>item</li></ul><pre>code</pre><table><tr><td>cell</td></tr></table>after</a>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`[before](/x)\n\n> quote\n\n- item\n\n\`\`\`\ncode\n\`\`\`\n\n| cell |\n| --- |\n\n[after](/x)`,
|
||||
)
|
||||
})
|
||||
|
||||
test("indents child lists from the actual parent marker width", () => {
|
||||
expect(convertHTMLToMarkdown(`<ol start="100"><li>outer<ul><li>inner</li></ul></li></ol>`)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<ol start="100"><li>outer<ul><li>inner</li></ul></li></ol>`)).toBe(
|
||||
`100. outer\n\n - inner`,
|
||||
)
|
||||
})
|
||||
|
||||
test("renders captions and definition lists with readable boundaries", () => {
|
||||
const html = `<table><caption>Cache modes</caption><tr><th>Name</th><th>Meaning</th></tr><tr><td>A</td><td>Local</td></tr></table><dl><dt>Cache</dt><dd>A local store</dd><dt>Origin</dt><dd>The remote source</dd></dl>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`Cache modes\n\n| Name | Meaning |\n| --- | --- |\n| A | Local |\n\n**Cache**\n: A local store\n\n**Origin**\n: The remote source`,
|
||||
)
|
||||
})
|
||||
|
||||
test("falls back to row-oriented text for table spans", () => {
|
||||
const html = `<table><tr><th colspan="2">Group</th></tr><tr><td>A</td><td rowspan="2">Shared</td></tr><tr><td>B</td></tr></table>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(`Group\n\nA | Shared\n\nB`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`Group\n\nA | Shared\n\nB`)
|
||||
})
|
||||
|
||||
test("suppresses head and hidden subtrees while retaining visible body content", () => {
|
||||
const html = `<head><title>noise</title></head><body><p>visible</p><div hidden>hidden</div><div aria-hidden="true">aria</div><div aria-hidden="false">shown</div></body>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(`visible\n\nshown`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`visible\n\nshown`)
|
||||
})
|
||||
|
||||
test("preserves pre breaks and normalizes multiline link titles", () => {
|
||||
const html = `<pre>first<br>second</pre><p><a href="/x" title="line one\n line two">link</a></p>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(`\`\`\`\nfirst\nsecond\n\`\`\`\n\n[link](/x "line one line two")`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`\`\`\`\nfirst\nsecond\n\`\`\`\n\n[link](/x "line one line two")`,
|
||||
)
|
||||
})
|
||||
|
||||
test("renders closed and open details according to visibility", () => {
|
||||
const html = `<details><summary>Closed</summary><p>secret</p></details><details open><summary>Open</summary><p>visible</p></details>`
|
||||
expect(convertHTMLToMarkdown(html)).toBe(`Closed\n\nOpen\n\nvisible`)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`Closed\n\nOpen\n\nvisible`)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -388,11 +399,6 @@ describe("WebFetchTool registration", () => {
|
||||
{ sessionID, action: "webfetch", resources: [url], save: ["*"], metadata: { url, format: "text", timeout: 4 } },
|
||||
])
|
||||
expect(requests).toMatchObject([{ url, headers: { accept: expect.stringContaining("text/plain;q=1.0") } }])
|
||||
|
||||
expect(yield* executeTool(registry, call({ url, format: "markdown" }))).toMatchObject({
|
||||
status: "completed",
|
||||
content: [{ type: "text", text: "hello" }],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -476,10 +482,6 @@ describe("WebFetchTool registration", () => {
|
||||
status: "completed",
|
||||
content: [{ type: "text", text: "Helloworld" }],
|
||||
})
|
||||
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1", format: "html" }))).toMatchObject({
|
||||
status: "completed",
|
||||
content: [{ type: "text", text: "<h1>Hello</h1><p>world</p><script>bad()</script>" }],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
|
||||
@@ -4,4 +4,9 @@ import { Event } from "./event.js"
|
||||
import { Worktree } from "./worktree.js"
|
||||
import { SessionEvent } from "./session-event.js"
|
||||
|
||||
export const SessionDurable = {
|
||||
definitions: Event.durableMap(SessionEvent.DurableDefinitions),
|
||||
schema: SessionEvent.Durable,
|
||||
} as const
|
||||
|
||||
export const Durable = Event.durableMap([...SessionEvent.DurableDefinitions, Worktree.Event.Resolved])
|
||||
|
||||
@@ -17,6 +17,14 @@ export const ResourcesChanged = Event.ephemeral({
|
||||
},
|
||||
})
|
||||
|
||||
export const BrowserOpenFailed = Event.ephemeral({
|
||||
type: "mcp.browser.open.failed",
|
||||
schema: {
|
||||
mcpName: Schema.String,
|
||||
url: Schema.String,
|
||||
},
|
||||
})
|
||||
|
||||
// Emitted whenever a server's connection status settles (connected, failed, needs_auth, closed) so
|
||||
// observers can refresh status without polling.
|
||||
export const StatusChanged = Event.ephemeral({
|
||||
|
||||
@@ -833,10 +833,13 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({
|
||||
"session",
|
||||
"input",
|
||||
event.data.sessionID,
|
||||
(store.session.input[event.data.sessionID] ?? []).filter((id) => id < event.data.to),
|
||||
(items) => {
|
||||
const boundary = items?.findIndex((id) => id === event.data.to) ?? -1
|
||||
return boundary < 0 ? items : items?.slice(0, boundary)
|
||||
},
|
||||
)
|
||||
message.update(event.data.sessionID, (draft, index) => {
|
||||
const position = draft.findIndex((item) => item.id >= event.data.to)
|
||||
const position = draft.findIndex((item) => item.id === event.data.to)
|
||||
if (position === -1) return
|
||||
for (const item of draft.splice(position)) index.delete(item.id)
|
||||
})
|
||||
|
||||
@@ -52,8 +52,9 @@ export function createSessionRows(sessionID: Accessor<string>) {
|
||||
)
|
||||
const visible = queued.size === 0 ? messages : messages.filter((message) => !queued.has(message.id))
|
||||
const boundary = revertBoundary()
|
||||
const boundaryIndex = boundary ? visible.findIndex((message) => message.id === boundary) : -1
|
||||
const rows = reduceSessionRows(
|
||||
boundary ? visible.filter((message) => message.id < boundary) : visible,
|
||||
boundaryIndex < 0 ? visible : visible.slice(0, boundaryIndex),
|
||||
inputs,
|
||||
turnTokens(),
|
||||
)
|
||||
|
||||
@@ -1105,7 +1105,7 @@ test("removes committed revert messages from local state", async () => {
|
||||
))
|
||||
|
||||
try {
|
||||
for (const [seq, inboxID] of ["msg_001", "msg_002", "msg_003"].entries()) {
|
||||
for (const [seq, inboxID] of ["msg_fff", "msg_000", "msg_001"].entries()) {
|
||||
emitEvent(events, {
|
||||
id: Event.ID.create(),
|
||||
created: seq,
|
||||
@@ -1121,13 +1121,13 @@ test("removes committed revert messages from local state", async () => {
|
||||
created: 3,
|
||||
type: "session.revert.committed",
|
||||
durable: durable(sessionID, 3),
|
||||
data: { sessionID, to: "msg_002" },
|
||||
data: { sessionID, to: "msg_000" },
|
||||
})
|
||||
|
||||
await wait(() => data.session.message.list(sessionID).length === 1)
|
||||
expect(data.session.message.list(sessionID).map((message) => message.id)).toEqual(["msg_001"])
|
||||
expect(data.session.message.get(sessionID, "msg_002")).toBeUndefined()
|
||||
expect(data.session.message.get(sessionID, "msg_003")).toBeUndefined()
|
||||
expect(data.session.message.list(sessionID).map((message) => message.id)).toEqual(["msg_fff"])
|
||||
expect(data.session.message.get(sessionID, "msg_000")).toBeUndefined()
|
||||
expect(data.session.message.get(sessionID, "msg_001")).toBeUndefined()
|
||||
} finally {
|
||||
app.renderer.destroy()
|
||||
}
|
||||
|
||||
@@ -52,7 +52,8 @@
|
||||
"mime-types": "3.0.2",
|
||||
"minimatch": "10.2.5",
|
||||
"npm-package-arg": "13.0.2",
|
||||
"resolve.exports": "catalog:"
|
||||
"resolve.exports": "catalog:",
|
||||
"xdg-basedir": "5.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tsconfig/bun": "catalog:",
|
||||
|
||||
@@ -1,19 +1,14 @@
|
||||
import os from "os"
|
||||
import path from "path"
|
||||
|
||||
const home = os.homedir()
|
||||
const data = process.env.XDG_DATA_HOME || (home ? path.join(home, ".local", "share") : undefined)
|
||||
const cache = process.env.XDG_CACHE_HOME || (home ? path.join(home, ".cache") : undefined)
|
||||
const config = process.env.XDG_CONFIG_HOME || (home ? path.join(home, ".config") : undefined)
|
||||
const state = process.env.XDG_STATE_HOME || (home ? path.join(home, ".local", "state") : undefined)
|
||||
import { xdgCache, xdgConfig, xdgData, xdgState } from "xdg-basedir"
|
||||
|
||||
/** The XDG base directories that root opencode's global paths. */
|
||||
export function roots(app: string) {
|
||||
return {
|
||||
data: path.join(data!, app),
|
||||
cache: path.join(cache!, app),
|
||||
config: path.join(config!, app),
|
||||
state: path.join(state!, app),
|
||||
data: path.join(xdgData!, app),
|
||||
cache: path.join(xdgCache!, app),
|
||||
config: path.join(xdgConfig!, app),
|
||||
state: path.join(xdgState!, app),
|
||||
tmp: path.join(os.tmpdir(), app),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
export * as NpmConfig from "./npm-config.js"
|
||||
|
||||
import { fileURLToPath } from "url"
|
||||
// @ts-expect-error npm does not publish types for this internal config API.
|
||||
import Config from "@npmcli/config"
|
||||
// @ts-expect-error npm does not publish types for this internal config API.
|
||||
import { definitions, flatten, nerfDarts, shorthands } from "@npmcli/config/lib/definitions/index.js"
|
||||
import { Effect } from "effect"
|
||||
|
||||
export const load = (dir: string) =>
|
||||
Effect.tryPromise({
|
||||
try: async () => {
|
||||
// @ts-expect-error npm does not publish types for this internal config API.
|
||||
const { default: Config } = await import("@npmcli/config")
|
||||
// @ts-expect-error npm does not publish types for this internal config API.
|
||||
const { default: npmDefinitions } = await import("@npmcli/config/lib/definitions/index.js")
|
||||
const { definitions, flatten, nerfDarts, shorthands } = npmDefinitions
|
||||
const config = new Config({
|
||||
// Resolved per call: on workerd import.meta.url is undefined and building
|
||||
// this URL at module scope fails startup validation; npm config never runs there.
|
||||
|
||||
@@ -30,6 +30,15 @@ export interface Interface {
|
||||
pkg: string,
|
||||
options?: { readonly subpaths?: readonly string[] },
|
||||
) => Effect.Effect<EntryPoint, InstallFailedError | EffectFlock.LockError>
|
||||
readonly install: (
|
||||
dir: string,
|
||||
input?: {
|
||||
add: {
|
||||
name: string
|
||||
version?: string
|
||||
}[]
|
||||
},
|
||||
) => Effect.Effect<void, EffectFlock.LockError | InstallFailedError>
|
||||
readonly which: (pkg: string, bin?: string) => Effect.Effect<string | undefined>
|
||||
}
|
||||
|
||||
@@ -134,6 +143,59 @@ const layer = Layer.effect(
|
||||
return resolveEntryPoint(first.name, first.path, options?.subpaths)
|
||||
}, Effect.scoped)
|
||||
|
||||
const install: Interface["install"] = Effect.fn("Npm.install")(function* (dir, input) {
|
||||
const canWrite = yield* afs.access(dir, { writable: true }).pipe(
|
||||
Effect.as(true),
|
||||
Effect.orElseSucceed(() => false),
|
||||
)
|
||||
if (!canWrite) return
|
||||
|
||||
const add = input?.add.map((pkg) => [pkg.name, pkg.version].filter(Boolean).join("@")) ?? []
|
||||
if (
|
||||
yield* Effect.gen(function* () {
|
||||
const nodeModulesExists = yield* afs.existsSafe(path.join(dir, "node_modules"))
|
||||
if (!nodeModulesExists) {
|
||||
yield* reify({ add, dir })
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}).pipe(Effect.withSpan("Npm.checkNodeModules"))
|
||||
)
|
||||
return
|
||||
|
||||
yield* Effect.gen(function* () {
|
||||
const pkg = yield* afs.readJson(path.join(dir, "package.json")).pipe(Effect.orElseSucceed(() => ({})))
|
||||
const lock = yield* afs.readJson(path.join(dir, "package-lock.json")).pipe(Effect.orElseSucceed(() => ({})))
|
||||
|
||||
const pkgAny = pkg as any
|
||||
const lockAny = lock as any
|
||||
const declared = new Set([
|
||||
...Object.keys(pkgAny?.dependencies || {}),
|
||||
...Object.keys(pkgAny?.devDependencies || {}),
|
||||
...Object.keys(pkgAny?.peerDependencies || {}),
|
||||
...Object.keys(pkgAny?.optionalDependencies || {}),
|
||||
...(input?.add || []).map((pkg) => pkg.name),
|
||||
])
|
||||
|
||||
const root = lockAny?.packages?.[""] || {}
|
||||
const locked = new Set([
|
||||
...Object.keys(root?.dependencies || {}),
|
||||
...Object.keys(root?.devDependencies || {}),
|
||||
...Object.keys(root?.peerDependencies || {}),
|
||||
...Object.keys(root?.optionalDependencies || {}),
|
||||
])
|
||||
|
||||
for (const name of declared) {
|
||||
if (!locked.has(name)) {
|
||||
yield* reify({ dir, add })
|
||||
return
|
||||
}
|
||||
}
|
||||
}).pipe(Effect.withSpan("Npm.checkDirty"))
|
||||
|
||||
return
|
||||
}, Effect.scoped)
|
||||
|
||||
const which = Effect.fn("Npm.which")(function* (pkg: string, bin?: string) {
|
||||
const dir = directory(pkg)
|
||||
const binDir = path.join(dir, "node_modules", ".bin")
|
||||
@@ -187,6 +249,7 @@ const layer = Layer.effect(
|
||||
|
||||
return Service.of({
|
||||
add,
|
||||
install,
|
||||
which,
|
||||
})
|
||||
}),
|
||||
@@ -200,6 +263,10 @@ export const node = makeGlobalNode({
|
||||
|
||||
const { runPromise } = makeRuntime(Service, LayerNode.compile(node))
|
||||
|
||||
export async function install(...args: Parameters<Interface["install"]>) {
|
||||
return runPromise((svc) => svc.install(...args))
|
||||
}
|
||||
|
||||
export async function add(...args: Parameters<Interface["add"]>) {
|
||||
return runPromise((svc) => svc.add(...args))
|
||||
}
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import os from "os"
|
||||
import path from "path"
|
||||
import { pathToFileURL } from "url"
|
||||
|
||||
const module = pathToFileURL(path.join(import.meta.dir, "../src/global-roots.ts")).href
|
||||
|
||||
describe("global roots", () => {
|
||||
test("uses XDG overrides", () => {
|
||||
const root = path.join(os.tmpdir(), "opencode-xdg-overrides")
|
||||
const env = {
|
||||
XDG_DATA_HOME: path.join(root, "data"),
|
||||
XDG_CACHE_HOME: path.join(root, "cache"),
|
||||
XDG_CONFIG_HOME: path.join(root, "config"),
|
||||
XDG_STATE_HOME: path.join(root, "state"),
|
||||
}
|
||||
|
||||
expect(run(env)).toEqual({
|
||||
data: path.join(env.XDG_DATA_HOME, "opencode"),
|
||||
cache: path.join(env.XDG_CACHE_HOME, "opencode"),
|
||||
config: path.join(env.XDG_CONFIG_HOME, "opencode"),
|
||||
state: path.join(env.XDG_STATE_HOME, "opencode"),
|
||||
tmp: path.join(os.tmpdir(), "opencode"),
|
||||
})
|
||||
})
|
||||
|
||||
test("empty XDG overrides use home directory defaults", () => {
|
||||
const home = path.join(os.tmpdir(), "opencode-xdg-home")
|
||||
|
||||
expect(
|
||||
run({
|
||||
XDG_DATA_HOME: "",
|
||||
XDG_CACHE_HOME: "",
|
||||
XDG_CONFIG_HOME: "",
|
||||
XDG_STATE_HOME: "",
|
||||
...(process.platform === "win32" ? { USERPROFILE: home } : { HOME: home }),
|
||||
}),
|
||||
).toEqual({
|
||||
data: path.join(home, ".local", "share", "opencode"),
|
||||
cache: path.join(home, ".cache", "opencode"),
|
||||
config: path.join(home, ".config", "opencode"),
|
||||
state: path.join(home, ".local", "state", "opencode"),
|
||||
tmp: path.join(os.tmpdir(), "opencode"),
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
function run(env: Record<string, string>) {
|
||||
const result = Bun.spawnSync({
|
||||
cmd: [
|
||||
process.execPath,
|
||||
"-e",
|
||||
`const { roots } = await import(${JSON.stringify(module)}); console.log(JSON.stringify(roots("opencode")))`,
|
||||
],
|
||||
env: { ...process.env, ...env },
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
})
|
||||
|
||||
expect(result.exitCode, result.stderr.toString()).toBe(0)
|
||||
return JSON.parse(result.stdout.toString())
|
||||
}
|
||||
@@ -24,6 +24,7 @@
|
||||
"@cloudflare/workers-types": "^4.20250808.0",
|
||||
"vitest": "3.2.7",
|
||||
"wrangler": "4.28.0",
|
||||
"xdg-basedir": "5.1.0",
|
||||
"unenv": "2.0.0-rc.24",
|
||||
"@effect/platform-node": "catalog:"
|
||||
}
|
||||
|
||||
@@ -51,8 +51,10 @@ export default defineWorkersConfig({
|
||||
// mime-types requires mime-db's JSON database at require time; keep the
|
||||
// lookup surface but back it with a static shim.
|
||||
{ find: /^mime-types$/, replacement: new URL("./test/shims/mime-types.mjs", import.meta.url).pathname },
|
||||
// Plugin installs never happen in the workerd profile (plugin discovery
|
||||
// is precompiled-only), so mock the package installation toolchain.
|
||||
// util/npm.ts imports the npm toolchain at module scope; plugin installs
|
||||
// never happen in the workerd profile (plugin discovery is precompiled-only),
|
||||
// and @npmcli/config touches process.stdout.isTTY during module init.
|
||||
{ find: /^@npmcli\/config(\/.*)?$/, replacement: mockProxy },
|
||||
{ find: /^@npmcli\/arborist(\/.*)?$/, replacement: mockProxy },
|
||||
{ find: /^pacote(\/.*)?$/, replacement: mockProxy },
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user