mirror of
https://github.com/anomalyco/opencode.git
synced 2026-07-26 13:15:26 -04:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6f56b481ed | |||
| 06dae383f7 | |||
| 7d204b5b57 | |||
| 49593c1ec4 | |||
| ff837fe949 | |||
| 4c6750d464 | |||
| 69f1ec22e3 | |||
| 823d327401 |
@@ -7,8 +7,8 @@ import { FSUtil } from "./fs-util"
|
||||
import { Location } from "./location"
|
||||
import { PositiveInt, RelativePath } from "./schema"
|
||||
import { FileSystemSearch } from "./filesystem/search"
|
||||
import { Entry, Match, PathError } from "./filesystem/schema"
|
||||
export { Entry, Match, PathError, Submatch } from "./filesystem/schema"
|
||||
import { Entry, Match } from "./filesystem/schema"
|
||||
export { Entry, Match, Submatch } from "./filesystem/schema"
|
||||
|
||||
export const ReadInput = Schema.Struct({
|
||||
path: RelativePath,
|
||||
@@ -58,10 +58,8 @@ export const Event = {
|
||||
}
|
||||
|
||||
export interface Interface {
|
||||
readonly read: (
|
||||
input: ReadInput,
|
||||
) => Effect.Effect<{ readonly content: Uint8Array; readonly mime: string }, PathError | FSUtil.Error>
|
||||
readonly list: (input?: ListInput) => Effect.Effect<Entry[], PathError | FSUtil.Error>
|
||||
readonly read: (input: ReadInput) => Effect.Effect<{ readonly content: Uint8Array; readonly mime: string }>
|
||||
readonly list: (input?: ListInput) => Effect.Effect<Entry[]>
|
||||
readonly find: (input: FindInput) => Effect.Effect<Entry[]>
|
||||
readonly glob: (input: GlobInput) => Effect.Effect<readonly Entry[]>
|
||||
readonly grep: (input: GrepInput) => Effect.Effect<readonly Match[]>
|
||||
@@ -79,9 +77,9 @@ const baseLayer = Layer.effect(
|
||||
const resolve = Effect.fnUntraced(function* (input?: RelativePath) {
|
||||
const absolute = path.resolve(location.directory, input ?? ".")
|
||||
if (!FSUtil.contains(location.directory, absolute))
|
||||
return yield* new PathError({ path: input ?? ".", reason: "lexical_escape" })
|
||||
const real = yield* fs.realPath(absolute)
|
||||
if (!FSUtil.contains(root, real)) return yield* new PathError({ path: input ?? ".", reason: "symlink_escape" })
|
||||
return yield* Effect.die(new Error("Path escapes the location"))
|
||||
const real = yield* fs.realPath(absolute).pipe(Effect.orDie)
|
||||
if (!FSUtil.contains(root, real)) return yield* Effect.die(new Error("Path escapes the location"))
|
||||
return { absolute, real, directory: location.directory, root }
|
||||
})
|
||||
return Service.of({
|
||||
@@ -90,18 +88,19 @@ const baseLayer = Layer.effect(
|
||||
grep: search.grep,
|
||||
read: Effect.fn("FileSystem.read")(function* (input) {
|
||||
const target = yield* resolve(input.path)
|
||||
const info = yield* fs.stat(target.real)
|
||||
if (info.type !== "File") return yield* new PathError({ path: input.path, reason: "not_file" })
|
||||
const info = yield* fs.stat(target.real).pipe(Effect.orDie)
|
||||
if (info.type !== "File") return yield* Effect.die(new Error("Path is not a file"))
|
||||
return {
|
||||
content: yield* fs.readFile(target.real),
|
||||
content: yield* fs.readFile(target.real).pipe(Effect.orDie),
|
||||
mime: FSUtil.mimeType(target.real),
|
||||
}
|
||||
}),
|
||||
list: Effect.fn("FileSystem.list")(function* (input = {}) {
|
||||
const target = yield* resolve(input.path)
|
||||
const info = yield* fs.stat(target.real)
|
||||
if (info.type !== "Directory") return yield* new PathError({ path: input.path ?? ".", reason: "not_directory" })
|
||||
const info = yield* fs.stat(target.real).pipe(Effect.orDie)
|
||||
if (info.type !== "Directory") return yield* Effect.die(new Error("Path is not a directory"))
|
||||
return yield* fs.readDirectoryEntries(target.real).pipe(
|
||||
Effect.orDie,
|
||||
Effect.map((items) =>
|
||||
items
|
||||
.flatMap((item) => {
|
||||
|
||||
@@ -21,8 +21,3 @@ export class Match extends Schema.Class<Match>("FileSystem.Match")({
|
||||
text: Schema.String,
|
||||
submatches: Schema.Array(Submatch),
|
||||
}) {}
|
||||
|
||||
export class PathError extends Schema.TaggedErrorClass<PathError>()("FileSystem.PathError", {
|
||||
path: Schema.String,
|
||||
reason: Schema.Literals(["lexical_escape", "symlink_escape", "not_file", "not_directory"]),
|
||||
}) {}
|
||||
|
||||
@@ -305,14 +305,7 @@ export const layer = Layer.effect(
|
||||
const llmFailure = failure instanceof LLMError ? failure : undefined
|
||||
if (llmFailure && !publisher.hasProviderError()) {
|
||||
yield* withPublication(publisher.failUnsettledTools("Provider did not return a tool result", true))
|
||||
yield* withPublication(
|
||||
events.publish(SessionEvent.Step.Failed, {
|
||||
sessionID: session.id,
|
||||
timestamp: yield* DateTime.now,
|
||||
assistantMessageID: yield* publisher.startAssistant(),
|
||||
error: { type: "unknown", message: llmFailure.reason.message },
|
||||
}),
|
||||
)
|
||||
yield* withPublication(publisher.failAssistant(llmFailure.reason.message))
|
||||
}
|
||||
if (stream._tag === "Failure" && Cause.hasInterrupts(stream.cause)) yield* FiberSet.clear(toolFibers)
|
||||
const settled = yield* restore(awaitToolFibers(toolFibers)).pipe(Effect.exit)
|
||||
@@ -327,6 +320,8 @@ export const layer = Layer.effect(
|
||||
) {
|
||||
yield* FiberSet.clear(toolFibers)
|
||||
yield* withPublication(publisher.failUnsettledTools("Tool execution interrupted"))
|
||||
if (publisher.hasActiveAssistant())
|
||||
yield* withPublication(publisher.failAssistant("Provider turn interrupted"))
|
||||
}
|
||||
if (settled._tag === "Failure" && !Cause.hasInterrupts(settled.cause)) {
|
||||
const failure = Cause.squash(settled.cause)
|
||||
|
||||
@@ -65,11 +65,14 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input)
|
||||
>()
|
||||
const timestamp = DateTime.now
|
||||
let assistantMessageID: SessionMessage.ID | undefined
|
||||
let assistantActive = false
|
||||
let assistantFailed = false
|
||||
let providerFailed = false
|
||||
|
||||
const startAssistant = Effect.fnUntraced(function* () {
|
||||
if (assistantMessageID !== undefined) return assistantMessageID
|
||||
assistantMessageID = SessionMessage.ID.create()
|
||||
assistantActive = true
|
||||
yield* events.publish(SessionEvent.Step.Started, {
|
||||
...input,
|
||||
assistantMessageID,
|
||||
@@ -190,6 +193,20 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input)
|
||||
yield* flushFragments()
|
||||
})
|
||||
|
||||
const failAssistant = Effect.fnUntraced(function* (message: string) {
|
||||
if (assistantFailed) return
|
||||
yield* flush()
|
||||
const assistantMessageID = yield* startAssistant()
|
||||
assistantActive = false
|
||||
assistantFailed = true
|
||||
yield* events.publish(SessionEvent.Step.Failed, {
|
||||
sessionID: input.sessionID,
|
||||
timestamp: yield* timestamp,
|
||||
assistantMessageID,
|
||||
error: { type: "unknown", message },
|
||||
})
|
||||
})
|
||||
|
||||
const failUnsettledTools = Effect.fn("SessionRunner.failUnsettledTools")(function* (
|
||||
message: string,
|
||||
hostedOnly = false,
|
||||
@@ -375,6 +392,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input)
|
||||
}
|
||||
case "step-finish":
|
||||
yield* flush()
|
||||
assistantActive = false
|
||||
yield* events.publish(SessionEvent.Step.Ended, {
|
||||
sessionID: input.sessionID,
|
||||
timestamp: yield* timestamp,
|
||||
@@ -388,13 +406,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input)
|
||||
return
|
||||
case "provider-error":
|
||||
providerFailed = true
|
||||
yield* flush()
|
||||
yield* events.publish(SessionEvent.Step.Failed, {
|
||||
sessionID: input.sessionID,
|
||||
timestamp: yield* timestamp,
|
||||
assistantMessageID: yield* startAssistant(),
|
||||
error: { type: "unknown", message: event.message },
|
||||
})
|
||||
yield* failAssistant(event.message)
|
||||
return
|
||||
}
|
||||
})
|
||||
@@ -402,10 +414,11 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input)
|
||||
return {
|
||||
publish,
|
||||
flush,
|
||||
failAssistant,
|
||||
failUnsettledTools,
|
||||
hasActiveAssistant: () => assistantActive,
|
||||
hasAssistantStarted: () => assistantMessageID !== undefined,
|
||||
hasProviderError: () => providerFailed,
|
||||
startAssistant,
|
||||
assistantMessageID: assistantMessageIDForTool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,12 +82,21 @@ const assistant = (message: SessionMessage.Assistant, model: Model) => {
|
||||
const result = toolResult(item, sameModel ? (item.provider?.resultMetadata ?? item.provider?.metadata) : undefined)
|
||||
return item.provider?.executed === true && result ? [call, result] : [call]
|
||||
})
|
||||
const meaningful = content.filter((part) => {
|
||||
if (part.type === "text") return part.text !== ""
|
||||
if (part.type !== "reasoning") return true
|
||||
return part.text !== "" || (part.providerMetadata !== undefined && Object.keys(part.providerMetadata).length > 0)
|
||||
})
|
||||
const results = message.content
|
||||
.filter((item): item is SessionMessage.AssistantTool => item.type === "tool" && item.provider?.executed !== true)
|
||||
.map((item) => toolResult(item, sameModel ? (item.provider?.resultMetadata ?? item.provider?.metadata) : undefined))
|
||||
.filter((message) => message !== undefined)
|
||||
.map(Message.tool)
|
||||
return [Message.make({ id: message.id, role: "assistant", content, metadata: message.metadata }), ...results]
|
||||
if (meaningful.length === 0) return results
|
||||
return [
|
||||
Message.make({ id: message.id, role: "assistant", content: meaningful, metadata: message.metadata }),
|
||||
...results,
|
||||
]
|
||||
}
|
||||
|
||||
function toLLMMessage(message: SessionMessage.Message, model: Model): Message[] {
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { Effect, Stream } from "effect"
|
||||
import { HttpClientResponse } from "effect/unstable/http"
|
||||
|
||||
export const collectBoundedResponseBody = (
|
||||
response: HttpClientResponse.HttpClientResponse,
|
||||
maximumBytes: number,
|
||||
tooLarge: () => Error,
|
||||
) =>
|
||||
Effect.gen(function* () {
|
||||
const contentLength = response.headers["content-length"]
|
||||
const parsedSize = contentLength ? Number.parseInt(contentLength, 10) : undefined
|
||||
const declaredSize =
|
||||
parsedSize !== undefined && Number.isSafeInteger(parsedSize) && parsedSize >= 0 ? parsedSize : undefined
|
||||
if (declaredSize !== undefined && declaredSize > maximumBytes) return yield* Effect.fail(tooLarge())
|
||||
let body = Buffer.allocUnsafe(Math.min(maximumBytes, declaredSize || 64 * 1024))
|
||||
let size = 0
|
||||
yield* Stream.runForEach(response.stream, (chunk) => {
|
||||
if (chunk.byteLength === 0) return Effect.void
|
||||
if (size + chunk.byteLength > maximumBytes) return Effect.fail(tooLarge())
|
||||
if (size + chunk.byteLength > body.byteLength) {
|
||||
const grown = Buffer.allocUnsafe(Math.min(maximumBytes, Math.max(size + chunk.byteLength, body.byteLength * 2)))
|
||||
body.copy(grown, 0, 0, size)
|
||||
body = grown
|
||||
}
|
||||
body.set(chunk, size)
|
||||
size += chunk.byteLength
|
||||
return Effect.void
|
||||
})
|
||||
return body.subarray(0, size)
|
||||
})
|
||||
@@ -13,23 +13,61 @@ export const MAX_MEDIA_INGEST_BYTES = 20 * 1024 * 1024
|
||||
const MAX_LINE_LENGTH = 2_000
|
||||
const MAX_LINE_SUFFIX = `... (line truncated to ${MAX_LINE_LENGTH} chars)`
|
||||
|
||||
export class BinaryFileError extends Error {
|
||||
constructor(readonly resource: string) {
|
||||
super(`Cannot read binary file: ${resource}`)
|
||||
this.name = "BinaryFileError"
|
||||
export class BinaryFileError extends Schema.TaggedErrorClass<BinaryFileError>()("ReadTool.BinaryFileError", {
|
||||
resource: Schema.String,
|
||||
}) {
|
||||
override get message() {
|
||||
return `Cannot read binary file: ${this.resource}`
|
||||
}
|
||||
}
|
||||
|
||||
export class MediaIngestLimitError extends Error {
|
||||
constructor(
|
||||
readonly resource: string,
|
||||
readonly maximumBytes: number,
|
||||
) {
|
||||
super(`Media exceeds ${maximumBytes} byte ingestion limit: ${resource}`)
|
||||
this.name = "MediaIngestLimitError"
|
||||
export class MediaIngestLimitError extends Schema.TaggedErrorClass<MediaIngestLimitError>()(
|
||||
"ReadTool.MediaIngestLimitError",
|
||||
{
|
||||
resource: Schema.String,
|
||||
maximumBytes: Schema.Number,
|
||||
},
|
||||
) {
|
||||
override get message() {
|
||||
return `Media exceeds ${this.maximumBytes} byte ingestion limit: ${this.resource}`
|
||||
}
|
||||
}
|
||||
|
||||
export class MalformedUtf8Error extends Schema.TaggedErrorClass<MalformedUtf8Error>()("ReadTool.MalformedUtf8Error", {
|
||||
resource: Schema.String,
|
||||
}) {
|
||||
override get message() {
|
||||
return `File is not valid UTF-8: ${this.resource}`
|
||||
}
|
||||
}
|
||||
|
||||
export class OffsetOutOfRangeError extends Schema.TaggedErrorClass<OffsetOutOfRangeError>()(
|
||||
"ReadTool.OffsetOutOfRangeError",
|
||||
{ offset: Schema.Number },
|
||||
) {
|
||||
override get message() {
|
||||
return `Offset ${this.offset} is out of range`
|
||||
}
|
||||
}
|
||||
|
||||
export class PathKindError extends Schema.TaggedErrorClass<PathKindError>()("ReadTool.PathKindError", {
|
||||
resource: Schema.String,
|
||||
expected: Schema.Literals(["a file", "a file or directory"]),
|
||||
}) {
|
||||
override get message() {
|
||||
return `Path is not ${this.expected}: ${this.resource}`
|
||||
}
|
||||
}
|
||||
|
||||
export type InspectError = FSUtil.Error | PathKindError
|
||||
export type ReadError =
|
||||
| FSUtil.Error
|
||||
| BinaryFileError
|
||||
| MediaIngestLimitError
|
||||
| MalformedUtf8Error
|
||||
| OffsetOutOfRangeError
|
||||
| PathKindError
|
||||
|
||||
export const PageInput = Schema.Struct({
|
||||
offset: PositiveInt.pipe(Schema.optional),
|
||||
limit: PositiveInt.check(Schema.isLessThanOrEqualTo(MAX_READ_LINES)).pipe(Schema.optional),
|
||||
@@ -52,13 +90,13 @@ export class ListPage extends Schema.Class<ListPage>("ReadTool.ListPage")({
|
||||
}) {}
|
||||
|
||||
export interface Interface {
|
||||
readonly inspect: (path: AbsolutePath) => Effect.Effect<"file" | "directory">
|
||||
readonly inspect: (path: AbsolutePath) => Effect.Effect<"file" | "directory", InspectError>
|
||||
readonly read: (
|
||||
path: AbsolutePath,
|
||||
resource: string,
|
||||
page?: PageInput,
|
||||
) => Effect.Effect<FileSystem.Content | TextPage>
|
||||
readonly list: (path: AbsolutePath, page?: PageInput) => Effect.Effect<ListPage>
|
||||
) => Effect.Effect<FileSystem.Content | TextPage, ReadError>
|
||||
readonly list: (path: AbsolutePath, page?: PageInput) => Effect.Effect<ListPage, FSUtil.Error>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/ReadToolFileSystem") {}
|
||||
@@ -111,11 +149,21 @@ const binary = (resource: string, bytes: Uint8Array) => {
|
||||
}
|
||||
return nonPrintable / bytes.length > 0.3
|
||||
}
|
||||
const decodeUtf8 = (resource: string, decoder: TextDecoder, bytes?: Uint8Array) =>
|
||||
Effect.try({
|
||||
try: () => decoder.decode(bytes, { stream: bytes !== undefined }),
|
||||
catch: (error) => {
|
||||
if (error instanceof TypeError) return new MalformedUtf8Error({ resource })
|
||||
throw error
|
||||
},
|
||||
})
|
||||
const decodeChunk = (resource: string, decoder: TextDecoder, bytes: Uint8Array) =>
|
||||
bytes.includes(0) ? Effect.fail(new BinaryFileError({ resource })) : decodeUtf8(resource, decoder, bytes)
|
||||
|
||||
export const inspect = Effect.fn("ReadTool.inspect")(function* (fs: FSUtil.Interface, input: string) {
|
||||
const info = yield* fs.stat(input).pipe(Effect.orDie)
|
||||
const info = yield* fs.stat(input)
|
||||
const type = info.type === "File" ? "file" : info.type === "Directory" ? "directory" : undefined
|
||||
if (!type) return yield* Effect.die(new Error("Path is not a file or directory"))
|
||||
if (!type) return yield* Effect.fail(new PathKindError({ resource: input, expected: "a file or directory" }))
|
||||
return type
|
||||
})
|
||||
|
||||
@@ -125,32 +173,30 @@ export const read = Effect.fn("ReadTool.read")(function* (
|
||||
resource: string,
|
||||
page: PageInput = {},
|
||||
) {
|
||||
const real = yield* fs.realPath(input).pipe(Effect.orDie)
|
||||
const real = yield* fs.realPath(input)
|
||||
return yield* Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const file = yield* fs.open(real, { flag: "r" }).pipe(Effect.orDie)
|
||||
const info = yield* file.stat.pipe(Effect.orDie)
|
||||
if (info.type !== "File") return yield* Effect.die(new Error("Path is not a file"))
|
||||
const file = yield* fs.open(real, { flag: "r" })
|
||||
const info = yield* file.stat
|
||||
if (info.type !== "File") return yield* Effect.fail(new PathKindError({ resource, expected: "a file" }))
|
||||
const first = Option.getOrElse(
|
||||
yield* file.readAlloc(Math.min(64 * 1024, Number(info.size) || 4 * 1024)).pipe(Effect.orDie),
|
||||
yield* file.readAlloc(Math.min(64 * 1024, Number(info.size) || 4 * 1024)),
|
||||
() => new Uint8Array(),
|
||||
)
|
||||
const mime = imageMime(first)
|
||||
if (mime) {
|
||||
if (info.size > MAX_MEDIA_INGEST_BYTES)
|
||||
return yield* Effect.die(new MediaIngestLimitError(resource, MAX_MEDIA_INGEST_BYTES))
|
||||
return yield* Effect.fail(new MediaIngestLimitError({ resource, maximumBytes: MAX_MEDIA_INGEST_BYTES }))
|
||||
const chunks = [first]
|
||||
let total = first.length
|
||||
while (total <= MAX_MEDIA_INGEST_BYTES) {
|
||||
const chunk = yield* file
|
||||
.readAlloc(Math.min(64 * 1024, MAX_MEDIA_INGEST_BYTES + 1 - total))
|
||||
.pipe(Effect.orDie)
|
||||
const chunk = yield* file.readAlloc(Math.min(64 * 1024, MAX_MEDIA_INGEST_BYTES + 1 - total))
|
||||
if (Option.isNone(chunk)) break
|
||||
chunks.push(chunk.value)
|
||||
total += chunk.value.length
|
||||
}
|
||||
if (total > MAX_MEDIA_INGEST_BYTES)
|
||||
return yield* Effect.die(new MediaIngestLimitError(resource, MAX_MEDIA_INGEST_BYTES))
|
||||
return yield* Effect.fail(new MediaIngestLimitError({ resource, maximumBytes: MAX_MEDIA_INGEST_BYTES }))
|
||||
return {
|
||||
uri: pathToFileURL(real).href,
|
||||
name: path.basename(real),
|
||||
@@ -162,19 +208,19 @@ export const read = Effect.fn("ReadTool.read")(function* (
|
||||
mime,
|
||||
}
|
||||
}
|
||||
if (startsWith(first, [0x25, 0x50, 0x44, 0x46]) || binary(resource, first))
|
||||
return yield* Effect.die(new BinaryFileError(resource))
|
||||
if (startsWith(first, [0x25, 0x50, 0x44, 0x46]) || extensions.has(path.extname(resource).toLowerCase()))
|
||||
return yield* Effect.fail(new BinaryFileError({ resource }))
|
||||
const paged = info.size > MAX_READ_BYTES || page.offset !== undefined || page.limit !== undefined
|
||||
if (!paged) {
|
||||
if (binary(resource, first)) return yield* Effect.fail(new BinaryFileError({ resource }))
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true })
|
||||
const text = [yield* Effect.sync(() => decoder.decode(first, { stream: true }))]
|
||||
const text = [yield* decodeUtf8(resource, decoder, first)]
|
||||
while (true) {
|
||||
const chunk = yield* file.readAlloc(64 * 1024).pipe(Effect.orDie)
|
||||
const chunk = yield* file.readAlloc(64 * 1024)
|
||||
if (Option.isNone(chunk)) break
|
||||
if (chunk.value.includes(0)) return yield* Effect.die(new BinaryFileError(resource))
|
||||
text.push(yield* Effect.sync(() => decoder.decode(chunk.value, { stream: true })))
|
||||
text.push(yield* decodeChunk(resource, decoder, chunk.value))
|
||||
}
|
||||
text.push(yield* Effect.sync(() => decoder.decode()))
|
||||
text.push(yield* decodeUtf8(resource, decoder))
|
||||
return {
|
||||
uri: pathToFileURL(real).href,
|
||||
name: path.basename(real),
|
||||
@@ -191,34 +237,29 @@ export const read = Effect.fn("ReadTool.read")(function* (
|
||||
let discard = false
|
||||
let line = 1
|
||||
let bytes = 0
|
||||
let found = false
|
||||
let truncated = false
|
||||
let next: number | undefined
|
||||
const append = (input: string) => {
|
||||
if (line < offset) {
|
||||
line++
|
||||
return
|
||||
return true
|
||||
}
|
||||
if (lines.length >= limit || bytes >= MAX_READ_BYTES) {
|
||||
truncated = true
|
||||
next ??= line++
|
||||
return
|
||||
next = line
|
||||
return false
|
||||
}
|
||||
found = true
|
||||
const text = input.length > MAX_LINE_LENGTH ? input.slice(0, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX : input
|
||||
const size = Buffer.byteLength(text, "utf-8") + (lines.length > 0 ? 1 : 0)
|
||||
if (bytes + size > MAX_READ_BYTES) {
|
||||
truncated = true
|
||||
next ??= line++
|
||||
return
|
||||
next = line
|
||||
return false
|
||||
}
|
||||
lines.push(text)
|
||||
bytes += size
|
||||
line++
|
||||
return true
|
||||
}
|
||||
const consume = (chunk: Uint8Array) => {
|
||||
if (chunk.includes(0)) throw new BinaryFileError(resource)
|
||||
let text = decoder.decode(chunk, { stream: true })
|
||||
const consume = (input: string) => {
|
||||
let text = input
|
||||
while (true) {
|
||||
const index = text.indexOf("\n")
|
||||
if (index === -1) {
|
||||
@@ -235,25 +276,44 @@ export const read = Effect.fn("ReadTool.read")(function* (
|
||||
pending = ""
|
||||
discard = false
|
||||
text = text.slice(index + 1)
|
||||
append(current.endsWith("\r") ? current.slice(0, -1) : current)
|
||||
if (!append(current.endsWith("\r") ? current.slice(0, -1) : current)) return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
yield* Effect.sync(() => consume(first))
|
||||
while (true) {
|
||||
const chunk = yield* file.readAlloc(64 * 1024).pipe(Effect.orDie)
|
||||
const consumeChunk = Effect.fnUntraced(function* (chunk: Uint8Array) {
|
||||
let start = 0
|
||||
while (start < chunk.length) {
|
||||
if (lines.length >= limit || bytes >= MAX_READ_BYTES) {
|
||||
next = line
|
||||
return false
|
||||
}
|
||||
const newline = chunk.indexOf(10, start)
|
||||
const end = newline === -1 ? chunk.length : newline + 1
|
||||
const segment = chunk.subarray(start, end)
|
||||
if (binary(resource, segment)) return yield* Effect.fail(new BinaryFileError({ resource }))
|
||||
if (!consume(yield* decodeUtf8(resource, decoder, segment))) return false
|
||||
start = end
|
||||
}
|
||||
return true
|
||||
})
|
||||
let done = !(yield* consumeChunk(first))
|
||||
while (!done) {
|
||||
const chunk = yield* file.readAlloc(64 * 1024)
|
||||
if (Option.isNone(chunk)) break
|
||||
yield* Effect.sync(() => consume(chunk.value))
|
||||
done = !(yield* consumeChunk(chunk.value))
|
||||
}
|
||||
const tail = yield* Effect.sync(() => decoder.decode())
|
||||
if (!discard) pending += tail
|
||||
if (pending) append(pending.endsWith("\r") ? pending.slice(0, -1) : pending)
|
||||
if (!found && offset !== 1) return yield* Effect.die(new Error(`Offset ${offset} is out of range`))
|
||||
if (!done) {
|
||||
const tail = yield* decodeUtf8(resource, decoder)
|
||||
if (!discard) pending += tail
|
||||
if (pending) append(pending.endsWith("\r") ? pending.slice(0, -1) : pending)
|
||||
}
|
||||
if (lines.length === 0 && offset !== 1) return yield* Effect.fail(new OffsetOutOfRangeError({ offset }))
|
||||
return new TextPage({
|
||||
type: "text-page",
|
||||
content: lines.join("\n"),
|
||||
mime: FSUtil.mimeType(real),
|
||||
offset,
|
||||
truncated,
|
||||
truncated: next !== undefined,
|
||||
...(next === undefined ? {} : { next }),
|
||||
})
|
||||
}),
|
||||
@@ -261,8 +321,8 @@ export const read = Effect.fn("ReadTool.read")(function* (
|
||||
})
|
||||
|
||||
export const list = Effect.fn("ReadTool.list")(function* (fs: FSUtil.Interface, input: string, page: PageInput = {}) {
|
||||
const real = yield* fs.realPath(input).pipe(Effect.orDie)
|
||||
const items = yield* fs.readDirectoryEntries(real).pipe(Effect.orDie)
|
||||
const real = yield* fs.realPath(input)
|
||||
const items = yield* fs.readDirectoryEntries(real)
|
||||
const offset = page.offset ?? 1
|
||||
const limit = Math.min(page.limit ?? MAX_READ_LINES, MAX_READ_LINES)
|
||||
const entries = yield* Effect.forEach(
|
||||
|
||||
@@ -57,8 +57,8 @@ export const layer = Layer.effectDiscard(
|
||||
const selected = path.isAbsolute(input.path) ? path.dirname(absolute) : location.directory
|
||||
if (!path.isAbsolute(input.path) && !FSUtil.contains(location.directory, absolute))
|
||||
return yield* Effect.die(new Error("Path escapes the allowed read root"))
|
||||
const real = yield* fs.realPath(absolute).pipe(Effect.orDie)
|
||||
const root = yield* fs.realPath(selected).pipe(Effect.orDie)
|
||||
const real = yield* fs.realPath(absolute)
|
||||
const root = yield* fs.realPath(selected)
|
||||
if (!FSUtil.contains(root, real))
|
||||
return yield* Effect.die(new Error("Path escapes the allowed read root"))
|
||||
const resource = path.relative(root, real).replaceAll("\\", "/") || "."
|
||||
@@ -83,7 +83,7 @@ export const layer = Layer.effectDiscard(
|
||||
.pipe(Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)))
|
||||
}
|
||||
if ("encoding" in content && content.encoding === "base64")
|
||||
return yield* Effect.fail(new ReadToolFileSystem.BinaryFileError(resource))
|
||||
return yield* Effect.fail(new ReadToolFileSystem.BinaryFileError({ resource }))
|
||||
return content
|
||||
}).pipe(
|
||||
Effect.mapError((error) => {
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
export * as WebFetchTool from "./webfetch"
|
||||
|
||||
import { ToolFailure } from "@opencode-ai/llm"
|
||||
import { Duration, Effect, Layer, Schema, Stream } from "effect"
|
||||
import { Duration, Effect, Layer, Schema } from "effect"
|
||||
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import { Parser } from "htmlparser2"
|
||||
import TurndownService from "turndown"
|
||||
import { PermissionV2 } from "../permission"
|
||||
import { collectBoundedResponseBody } from "./http-body"
|
||||
import { Tool } from "./tool"
|
||||
import { Tools } from "./tools"
|
||||
|
||||
@@ -86,24 +87,11 @@ const execute = (http: HttpClient.HttpClient, url: string, format: Format, userA
|
||||
http.execute(request(url, format, userAgent)).pipe(Effect.flatMap(HttpClientResponse.filterStatusOk))
|
||||
|
||||
const collectBody = (response: HttpClientResponse.HttpClientResponse) =>
|
||||
Effect.gen(function* () {
|
||||
const contentLength = response.headers["content-length"]
|
||||
if (contentLength && Number.parseInt(contentLength, 10) > MAX_RESPONSE_BYTES) {
|
||||
return yield* Effect.fail(new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`))
|
||||
}
|
||||
const chunks: Uint8Array[] = []
|
||||
let size = 0
|
||||
yield* Stream.runForEach(response.stream, (chunk) =>
|
||||
Effect.gen(function* () {
|
||||
size += chunk.byteLength
|
||||
if (size > MAX_RESPONSE_BYTES)
|
||||
return yield* Effect.fail(new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`))
|
||||
chunks.push(chunk)
|
||||
return undefined
|
||||
}),
|
||||
)
|
||||
return Buffer.concat(chunks, size)
|
||||
})
|
||||
collectBoundedResponseBody(
|
||||
response,
|
||||
MAX_RESPONSE_BYTES,
|
||||
() => new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`),
|
||||
)
|
||||
|
||||
const mimeFrom = (contentType: string) => contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""
|
||||
const isImageAttachment = (mime: string) =>
|
||||
@@ -171,12 +159,16 @@ export const layer = Layer.effectDiscard(
|
||||
orElse: () => Effect.fail(new Error("Request timed out")),
|
||||
}),
|
||||
)
|
||||
const content = convert(new TextDecoder().decode(body), contentType, input.format)
|
||||
const content = new TextDecoder().decode(body)
|
||||
const output = yield* Effect.try({
|
||||
try: () => convert(content, contentType, input.format),
|
||||
catch: (error) => error,
|
||||
})
|
||||
return {
|
||||
url: input.url,
|
||||
contentType,
|
||||
format: input.format,
|
||||
output: content,
|
||||
output,
|
||||
}
|
||||
}).pipe(Effect.mapError(() => new ToolFailure({ message: `Unable to fetch ${input.url}` }))),
|
||||
}),
|
||||
|
||||
@@ -9,6 +9,7 @@ import { PositiveInt } from "../schema"
|
||||
import { PermissionV2 } from "../permission"
|
||||
import { Tool } from "./tool"
|
||||
import { Tools } from "./tools"
|
||||
import { collectBoundedResponseBody } from "./http-body"
|
||||
import { checksum } from "../util/encode"
|
||||
|
||||
export const name = "websearch"
|
||||
@@ -164,10 +165,12 @@ const callMcp = <F extends Schema.Struct.Fields>(
|
||||
)
|
||||
return yield* Effect.gen(function* () {
|
||||
const response = yield* HttpClient.filterStatusOk(http).execute(request)
|
||||
const body = yield* response.text
|
||||
if (Buffer.byteLength(body, "utf8") > MAX_RESPONSE_BYTES)
|
||||
return yield* Effect.fail(new Error(`${tool} response exceeded ${MAX_RESPONSE_BYTES} bytes`))
|
||||
return yield* parseResponse(body)
|
||||
const body = yield* collectBoundedResponseBody(
|
||||
response,
|
||||
MAX_RESPONSE_BYTES,
|
||||
() => new Error(`${tool} response exceeded ${MAX_RESPONSE_BYTES} bytes`),
|
||||
)
|
||||
return yield* parseResponse(body.toString("utf8"))
|
||||
}).pipe(
|
||||
Effect.timeoutOrElse({
|
||||
duration: Duration.seconds(25),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import fs from "fs/promises"
|
||||
import path from "path"
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Cause, Effect, Exit, Layer } from "effect"
|
||||
import { Effect, Exit, Layer } from "effect"
|
||||
import { FileSystem } from "@opencode-ai/core/filesystem"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { Location } from "@opencode-ai/core/location"
|
||||
@@ -31,13 +31,6 @@ const withTmp = <A, E, R>(f: (directory: string) => Effect.Effect<A, E, R>) =>
|
||||
).pipe(Effect.flatMap((tmp) => f(tmp.path)))
|
||||
|
||||
describe("FileSystem", () => {
|
||||
const expectFail = (exit: Exit.Exit<unknown, unknown>) => {
|
||||
expect(Exit.isFailure(exit)).toBe(true)
|
||||
if (Exit.isSuccess(exit)) return
|
||||
expect(Cause.hasFails(exit.cause)).toBe(true)
|
||||
expect(Cause.hasDies(exit.cause)).toBe(false)
|
||||
}
|
||||
|
||||
it.live("reads text and binary files", () =>
|
||||
withTmp((directory) =>
|
||||
Effect.gen(function* () {
|
||||
@@ -67,39 +60,13 @@ describe("FileSystem", () => {
|
||||
),
|
||||
)
|
||||
|
||||
it.live("fails for missing paths", () =>
|
||||
it.live("rejects lexical escapes", () =>
|
||||
withTmp((directory) =>
|
||||
Effect.gen(function* () {
|
||||
const exit = yield* (yield* FileSystem.Service)
|
||||
.read({ path: RelativePath.make("missing.txt") })
|
||||
const result = yield* (yield* FileSystem.Service)
|
||||
.read({ path: RelativePath.make("../outside.txt") })
|
||||
.pipe(Effect.exit)
|
||||
expectFail(exit)
|
||||
}).pipe(provide(directory)),
|
||||
),
|
||||
)
|
||||
|
||||
it.live("fails for wrong path kinds", () =>
|
||||
withTmp((directory) =>
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.promise(() => fs.mkdir(path.join(directory, "src")))
|
||||
yield* Effect.promise(() => fs.writeFile(path.join(directory, "README.md"), "# Test"))
|
||||
const service = yield* FileSystem.Service
|
||||
expectFail(yield* service.read({ path: RelativePath.make("src") }).pipe(Effect.exit))
|
||||
expectFail(yield* service.list({ path: RelativePath.make("README.md") }).pipe(Effect.exit))
|
||||
}).pipe(provide(directory)),
|
||||
),
|
||||
)
|
||||
|
||||
it.live("fails for lexical and symlink escapes", () =>
|
||||
withTmp((directory) =>
|
||||
Effect.gen(function* () {
|
||||
const outside = path.join(directory, "..", "outside.txt")
|
||||
yield* Effect.promise(() => fs.writeFile(outside, "outside"))
|
||||
yield* Effect.promise(() => fs.symlink(outside, path.join(directory, "linked.txt")))
|
||||
const service = yield* FileSystem.Service
|
||||
expectFail(yield* service.read({ path: RelativePath.make("../outside.txt") }).pipe(Effect.exit))
|
||||
expectFail(yield* service.read({ path: RelativePath.make("linked.txt") }).pipe(Effect.exit))
|
||||
yield* Effect.promise(() => fs.rm(outside))
|
||||
expect(Exit.isFailure(result)).toBe(true)
|
||||
}).pipe(provide(directory)),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -14,6 +14,39 @@ const id = (value: string) => SessionMessage.ID.make(`msg_${value}`)
|
||||
const model = Model.make({ id: "model", provider: "provider", route: OpenAIChat.route })
|
||||
|
||||
describe("toLLMMessages", () => {
|
||||
test("omits empty assistant turns", () => {
|
||||
const assistant = (value: string, content: SessionMessage.Assistant["content"]) =>
|
||||
new SessionMessage.Assistant({
|
||||
id: id(value),
|
||||
type: "assistant",
|
||||
agent: "build",
|
||||
model: { id: ModelV2.ID.make("model"), providerID: ProviderV2.ID.make("provider") },
|
||||
content,
|
||||
time: { created, completed: created },
|
||||
})
|
||||
const messages = toLLMMessages(
|
||||
[
|
||||
assistant("empty", []),
|
||||
assistant("empty-text", [new SessionMessage.AssistantText({ type: "text", id: "empty", text: "" })]),
|
||||
assistant("empty-reasoning", [
|
||||
new SessionMessage.AssistantReasoning({ type: "reasoning", id: "empty-reasoning", text: "" }),
|
||||
]),
|
||||
assistant("text", [new SessionMessage.AssistantText({ type: "text", id: "text", text: "Partial" })]),
|
||||
assistant("reasoning", [
|
||||
new SessionMessage.AssistantReasoning({
|
||||
type: "reasoning",
|
||||
id: "reasoning",
|
||||
text: "",
|
||||
providerMetadata: { anthropic: { signature: "sig_1" } },
|
||||
}),
|
||||
]),
|
||||
],
|
||||
model,
|
||||
)
|
||||
|
||||
expect(messages.map((message) => message.id)).toEqual([id("text"), id("reasoning")])
|
||||
})
|
||||
|
||||
test("maps every top-level V2 Session message type", () => {
|
||||
const file = new FileAttachment({ uri: "data:image/png;base64,aGVsbG8=", mime: "image/png", name: "hello.png" })
|
||||
const messages = toLLMMessages(
|
||||
|
||||
@@ -547,6 +547,8 @@ const verifyPartialFlushOnInterruption = (kind: FragmentKind) =>
|
||||
{ type: "user", text: prompt },
|
||||
{
|
||||
type: "assistant",
|
||||
finish: "error",
|
||||
error: { type: "unknown", message: "Provider turn interrupted" },
|
||||
content: [
|
||||
kind === "tool input"
|
||||
? { type: "tool", id: fragmentID(kind, "interrupted"), state: { status: "error" } }
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { NodeFileSystem } from "@effect/platform-node"
|
||||
import path from "path"
|
||||
import { Effect, FileSystem, Layer } from "effect"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { ReadToolFileSystem } from "@opencode-ai/core/tool/read-filesystem"
|
||||
import { testEffect } from "./lib/effect"
|
||||
|
||||
const it = testEffect(FSUtil.layer.pipe(Layer.provideMerge(NodeFileSystem.layer)))
|
||||
const fixture = Effect.gen(function* () {
|
||||
const fs = yield* FSUtil.Service
|
||||
const files = yield* FileSystem.FileSystem
|
||||
const directory = yield* files.makeTempDirectoryScoped()
|
||||
return { fs, files, directory }
|
||||
})
|
||||
|
||||
describe("ReadToolFileSystem", () => {
|
||||
it.effect("fails with a typed filesystem error when a resolved file disappears", () =>
|
||||
Effect.gen(function* () {
|
||||
const { fs, directory } = yield* fixture
|
||||
const file = path.join(directory, "missing.txt")
|
||||
|
||||
const error = yield* ReadToolFileSystem.read(fs, file, "missing.txt").pipe(Effect.flip)
|
||||
|
||||
expect(error).toMatchObject({ _tag: "PlatformError" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails when a file becomes the wrong path kind", () =>
|
||||
Effect.gen(function* () {
|
||||
const { fs, directory } = yield* fixture
|
||||
|
||||
const error = yield* ReadToolFileSystem.read(fs, directory, "folder").pipe(Effect.flip)
|
||||
|
||||
expect(error).toBeInstanceOf(ReadToolFileSystem.PathKindError)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("fails with a typed filesystem error when directory listing fails", () =>
|
||||
Effect.gen(function* () {
|
||||
const { fs, files, directory } = yield* fixture
|
||||
const file = path.join(directory, "file.txt")
|
||||
yield* files.writeFileString(file, "hello")
|
||||
|
||||
const error = yield* ReadToolFileSystem.list(fs, file).pipe(Effect.flip)
|
||||
|
||||
expect(error).toBeInstanceOf(FSUtil.FileSystemError)
|
||||
if (error instanceof FSUtil.FileSystemError) expect(error.method).toBe("readDirectoryEntries")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("reports binary and malformed UTF-8 content as typed errors", () =>
|
||||
Effect.gen(function* () {
|
||||
const { fs, files, directory } = yield* fixture
|
||||
const binary = path.join(directory, "archive.dat")
|
||||
const malformed = path.join(directory, "malformed.txt")
|
||||
yield* files.writeFile(binary, Uint8Array.of(0, 1, 2, 3))
|
||||
const malformedContent = new Uint8Array(64 * 1024 + 1).fill(97)
|
||||
malformedContent[64 * 1024] = 0x80
|
||||
yield* files.writeFile(malformed, malformedContent)
|
||||
|
||||
const binaryError = yield* ReadToolFileSystem.read(fs, binary, "archive.dat").pipe(Effect.flip)
|
||||
const malformedError = yield* ReadToolFileSystem.read(fs, malformed, "malformed.txt").pipe(Effect.flip)
|
||||
|
||||
expect(binaryError).toBeInstanceOf(ReadToolFileSystem.BinaryFileError)
|
||||
expect(binaryError.message).toBe("Cannot read binary file: archive.dat")
|
||||
expect(malformedError).toBeInstanceOf(ReadToolFileSystem.MalformedUtf8Error)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("reports out-of-range pagination as a typed error", () =>
|
||||
Effect.gen(function* () {
|
||||
const { fs, files, directory } = yield* fixture
|
||||
const file = path.join(directory, "short.txt")
|
||||
yield* files.writeFileString(file, "one\n")
|
||||
|
||||
const error = yield* ReadToolFileSystem.read(fs, file, "short.txt", { offset: 2 }).pipe(Effect.flip)
|
||||
|
||||
expect(error).toBeInstanceOf(ReadToolFileSystem.OffsetOutOfRangeError)
|
||||
expect(error.message).toBe("Offset 2 is out of range")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("stops reading after the requested page is complete", () =>
|
||||
Effect.gen(function* () {
|
||||
const { fs, files, directory } = yield* fixture
|
||||
const prefix = new TextEncoder().encode("one\n")
|
||||
for (const [name, trailing] of [
|
||||
["malformed.txt", 0x80],
|
||||
["nul.txt", 0],
|
||||
] as const) {
|
||||
const file = path.join(directory, name)
|
||||
yield* files.writeFile(file, Uint8Array.from([...prefix, trailing]))
|
||||
|
||||
const result = yield* ReadToolFileSystem.read(fs, file, name, { limit: 1 })
|
||||
|
||||
expect(result).toMatchObject({ type: "text-page", content: "one", truncated: true, next: 2 })
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves the media ingestion limit message", () =>
|
||||
Effect.gen(function* () {
|
||||
const { fs, files, directory } = yield* fixture
|
||||
const file = path.join(directory, "oversized.png")
|
||||
yield* files.writeFile(file, Uint8Array.of(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a))
|
||||
yield* files.truncate(file, ReadToolFileSystem.MAX_MEDIA_INGEST_BYTES + 1)
|
||||
|
||||
const error = yield* ReadToolFileSystem.read(fs, file, "oversized.png").pipe(Effect.flip)
|
||||
|
||||
expect(error).toBeInstanceOf(ReadToolFileSystem.MediaIngestLimitError)
|
||||
expect(error.message).toBe(
|
||||
`Media exceeds ${ReadToolFileSystem.MAX_MEDIA_INGEST_BYTES} byte ingestion limit: oversized.png`,
|
||||
)
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -1,5 +1,5 @@
|
||||
import { beforeEach, describe, expect } from "bun:test"
|
||||
import { Effect, Exit, Layer } from "effect"
|
||||
import { Effect, Exit, Layer, PlatformError } from "effect"
|
||||
import { Config } from "@opencode-ai/core/config"
|
||||
import { ConfigAttachments } from "@opencode-ai/core/config/attachments"
|
||||
import { FileSystem } from "@opencode-ai/core/filesystem"
|
||||
@@ -18,6 +18,8 @@ import { testEffect } from "./lib/effect"
|
||||
import { toolIdentity, executeTool, settleTool, toolDefinitions } from "./lib/tool"
|
||||
|
||||
const assertions: PermissionV2.AssertInput[] = []
|
||||
const missingPath = "__missing_read_target__.txt"
|
||||
const missingAbsolutePath = `${process.cwd()}/${missingPath}`
|
||||
const readCalls: {
|
||||
input: AbsolutePath
|
||||
page: ReadToolFileSystem.PageInput
|
||||
@@ -32,7 +34,7 @@ let readResult: FileSystem.Content | ReadToolFileSystem.TextPage = {
|
||||
encoding: "utf8",
|
||||
mime: "text/plain",
|
||||
}
|
||||
let readFailure: unknown
|
||||
let readFailure: ReadToolFileSystem.ReadError | undefined
|
||||
let configEntries: Config.Entry[] = []
|
||||
const reader = Layer.succeed(
|
||||
ReadToolFileSystem.Service,
|
||||
@@ -40,7 +42,7 @@ const reader = Layer.succeed(
|
||||
inspect: () => (resolveFailure === undefined ? Effect.succeed(resolvedType) : Effect.die(resolveFailure)),
|
||||
read: (input, _resource, page = {}) => {
|
||||
readCalls.push({ input, page })
|
||||
if (readFailure !== undefined) return Effect.die(readFailure)
|
||||
if (readFailure !== undefined) return Effect.fail(readFailure)
|
||||
return Effect.succeed(readResult)
|
||||
},
|
||||
list: (_path, input = {}) =>
|
||||
@@ -70,7 +72,24 @@ const config = Layer.succeed(Config.Service, Config.Service.of({ entries: () =>
|
||||
const image = Image.layer.pipe(Layer.provide(config))
|
||||
const testFileSystem = Layer.effect(
|
||||
FSUtil.Service,
|
||||
FSUtil.Service.use((fs) => Effect.succeed(FSUtil.Service.of({ ...fs, realPath: (path) => Effect.succeed(path) }))),
|
||||
FSUtil.Service.use((fs) =>
|
||||
Effect.succeed(
|
||||
FSUtil.Service.of({
|
||||
...fs,
|
||||
realPath: (path) =>
|
||||
path === missingAbsolutePath
|
||||
? Effect.fail(
|
||||
PlatformError.systemError({
|
||||
_tag: "NotFound",
|
||||
module: "FileSystem",
|
||||
method: "realPath",
|
||||
pathOrDescriptor: path,
|
||||
}),
|
||||
)
|
||||
: Effect.succeed(path),
|
||||
}),
|
||||
),
|
||||
),
|
||||
).pipe(Layer.provide(FSUtil.defaultLayer))
|
||||
const infrastructure = Layer.mergeAll(
|
||||
testFileSystem,
|
||||
@@ -412,9 +431,32 @@ describe("ReadTool", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("returns expected filesystem failures to the model", () =>
|
||||
Effect.gen(function* () {
|
||||
readFailure = new ReadToolFileSystem.BinaryFileError({ resource: "archive.dat" })
|
||||
const registry = yield* ToolRegistry.Service
|
||||
|
||||
expect(
|
||||
yield* executeTool(registry, {
|
||||
sessionID,
|
||||
...toolIdentity,
|
||||
call: {
|
||||
type: "tool-call",
|
||||
id: "call-binary",
|
||||
name: "read",
|
||||
input: { path: "archive.dat", offset: 2, limit: 1 },
|
||||
},
|
||||
}),
|
||||
).toEqual({ type: "error", value: "Cannot read binary file: archive.dat" })
|
||||
expect(readCalls).toEqual([
|
||||
{ input: AbsolutePath.make(`${process.cwd()}/archive.dat`), page: { offset: 2, limit: 1 } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves unexpected filesystem defects", () =>
|
||||
Effect.gen(function* () {
|
||||
readFailure = new ReadToolFileSystem.BinaryFileError("archive.dat")
|
||||
resolveFailure = new Error("unexpected")
|
||||
const registry = yield* ToolRegistry.Service
|
||||
|
||||
expect(
|
||||
@@ -422,18 +464,10 @@ describe("ReadTool", () => {
|
||||
yield* executeTool(registry, {
|
||||
sessionID,
|
||||
...toolIdentity,
|
||||
call: {
|
||||
type: "tool-call",
|
||||
id: "call-binary",
|
||||
name: "read",
|
||||
input: { path: "archive.dat", offset: 2, limit: 1 },
|
||||
},
|
||||
call: { type: "tool-call", id: "call-defect", name: "read", input: { path: "README.md" } },
|
||||
}).pipe(Effect.exit),
|
||||
),
|
||||
).toBe(true)
|
||||
expect(readCalls).toEqual([
|
||||
{ input: AbsolutePath.make(`${process.cwd()}/archive.dat`), page: { offset: 2, limit: 1 } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -453,6 +487,22 @@ describe("ReadTool", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("returns missing paths as model-visible tool failures", () =>
|
||||
Effect.gen(function* () {
|
||||
const registry = yield* ToolRegistry.Service
|
||||
|
||||
expect(
|
||||
yield* executeTool(registry, {
|
||||
sessionID,
|
||||
...toolIdentity,
|
||||
call: { type: "tool-call", id: "call-missing-path", name: "read", input: { path: missingPath } },
|
||||
}),
|
||||
).toEqual({ type: "error", value: `Unable to read ${missingPath}` })
|
||||
expect(assertions).toEqual([])
|
||||
expect(readCalls).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lists a bounded directory page through read", () =>
|
||||
Effect.gen(function* () {
|
||||
resolvedType = "directory"
|
||||
|
||||
@@ -176,6 +176,25 @@ describe("WebFetchTool registration", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("returns an error result when HTML-to-Markdown conversion throws", () =>
|
||||
Effect.gen(function* () {
|
||||
reset()
|
||||
respond = () =>
|
||||
Effect.succeed(
|
||||
new Response("<div>".repeat(10_000) + "content" + "</div>".repeat(10_000), {
|
||||
headers: { "content-type": "text/html" },
|
||||
}),
|
||||
)
|
||||
const registry = yield* ToolRegistry.Service
|
||||
const url = "https://1.1.1.1/deep-html"
|
||||
|
||||
expect(yield* executeTool(registry, call({ url, format: "markdown" }))).toEqual({
|
||||
type: "error",
|
||||
value: `Unable to fetch ${url}`,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects declared and streamed oversized bodies", () =>
|
||||
Effect.gen(function* () {
|
||||
reset()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { beforeEach, describe, expect, test } from "bun:test"
|
||||
import { Effect, Layer, Schema } from "effect"
|
||||
import { HttpClient, HttpClientResponse } from "effect/unstable/http"
|
||||
import { PermissionV2 } from "@opencode-ai/core/permission"
|
||||
@@ -66,8 +66,14 @@ interface Request {
|
||||
const requests: Request[] = []
|
||||
const assertions: PermissionV2.AssertInput[] = []
|
||||
let responseBody = payload("search results")
|
||||
let makeResponse = () => new Response(responseBody, { status: 200 })
|
||||
let config: WebSearchTool.Config = { enableExa: false, enableParallel: false }
|
||||
|
||||
beforeEach(() => {
|
||||
responseBody = payload("search results")
|
||||
makeResponse = () => new Response(responseBody, { status: 200 })
|
||||
})
|
||||
|
||||
const http = Layer.succeed(
|
||||
HttpClient.HttpClient,
|
||||
HttpClient.make((request) =>
|
||||
@@ -78,7 +84,7 @@ const http = Layer.succeed(
|
||||
headers: request.headers,
|
||||
body: JSON.parse(new TextDecoder().decode(request.body.body)),
|
||||
})
|
||||
return HttpClientResponse.fromWeb(request, new Response(responseBody, { status: 200 }))
|
||||
return HttpClientResponse.fromWeb(request, makeResponse())
|
||||
}),
|
||||
),
|
||||
)
|
||||
@@ -270,7 +276,22 @@ describe("WebSearchTool registration", () => {
|
||||
Effect.gen(function* () {
|
||||
requests.length = 0
|
||||
assertions.length = 0
|
||||
responseBody = "x".repeat(WebSearchTool.MAX_RESPONSE_BYTES + 1)
|
||||
let chunksRead = 0
|
||||
let cancelled = false
|
||||
makeResponse = () =>
|
||||
new Response(
|
||||
new ReadableStream({
|
||||
pull(controller) {
|
||||
chunksRead++
|
||||
if (chunksRead === 10) throw new Error("response was not stopped at the byte limit")
|
||||
controller.enqueue(new Uint8Array(64 * 1024))
|
||||
},
|
||||
cancel() {
|
||||
cancelled = true
|
||||
},
|
||||
}),
|
||||
{ status: 200 },
|
||||
)
|
||||
config = { provider: "exa", enableExa: false, enableParallel: false }
|
||||
const registry = yield* ToolRegistry.Service
|
||||
|
||||
@@ -281,6 +302,8 @@ describe("WebSearchTool registration", () => {
|
||||
call: { type: "tool-call", id: "call-large-response", name: "websearch", input: { query: "too much" } },
|
||||
}),
|
||||
).toEqual({ type: "error", value: "Unable to search the web for too much" })
|
||||
expect(chunksRead).toBeLessThan(10)
|
||||
expect(cancelled).toBe(true)
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
import { EOL } from "os"
|
||||
import { Effect } from "effect"
|
||||
import { FileSystem } from "@opencode-ai/core/filesystem"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { LocationServiceMap } from "@opencode-ai/core/location-layer"
|
||||
import { Location } from "@opencode-ai/core/location"
|
||||
import { AbsolutePath, RelativePath } from "@opencode-ai/core/schema"
|
||||
import { CliError, effectCmd } from "../../effect-cmd"
|
||||
import { effectCmd } from "../../effect-cmd"
|
||||
import { cmd } from "../cmd"
|
||||
|
||||
const filesystem = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
|
||||
@@ -14,8 +13,6 @@ const filesystem = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
|
||||
Effect.provide(LocationServiceMap.layer),
|
||||
)
|
||||
|
||||
const fileError = (error: FileSystem.PathError | FSUtil.Error) => new CliError({ message: error.message })
|
||||
|
||||
const FileSearchCommand = effectCmd({
|
||||
command: "search <query>",
|
||||
describe: "search files by query",
|
||||
@@ -41,9 +38,7 @@ const FileReadCommand = effectCmd({
|
||||
description: "File path to read",
|
||||
}),
|
||||
handler: Effect.fn("Cli.debug.file.read")(function* (args) {
|
||||
const file = yield* filesystem(
|
||||
FileSystem.Service.use((svc) => svc.read({ path: RelativePath.make(args.path) })),
|
||||
).pipe(Effect.mapError(fileError))
|
||||
const file = yield* filesystem(FileSystem.Service.use((svc) => svc.read({ path: RelativePath.make(args.path) })))
|
||||
process.stdout.write(
|
||||
JSON.stringify(
|
||||
{ content: Buffer.from(file.content).toString("base64"), encoding: "base64", mime: file.mime },
|
||||
@@ -64,9 +59,7 @@ const FileListCommand = effectCmd({
|
||||
description: "File path to list",
|
||||
}),
|
||||
handler: Effect.fn("Cli.debug.file.list")(function* (args) {
|
||||
const files = yield* filesystem(
|
||||
FileSystem.Service.use((svc) => svc.list({ path: RelativePath.make(args.path) })),
|
||||
).pipe(Effect.mapError(fileError))
|
||||
const files = yield* filesystem(FileSystem.Service.use((svc) => svc.list({ path: RelativePath.make(args.path) })))
|
||||
process.stdout.write(JSON.stringify(files, null, 2) + EOL)
|
||||
}),
|
||||
})
|
||||
|
||||
@@ -10,10 +10,6 @@ import ignore from "ignore"
|
||||
import path from "path"
|
||||
import { HttpApiBuilder } from "effect/unstable/httpapi"
|
||||
import { InstanceHttpApi } from "../api"
|
||||
import { InvalidRequestError } from "../errors"
|
||||
|
||||
const invalidRequest = (error: FileSystem.PathError | FSUtil.Error) =>
|
||||
new InvalidRequestError({ message: error.message, kind: error._tag })
|
||||
|
||||
export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handlers) =>
|
||||
Effect.gen(function* () {
|
||||
@@ -83,9 +79,7 @@ export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handl
|
||||
.readFileString(path.join(location.project.directory, ".ignore"))
|
||||
.pipe(Effect.catch(() => Effect.succeed("")))
|
||||
if (ignorefile) ignored.add(ignorefile)
|
||||
return (yield* fs
|
||||
.list({ path: RelativePath.make(ctx.query.path) })
|
||||
.pipe(Effect.mapError(invalidRequest))).map((item) => ({
|
||||
return (yield* fs.list({ path: RelativePath.make(ctx.query.path) })).map((item) => ({
|
||||
name: path.basename(item.path),
|
||||
path: item.path,
|
||||
absolute: path.resolve(location.directory, item.path),
|
||||
@@ -107,7 +101,6 @@ export const fileHandlers = HttpApiBuilder.group(InstanceHttpApi, "file", (handl
|
||||
return yield* filesystem(
|
||||
FileSystem.Service.use((fs) => fs.read({ path: RelativePath.make(ctx.query.path) })),
|
||||
).pipe(
|
||||
Effect.mapError(invalidRequest),
|
||||
Effect.flatMap((item) =>
|
||||
Effect.gen(function* () {
|
||||
const text = item.content.includes(0)
|
||||
|
||||
@@ -1,25 +1,9 @@
|
||||
import type { FileSystemEntry } from "@opencode-ai/sdk/v2/types"
|
||||
import type { Effect } from "effect"
|
||||
import type { PlatformError } from "effect/PlatformError"
|
||||
|
||||
export type FileSystemError =
|
||||
| PlatformError
|
||||
| {
|
||||
readonly _tag: "FileSystemError"
|
||||
readonly method: string
|
||||
readonly cause?: unknown
|
||||
}
|
||||
| {
|
||||
readonly _tag: "FileSystem.PathError"
|
||||
readonly path: string
|
||||
readonly reason: "lexical_escape" | "symlink_escape" | "not_file" | "not_directory"
|
||||
}
|
||||
|
||||
export interface FileSystem {
|
||||
read(input: {
|
||||
readonly path: string
|
||||
}): Effect.Effect<{ readonly content: Uint8Array; readonly mime: string }, FileSystemError>
|
||||
list(input?: { readonly path?: string }): Effect.Effect<FileSystemEntry[], FileSystemError>
|
||||
read(input: { readonly path: string }): Effect.Effect<{ readonly content: Uint8Array; readonly mime: string }>
|
||||
list(input?: { readonly path?: string }): Effect.Effect<FileSystemEntry[]>
|
||||
find(input: {
|
||||
readonly query: string
|
||||
readonly type?: "file" | "directory"
|
||||
|
||||
@@ -7,7 +7,7 @@ export type { AISDK, AISDKHooks } from "./aisdk.js"
|
||||
export type { Catalog, CatalogDraft, CatalogProviderRecord } from "./catalog.js"
|
||||
export type { Command, CommandDraft } from "./command.js"
|
||||
export type { Event, EventMap } from "./event.js"
|
||||
export type { FileSystem, FileSystemError } from "./filesystem.js"
|
||||
export type { FileSystem } from "./filesystem.js"
|
||||
export type { Integration, IntegrationDraft, IntegrationMethod, IntegrationMethodRegistration } from "./integration.js"
|
||||
export type { Location } from "./location.js"
|
||||
export type { Npm } from "./npm.js"
|
||||
|
||||
@@ -1,28 +1,21 @@
|
||||
import { FileSystem } from "@opencode-ai/core/filesystem"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { RelativePath } from "@opencode-ai/core/schema"
|
||||
import { Effect } from "effect"
|
||||
import { HttpServerResponse } from "effect/unstable/http"
|
||||
import { HttpApiBuilder } from "effect/unstable/httpapi"
|
||||
import { Api } from "../api"
|
||||
import { InvalidRequestError } from "../errors"
|
||||
import { response } from "../groups/location"
|
||||
|
||||
const invalidRequest = (error: FileSystem.PathError | FSUtil.Error) =>
|
||||
new InvalidRequestError({ message: error.message, kind: error._tag })
|
||||
|
||||
export const FileSystemHandler = HttpApiBuilder.group(Api, "server.fs", (handlers) =>
|
||||
Effect.gen(function* () {
|
||||
return handlers
|
||||
.handleRaw("fs.read", (ctx) =>
|
||||
Effect.gen(function* () {
|
||||
const file = yield* (yield* FileSystem.Service)
|
||||
.read({
|
||||
path: RelativePath.make(
|
||||
decodeURIComponent(new URL(ctx.request.url, "http://localhost").pathname.slice(13)),
|
||||
),
|
||||
})
|
||||
.pipe(Effect.mapError(invalidRequest))
|
||||
const file = yield* (yield* FileSystem.Service).read({
|
||||
path: RelativePath.make(
|
||||
decodeURIComponent(new URL(ctx.request.url, "http://localhost").pathname.slice(13)),
|
||||
),
|
||||
})
|
||||
return HttpServerResponse.uint8Array(file.content, { contentType: file.mime })
|
||||
}),
|
||||
)
|
||||
@@ -30,7 +23,7 @@ export const FileSystemHandler = HttpApiBuilder.group(Api, "server.fs", (handler
|
||||
response(
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* FileSystem.Service
|
||||
return yield* fs.list(ctx.query).pipe(Effect.mapError(invalidRequest))
|
||||
return yield* fs.list(ctx.query)
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -330,7 +330,7 @@ function CatalogDatum(props: { label: string; value: string }) {
|
||||
function ModelOverview(props: { data: StatsModelData | null }) {
|
||||
return (
|
||||
<section data-section="model-panel">
|
||||
<SectionTitle title="Overview" description="Recent OpenCode Go tokens, sessions, and market position." />
|
||||
<SectionTitle title="Overview" description="Recent OpenCode Go tokens, unique users, and market position." />
|
||||
<Show
|
||||
when={props.data}
|
||||
fallback={
|
||||
@@ -340,6 +340,7 @@ function ModelOverview(props: { data: StatsModelData | null }) {
|
||||
{(data) => (
|
||||
<div data-component="model-metric-grid">
|
||||
<MetricCard label="Tokens" value={formatTokens(data().totals.tokens)} detail="last two months" />
|
||||
<MetricCard label="Unique Users" value={formatUsers(data().totals.uniqueUsers)} detail="last two months" />
|
||||
<MetricCard label="Sessions" value={formatInteger(data().totals.sessions)} detail="completed sessions" />
|
||||
<MetricCard
|
||||
label="Token Share"
|
||||
|
||||
@@ -5,28 +5,36 @@ import {
|
||||
StartQueryExecutionCommand,
|
||||
type Row,
|
||||
} from "@aws-sdk/client-athena"
|
||||
import { Effect, Layer, Schema } from "effect"
|
||||
import { Effect, Layer } from "effect"
|
||||
import * as Context from "effect/Context"
|
||||
import { Resource } from "sst/resource"
|
||||
|
||||
const ATHENA_MAX_POLL_ATTEMPTS = 60
|
||||
const ATHENA_MAX_POLL_ATTEMPTS = 300
|
||||
const ATHENA_PAGE_SIZE = 1000
|
||||
|
||||
export type AthenaData = Record<string, string>
|
||||
|
||||
export class AthenaQueryError extends Schema.TaggedErrorClass<AthenaQueryError>()("AthenaQueryError", {
|
||||
message: Schema.String,
|
||||
queryExecutionId: Schema.optional(Schema.String),
|
||||
cause: Schema.optional(Schema.Defect()),
|
||||
}) {}
|
||||
export class AthenaQueryError extends Error {
|
||||
readonly _tag = "AthenaQueryError"
|
||||
readonly queryExecutionId?: string
|
||||
|
||||
export class AthenaQueryTimeoutError extends Schema.TaggedErrorClass<AthenaQueryTimeoutError>()(
|
||||
"AthenaQueryTimeoutError",
|
||||
{
|
||||
message: Schema.String,
|
||||
queryExecutionId: Schema.String,
|
||||
},
|
||||
) {}
|
||||
constructor(input: { message: string; queryExecutionId?: string; cause?: unknown }) {
|
||||
super(input.message, { cause: input.cause })
|
||||
this.name = "AthenaQueryError"
|
||||
this.queryExecutionId = input.queryExecutionId
|
||||
}
|
||||
}
|
||||
|
||||
export class AthenaQueryTimeoutError extends Error {
|
||||
readonly _tag = "AthenaQueryTimeoutError"
|
||||
readonly queryExecutionId: string
|
||||
|
||||
constructor(input: { message: string; queryExecutionId: string }) {
|
||||
super(input.message)
|
||||
this.name = "AthenaQueryTimeoutError"
|
||||
this.queryExecutionId = input.queryExecutionId
|
||||
}
|
||||
}
|
||||
|
||||
export declare namespace Athena {
|
||||
export interface Service {
|
||||
@@ -57,7 +65,7 @@ export class Athena extends Context.Service<Athena, Athena.Service>()("@opencode
|
||||
})
|
||||
const queryExecutionId = started.QueryExecutionId
|
||||
if (!queryExecutionId)
|
||||
return yield* new AthenaQueryError({ message: "Athena did not return a query execution id" })
|
||||
return yield* Effect.fail(new AthenaQueryError({ message: "Athena did not return a query execution id" }))
|
||||
|
||||
yield* poll(client, queryExecutionId)
|
||||
return yield* results(client, queryExecutionId)
|
||||
@@ -87,16 +95,20 @@ const poll: (
|
||||
|
||||
if (status?.State === "SUCCEEDED") return
|
||||
if (status?.State === "FAILED" || status?.State === "CANCELLED")
|
||||
return yield* new AthenaQueryError({
|
||||
message: `Athena stats query ${status.State.toLowerCase()}: ${status.StateChangeReason ?? "unknown reason"}`,
|
||||
queryExecutionId,
|
||||
})
|
||||
return yield* Effect.fail(
|
||||
new AthenaQueryError({
|
||||
message: `Athena stats query ${status.State.toLowerCase()}: ${status.StateChangeReason ?? "unknown reason"}`,
|
||||
queryExecutionId,
|
||||
}),
|
||||
)
|
||||
|
||||
if (attempt >= ATHENA_MAX_POLL_ATTEMPTS - 1)
|
||||
return yield* new AthenaQueryTimeoutError({
|
||||
message: `Athena stats query ${queryExecutionId} did not complete`,
|
||||
queryExecutionId,
|
||||
})
|
||||
return yield* Effect.fail(
|
||||
new AthenaQueryTimeoutError({
|
||||
message: `Athena stats query ${queryExecutionId} did not complete`,
|
||||
queryExecutionId,
|
||||
}),
|
||||
)
|
||||
|
||||
return yield* poll(client, queryExecutionId, attempt + 1)
|
||||
})
|
||||
|
||||
@@ -44,16 +44,29 @@ export class DrizzleClient extends Context.Service<DrizzleClient, Drizzle>()("@o
|
||||
)
|
||||
}
|
||||
|
||||
export class DatabaseError extends Schema.TaggedErrorClass<DatabaseError>()("DatabaseError", {
|
||||
cause: Schema.Defect(),
|
||||
}) {}
|
||||
export class DatabaseError extends Error {
|
||||
readonly _tag = "DatabaseError"
|
||||
|
||||
constructor(input: { cause: unknown }) {
|
||||
super("Database operation failed", { cause: input.cause })
|
||||
this.name = "DatabaseError"
|
||||
}
|
||||
|
||||
static make(input: { cause: unknown }) {
|
||||
return new DatabaseError(input)
|
||||
}
|
||||
}
|
||||
|
||||
export const catchDbError = Effect.mapError((cause) => DatabaseError.make({ cause }))
|
||||
|
||||
export class MigrationError extends Schema.TaggedErrorClass<MigrationError>()("MigrationError", {
|
||||
message: Schema.String,
|
||||
cause: Schema.optional(Schema.Defect()),
|
||||
}) {}
|
||||
export class MigrationError extends Error {
|
||||
readonly _tag = "MigrationError"
|
||||
|
||||
constructor(input: { message: string; cause?: unknown }) {
|
||||
super(input.message, { cause: input.cause })
|
||||
this.name = "MigrationError"
|
||||
}
|
||||
}
|
||||
|
||||
export const migrate = Effect.fn("Database.migrate")(function* () {
|
||||
const settings = yield* DatabaseConfig
|
||||
@@ -68,9 +81,11 @@ export const migrate = Effect.fn("Database.migrate")(function* () {
|
||||
catch: (cause) => new MigrationError({ message: "Failed to apply database migrations", cause }),
|
||||
})
|
||||
if (result)
|
||||
return yield* new MigrationError({
|
||||
message: `Failed to initialize database migrations: ${result.exitCode}`,
|
||||
})
|
||||
return yield* Effect.fail(
|
||||
new MigrationError({
|
||||
message: `Failed to initialize database migrations: ${result.exitCode}`,
|
||||
}),
|
||||
)
|
||||
yield* Effect.logInfo("database migrations complete").pipe(
|
||||
Effect.annotateLogs({ migrationsDir: settings.migrationsDir }),
|
||||
)
|
||||
|
||||
@@ -54,6 +54,7 @@ export type StatsModelData = {
|
||||
tokenChange: number
|
||||
totals: {
|
||||
sessions: number
|
||||
uniqueUsers: number
|
||||
tokens: number
|
||||
cost: number
|
||||
tokensPerSession: number
|
||||
@@ -381,6 +382,7 @@ function buildStatsModelData(
|
||||
tokenChange: percentChange(current.totalTokens, previous.totalTokens),
|
||||
totals: {
|
||||
sessions: current.sessions,
|
||||
uniqueUsers: current.uniqueUsers,
|
||||
tokens: current.totalTokens,
|
||||
cost: round(microcentsToDollars(current.totalCostMicrocents), 2),
|
||||
tokensPerSession: current.sessions > 0 ? Math.round(current.totalTokens / current.sessions) : 0,
|
||||
|
||||
@@ -17,6 +17,8 @@ export type StatDimension = "model" | "provider" | "geo" | "geo_model"
|
||||
export function buildStatsQuery(periodStart: Date, periodEnd: Date, dimension: StatDimension) {
|
||||
const periodStartValue = sqlString(periodStart.toISOString())
|
||||
const periodEndValue = sqlString(periodEnd.toISOString())
|
||||
const periodStartDateValue = sqlString(periodStart.toISOString().slice(0, 10))
|
||||
const periodEndDateValue = sqlString(periodEnd.toISOString().slice(0, 10))
|
||||
const sourceTable = [Resource.InferenceEvent.catalog, Resource.InferenceEvent.database, Resource.InferenceEvent.table]
|
||||
.map(sqlIdentifier)
|
||||
.join(".")
|
||||
@@ -95,6 +97,9 @@ WITH normalized AS (
|
||||
WHERE event_type = 'completions'
|
||||
AND model IS NOT NULL
|
||||
AND model <> ''
|
||||
AND source = 'lite'
|
||||
AND event_date >= ${periodStartDateValue}
|
||||
AND event_date <= ${periodEndDateValue}
|
||||
AND event_timestamp >= ${periodStartValue}
|
||||
AND event_timestamp < ${periodEndValue}
|
||||
), filtered AS (
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Buffer } from "node:buffer"
|
||||
import { FirehoseClient, PutRecordBatchCommand } from "@aws-sdk/client-firehose"
|
||||
import { Effect, Layer, Schema } from "effect"
|
||||
import { Effect, Layer } from "effect"
|
||||
import * as Context from "effect/Context"
|
||||
import { Resource } from "sst/resource"
|
||||
|
||||
@@ -12,11 +12,16 @@ type IngestEvent = Record<string, unknown>
|
||||
type LakeRoute = { database: string; table: string }
|
||||
type FirehoseRecord = { Data: Uint8Array }
|
||||
|
||||
export class IngestError extends Schema.TaggedErrorClass<IngestError>()("IngestError", {
|
||||
message: Schema.String,
|
||||
failed: Schema.Number,
|
||||
cause: Schema.optional(Schema.Defect()),
|
||||
}) {}
|
||||
export class IngestError extends Error {
|
||||
readonly _tag = "IngestError"
|
||||
readonly failed: number
|
||||
|
||||
constructor(input: { message: string; failed: number; cause?: unknown }) {
|
||||
super(input.message, { cause: input.cause })
|
||||
this.name = "IngestError"
|
||||
this.failed = input.failed
|
||||
}
|
||||
}
|
||||
|
||||
export declare namespace Ingest {
|
||||
export interface Service {
|
||||
@@ -37,10 +42,12 @@ export class Ingest extends Context.Service<Ingest, Ingest.Service>()("@opencode
|
||||
yield* Effect.logWarning(
|
||||
`lake ingest rejected ${JSON.stringify({ records: counts.records, unsupported: counts.unsupported })}`,
|
||||
)
|
||||
return yield* new IngestError({
|
||||
message: "Unsupported lake event type",
|
||||
failed: counts.unsupported,
|
||||
})
|
||||
return yield* Effect.fail(
|
||||
new IngestError({
|
||||
message: "Unsupported lake event type",
|
||||
failed: counts.unsupported,
|
||||
}),
|
||||
)
|
||||
}
|
||||
if (counts.records === 0) return { records: 0 }
|
||||
|
||||
@@ -66,7 +73,7 @@ export class Ingest extends Context.Service<Ingest, Ingest.Service>()("@opencode
|
||||
|
||||
if (failed > 0) {
|
||||
yield* Effect.logWarning(`lake ingest incomplete ${JSON.stringify({ records: counts.records, failed })}`)
|
||||
return yield* new IngestError({ message: "Failed to ingest all lake records", failed })
|
||||
return yield* Effect.fail(new IngestError({ message: "Failed to ingest all lake records", failed }))
|
||||
}
|
||||
|
||||
yield* Effect.logInfo(`lake ingest complete ${JSON.stringify({ records: counts.records, batches })}`)
|
||||
|
||||
@@ -13,6 +13,10 @@
|
||||
"outputs": [],
|
||||
"passThroughEnv": ["*"]
|
||||
},
|
||||
"@opencode-ai/core#test": {
|
||||
"dependsOn": ["^build"],
|
||||
"outputs": []
|
||||
},
|
||||
"@opencode-ai/app#test": {
|
||||
"dependsOn": ["^build"],
|
||||
"outputs": []
|
||||
|
||||
Reference in New Issue
Block a user