Files
luma/Sources/LumaCore/Models/WidgetUpdate.swift
Ole André Vadla Ravnås 8d6f91fe1e Plumb image attachments through to the LLM
Console widgets get an appendImage / respond.image API on the
agent. read_widget_state surfaces image entries as attachments
scaled to at most max_dimension pixels (default 1568) on the
longest edge. Anthropic nests image blocks inside tool_result;
OpenAI gets a synthetic follow-up user message. The host scales
via CoreImage on macOS and GdkPixbuf on GTK.
2026-05-22 12:15:53 +02:00

586 lines
21 KiB
Swift

import Foundation
public struct WidgetUpdate: Sendable, Identifiable {
public let id: UUID
public let instanceID: UUID
public let widget: String
public let kind: Kind
public init(id: UUID = UUID(), instanceID: UUID, widget: String, kind: Kind) {
self.id = id
self.instanceID = instanceID
self.widget = widget
self.kind = kind
}
public enum Kind: Sendable {
case counterSet(WidgetCounterValue)
case histogramSet([WidgetHistogramBucket])
case histogramIncrement(label: String, by: Double)
case graphPoint(WidgetGraphPoint)
case listUpsert(WidgetListItem)
case listRemove(itemID: String)
case tableUpsert(WidgetTableRow)
case tableRemove(rowID: String)
case hexSet(WidgetHexState)
case consoleAppend(WidgetConsoleEntry)
/// Posted by the agent when `onConsoleInput` has finished posting
/// replies for `inputEntryID`. Lets request/response waiters exit
/// before their timeout elapses.
case consoleReplyDone(inputEntryID: String)
case clear
/// In-memory snapshot replay. Emitted locally when widget state is
/// hydrated from disk so listening views can refresh in one shot.
/// Never serialized over the wire.
case snapshot(WidgetState)
}
public func toWireJSON() -> [String: Any] {
var obj: [String: Any] = [
"instance_id": instanceID.uuidString,
"widget": widget,
]
switch kind {
case .counterSet(let value):
obj["kind"] = "counter-set"
var counter: [String: Any] = ["value": value.value]
if let unit = value.unit { counter["unit"] = unit }
if let delta = value.delta { counter["delta"] = delta }
obj["counter"] = counter
case .histogramSet(let buckets):
obj["kind"] = "histogram-set"
obj["buckets"] = buckets.map { ["label": $0.label, "count": $0.count] }
case .histogramIncrement(let label, let by):
obj["kind"] = "histogram-increment"
obj["label"] = label
obj["by"] = by
case .graphPoint(let point):
obj["kind"] = "graph-point"
obj["point"] = ["series": point.series, "x": point.x, "y": point.y]
case .listUpsert(let item):
obj["kind"] = "list-upsert"
var itemObj: [String: Any] = ["id": item.id, "title": item.title]
if let s = item.subtitle { itemObj["subtitle"] = s }
if let a = item.accessory { itemObj["accessory"] = a }
obj["item"] = itemObj
case .listRemove(let itemID):
obj["kind"] = "list-remove"
obj["item"] = itemID
case .tableUpsert(let row):
obj["kind"] = "table-upsert"
obj["row"] = ["id": row.id, "cells": row.cells]
case .tableRemove(let rowID):
obj["kind"] = "table-remove"
obj["row"] = rowID
case .hexSet(let state):
obj["kind"] = "hex-set"
obj["hex"] = [
"bytes": state.bytes.base64EncodedString(),
"base_address": state.baseAddress,
]
case .consoleAppend(let entry):
obj["kind"] = "console-append"
obj["entry"] = entry.toWireJSON()
case .consoleReplyDone(let inputEntryID):
obj["kind"] = "console-reply-done"
obj["reply_to"] = inputEntryID
case .clear:
obj["kind"] = "clear"
case .snapshot:
preconditionFailure("WidgetUpdate.snapshot is in-memory only")
}
return obj
}
public static func fromWireJSON(_ obj: [String: Any]) -> WidgetUpdate? {
guard let instanceIDStr = obj["instance_id"] as? String,
let instanceID = UUID(uuidString: instanceIDStr),
let widget = obj["widget"] as? String,
let kindStr = obj["kind"] as? String
else { return nil }
guard let kind = decodeKind(kindStr, from: obj) else { return nil }
return WidgetUpdate(instanceID: instanceID, widget: widget, kind: kind)
}
private static func decodeKind(_ kindStr: String, from obj: [String: Any]) -> Kind? {
switch kindStr {
case "counter-set":
guard let counter = obj["counter"] as? [String: Any],
let value = decodeDouble(counter["value"])
else { return nil }
return .counterSet(WidgetCounterValue(
value: value,
unit: counter["unit"] as? String,
delta: decodeDouble(counter["delta"])
))
case "histogram-set":
guard let buckets = obj["buckets"] as? [[String: Any]] else { return nil }
let parsed = buckets.compactMap { b -> WidgetHistogramBucket? in
guard let label = b["label"] as? String, let count = decodeDouble(b["count"]) else { return nil }
return WidgetHistogramBucket(label: label, count: count)
}
return .histogramSet(parsed)
case "histogram-increment":
guard let label = obj["label"] as? String, let by = decodeDouble(obj["by"]) else { return nil }
return .histogramIncrement(label: label, by: by)
case "graph-point":
guard let pointObj = obj["point"] as? [String: Any],
let series = pointObj["series"] as? String,
let x = decodeDouble(pointObj["x"]),
let y = decodeDouble(pointObj["y"])
else { return nil }
return .graphPoint(WidgetGraphPoint(series: series, x: x, y: y))
case "list-upsert":
guard let itemObj = obj["item"] as? [String: Any],
let id = itemObj["id"] as? String,
let title = itemObj["title"] as? String
else { return nil }
return .listUpsert(WidgetListItem(
id: id,
title: title,
subtitle: itemObj["subtitle"] as? String,
accessory: itemObj["accessory"] as? String
))
case "list-remove":
guard let itemID = obj["item"] as? String else { return nil }
return .listRemove(itemID: itemID)
case "table-upsert":
guard let rowObj = obj["row"] as? [String: Any],
let id = rowObj["id"] as? String,
let cells = rowObj["cells"] as? [String: String]
else { return nil }
return .tableUpsert(WidgetTableRow(id: id, cells: cells))
case "table-remove":
guard let rowID = obj["row"] as? String else { return nil }
return .tableRemove(rowID: rowID)
case "hex-set":
guard let hex = obj["hex"] as? [String: Any],
let b64 = hex["bytes"] as? String,
let bytes = Data(base64Encoded: b64)
else { return nil }
let baseAddress: UInt64 = (hex["base_address"] as? NSNumber)?.uint64Value ?? 0
return .hexSet(WidgetHexState(bytes: bytes, baseAddress: baseAddress))
case "console-append":
guard let entryObj = obj["entry"] as? [String: Any],
let entry = WidgetConsoleEntry.fromWireJSON(entryObj)
else { return nil }
return .consoleAppend(entry)
case "console-reply-done":
guard let replyTo = obj["reply_to"] as? String else { return nil }
return .consoleReplyDone(inputEntryID: replyTo)
case "clear":
return .clear
default:
return nil
}
}
private static func decodeDouble(_ raw: Any?) -> Double? {
if let n = raw as? NSNumber { return n.doubleValue }
return nil
}
}
public struct WidgetCounterValue: Codable, Sendable, Equatable {
public var value: Double
public var unit: String?
public var delta: Double?
public init(value: Double, unit: String? = nil, delta: Double? = nil) {
self.value = value
self.unit = unit
self.delta = delta
}
}
public struct WidgetHistogramBucket: Codable, Sendable, Equatable {
public let label: String
public var count: Double
public init(label: String, count: Double) {
self.label = label
self.count = count
}
}
public struct WidgetGraphPoint: Codable, Sendable, Equatable {
public let series: String
public let x: Double
public let y: Double
public init(series: String, x: Double, y: Double) {
self.series = series
self.x = x
self.y = y
}
}
public struct WidgetListItem: Codable, Sendable, Identifiable, Equatable {
public let id: String
public var title: String
public var subtitle: String?
public var accessory: String?
public init(id: String, title: String, subtitle: String? = nil, accessory: String? = nil) {
self.id = id
self.title = title
self.subtitle = subtitle
self.accessory = accessory
}
}
public struct WidgetTableRow: Codable, Sendable, Identifiable, Equatable {
public let id: String
public var cells: [String: String]
public init(id: String, cells: [String: String]) {
self.id = id
self.cells = cells
}
}
public struct WidgetHexState: Codable, Sendable, Equatable {
public var bytes: Data
public var baseAddress: UInt64
public init(bytes: Data, baseAddress: UInt64 = 0) {
self.bytes = bytes
self.baseAddress = baseAddress
}
}
public struct ConsoleImage: Codable, Sendable, Equatable {
public var mediaType: String
public var data: Data
public var width: Int
public var height: Int
public init(mediaType: String, data: Data, width: Int, height: Int) {
self.mediaType = mediaType
self.data = data
self.width = width
self.height = height
}
}
public struct WidgetConsoleEntry: Codable, Sendable, Equatable, Identifiable {
public enum Kind: String, Codable, Sendable {
case input
case output
case image
case error
}
public let id: String
public let kind: Kind
public var text: String
public var value: JSInspectValue?
public var image: ConsoleImage?
public var replyTo: String?
public init(
id: String = UUID().uuidString,
kind: Kind,
text: String,
value: JSInspectValue? = nil,
image: ConsoleImage? = nil,
replyTo: String? = nil
) {
self.id = id
self.kind = kind
self.text = text
self.value = value
self.image = image
self.replyTo = replyTo
}
public func toWireJSON() -> [String: Any] {
var obj: [String: Any] = [
"id": id,
"kind": kind.rawValue,
"text": text,
]
if let value, let encoded = try? JSONEncoder().encode(value),
let valueJSON = try? JSONSerialization.jsonObject(with: encoded)
{
obj["value"] = valueJSON
}
if let image {
obj["image"] = [
"media_type": image.mediaType,
"data_base64": image.data.base64EncodedString(),
"width": image.width,
"height": image.height,
]
}
if let replyTo { obj["reply_to"] = replyTo }
return obj
}
public static func fromWireJSON(_ obj: [String: Any]) -> WidgetConsoleEntry? {
guard let id = obj["id"] as? String,
let kindStr = obj["kind"] as? String,
let kind = Kind(rawValue: kindStr),
let text = obj["text"] as? String
else { return nil }
var value: JSInspectValue? = nil
if let raw = obj["value"],
let data = try? JSONSerialization.data(withJSONObject: raw)
{
value = try? JSONDecoder().decode(JSInspectValue.self, from: data)
}
let image = (obj["image"] as? [String: Any]).flatMap(parseConsoleImage)
let replyTo = obj["reply_to"] as? String
return WidgetConsoleEntry(id: id, kind: kind, text: text, value: value, image: image, replyTo: replyTo)
}
}
private func parseConsoleImage(_ obj: [String: Any]) -> ConsoleImage? {
guard let mediaType = obj["media_type"] as? String,
let base64 = obj["data_base64"] as? String,
let data = Data(base64Encoded: base64),
let width = obj["width"] as? Int,
let height = obj["height"] as? Int
else { return nil }
return ConsoleImage(mediaType: mediaType, data: data, width: width, height: height)
}
public struct WidgetStateSnapshot: Sendable {
public let sessionID: UUID
public let instanceID: UUID
public let widget: String
public let state: WidgetState
public init(sessionID: UUID, instanceID: UUID, widget: String, state: WidgetState) {
self.sessionID = sessionID
self.instanceID = instanceID
self.widget = widget
self.state = state
}
public static func fromWireJSON(_ obj: [String: Any]) -> WidgetStateSnapshot? {
guard let sessionStr = obj["session_id"] as? String,
let sessionID = UUID(uuidString: sessionStr),
let instanceStr = obj["instance_id"] as? String,
let instanceID = UUID(uuidString: instanceStr),
let widget = obj["widget"] as? String,
let stateObj = obj["state"] as? [String: Any]
else { return nil }
var counter: WidgetCounterValue?
if let c = stateObj["counter"] as? [String: Any], let value = (c["value"] as? NSNumber)?.doubleValue {
counter = WidgetCounterValue(
value: value,
unit: c["unit"] as? String,
delta: (c["delta"] as? NSNumber)?.doubleValue
)
}
var histogram: [WidgetHistogramBucket] = []
if let buckets = stateObj["buckets"] as? [[String: Any]] {
for b in buckets {
guard let label = b["label"] as? String, let count = (b["count"] as? NSNumber)?.doubleValue else { continue }
histogram.append(WidgetHistogramBucket(label: label, count: count))
}
}
var graphSeries: [String: [WidgetGraphPoint]] = [:]
if let pts = stateObj["points"] as? [[String: Any]] {
for p in pts {
guard let series = p["series"] as? String,
let x = (p["x"] as? NSNumber).map({ $0.doubleValue }),
let y = (p["y"] as? NSNumber).map({ $0.doubleValue })
else { continue }
graphSeries[series, default: []].append(WidgetGraphPoint(series: series, x: x, y: y))
}
}
var listItems: [WidgetListItem] = []
if let items = stateObj["items"] as? [[String: Any]] {
for it in items {
guard let id = it["id"] as? String, let title = it["title"] as? String else { continue }
listItems.append(WidgetListItem(
id: id,
title: title,
subtitle: it["subtitle"] as? String,
accessory: it["accessory"] as? String
))
}
}
var tableRows: [WidgetTableRow] = []
if let rows = stateObj["rows"] as? [[String: Any]] {
for row in rows {
guard let id = row["id"] as? String, let cells = row["cells"] as? [String: String] else { continue }
tableRows.append(WidgetTableRow(id: id, cells: cells))
}
}
var hex: WidgetHexState?
if let h = stateObj["hex"] as? [String: Any],
let b64 = h["bytes"] as? String,
let bytes = Data(base64Encoded: b64)
{
hex = WidgetHexState(
bytes: bytes,
baseAddress: (h["base_address"] as? NSNumber)?.uint64Value ?? 0
)
}
var consoleEntries: [WidgetConsoleEntry] = []
if let entries = stateObj["entries"] as? [[String: Any]] {
for e in entries {
if let parsed = WidgetConsoleEntry.fromWireJSON(e) {
consoleEntries.append(parsed)
}
}
}
return WidgetStateSnapshot(
sessionID: sessionID,
instanceID: instanceID,
widget: widget,
state: WidgetState(
counter: counter,
histogram: histogram,
graphSeries: graphSeries,
listItems: listItems,
tableRows: tableRows,
hex: hex,
consoleEntries: consoleEntries
)
)
}
}
public struct WidgetState: Codable, Sendable, Equatable {
public var counter: WidgetCounterValue?
public var histogram: [WidgetHistogramBucket]
public var graphSeries: [String: [WidgetGraphPoint]]
public var listItems: [WidgetListItem]
public var tableRows: [WidgetTableRow]
public var hex: WidgetHexState?
public var consoleEntries: [WidgetConsoleEntry]
public init(
counter: WidgetCounterValue? = nil,
histogram: [WidgetHistogramBucket] = [],
graphSeries: [String: [WidgetGraphPoint]] = [:],
listItems: [WidgetListItem] = [],
tableRows: [WidgetTableRow] = [],
hex: WidgetHexState? = nil,
consoleEntries: [WidgetConsoleEntry] = []
) {
self.counter = counter
self.histogram = histogram
self.graphSeries = graphSeries
self.listItems = listItems
self.tableRows = tableRows
self.hex = hex
self.consoleEntries = consoleEntries
}
private enum CodingKeys: String, CodingKey {
case counter, histogram, graphSeries, listItems, tableRows, hex, consoleEntries
}
public init(from decoder: Decoder) throws {
let c = try decoder.container(keyedBy: CodingKeys.self)
counter = try c.decodeIfPresent(WidgetCounterValue.self, forKey: .counter)
histogram = try c.decodeIfPresent([WidgetHistogramBucket].self, forKey: .histogram) ?? []
graphSeries = try c.decodeIfPresent([String: [WidgetGraphPoint]].self, forKey: .graphSeries) ?? [:]
listItems = try c.decodeIfPresent([WidgetListItem].self, forKey: .listItems) ?? []
tableRows = try c.decodeIfPresent([WidgetTableRow].self, forKey: .tableRows) ?? []
hex = try c.decodeIfPresent(WidgetHexState.self, forKey: .hex)
consoleEntries = try c.decodeIfPresent([WidgetConsoleEntry].self, forKey: .consoleEntries) ?? []
}
public mutating func apply(_ kind: WidgetUpdate.Kind) {
if case .snapshot(let snapshot) = kind {
self = snapshot
return
}
switch kind {
case .counterSet(let value):
counter = value
case .histogramSet(let buckets):
histogram = buckets
case .histogramIncrement(let label, let by):
if let index = histogram.firstIndex(where: { $0.label == label }) {
histogram[index].count += by
} else {
histogram.append(WidgetHistogramBucket(label: label, count: by))
}
case .graphPoint(let point):
graphSeries[point.series, default: []].append(point)
case .listUpsert(let item):
if let index = listItems.firstIndex(where: { $0.id == item.id }) {
listItems[index] = item
} else {
listItems.append(item)
}
case .listRemove(let id):
listItems.removeAll { $0.id == id }
case .tableUpsert(let row):
if let index = tableRows.firstIndex(where: { $0.id == row.id }) {
tableRows[index] = row
} else {
tableRows.append(row)
}
case .tableRemove(let id):
tableRows.removeAll { $0.id == id }
case .hexSet(let value):
hex = value
case .consoleAppend(let entry):
consoleEntries.append(entry)
case .consoleReplyDone:
break
case .clear:
counter = nil
histogram.removeAll()
graphSeries.removeAll()
listItems.removeAll()
tableRows.removeAll()
hex = nil
consoleEntries.removeAll()
case .snapshot:
break
}
}
public mutating func cap(to kind: InstrumentWidget.Kind) {
switch kind {
case .counter:
break
case .histogram(let cfg):
if histogram.count > cfg.maxBuckets {
histogram = Array(histogram.suffix(cfg.maxBuckets))
}
case .graph(let cfg):
for (seriesID, points) in graphSeries where points.count > cfg.maxPoints {
graphSeries[seriesID] = Array(points.suffix(cfg.maxPoints))
}
case .list(let cfg):
if listItems.count > cfg.maxItems {
listItems = Array(listItems.suffix(cfg.maxItems))
}
case .table(let cfg):
if tableRows.count > cfg.maxRows {
tableRows = Array(tableRows.suffix(cfg.maxRows))
}
case .hex(let cfg):
if let h = hex, h.bytes.count > cfg.maxBytes {
let drop = h.bytes.count - cfg.maxBytes
hex = WidgetHexState(
bytes: h.bytes.suffix(cfg.maxBytes),
baseAddress: h.baseAddress &+ UInt64(drop)
)
}
case .console(let cfg):
if consoleEntries.count > cfg.maxEntries {
consoleEntries = Array(consoleEntries.suffix(cfg.maxEntries))
}
}
}
}