mirror of
https://github.com/Twigpine/zero.git
synced 2026-11-11 18:00:54 -10:00
* fix(agent): abort on image-rejection 400s instead of retrying in a loop
* feat(imageinput): add ReadClipboardImage for OS clipboard image extraction
* feat(tui): clipboard image paste — detect and attach screenshots from OS clipboard
* fix(imageinput): use temp file for Windows clipboard image (PowerShell stdout can't emit raw binary)
* test: handle both image-present and image-absent clipboard states
* fix(vision): add kimi-k2 and qwen-vl to vision-capable name heuristic
* fix(tui): render image errors as red error rows instead of grey system notes
* feat(vision): check models.dev InputModalities for vision support instead of only name heuristics
The vision gate now checks three sources:
1. Curated model registry (catalog authority + name heuristic)
2. Discovered model list from models.dev (InputModalities contains 'image')
3. Name heuristic fallback for unknown models
This means custom/ollama/cloud models (kimi-k2, qwen-vl, etc.) are correctly
detected as vision-capable when models.dev reports them with image input
modality — no more manual name matching needed.
* chore: gofmt clipboard_test.go
* fix(imageinput): remove shell injection in the Linux clipboard reader
The Linux clipboard reader built shell commands by concatenating a
clipboard-derived MIME type into `sh -c`:
exec.Command("sh", "-c", "wl-paste --type "+t+" 2>/dev/null")
exec.Command("sh", "-c", "xclip ... -t "+t+" -o 2>/dev/null")
`t` comes from `wl-paste --list-types` / `xclip TARGETS` and is only filtered by
`HasPrefix(t, "image/")`, so a hostile clipboard offerer could register a target
like `image/png; rm -rf ~` — it passes the prefix check and then executes
through the shell. A real (if low-likelihood) local code-exec vector.
Drop the shell entirely: every helper now runs via exec.Command(prog, args...)
with the MIME type passed as a discrete argument (matching the existing no-shell
pattern in pdf.go). Stderr is discarded by not assigning it. Factored into
runClipboardStdout + imageMIMETypes helpers.
Test: imageMIMETypes filters to image/* lines and returns even an
injection-shaped type verbatim (safe now that it never reaches a shell).
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
* test(tools): de-flake TestWriteStdinInterruptTerminatesSession
The test had two Windows-CI flakes that reddened unrelated PRs:
1. It relied on write_stdin's 1000ms yield window being long enough for the
async SIGKILL + reap to land before asserting the session was no longer
running. On a loaded runner the reap exceeded that window, so write_stdin
reported "still running" and the assertion failed. Fix: after terminating,
wait on session.done deterministically (30s safety timeout that fails loudly
if the kill genuinely hangs); the common case returns the instant the
process exits.
2. The SIGKILL'd child had the test's t.TempDir() as its cwd, and Windows may
not release the directory handle the instant the process is reaped — so
t.TempDir()'s immediate RemoveAll on cleanup failed with "being used by
another process". Fix: resilientTempDir retries RemoveAll for up to 5s
(best-effort; a leaked temp dir never fails the test).
Verified 20x locally; full internal/tools suite green.
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
---------
Co-authored-by: Claude Opus 4.8 (1M context) <[email protected]>
177 lines
6.1 KiB
Go
177 lines
6.1 KiB
Go
package imageinput
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"runtime"
|
|
"strings"
|
|
|
|
"github.com/Gitlawb/zero/internal/zeroruntime"
|
|
)
|
|
|
|
// ReadClipboardImage returns the raw image bytes and media type from the OS
|
|
// clipboard, or (nil, "", nil) when the clipboard has no image. Called when
|
|
// text clipboard is empty (the user pasted a screenshot). The media type is
|
|
// sniffed from the bytes, not trusted from the clipboard.
|
|
func ReadClipboardImage() ([]byte, string, error) {
|
|
data, err := readClipboardImageBytes()
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
if data == nil {
|
|
return nil, "", nil
|
|
}
|
|
// Sniff the media type from the bytes — don't trust the clipboard's claim.
|
|
sniffLen := len(data)
|
|
if sniffLen > 512 {
|
|
sniffLen = 512
|
|
}
|
|
mediaType := zeroruntime.NormalizeImageMediaType(http.DetectContentType(data[:sniffLen]))
|
|
if mediaType == "" {
|
|
return nil, "", fmt.Errorf("clipboard image is not a supported type (allowed: png, jpeg, gif, webp)")
|
|
}
|
|
return data, mediaType, nil
|
|
}
|
|
|
|
// readClipboardImageBytes calls the platform-specific clipboard tool to extract
|
|
// image bytes. Returns (nil, nil) when no image is present.
|
|
func readClipboardImageBytes() ([]byte, error) {
|
|
switch runtime.GOOS {
|
|
case "windows":
|
|
return readClipboardImageWindows()
|
|
case "darwin":
|
|
return readClipboardImageDarwin()
|
|
case "linux":
|
|
return readClipboardImageLinux()
|
|
default:
|
|
return nil, nil
|
|
}
|
|
}
|
|
|
|
// readClipboardImageWindows uses PowerShell to check for and read a clipboard
|
|
// image. The image is saved as PNG to a temp file, read back, and the temp file
|
|
// deleted. Returns (nil, nil) when no image is on the clipboard.
|
|
func readClipboardImageWindows() ([]byte, error) {
|
|
// Check if the clipboard contains an image.
|
|
check := `Add-Type -AssemblyName System.Windows.Forms; Add-Type -AssemblyName System.Drawing; [System.Windows.Forms.Clipboard]::ContainsImage()`
|
|
out, err := exec.Command("powershell", "-NoProfile", "-Command", check).Output()
|
|
if err != nil {
|
|
return nil, nil // clipboard not available, treat as no image
|
|
}
|
|
if strings.TrimSpace(string(out)) != "True" {
|
|
return nil, nil
|
|
}
|
|
// Save the clipboard image as PNG to a temp file, then read the bytes.
|
|
// PowerShell stdout can't reliably emit raw binary — $ms.ToArray() prints
|
|
// a .NET byte array as space-separated text, not raw bytes. A temp file
|
|
// is the correct binary-safe path.
|
|
tmpFile, err := os.CreateTemp("", "zero-clipboard-*.png")
|
|
if err != nil {
|
|
return nil, nil
|
|
}
|
|
tmpPath := tmpFile.Name()
|
|
tmpFile.Close()
|
|
defer os.Remove(tmpPath)
|
|
script := `Add-Type -AssemblyName System.Windows.Forms; Add-Type -AssemblyName System.Drawing; $img = [System.Windows.Forms.Clipboard]::GetImage(); if ($img -ne $null) { $img.Save('` + tmpPath + `', [System.Drawing.Imaging.ImageFormat]::Png) }`
|
|
cmd := exec.Command("powershell", "-NoProfile", "-Command", script)
|
|
if err := cmd.Run(); err != nil {
|
|
return nil, nil
|
|
}
|
|
data, err := os.ReadFile(tmpPath)
|
|
if err != nil || len(data) == 0 {
|
|
return nil, nil
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
// readClipboardImageDarwin uses osascript to check for and read a clipboard
|
|
// image. Returns (nil, nil) when no image is present.
|
|
func readClipboardImageDarwin() ([]byte, error) {
|
|
// Check clipboard info for image classes.
|
|
check := `osascript -e 'clipboard info'`
|
|
out, err := exec.Command("sh", "-c", check).Output()
|
|
if err != nil {
|
|
return nil, nil
|
|
}
|
|
info := string(out)
|
|
if !strings.Contains(info, "PNG") && !strings.Contains(info, "JPEG") && !strings.Contains(info, "TIFF") && !strings.Contains(info, "GIF") {
|
|
return nil, nil
|
|
}
|
|
// Write clipboard image to a temp file via AppleScript, then read it.
|
|
// Using pngpaste if available, falling back to a Python one-liner.
|
|
cmd := exec.Command("sh", "-c", `pngpaste - 2>/dev/null || python3 -c "
|
|
import AppKit, sys
|
|
pb = AppKit.NSPasteboard.generalPasteboard()
|
|
data = pb.dataForType_(AppKit.NSPasteboardTypePNG)
|
|
if data:
|
|
sys.stdout.buffer.write(data.bytes())
|
|
"`)
|
|
var stdout bytes.Buffer
|
|
cmd.Stdout = &stdout
|
|
if err := cmd.Run(); err != nil {
|
|
return nil, nil
|
|
}
|
|
if stdout.Len() == 0 {
|
|
return nil, nil
|
|
}
|
|
return stdout.Bytes(), nil
|
|
}
|
|
|
|
// readClipboardImageLinux tries wl-paste (Wayland) then xclip (X11) to read
|
|
// clipboard image bytes. Returns (nil, nil) when no image or no tool available.
|
|
//
|
|
// The MIME type comes from the clipboard (wl-paste --list-types / xclip
|
|
// TARGETS), so it is NEVER interpolated into a shell — every command runs via
|
|
// exec.Command(prog, args...) with the type passed as a discrete argument.
|
|
// (A hostile clipboard offerer could otherwise register a target like
|
|
// "image/png; rm -rf ~" that passes the "image/" prefix check.)
|
|
func readClipboardImageLinux() ([]byte, error) {
|
|
// Try Wayland first.
|
|
if types, err := runClipboardStdout("wl-paste", "--list-types"); err == nil {
|
|
for _, t := range imageMIMETypes(types) {
|
|
if data, err := runClipboardStdout("wl-paste", "--type", t); err == nil && len(data) > 0 {
|
|
return data, nil
|
|
}
|
|
}
|
|
}
|
|
// Fall back to X11 xclip.
|
|
if types, err := runClipboardStdout("xclip", "-selection", "clipboard", "-t", "TARGETS", "-o"); err == nil {
|
|
for _, t := range imageMIMETypes(types) {
|
|
if data, err := runClipboardStdout("xclip", "-selection", "clipboard", "-t", t, "-o"); err == nil && len(data) > 0 {
|
|
return data, nil
|
|
}
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// runClipboardStdout runs a clipboard helper and returns only its stdout.
|
|
// Stderr is discarded (the helpers are noisy when the clipboard is empty or the
|
|
// tool is missing); a missing tool surfaces as the command error, treated as
|
|
// "no image" by the callers.
|
|
func runClipboardStdout(name string, args ...string) ([]byte, error) {
|
|
cmd := exec.Command(name, args...)
|
|
var stdout bytes.Buffer
|
|
cmd.Stdout = &stdout
|
|
if err := cmd.Run(); err != nil {
|
|
return nil, err
|
|
}
|
|
return stdout.Bytes(), nil
|
|
}
|
|
|
|
// imageMIMETypes extracts the "image/*" lines from a newline-separated type
|
|
// list, each safe to pass as a discrete argument (no shell).
|
|
func imageMIMETypes(list []byte) []string {
|
|
var out []string
|
|
for _, line := range strings.Split(string(list), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if strings.HasPrefix(line, "image/") {
|
|
out = append(out, line)
|
|
}
|
|
}
|
|
return out
|
|
}
|