Niko1221/Strata
1
0 Fork 0
mirror of https://github.com/Niko1221/Strata.git synced 2026-11-12 18:51:35 -10:00
Code Issues Packages Projects Releases 1 Wiki Activity
Files
main
Strata/tools/vision/strata_vision.cpp
T
Niko1221 828a2559a9 vision: the image-token limit has a default and is enforced (I7, #3690) 
The reader was clamped only when the config carried vision.max_tokens; an old, hand-edited or migrated config left it unclamped and
one picture became 93,700 image tokens. Vision.token_limit gives every reader a limit (GPU 1024, CPU 300, the numbers setup writes)
unless the config names one (a value the person set is kept, 0 still means none). strata-vision refuses a picture that comes back
over the limit plus an eighth with a plain ERR instead of passing 90 times the budget on, and prints a WARNING when its own warm-up
picture is over it. The server logs a picture that is more than 4 times the limit with the numbers to send with a report.
2026-10-22 13:10:12 +02:00

344 lines
18 KiB
C++
Raw Permalink Blame History

// tools/vision/strata_vision.cpp - the image half of Strata's multimodal path.
//
// Turns an image into the embeddings the text model reads in place of its <|image_pad|> tokens, with llama.cpp's
// mtmd library and the model's mmproj file (vision encoder + projector). The engine does the rest: it places the
// rows at the image's pad tokens and gives them their 2-D M-RoPE positions (see --serve GENI in generate.cpp).
//
// strata-vision --mmproj <mmproj.gguf> --model <text model .gguf, first split> [--gpu] [--threads N]
// [--max-tokens N] [--min-tokens N] [--flash-attn on|off|auto]
// Flash attention defaults to auto, and to off on the CPU when ggml is built with AVX-512 (see main).
//
// Resident: prints "READY <n_embd>", then per stdin line
// ENC <image path> <output path> -> "OK <n_tokens> <nx> <ny> <ms>" or "ERR <message>"
// QUIT
// The output file is int32 {0x31455653 'SVE1', n_tokens, nx, ny, n_embd} then float32 [n_tokens][n_embd],
// row i at grid position (x = i % nx, y = i / nx). The text model is opened vocab-only (no weights).
#include "ggml-backend.h"
#include "ggml-cpu.h"
#include "gguf.h"
#include "gguf_meta.h"
#include "gpu_arch.h"
#include "llama.h"
#include "mtmd.h"
#include "mtmd-helper.h"
#include <algorithm>
#if !defined(_WIN32)
#include <fcntl.h>
#include <unistd.h>
#elif defined(STRATA_VISION_HIP)
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#endif
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <sstream>
#include <string>
#include <thread>
#include <vector>
namespace {
// the GPU architectures ggml-hip printed while it started (" Device 0: ..., gfx1201:sramecc-:xnack- (0x1201), ...")
std::vector<std::string> g_gfx;
void quiet_log(ggml_log_level level, const char* text, void*) {
if (level == GGML_LOG_LEVEL_INFO && std::strstr(text, "Device ") != nullptr) {
const std::string gfx = strata_gfx_in(text);
if (!gfx.empty()) g_gfx.push_back(gfx);
}
if (level >= GGML_LOG_LEVEL_WARN) std::fputs(text, stderr);
}
// "ENC <image> <out>": paths may contain spaces, so the image path ends at the last " " before the output path;
// the server always writes paths without spaces, but a user calling the tool by hand might not
bool parse_enc(const std::string& line, std::string& img, std::string& out) {
if (line.rfind("ENC ", 0) != 0) return false;
const std::string rest = line.substr(4);
const size_t sp = rest.rfind(' ');
if (sp == std::string::npos || sp == 0) return false;
img = rest.substr(0, sp);
out = rest.substr(sp + 1);
return !img.empty() && !out.empty();
}
} // namespace
int main(int argc, char** argv) {
#if defined(_WIN32) && defined(STRATA_VISION_HIP)
// hipblas.dll is delay-loaded (tools/vision/CMakeLists.txt): it and the ROCm DLLs behind it (rocblas.dll finds its kernel
// library and ../.kpack beside itself) stay in rocm\bin of the engine package, which is not on the DLL search path of a
// program started with nothing but System32 in PATH. Put that folder on it before the first call.
{
wchar_t exe[MAX_PATH];
const DWORD n = GetModuleFileNameW(nullptr, exe, MAX_PATH);
std::wstring dir(exe, n);
const size_t cut = dir.find_last_of(L"\\/");
if (cut != std::wstring::npos) {
dir.resize(cut);
const std::wstring rocm = dir + L"\\rocm\\bin";
if (GetFileAttributesW(rocm.c_str()) != INVALID_FILE_ATTRIBUTES) SetDllDirectoryW(rocm.c_str());
}
}
#endif
std::string mmproj, model;
bool gpu = false;
int threads = 0, max_tokens = 0, min_tokens = 0;
llama_flash_attn_type fa = LLAMA_FLASH_ATTN_TYPE_AUTO;
bool fa_given = false;
for (int i = 1; i < argc; ++i) {
const std::string a = argv[i];
auto next = [&]() -> std::string {
if (i + 1 >= argc) { std::fprintf(stderr, "%s needs a value\n", a.c_str()); std::exit(2); }
return argv[++i];
};
if (a == "--mmproj") mmproj = next();
else if (a == "--model") model = next();
else if (a == "--gpu") gpu = true;
else if (a == "--version") {
#ifdef STRATA_VISION_HIP
std::printf("strata-vision hip (GPU architectures: %s)\n", STRATA_VISION_HIP_ARCHS);
#elif defined(STRATA_VISION_METAL)
std::printf("strata-vision metal\n");
#elif defined(STRATA_VISION_SYCL)
std::printf("strata-vision sycl (Intel GPU)\n");
#else
std::printf("strata-vision\n");
#endif
return 0;
}
else if (a == "--threads") threads = std::atoi(next().c_str());
else if (a == "--max-tokens") max_tokens = std::atoi(next().c_str());
else if (a == "--min-tokens") min_tokens = std::atoi(next().c_str()); // mtmd image_min_tokens (#767)
else if (a == "--flash-attn") { // FA keeps K and V in FP16; off = the attention in FP32
const std::string v = next();
fa = v == "on" ? LLAMA_FLASH_ATTN_TYPE_ENABLED : v == "off" ? LLAMA_FLASH_ATTN_TYPE_DISABLED
: LLAMA_FLASH_ATTN_TYPE_AUTO;
fa_given = true;
}
else { std::fprintf(stderr, "unknown argument %s\n", a.c_str()); return 2; }
}
if (mmproj.empty() || model.empty()) {
std::fprintf(stderr, "usage: strata-vision --mmproj <mmproj.gguf> --model <model.gguf> [--gpu] [--threads N] "
"[--max-tokens N] [--min-tokens N] [--flash-attn on|off|auto]\n");
return 2;
}
// On the CPU the GPU stays unseen: a CUDA build otherwise opens a context there (measured: 0.4-0.7 GB of VRAM,
// 150-260 expert slots less for the engine beside it). Before anything reaches the CUDA runtime.
if (!gpu) {
#ifdef _WIN32
_putenv_s("CUDA_VISIBLE_DEVICES", "-1");
_putenv_s("HIP_VISIBLE_DEVICES", "-1");
#else
setenv("CUDA_VISIBLE_DEVICES", "-1", 1);
setenv("HIP_VISIBLE_DEVICES", "-1", 1);
#endif
}
llama_log_set(quiet_log, nullptr);
mtmd_helper_log_set(quiet_log, nullptr);
llama_backend_init();
#if defined(STRATA_VISION_HIP) || defined(STRATA_VISION_SYCL)
#ifdef STRATA_VISION_SYCL
#define STRATA_VISION_BUILT_FOR "Intel GPUs"
#else
#define STRATA_VISION_BUILT_FOR STRATA_VISION_HIP_ARCHS
#endif
if (gpu) {
// The AMD / Intel GPU reader: starting the backends prints the GPU line. With --gpu and no GPU, mtmd would quietly use the CPU
// (a second, slower CPU reader inside the HIP program), and a card the build has no code for would abort at its first
// kernel. Both end here with a line the server reads: it then starts the CPU reader (serve/server.py, Vision._start).
bool have_gpu = false;
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
const auto type = ggml_backend_dev_type(ggml_backend_dev_get(i));
have_gpu = have_gpu || type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU;
}
if (!have_gpu) {
std::printf("ERR this picture reader found no GPU (it was built for: %s)\n", STRATA_VISION_BUILT_FOR);
std::fflush(stdout);
return 3;
}
#ifdef STRATA_VISION_HIP
const char* check = std::getenv("STRATA_VISION_ARCH_CHECK");
if (!g_gfx.empty() && (check == nullptr || std::atoi(check) != 0) &&
!strata_arch_listed(STRATA_VISION_HIP_ARCHS, g_gfx.front())) {
std::printf("ERR this GPU (%s) is not in the picture reader's build (%s)\n", g_gfx.front().c_str(),
STRATA_VISION_HIP_ARCHS);
std::fflush(stdout);
return 3;
}
#endif
}
#endif
#ifdef STRATA_VISION_METAL
if (gpu) {
// The Mac: the same program reads on the GPU (--gpu, ggml's Metal backend) or on the CPU. With --gpu and no Metal device (a
// virtual machine without a GPU, say), mtmd would quietly use the CPU, which reads a picture in ~50 s: say so instead, and
// the server starts the program again without --gpu (serve/server.py, Vision._start) with one log line.
bool have_gpu = false;
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
const auto type = ggml_backend_dev_type(ggml_backend_dev_get(i));
have_gpu = have_gpu || type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU;
}
if (!have_gpu) {
std::printf("ERR this Mac has no Metal GPU for the picture reader\n");
std::fflush(stdout);
return 3;
}
}
#endif
const auto load_t0 = std::chrono::steady_clock::now();
#if !defined(_WIN32) && !defined(__APPLE__) // macOS has no posix_fadvise
// ask the OS for the projector file up front (128 KiB steps) so the loader's sequential reads find it cached;
// STRATA_READ_AHEAD=0 turns it off, as in the engine
if (const char* ra = std::getenv("STRATA_READ_AHEAD"); ra == nullptr || std::atoi(ra) != 0) {
const int fd = open(mmproj.c_str(), O_RDONLY);
if (fd >= 0) {
const off_t n = lseek(fd, 0, SEEK_END);
for (off_t at = 0; at < n; at += (off_t) (128 << 10))
(void) posix_fadvise(fd, at, std::min<off_t>((off_t) (128 << 10), n - at), POSIX_FADV_WILLNEED);
close(fd);
}
}
#endif
// the vocabulary and header only: the weights (and their tensor types, which can be Strata-private ones such as
// Gyro's TQK6 / TQK7) are never touched - see gguf_meta.h
std::string how;
llama_model* text = strata_open_text_model(model, &how);
if (!text) { std::printf("ERR cannot open the text model %s (%s)\n", model.c_str(), how.c_str()); std::fflush(stdout); return 1; }
mtmd_context_params cp = mtmd_context_params_default();
cp.use_gpu = gpu;
cp.print_timings = false;
cp.warmup = false;
// On the CPU "auto" turns flash attention on. ggml's fast (tiled) CPU kernel for it needs the head size, 72 in this
// encoder, to be a multiple of the vector width: 8 floats with AVX2 (the release builds), 16 with AVX-512 (a build
// from source on a CPU that has it), where ggml falls back to a kernel that is several times slower and accumulates
// in FP16. A 1024x1024 picture on a Ryzen 7 7700X, 8 threads: AVX2 8-10 s with it, 14-15 s without; AVX-512 44 s
// with it (26% off the FP32 attention's output), 13 s without.
if (!gpu && !fa_given && ggml_cpu_has_avx512()) fa = LLAMA_FLASH_ATTN_TYPE_DISABLED;
cp.flash_attn_type = fa;
// on the CPU without --threads: one per core (mtmd's own default is 4 threads)
// (at most 16: a 256-thread server does not need 128 encoder threads; --threads overrides)
if (threads <= 0 && !gpu) threads = std::max(1u, std::min(16u, std::thread::hardware_concurrency() / 2));
if (threads > 0) cp.n_threads = threads;
if (max_tokens > 0) cp.image_max_tokens = max_tokens;
if (min_tokens > 0) cp.image_min_tokens = min_tokens;
mtmd_context* ctx = mtmd_init_from_file(mmproj.c_str(), text, cp);
if (!ctx || !mtmd_support_vision(ctx)) {
std::printf("ERR cannot load the vision encoder %s\n", mmproj.c_str());
std::fflush(stdout);
return 1;
}
std::fprintf(stderr, "strata-vision: model files loaded in %.1f s\n",
std::chrono::duration<double>(std::chrono::steady_clock::now() - load_t0).count());
// a vocab-only model reports no hparams, so the width comes from the projector: its output is the model's input
int n_embd = 0;
{
gguf_init_params gp{true, nullptr};
gguf_context* gg = gguf_init_from_file(mmproj.c_str(), gp);
const int64_t k = gg ? gguf_find_key(gg, "clip.vision.projection_dim") : -1;
if (k >= 0) n_embd = (int) gguf_get_val_u32(gg, k);
if (gg) gguf_free(gg);
}
if (n_embd <= 0) { std::printf("ERR the vision encoder has no projection_dim\n"); std::fflush(stdout); return 1; }
// Warm up at the LARGEST picture before READY: the encoder's GPU work buffers are allocated now, not at the first
// real picture. The server starts this process before the engine, so the engine sizes its expert cache from
// what is really left; allocating ~1 GB later, on a GPU the engine has filled, made Windows page GPU memory and
// the engine crawl to a standstill. (A square image well above any cap; mtmd scales it to the token limit.)
// On the CPU there is no VRAM to reserve, and the warm-up would only delay the engine's start by one encode
// (~6 s at 1,024 tokens).
if (!gpu) std::fprintf(stderr, "strata-vision: on the CPU, %d threads, no warm-up\n", threads);
else {
const auto warm_t0 = std::chrono::steady_clock::now();
const uint32_t side = 2048;
std::vector<unsigned char> rgb((size_t) side * side * 3, 128);
mtmd_bitmap* bm = mtmd_bitmap_init(side, side, rgb.data());
mtmd_input_chunks* chunks = mtmd_input_chunks_init();
const std::string marker = mtmd_default_marker();
mtmd_input_text txt{marker.c_str(), marker.size(), false, true};
const mtmd_bitmap* bms[1] = {bm};
int warm_tokens = 0;
if (bm && mtmd_tokenize(ctx, chunks, &txt, bms, 1) == 0) {
for (size_t c = 0; c < mtmd_input_chunks_size(chunks); ++c) {
const mtmd_input_chunk* ch = mtmd_input_chunks_get(chunks, c);
if (mtmd_input_chunk_get_type(ch) == MTMD_INPUT_CHUNK_TYPE_IMAGE && mtmd_encode_chunk(ctx, ch) == 0)
warm_tokens = (int) mtmd_input_chunk_get_n_tokens(ch);
}
}
std::fprintf(stderr, "strata-vision: warmed up at %d image tokens in %.1f s\n", warm_tokens,
std::chrono::duration<double>(std::chrono::steady_clock::now() - warm_t0).count());
// I7 (#3690): a picture that became 93,700 image tokens under --max-tokens 1024. The limit is not applied by this build of mtmd for
// this model, and the log line above is the evidence to ask for.
if (max_tokens > 0 && warm_tokens > max_tokens + max_tokens / 8)
std::fprintf(stderr, "strata-vision: WARNING the warm-up picture became %d image tokens, over the limit of %d (--max-tokens): "
"the limit is not applied for this model; pictures over it are refused\n", warm_tokens, max_tokens);
mtmd_input_chunks_free(chunks);
if (bm) mtmd_bitmap_free(bm);
}
std::printf("READY %d\n", n_embd);
std::fflush(stdout);
std::string line;
while (std::getline(std::cin, line)) {
while (!line.empty() && (line.back() == '\r' || line.back() == '\n')) line.pop_back();
if (line == "QUIT") break;
std::string img, out;
if (!parse_enc(line, img, out)) { std::printf("ERR expected: ENC <image> <output>\n"); std::fflush(stdout); continue; }
const auto t0 = std::chrono::steady_clock::now();
mtmd_helper_bitmap_wrapper bw = mtmd_helper_bitmap_init_from_file(ctx, img.c_str(), false,
mtmd_helper_init_opt_default());
if (!bw.bitmap) { std::printf("ERR cannot read the image %s\n", img.c_str()); std::fflush(stdout); continue; }
mtmd_input_chunks* chunks = mtmd_input_chunks_init();
const std::string prompt = mtmd_default_marker();
mtmd_input_text txt{prompt.c_str(), prompt.size(), false, true};
const mtmd_bitmap* bm[1] = {bw.bitmap};
std::string err;
const mtmd_input_chunk* ichunk = nullptr;
if (mtmd_tokenize(ctx, chunks, &txt, bm, 1) != 0) err = "the image could not be preprocessed";
for (size_t c = 0; err.empty() && c < mtmd_input_chunks_size(chunks); ++c) {
const mtmd_input_chunk* ch = mtmd_input_chunks_get(chunks, c);
if (mtmd_input_chunk_get_type(ch) == MTMD_INPUT_CHUNK_TYPE_IMAGE) ichunk = ch;
}
if (err.empty() && !ichunk) err = "no image chunk";
if (err.empty() && mtmd_encode_chunk(ctx, ichunk) != 0) err = "the vision encoder failed";
if (err.empty()) {
const mtmd_image_tokens* it = mtmd_input_chunk_get_tokens_image(ichunk);
const int n = (int) mtmd_input_chunk_get_n_tokens(ichunk);
// the grid from the decoder positions (nx/ny getters are deprecated): x and y of the last token
const mtmd_decoder_pos last = mtmd_image_tokens_get_decoder_pos(it, 0, (size_t) n - 1);
const int nx = (int) last.x + 1, ny = (int) last.y + 1;
if (nx * ny != n) err = "the image grid is not rectangular (" + std::to_string(n) + " tokens)";
// never silently produce many times the budget (I7, #3690): over the limit (a rounding margin of an eighth) the picture is refused
if (err.empty() && max_tokens > 0 && n > max_tokens + max_tokens / 8)
err = "the picture became " + std::to_string(n) + " image tokens, over the limit of " + std::to_string(max_tokens) +
" (vision.max_tokens); send a smaller picture or raise the limit";
const float* embd = mtmd_get_output_embd(ctx);
FILE* f = err.empty() ? std::fopen(out.c_str(), "wb") : nullptr;
if (err.empty() && !f) err = "cannot write " + out;
if (f) {
const int32_t hdr[5] = {0x31455653, n, nx, ny, n_embd};
const bool ok = std::fwrite(hdr, sizeof hdr, 1, f) == 1 &&
std::fwrite(embd, sizeof(float) * (size_t) n_embd, (size_t) n, f) == (size_t) n;
std::fclose(f);
if (!ok) err = "short write to " + out;
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
if (ok) std::printf("OK %d %d %d %.0f\n", n, nx, ny, ms);
}
}
if (!err.empty()) std::printf("ERR %s\n", err.c_str());
std::fflush(stdout);
mtmd_input_chunks_free(chunks);
mtmd_bitmap_free(bw.bitmap);
if (bw.video_ctx) mtmd_helper_video_free(bw.video_ctx);
}
mtmd_free(ctx);
llama_model_free(text);
llama_backend_free();
return 0;
}
Reference in new issue View Git Blame Copy Permalink