mirror of
https://github.com/gufo-org/gufo.git
synced 2026-11-07 09:35:59 -10:00
feat(embeddings): support video inputs (#869)
This commit is contained in:
Federico Izzo authored and GitHub committed
1 parent
de15fdb91f
commit
a3edb56f8f
55 files changed
+4252
-54
No files matched your search
@@ -4,5 +4,7 @@
|
||||
tests/fixtures/** text eol=lf
|
||||
# Compressed fixtures are binary (zip CRLF bytes must survive).
|
||||
*.npz binary
|
||||
*.mp4 binary
|
||||
*.webm binary
|
||||
# cmd.exe parses batch files reliably only with CRLF line endings.
|
||||
*.cmd text eol=crlf
|
||||
@@ -270,6 +270,12 @@ jobs:
|
||||
if (Test-Path "$build\$dir") { Copy-Item "$build\$dir" $dist -Recurse }
|
||||
}
|
||||
Copy-Item "$build\vcpkg_installed\x64-windows-release\bin\*.dll" $dist -Force
|
||||
if (Test-Path "$build\gufo-video-libraries.txt") {
|
||||
Copy-Item "$build\gufo-video-libraries.txt" $dist
|
||||
$videoTest = Get-ChildItem $build -Recurse -Filter 'embedding_gemma2_video_processor_test.exe' | Select-Object -First 1
|
||||
if (-not $videoTest) { throw 'video decoder CPU test was not built' }
|
||||
Copy-Item $videoTest.FullName $dist
|
||||
}
|
||||
Copy-Item "$env:GUFO_PACKAGING_ROOT/licenses/llvm-openmp.txt" "$dist/llvm-openmp-LICENSE.txt"
|
||||
'LLVM OpenMP 18.1.8, source: https://github.com/llvm/llvm-project/tree/3b5b5c1ec4a3095ab096dd780e84d7ab81f3d7ff/openmp; Apache-2.0 WITH LLVM-exception; built with OPENMP_MSVC_NAME_SCHEME=ON.' |
|
||||
Set-Content "$dist/llvm-openmp-version.txt" -Encoding ascii
|
||||
@@ -316,6 +322,10 @@ jobs:
|
||||
$output = & ./gufo.exe $argument
|
||||
if ($LASTEXITCODE -ne 0 -or -not $output) { throw "bundled gufo $argument failed" }
|
||||
}
|
||||
if (Test-Path './embedding_gemma2_video_processor_test.exe') {
|
||||
& ./embedding_gemma2_video_processor_test.exe tests/fixtures/embedding_gemma2/video_golden.json
|
||||
if ($LASTEXITCODE -ne 0) { throw 'bundled video decoder checks failed' }
|
||||
}
|
||||
& ./gufo.exe diagnose --section platform
|
||||
if ($LASTEXITCODE -ne 0) { throw 'extracted bundle platform diagnosis failed' }
|
||||
} finally { Pop-Location }
|
||||
@@ -341,6 +351,9 @@ jobs:
|
||||
New-Item -ItemType Directory -Force -Path $dist | Out-Null
|
||||
Copy-Item "$bundle\gufo.exe", "$bundle\*.dll", LICENSE, NOTICE, THIRD_PARTY_NOTICES.md $dist
|
||||
Copy-Item "$bundle/llvm-openmp-LICENSE.txt", "$bundle/llvm-openmp-version.txt" $dist
|
||||
if (Test-Path "$bundle/gufo-video-libraries.txt") {
|
||||
Copy-Item "$bundle/gufo-video-libraries.txt" $dist
|
||||
}
|
||||
Copy-Item licenses "$dist\third-party" -Recurse
|
||||
Copy-Item src\models\deepseek_v4_flash\LICENSE.ds4 "$dist\third-party"
|
||||
foreach ($dir in 'rocblas', 'hipblaslt') {
|
||||
@@ -361,6 +374,11 @@ jobs:
|
||||
if ($LASTEXITCODE -ne 0) { throw 'cannot extract Windows release' }
|
||||
python "$env:GUFO_PACKAGING_ROOT/tools/windows/check_bundle.py" $extracted
|
||||
if ($LASTEXITCODE -ne 0) { throw 'release ZIP has unshipped DLL dependencies' }
|
||||
# Use an external CPU harness beside the extracted release DLLs.
|
||||
# Tests and fixtures stay out of the published ZIP.
|
||||
if (Test-Path "$bundle/embedding_gemma2_video_processor_test.exe") {
|
||||
Copy-Item "$bundle/embedding_gemma2_video_processor_test.exe" $extracted
|
||||
}
|
||||
$env:PATH = "$extracted;$env:SystemRoot/System32;$env:SystemRoot"
|
||||
Push-Location $extracted
|
||||
try {
|
||||
@@ -371,6 +389,10 @@ jobs:
|
||||
}
|
||||
$help = & ./gufo.exe --help
|
||||
if ($LASTEXITCODE -ne 0 -or -not $help) { throw 'release help failed' }
|
||||
if (Test-Path './embedding_gemma2_video_processor_test.exe') {
|
||||
& ./embedding_gemma2_video_processor_test.exe "$bundle/tests/fixtures/embedding_gemma2/video_golden.json"
|
||||
if ($LASTEXITCODE -ne 0) { throw 'bundled video decoder checks failed' }
|
||||
}
|
||||
& ./gufo.exe diagnose --section platform
|
||||
if ($LASTEXITCODE -ne 0) { throw 'release platform diagnosis failed' }
|
||||
} finally { Pop-Location }
|
||||
|
||||
@@ -89,6 +89,7 @@ find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(WEBP REQUIRED IMPORTED_TARGET libwebp)
|
||||
pkg_check_modules(SNDFILE REQUIRED IMPORTED_TARGET sndfile)
|
||||
pkg_check_modules(SOXR REQUIRED IMPORTED_TARGET soxr)
|
||||
pkg_check_modules(AV REQUIRED IMPORTED_TARGET libavformat libavcodec libavutil libswscale)
|
||||
find_package(OpenSSL REQUIRED COMPONENTS Crypto)
|
||||
set(GUFO_FFMPEG_EXECUTABLE "ffmpeg" CACHE STRING
|
||||
"FFmpeg executable used for generated media")
|
||||
@@ -1090,6 +1091,25 @@ if (BUILD_TESTING)
|
||||
set_tests_properties(kernel_bench_test PROPERTIES
|
||||
LABELS "cpu;benchmarks;statistics;json")
|
||||
|
||||
add_library(platform_library_fixture SHARED tests/core/platform_library_fixture.cpp)
|
||||
set_target_properties(platform_library_fixture PROPERTIES
|
||||
WINDOWS_EXPORT_ALL_SYMBOLS ON
|
||||
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tests/core/fixtures"
|
||||
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tests/core")
|
||||
add_executable(platform_library_test tests/core/platform_library_test.cpp)
|
||||
target_link_libraries(platform_library_test PRIVATE gufo_platform)
|
||||
add_dependencies(platform_library_test platform_library_fixture)
|
||||
add_test(NAME platform_library_test COMMAND platform_library_test
|
||||
"$<TARGET_FILE:platform_library_fixture>")
|
||||
set_tests_properties(platform_library_test PROPERTIES LABELS "cpu;platform")
|
||||
if(NOT WIN32)
|
||||
add_test(NAME platform_library_relocated_test COMMAND platform_library_test
|
||||
"/gufo-missing-build-path/$<TARGET_FILE_NAME:platform_library_fixture>")
|
||||
set_tests_properties(platform_library_relocated_test PROPERTIES
|
||||
LABELS "cpu;platform"
|
||||
ENVIRONMENT "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:platform_library_fixture>")
|
||||
endif()
|
||||
|
||||
# the publish suite drives descriptors through dup2
|
||||
if (NOT WIN32)
|
||||
add_executable(platform_publish_test
|
||||
|
||||
@@ -197,6 +197,7 @@ limits disallow those pins. Gufo leaves `HSA_USERPTR_FOR_PAGED_MEM` unchanged.
|
||||
| ROCm HIP compiler/runtime, hipBLAS, hipBLASLt, rocBLAS | GPU execution and matrix multiplication |
|
||||
| hipCUB, rocPRIM, rocWMMA headers | Compiled GPU kernels |
|
||||
| ICU, libcurl, OpenSSL, libpng, libjpeg, libwebp | Tokenization, HTTPS, hashing and images |
|
||||
| FFmpeg development libraries | Video embedding decoding and RGB conversion |
|
||||
| libsndfile, libsoxr | Audio decoding and reference-compatible resampling |
|
||||
| FFmpeg and ffprobe | Video/audio output; invoked as separate executables |
|
||||
|
||||
@@ -208,7 +209,7 @@ For example, on Debian/Ubuntu the ordinary system libraries are:
|
||||
|
||||
```sh
|
||||
sudo apt install build-essential cmake ninja-build pkg-config \
|
||||
libicu-dev libcurl4-openssl-dev libssl-dev libpng-dev libjpeg-dev libwebp-dev libsndfile1-dev libsoxr-dev ffmpeg
|
||||
libicu-dev libcurl4-openssl-dev libssl-dev libpng-dev libjpeg-dev libwebp-dev libsndfile1-dev libsoxr-dev libavformat-dev libavcodec-dev libavutil-dev libswscale-dev ffmpeg
|
||||
|
||||
# ROCm libraries from the table, named as AMD's repository ships them.
|
||||
sudo apt install hipblas-dev hipblaslt-dev rocblas-dev \
|
||||
|
||||
@@ -28,7 +28,7 @@ Non-Nix builds must retain the notices of the versions they actually distribute.
|
||||
| libwebp | Linked; WebP decoding | `BSD-3-Clause` | 1.6.0; flake.lock | [webmproject/libwebp](https://chromium.googlesource.com/webm/libwebp) |
|
||||
| libsndfile | Linked; audio decoding | `LGPL-2.1-or-later` | 1.2.2; flake.lock | [libsndfile](https://github.com/libsndfile/libsndfile) |
|
||||
| libsoxr | Linked; audio resampling | `LGPL-2.1-or-later` | 0.1.3; flake.lock | [SoX Resampler](https://sourceforge.net/projects/soxr/) |
|
||||
| FFmpeg | Separate ffmpeg/ffprobe executables for media | `GPL-3.0-or-later` (Nix build with GPL/version3 components) | 8.1.2; flake.lock | [FFmpeg](https://github.com/FFmpeg/FFmpeg) |
|
||||
| FFmpeg | Linked libavformat/libavcodec/libavutil/libswscale for video embeddings; ffmpeg/ffprobe executables for media | `GPL-3.0-or-later` (Nix build with GPL/version3 components) | 9.0.2; flake.lock | [FFmpeg](https://github.com/FFmpeg/FFmpeg) |
|
||||
| LLVM OpenMP | Windows runtime built from upstream source; Microsoft-compatible DLL naming | `Apache-2.0 WITH LLVM-exception` | 18.1.8 (`3b5b5c1ec4a3095ab096dd780e84d7ab81f3d7ff`) | [LLVM](https://github.com/llvm/llvm-project/tree/llvmorg-18.1.8/openmp); [license](licenses/llvm-openmp.txt) |
|
||||
| GNU C/C++/OpenMP runtimes | System runtime libraries; no Gufo source import | `LGPL-2.1-or-later AND (GPL-3.0-or-later WITH GCC-exception-3.1)` | glibc/GCC packages in flake.lock | [GNU](https://www.gnu.org/software/) |
|
||||
| llama.cpp / ggml | Adapted quantization, attention and model-private HIP kernels | `MIT` | `5c0e9468378eba6bf3cc1989ff5d62fbbe4d9e3a`; attention `e9fa0781f1c25fc4fe8c86be1edc6970661ad6f0` | [llama.cpp](https://github.com/ggml-org/llama.cpp) |
|
||||
@@ -54,8 +54,9 @@ This software is based in part on the work of the Independent JPEG Group.
|
||||
The mikealanni/gufo fork retains the same Copyright (c) 2026 gufo contributors
|
||||
notice and MIT text as Gufo's root `LICENSE`.
|
||||
|
||||
FFmpeg is invoked as a separate process, not linked into Gufo. The pinned Nix
|
||||
build enables GPL/version3 components; a system FFmpeg build can have different
|
||||
Gufo loads FFmpeg libraries on demand for video embeddings and invokes
|
||||
ffmpeg/ffprobe as separate processes for media tools. The pinned Nix build
|
||||
enables GPL/version3 components; a system FFmpeg build can have different
|
||||
terms. Redistributors bundling FFmpeg or other copyleft dependencies must also
|
||||
provide their applicable license texts and corresponding source/build materials,
|
||||
not just this MIT license. The pinned Nixpkgs expressions and `flake.lock`
|
||||
|
||||
+5
-1
@@ -16,7 +16,7 @@ set(gufo_pr_targets
|
||||
arg_parser_test json_test json_constraint_test gguf_reader_test gguf_identity_test
|
||||
heap_placement_test
|
||||
platform_publish_test platform_prefault_test platform_mapped_file_test
|
||||
platform_fs_test
|
||||
platform_fs_test platform_library_test
|
||||
logit_sampler_test ggml_dequant_test
|
||||
quote_tracker_test logprobs_test sampling_test
|
||||
openai_chat_test http_server_test http_server_signals_test trace_test
|
||||
@@ -32,12 +32,16 @@ set(gufo_pr_targets
|
||||
ds4_sampling_test ds4_chat_template_test ds4_cli_test
|
||||
qwen3_asr_config_test qwen3_asr_audio_api_test
|
||||
embedding_gemma2_config_test embedding_gemma2_embeddings_api_test
|
||||
embedding_gemma2_video_processor_test
|
||||
embedding_gemma2_image_processor_test
|
||||
embedding_gemma2_batch_scheduler_test
|
||||
qwen3_tts_config_test qwen3_tts_audio_api_test
|
||||
qwen_image_21_test
|
||||
video_api_test minimax_h3_sampling_test minimax_h3_runtime_test)
|
||||
set(gufo_pr_tests ${gufo_pr_targets})
|
||||
if(NOT WIN32)
|
||||
list(APPEND gufo_pr_tests platform_library_relocated_test)
|
||||
endif()
|
||||
list(REMOVE_ITEM gufo_pr_tests
|
||||
qwen38_flash_next_config_test qwen38_flash_next_mtp_sampling_test
|
||||
qwen38_flash_next_rope_scaling_test
|
||||
|
||||
+6
-5
@@ -644,13 +644,13 @@ atomic publication.
|
||||
|
||||
## Embeddings
|
||||
|
||||
EmbeddingGemma 2 (`google/embeddinggemma-2`) serves text, image and audio embeddings from its own model process. It is a 768-dimension, BF16, 8192-token-context encoder:
|
||||
EmbeddingGemma 2 (`google/embeddinggemma-2`) serves text, image, video and audio embeddings from its own model process. It is a 768-dimension, BF16, 8192-token-context encoder:
|
||||
|
||||
```sh
|
||||
./result/bin/gufo serve --port 8080 embedding --model /persist/models/embeddinggemma-2
|
||||
```
|
||||
|
||||
`--model` must name an EmbeddingGemma 2 checkpoint directory, and `--served-model-name` overrides the default model ID `embeddinggemma-2`. The process refuses to start on any other directory and in a build without the HIP backend. There is no `--sessions` or `--context` option, because requests share one encoder. Requests that wait for the device share batches of up to 8,192 tokens, except that a request with an image or audio runs in a batch of its own. When the device is free, the oldest waiting request runs one batch for itself and the requests behind it. The batch takes whole requests in arrival order until the next one does not fit or has an image or audio. A lone request runs at once. A request that fits in one batch never spans two. A request larger than one batch is split, and after each of its batches it goes behind the requests that arrived meanwhile, so a short request waits for at most one batch of a long one. Route this process through llama-swap when one public URL should offer embeddings alongside text, speech, or images.
|
||||
`--model` must name an EmbeddingGemma 2 checkpoint directory, and `--served-model-name` overrides the default model ID `embeddinggemma-2`. The process refuses to start on any other directory and in a build without the HIP backend. There is no `--sessions` or `--context` option, because requests share one encoder. Requests that wait for the device share batches of up to 8,192 tokens, except that a request with an image, video or audio runs in a batch of its own. When the device is free, the oldest waiting request runs one batch for itself and the requests behind it. The batch takes whole requests in arrival order until the next one does not fit or has an image, video or audio. A lone request runs at once. A request that fits in one batch never spans two. A request larger than one batch is split, and after each of its batches it goes behind the requests that arrived meanwhile, so a short request waits for at most one batch of a long one. Route this process through llama-swap when one public URL should offer embeddings alongside text, speech, or images.
|
||||
|
||||
`POST /v1/embeddings` follows the OpenAI request shape:
|
||||
|
||||
@@ -684,12 +684,13 @@ The response holds one embedding per input in request order:
|
||||
- `dimensions` is optional and must be 128, 256, 512, or 768, which is the default. Any other value returns 400 `invalid_dimensions`. A smaller value keeps the leading components of the normalized vector, which is Matryoshka truncation, and renormalizes them to unit length.
|
||||
- `user` is accepted and ignored when it is a string, and a non-string `user` returns 400 `invalid_parameter_type`.
|
||||
- `prompt_name` is optional; see task prefixes below.
|
||||
- `video_options` optionally selects video FPS, frame cap, overflow strategy and timestamps; see [video inputs](models/embeddinggemma-2/README.md#video).
|
||||
- `truncate` is optional and `false` by default; see the token limit below. A non-boolean `truncate` returns 400 `invalid_parameter_type`. The server rejects other members with 400 `unsupported_field`.
|
||||
- Every vector is mean-pooled and L2-normalized, so the dot product of two vectors is their cosine similarity.
|
||||
- Each input may hold at most 8192 tokens, counting the `<bos>` and `<eos>` markers the tokenizer adds. By default the server refuses longer inputs instead of truncating them, with 400 `context_length_exceeded`, and the message names the input index. With `"truncate": true` it keeps the first 8190 tokens of each longer input between the markers instead; only `--max-request-bytes` then limits a text input's length. An input with images or audio is truncated only when all of its media tokens fit in the kept tokens; otherwise it is still refused. `usage.prompt_tokens` and `usage.total_tokens` count every token embedded for every input, markers included.
|
||||
- Errors use the standard error object. Other 400 codes are `too_many_audio`, `invalid_audio_input`, `too_many_images`, `invalid_image_input`, `parse_error`, `invalid_request_body`, `missing_input`, `empty_input`, `too_many_inputs`, `invalid_encoding_format`, and `invalid_parameter_type`. A client disconnect returns 499 `cancelled`, and a request whose client disconnects before its next batch does not use the device again. A failed encode returns 500 `embedding_failed` to every request with inputs in the failed batch. A service that is not ready returns 503 `embedding_service_unavailable`. A GPU failure that the device probe confirms returns 503 `device_lost` and shuts the server down as described under [Lifecycle and limits](#lifecycle-and-limits). The usual authentication, `--max-request-bytes`, and request logging apply.
|
||||
- Each input may hold at most 8192 tokens, counting the `<bos>` and `<eos>` markers the tokenizer adds. By default the server refuses longer inputs instead of truncating them, with 400 `context_length_exceeded`, and the message names the input index. With `"truncate": true` it keeps the first 8190 tokens of each longer input between the markers instead; only `--max-request-bytes` then limits a text input's length. An input with images, video or audio is truncated only when all of its media tokens fit in the kept tokens; otherwise it is still refused. `usage.prompt_tokens` and `usage.total_tokens` count every token embedded for every input, markers included.
|
||||
- Errors use the standard error object. Other 400 codes are `too_many_videos`, `invalid_video_input`, `invalid_video_options`, `too_many_audio`, `invalid_audio_input`, `too_many_images`, `invalid_image_input`, `parse_error`, `invalid_request_body`, `missing_input`, `empty_input`, `too_many_inputs`, `invalid_encoding_format`, and `invalid_parameter_type`. A client disconnect returns 499 `cancelled`, and a request whose client disconnects before its next batch does not use the device again. A failed encode returns 500 `embedding_failed` to every request with inputs in the failed batch. A service that is not ready returns 503 `embedding_service_unavailable`. A GPU failure that the device probe confirms returns 503 `device_lost` and shuts the server down as described under [Lifecycle and limits](#lifecycle-and-limits). The usual authentication, `--max-request-bytes`, and request logging apply.
|
||||
|
||||
Typed items follow llama.cpp's form: `{"content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,..."}},{"type":"text","text":"\n"}]}`. Each item produces one vector in the same space as text. Text parts retain their order and whitespace. PNG, JPEG and WebP use the existing image reader, including public-HTTPS checks and the shared 20 MiB / 15-second budget. Embeddings accept at most 32 images per request, 32 megapixels per image, and 32,768 pixels on either side. Image soft tokens and their markers count toward context and usage. A request with images runs in its own device batch, in arrival order, and its soft tokens count toward the 8,192-token batch. The first image request loads the vision weights in that batch. See [image embeddings](models/embeddinggemma-2/README.md#images) for exact preprocessing and index migration. Audio uses typed `input_audio` parts with raw base64, audio data URLs or public HTTPS URLs; WAV, FLAC, MP3 and Ogg are supported. Up to 32 clips of 30 seconds each share the image read budget. Actual subsampled audio tokens and their boundary markers count toward context and usage. See [audio embeddings](models/embeddinggemma-2/README.md#audio) for decoding, preprocessing, bounds and examples. Video content parts are unsupported.
|
||||
Typed items follow llama.cpp's form: `{"content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,..."}},{"type":"text","text":"\n"}]}`. Each item produces one vector in the same space as text. Text parts retain their order and whitespace. PNG, JPEG and WebP use the existing image reader, including public-HTTPS checks and the shared 20 MiB / 15-second budget. Embeddings accept at most 32 images per request, 32 megapixels per image, and 32,768 pixels on either side. Image soft tokens and their markers count toward context and usage. A request with images runs in its own device batch, in arrival order, and its soft tokens count toward the 8,192-token batch. The first image request loads the vision weights in that batch. See [image embeddings](models/embeddinggemma-2/README.md#images) for exact preprocessing and index migration. Audio uses typed `input_audio` parts with raw base64, audio data URLs or public HTTPS URLs; WAV, FLAC, MP3 and Ogg are supported. Up to 32 clips of 30 seconds each share the image read budget. Actual subsampled audio tokens and their boundary markers count toward context and usage. See [audio embeddings](models/embeddinggemma-2/README.md#audio) for decoding, preprocessing, bounds and examples. Video uses typed `input_video` parts with raw base64, video data URLs or public HTTPS URLs, producing one jointly encoded vector per input. MP4/MOV and WebM/Matroska share the same media read budget. Defaults sample 1 FPS, uniformly capped at 32 frames with 140 soft tokens per frame. See [video embeddings](models/embeddinggemma-2/README.md#video) for `video_options`, timestamp framing, audio-track treatment and decoding limits.
|
||||
|
||||
**llama.cpp's `POST /embeddings`.** Clients written for llama-server's non-OpenAI route, such as Frigate 0.18's semantic search, can use `POST /embeddings` (or its older alias `POST /embedding`) on the same process. It shares the model, tokenization, image reader, pooling, limits and errors of `/v1/embeddings`, and differs in three ways:
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
The probes are English, Spanish, and Chinese queries and passages, code, emoji, whitespace runs, literal special-token text, and one long document. They measure agreement with the reference implementation, not retrieval quality on a benchmark, and Gufo reports no MTEB score. Activations stay float32 between BF16 matrix products, so Gufo is closer to the float32 reference than the model's BF16 execution is.
|
||||
|
||||
Google published the retrieval accuracy of this checkpoint on MTEB, code, and multilingual tasks, and Gufo did not re-measure it. Video embeddings are unsupported; image and audio qualification is below. Quantized variants are unsupported and unmeasured. The model card states a 1,024-token sliding window. The checkpoint's configuration and the Hugging Face code use a radius of 512 positions on each side, or 1,025 keys, and Gufo follows them. Hugging Face's `sliding_window_bidirectional_overlay` in `transformers/masking_utils.py` keeps keys where `abs(q_idx - kv_idx) <= sliding_window`, and `create_bidirectional_sliding_window_mask` passes `config.sliding_window` (512). The `+ 1` in `modeling_embedding_gemma2.py` applies only to the flash-attention path.
|
||||
Google published the retrieval accuracy of this checkpoint on MTEB, code, and multilingual tasks, and Gufo did not re-measure it. Image, video and audio qualification is below. Quantized variants are unsupported and unmeasured. The model card states a 1,024-token sliding window. The checkpoint's configuration and the Hugging Face code use a radius of 512 positions on each side, or 1,025 keys, and Gufo follows them. Hugging Face's `sliding_window_bidirectional_overlay` in `transformers/masking_utils.py` keeps keys where `abs(q_idx - kv_idx) <= sliding_window`, and `create_bidirectional_sliding_window_mask` passes `config.sliding_window` (512). The `+ 1` in `modeling_embedding_gemma2.py` applies only to the flash-attention path.
|
||||
|
||||
## Reproduce
|
||||
|
||||
@@ -102,7 +102,7 @@ Feature and decoder fixtures are independent of the production C++ implementatio
|
||||
These establish reference agreement, rather than speech recognition, audio
|
||||
classification or retrieval benchmark accuracy. The environmental and music
|
||||
probes are synthetic; the speech recording is the existing reviewed functional
|
||||
fixture. Video remains unsupported.
|
||||
fixture. Video qualification is below.
|
||||
|
||||
The [retained evidence](artifacts/gfx1151-2026-10-26-audio.json) records revision,
|
||||
binary/harness/fixture hashes, every reference cosine, memory observations,
|
||||
@@ -112,3 +112,86 @@ idle main/candidate controls preserve byte-identical vectors and work counts;
|
||||
every request and phase is within the 5% and 10 ms margins. No overall speedup
|
||||
is claimed. Native Windows compilation is checked by CI; local numerical and
|
||||
HTTP evidence here is Linux gfx1151 only.
|
||||
|
||||
|
||||
## Video reference parity
|
||||
|
||||
Qualified on gfx1151 with checkpoint
|
||||
`914f7f89142e33e77833254d9c9b90c3cef7303b` and the official transformers
|
||||
implementation at `50e35cc0ca18ec19b80653690de553e6decf74f5`.
|
||||
`tools/models/embedding_gemma2/reference_video.py` decodes retained encoded clips
|
||||
with PyAV and runs the official processor on RGB arrays with explicit frame
|
||||
count, average FPS and duration metadata. This also covers containers such as
|
||||
WebM that omit PyAV's header frame count. Reference metadata retains the PyAV,
|
||||
FFmpeg library, torch and Hugging Face Hub versions and attention backends.
|
||||
The teacher uses float32; no Gufo code participates in fixture generation.
|
||||
|
||||
Thirteen probes cover sub-second and fractional-FPS clips, portrait and landscape,
|
||||
32-frame and overflow boundaries, floating linspace rounding, uniform/truncate selection, timestamped
|
||||
upsampling, repeated clips, embedded soundtracks, and joint text/video/image/audio.
|
||||
All selected frame indices, unpadded patch shapes and resized patch bytes match
|
||||
in the CPU test. HTTP expanded-token usage agrees with reference token counts;
|
||||
the worst cosine across thirteen video/mixed vectors is 0.99997998 versus the teacher, with
|
||||
finite normalized outputs. Existing image controls exceed 0.9999 and text/image
|
||||
GPU golden tests pass with weights loaded. This establishes reference parity on
|
||||
these fixtures, not a video retrieval benchmark.
|
||||
|
||||
The full focused video HTTP run retains 29 requests, including Matryoshka
|
||||
128/256/512, concurrent/repeated requests, malformed and oversized media,
|
||||
private/local URL refusal, pixel/duration/frame/request-work limits, context
|
||||
overflow with and without truncation, cancellation correlated with its server
|
||||
request ID, and successful recovery. Peak logged process RSS was 628 MiB in
|
||||
that run; this is host RSS, not a GPU memory measurement. Raw reports, responses,
|
||||
server logs and speed observations are retained under `artifacts/issue-761/`.
|
||||
|
||||
Reproduce the bounded smoke check with `model_services.py embedding` (see
|
||||
[functional tests](../../../tests/functional/README.md)); use
|
||||
`--embedding-suite full` for all modality suites. Generate goldens in the pinned
|
||||
reference environment with:
|
||||
|
||||
```sh
|
||||
PYTHONPATH=/path/to/pinned/transformers/src python3 \
|
||||
tools/models/embedding_gemma2/reference_video.py \
|
||||
--model models/embeddinggemma-2 \
|
||||
--output tests/fixtures/embedding_gemma2/video_golden.json
|
||||
```
|
||||
|
||||
That source needs Hugging Face Hub >=1.31.0 (qualification used 1.31.0), in
|
||||
addition to the reference tools in `nix develop`. Existing encoded fixtures are
|
||||
reused and their SHA256 values are recorded. `--case NAME` generates only one
|
||||
probe for focused investigation; a subset is not full qualification.
|
||||
|
||||
|
||||
The [retained qualification evidence](artifacts/gfx1151-2026-10-29-video.json)
|
||||
records revisions, checkpoint/binary/harness/fixture hashes, reference cosines,
|
||||
per-request latency and memory observations. The linspace rounding regression
|
||||
and affected overflow controls also pass focused HTTP replay after the review
|
||||
fix. The final three-request smoke check takes about four seconds including
|
||||
startup on this host. The standard 11-workload speed benchmark retains 60
|
||||
requests per workload. All 36 resident text/image timing pairs and nine warm-up
|
||||
pairs pass the per-request 5% and 10 ms thresholds against main `6feba906`, with
|
||||
identical work counts. No aggregate speedup is claimed. Initial harness failures
|
||||
and missing-model skips remain explicitly unqualified in the evidence record.
|
||||
Three fresh-context review rounds found one sampling issue, which was fixed;
|
||||
the last two rounds reported no findings.
|
||||
|
||||
|
||||
### CI follow-up
|
||||
|
||||
The initial Windows package built and passed CPU tests, but its dependency
|
||||
validator misclassified the Windows CNG `ncrypt.dll` imported by FFmpeg. Adding
|
||||
that inbox DLL to the allowlist fixed the native Windows package job. Codec
|
||||
DLLs remain mandatory. Tests also cover explicit manifests for libraries loaded
|
||||
on demand, and the extracted release ZIP is checked with an external video CPU
|
||||
harness without shipping test executables or fixtures to users.
|
||||
|
||||
Devlab additionally measured a consistent 18–24 ms process-init regression in
|
||||
embedding, ASR and TTS: eager FFmpeg linkage initialized its transitive libraries
|
||||
for every service. FFmpeg now loads on the first video request through an OS
|
||||
library leaf. It is absent from the executable's startup dependency graph.
|
||||
Three local matched startup pairs pass the 5%/10 ms thresholds; video CPU
|
||||
fixtures, all 30 video HTTP requests and the owned multimodal smoke check pass.
|
||||
The [CI follow-up evidence](artifacts/gfx1151-2026-10-29-video-ci.json) retains the
|
||||
original flags, matched startup observations, binary/harness identity and checks.
|
||||
Two additional independent reviews covered the fix; a release validation gap was
|
||||
addressed and the final review reported no findings.
|
||||
@@ -1,6 +1,6 @@
|
||||
# EmbeddingGemma 2
|
||||
|
||||
Native safetensors text, image and audio embeddings on gfx1151. Gufo runs [google/embeddinggemma-2](https://huggingface.co/google/embeddinggemma-2), a 768-dimensional embedding model for 100+ languages and code, through `POST /v1/embeddings`. Text, images and audio share one vector space. Video inputs remain unsupported. Python runs only in offline reference checks.
|
||||
Native safetensors text, image, video and audio embeddings on gfx1151. Gufo runs [google/embeddinggemma-2](https://huggingface.co/google/embeddinggemma-2), a 768-dimensional embedding model for 100+ languages and code, through `POST /v1/embeddings`. Text, images, video and audio share one vector space. Python runs only in offline reference checks.
|
||||
|
||||
[Benchmarks](BENCHMARKS.md) · [Quality](QUALITY.md) · [Experiments](EXPERIMENTS.md) · [ROCm 10.1](ROCM.md)
|
||||
|
||||
@@ -111,6 +111,69 @@ matrix inputs. An audio request runs in its own device batch. Cancellation is
|
||||
checked during preprocessing and between audio layers; failures drain queued
|
||||
work and use the existing device-loss probe before the next request.
|
||||
|
||||
## Video
|
||||
|
||||
Typed `input_video` parts accept the llama.cpp `data`/`url` contract:
|
||||
|
||||
```json
|
||||
{
|
||||
"input": [{"content": [
|
||||
{"type": "text", "text": "a camera clip: "},
|
||||
{"type": "input_video", "input_video": {"data": "<base64 MP4>"}},
|
||||
{"type": "text", "text": " near the entrance"}
|
||||
]}],
|
||||
"video_options": {"fps": 1, "max_frames": 32,
|
||||
"overflow_strategy": "uniform", "add_timestamps": false}
|
||||
}
|
||||
```
|
||||
|
||||
`data` takes precedence over `url`. Either accepts raw base64, a base64 video
|
||||
data URL, or a public HTTPS URL. Local paths and private destinations are
|
||||
refused. Supported containers are MP4/MOV and WebM/Matroska, with codecs supplied
|
||||
by the linked FFmpeg build (including H.264 and VP9 in the qualified build).
|
||||
Data URL media types are `video/mp4`, `video/quicktime`, `video/webm` and
|
||||
`video/x-matroska`. All media share a 20 MiB byte budget and 15-second read
|
||||
deadline; the server's HTTP body limit also applies before base64 decoding.
|
||||
|
||||
Defaults match the pinned processor: 1 FPS, at most 32 frames, uniform overflow,
|
||||
140 soft-token budget per frame, and timestamps disabled. `video_options` applies
|
||||
to every video in the request. `fps` is a finite number in `(0, 120]`,
|
||||
`max_frames` an integer in `[1, 32]`, `overflow_strategy` is `uniform` or
|
||||
`truncate`, and `add_timestamps` is boolean. Other processor controls, including
|
||||
vision budgets, remain tracked in #763.
|
||||
|
||||
Sampling uses the decoded frame count and average source FPS: select
|
||||
`max(1, floor(duration * requested_fps))` indices by flooring source-frame
|
||||
positions, then apply the cap using uniform linspace/floor indices or take the
|
||||
first indices for `truncate`. Upsampling repeats selected frames. Timestamps
|
||||
use the selected source indices divided by source FPS, formatted as `mm:ss`.
|
||||
The decoded frame count is used even when a container omits its frame count.
|
||||
FFmpeg decodes RGB pixels without applying display rotation, as the pinned PyAV
|
||||
reference does. Frame size changes within a clip are refused.
|
||||
|
||||
Each frame uses image boundaries around **video** placeholders, preserving its
|
||||
position among text, images, other videos and separate audio parts. All features
|
||||
pass through the shared text backbone and produce one normalized vector per
|
||||
input. Expanded placeholders count toward usage and the 8,192-token context;
|
||||
`truncate` never drops part of a clip. Vision weights load lazily and remain
|
||||
shared with image requests. FFmpeg runtime libraries load on the first video
|
||||
request, preserving startup latency for other services. Missing decoder runtime
|
||||
libraries are a server error (500), not malformed media.
|
||||
|
||||
The video path takes the first video stream and ignores embedded audio tracks,
|
||||
matching the official visual decoder. To embed sound jointly, provide a separate
|
||||
`input_audio` part in the desired reference order; Gufo does not automatically
|
||||
extract or interleave the container's audio. The fixtures cover both an embedded
|
||||
soundtrack and explicit text/video/image/audio interleaving.
|
||||
|
||||
Limits per clip are 300 seconds, 4,096 decoded frames, 4 megapixels per frame,
|
||||
512 megapixels of decoded work across two passes, eight container streams and a
|
||||
15-second decode/processing deadline. A request accepts at most 32 clips and
|
||||
64 selected frames. Decoding first counts frames, then retains only the selected
|
||||
frames' unpadded patches; it never stores the entire decoded clip. Cancellation
|
||||
is checked during decoding, preprocessing and between GPU frame passes.
|
||||
Malformed or over-budget clips return 400 `invalid_video_input`.
|
||||
|
||||
## Dimensions
|
||||
|
||||
`dimensions` selects a Matryoshka size of 128, 256, 512, or 768, and defaults to 768. Gufo keeps the first components of the full vector and renormalizes. Quality falls slowly down to 256 and faster at 128, which Google recommends only for text. Vectors of different sizes, or from different models, are not comparable. Changing either requires re-embedding an index.
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"date": "2026-10-29",
|
||||
"candidate_code_revision": "739a8fdd106631724d3bb4fdbdf8ba1069529276",
|
||||
"binary_sha256": "5e164b64cea3f69051ab932843b6b76f9b6b5459f2c1c2ac4977dd8639831b3a",
|
||||
"windows_failure": {
|
||||
"run": 40848600507,
|
||||
"cause": "avformat-63.dll requires unbundled ncrypt.dll",
|
||||
"fix": "Recognize inbox Windows CNG runtime; retain codec DLL checks",
|
||||
"fixed_build_run": 40852625849,
|
||||
"fixed_build_job": 123375625555,
|
||||
"build_status": "success"
|
||||
},
|
||||
"linux_startup_failure": {
|
||||
"run": 40848680299,
|
||||
"flags": {
|
||||
"embedding-gemma2.phases": [
|
||||
{
|
||||
"group": "startup",
|
||||
"key": "startup.init",
|
||||
"label": "init",
|
||||
"base": [
|
||||
72.79733242467046,
|
||||
76.0394437238574
|
||||
],
|
||||
"head": [
|
||||
94.45651294663548,
|
||||
94.20524211600423
|
||||
],
|
||||
"verdict": "slower",
|
||||
"checked": true
|
||||
}
|
||||
],
|
||||
"qwen3-asr.phases": [
|
||||
{
|
||||
"group": "startup",
|
||||
"key": "startup.init",
|
||||
"label": "init",
|
||||
"base": [
|
||||
74.03909182175994,
|
||||
78.01739312708378
|
||||
],
|
||||
"head": [
|
||||
98.19441335275769,
|
||||
96.01445822045207
|
||||
],
|
||||
"verdict": "slower",
|
||||
"checked": true
|
||||
}
|
||||
],
|
||||
"qwen3-tts-base.phases": [
|
||||
{
|
||||
"group": "startup",
|
||||
"key": "startup.init",
|
||||
"label": "init",
|
||||
"base": [
|
||||
74.79083118960261,
|
||||
76.10890129581094
|
||||
],
|
||||
"head": [
|
||||
96.41655767336488,
|
||||
97.4049479700625
|
||||
],
|
||||
"verdict": "slower",
|
||||
"checked": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"cause": "Eager FFmpeg linking loaded codec/network DSOs before every service initialization",
|
||||
"fix": "First-use typed symbol binding through platform Library; no FFmpeg load-time dependencies"
|
||||
},
|
||||
"matched_startup": {
|
||||
"base_revision": "6feba906",
|
||||
"arms": [
|
||||
{
|
||||
"side": "base",
|
||||
"startup": {
|
||||
"listen_source": "server",
|
||||
"init": 54.3047170212958,
|
||||
"weights": 1922.0,
|
||||
"vision": null,
|
||||
"structured": null,
|
||||
"listen": 0.099
|
||||
}
|
||||
},
|
||||
{
|
||||
"side": "head",
|
||||
"startup": {
|
||||
"listen_source": "server",
|
||||
"init": 52.17735798214562,
|
||||
"weights": 1642.0,
|
||||
"vision": null,
|
||||
"structured": null,
|
||||
"listen": 0.123
|
||||
}
|
||||
},
|
||||
{
|
||||
"side": "head",
|
||||
"startup": {
|
||||
"listen_source": "server",
|
||||
"init": 51.826708018779755,
|
||||
"weights": 1635.0,
|
||||
"vision": null,
|
||||
"structured": null,
|
||||
"listen": 0.103
|
||||
}
|
||||
},
|
||||
{
|
||||
"side": "base",
|
||||
"startup": {
|
||||
"listen_source": "server",
|
||||
"init": 52.05013099475764,
|
||||
"weights": 1640.0,
|
||||
"vision": null,
|
||||
"structured": null,
|
||||
"listen": 0.11
|
||||
}
|
||||
},
|
||||
{
|
||||
"side": "base",
|
||||
"startup": {
|
||||
"listen_source": "server",
|
||||
"init": 53.738362999865785,
|
||||
"weights": 1648.0,
|
||||
"vision": null,
|
||||
"structured": null,
|
||||
"listen": 0.098
|
||||
}
|
||||
},
|
||||
{
|
||||
"side": "head",
|
||||
"startup": {
|
||||
"listen_source": "server",
|
||||
"init": 51.58790500718169,
|
||||
"weights": 1648.0,
|
||||
"vision": null,
|
||||
"structured": null,
|
||||
"listen": 0.116
|
||||
}
|
||||
}
|
||||
],
|
||||
"init_pairs": [
|
||||
{
|
||||
"base_ms": 54.3047170212958,
|
||||
"head_ms": 52.17735798214562,
|
||||
"flag": false
|
||||
},
|
||||
{
|
||||
"base_ms": 52.05013099475764,
|
||||
"head_ms": 51.826708018779755,
|
||||
"flag": false
|
||||
},
|
||||
{
|
||||
"base_ms": 53.738362999865785,
|
||||
"head_ms": 51.58790500718169,
|
||||
"flag": false
|
||||
}
|
||||
],
|
||||
"thresholds": {
|
||||
"fraction": 0.05,
|
||||
"milliseconds": 10
|
||||
}
|
||||
},
|
||||
"checks": {
|
||||
"cpu_platform_library": "passed; real DLL/so fixture, missing library and missing symbol",
|
||||
"cpu_api_video_processor": "passed; 13 independent reference probes and resource checks",
|
||||
"windows_bundle_unit_tests": "7 passed",
|
||||
"windows_library_cross_compile": "passed (MinGW x86-64 C++20)",
|
||||
"http_video": {
|
||||
"status": "passed",
|
||||
"requests": 30,
|
||||
"cancellation": "2026-10-29 11:04:40 [WARN] [http] request=r31 event=completed method=POST path=/v1/embeddings status=499 duration_ms=25.1 outcome=completed error_code=cancelled rss_mib=507 host_available_mib=118414"
|
||||
},
|
||||
"owned_multimodal_smoke": "passed"
|
||||
},
|
||||
"packaging": "Generated DLL manifest is checked in validation bundles and exact release ZIPs. CPU video harness validates extracted DLLs; test binary and fixtures stay out of the released ZIP.",
|
||||
"reviews": [
|
||||
{
|
||||
"findings": 1,
|
||||
"addressed": "Copy runtime manifest into release ZIP and validate extracted release DLLs using an external CPU video harness"
|
||||
},
|
||||
{
|
||||
"findings": 0
|
||||
}
|
||||
],
|
||||
"harness_sha256": {
|
||||
"tools/windows/check_bundle.py": "fda8a4001c20a661686119f2e6cb1544ce82d07a877cdf81891511b98474bef8",
|
||||
"tests/tools/windows_bundle_test.py": "495833e0237423e04d71aa914465a55c91bbf347bf57b7f38412daa23b533536",
|
||||
"tests/functional/embedding_video.py": "1ec21a20b79b468f031bc214d263fb1dd3b06ac6efafc116f705ef09d0db0967"
|
||||
},
|
||||
"runtime_loader_followup": {
|
||||
"windows_failure": "94f99ff1: CPU video test Windows error 126; configured DLL path used forward slashes. Normalize to native separators and use wide LoadLibraryExW. Move fixture outside executable directory to exercise absolute-path fallback.",
|
||||
"linux_failure": "94f99ff1: relocated package video HTTP returned 500 because build-host Nix path is absent. Resolve versioned FFmpeg filename during configuration; fall back to loader search for relocated pinned closure.",
|
||||
"regression_tests": "Native library load, relocated Linux library load, missing symbol/library, CPU video reference probes; Windows hosted rerun pending."
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -114,6 +114,7 @@
|
||||
ps.jsonschema # independent structured-output schema oracle
|
||||
lpipsRocm
|
||||
ps.numpy
|
||||
ps.av # official video decoder oracle
|
||||
ps.librosa # pinned official EmbeddingGemma audio decoder oracle
|
||||
ps.scipy
|
||||
ps.matplotlib # benchmark charts (tools/bench/model-bench.py render)
|
||||
@@ -362,6 +363,7 @@
|
||||
pkgsSys.libwebp
|
||||
pkgsSys.libsndfile
|
||||
pkgsSys.soxr
|
||||
pkgsSys.ffmpeg-headless
|
||||
pkgsSys.openssl
|
||||
];
|
||||
src = staticAnalysisSource;
|
||||
@@ -481,6 +483,7 @@
|
||||
pkgsSys.libwebp
|
||||
pkgsSys.libsndfile
|
||||
pkgsSys.soxr
|
||||
pkgsSys.ffmpeg-headless
|
||||
pkgsSys.openssl
|
||||
];
|
||||
src = testSource;
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
#include "src/cli/serve/logging.hpp"
|
||||
#include "src/core/image.hpp"
|
||||
#include "src/core/json.hpp"
|
||||
#include "src/core/platform/library.hpp"
|
||||
#include "src/models/embedding_gemma2/audio_decoder.hpp"
|
||||
#include "src/models/embedding_gemma2/audio_processor.hpp"
|
||||
#include "src/models/embedding_gemma2/image_processor.hpp"
|
||||
@@ -61,6 +62,8 @@ struct EmbeddingService::Impl {
|
||||
options.model_root.string(), &image_processor_error);
|
||||
(void)models::embedding_gemma2::ValidateAudioProcessor(
|
||||
options.model_root.string(), &audio_processor_error);
|
||||
(void)models::embedding_gemma2::ValidateVideoProcessor(
|
||||
options.model_root.string(), &video_processor_error);
|
||||
encoder = models::embedding_gemma2::hip::EncoderRuntime::Create(
|
||||
options.model_root.string(), &error);
|
||||
if (encoder == nullptr) {
|
||||
@@ -88,6 +91,8 @@ struct EmbeddingService::Impl {
|
||||
EmbeddingService::kMaxInputTokens * tokenizer->max_token_bytes();
|
||||
std::size_t image_count = 0;
|
||||
std::size_t audio_count = 0;
|
||||
std::size_t video_count = 0;
|
||||
std::size_t video_frames = 0;
|
||||
for (std::size_t index = 0; index < inputs.size(); ++index) {
|
||||
if (is_cancelled && is_cancelled())
|
||||
return EmbeddingStatus::kCancelled;
|
||||
@@ -101,6 +106,42 @@ struct EmbeddingService::Impl {
|
||||
return EmbeddingStatus::kCancelled;
|
||||
if (part.type == EmbeddingPart::Type::kText) {
|
||||
text += part.value;
|
||||
} else if (part.type == EmbeddingPart::Type::kVideo) {
|
||||
if (!encoder->config().supported_video ||
|
||||
!video_processor_error.empty()) {
|
||||
if (error)
|
||||
*error = video_processor_error.empty()
|
||||
? "unsupported embedding vision configuration"
|
||||
: video_processor_error;
|
||||
return EmbeddingStatus::kInvalidVideoInput;
|
||||
}
|
||||
try {
|
||||
if (++video_count > EmbeddingService::kMaxVideos)
|
||||
throw std::invalid_argument(
|
||||
"embedding requests accept at most 32 videos");
|
||||
const auto bytes = core::ReadVideoUrl(part.value, budget);
|
||||
auto frames =
|
||||
ProcessVideo(bytes, request.video, is_cancelled,
|
||||
EmbeddingService::kMaxVideoFrames - video_frames);
|
||||
video_frames += frames.size();
|
||||
if (video_frames > EmbeddingService::kMaxVideoFrames)
|
||||
throw std::invalid_argument(
|
||||
"embedding requests accept at most 64 sampled video frames");
|
||||
text += FrameVideo(frames, request.video);
|
||||
for (auto& frame : frames)
|
||||
sequence.images.push_back(
|
||||
{0, std::move(frame.patches), kVideoTokenId});
|
||||
} catch (const platform::LibraryError& failure) {
|
||||
if (error)
|
||||
*error = failure.what();
|
||||
return EmbeddingStatus::kFailed;
|
||||
} catch (const std::exception& failure) {
|
||||
if (is_cancelled && is_cancelled())
|
||||
return EmbeddingStatus::kCancelled;
|
||||
if (error)
|
||||
*error = failure.what();
|
||||
return EmbeddingStatus::kInvalidVideoInput;
|
||||
}
|
||||
} else if (part.type == EmbeddingPart::Type::kAudio) {
|
||||
if (!encoder->config().supported_audio ||
|
||||
!audio_processor_error.empty()) {
|
||||
@@ -197,19 +238,20 @@ struct EmbeddingService::Impl {
|
||||
EmbeddingService::kMaxInputTokens)
|
||||
: tokenizer->Encode(text);
|
||||
if (!sequence.images.empty()) {
|
||||
std::size_t expected = 0;
|
||||
for (const auto& image : sequence.images)
|
||||
expected += image.patches.soft_tokens();
|
||||
// Validate after joining and tokenizing: a literal placeholder may
|
||||
// straddle adjacent text parts. It must never capture another image's
|
||||
// features or leave an unfilled image token in the text backbone.
|
||||
if (static_cast<std::size_t>(std::ranges::count(
|
||||
sequence.tokens, kImageTokenId)) != expected) {
|
||||
if (error)
|
||||
*error =
|
||||
"text must not contain image placeholders when images are "
|
||||
"supplied";
|
||||
return EmbeddingStatus::kInvalidInput;
|
||||
for (const auto token : {kImageTokenId, kVideoTokenId}) {
|
||||
std::size_t expected = 0;
|
||||
for (const auto& image : sequence.images)
|
||||
if (image.token_id == token)
|
||||
expected += image.patches.soft_tokens();
|
||||
if (static_cast<std::size_t>(
|
||||
std::ranges::count(sequence.tokens, token)) != expected) {
|
||||
if (error)
|
||||
*error =
|
||||
"text must not contain image/video placeholders when vision "
|
||||
"inputs are supplied";
|
||||
return token == kVideoTokenId ? EmbeddingStatus::kInvalidVideoInput
|
||||
: EmbeddingStatus::kInvalidInput;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!sequence.audio.empty()) {
|
||||
@@ -227,6 +269,14 @@ struct EmbeddingService::Impl {
|
||||
}
|
||||
if (request.truncate &&
|
||||
sequence.tokens.size() > EmbeddingService::kMaxInputTokens) {
|
||||
const auto last_video = std::ranges::find(
|
||||
sequence.tokens.rbegin(), sequence.tokens.rend(), kVideoTokenId);
|
||||
if (last_video != sequence.tokens.rend() &&
|
||||
static_cast<std::size_t>(sequence.tokens.rend() - last_video) + 1 >
|
||||
EmbeddingService::kMaxInputTokens - 1) {
|
||||
batch->failed_index = index;
|
||||
return EmbeddingStatus::kInputTooLong;
|
||||
}
|
||||
const auto last = std::ranges::find(
|
||||
sequence.tokens.rbegin(), sequence.tokens.rend(), kAudioTokenId);
|
||||
if (!sequence.audio.empty() && last != sequence.tokens.rend() &&
|
||||
@@ -246,7 +296,7 @@ struct EmbeddingService::Impl {
|
||||
std::size_t cursor = 0;
|
||||
for (auto& image : sequence.images) {
|
||||
while (cursor < sequence.tokens.size() &&
|
||||
sequence.tokens[cursor] != kImageTokenId)
|
||||
sequence.tokens[cursor] != image.token_id)
|
||||
++cursor;
|
||||
image.offset = cursor;
|
||||
cursor += image.patches.soft_tokens();
|
||||
@@ -287,6 +337,7 @@ struct EmbeddingService::Impl {
|
||||
std::string initialization_error;
|
||||
std::string image_processor_error;
|
||||
std::string audio_processor_error;
|
||||
std::string video_processor_error;
|
||||
// Serializes the injected runner, which has no device turns of its own.
|
||||
std::mutex runner_mutex;
|
||||
std::atomic<bool> device_lost{false};
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include "src/models/embedding_gemma2/hip/encoder_runtime.hpp"
|
||||
#include "src/models/embedding_gemma2/video_processor.hpp"
|
||||
|
||||
namespace gufo::server {
|
||||
|
||||
@@ -28,7 +29,7 @@ struct EmbeddingBatch {
|
||||
};
|
||||
|
||||
struct EmbeddingPart {
|
||||
enum class Type { kText, kImage, kAudio };
|
||||
enum class Type { kText, kImage, kAudio, kVideo };
|
||||
Type type{Type::kText};
|
||||
std::string value;
|
||||
};
|
||||
@@ -42,6 +43,7 @@ enum class EmbeddingStatus {
|
||||
kInputTooLong,
|
||||
kInvalidInput,
|
||||
kInvalidAudioInput,
|
||||
kInvalidVideoInput,
|
||||
kFailed,
|
||||
/// The device failed a probe after an error. The service stays lost.
|
||||
kDeviceLost,
|
||||
@@ -53,6 +55,7 @@ struct EmbeddingRequestOptions {
|
||||
/// first tokens, keeping the end marker, instead of refusing it. An input
|
||||
/// whose images would not fit whole is still refused.
|
||||
bool truncate{false};
|
||||
models::embedding_gemma2::VideoOptions video;
|
||||
};
|
||||
|
||||
struct EmbeddingServiceOptions {
|
||||
@@ -97,6 +100,8 @@ public:
|
||||
|
||||
static constexpr std::size_t kMaxImages = 32;
|
||||
static constexpr std::size_t kMaxAudio = 32;
|
||||
static constexpr std::size_t kMaxVideos = 32;
|
||||
static constexpr std::size_t kMaxVideoFrames = 64;
|
||||
|
||||
explicit EmbeddingService(EmbeddingServiceOptions options);
|
||||
~EmbeddingService();
|
||||
|
||||
@@ -167,7 +167,7 @@ bool LooksLikeAudio(std::string_view base64) {
|
||||
bool IsKnownField(std::string_view name, bool native) {
|
||||
return name == "input" || name == "model" || name == "encoding_format" ||
|
||||
name == "dimensions" || name == "user" || name == "truncate" ||
|
||||
name == "prompt_name" ||
|
||||
name == "prompt_name" || name == "video_options" ||
|
||||
(native && (name == "content" || name == "embd_normalize"));
|
||||
}
|
||||
|
||||
@@ -220,6 +220,7 @@ HttpResponse Embed(const HttpRequest& request, EmbeddingService& service,
|
||||
std::vector<EmbeddingInput> inputs;
|
||||
std::size_t image_count = 0;
|
||||
std::size_t audio_count = 0;
|
||||
std::size_t video_count = 0;
|
||||
const auto parse_item = [&](const json::Value& item) -> HttpResponse {
|
||||
EmbeddingInput parsed;
|
||||
if (item.is_string()) {
|
||||
@@ -316,6 +317,28 @@ HttpResponse Embed(const HttpRequest& request, EmbeddingService& service,
|
||||
return BadRequest("embedding requests accept at most 32 images",
|
||||
"too_many_images");
|
||||
parsed.content.push_back({EmbeddingPart::Type::kImage, url->str()});
|
||||
} else if (type == "input_video") {
|
||||
const auto* video = part.find("input_video");
|
||||
if (part.size() != 2 || !video || !video->is_object())
|
||||
return BadRequest("video parts require an 'input_video' object",
|
||||
"invalid_parameter_type");
|
||||
for (const auto& [key, value] : video->members()) {
|
||||
if ((key != "data" && key != "url") || !value.is_string())
|
||||
return BadRequest(
|
||||
"input_video accepts string data and url fields",
|
||||
"invalid_parameter_type");
|
||||
}
|
||||
const auto* source = video->find("data");
|
||||
if (!source)
|
||||
source = video->find("url");
|
||||
if (!source || source->str().empty())
|
||||
return BadRequest("input_video requires nonempty data or url",
|
||||
"invalid_parameter_type");
|
||||
if (++video_count > EmbeddingService::kMaxVideos)
|
||||
return BadRequest("embedding requests accept at most 32 videos",
|
||||
"too_many_videos");
|
||||
parsed.content.push_back(
|
||||
{EmbeddingPart::Type::kVideo, source->str()});
|
||||
} else if (type == "input_audio") {
|
||||
const auto* audio = part.find("input_audio");
|
||||
if (part.size() != 2 || !audio || !audio->is_object())
|
||||
@@ -456,6 +479,42 @@ HttpResponse Embed(const HttpRequest& request, EmbeddingService& service,
|
||||
}
|
||||
|
||||
EmbeddingRequestOptions options;
|
||||
if (const auto* video = Optional(root, "video_options")) {
|
||||
if (!video->is_object())
|
||||
return BadRequest("video_options must be an object",
|
||||
"invalid_parameter_type");
|
||||
for (const auto& [key, value] : video->members()) {
|
||||
if (key == "fps") {
|
||||
if (!value.is_number() || !std::isfinite(value.as_double()) ||
|
||||
value.as_double() <= 0 || value.as_double() > 120)
|
||||
return BadRequest("video fps must be in (0, 120]",
|
||||
"invalid_video_options");
|
||||
options.video.fps = value.as_double();
|
||||
} else if (key == "max_frames") {
|
||||
const auto n = value.as_size(0);
|
||||
if (!value.is_number() || n < 1 || n > 32 ||
|
||||
value.as_double() != static_cast<double>(n))
|
||||
return BadRequest("video max_frames must be an integer in [1, 32]",
|
||||
"invalid_video_options");
|
||||
options.video.max_frames = n;
|
||||
} else if (key == "add_timestamps") {
|
||||
if (!value.is_bool())
|
||||
return BadRequest("add_timestamps must be boolean",
|
||||
"invalid_video_options");
|
||||
options.video.timestamps = value.as_bool();
|
||||
} else if (key == "overflow_strategy") {
|
||||
if (!value.is_string() ||
|
||||
(value.str() != "uniform" && value.str() != "truncate"))
|
||||
return BadRequest(
|
||||
"video overflow_strategy must be uniform or truncate",
|
||||
"invalid_video_options");
|
||||
options.video.truncate_overflow = value.str() == "truncate";
|
||||
} else
|
||||
return BadRequest("unsupported video option: " + key,
|
||||
"invalid_video_options");
|
||||
}
|
||||
}
|
||||
|
||||
if (const json::Value* truncate = Optional(root, "truncate");
|
||||
truncate != nullptr) {
|
||||
if (!truncate->is_bool()) {
|
||||
@@ -479,6 +538,8 @@ HttpResponse Embed(const HttpRequest& request, EmbeddingService& service,
|
||||
std::to_string(EmbeddingService::kMaxInputTokens) +
|
||||
"-token context window",
|
||||
"context_length_exceeded");
|
||||
case EmbeddingStatus::kInvalidVideoInput:
|
||||
return BadRequest(error, "invalid_video_input");
|
||||
case EmbeddingStatus::kInvalidAudioInput:
|
||||
return BadRequest(error, "invalid_audio_input");
|
||||
case EmbeddingStatus::kInvalidInput:
|
||||
|
||||
+28
-13
@@ -417,22 +417,32 @@ std::vector<std::uint8_t> ReadImageFile(const std::filesystem::path& path) {
|
||||
return bytes;
|
||||
}
|
||||
|
||||
enum class MediaKind { kImage, kAudio, kVideo };
|
||||
|
||||
static std::vector<std::uint8_t> ReadMediaUrl(std::string_view url,
|
||||
ImageReadBudget& budget,
|
||||
bool audio);
|
||||
MediaKind media_kind);
|
||||
|
||||
std::vector<std::uint8_t> ReadImageUrl(std::string_view url,
|
||||
ImageReadBudget& budget) {
|
||||
return ReadMediaUrl(url, budget, false);
|
||||
return ReadMediaUrl(url, budget, MediaKind::kImage);
|
||||
}
|
||||
std::vector<std::uint8_t> ReadAudioUrl(std::string_view url,
|
||||
ImageReadBudget& budget) {
|
||||
if (url.starts_with("data:") || url.starts_with("https://") ||
|
||||
url.starts_with("http://") || url.starts_with("file:"))
|
||||
return ReadMediaUrl(url, budget, true);
|
||||
return ReadMediaUrl(url, budget, MediaKind::kAudio);
|
||||
// Raw base64 has no scheme. The strict decoder rejects paths and bad padding.
|
||||
return ReadMediaUrl("data:audio/wav;base64," + std::string(url), budget,
|
||||
true);
|
||||
MediaKind::kAudio);
|
||||
}
|
||||
std::vector<std::uint8_t> ReadVideoUrl(std::string_view url,
|
||||
ImageReadBudget& budget) {
|
||||
if (url.starts_with("data:") || url.starts_with("https://") ||
|
||||
url.starts_with("http://") || url.starts_with("file:"))
|
||||
return ReadMediaUrl(url, budget, MediaKind::kVideo);
|
||||
return ReadMediaUrl("data:video/mp4;base64," + std::string(url), budget,
|
||||
MediaKind::kVideo);
|
||||
}
|
||||
std::vector<std::uint8_t> ReadImageUrl(std::string_view url) {
|
||||
ImageReadBudget budget;
|
||||
@@ -465,7 +475,7 @@ bool IsPublicImageAddress(std::span<const std::uint8_t> address) noexcept {
|
||||
|
||||
static std::vector<std::uint8_t> ReadMediaUrl(std::string_view url,
|
||||
ImageReadBudget& budget,
|
||||
bool audio) {
|
||||
MediaKind media_kind) {
|
||||
if (!budget.accepts_images)
|
||||
throw std::invalid_argument("model does not support image input");
|
||||
const auto remaining_ms =
|
||||
@@ -488,15 +498,20 @@ static std::vector<std::uint8_t> ReadMediaUrl(std::string_view url,
|
||||
if (!header.ends_with(";base64"))
|
||||
throw std::invalid_argument("image data URL must be base64-encoded");
|
||||
const std::string media = header.substr(0, header.find(';'));
|
||||
if (audio ? (media != "audio/wav" && media != "audio/x-wav" &&
|
||||
media != "audio/mpeg" && media != "audio/mp3" &&
|
||||
media != "audio/flac" && media != "audio/ogg")
|
||||
: (media != "image/png" && media != "image/jpeg" &&
|
||||
media != "image/jpg" && media != "image/webp"))
|
||||
if (media_kind == MediaKind::kVideo
|
||||
? (media != "video/mp4" && media != "video/webm" &&
|
||||
media != "video/x-matroska" && media != "video/quicktime")
|
||||
: media_kind == MediaKind::kAudio
|
||||
? (media != "audio/wav" && media != "audio/x-wav" &&
|
||||
media != "audio/mpeg" && media != "audio/mp3" &&
|
||||
media != "audio/flac" && media != "audio/ogg")
|
||||
: (media != "image/png" && media != "image/jpeg" &&
|
||||
media != "image/jpg" && media != "image/webp"))
|
||||
throw std::invalid_argument(
|
||||
audio ? "unsupported audio data URL type"
|
||||
: "image data URL type \"" + media +
|
||||
"\" is not supported; use PNG, JPEG or WebP");
|
||||
media_kind != MediaKind::kImage
|
||||
? "unsupported media data URL type"
|
||||
: "image data URL type \"" + media +
|
||||
"\" is not supported; use PNG, JPEG or WebP");
|
||||
const auto encoded = url.substr(comma + 1);
|
||||
// Check the decoded size before allocating, including base64 padding.
|
||||
const auto padding = encoded.ends_with("==") ? 2U
|
||||
|
||||
@@ -49,5 +49,9 @@ struct ImageReadBudget {
|
||||
[[nodiscard]] std::vector<std::uint8_t> ReadAudioUrl(std::string_view url,
|
||||
ImageReadBudget& budget);
|
||||
|
||||
/// Video uses the same shared byte/deadline and public HTTPS protections.
|
||||
[[nodiscard]] std::vector<std::uint8_t> ReadVideoUrl(std::string_view url,
|
||||
ImageReadBudget& budget);
|
||||
|
||||
} // namespace gufo::core
|
||||
#endif
|
||||
@@ -2,6 +2,7 @@
|
||||
# (GUFO_PLATFORM_ONLY) share this list.
|
||||
add_library(gufo_platform STATIC
|
||||
env.cpp
|
||||
library.cpp
|
||||
mapped_file.cpp
|
||||
net.cpp
|
||||
shutdown_signal.cpp
|
||||
@@ -14,7 +15,7 @@ add_library(gufo_platform STATIC
|
||||
"$<$<PLATFORM_ID:Windows>:${CMAKE_CURRENT_SOURCE_DIR}/pages_windows.cpp>"
|
||||
"$<$<PLATFORM_ID:Windows>:${CMAKE_CURRENT_SOURCE_DIR}/page_pin_windows.cpp>")
|
||||
target_include_directories(gufo_platform PUBLIC ${PROJECT_SOURCE_DIR})
|
||||
target_link_libraries(gufo_platform PUBLIC Threads::Threads)
|
||||
target_link_libraries(gufo_platform PUBLIC Threads::Threads ${CMAKE_DL_LIBS})
|
||||
if (WIN32)
|
||||
target_link_libraries(gufo_platform PUBLIC ${GUFO_PLATFORM_LIBS})
|
||||
endif()
|
||||
@@ -0,0 +1,70 @@
|
||||
#include "src/core/platform/library.hpp"
|
||||
|
||||
#include <filesystem>
|
||||
#include <stdexcept>
|
||||
|
||||
#ifdef _WIN32
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
|
||||
#else
|
||||
#include <dlfcn.h>
|
||||
#endif
|
||||
|
||||
namespace gufo::platform {
|
||||
Library::Library(const std::string& name) {
|
||||
#ifdef _WIN32
|
||||
// Prefer the packaged DLL beside the executable. Development/test builds
|
||||
// also have a configured absolute library path; neither mode searches PATH
|
||||
// or the current working directory for a runtime dependency.
|
||||
auto path = std::filesystem::path(name);
|
||||
path.make_preferred();
|
||||
handle_ = LoadLibraryExW(
|
||||
path.filename().c_str(), nullptr,
|
||||
LOAD_LIBRARY_SEARCH_APPLICATION_DIR | LOAD_LIBRARY_SEARCH_SYSTEM32);
|
||||
if (!handle_ && path.is_absolute())
|
||||
handle_ = LoadLibraryExW(
|
||||
path.c_str(), nullptr,
|
||||
LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_SYSTEM32);
|
||||
if (!handle_)
|
||||
throw LibraryError("cannot load runtime library " + name +
|
||||
": Windows error " + std::to_string(GetLastError()));
|
||||
#else
|
||||
handle_ = dlopen(name.c_str(), RTLD_NOW | RTLD_LOCAL);
|
||||
// Portable packages relocate the pinned closure and supply its library
|
||||
// directories through the ELF loader. The build-host Nix path is absent.
|
||||
if (!handle_ && std::filesystem::path(name).is_absolute())
|
||||
handle_ = dlopen(std::filesystem::path(name).filename().c_str(),
|
||||
RTLD_NOW | RTLD_LOCAL);
|
||||
if (!handle_)
|
||||
throw LibraryError("cannot load runtime library " + name + ": " +
|
||||
dlerror());
|
||||
#endif
|
||||
}
|
||||
Library::~Library() {
|
||||
#ifdef _WIN32
|
||||
if (handle_)
|
||||
FreeLibrary(static_cast<HMODULE>(handle_));
|
||||
#else
|
||||
if (handle_)
|
||||
dlclose(handle_);
|
||||
#endif
|
||||
}
|
||||
void* Library::Symbol(const char* name) const {
|
||||
#ifdef _WIN32
|
||||
auto* address = reinterpret_cast<void*>(
|
||||
GetProcAddress(static_cast<HMODULE>(handle_), name));
|
||||
if (!address)
|
||||
throw LibraryError(std::string("missing runtime symbol ") + name);
|
||||
#else
|
||||
dlerror();
|
||||
auto* address = dlsym(handle_, name);
|
||||
if (const auto* error = dlerror())
|
||||
throw LibraryError(std::string("missing runtime symbol ") + name + ": " +
|
||||
error);
|
||||
#endif
|
||||
return address;
|
||||
}
|
||||
} // namespace gufo::platform
|
||||
@@ -0,0 +1,27 @@
|
||||
#ifndef GUFO_CORE_PLATFORM_LIBRARY_HPP_
|
||||
#define GUFO_CORE_PLATFORM_LIBRARY_HPP_
|
||||
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
namespace gufo::platform {
|
||||
class LibraryError : public std::runtime_error {
|
||||
public:
|
||||
using std::runtime_error::runtime_error;
|
||||
};
|
||||
|
||||
/// Loads one runtime library on demand. The caller keeps this object alive
|
||||
/// while using any symbol obtained from it. Failures throw std::runtime_error.
|
||||
class Library {
|
||||
public:
|
||||
explicit Library(const std::string& name);
|
||||
~Library();
|
||||
Library(const Library&) = delete;
|
||||
Library& operator=(const Library&) = delete;
|
||||
[[nodiscard]] void* Symbol(const char* name) const;
|
||||
|
||||
private:
|
||||
void* handle_{nullptr};
|
||||
};
|
||||
} // namespace gufo::platform
|
||||
#endif
|
||||
@@ -3,6 +3,7 @@
|
||||
add_library(gufo_embedding_gemma2 STATIC
|
||||
audio_processor.cpp
|
||||
audio_decoder.cpp
|
||||
video_processor.cpp
|
||||
batch_scheduler.cpp
|
||||
image_processor.cpp
|
||||
config.cpp
|
||||
@@ -16,6 +17,35 @@ target_include_directories(gufo_embedding_gemma2 PUBLIC ${PROJECT_SOURCE_DIR})
|
||||
target_link_libraries(gufo_embedding_gemma2 PUBLIC Threads::Threads
|
||||
gufo_platform PkgConfig::SNDFILE PkgConfig::SOXR)
|
||||
|
||||
# Video libraries load on the first video request. Linking them into gufo
|
||||
# adds their codec/network dependency initialization to every service startup.
|
||||
target_include_directories(gufo_embedding_gemma2 PRIVATE ${AV_INCLUDE_DIRS})
|
||||
target_compile_options(gufo_embedding_gemma2 PRIVATE ${AV_CFLAGS_OTHER})
|
||||
set(video_libraries "")
|
||||
foreach(component IN ITEMS avformat avcodec avutil swscale)
|
||||
string(TOUPPER "${component}" upper)
|
||||
if(WIN32)
|
||||
string(REGEX MATCH "^[0-9]+" major "${AV_lib${component}_VERSION}")
|
||||
if(NOT major)
|
||||
message(FATAL_ERROR "Missing FFmpeg ${component} runtime version")
|
||||
endif()
|
||||
list(GET AV_LIBRARY_DIRS 0 runtime_libdir)
|
||||
cmake_path(SET runtime NORMALIZE "${runtime_libdir}/../bin/${component}-${major}.dll")
|
||||
string(APPEND video_libraries "${component}-${major}.dll\n")
|
||||
else()
|
||||
find_library(GUFO_${upper}_LIBRARY NAMES ${component}
|
||||
HINTS ${AV_LIBRARY_DIRS} REQUIRED)
|
||||
# Keep the versioned filename for loading the relocated Linux closure.
|
||||
file(REAL_PATH "${GUFO_${upper}_LIBRARY}" runtime)
|
||||
endif()
|
||||
target_compile_definitions(gufo_embedding_gemma2 PRIVATE
|
||||
"GUFO_${upper}_LIBRARY=\"${runtime}\"")
|
||||
endforeach()
|
||||
if(WIN32)
|
||||
file(GENERATE OUTPUT "${PROJECT_BINARY_DIR}/gufo-video-libraries.txt"
|
||||
CONTENT "${video_libraries}")
|
||||
endif()
|
||||
|
||||
# Float32 CPU oracle for the HIP runtime; tests link it explicitly.
|
||||
add_library(gufo_embedding_gemma2_reference STATIC EXCLUDE_FROM_ALL
|
||||
reference/cpu_encoder.cpp
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include "src/models/embedding_gemma2/audio_processor.hpp"
|
||||
#include "src/models/embedding_gemma2/config.hpp"
|
||||
#include "src/models/embedding_gemma2/image_processor.hpp"
|
||||
|
||||
namespace gufo::models::embedding_gemma2 {
|
||||
@@ -18,6 +19,8 @@ namespace gufo::models::embedding_gemma2 {
|
||||
struct SequenceImage {
|
||||
std::size_t offset{0};
|
||||
ImagePatches patches;
|
||||
/// Video frames share the vision tower but use their own placeholders.
|
||||
std::uint32_t token_id{kImageTokenId};
|
||||
};
|
||||
struct EncoderSequence {
|
||||
std::vector<std::uint32_t> tokens;
|
||||
|
||||
@@ -180,6 +180,8 @@ std::optional<ModelConfig> ParseModelConfig(const std::string& json_contents) {
|
||||
root.member_size("image_token_id") == 258880 &&
|
||||
root.member_size("eoi_token_id") == 258882;
|
||||
}
|
||||
config.supported_video = config.supported_vision &&
|
||||
root.member_size("video_token_id") == kVideoTokenId;
|
||||
if (const auto* audio = root.find("audio_config");
|
||||
audio && audio->is_object()) {
|
||||
const auto* channels = audio->find("subsampling_conv_channels");
|
||||
|
||||
@@ -14,6 +14,8 @@ namespace gufo::models::embedding_gemma2 {
|
||||
/// reference applies the same constant but keeps the product in float32.
|
||||
inline constexpr float kEmbedScale = 22.625F;
|
||||
inline constexpr std::uint32_t kImageTokenId = 258880;
|
||||
inline constexpr std::uint32_t kVideoTokenId = 258884;
|
||||
inline constexpr const char* kVideoToken = "<|video|>";
|
||||
inline constexpr std::uint32_t kAudioTokenId = 258881;
|
||||
inline constexpr const char* kAudioToken = "<|audio|>";
|
||||
inline constexpr const char* kBeginAudioToken = "<|audio>";
|
||||
@@ -62,6 +64,7 @@ struct ModelConfig {
|
||||
std::string hidden_activation;
|
||||
TextConfig text;
|
||||
bool supported_vision{false};
|
||||
bool supported_video{false};
|
||||
bool supported_audio{false};
|
||||
};
|
||||
|
||||
|
||||
@@ -493,7 +493,13 @@ void EncoderRuntime::Impl::RunBatch(
|
||||
clip.features.soft_tokens(), stream);
|
||||
}
|
||||
for (const auto& image : sequences[s]->images) {
|
||||
if (sequences[s]->is_cancelled && sequences[s]->is_cancelled())
|
||||
throw std::runtime_error(kCancelledMessage);
|
||||
RunVision(image.patches, host_offsets[s] + image.offset);
|
||||
// Bound outstanding video work to one frame, so a disconnected long
|
||||
// clip releases the device before encoding the remaining frames.
|
||||
if (image.token_id == kVideoTokenId)
|
||||
HIP_CHECK(hipStreamSynchronize(stream));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -743,7 +749,9 @@ bool EncoderRuntime::EmbedMultimodal(
|
||||
return fail("embedding_gemma2: invalid image patches or token offset");
|
||||
}
|
||||
for (std::size_t j = 0; j < n; ++j) {
|
||||
if (sequence.tokens[image.offset + j] != kImageTokenId) {
|
||||
if ((image.token_id != kImageTokenId &&
|
||||
image.token_id != kVideoTokenId) ||
|
||||
sequence.tokens[image.offset + j] != image.token_id) {
|
||||
return fail("embedding_gemma2: image placeholder mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,7 +160,10 @@ bool ValidateImageProcessor(const std::string& model_dir, std::string* error) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ImagePatches ProcessImage(const core::Image& image) {
|
||||
ImagePatches ProcessImage(const core::Image& image,
|
||||
std::size_t soft_token_budget) {
|
||||
if (soft_token_budget != 140 && soft_token_budget != 280)
|
||||
throw std::invalid_argument("unsupported embedding vision budget");
|
||||
if (image.width == 0 || image.height == 0 || image.width > kMaxImageSide ||
|
||||
image.height > kMaxImageSide ||
|
||||
std::size_t{image.width} * image.height > core::kMaxImagePixels ||
|
||||
@@ -168,7 +171,7 @@ ImagePatches ProcessImage(const core::Image& image) {
|
||||
throw std::invalid_argument(
|
||||
"invalid embedding image dimensions or RGB data");
|
||||
}
|
||||
constexpr int kBudget = 280;
|
||||
const int kBudget = static_cast<int>(soft_token_budget);
|
||||
constexpr int kStride = 48;
|
||||
const double factor =
|
||||
std::sqrt((kBudget * 9.0 * 256) /
|
||||
|
||||
@@ -30,7 +30,8 @@ struct ImagePatches {
|
||||
|
||||
/// Gemma4ImageProcessor: aspect-preserving 280-soft-token budget, uint8
|
||||
/// antialiased bicubic resize, rescale, then patchify. Does not crop.
|
||||
[[nodiscard]] ImagePatches ProcessImage(const core::Image& image);
|
||||
[[nodiscard]] ImagePatches ProcessImage(const core::Image& image,
|
||||
std::size_t soft_token_budget = 280);
|
||||
|
||||
} // namespace gufo::models::embedding_gemma2
|
||||
#endif
|
||||
@@ -0,0 +1,459 @@
|
||||
#include "src/models/embedding_gemma2/video_processor.hpp"
|
||||
|
||||
extern "C" {
|
||||
#include <libavcodec/avcodec.h>
|
||||
#include <libavformat/avformat.h>
|
||||
#include <libavutil/imgutils.h>
|
||||
#include <libswscale/swscale.h>
|
||||
}
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iterator>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "src/core/json.hpp"
|
||||
#include "src/core/platform/library.hpp"
|
||||
#include "src/models/embedding_gemma2/config.hpp"
|
||||
|
||||
namespace gufo::models::embedding_gemma2 {
|
||||
namespace {
|
||||
// Thread-safe first-use loading preserves text/image/audio startup latency.
|
||||
// Hold handles before the typed symbols so every function pointer stays valid.
|
||||
struct FFmpegApi {
|
||||
platform::Library avformat{GUFO_AVFORMAT_LIBRARY};
|
||||
platform::Library avcodec{GUFO_AVCODEC_LIBRARY};
|
||||
platform::Library avutil{GUFO_AVUTIL_LIBRARY};
|
||||
platform::Library swscale{GUFO_SWSCALE_LIBRARY};
|
||||
decltype(&::avio_alloc_context) avio_alloc_context{
|
||||
reinterpret_cast<decltype(&::avio_alloc_context)>(
|
||||
avformat.Symbol("avio_alloc_context"))};
|
||||
decltype(&::avio_context_free) avio_context_free{
|
||||
reinterpret_cast<decltype(&::avio_context_free)>(
|
||||
avformat.Symbol("avio_context_free"))};
|
||||
decltype(&::avformat_alloc_context) avformat_alloc_context{
|
||||
reinterpret_cast<decltype(&::avformat_alloc_context)>(
|
||||
avformat.Symbol("avformat_alloc_context"))};
|
||||
decltype(&::avformat_open_input) avformat_open_input{
|
||||
reinterpret_cast<decltype(&::avformat_open_input)>(
|
||||
avformat.Symbol("avformat_open_input"))};
|
||||
decltype(&::avformat_close_input) avformat_close_input{
|
||||
reinterpret_cast<decltype(&::avformat_close_input)>(
|
||||
avformat.Symbol("avformat_close_input"))};
|
||||
decltype(&::avformat_find_stream_info) avformat_find_stream_info{
|
||||
reinterpret_cast<decltype(&::avformat_find_stream_info)>(
|
||||
avformat.Symbol("avformat_find_stream_info"))};
|
||||
decltype(&::av_read_frame) av_read_frame{
|
||||
reinterpret_cast<decltype(&::av_read_frame)>(
|
||||
avformat.Symbol("av_read_frame"))};
|
||||
decltype(&::avcodec_alloc_context3) avcodec_alloc_context3{
|
||||
reinterpret_cast<decltype(&::avcodec_alloc_context3)>(
|
||||
avcodec.Symbol("avcodec_alloc_context3"))};
|
||||
decltype(&::avcodec_parameters_to_context) avcodec_parameters_to_context{
|
||||
reinterpret_cast<decltype(&::avcodec_parameters_to_context)>(
|
||||
avcodec.Symbol("avcodec_parameters_to_context"))};
|
||||
decltype(&::avcodec_open2) avcodec_open2{
|
||||
reinterpret_cast<decltype(&::avcodec_open2)>(
|
||||
avcodec.Symbol("avcodec_open2"))};
|
||||
decltype(&::avcodec_find_decoder) avcodec_find_decoder{
|
||||
reinterpret_cast<decltype(&::avcodec_find_decoder)>(
|
||||
avcodec.Symbol("avcodec_find_decoder"))};
|
||||
decltype(&::avcodec_free_context) avcodec_free_context{
|
||||
reinterpret_cast<decltype(&::avcodec_free_context)>(
|
||||
avcodec.Symbol("avcodec_free_context"))};
|
||||
decltype(&::avcodec_send_packet) avcodec_send_packet{
|
||||
reinterpret_cast<decltype(&::avcodec_send_packet)>(
|
||||
avcodec.Symbol("avcodec_send_packet"))};
|
||||
decltype(&::avcodec_receive_frame) avcodec_receive_frame{
|
||||
reinterpret_cast<decltype(&::avcodec_receive_frame)>(
|
||||
avcodec.Symbol("avcodec_receive_frame"))};
|
||||
decltype(&::av_packet_alloc) av_packet_alloc{
|
||||
reinterpret_cast<decltype(&::av_packet_alloc)>(
|
||||
avcodec.Symbol("av_packet_alloc"))};
|
||||
decltype(&::av_packet_free) av_packet_free{
|
||||
reinterpret_cast<decltype(&::av_packet_free)>(
|
||||
avcodec.Symbol("av_packet_free"))};
|
||||
decltype(&::av_packet_unref) av_packet_unref{
|
||||
reinterpret_cast<decltype(&::av_packet_unref)>(
|
||||
avcodec.Symbol("av_packet_unref"))};
|
||||
decltype(&::av_malloc) av_malloc{
|
||||
reinterpret_cast<decltype(&::av_malloc)>(avutil.Symbol("av_malloc"))};
|
||||
decltype(&::av_free) av_free{
|
||||
reinterpret_cast<decltype(&::av_free)>(avutil.Symbol("av_free"))};
|
||||
decltype(&::av_freep) av_freep{
|
||||
reinterpret_cast<decltype(&::av_freep)>(avutil.Symbol("av_freep"))};
|
||||
decltype(&::av_dict_set) av_dict_set{
|
||||
reinterpret_cast<decltype(&::av_dict_set)>(avutil.Symbol("av_dict_set"))};
|
||||
decltype(&::av_dict_free) av_dict_free{
|
||||
reinterpret_cast<decltype(&::av_dict_free)>(
|
||||
avutil.Symbol("av_dict_free"))};
|
||||
decltype(&::av_frame_alloc) av_frame_alloc{
|
||||
reinterpret_cast<decltype(&::av_frame_alloc)>(
|
||||
avutil.Symbol("av_frame_alloc"))};
|
||||
decltype(&::av_frame_free) av_frame_free{
|
||||
reinterpret_cast<decltype(&::av_frame_free)>(
|
||||
avutil.Symbol("av_frame_free"))};
|
||||
decltype(&::av_frame_unref) av_frame_unref{
|
||||
reinterpret_cast<decltype(&::av_frame_unref)>(
|
||||
avutil.Symbol("av_frame_unref"))};
|
||||
decltype(&::sws_getContext) sws_getContext{
|
||||
reinterpret_cast<decltype(&::sws_getContext)>(
|
||||
swscale.Symbol("sws_getContext"))};
|
||||
decltype(&::sws_freeContext) sws_freeContext{
|
||||
reinterpret_cast<decltype(&::sws_freeContext)>(
|
||||
swscale.Symbol("sws_freeContext"))};
|
||||
decltype(&::sws_scale) sws_scale{
|
||||
reinterpret_cast<decltype(&::sws_scale)>(swscale.Symbol("sws_scale"))};
|
||||
};
|
||||
FFmpegApi& FFmpeg() {
|
||||
static FFmpegApi api;
|
||||
return api;
|
||||
}
|
||||
|
||||
struct Budget {
|
||||
const std::function<bool()>& cancelled;
|
||||
std::chrono::steady_clock::time_point deadline{
|
||||
std::chrono::steady_clock::now() + std::chrono::seconds(15)};
|
||||
std::size_t pixels{0};
|
||||
bool Stop() const {
|
||||
return (cancelled && cancelled()) ||
|
||||
std::chrono::steady_clock::now() > deadline;
|
||||
}
|
||||
void Check() const {
|
||||
if (Stop())
|
||||
throw std::invalid_argument(
|
||||
"embedding video decoding cancelled or timed out");
|
||||
}
|
||||
};
|
||||
struct Decoder {
|
||||
std::span<const std::uint8_t> bytes;
|
||||
Budget& budget;
|
||||
std::size_t position{0};
|
||||
AVIOContext* io{nullptr};
|
||||
AVFormatContext* format{nullptr};
|
||||
AVCodecContext* codec{nullptr};
|
||||
AVFrame* frame{nullptr};
|
||||
AVPacket* packet{nullptr};
|
||||
int stream{-1};
|
||||
~Decoder() {
|
||||
FFmpeg().av_packet_free(&packet);
|
||||
FFmpeg().av_frame_free(&frame);
|
||||
FFmpeg().avcodec_free_context(&codec);
|
||||
FFmpeg().avformat_close_input(&format);
|
||||
if (io) {
|
||||
FFmpeg().av_freep(&io->buffer);
|
||||
FFmpeg().avio_context_free(&io);
|
||||
}
|
||||
}
|
||||
static int Read(void* context, std::uint8_t* out, int count) {
|
||||
auto& self = *static_cast<Decoder*>(context);
|
||||
if (self.budget.Stop())
|
||||
return AVERROR_EXIT;
|
||||
const auto n =
|
||||
std::min<std::size_t>(count, self.bytes.size() - self.position);
|
||||
if (!n)
|
||||
return AVERROR_EOF;
|
||||
std::memcpy(out, self.bytes.data() + self.position, n);
|
||||
self.position += n;
|
||||
return static_cast<int>(n);
|
||||
}
|
||||
static std::int64_t Seek(void* context, std::int64_t offset, int whence) {
|
||||
auto& self = *static_cast<Decoder*>(context);
|
||||
if (self.budget.Stop())
|
||||
return AVERROR_EXIT;
|
||||
if (whence == AVSEEK_SIZE)
|
||||
return static_cast<std::int64_t>(self.bytes.size());
|
||||
whence &= ~AVSEEK_FORCE;
|
||||
const auto base =
|
||||
whence == SEEK_SET ? 0LL
|
||||
: whence == SEEK_CUR ? static_cast<long long>(self.position)
|
||||
: whence == SEEK_END ? static_cast<long long>(self.bytes.size())
|
||||
: -1LL;
|
||||
if (base < 0 || offset < -base ||
|
||||
offset > static_cast<std::int64_t>(self.bytes.size()) - base)
|
||||
return AVERROR(EINVAL);
|
||||
self.position = static_cast<std::size_t>(base + offset);
|
||||
return static_cast<std::int64_t>(self.position);
|
||||
}
|
||||
void Open() {
|
||||
budget.Check();
|
||||
auto* buffer = static_cast<unsigned char*>(FFmpeg().av_malloc(32768));
|
||||
if (!buffer)
|
||||
throw std::bad_alloc();
|
||||
io = FFmpeg().avio_alloc_context(buffer, 32768, 0, this, Read, nullptr,
|
||||
Seek);
|
||||
if (!io) {
|
||||
FFmpeg().av_free(buffer);
|
||||
throw std::bad_alloc();
|
||||
}
|
||||
format = FFmpeg().avformat_alloc_context();
|
||||
if (!format)
|
||||
throw std::bad_alloc();
|
||||
format->pb = io;
|
||||
format->flags |= AVFMT_FLAG_CUSTOM_IO;
|
||||
format->interrupt_callback = {
|
||||
[](void* p) { return static_cast<Budget*>(p)->Stop() ? 1 : 0; },
|
||||
&budget};
|
||||
format->max_streams = 8;
|
||||
format->probesize = 1024 * 1024;
|
||||
format->max_analyze_duration = 2 * AV_TIME_BASE;
|
||||
AVDictionary* options = nullptr;
|
||||
FFmpeg().av_dict_set(&options, "format_whitelist", "mov,matroska,webm", 0);
|
||||
FFmpeg().av_dict_set(&options, "protocol_whitelist", "", 0);
|
||||
// MOV external data references must never trigger additional IO.
|
||||
FFmpeg().av_dict_set(&options, "enable_drefs", "0", 0);
|
||||
const int opened =
|
||||
FFmpeg().avformat_open_input(&format, nullptr, nullptr, &options);
|
||||
FFmpeg().av_dict_free(&options);
|
||||
if (opened < 0)
|
||||
throw std::invalid_argument(
|
||||
"embedding video container could not be opened");
|
||||
// Probing may itself decode a frame. Apply limits before that decoder is
|
||||
// created, not only to the persistent codec used for the two passes.
|
||||
AVDictionary* probe_options[8]{};
|
||||
for (auto& option : probe_options) {
|
||||
FFmpeg().av_dict_set(&option, "max_pixels", "4194304", 0);
|
||||
FFmpeg().av_dict_set(&option, "threads", "1", 0);
|
||||
}
|
||||
const int probed =
|
||||
FFmpeg().avformat_find_stream_info(format, probe_options);
|
||||
for (auto& option : probe_options)
|
||||
FFmpeg().av_dict_free(&option);
|
||||
if (probed < 0)
|
||||
throw std::invalid_argument(
|
||||
"embedding video supports valid MP4/MOV and WebM/Matroska "
|
||||
"containers");
|
||||
for (unsigned i = 0; i < format->nb_streams; ++i) {
|
||||
if (format->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
|
||||
stream = static_cast<int>(i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (format->duration != AV_NOPTS_VALUE &&
|
||||
format->duration > 300LL * AV_TIME_BASE)
|
||||
throw std::invalid_argument("embedding video exceeds 300 seconds");
|
||||
if (stream < 0)
|
||||
throw std::invalid_argument("embedding video has no video stream");
|
||||
const auto* parameters = format->streams[stream]->codecpar;
|
||||
const auto* implementation =
|
||||
FFmpeg().avcodec_find_decoder(parameters->codec_id);
|
||||
if (!implementation)
|
||||
throw std::invalid_argument("unsupported embedding video codec");
|
||||
codec = FFmpeg().avcodec_alloc_context3(implementation);
|
||||
if (!codec)
|
||||
throw std::bad_alloc();
|
||||
if (FFmpeg().avcodec_parameters_to_context(codec, parameters) < 0)
|
||||
throw std::invalid_argument("invalid embedding video codec parameters");
|
||||
codec->thread_count = 1;
|
||||
codec->max_pixels = 4 * 1024 * 1024;
|
||||
codec->err_recognition = AV_EF_EXPLODE;
|
||||
if (FFmpeg().avcodec_open2(codec, implementation, nullptr) < 0)
|
||||
throw std::invalid_argument("embedding video decoder could not open");
|
||||
frame = FFmpeg().av_frame_alloc();
|
||||
packet = FFmpeg().av_packet_alloc();
|
||||
if (!frame || !packet)
|
||||
throw std::bad_alloc();
|
||||
}
|
||||
std::size_t Decode(
|
||||
const std::function<void(std::size_t, const AVFrame*)>& consume) {
|
||||
std::size_t count = 0;
|
||||
int width = 0, height = 0;
|
||||
auto receive = [&] {
|
||||
for (;;) {
|
||||
budget.Check();
|
||||
const int status = FFmpeg().avcodec_receive_frame(codec, frame);
|
||||
if (status == AVERROR(EAGAIN) || status == AVERROR_EOF)
|
||||
break;
|
||||
if (status < 0)
|
||||
throw std::invalid_argument("embedding video frame decoding failed");
|
||||
if (count == 0) {
|
||||
width = frame->width;
|
||||
height = frame->height;
|
||||
}
|
||||
if (frame->width != width || frame->height != height)
|
||||
throw std::invalid_argument(
|
||||
"embedding video frame dimensions change within clip");
|
||||
const auto pixels =
|
||||
static_cast<std::size_t>(frame->width) * frame->height;
|
||||
if (++count > 4096 || frame->width < 1 || frame->height < 1 ||
|
||||
pixels > 4 * 1024 * 1024 ||
|
||||
pixels > 512 * 1024 * 1024 - budget.pixels)
|
||||
throw std::invalid_argument(
|
||||
"embedding video exceeds decoded work bounds");
|
||||
budget.pixels += pixels;
|
||||
consume(count - 1, frame);
|
||||
FFmpeg().av_frame_unref(frame);
|
||||
}
|
||||
};
|
||||
int status;
|
||||
while ((status = FFmpeg().av_read_frame(format, packet)) >= 0) {
|
||||
budget.Check();
|
||||
if (packet->stream_index == stream) {
|
||||
if (FFmpeg().avcodec_send_packet(codec, packet) < 0)
|
||||
throw std::invalid_argument("embedding video packet decoding failed");
|
||||
receive();
|
||||
}
|
||||
FFmpeg().av_packet_unref(packet);
|
||||
}
|
||||
if (status != AVERROR_EOF ||
|
||||
FFmpeg().avcodec_send_packet(codec, nullptr) < 0)
|
||||
throw std::invalid_argument(
|
||||
"embedding video is truncated or decoding failed");
|
||||
receive();
|
||||
return count;
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
std::vector<std::size_t> SampleVideoFrames(std::size_t frames, double fps,
|
||||
const VideoOptions& options) {
|
||||
if (!frames || frames > 4096 || !std::isfinite(fps) || fps <= 0 ||
|
||||
!std::isfinite(options.fps) || options.fps <= 0 || options.fps > 120 ||
|
||||
!options.max_frames || options.max_frames > 32 || frames / fps > 300)
|
||||
throw std::invalid_argument(
|
||||
"invalid embedding video rate, duration or frame count");
|
||||
const auto sampled = std::max<std::size_t>(
|
||||
1, static_cast<std::size_t>(frames / fps * options.fps));
|
||||
const auto kept = std::min(sampled, options.max_frames);
|
||||
std::vector<std::size_t> indices;
|
||||
// NumPy linspace multiplies by a double step, then floors for dtype=int.
|
||||
// Rational integer division differs at floating rounding boundaries.
|
||||
const double step =
|
||||
kept > 1 ? static_cast<double>(sampled - 1) / (kept - 1) : 0;
|
||||
for (std::size_t j = 0; j < kept; ++j) {
|
||||
const auto i = sampled <= kept || options.truncate_overflow ? j
|
||||
: j + 1 == kept && kept > 1
|
||||
? sampled - 1
|
||||
: static_cast<std::size_t>(j * step);
|
||||
const auto index =
|
||||
i == 0 ? 0
|
||||
: std::min(frames - 1,
|
||||
static_cast<std::size_t>(i * (fps / options.fps)));
|
||||
indices.push_back(index);
|
||||
}
|
||||
return indices;
|
||||
}
|
||||
|
||||
bool ValidateVideoProcessor(const std::string& model_dir, std::string* error) {
|
||||
try {
|
||||
std::ifstream file(std::filesystem::path(model_dir) /
|
||||
"processor_config.json");
|
||||
const auto root =
|
||||
json::parse(std::string(std::istreambuf_iterator<char>(file), {}));
|
||||
const auto* c = root.find("video_processor");
|
||||
const auto flag = [&](const char* name, bool expected) {
|
||||
const auto* v = c ? c->find(name) : nullptr;
|
||||
return v && v->is_bool() && v->as_bool() == expected;
|
||||
};
|
||||
if (c &&
|
||||
c->member_str("video_processor_type") ==
|
||||
"EmbeddingGemma2VideoProcessor" &&
|
||||
flag("do_convert_rgb", true) && flag("do_resize", true) &&
|
||||
flag("do_rescale", true) && flag("do_normalize", false) &&
|
||||
flag("do_sample_frames", true) && flag("add_timestamps", false) &&
|
||||
c->member_double("fps") == 1 && c->member_size("max_frames") == 32 &&
|
||||
c->member_str("overflow_strategy") == "uniform" &&
|
||||
c->member_size("max_soft_tokens") == 140 &&
|
||||
c->member_size("patch_size") == 16 &&
|
||||
c->member_size("pooling_kernel_size") == 3 &&
|
||||
c->member_size("resample") == 3 &&
|
||||
std::abs(c->member_double("rescale_factor") - 1.0 / 255) < 1e-15)
|
||||
return true;
|
||||
} catch (const std::exception&) {
|
||||
}
|
||||
if (error)
|
||||
*error =
|
||||
"embedding_gemma2: unsupported or missing video processor "
|
||||
"configuration";
|
||||
return false;
|
||||
}
|
||||
|
||||
std::vector<VideoFrame> ProcessVideo(std::span<const std::uint8_t> bytes,
|
||||
const VideoOptions& options,
|
||||
const std::function<bool()>& is_cancelled,
|
||||
std::size_t frame_budget) {
|
||||
if (bytes.empty() || bytes.size() > 20 * 1024 * 1024)
|
||||
throw std::invalid_argument("embedding video is empty or exceeds 20 MiB");
|
||||
Budget budget{is_cancelled};
|
||||
budget.Check();
|
||||
(void)FFmpeg(); // Finish loading before a Decoder can need cleanup.
|
||||
std::size_t count;
|
||||
double fps;
|
||||
{
|
||||
Decoder decoder{bytes, budget};
|
||||
decoder.Open();
|
||||
fps = av_q2d(decoder.format->streams[decoder.stream]->avg_frame_rate);
|
||||
count = decoder.Decode([](std::size_t, const AVFrame*) {});
|
||||
const auto declared = decoder.format->streams[decoder.stream]->nb_frames;
|
||||
if (declared > 0 && static_cast<std::size_t>(declared) != count)
|
||||
throw std::invalid_argument(
|
||||
"embedding video frame count differs from container; truncated "
|
||||
"input");
|
||||
}
|
||||
const auto selected = SampleVideoFrames(count, fps, options);
|
||||
if (selected.size() > frame_budget)
|
||||
throw std::invalid_argument(
|
||||
"embedding request video frame budget exhausted");
|
||||
Decoder decoder{bytes, budget};
|
||||
decoder.Open();
|
||||
std::vector<VideoFrame> frames;
|
||||
decoder.Decode([&](std::size_t index, const AVFrame* frame) {
|
||||
if (frames.size() >= selected.size() || index != selected[frames.size()])
|
||||
return;
|
||||
std::unique_ptr<SwsContext, decltype(FFmpeg().sws_freeContext)> conversion(
|
||||
FFmpeg().sws_getContext(frame->width, frame->height,
|
||||
static_cast<AVPixelFormat>(frame->format),
|
||||
frame->width, frame->height, AV_PIX_FMT_RGB24,
|
||||
SWS_BICUBIC, nullptr, nullptr, nullptr),
|
||||
FFmpeg().sws_freeContext);
|
||||
if (!conversion)
|
||||
throw std::invalid_argument("embedding video RGB conversion failed");
|
||||
core::Image image{
|
||||
static_cast<std::uint32_t>(frame->width),
|
||||
static_cast<std::uint32_t>(frame->height),
|
||||
std::vector<std::uint8_t>(static_cast<std::size_t>(frame->width) *
|
||||
frame->height * 3)};
|
||||
std::uint8_t* output[] = {image.pixels.data()};
|
||||
int stride[] = {frame->width * 3};
|
||||
if (FFmpeg().sws_scale(conversion.get(), frame->data, frame->linesize, 0,
|
||||
frame->height, output, stride) != frame->height)
|
||||
throw std::invalid_argument("embedding video RGB conversion failed");
|
||||
auto patches = ProcessImage(image, 140);
|
||||
do {
|
||||
frames.push_back({index, index / fps, patches});
|
||||
} while (frames.size() < selected.size() &&
|
||||
selected[frames.size()] == index);
|
||||
budget.Check();
|
||||
});
|
||||
if (frames.size() != selected.size())
|
||||
throw std::invalid_argument("embedding video lost selected frames");
|
||||
return frames;
|
||||
}
|
||||
|
||||
std::string FrameVideo(const std::vector<VideoFrame>& frames,
|
||||
const VideoOptions& options) {
|
||||
std::string text;
|
||||
for (const auto& frame : frames) {
|
||||
if (options.timestamps) {
|
||||
if (!text.empty())
|
||||
text += ' ';
|
||||
char timestamp[32];
|
||||
const int seconds = static_cast<int>(frame.timestamp);
|
||||
std::snprintf(timestamp, sizeof(timestamp), "%02d:%02d ", seconds / 60,
|
||||
seconds % 60);
|
||||
text += timestamp;
|
||||
}
|
||||
text += kBeginImageToken;
|
||||
for (std::size_t j = 0; j < frame.patches.soft_tokens(); ++j)
|
||||
text += kVideoToken;
|
||||
text += kEndImageToken;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
} // namespace gufo::models::embedding_gemma2
|
||||
@@ -0,0 +1,42 @@
|
||||
#ifndef GUFO_MODELS_EMBEDDING_GEMMA2_VIDEO_PROCESSOR_HPP_
|
||||
#define GUFO_MODELS_EMBEDDING_GEMMA2_VIDEO_PROCESSOR_HPP_
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "src/models/embedding_gemma2/image_processor.hpp"
|
||||
|
||||
namespace gufo::models::embedding_gemma2 {
|
||||
struct VideoOptions {
|
||||
double fps{1};
|
||||
std::size_t max_frames{32};
|
||||
bool timestamps{false};
|
||||
bool truncate_overflow{false};
|
||||
};
|
||||
struct VideoFrame {
|
||||
std::size_t index{0};
|
||||
double timestamp{0};
|
||||
ImagePatches patches;
|
||||
};
|
||||
/// Exact FPS/floor selection followed by uniform linspace/floor overflow.
|
||||
[[nodiscard]] std::vector<std::size_t> SampleVideoFrames(
|
||||
std::size_t frames, double fps, const VideoOptions& options = {});
|
||||
[[nodiscard]] bool ValidateVideoProcessor(const std::string& model_dir,
|
||||
std::string* error = nullptr);
|
||||
/// In-memory FFmpeg decoder; never opens nested URLs or files. Selects the
|
||||
/// first video stream and ignores audio, matching the visual reference path.
|
||||
/// Bounds: 20 MiB, 300 seconds, 4096 decoded frames, 4M pixels/frame,
|
||||
/// 512M decoded pixels over both passes, and 15 seconds including conversion.
|
||||
[[nodiscard]] std::vector<VideoFrame> ProcessVideo(
|
||||
std::span<const std::uint8_t> bytes, const VideoOptions& options = {},
|
||||
const std::function<bool()>& is_cancelled = {},
|
||||
std::size_t frame_budget = 32);
|
||||
/// Reference per-frame image boundaries, video placeholders and optional mm:ss.
|
||||
[[nodiscard]] std::string FrameVideo(const std::vector<VideoFrame>& frames,
|
||||
const VideoOptions& options);
|
||||
} // namespace gufo::models::embedding_gemma2
|
||||
#endif
|
||||
@@ -0,0 +1,3 @@
|
||||
extern "C" int GufoLibraryProbe() {
|
||||
return 761;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "src/core/platform/library.hpp"
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
try {
|
||||
if (argc != 2)
|
||||
throw std::runtime_error("expected fixture library path");
|
||||
// CMake supplies forward-slash paths on Windows too. Keep the fixture
|
||||
// outside the executable directory so this exercises the absolute fallback.
|
||||
gufo::platform::Library library(
|
||||
std::filesystem::path(argv[1]).generic_string());
|
||||
auto probe =
|
||||
reinterpret_cast<int (*)()>(library.Symbol("GufoLibraryProbe"));
|
||||
if (probe() != 761)
|
||||
throw std::runtime_error("loaded the wrong library or symbol");
|
||||
bool missing_symbol = false;
|
||||
try {
|
||||
(void)library.Symbol("GufoMissingSymbol");
|
||||
} catch (const gufo::platform::LibraryError&) {
|
||||
missing_symbol = true;
|
||||
}
|
||||
if (!missing_symbol)
|
||||
throw std::runtime_error("missing symbol was accepted");
|
||||
bool missing_library = false;
|
||||
try {
|
||||
gufo::platform::Library missing("gufo-missing-library-761");
|
||||
} catch (const gufo::platform::LibraryError&) {
|
||||
missing_library = true;
|
||||
}
|
||||
if (!missing_library)
|
||||
throw std::runtime_error("missing library was accepted");
|
||||
return 0;
|
||||
} catch (const std::exception& error) {
|
||||
std::cerr << error.what() << '\n';
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"model_revision":"914f7f89142e33e77833254d9c9b90c3cef7303b","transformers_revision":"50e35cc0ca18ec19b80653690de553e6decf74f5","transformers":"5.21.0.dev0","torch":"2.14.1","av":"19.0.1","dtype":"float32","cases":[{"name":"short","text":"<|video|>","videos":[{"path":"video/short.mp4","sha256":"c3db490892da41604ae1d95a4cab2938c92676430a80e44d35686020a8cd5166","indices":[0],"fps":5.0,"rgb_sha256":["febb84bc4b551b69b83ee6bda85aa81151c7ea18b12d80ded47b26c1a980f80c"],"patches":[{"rows":30,"columns":39,"sha256":"0e57e72fe0b21b1b939a3587c2e9d8e17601ebe4168616bd5fea454e334d55fd"}]}],"options":{},"token_ids":[2,255999,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258884,258882,1],"embedding":[0.003876516,-0.03338786,0.03144105,0.01774808,-0.01896732,0.006312254,-0.03618869,-0.05684762,0.002288234,-0.1107745,-0.0512111,-0.01808877,-0.05045011,0.02034561,-0.007554227,-0.03726561,0.04640371,0.006195985,0.0268251,-3.476308e-05,-0.001080072,-0.01852308,-0.02285528,-0.006843851,-0.008637535,0.02011538,0.04172415,-0.01003387,-0.07222667,0.2730098,-0.01374793,-0.007058692,0.03128324,-0.0123558,0.06476951,-0.05424787,0.01592976,-0.02288595,-0.01124671,-0.01680539,0.02312169,0.01061238,-0.01972442,-0.004532161,-0.06125463,-0.02433864,-0.03903744,-0.03410117,-0.04066142,-0.03484535,-0.006117698,-0.01174453,-0.02029616,-0.02036407,0.02623061,0.01759216,-0.01701039,-0.04237574,0.001892259,-0.004318431,0.00960221,-0.01485983,0.00394979,0.07868249,-0.005810675,-0.006379002,0.008901715,-0.008666565,0.009082009,0.01412046,0.01189293,-0.04247107,0.01137242,-0.04514333,0.04698243,-0.02772602,-0.1118709,0.004948382,0.003167819,-0.006160828,-0.01577812,-0.04870902,-0.01839819,-0.0157383,-0.0008339002,-0.04444179,-0.02017964,-0.0003771186,-0.03363997,0.007737723,0.02601541,-0.02110727,0.0007181308,0.001504829,-0.04453949,-0.07708732,0.003359881,-0.01814633,0.03023413,0.04781501,-0.005068962,-0.0188656,-0.01139743,-0.0063434,0.05295325,-0.05952259,0.03134124,0.008516416,-0.008970609,-0.03832433,0.01858247,-0.2844111,0.001943276,0.02762808,-0.02629462,0.3635418,-0.04012704,-0.0225022,0.04564392,0.01232989,-0.0005613292,0.04169965,-0.01254948,0.0005181573,0.03149263,0.00756318,0.03390954,-0.01712739,0.02803318,0.01812534,0.04233173,0.0385979,0.01864564,-0.01705077,0.007246386,-0.01443585,-0.02048707,-0.008170081,0.08948741,-0.03327851,-0.02179497,-0.01104186,0.04078136,-0.02591984,-0.03001528,0.01696961,-0.007670771,-0.01725151,0.01357225,0.03057071,0.004390941,0.0004760481,0.01242044,-0.008070325,-0.0459843,0.008867458,-0.02522982,-0.04407372,0.005459108,-0.03551684,-0.02296877,-0.01516054,-0.02249851,0.008281268,0.04128857,0.0125641,-0.04016386,-0.01265633,-0.01927898,0.02448632,-0.01486249,0.05239045,-0.03435573,-0.00219044,0.05913713,0.004055265,0.02499799,-0.00524374,-0.003340539,0.02298757,-0.02889778,-0.06993925,0.03921454,0.002727761,0.01095657,0.0004331961,-0.008150792,-0.02130126,-0.03461112,0.02899666,-0.01761106,-0.0212787,0.03680353,-0.02274824,-0.01008244,0.03729492,0.05397943,-0.02768912,-0.01324689,-0.03250455,-0.01068547,0.03581072,-0.01424066,0.01179323,0.008357766,0.01010743,-0.00670562,0.04287989,0.01596705,0.02185446,0.02537689,-0.07958116,-0.02695167,-0.02351543,0.02677926,0.01840894,0.007242139,0.07210323,-0.06115875,-0.006711443,-0.04785391,0.03985091,0.07539025,0.009185832,-0.0610294,0.01921858,0.05666391,0.02922833,0.03161838,0.04150334,0.01291316,-0.01713471,0.02936063,-0.007839663,-0.0582708,0.01955836,-0.0002061019,-0.02103102,0.005856875,0.04439465,-0.03707247,0.03952084,-0.02861205,-0.03541173,0.01111766,-0.0163459,-0.01028605,0.03507983,-0.01262478,-0.02352679,-0.03453816,-0.0008230018,0.007397925,-0.01672688,0.02490266,-0.03408696,-0.01293998,-0.009931975,-0.008142025,0.01841283,0.0009907854,0.04035531,0.0226233,-0.02200643,0.008613118,0.004468092,-0.004485655,-0.02249593,0.01494714,-0.0430726,-0.001149846,0.005301391,0.009194931,-4.130771e-05,0.02263415,0.01628245,0.0177795,-0.00127595,-0.002290176,-0.008757639,-0.002569642,-0.01888969,-0.01496594,-0.0286353,-0.01496118,-0.002832859,0.009175042,0.02039406,0.02163801,0.02762593,0.01667076,0.007580813,-0.00864099,0.0Line truncated
|
||||
@@ -30,6 +30,15 @@ nix develop -c python3 tests/functional/model_services.py tts \
|
||||
--output artifacts/model-services/tts-base-1
|
||||
```
|
||||
|
||||
Embedding uses `--embedding-suite smoke` by default, including Devlab's PR tier.
|
||||
It retains independent reference-vector checks for all four modalities in one
|
||||
small batch, then checks an invalid clip and successful text recovery. This is
|
||||
a smoke check, not full model qualification. Select `--embedding-suite full`
|
||||
for the existing numerical/API suites plus video goldens, sampling/timestamps,
|
||||
dimensions, concurrent requests, context overflow, cancellation and recovery.
|
||||
Devlab's full tier selects that suite. Both retain individual request timings;
|
||||
manual invocations do not qualify performance against main.
|
||||
|
||||
The default audio root is the packaged `fixtures/audio/` English WAV and reviewed
|
||||
transcript, authorized by the maintainer for publication. Override `--audio-root`
|
||||
only with a folder containing that English pair. ASR generates silence in memory
|
||||
@@ -41,7 +50,7 @@ VoiceDesign model (fixed English voice instructions). Neither requires a clone.
|
||||
|
||||
| Suite | Required checks |
|
||||
| --- | --- |
|
||||
| Embedding | Text, image and audio HTTP suites: independent reference vectors, dimensions, normalization, encodings, batch ordering, concurrent requests, image/audio formats, multiple clips, audio/text/image interleaving, invalid inputs, cancellation and recovery |
|
||||
| Embedding | The default three-request smoke check covers text, image, video, mixed video/audio/image input, malformed-video rejection and recovery. `--embedding-suite full` retains the text, image, audio and video HTTP suites: independent reference vectors, dimensions, normalization, encodings, batch ordering, concurrent requests, image/audio formats, multiple clips, audio/text/image interleaving, invalid inputs, cancellation and recovery |
|
||||
| ASR | English transcript WER ≤ 0.15, exact greedy replay, JSON/text/verbose formats, complete SSE matching buffered output, concurrent speech/silence, rejected options, malformed WAV recovery, stream cancellation and subsequent valid inference |
|
||||
| TTS | Voice discovery, valid non-silent 24-kHz mono PCM16 audio, two fixed texts with ASR WER ≤ 0.25, exact same-seed warm replay, WAV/PCM/SSE equality, different concurrent texts/seeds, rejected inputs, stream cancellation and subsequent valid inference |
|
||||
|
||||
|
||||
@@ -80,7 +80,8 @@ def select(paths, data, tier='pr', ar=False, benchmark=False):
|
||||
for field in ('groups', 'deferred', 'release_only', 'out_of_scope', 'obligations'):
|
||||
result[field].extend(planned[field])
|
||||
else:
|
||||
suites = {'embedding': ['embedding-text', 'embedding-images'],
|
||||
suites = {'embedding': (['embedding-smoke'] if tier == 'pr' else
|
||||
['embedding-text', 'embedding-images', 'embedding-audio', 'embedding-video']),
|
||||
'asr': ['asr-http'], 'tts': ['tts-http']}[model['category']]
|
||||
result['groups'].append({'profile': key + '/native', 'configuration': 'standard',
|
||||
'settings': {}, 'suites': suites})
|
||||
|
||||
@@ -54,6 +54,8 @@ def execute(plan, binary, baseline, assets_path, output, report, logs):
|
||||
else:
|
||||
destination = output / key
|
||||
command = service_args(key, binary, baseline, assets, destination)
|
||||
if model['category'] == 'embedding':
|
||||
command += ['--embedding-suite', 'smoke' if plan['tier'] == 'pr' else 'full']
|
||||
with (output / (key + '.log')).open('w') as log:
|
||||
result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, timeout=900)
|
||||
path = destination / 'report.json'
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Video HTTP reference parity; --smoke is a bounded multimodal service check."""
|
||||
import argparse
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import io
|
||||
import http.client
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import socket
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
from embedding_images import cosine, picture
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parents[1] / 'fixtures' / 'embedding_gemma2'
|
||||
|
||||
|
||||
def source(path):
|
||||
return base64.b64encode(path.read_bytes()).decode()
|
||||
|
||||
|
||||
def mixed_audio():
|
||||
import numpy as np
|
||||
samples = np.sin(np.arange(3200, dtype=np.float32)*0.07)*0.1
|
||||
# IEEE float WAV retains the reference float32 waveform exactly.
|
||||
raw = samples.astype('<f4').tobytes()
|
||||
return base64.b64encode(b'RIFF' + struct.pack('<I', 36+len(raw)) + b'WAVEfmt ' +
|
||||
struct.pack('<IHHIIHH', 16, 3, 1, 16000, 64000, 4, 32) + b'data' +
|
||||
struct.pack('<I', len(raw)) + raw).decode()
|
||||
|
||||
|
||||
def content(case):
|
||||
from PIL import Image
|
||||
text = case['text']
|
||||
parts, video = [], 0
|
||||
while text:
|
||||
next_marker = min(((text.find(marker), marker) for marker in
|
||||
('<|video|>', '<|image|>', '<|audio|>') if marker in text), default=None)
|
||||
if next_marker is None:
|
||||
parts.append({'type': 'text', 'text': text})
|
||||
break
|
||||
at, marker = next_marker
|
||||
if at:
|
||||
parts.append({'type': 'text', 'text': text[:at]})
|
||||
text = text[at+len(marker):]
|
||||
if marker == '<|video|>':
|
||||
parts.append({'type': 'input_video', 'input_video': {'data': source(FIXTURES / case['videos'][video]['path'])}})
|
||||
video += 1
|
||||
elif marker == '<|audio|>':
|
||||
parts.append({'type': 'input_audio', 'input_audio': {'data': mixed_audio()}})
|
||||
else:
|
||||
output = io.BytesIO()
|
||||
Image.new('RGB', (32, 32), (255, 0, 0)).save(output, 'PNG')
|
||||
parts.append({'type': 'image_url', 'image_url': {'url': 'data:image/png;base64,' +
|
||||
base64.b64encode(output.getvalue()).decode()}})
|
||||
return {'content': parts}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8080/v1')
|
||||
parser.add_argument('--model', default='embeddinggemma-2')
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--smoke', action='store_true')
|
||||
parser.add_argument('--server-log', type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
parser.error('output must not exist')
|
||||
fixture = json.loads((FIXTURES / 'video_golden.json').read_text())
|
||||
cases = {c['name']: c for c in fixture['cases']}
|
||||
results = []
|
||||
report = {'status': 'running', 'smoke': args.smoke, 'requests': results,
|
||||
'reference': {k: v for k, v in fixture.items() if k != 'cases'}}
|
||||
|
||||
def post(inputs, **options):
|
||||
request = urllib.request.Request(args.base_url.rstrip('/') + '/embeddings',
|
||||
data=json.dumps({'model': args.model, 'input': inputs, **options}).encode(),
|
||||
headers={'Content-Type': 'application/json'})
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
body, status = json.load(response), response.status
|
||||
except urllib.error.HTTPError as e:
|
||||
body, status = json.load(e), e.code
|
||||
results.append({'status': status, 'elapsed_ms': (time.perf_counter()-start)*1000,
|
||||
'options': options, 'input': inputs, 'response': body})
|
||||
return status, body
|
||||
|
||||
def check(body, expected):
|
||||
assert [item['index'] for item in body['data']] == list(range(len(expected))), body
|
||||
assert body['usage']['prompt_tokens'] == sum(len(c['token_ids']) for c in expected), body['usage']
|
||||
for item, case in zip(body['data'], expected):
|
||||
vector = item['embedding']
|
||||
assert len(vector) == 768 and all(math.isfinite(x) for x in vector)
|
||||
assert abs(sum(x*x for x in vector)-1) < 1e-5
|
||||
agreement = cosine(vector, case['embedding'])
|
||||
assert agreement >= 0.9999, (case['name'], agreement)
|
||||
item['reference_cosine'] = agreement
|
||||
|
||||
try:
|
||||
# One small real batch covers all four towers and interleaving. Text and
|
||||
# image references remain independent of the newly introduced video path.
|
||||
text = next(c for c in json.loads((FIXTURES/'embedding_golden.json').read_text())['cases'] if c['name'] == 'plain')
|
||||
image = json.loads((FIXTURES/'image_golden.json').read_text())['cases'][0]
|
||||
probes = [text, image, cases['short'], cases['mixed']]
|
||||
inputs = [text['text'], {'content': [{'type': 'image_url', 'image_url': {'url': picture(image['images'][0])}}]},
|
||||
content(cases['short']), content(cases['mixed'])]
|
||||
status, body = post(inputs)
|
||||
assert status == 200, body
|
||||
check(body, probes)
|
||||
if not args.smoke:
|
||||
for case in fixture['cases']:
|
||||
status, body = post([content(case)], video_options=case['options'])
|
||||
assert status == 200, body
|
||||
check(body, [case])
|
||||
for dimensions in (128, 256, 512):
|
||||
status, body = post([content(cases['short'])], dimensions=dimensions)
|
||||
assert status == 200, body
|
||||
vec = body['data'][0]['embedding']
|
||||
assert len(vec) == dimensions and abs(sum(v*v for v in vec)-1) < 1e-5
|
||||
assert cosine(vec, cases['short']['embedding'][:dimensions]) >= 0.9999
|
||||
with concurrent.futures.ThreadPoolExecutor(2) as pool:
|
||||
replies = list(pool.map(lambda _: post([content(cases['short'])]), range(2)))
|
||||
for status, body in replies:
|
||||
assert status == 200, body
|
||||
check(body, [cases['short']])
|
||||
invalid = ['file:///etc/passwd', 'https://127.0.0.1/x', 'data:video/mp4;base64,AAAA']
|
||||
for url in invalid:
|
||||
status, body = post([{'content': [{'type': 'input_video', 'input_video': {'url': url}}]}])
|
||||
assert status in (400, 413), body
|
||||
# The HTTP layer refuses oversized Content-Length before reading a
|
||||
# body. Sending the rejected body can race the server's close.
|
||||
endpoint = urllib.parse.urlsplit(args.base_url)
|
||||
client = http.client.HTTPConnection(endpoint.hostname, endpoint.port, timeout=30)
|
||||
try:
|
||||
client.putrequest('POST', endpoint.path.rstrip('/') + '/embeddings')
|
||||
client.putheader('Content-Type', 'application/json')
|
||||
client.putheader('Content-Length', str(30*1024*1024))
|
||||
client.endheaders()
|
||||
response = client.getresponse()
|
||||
rejected = response.read().decode()
|
||||
assert response.status == 413, rejected
|
||||
report['oversized_http'] = {'content_length': 30*1024*1024, 'status': response.status, 'response': rejected}
|
||||
finally:
|
||||
client.close()
|
||||
for name in ('oversized', 'overduration', 'overframes'):
|
||||
status, body = post([{'content': [{'type': 'input_video', 'input_video': {
|
||||
'data': source(FIXTURES/'video'/f'{name}.mp4')}}]}])
|
||||
assert status == 400 and body['error']['code'] == 'invalid_video_input', (name, body)
|
||||
limited = {'content': [content(cases['overflow'])['content'][0]] * 3}
|
||||
status, body = post([limited])
|
||||
assert status == 400 and body['error']['code'] == 'invalid_video_input', body
|
||||
too_long = content(cases['overflow'])
|
||||
too_long['content'].insert(0, {'type': 'text', 'text': 'hello '*8500})
|
||||
for truncate in (False, True):
|
||||
status, body = post([too_long], truncate=truncate)
|
||||
assert status == 400 and body['error']['code'] == 'context_length_exceeded', body
|
||||
if not args.smoke:
|
||||
assert args.server_log and args.server_log.is_file(), 'full suite needs --server-log to verify cancellation'
|
||||
before = len(args.server_log.read_text().splitlines())
|
||||
endpoint = urllib.parse.urlsplit(args.base_url)
|
||||
payload = json.dumps({'model': args.model, 'input': [content(cases['overflow'])]}).encode()
|
||||
with socket.create_connection((endpoint.hostname, endpoint.port or 80), timeout=30) as client:
|
||||
client.sendall((f"POST {endpoint.path.rstrip('/')}/embeddings HTTP/1.1\r\n"
|
||||
f"Host: {endpoint.netloc}\r\nContent-Type: application/json\r\n"
|
||||
f"Content-Length: {len(payload)}\r\nConnection: close\r\n\r\n").encode()+payload)
|
||||
time.sleep(0.02)
|
||||
client.shutdown(socket.SHUT_RDWR)
|
||||
deadline, cancellation = time.monotonic()+15, None
|
||||
while time.monotonic() < deadline:
|
||||
lines = args.server_log.read_text().splitlines()[before:]
|
||||
received = [re.search(r'request=(\S+)', line).group(1) for line in lines
|
||||
if 'event=received ' in line and 'path=/v1/embeddings ' in line]
|
||||
if received:
|
||||
assert len(received) == 1, received
|
||||
cancellation = next((line for line in lines if f'request={received[0]} ' in line
|
||||
and 'event=completed ' in line and 'error_code=cancelled' in line), None)
|
||||
if cancellation:
|
||||
break
|
||||
time.sleep(0.02)
|
||||
assert cancellation, 'server must observe cancellation for this request'
|
||||
report['cancelled_request'] = {'request': json.loads(payload), 'completion': cancellation}
|
||||
status, body = post([{'content': [{'type': 'input_video', 'input_video': {'data': 'AAAA'}}]}])
|
||||
assert status == 400 and body['error']['code'] == 'invalid_video_input', body
|
||||
status, body = post([text['text']])
|
||||
assert status == 200, body
|
||||
check(body, [text])
|
||||
report['status'] = 'passed'
|
||||
finally:
|
||||
if report['status'] != 'passed':
|
||||
report['status'] = 'failed'
|
||||
args.output.write_text(json.dumps(report, indent=2)+'\n')
|
||||
print(f"PASS embedding {'smoke' if args.smoke else 'video'}: {len(results)} requests")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -23,6 +23,7 @@ HERE = Path(__file__).resolve().parent
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('service', choices=('embedding', 'asr', 'tts'))
|
||||
parser.add_argument('--embedding-suite', choices=('smoke', 'full'), default='smoke')
|
||||
parser.add_argument('--binary', type=Path, required=True)
|
||||
parser.add_argument('--model', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
@@ -70,7 +71,7 @@ def main():
|
||||
'timing_qualification': 'observations only; no matched baseline comparison',
|
||||
'service_harness_sha256': {name: sha256(HERE / name) for name in
|
||||
('model_services.py', 'service_http.py', 'asr_service.py', 'tts_service.py',
|
||||
'embeddings.py', 'embedding_images.py', 'embedding_audio.py')}}
|
||||
'embeddings.py', 'embedding_images.py', 'embedding_audio.py', 'embedding_video.py')}}
|
||||
evidence = Evidence(output, report)
|
||||
evidence.save()
|
||||
url = f'http://127.0.0.1:{port}'
|
||||
@@ -83,11 +84,16 @@ def main():
|
||||
models = evidence.json(url + '/v1/models', label='models', method='GET')
|
||||
assert public_model in [item['id'] for item in models['data']], models
|
||||
if args.service == 'embedding':
|
||||
for script in ('embeddings.py', 'embedding_images.py', 'embedding_audio.py'):
|
||||
scripts = ('embedding_video.py',) if args.embedding_suite == 'smoke' else (
|
||||
'embeddings.py', 'embedding_images.py', 'embedding_audio.py', 'embedding_video.py')
|
||||
report['embedding_suite'] = args.embedding_suite
|
||||
for script in scripts:
|
||||
with evidence.case(script):
|
||||
launch = [sys.executable, str(HERE / script), '--base-url', url + '/v1',
|
||||
'--model', public_model, '--output', str(output / (script + '.json'))]
|
||||
if script == 'embedding_audio.py':
|
||||
if script == 'embedding_video.py' and args.embedding_suite == 'smoke':
|
||||
launch += ['--smoke']
|
||||
if script in ('embedding_audio.py', 'embedding_video.py'):
|
||||
launch += ['--server-log', str(output / 'server.log')]
|
||||
with (output / (script + '.log')).open('w') as log:
|
||||
result = subprocess.run(launch, stdout=log, stderr=subprocess.STDOUT, timeout=600)
|
||||
|
||||
@@ -88,6 +88,13 @@ class PlanTest(unittest.TestCase):
|
||||
self.assertEqual(self.models('src/models/qwen_image_21/model.cpp'), set())
|
||||
self.assertEqual(self.models('src/models/minimax_h3/runtime.cpp'), set())
|
||||
|
||||
def test_embedding_smoke_and_full_coverage(self):
|
||||
pr = self.plan('src/models/embedding_gemma2/video_processor.cpp')
|
||||
self.assertEqual(pr['groups'][0]['suites'], ['embedding-smoke'])
|
||||
full = self.plan('src/models/embedding_gemma2/video_processor.cpp', tier='full')
|
||||
self.assertEqual(full['groups'][0]['suites'],
|
||||
['embedding-text', 'embedding-images', 'embedding-audio', 'embedding-video'])
|
||||
|
||||
def test_full_tts_and_optional_ar(self):
|
||||
self.assertEqual(self.models('src/models/qwen3_tts/config.cpp', tier='full'),
|
||||
{'qwen3-tts-base', 'qwen3-tts-custom', 'qwen3-tts-design'})
|
||||
|
||||
@@ -150,3 +150,10 @@ target_compile_definitions(embedding_gemma2_audio_decoder_test PRIVATE
|
||||
add_test(NAME embedding_gemma2_audio_decoder_test COMMAND embedding_gemma2_audio_decoder_test)
|
||||
set_tests_properties(embedding_gemma2_audio_decoder_test PROPERTIES
|
||||
LABELS "cpu;models;embedding-gemma2;audio;quality")
|
||||
|
||||
add_executable(embedding_gemma2_video_processor_test video_processor_test.cpp)
|
||||
target_link_libraries(embedding_gemma2_video_processor_test PRIVATE gufo_embedding_gemma2 gufo_core)
|
||||
target_compile_definitions(embedding_gemma2_video_processor_test PRIVATE
|
||||
GUFO_VIDEO_GOLDEN="${PROJECT_SOURCE_DIR}/tests/fixtures/embedding_gemma2/video_golden.json")
|
||||
add_test(NAME embedding_gemma2_video_processor_test COMMAND embedding_gemma2_video_processor_test)
|
||||
set_tests_properties(embedding_gemma2_video_processor_test PROPERTIES LABELS "cpu;models;embedding-gemma2;video;quality")
|
||||
@@ -148,6 +148,12 @@ int main() {
|
||||
const auto vision = embedding_gemma2::ParseModelConfig(vision_config);
|
||||
Check(vision && vision->supported_vision,
|
||||
"released vision geometry is supported");
|
||||
Check(vision->supported_video, "released video placeholder is supported");
|
||||
const auto wrong_video = embedding_gemma2::ParseModelConfig(Replace(
|
||||
vision_config, "\"video_token_id\": 258884", "\"video_token_id\": 42"));
|
||||
Check(wrong_video && wrong_video->supported_vision &&
|
||||
!wrong_video->supported_video,
|
||||
"unsupported video token preserves image support");
|
||||
for (const auto& changed :
|
||||
{Replace(vision_config, "\"head_dim\": 64", "\"head_dim\": 128"),
|
||||
Replace(vision_config, "\"pooling_kernel_size\": 3",
|
||||
|
||||
@@ -520,6 +520,7 @@ void TestLimitsAndFailures() {
|
||||
|
||||
void TestTypedInput() {
|
||||
std::vector<EmbeddingInput> captured;
|
||||
gufo::server::EmbeddingRequestOptions captured_options;
|
||||
EmbeddingService service(EmbeddingServiceOptions{
|
||||
.model_root = {},
|
||||
.validate_model = false,
|
||||
@@ -527,9 +528,10 @@ void TestTypedInput() {
|
||||
.prompts = {},
|
||||
.default_prompt_name = {},
|
||||
.runner = [&](const std::vector<EmbeddingInput>& inputs,
|
||||
const gufo::server::EmbeddingRequestOptions&,
|
||||
const gufo::server::EmbeddingRequestOptions& options,
|
||||
EmbeddingBatch* batch, std::string*) {
|
||||
captured = inputs;
|
||||
captured_options = options;
|
||||
for (std::size_t i = 0; i < inputs.size(); ++i) {
|
||||
batch->embeddings.push_back(FakeEmbedding(std::to_string(i)));
|
||||
batch->token_counts.push_back(i + 2);
|
||||
@@ -607,6 +609,48 @@ void TestTypedInput() {
|
||||
many += "]}]}";
|
||||
ExpectError(Send(service, many), 400, "too_many_images",
|
||||
"image count bounded before reading");
|
||||
Check(
|
||||
Send(
|
||||
service,
|
||||
R"({"input":[{"content":[{"type":"text","text":"before"},{"type":"input_video","input_video":{"data":"AAAA","url":"ignored"}},{"type":"text","text":"after"}]}],"video_options":{"fps":0.5,"max_frames":3,"add_timestamps":true,"overflow_strategy":"truncate"}})")
|
||||
.status == 200,
|
||||
"video content and controls accepted");
|
||||
Check(captured[0].content.size() == 3 &&
|
||||
captured[0].content[1].type == EmbeddingPart::Type::kVideo &&
|
||||
captured[0].content[1].value == "AAAA" &&
|
||||
captured[0].content[0].value == "before" &&
|
||||
captured[0].content[2].value == "after",
|
||||
"video data precedence and interleaving");
|
||||
Check(captured_options.video.fps == 0.5 &&
|
||||
captured_options.video.max_frames == 3 &&
|
||||
captured_options.video.timestamps &&
|
||||
captured_options.video.truncate_overflow,
|
||||
"video controls reach runtime");
|
||||
for (
|
||||
const auto* body :
|
||||
{R"({"input":[{"content":[{"type":"input_video","input_video":{}}]}]})",
|
||||
R"({"input":[{"content":[{"type":"input_video","input_video":{"data":3}}]}]})",
|
||||
R"({"input":[{"content":[{"type":"input_video","input_video":{"url":"x","extra":"x"}}]}]})"})
|
||||
ExpectError(Send(service, body), 400, "invalid_parameter_type",
|
||||
"invalid video schema");
|
||||
for (const auto* settings :
|
||||
{R"({"fps":0})", R"({"fps":121})", R"({"max_frames":0})",
|
||||
R"({"max_frames":33})", R"({"max_frames":1.5})",
|
||||
R"({"add_timestamps":1})", R"({"overflow_strategy":"other"})",
|
||||
R"({"unknown":true})"})
|
||||
ExpectError(
|
||||
Send(service, std::string(R"({"input":"hello","video_options":)") +
|
||||
settings + "}"),
|
||||
400, "invalid_video_options", "invalid video control");
|
||||
std::string video_many = R"({"input":[{"content":[)";
|
||||
for (int i = 0; i < 33; ++i) {
|
||||
if (i)
|
||||
video_many += ',';
|
||||
video_many += R"({"type":"input_video","input_video":{"data":"AAAA"}})";
|
||||
}
|
||||
video_many += "]}]}";
|
||||
ExpectError(Send(service, video_many), 400, "too_many_videos",
|
||||
"video request count bound");
|
||||
}
|
||||
|
||||
// #764: llama.cpp's non-OpenAI `POST /embeddings`, as Frigate 0.18 sends it
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
// Independent pinned official processor fixtures, including encoded media.
|
||||
#include "src/models/embedding_gemma2/video_processor.hpp"
|
||||
|
||||
#include <cmath>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "src/core/crypto/sha256.hpp"
|
||||
#include "src/core/json.hpp"
|
||||
|
||||
namespace model = gufo::models::embedding_gemma2;
|
||||
namespace {
|
||||
void Check(bool ok, const std::string& message) {
|
||||
if (!ok)
|
||||
throw std::runtime_error(message);
|
||||
}
|
||||
template<class F>
|
||||
void Refused(F run, const char* name) {
|
||||
bool refused = false;
|
||||
try {
|
||||
run();
|
||||
} catch (const std::exception&) {
|
||||
refused = true;
|
||||
}
|
||||
Check(refused, name);
|
||||
}
|
||||
} // namespace
|
||||
int main(int argc, char** argv) {
|
||||
try {
|
||||
const std::filesystem::path golden = argc > 1 ? argv[1] : GUFO_VIDEO_GOLDEN;
|
||||
std::ifstream file(golden);
|
||||
const auto root = gufo::json::parse(
|
||||
std::string(std::istreambuf_iterator<char>(file), {}));
|
||||
const std::filesystem::path fixture = golden.parent_path();
|
||||
for (const auto& probe : root.find("cases")->items()) {
|
||||
model::VideoOptions options;
|
||||
const auto& settings = *probe.find("options");
|
||||
options.fps = settings.member_double("fps", 1);
|
||||
options.timestamps = (settings.find("add_timestamps") &&
|
||||
settings.find("add_timestamps")->as_bool());
|
||||
options.max_frames = settings.member_size("max_frames", 32);
|
||||
options.truncate_overflow =
|
||||
settings.member_str("overflow_strategy") == "truncate";
|
||||
for (const auto& video : probe.find("videos")->items()) {
|
||||
std::ifstream media(fixture / video.member_str("path"),
|
||||
std::ios::binary);
|
||||
std::vector<std::uint8_t> bytes(std::istreambuf_iterator<char>(media),
|
||||
{});
|
||||
Check(gufo::crypto::Sha256Hex(bytes) == video.member_str("sha256"),
|
||||
"media hash");
|
||||
const auto frames = model::ProcessVideo(bytes, options);
|
||||
const auto& indices = video.find("indices")->items();
|
||||
const auto& expected = video.find("patches")->items();
|
||||
Check(frames.size() == indices.size(), "sampled frame count");
|
||||
for (std::size_t i = 0; i < frames.size(); ++i) {
|
||||
const auto& frame = frames[i];
|
||||
Check(frame.index == indices[i].as_size(), "selected frame index");
|
||||
Check(frame.patches.rows == expected[i].member_size("rows") &&
|
||||
frame.patches.columns == expected[i].member_size("columns"),
|
||||
"patch geometry");
|
||||
std::vector<std::uint8_t> pixels;
|
||||
for (float value : frame.patches.pixels)
|
||||
pixels.push_back(
|
||||
static_cast<std::uint8_t>(std::lround(value * 255)));
|
||||
Check(gufo::crypto::Sha256Hex(pixels) ==
|
||||
expected[i].member_str("sha256"),
|
||||
probe.member_str("name") +
|
||||
" decoded RGB and resized patch bytes");
|
||||
}
|
||||
Refused([&] { (void)model::ProcessVideo(bytes, options, {}, 0); },
|
||||
"request frame budget");
|
||||
const auto framed = model::FrameVideo(frames, options);
|
||||
Check(!framed.empty() && framed.find("<|video|>") != std::string::npos,
|
||||
"video framing");
|
||||
Refused(
|
||||
[&] { (void)model::ProcessVideo(bytes, {}, [] { return true; }); },
|
||||
"cancelled decoder");
|
||||
}
|
||||
std::cout << "PASS " << probe.member_str("name") << '\n';
|
||||
}
|
||||
for (const auto* name :
|
||||
{"oversized.mp4", "overduration.mp4", "overframes.mp4"}) {
|
||||
std::ifstream media(fixture / "video" / name, std::ios::binary);
|
||||
Check(media.good(), std::string("missing resource fixture ") + name);
|
||||
std::vector<std::uint8_t> bytes(std::istreambuf_iterator<char>(media),
|
||||
{});
|
||||
Refused([&] { (void)model::ProcessVideo(bytes); }, name);
|
||||
}
|
||||
Check(model::SampleVideoFrames(60, 30000.0 / 1001) ==
|
||||
std::vector<std::size_t>{0, 29},
|
||||
"fractional floor sampling");
|
||||
model::VideoOptions tiny_fps;
|
||||
tiny_fps.fps = 1e-320;
|
||||
Check(model::SampleVideoFrames(60, 30, tiny_fps) ==
|
||||
std::vector<std::size_t>{0},
|
||||
"tiny positive FPS avoids nonfinite arithmetic");
|
||||
model::VideoOptions rounding;
|
||||
rounding.max_frames = 23;
|
||||
const auto rounded = model::SampleVideoFrames(31, 1, rounding);
|
||||
Check(rounded.size() == 23 && rounded[11] == 14 && rounded.back() == 30,
|
||||
"uniform overflow matches NumPy floating linspace floor");
|
||||
const auto capped = model::SampleVideoFrames(66, 2);
|
||||
Check(capped.size() == 32 && capped.front() == 0 && capped.back() == 64 &&
|
||||
capped[30] == 60,
|
||||
"uniform overflow boundary");
|
||||
Refused([] { (void)model::SampleVideoFrames(1, 0); }, "missing fps");
|
||||
Refused([] { (void)model::SampleVideoFrames(4097, 25); },
|
||||
"frame work limit");
|
||||
Refused([] { (void)model::SampleVideoFrames(301, 1); }, "duration limit");
|
||||
Refused([] { (void)model::ProcessVideo({}); }, "empty video");
|
||||
const std::vector<std::uint8_t> garbage{1, 2, 3, 4};
|
||||
Refused([&] { (void)model::ProcessVideo(garbage); }, "malformed video");
|
||||
const std::vector<std::uint8_t> oversized(20 * 1024 * 1024 + 1);
|
||||
Refused([&] { (void)model::ProcessVideo(oversized); },
|
||||
"encoded video bound");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "FAIL video_processor_test: " << e.what() << '\n';
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,35 @@ class WindowsBundleTest(unittest.TestCase):
|
||||
(root / "libmpg123.dll").touch()
|
||||
self.assertEqual(bundle.check_bundle(root), [])
|
||||
|
||||
def test_video_imports_allow_ncrypt_but_require_bundled_codecs(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
for name in ("gufo.exe", "avformat-63.dll"):
|
||||
(root / name).touch()
|
||||
imports = {
|
||||
"gufo.exe": {"avformat-63.dll"},
|
||||
"avformat-63.dll": {"NCRYPT.DLL", "avcodec-63.dll"},
|
||||
"avcodec-63.dll": {"kernel32.dll"},
|
||||
}
|
||||
with patch.object(bundle, "imported_dlls", side_effect=lambda p: imports[p.name]):
|
||||
self.assertEqual(
|
||||
bundle.check_bundle(root),
|
||||
["avformat-63.dll requires unbundled avcodec-63.dll"],
|
||||
)
|
||||
(root / "avcodec-63.dll").touch()
|
||||
self.assertEqual(bundle.check_bundle(root), [])
|
||||
|
||||
def test_lazy_video_libraries_are_required_without_pe_imports(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "gufo.exe").touch()
|
||||
(root / "gufo-video-libraries.txt").write_text("avformat-63.dll\n", encoding="utf-8")
|
||||
with patch.object(bundle, "imported_dlls", return_value={"kernel32.dll"}):
|
||||
self.assertEqual(bundle.check_bundle(root),
|
||||
["video runtime requires unbundled avformat-63.dll"])
|
||||
(root / "AVFORMAT-63.DLL").touch()
|
||||
self.assertEqual(bundle.check_bundle(root), [])
|
||||
|
||||
def test_developer_and_crt_libraries_are_not_system_dependencies(self):
|
||||
for name in ("libomp140.x86_64.dll", "libomp.dll", "msvcp140.dll", "vcruntime140.dll"):
|
||||
with self.subTest(name=name):
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pinned official video decoder/processor/model oracle; imports no Gufo code.
|
||||
|
||||
Run with the pinned transformers source on PYTHONPATH, in nix develop.
|
||||
Media is encoded once and retained beside the fixture so codec changes cannot
|
||||
silently replace an already qualified vector. --media-only creates the clips.
|
||||
"""
|
||||
import argparse
|
||||
from fractions import Fraction
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import av
|
||||
import numpy as np
|
||||
import torch
|
||||
import transformers
|
||||
import huggingface_hub
|
||||
from transformers import AutoConfig, AutoModel, AutoProcessor
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
REVISION = '50e35cc0ca18ec19b80653690de553e6decf74f5'
|
||||
CHECKPOINT = '914f7f89142e33e77833254d9c9b90c3cef7303b'
|
||||
SPECS = [
|
||||
('short', 64, 48, 3, '5', 'libx264', 'mp4'),
|
||||
('fractional', 96, 48, 60, '30000/1001', 'libx264', 'mp4'),
|
||||
('portrait', 32, 96, 7, '2', 'libvpx-vp9', 'webm'),
|
||||
('boundary', 48, 48, 32, '1', 'libx264', 'mp4'),
|
||||
('overflow', 48, 48, 66, '2', 'libx264', 'mp4'),
|
||||
]
|
||||
|
||||
|
||||
def media(root):
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
for name, width, height, frames, fps, codec, extension in SPECS:
|
||||
path = root / f'{name}.{extension}'
|
||||
if path.exists():
|
||||
continue
|
||||
with av.open(str(path), 'w') as container:
|
||||
stream = container.add_stream(codec, rate=Fraction(fps))
|
||||
stream.width, stream.height, stream.pix_fmt = width, height, 'yuv420p'
|
||||
for t in range(frames):
|
||||
y, x, c = np.indices((height, width, 3))
|
||||
rgb = ((x*17 + y*31 + c*67 + t*29 + x*y % 251) % 256).astype(np.uint8)
|
||||
for packet in stream.encode(av.VideoFrame.from_ndarray(rgb, format='rgb24')):
|
||||
container.mux(packet)
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
|
||||
for name, size, rate, frames in [('oversized', '2048x2050', '1', 1),
|
||||
('overduration', '16x16', '1/301', 1),
|
||||
('overframes', '16x16', '30', 4097)]:
|
||||
path = root / f'{name}.mp4'
|
||||
if not path.exists():
|
||||
subprocess.run(['ffmpeg', '-v', 'error', '-f', 'lavfi', '-i',
|
||||
f'color=c=black:s={size}:r={rate}', '-frames:v', str(frames),
|
||||
'-c:v', 'libx264', '-threads', '1', '-pix_fmt', 'yuv420p', str(path)], check=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--model', default='models/embeddinggemma-2')
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--media-only', action='store_true')
|
||||
parser.add_argument('--case', action='append', help='Generate only the named probes')
|
||||
args = parser.parse_args()
|
||||
root = args.output.parent / 'video'
|
||||
media(root)
|
||||
audio_track = root / 'with_audio.mp4'
|
||||
if not audio_track.exists():
|
||||
subprocess.run(['ffmpeg', '-v', 'error', '-i', str(root/'short.mp4'),
|
||||
'-f', 'lavfi', '-i', 'sine=frequency=440:duration=0.6', '-map', '0:v:0',
|
||||
'-map', '1:a:0', '-c:v', 'copy', '-c:a', 'aac', '-shortest', str(audio_track)], check=True)
|
||||
if args.media_only:
|
||||
return
|
||||
torch.set_num_threads(8)
|
||||
processor = AutoProcessor.from_pretrained(args.model)
|
||||
config = AutoConfig.from_pretrained(args.model)
|
||||
model = AutoModel.from_pretrained(args.model, config=config, dtype=torch.float32,
|
||||
attn_implementation={'text_config': 'sdpa', 'vision_config': 'sdpa', 'audio_config': 'eager'}).eval()
|
||||
cases = []
|
||||
probes = [(s[0], [root / f'{s[0]}.{s[-1]}'], '<|video|>', {}) for s in SPECS]
|
||||
probes += [('linspace_rounding', [root/'boundary.mp4'], '<|video|>', {'fps': 0.96875, 'max_frames': 23}),
|
||||
('audio_track', [audio_track], '<|video|>', {}),
|
||||
('upsample', [root/'short.mp4'], '<|video|>', {'fps': 10, 'add_timestamps': True}),
|
||||
('sampling_control', [root/'portrait.webm'], '<|video|>', {'fps': 0.5, 'max_frames': 1}),
|
||||
('timestamps', [root/'portrait.webm'], 'before <|video|> after', {'add_timestamps': True}),
|
||||
('truncate', [root/'overflow.mp4'], '<|video|>', {'max_frames': 3, 'overflow_strategy': 'truncate'}),
|
||||
('repeat', [root/'short.mp4', root/'short.mp4'], '<|video|> then <|video|>', {}),
|
||||
('mixed', [root/'short.mp4', root/'portrait.webm'],
|
||||
'before <|video|> middle <|image|> <|audio|> after <|video|>', {})]
|
||||
for name, paths, text, options in probes:
|
||||
if args.case and name not in args.case:
|
||||
continue
|
||||
videos, records, metadatas = [], [], []
|
||||
for path in paths:
|
||||
def sample(metadata, **kwargs):
|
||||
return processor.video_processor.sample_frames(metadata, fps=options.get('fps', 1),
|
||||
max_frames=options.get('max_frames', 32),
|
||||
overflow_strategy=options.get('overflow_strategy', 'uniform'))
|
||||
with av.open(str(path)) as container:
|
||||
stream = container.streams.video[0]
|
||||
fps = float(stream.average_rate)
|
||||
decoded = np.stack([frame.to_ndarray(format='rgb24') for frame in container.decode(video=0)])
|
||||
metadata = VideoMetadata(total_num_frames=len(decoded), fps=fps,
|
||||
duration=len(decoded)/fps, width=decoded.shape[2], height=decoded.shape[1], video_backend='pyav')
|
||||
indices = sample(metadata)
|
||||
frames = decoded[indices]
|
||||
metadata.frames_indices = indices
|
||||
videos.append(decoded)
|
||||
metadatas.append(VideoMetadata(total_num_frames=len(decoded), fps=fps,
|
||||
duration=len(decoded)/fps, width=decoded.shape[2], height=decoded.shape[1], video_backend='pyav'))
|
||||
encoded = processor.video_processor([frames], do_sample_frames=False, return_tensors='pt')
|
||||
records.append({'path': 'video/' + path.name, 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
'indices': metadata.frames_indices.tolist(), 'fps': metadata.fps,
|
||||
'rgb_sha256': [hashlib.sha256(frame.tobytes()).hexdigest() for frame in frames],
|
||||
'patches': []})
|
||||
for pixels, positions in zip(encoded['pixel_values_videos'], encoded['video_position_ids']):
|
||||
valid = (positions != -1).all(-1)
|
||||
raw = (pixels[valid] * 255).round().to(torch.uint8).numpy().tobytes()
|
||||
records[-1]['patches'].append({'rows': int(positions[valid, 1].max())+1,
|
||||
'columns': int(positions[valid, 0].max())+1,
|
||||
'sha256': hashlib.sha256(raw).hexdigest()})
|
||||
# Pass encoded paths to exercise official decoding and its metadata.
|
||||
kwargs = {}
|
||||
if name == 'mixed':
|
||||
from PIL import Image
|
||||
kwargs['images'] = [[Image.new('RGB', (32, 32), (255, 0, 0))]]
|
||||
kwargs['audio'] = [np.sin(np.arange(3200, dtype=np.float32)*0.07)*0.1]
|
||||
kwargs['audio_kwargs'] = {'sampling_rate': 16000}
|
||||
inputs = processor(text=[text], videos=videos,
|
||||
videos_kwargs={'video_metadata': metadatas, **options},
|
||||
return_tensors='pt', **kwargs)
|
||||
with torch.inference_mode():
|
||||
hidden = model(**inputs).last_hidden_state
|
||||
mask = inputs['attention_mask'].unsqueeze(-1)
|
||||
vector = torch.nn.functional.normalize((hidden*mask).sum(1)/mask.sum(1), dim=-1)[0].tolist()
|
||||
cases.append({'name': name, 'text': text, 'videos': records, 'options': options,
|
||||
'token_ids': inputs['input_ids'][0].tolist(),
|
||||
'embedding': [float(f'{v:.7g}') for v in vector]})
|
||||
print(name, len(cases[-1]['token_ids']), 'tokens', flush=True)
|
||||
args.output.write_text(json.dumps({'model_revision': CHECKPOINT,
|
||||
'transformers_revision': REVISION, 'transformers': transformers.__version__,
|
||||
'torch': torch.__version__, 'av': av.__version__,
|
||||
'huggingface_hub': huggingface_hub.__version__,
|
||||
'av_libraries': av.library_versions,
|
||||
'attention': {'text_config': 'sdpa', 'vision_config': 'sdpa', 'audio_config': 'eager'}, 'dtype': 'float32', 'cases': cases}, separators=(',', ':')))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -14,7 +14,7 @@ import pefile
|
||||
# Windows 11 inbox libraries, not development tools, CRT or GPU SDK runtimes.
|
||||
SYSTEM_DLLS = frozenset(
|
||||
"""advapi32.dll bcrypt.dll cfgmgr32.dll crypt32.dll gdi32.dll iphlpapi.dll
|
||||
kernel32.dll ntdll.dll ole32.dll oleaut32.dll psapi.dll secur32.dll setupapi.dll
|
||||
kernel32.dll ncrypt.dll ntdll.dll ole32.dll oleaut32.dll psapi.dll secur32.dll setupapi.dll
|
||||
shell32.dll shlwapi.dll user32.dll version.dll winmm.dll ws2_32.dll""".split()
|
||||
)
|
||||
|
||||
@@ -45,6 +45,18 @@ def check_bundle(directory):
|
||||
if "gufo.exe" not in images:
|
||||
raise ValueError("bundle is missing gufo.exe")
|
||||
missing = []
|
||||
# Lazy-loaded video libraries do not appear in gufo.exe's import table.
|
||||
# Older releases have no manifest and retain their import-only contract.
|
||||
manifest = directory / "gufo-video-libraries.txt"
|
||||
if manifest.is_file():
|
||||
names = manifest.read_text(encoding="utf-8").splitlines()
|
||||
if not names or any(not name.lower().endswith(".dll") or
|
||||
"/" in name or "\\" in name for name in names):
|
||||
raise ValueError("invalid video runtime library manifest")
|
||||
for name in names:
|
||||
if name.lower() not in images:
|
||||
missing.append(f"video runtime requires unbundled {name.lower()}")
|
||||
|
||||
for name, path in sorted(images.items()):
|
||||
for dependency in sorted(imported_dlls(path)):
|
||||
if dependency not in images and not is_system_dll(dependency):
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
"libwebp",
|
||||
"libsndfile",
|
||||
"soxr",
|
||||
{"name": "ffmpeg", "default-features": false, "features": ["avcodec", "avformat", "swscale"]},
|
||||
"pkgconf"
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user