From 465cd61dfb975a6d6bdf3f8403489ef960d2f622 Mon Sep 17 00:00:00 2001 From: Niko1221 Date: Mon, 12 Oct 2026 21:04:17 +0200 Subject: [PATCH] Formats cost nothing unless used: Q6_K gate/up experts in the normal build; the K-quant MMQ prompt kernels on by default, on CUDA as their own library (strata_kq) opened only for a model with these experts and STRATA_MMQ_KQ=1 --- CMakeLists.txt | 69 ++++++++++---- data/quant_quality/v1/docs.txt | 95 +++++++++----------- data/quant_quality/v1/set.ids.json | 2 +- include/strata/prefill/mmq_kq_abi.hpp | 30 +++++++ src/prefill/mmq_kq_lib.cu | 60 +++++++++++++ src/prefill/mmq_kq_loader.cpp | 124 ++++++++++++++++++++++++++ src/prefill/moe_mmq.cu | 46 ++++++++-- src/program/generate.cpp | 9 ++ sycl/CMakeLists.txt | 6 ++ 9 files changed, 364 insertions(+), 77 deletions(-) create mode 100644 include/strata/prefill/mmq_kq_abi.hpp create mode 100644 src/prefill/mmq_kq_lib.cu create mode 100644 src/prefill/mmq_kq_loader.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f6426a45..f35277bd4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,14 +70,17 @@ if(STRATA_HIP_GFX906) endif() option(STRATA_ENABLE_HIP "Build the HIP targets for wave32 AMD GPUs (gfx1100, gfx1201; gfx1012/gfx1030 unvalidated)" OFF) option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF) -# CUDA: MMQ prompt kernels for Q4_K / Q5_K / Q5_1 (Unsloth's UD-Q4_K_XL experts). Off by default: the engine loads -# every kernel when it starts (CUDA_MODULE_LOADING=EAGER), so these took ~5 MB of VRAM - 1-3 expert slots - from every -# model, and the UD-Q4_K_XL prompt is SSD-bound with or without them (docs/UNSLOTH_Q4.md) -option(STRATA_MMQ_KQUANTS "Build the MMQ prompt kernels for Q4_K / Q5_K / Q5_1 / Q6_K: the CUDA experts of Unsloth's UD-Q4_K_XL, and on HIP the dense GGUF projections (STRATA_DENSE_MMQ)" OFF) -# CUDA: Q6_K gate/up in the grouped expert kernels (Unsloth's UD-Q6_K_XL, docs/UNSLOTH_Q6.md). Off by default: every -# grouped-kernel instance is loaded when the engine starts (CUDA_MODULE_LOADING=EAGER), so one more costs VRAM that -# every other model would lose as expert slots (PR #860 review) -option(STRATA_Q6K_EXPERTS "Build the Q6_K gate/up grouped expert kernels (UD-Q6_K_XL)" OFF) +# MMQ prompt kernels for Q4_K / Q5_K / Q5_1 / Q6_K (Unsloth's UD-Q4/Q5/Q6_K_XL experts; on HIP also the dense GGUF +# projections, STRATA_DENSE_MMQ). On by default since MODEL-UPDATE Phase 1: on CUDA they are their own library +# (strata_kq, loaded only for a model with these formats: the engine loads every kernel of its own binary at start, +# CUDA_MODULE_LOADING=EAGER, and these took ~3.8 MB of VRAM - 3 IQ3_XXS expert slots on a P100 - from every model); on +# HIP a code object loads on first use, so they cost other models nothing there (R9700: the same slots). Used at run +# time with STRATA_MMQ_KQ=1 (opt-in: it changes the prompt's rounding against the FP16 path). +option(STRATA_MMQ_KQUANTS "Build the MMQ prompt kernels for Q4_K / Q5_K / Q5_1 / Q6_K: the experts of Unsloth's K-quant files, and on HIP the dense GGUF projections (STRATA_DENSE_MMQ)" ON) +# Q6_K gate/up in the grouped expert kernels (Unsloth's UD-Q6_K_XL, and layer 2 of UD-Q5_K_XL; docs/QUANTS.md). On by +# default since MODEL-UPDATE Phase 1: one more instance per grouped kernel in iq_kernels.cu, +19 KB of sm_60 code, +# measured to leave every expert-cache slot count as it was (P100, R9700). -DSTRATA_Q6K_EXPERTS=OFF drops it. +option(STRATA_Q6K_EXPERTS "Build the Q6_K gate/up grouped expert kernels (UD-Q6_K_XL, UD-Q5_K_XL)" ON) if(STRATA_Q6K_EXPERTS) add_compile_definitions(STRATA_Q6K_EXPERTS=1) endif() @@ -1365,14 +1368,18 @@ if(STRATA_NATIVE_EXPERTS) add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/moe_fused.cu src/prefill/moe_fused_iq.cu src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu) # q8_0: the drafter's matrices (E-9); STRATA_ORCA_Q4KS_MMQ adds q5_0 (#296, in _strata_mmq_types), and - # STRATA_MMQ_KQUANTS q4_k q5_k q5_1, the experts of Unsloth's UD-Q4_K_XL (docs/UNSLOTH_Q4.md; the dispatch - # follows eddoursul/Strata 7706c1b); q6_k is HIP only + # STRATA_MMQ_KQUANTS q4_k q5_k q5_1 q6_k, the experts of Unsloth's UD-Q4/Q5/Q6_K_XL (docs/QUANTS.md; the dispatch + # follows eddoursul/Strata 7706c1b). MODEL-UPDATE Phase 1: on CUDA they are a library of their own, strata_kq + # (below), loaded only for a model with these experts: the engine loads every kernel of its own binary at start + # (CUDA_MODULE_LOADING=EAGER), and these four took ~3.8 MB of VRAM (3 expert slots of IQ3_XXS on a P100) from every + # model. gfx906 (HIP: code objects load on first use) keeps them in strata_mmq. set(_strata_mmq_cuda_types ${_strata_mmq_types}) - if(STRATA_MMQ_KQUANTS) - list(APPEND _strata_mmq_cuda_types q4_k q5_k q5_1) - if(STRATA_Q6K_EXPERTS OR STRATA_HIP_GFX906) # UD-Q6_K_XL gate/up (docs/UNSLOTH_Q6.md), - list(APPEND _strata_mmq_cuda_types q6_k) # and gfx906: moe_mmq.cu calls it under __HIPCC__ - endif() + set(_strata_kq_types q4_k q5_k q5_1 q6_k) + set(_strata_kq_dso OFF) + if(STRATA_MMQ_KQUANTS AND STRATA_HIP_GFX906) + list(APPEND _strata_mmq_cuda_types ${_strata_kq_types}) + elseif(STRATA_MMQ_KQUANTS) + set(_strata_kq_dso ON) endif() foreach(_t IN LISTS _strata_mmq_cuda_types) target_sources(strata_mmq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu) @@ -1391,6 +1398,35 @@ if(STRATA_NATIVE_EXPERTS) if(STRATA_MMQ_KQUANTS) target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_KQUANTS=1) endif() + if(_strata_kq_dso) + # strata_kq: the K-quant MMQ instances with their own copy of ggml_cuda_host.cu's symbols and the static CUDA + # runtime, next to the engine (libstrata_kq.so / strata_kq.dll); moe_mmq.cu opens it when a model asks for one + # of these formats (STRATA_MMQ_KQ=1). Without the file the engine runs them on the FP16 prompt path, as before. + target_compile_definitions(strata_mmq PRIVATE STRATA_KQ_DSO=1) + target_sources(strata_mmq PRIVATE src/prefill/mmq_kq_loader.cpp) + if(NOT WIN32) + target_link_libraries(strata_mmq PUBLIC ${CMAKE_DL_LIBS}) + endif() + add_library(strata_kq SHARED src/prefill/mmq_kq_lib.cu src/prefill/ggml_cuda_host.cu) + foreach(_t IN LISTS _strata_kq_types) + target_sources(strata_kq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu) + endforeach() + target_include_directories(strata_kq PRIVATE ${_strata_gg}/include ${_strata_gg}/src ${_strata_mmq} + ${CMAKE_CURRENT_SOURCE_DIR}/include) + target_compile_options(strata_kq PRIVATE $<$:--extended-lambda -use_fast_math>) + if(MSVC) + target_compile_options(strata_kq PRIVATE $<$:-Xcompiler=/bigobj>) + endif() + set_target_properties(strata_kq PROPERTIES CUDA_RUNTIME_LIBRARY Static POSITION_INDEPENDENT_CODE ON + CXX_VISIBILITY_PRESET hidden CUDA_VISIBILITY_PRESET hidden) + # ggml-base is static: position-independent so a shared library can hold it (the same code otherwise) + set_target_properties(ggml-base PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_link_libraries(strata_kq PRIVATE ggml-base) + if(TARGET CUDA::cuda_driver) + target_link_libraries(strata_kq PRIVATE CUDA::cuda_driver) + endif() + add_dependencies(strata_mmq strata_kq) + endif() if(STRATA_ORCA_Q4KS_MMQ) target_compile_definitions(strata_mmq PRIVATE STRATA_ORCA_Q4KS_MMQ=1) endif() @@ -1413,7 +1449,8 @@ if(STRATA_NATIVE_EXPERTS) add_executable(prefill_mmq_kquant_test tests/cuda/prefill_mmq_kquant_test.cpp) target_link_libraries(prefill_mmq_kquant_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target}) add_test(NAME prefill_mmq_kquant_test COMMAND prefill_mmq_kquant_test) - set_tests_properties(prefill_mmq_kquant_test PROPERTIES TIMEOUT 120) + # the K-quant MMQ is opt-in at run time (STRATA_MMQ_KQ=1), and on CUDA it is strata_kq next to the test + set_tests_properties(prefill_mmq_kquant_test PROPERTIES TIMEOUT 120 ENVIRONMENT STRATA_MMQ_KQ=1) endif() elseif(STRATA_PREFILL_MMQ) # Compile only the pinned GGML MMQ and activation-quantizer sources used by prefill. This uses diff --git a/data/quant_quality/v1/docs.txt b/data/quant_quality/v1/docs.txt index e73393f86..3705b5cbb 100644 --- a/data/quant_quality/v1/docs.txt +++ b/data/quant_quality/v1/docs.txt @@ -1,56 +1,49 @@ -## What is in the file +- **GPU (VRAM):** attention and DeltaNet mixers, the gated-residual weights, routers, shared experts, output head, the MTP + draft layer, the KV cache (from 64K: only its most-read part, the rest streams from RAM), and an **expert cache** that fills the rest of VRAM with the most-used experts (it adapts to + the conversation while you chat). +- **RAM:** all 24,576 experts, pinned. The CPU computes the experts that are not on the GPU **in place**, at the same time + as the GPU works on the cached ones (AVX-512 / AVX2 kernels, ggml's for the i-quants). +- **SSD:** the 28.8 GB n-gram table, a few rows per token, read unbuffered past the OS cache (`--ple-io direct`, the + default, made for SSDs; on a rotational disk `--ple-io ram` keeps the table in RAM, #605). A start-up probe warns when direct reads measure slow (under 15K rows/s) but never switches by itself (since 0.1.43.1; `STRATA_PLE_AUTO_MMAP=1` brings back the 0.1.42 switch to `--ple-io mmap`): the bar is calibrated for NVMe, a SATA SSD tops out at 10-14K rows/s with direct's 16 reads in flight, and on one (Linux, 77.8K-token prompt) mmap read 2.1K rows/s against direct's 14.4K (#2040), while on a Windows box with a slow mapped file mmap stalled prompts (#1984). The engine reads the table + in the format the GGUF has it: IQ4_NL (the default table), Q4_0, Q5_0, Q5_1, Q8_0, FP8 (E4M3 with a scale) or BF16. + Measured on 4,000 random rows against the checkpoint's own BF16 table, the mean per-row error is Q8_0 0.53%, FP8 2.64%, + Q5_1 3.78%, Q5_0 4.25%, IQ4_NL 7.60%, Q4_1 7.80%, Q4_0 8.55% (a Q8_0 table is 54 GB, an IQ4_NL one 28.8 GB). On the Q2_0 model, 700 teacher-forced tokens, the mean KL against the BF16 table is 0.0117 (Q5_0), 0.0121 (Q5_1), 0.0125 + (Q8_0), 0.0128 (FP8), 0.0140 (Q4_0) and 0.0146 (IQ4_NL), with the perplexity within 1% of BF16's either way: any change to the + table moves the 2-bit model by about 0.012, so the formats are hard to tell apart. Setup does not offer another table; + it only changes which GGUF the engine is given. +- **Speculation:** the model's own MTP layer drafts up to 3 tokens; one pass over all 48 layers checks them. 2.4-3.2 + tokens per pass on average. When the reply repeats the context (code edits, quoted text), **prompt lookup** (engine + 0.1.7) drafts up to 5 tokens from the earlier copy, but only where its measured acceptance and cost say it pays: + code edits 6-11% faster, other text unchanged. The drafts are checked like the MTP's, so the output is the same. +- **Prompts** are processed in chunks of up to 8,192 tokens (`--prefill auto`; 32,768 opt-in) with the experts + streamed to the GPU over PCIe. Since engine 0.1.39b (#583) the streamed ring is sized in bytes for the pack (a + native pack with bigger blobs gets fewer ring slots and keeps more cache slots) and `--prefill auto` picks the + largest chunk that keeps that ring full. It only gives up ring slots where 0.1.39's rule held the chunk at 4,096 + tokens or less; from 6,144 on it keeps 0.1.39's ring and only looks for a bigger chunk beside it (shrinking the + ring there measured slower: the Coder -12%), and a prompt that fits 0.1.39's chunk keeps 0.1.39's ring (one + chunk: a smaller ring only slowed it). RTX 5070, 32K prompts against `STRATA_RING_BYTES=0`, 2-3 interleaved + rounds: IQ3_XXS +18.5% at a 1,500-slot cache and +9% in another memory state, the Coder +3%, IQ3_S unchanged; 4K + prompts unchanged (-0.7% to +1.1%, bit-identical). Q2_0 (a fixed `--prefill 2048`) is not affected. It changes + the bits of a long prompt against 0.1.39 (the experts go through a different + mix of cached and streamed groups). Measured quality, teacher-forced over the next 2,001 tokens of a long document + against the FP16 prompt path (IQ3_XXS, `--prefill auto`): 8K prompt KL 0.042 (0.1.39: 0.054), top-1 agreement + 93.5% (92.2%); 32K prompt KL 0.020 (0.019), top-1 95.3% (95.0%) - the same band as before. `STRATA_RING_BYTES=0` + restores 0.1.39's ring, loan and chunk choice. -Six shards, 169,165,382,688 bytes (169.2 GB) together ([sizes and SHA-256 from the HF tree API](https://huggingface.co/api/models/unsloth/Qwen3.8-Flash-Next-GGUF/tree/38bb39ee97821de2c9009abb7e93950eec396e66/UD-Q6_K_XL); -shards 1, 2 and 6 were also checked bit for bit against their Hugging Face LFS hashes after download). Shard 1 holds -only metadata; shard 2 only the output head (`output.weight`, `output_hc_*`, all Q8_0 but `output_hc_norm`, F32); -shard 3 is exactly the PLE table, nothing else. +The full story, with measurements, bottlenecks and what comes next: **[docs/paper/Strata-Paper.pdf](paper/Strata-Paper.pdf)**. -| File | Bytes | SHA-256 | -| --- | --- | --- | -| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00001-of-00006.gguf` | 10,946,624 | `8bdc6bad55b6699f46f60708829932ff4018d7ac9bd309c388534fa31753bdb7` | -| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00002-of-00006.gguf` | 682,434,912 | `494ca4ed3dbf97bc28da88af3890b8877b9032f909812d00c0526a9ca5e91d2e` | -| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00003-of-00006.gguf` | 54,400,261,312 | `34efd79a80a1ce540a517a5d56171924b66ce1c38b04c904f17ad6d8ef17cf20` | -| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00004-of-00006.gguf` | 49,814,108,512 | `0ea4b599880a5a52fcd9c188ba1443f646c69c96571904ddbd6a79246a01a997` | -| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00005-of-00006.gguf` | 49,419,317,344 | `9948e81ae8368144b135a7403d8c0f2cf9a6c2f9f692313865855b2832e696c9` | -| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00006-of-00006.gguf` | 14,838,313,984 | `e4ae255234f42b18012f6d6eec5bf615b57479b7e0a0233c47436a321ffeec3f` | +--- -Formats, read from the shard headers with `tools/gguf_reader.py` (all six shards verified: shard 4 = blocks 0-20, -520 tensors, shard 5 = blocks 20-41, 530 tensors, and per layer the types above): +## Credits and licenses -- **Routed experts:** `ffn_gate_exps` and `ffn_up_exps` are Q6_K (seen in every layer of shard 6: blocks 41-47); - `ffn_down_exps` is Q8_0 (per layer 891.3 MB at 640×2560×512, against Q6_K's 688.1 MB at 2560×640×512 for gate or - up). Gate and up are Q6_K in 47 of 48 layers and Q8_0 in layer 2; down is Q8_0 in every layer (the pack's - `native_experts.txt`: 47 rows of ggml types 14/8, and layer 2 is 8/8). Q6_K gate/up is native: the - engine multiplies them directly on the GPU in every path check below, and the CPU has the ggml block layout (210 - bytes per 256 elements); the per-role layout in `native_experts.txt` v4 handles shards whose boundaries cut - through a layer like UD-Q4_K_XL's layer 11 case does. -- **Everything else:** attention and shared-expert projections, SSM, hyper-connections, the PLE key/value, the token - embedding and the output head are Q8_0, or F32/BF16 where the engine reads floats — the same pattern - [UD-Q4_K_XL](UNSLOTH_Q4.md) handles with `tools/iq_pack.py --compat-bf16`. -- **The PLE table** (`per_layer_token_embd.weight`, [160, 320001536]): **Q8_0**, its own shard 3. A row is - 5 blocks of 32 elements at 34 bytes = 170 bytes, and the whole table is 54.4 GB, against the 28.8 GB IQ4_NL of - UD-Q4_K_XL. Engine 0.1.38 reads Q8_0 rows natively: `PleTable` accepts the type, the row reader is already - parameterized by bytes-per-row (its selftest runs 90, 110 and 170 byte rows), and no GPU kernel changes — - `gather` outputs floats. (UD-Q4_K_XL's table is IQ4_NL; the FP8 table of [tools/ple_fp8_pack.py](../tools/ple_fp8_pack.py) - needs the original checkpoint, which this file is not a substitute for. Converting this Q8_0 table to FP8 would - trade table precision away for 6% smaller rows; Q8_0 read directly has none that conversion's error.) +Strata itself: [MIT](../LICENSE). The model files are not part of it; their licenses apply to them (below). -Q6_K gate/up costs 210 bytes per 256 elements, against Q4_K's 144, so the routed experts' share of the file grows -about 45% per element read; expected decode speed accordingly lower than UD-Q4_K_XL's on the same hardware. The -table row gather grows 170/90 = 1.89x over IQ4_NL. - -## The pack (by hand) - -Same workflow as [UD-Q4_K_XL](UNSLOTH_Q4.md#the-pack-by-hand), from the repository root: - -```sh -.venv/bin/python tools/iq_pack.py \ - --gguf /path/to/Qwen3.8-Flash-Next-UD-Q6_K_XL-00001-of-00006.gguf \ - --out packs/ud-q6_k_xl --compat-bf16 -``` - -**`--compat-bf16` is required** for the same reason as there (the 195 small Q8_0 projections the engine reads as -BF16). Do not add `--experts-bin`; the experts stay in the GGUF files. Built here end to end on this file -(`packs/ud-q6_k_xl`, the packer's own tests cover the shard and conversion logic via -`.venv/bin/python -m unittest discover -s tools -p test_iq_pack.py`): 1079 tensors indexed, 303 served natively, 460 -tensors converted (26 \ No newline at end of file +- Model: [Qwen/Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next) by the Qwen team; quantizations: + [ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF). + The Coder: [ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF) + (Apache-2.0 per its card); its support in Strata came from @pjgmobile's PR #54. + Swift 1.5: [ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF](https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF) + by UkisAI. Their licenses apply to the weights. +- [llama.cpp / ggml](https://github.com/ggml-org/llama.cpp) (MIT): the i-quant formats, the GPU dot products and + dequantizers transcribed in `src/kernels/cuda/iq_kernels.cu`, the CPU backend linked for the i-quant experts, the + `mtmd` library behind the image encoder (`tools/vision/`), and `gguf-py` use \ No newline at end of file diff --git a/data/quant_quality/v1/set.ids.json b/data/quant_quality/v1/set.ids.json index f487b6495..67081e5c9 100644 --- a/data/quant_quality/v1/set.ids.json +++ b/data/quant_quality/v1/set.ids.json @@ -1 +1 @@ -{"version": "v1", "seg_tokens": 1000, "segments": [{"name": "code", "ids": [12237, 85286, 54802, 19913, 343, 18, 4108, 6396, 471, 874, 50285, 980, 11, 874, 50285, 1644, 2195, 88, 13, 271, 34447, 1521, 279, 1510, 13985, 1644, 63, 5187, 1828, 6129, 4021, 1724, 411, 35426, 25, 1141, 1510, 21563, 2581, 42750, 1954, 900, 63, 198, 8746, 682, 874, 4314, 364, 913, 220, 19, 17, 27718, 48, 17, 62, 15, 63, 681, 864, 369, 24733, 1056, 279, 6522, 13, 2844, 369, 4924, 264, 387, 15, 782, 21, 198, 64919, 471, 279, 3443, 411, 4560, 369, 26208, 364, 369, 8047, 314, 279, 5129, 5224, 286, 13, 271, 1919, 369, 279, 5604, 7873, 364, 1510, 3431, 53683, 20123, 14, 63, 318, 77646, 16137, 16, 20, 25, 328, 21563, 19913, 343, 18, 6396, 318, 76, 2117, 11, 198, 2083, 50285, 980, 23285, 61406, 1049, 68434, 279, 4108, 1132, 26, 15167, 795, 369, 2496, 1301, 13, 198, 12237, 198, 1445, 1259, 20409, 548, 1120, 31144, 271, 464, 795, 8588, 198, 464, 51372, 198, 464, 1968, 198, 1445, 18904, 1120, 5586, 271, 21563, 19913, 47540, 283, 220, 15, 87, 19, 21, 20, 20, 19, 22, 19, 22, 220, 653, 328, 21563, 19913, 1, 2545, 12692, 1068, 271, 2, 50285, 980, 1756, 2663, 13, 220, 19, 17, 283, 1167, 17, 62, 15, 11, 279, 34100, 48, 12, 7146, 46, 69764, 17830, 517, 10845, 11, 26334, 494, 50285, 1644, 2195, 88, 430, 314, 220, 15, 13, 16, 24, 13, 198, 21563, 2581, 24973, 25, 6249, 18078, 11, 592, 60, 283, 313, 198, 262, 220, 15, 25, 328, 37, 18, 17, 487, 220, 16, 25, 328, 37, 16, 21, 487, 220, 17, 25, 328, 48, 19, 62, 15, 487, 220, 18, 25, 328, 48, 19, 62, 16, 487, 220, 19, 25, 328, 48, 19, 62, 17, 487, 220, 20, 25, 328, 48, 19, 62, 18, 487, 220, 21, 25, 328, 48, 20, 62, 15, 487, 220, 22, 25, 328, 48, 20, 62, 16, 487, 198, 262, 220, 23, 25, 328, 48, 23, 62, 15, 487, 220, 24, 25, 328, 48, 23, 62, 16, 487, 220, 16, 15, 25, 328, 48, 17, 9802, 487, 220, 16, 16, 25, 328, 48, 18, 9802, 487, 220, 16, 17, 25, 328, 48, 19, 9802, 487, 220, 16, 18, 25, 328, 48, 20, 9802, 487, 220, 16, 19, 25, 328, 48, 21, 9802, 487, 198, 262, 220, 16, 20, 25, 328, 48, 23, 9802, 487, 220, 16, 21, 25, 328, 59865, 17, 62, 5955, 50, 487, 220, 16, 22, 25, 328, 59865, 17, 6645, 50, 487, 220, 16, 23, 25, 328, 59865, 18, 62, 5955, 50, 487, 220, 16, 24, 25, 328, 59865, 16, 1063, 487, 220, 17, 15, 25, 328, 59865, 19, 1548, 43, 487, 198, 262, 220, 17, 16, 25, 328, 59865, 18, 1063, 487, 220, 17, 17, 25, 328, 59865, 17, 1063, 487, 220, 17, 18, 25, 328, 59865, 19, 6645, 50, 487, 220, 17, 19, 25, 328, 40, 23, 487, 220, 17, 20, 25, 328, 40, 16, 21, 487, 220, 17, 21, 25, 328, 40, 18, 17, 487, 220, 17, 22, 25, 328, 40, 21, 19, 487, 198, 262, 220, 17, 23, 25, 328, 37, 21, 19, 487, 220, 17, 24, 25, 328, 59865, 16, 1203, 487, 220, 18, 15, 25, 328, 19278, 16, 21, 487, 220, 18, 16, 25, 328, 48, 19, 62, 15, 62, 19, 62, 19, 487, 220, 18, 17, 25, 328, 48, 19, 62, 15, 62, 19, 62, 23, 487, 220, 18, 18, 25, 328, 48, 19, 62, 15, 62, 23, 62, 23, 487, 198, 262, 220, 18, 19, 25, 328, 51, 48, 16, 62, 15, 487, 220, 18, 20, 25, 328, 51, 48, 17, 62, 15, 487, 220, 18, 21, 25, 328, 59865, 19, 1548, 43, 62, 19, 62, 19, 487, 220, 18, 22, 25, 328, 59865, 19, 1548, 43, 62, 19, 62, 23, 487, 220, 18, 23, 25, 328, 59865, 19, 1548, 43, 62, 23, 62, 23, 487, 198, 262, 220, 18, 24, 25, 328, 28463, 11361, 19, 487, 220, 19, 15, 25, 328, 35125, 11361, 19, 487, 220, 19, 16, 25, 328, 48, 19, 62, 15, 62, 23, 62, 23, 487, 220, 19, 17, 25, 328, 48, 17, 62, 15, 487, 198, 92, 271, 2, 54802, 19913, 10836, 869, 913, 13746, 198, 21563, 19913, 57818, 283, 313, 198, 262, 220, 15, 25, 3376, 84, 23, 487, 220, 16, 681, 220, 16, 25, 3376, 72, 23, 487, 220, 16, 681, 220, 17, 25, 3376, 84, 16, 21, 487, 220, 17, 681, 220, 18, 25, 3376, 72, 16, 21, 487, 220, 17, 681, 220, 19, 25, 3376, 84, 18, 17, 487, 220, 19, 681, 220, 20, 25, 3376, 72, 18, 17, 487, 220, 19, 681, 198, 262, 220, 21, 25, 3376, 69, 18, 17, 487, 220, 19, 681, 220, 22, 25, 3376, 2554, 487, 220, 16, 681, 220, 23, 25, 3376, 889, 487, 2168, 681, 220, 24, 25, 3376, 1595, 487, 2168, 681, 198, 262, 220, 16, 15, 25, 3376, 84, 21, 19, 487, 220, 23, 681, 220, 16, 16, 25, 3376, 72, 21, 19, 487, 220, 23, 681, 220, 16, 17, 25, 3376, 69, 21, 19, 487, 220, 23, 681, 198, 92, 271, 2, 50285, 980, 2424, 16547, 364, 279, 3107, 56373, 411, 1558, 5533, 25, 318, 4425, 5255, 11, 5639, 791, 2424, 8, 198, 2, 1061, 369, 1092, 3520, 264, 791, 2308, 3677, 4763, 1698, 1716, 470, 11, 321, 424, 369, 279, 4954, 279, 61369, 4191, 13, 198, 38637, 57038, 83072, 25, 6249, 16793, 11, 13992, 18078, 11, 514, 4891, 283, 313, 198, 262, 328, 37, 18, 17, 763, 318, 16, 11, 220, 19, 681, 328, 37, 16, 21, 763, 318, 16, 11, 220, 17, 681, 328, 19278, 16, 21, 763, 318, 16, 11, 220, 17, 681, 328, 37, 21, 19, 763, 318, 16, 11, 220, 23, 681, 198, 262, 328, 48, 19, 62, 15, 763, 318, 18, 17, 11, 220, 16, 23, 681, 328, 48, 19, 62, 16, 763, 318, 18, 17, 11, 220, 17, 15, 681, 328, 48, 20, 62, 15, 763, 318, 18, 17, 11, 220, 17, 17, 681, 328, 48]}, {"name": "docs", "ids": [550, 3437, 369, 303, 279, 999, 271, 40087, 71590, 11, 220, 16, 21, 24, 11, 16, 21, 20, 11, 18, 23, 17, 11, 21, 23, 23, 5639, 318, 16, 21, 24, 13, 17, 18295, 8, 3658, 11406, 51368, 321, 21050, 12, 17, 20, 21, 494, 279, 48855, 4757, 5165, 9251, 2349, 1074, 71, 34098, 1509, 6616, 10195, 19652, 14, 10545, 67194, 43765, 16451, 18, 13, 23, 12, 23129, 12, 5666, 11668, 38, 19913, 60854, 14, 18, 23, 5876, 18, 24, 2059, 24, 22, 23, 17, 16, 442, 17, 66, 24, 15, 15, 24, 12163, 22, 68, 24, 18, 24, 20, 15, 92520, 18, 24, 21, 68, 21, 21, 14, 4505, 27325, 21, 9802, 6645, 43, 1174, 198, 898, 2271, 220, 16, 11, 220, 17, 321, 220, 21, 998, 1048, 9770, 2612, 364, 2612, 2272, 836, 462, 34098, 18036, 436, 8225, 48570, 1238, 4003, 553, 92071, 220, 16, 9687, 198, 3138, 10836, 26, 50319, 220, 17, 1132, 279, 2468, 1901, 27718, 2911, 24354, 7561, 1510, 2911, 1469, 66, 44272, 7561, 660, 1167, 23, 62, 15, 694, 1510, 2911, 1469, 66, 18601, 7561, 426, 18, 17, 1174, 198, 898, 552, 220, 18, 369, 6681, 279, 387, 841, 1898, 11, 4161, 745, 13, 271, 91, 2793, 735, 29041, 735, 21050, 12, 17, 20, 21, 735, 198, 91, 12091, 735, 12091, 735, 12091, 735, 198, 91, 1510, 48, 16451, 18, 13, 23, 12, 23129, 12, 5666, 12, 4505, 27325, 21, 9802, 6645, 43, 12, 15, 15, 15, 15, 16, 8404, 12, 15, 15, 15, 15, 21, 75021, 1644, 63, 735, 220, 16, 15, 11, 24, 19, 21, 11, 21, 17, 19, 735, 1510, 23, 65, 7385, 21, 13458, 20, 20, 65, 21, 21, 24, 24, 69, 19, 21, 69, 21, 15, 22, 15, 23, 23, 17, 24, 24, 18, 17, 529, 19, 15, 16, 23, 67, 22, 565, 24, 8670, 18, 15, 24, 66, 18, 23, 23, 20, 18, 19, 3510, 18, 16, 22, 20, 18, 82800, 22, 63, 735, 198, 91, 1510, 48, 16451, 18, 13, 23, 12, 23129, 12, 5666, 12, 4505, 27325, 21, 9802, 6645, 43, 12, 15, 15, 15, 15, 17, 8404, 12, 15, 15, 15, 15, 21, 75021, 1644, 63, 735, 220, 21, 23, 17, 11, 19, 18, 19, 11, 24, 16, 17, 735, 1510, 19, 24, 19, 895, 19, 290, 18, 1932, 69, 24, 22, 8635, 17, 23, 3131, 23, 23, 2492, 18, 23, 24, 15, 65, 23, 23, 22, 22, 65, 24, 15, 18, 17, 69, 24, 15, 24, 23, 16, 17, 67, 15, 15, 66, 15, 20, 17, 21, 64, 24, 895, 20, 68, 24, 16, 67, 17, 68, 63, 735, 198, 91, 1510, 48, 16451, 18, 13, 23, 12, 23129, 12, 5666, 12, 4505, 27325, 21, 9802, 6645, 43, 12, 15, 15, 15, 15, 18, 8404, 12, 15, 15, 15, 15, 21, 75021, 1644, 63, 735, 220, 20, 19, 11, 19, 15, 15, 11, 17, 21, 16, 11, 18, 16, 17, 735, 1510, 18, 19, 90450, 22, 24, 64, 23, 15, 64, 16, 341, 20, 19, 15, 64, 20, 16, 22, 64, 20, 67, 20, 21, 16, 22, 16, 24, 17, 19, 65, 21, 21, 341, 16, 66, 18, 23, 65, 15, 19, 66, 24, 15, 19, 69, 16, 22, 327, 21, 67, 23, 797, 16, 22, 9500, 17, 15, 63, 735, 198, 91, 1510, 48, 16451, 18, 13, 23, 12, 23129, 12, 5666, 12, 4505, 27325, 21, 9802, 6645, 43, 12, 15, 15, 15, 15, 19, 8404, 12, 15, 15, 15, 15, 21, 75021, 1644, 63, 735, 220, 19, 24, 11, 23, 16, 19, 11, 16, 15, 23, 11, 20, 16, 17, 735, 1510, 15, 12150, 19, 65, 20, 24, 24, 23, 23, 15, 64, 20, 64, 20, 17, 69, 4243, 24, 66, 16, 23, 23, 4494, 16, 19, 19, 18, 69, 21, 19, 21, 66, 21, 24, 66, 24, 21, 20, 22, 16, 24, 15, 19, 613, 8670, 21, 64, 22, 24, 17, 19, 21, 64, 15, 16, 64, 24, 24, 22, 63, 735, 198, 91, 1510, 48, 16451, 18, 13, 23, 12, 23129, 12, 5666, 12, 4505, 27325, 21, 9802, 6645, 43, 12, 15, 15, 15, 15, 20, 8404, 12, 15, 15, 15, 15, 21, 75021, 1644, 63, 735, 220, 19, 24, 11, 19, 16, 24, 11, 18, 16, 22, 11, 18, 19, 19, 735, 1510, 24, 24, 19, 23, 68, 23, 16, 5734, 23, 18, 21, 23, 16, 19, 19, 65, 16, 18, 20, 64, 22, 19, 15, 18, 67, 23, 66, 15, 69, 17, 9500, 24, 64, 21, 66, 17, 69, 24, 69, 21, 24, 17, 18, 16, 18, 23, 21, 20, 23, 20, 20, 65, 17, 23, 18, 17, 68, 21, 24, 21, 66, 24, 63, 735, 198, 91, 1510, 48, 16451, 18, 13, 23, 12, 23129, 12, 5666, 12, 4505, 27325, 21, 9802, 6645, 43, 12, 15, 15, 15, 15, 21, 8404, 12, 15, 15, 15, 15, 21, 75021, 1644, 63, 735, 220, 16, 19, 11, 23, 18, 23, 11, 18, 16, 18, 11, 24, 23, 19, 735, 1510, 68, 19, 5734, 17, 20, 20, 17, 18, 19, 69, 19, 17, 65, 16, 23, 15, 16, 17, 69, 21, 67, 21, 92520, 20, 12849, 21, 16, 20, 65, 20, 22, 19, 22, 24, 65, 22, 68, 15, 64, 15, 17, 18, 18, 66, 19, 22, 19, 18, 21, 64, 18, 17, 16, 27063, 732, 18, 69, 63, 735, 271, 43121, 11, 1301, 494, 279, 50319, 6881, 440, 1510, 15449, 4691, 70, 1644, 21614, 6971, 63, 318, 528, 4590, 71590, 23185, 25, 50319, 220, 19, 283, 9714, 220, 15, 12, 17, 15, 11, 198, 20, 17, 15, 74451, 11, 50319, 220, 20, 283, 9714, 220, 17, 15, 12, 19, 16, 11, 220, 20, 18, 15, 74451, 11, 321, 791, 6000, 279, 4346, 3294, 1590, 271, 12, 2972, 49, 17978, 11312, 64700, 1510, 529, 77, 52954, 2615, 1630, 63, 321, 1510, 529, 77, 7980, 2615, 1630, 63, 513, 1167, 21, 9802, 318, 27392, 303, 1396, 6000, 314, 50319, 220, 21, 25, 9714, 220, 19, 16, 12, 19, 22, 1174, 198]}, {"name": "english", "ids": [760, 324, 55965, 506, 279, 809, 314, 279, 17631, 1387, 12615, 995, 978, 26606, 364, 4409, 34227, 1578, 11, 694, 279, 6092, 1990, 3106, 4184, 680, 310, 424, 1396, 7548, 13, 35254, 25624, 852, 995, 1801, 279, 5728, 11, 321, 18022, 995, 3397, 5158, 424, 1441, 13, 1049, 557, 4777, 15513, 421, 383, 7548, 12967, 11, 948, 279, 49813, 995, 2047, 321, 279, 7397, 2981, 995, 978, 304, 18988, 4101, 11, 799, 1637, 1000, 4088, 279, 2984, 314, 279, 9506, 21268, 11, 14617, 279, 10679, 5817, 440, 279, 1328, 421, 17750, 4661, 279, 5373, 314, 279, 72103, 4998, 11, 24562, 279, 7493, 321, 28279, 7123, 11, 321, 1353, 13, 271, 2381, 1379, 314, 421, 854, 279, 1637, 995, 978, 36306, 63918, 11, 852, 10298, 279, 509, 429, 75, 702, 321, 11324, 1292, 4221, 279, 9856, 13, 3095, 1024, 29528, 5326, 6348, 680, 11, 279, 1328, 5642, 310, 1024, 63634, 84907, 11, 852, 557, 60163, 321, 6306, 279, 4220, 2478, 264, 11939, 314, 264, 13542, 1603, 12967, 13, 1216, 1018, 748, 11, 52039, 11, 364, 279, 1118, 2250, 5381, 13, 1848, 279, 11400, 7548, 264, 13071, 3571, 303, 3988, 494, 279, 9422, 11, 321, 551, 1669, 5387, 506, 279, 1844, 314, 279, 20618, 9799, 264, 2526, 48345, 14882, 4226, 1141, 1560, 3162, 279, 15484, 22317, 440, 264, 20496, 492, 564, 321, 18022, 70524, 506, 279, 32647, 13, 271, 1465, 1467, 1092, 780, 36919, 995, 3125, 1384, 310, 635, 11, 864, 557, 4161, 21100, 13, 1216, 5947, 1441, 279, 854, 11, 279, 4944, 314, 279, 9662, 11, 279, 803, 23235, 383, 279, 55618, 11, 321, 279, 2029, 421, 551, 1362, 1436, 1132, 799, 1637, 383, 9249, 13, 4844, 551, 14428, 1142, 1441, 279, 7493, 321, 28279, 7123, 11, 26164, 279, 21268, 440, 780, 26338, 12854, 9990, 11, 321, 3125, 279, 72103, 7110, 11, 852, 7712, 734, 19083, 279, 2199, 36516, 7803, 13, 561, 14882, 557, 15348, 290, 303, 449, 6258, 2843, 13, 11116, 10390, 692, 11, 264, 20922, 2812, 22635, 852, 995, 978, 48345, 7249, 494, 279, 28053, 11, 995, 10548, 780, 31111, 948, 279, 28774, 67490, 3808, 321, 995, 978, 31117, 440, 780, 7303, 13, 271, 760, 3255, 557, 303, 279, 2136, 5392, 279, 2614, 1936, 11, 321, 84907, 557, 46523, 539, 424, 11, 9712, 1521, 279, 9994, 8280, 1384, 303, 780, 2812, 13398, 13, 3437, 19013, 440, 1384, 557, 2414, 745, 13, 1848, 279, 4088, 1142, 3055, 279, 21268, 421, 11129, 11, 279, 72103, 7110, 995, 3125, 1384, 421, 279, 26606, 3000, 995, 6237, 13542, 279, 4220, 854, 13, 1049, 995, 6316, 11, 424, 995, 67474, 1141, 5215, 11, 321, 424, 995, 2725, 6681, 1092, 424, 557, 5617, 310, 635, 13, 561, 3000, 1362, 8543, 264, 14882, 3023, 494, 279, 22317, 11, 694, 424, 1362, 524, 5125, 421, 264, 14882, 557, 303, 11914, 13, 8020, 264, 1637, 1362, 635, 421, 11, 321, 1132, 413, 264, 1637, 557, 1017, 310, 1353, 13, 271, 15450, 303, 279, 6092, 6807, 2963, 883, 421, 5728, 1381, 430, 3384, 424, 995, 978, 264, 26802, 6365, 314, 8842, 11, 264, 43472, 46370, 2272, 279, 13001, 314, 12282, 13, 84907, 1503, 524, 1683, 424, 557, 3977, 314, 279, 2993, 13, 1216, 15028, 424, 5820, 1521, 17540, 13235, 279, 2235, 51624, 321, 1362, 524, 4847, 4300, 5402, 310, 2842, 2019, 680, 310, 279, 20618, 11, 321, 421, 424, 8308, 1521, 18022, 4674, 310, 381, 279, 799, 310, 809, 424, 13, 27934, 1159, 513, 1040, 421, 13, 2302, 680, 4119, 836, 7715, 11, 321, 1396, 748, 3428, 11, 1973, 5316, 8842, 424, 11, 781, 1423, 264, 491, 799, 13, 271, 1465, 369, 16570, 52616, 1381, 321, 4138, 430, 449, 22839, 383, 279, 37126, 9662, 8520, 421, 488, 628, 1436, 494, 279, 1844, 314, 279, 324, 55965, 383, 264, 2708, 1834, 25, 1330, 7493, 4022, 73090, 10932, 303, 6761, 383, 279, 32946, 11, 1754, 799, 14936, 539, 24296, 421, 1830, 46219, 11, 9039, 321, 24052, 9361, 310, 264, 2444, 2956, 303, 2361, 3046, 13, 561, 5757, 513, 1546, 1603, 13, 2302, 2213, 4409, 4156, 13, 1216, 1990, 5601, 680, 310, 279, 20618, 383, 79833, 13382, 948, 551, 369, 2046, 11, 73797, 279, 7123, 11, 321, 5686, 13, 271, 25337, 1384, 3069, 11, 321, 551, 668, 3184, 488, 264, 3255, 421, 682, 4161, 310, 635, 440, 324, 21204, 19154, 13, 733, 780, 2018, 1007, 383, 279, 9662, 8520, 11, 264, 16070, 1785, 59144, 264, 6103, 9704, 303, 279, 9039, 314, 264, 94496, 383, 799, 314, 279, 73090, 506, 279, 2944, 809, 314, 279, 1287, 13, 561, 9704, 557, 2526, 11, 1575, 4613, 279, 13001, 11, 321, 279, 3061, 20364, 424, 430, 3238, 10304, 13, 1049, 1000, 599, 978, 6749, 506, 2261, 279, 1727, 13145, 3814, 11, 4590, 5381, 2843, 13, 357, 51715, 383, 279, 13017, 14882, 11, 11946, 279, 62970, 383, 2361, 2531, 11, 13293, 421, 279, 164840, 6502, 33172, 2086, 11, 264, 36334, 16365, 5020, 1172, 279, 13087, 2696, 11, 321, 4439, 279, 10390, 692, 310, 2842, 13, 3095, 781, 8660, 279, 94496, 781, 1669, 264, 17504, 421, 557, 264, 2342, 2756, 494, 20738, 13, 997, 46333, 424, 2695, 264, 6353, 13, 997, 46333, 279, 94496, 1238, 279, 17504, 4490, 1000, 599, 2695, 264, 2182, 321, 279, 3240, 314, 264, 2526, 3626, 13, 271, 760, 51715, 557, 524, 26802, 261, 1056, 279, 3061, 13, 2838, 995, 4777, 978, 1017, 11, 3160, 3213, 310, 6512, 11, 321, 21520, 3213, 310, 2466, 364, 279, 14882, 310, 2842, 13, 561, 795, 995, 6625, 279, 1788, 3377, 26, 424, 995, 1066, 524, 6625, 424, 52039, 3213, 364, 5316, 310, 2373, 13, 84907, 15028, 883, 421, 3428, 13, 6983, 1785, 551, 682, 6237, 383, 682, 978, 1546, 1603, 506, 28043, 279, 2433, 1141, 24990, 3481, 310, 708, 4808, 11, 321, 1691, 2599, 1603, 506, 279, 2433, 18022, 3481, 13, 561, 12558, 1881, 279, 1330, 369, 1332, 1208, 1990, 7005, 836, 2426, 11, 524, 539, 3604, 279, 5484, 579, 2531, 2577, 1056, 279, 5484, 11, 694, 539, 59291, 948, 279, 2531, 4924, 682, 28599, 5324, 13, 271, 4272, 383, 79833, 13382, 11, 948, 551, 628, 11]}, {"name": "multilingual", "ids": [2495, 450, 71234, 176217, 48, 16451, 18, 13, 23, 12, 23129, 12, 5666, 9251, 2349, 1074, 71, 34098, 1509, 6616, 43765, 16451, 43765, 16451, 18, 13, 23, 12, 23129, 12, 5666, 31230, 6993, 16927, 224986, 6464, 197102, 13, 198, 31538, 5810, 4227, 198214, 11, 19611, 48473, 189164, 12, 4312, 598, 11, 6407, 91957, 161691, 15008, 8162, 182720, 13, 8963, 77477, 294, 11, 205397, 5927, 11, 229411, 198, 33, 20143, 1962, 192900, 5280, 188474, 31926, 1962, 46472, 6481, 70, 779, 51592, 13, 83430, 82, 2652, 41759, 49188, 188474, 6464, 13, 271, 550, 41201, 56104, 5810, 1477, 30, 271, 46445, 17104, 1477, 6993, 39124, 37521, 224986, 29469, 12, 4716, 82, 18184, 37525, 13, 17530, 9372, 5810, 74905, 220, 65121, 34913, 90283, 287, 13, 271, 12, 2972, 29140, 154563, 92503, 64700, 12981, 56104, 2659, 63538, 303, 16927, 211571, 12485, 200366, 13, 220, 21, 15, 56212, 453, 47584, 27923, 12223, 198, 220, 195937, 11, 10396, 3717, 72621, 63615, 13, 198, 12, 2972, 43, 13051, 188474, 57491, 64700, 12981, 56104, 1477, 6993, 54854, 2421, 11, 557, 3717, 5521, 839, 267, 318, 164509, 4227, 157536, 11, 5927, 12030, 12485, 415, 94388, 5280, 198, 220, 220, 18, 17, 42, 56212, 553, 271, 20353, 29, 198, 10948, 1721, 337, 29028, 28392, 25, 220497, 220, 20, 15, 22, 15, 318, 16, 17, 18295, 681, 89540, 220, 20, 220, 22, 21, 15, 15, 11, 220, 21, 19, 18295, 21360, 510, 337, 1721, 337, 29, 55771, 25, 27255, 220, 24, 15, 22, 15, 59781, 318, 16, 21, 18295, 681, 89540, 220, 24, 220, 18, 24, 15, 15, 55, 11, 220, 19, 22, 18295, 21360, 510, 337, 1419, 371, 29, 198, 10948, 1721, 1251, 29, 271, 91, 178297, 735, 238791, 92503, 735, 436, 13051, 188474, 57491, 735, 198, 91, 12091, 735, 12091, 25, 735, 12091, 25, 735, 198, 91, 2972, 48, 17, 62, 15, 332, 735, 220, 24, 19, 56212, 2600, 735, 220, 17, 11, 21, 20, 15, 56212, 2600, 735, 198, 91, 2972, 59865, 17, 6645, 50, 332, 735, 220, 22, 24, 56212, 2600, 735, 220, 17, 11, 15, 24, 15, 56212, 2600, 735, 198, 91, 2972, 59865, 18, 62, 5955, 50, 332, 735, 220, 21, 17, 56212, 2600, 735, 220, 16, 11, 22, 20, 15, 56212, 2600, 735, 198, 91, 2972, 59865, 18, 1063, 332, 735, 220, 20, 18, 56212, 2600, 735, 220, 16, 11, 21, 17, 15, 56212, 2600, 735, 198, 91, 2972, 34120, 332, 735, 220, 20, 20, 56212, 2600, 735, 220, 17, 11, 16, 23, 15, 56212, 2600, 735, 271, 510, 1251, 1721, 1251, 29, 271, 91, 178297, 735, 238791, 92503, 735, 436, 13051, 188474, 57491, 735, 198, 91, 12091, 735, 12091, 25, 735, 12091, 25, 735, 198, 91, 2972, 48, 17, 62, 15, 332, 735, 220, 21, 15, 56212, 2600, 735, 220, 16, 11, 16, 21, 15, 56212, 2600, 735, 198, 91, 2972, 59865, 17, 6645, 50, 332, 735, 220, 20, 17, 56212, 2600, 735, 220, 16, 11, 16, 16, 15, 56212, 2600, 735, 198, 91, 2972, 34120, 332, 735, 220, 19, 19, 56212, 2600, 735, 220, 16, 11, 19, 17, 15, 56212, 2600, 735, 271, 510, 1251, 1419, 371, 29, 198, 510, 1938, 29, 271, 45, 28392, 25, 1167, 17, 62, 15, 5280, 7943, 220, 15, 13, 16, 13, 18, 21, 11, 2659, 37402, 9625, 22126, 5280, 220, 15, 13, 16, 13, 17, 21, 318, 19, 42, 92503, 11, 220, 18, 17, 42, 17058, 12389, 553, 8148, 162946, 6231, 198, 8320, 15212, 62663, 303, 498, 82835, 50, 20668, 9251, 13715, 14, 82835, 50, 20668, 2, 19755, 32848, 38012, 553, 49584, 189202, 5280, 17966, 18588, 1354, 5810, 195937, 25, 49584, 198, 5182, 55, 220, 18, 15, 24, 15, 318, 17, 19, 18295, 8, 49214, 74905, 220, 16, 15, 15, 12, 16, 19, 15, 56212, 453, 47584, 27923, 181737, 13, 84697, 882, 1798, 632, 271, 2495, 450, 19092, 216660, 176217, 48, 16451, 18, 13, 23, 12, 23129, 12, 5666, 9251, 2349, 1074, 71, 34098, 1509, 6616, 43765, 16451, 43765, 16451, 18, 13, 23, 12, 23129, 12, 5666, 31230, 1668, 632, 6464, 4472, 13, 351, 16811, 198, 354, 6453, 79653, 293, 6, 5681, 11, 24111, 12506, 11, 7528, 59006, 293, 6, 221400, 632, 208450, 13, 7305, 243195, 11, 85388, 3717, 1970, 11, 12647, 909, 5167, 1778, 198, 20812, 399, 785, 9462, 24449, 8098, 1778, 24449, 12636, 401, 1970, 13, 215329, 808, 16343, 647, 14487, 6464, 13, 271, 550, 176591, 1725, 796, 183208, 907, 271, 63611, 324, 46450, 2305, 10529, 322, 7639, 1668, 24739, 6464, 401, 48852, 47021, 16766, 13, 1188, 3817, 7618, 3656, 47288, 220, 65121, 293, 21372, 3724, 13, 271, 12, 2972, 25933, 35768, 3424, 190724, 170592, 1147, 183208, 3656, 175873, 1147, 86333, 274, 78502, 11720, 6652, 5992, 71388, 9915, 13, 220, 21, 15, 10885, 1298, 184286, 11, 272, 16811, 5346, 72931, 1649, 14487, 29579, 13, 198, 12, 2972, 66621, 14487, 9640, 170592, 1147, 183208, 3656, 175873, 3687, 15110, 1323, 3633, 1649, 8740, 57519, 9825, 318, 3266, 632, 2128, 11, 3717, 1970, 5724, 632, 187282, 401, 9915, 401, 220, 18, 17, 42, 10885, 553, 271, 20353, 29, 198, 10948, 1721, 337, 29028, 28392, 535, 220497, 220, 20, 15, 22, 15, 318, 16, 17, 5805, 681, 89540, 220, 20, 220, 22, 21, 15, 15, 11, 220, 21, 19, 5805, 401, 21360, 510, 337, 1721, 337, 29, 55771, 535, 27255, 220, 24, 15, 22, 15, 59781, 318, 16, 21, 5805, 681, 89540, 220, 24, 220, 18, 24, 15, 15, 55, 11, 220, 19, 22, 5805, 401, 21360, 510, 337, 1419, 371, 29, 198, 10948, 1721, 1251, 29, 271, 91, 196281, 735, 240637, 3424, 190724, 735, 36983, 14487, 9640, 735, 198, 91, 12091, 735, 12091, 25, 735, 12091, 25, 735, 198, 91, 2972, 48, 17, 62, 15, 332, 735, 220, 24, 19, 10885, 2600, 735, 220, 17, 11, 21, 20, 15, 10885, 2600, 735, 198, 91, 2972, 59865, 17, 6645, 50, 332, 735, 220, 22, 24, 10885, 2600, 735, 220, 17, 11, 15, 24, 15, 10885, 2600, 735, 198, 91, 2972, 59865, 18, 62, 5955, 50, 332, 735, 220, 21, 17, 10885, 2600, 735, 220, 16, 11, 22, 20, 15]}, {"name": "agent", "ids": [248045, 8678, 198, 2523, 513, 264, 10505, 8057, 3133, 303, 279, 1156, 579, 12182, 13, 1394, 628, 1562, 7141, 310, 1301, 3425, 11, 2624, 279, 1970, 321, 1542, 10966, 13, 13262, 4203, 2526, 321, 10033, 1092, 488, 1467, 13, 248046, 198, 248045, 846, 198, 760, 1228, 1510, 1877, 20394, 24647, 63, 14107, 2394, 13278, 579, 10562, 13, 2885, 488, 1423, 680, 3069, 321, 4884, 424, 30, 248046, 198, 248045, 74455, 198, 248068, 198, 40, 1220, 1542, 279, 20738, 1228, 1118, 310, 1436, 279, 4880, 1412, 1876, 11, 1179, 1353, 506, 279, 709, 424, 6813, 13, 198, 248069, 271, 248058, 198, 4754, 591, 763, 328, 5917, 10494, 487, 328, 15889, 763, 5046, 5454, 763, 328, 12305, 471, 76, 26864, 6813, 12333, 2914, 16799, 6971, 471, 74, 1228, 20394, 24647, 471, 80, 29958, 198, 248059, 248046, 198, 248045, 846, 198, 248066, 198, 37, 21641, 498, 16, 15, 15, 4, 60, 198, 2960, 8442, 32009, 48956, 34589, 410, 198, 7443, 40015, 62, 1228, 20394, 24647, 42390, 40015, 271, 262, 687, 1228, 20394, 24647, 4406, 198, 285, 1992, 4563, 24647, 437, 24, 15, 82, 871, 606, 220, 24, 15, 198, 285, 1992, 4563, 24647, 437, 20, 76, 871, 606, 220, 18, 15, 15, 198, 29, 950, 1992, 4563, 24647, 437, 16, 71, 18, 15, 76, 871, 606, 220, 20, 19, 15, 15, 198, 36, 950, 14953, 25, 8060, 7836, 25, 359, 16, 71, 18, 15, 76, 6, 271, 23095, 12333, 2914, 16799, 6971, 25, 16, 19, 25, 14953, 198, 16, 4490, 11, 220, 16, 16, 48600, 2140, 303, 220, 15, 13, 17, 16, 82, 198, 248067, 248046, 198, 248045, 74455, 198, 248068, 198, 10570, 7896, 944, 694, 264, 10539, 869, 14107, 13, 6558, 728, 1301, 279, 6518, 13, 198, 248069, 271, 248058, 198, 4754, 591, 763, 328, 851, 2378, 487, 328, 15889, 763, 5046, 2267, 763, 328, 3431, 34093, 16799, 6971, 487, 328, 2388, 6325, 763, 220, 16, 11, 328, 400, 6325, 763, 220, 19, 15, 3307, 198, 248059, 248046, 198, 248045, 846, 198, 248066, 198, 464, 311, 271, 87662, 283, 5046, 82, 763, 220, 16, 11, 328, 76, 763, 220, 21, 15, 11, 328, 71, 763, 220, 18, 21, 15, 15, 11, 328, 67, 763, 220, 23, 21, 19, 15, 15, 92, 198, 27360, 283, 311, 18351, 2515, 83501, 11189, 67, 10, 2299, 58, 3445, 15518, 2387, 3, 871, 1358, 727, 4563, 24647, 7008, 25, 592, 8, 1411, 514, 25, 198, 262, 4071, 14048, 264, 7836, 1040, 359, 24, 15, 82, 504, 359, 20, 76, 6, 466, 359, 16, 71, 18, 15, 76, 6, 1083, 6283, 71483, 198, 262, 1414, 283, 1414, 16674, 973, 14344, 363, 198, 262, 413, 524, 1414, 25, 198, 285, 4673, 14953, 437, 3092, 7836, 871, 198, 262, 295, 283, 696, 32379, 10749, 7008, 8, 198, 262, 413, 295, 369, 2168, 25, 198, 285, 4673, 14953, 925, 1, 11469, 7836, 25, 313, 1272, 0, 81, 54084, 198, 262, 460, 514, 1212, 5982, 7, 16, 578, 348, 696, 1797, 14059, 11722, 5982, 7, 17, 7025, 198, 248067, 248046, 198, 248045, 74455, 198, 248068, 198, 760, 19567, 369, 74483, 506, 2107, 10029, 321, 8798, 6681, 799, 1324, 58533, 6505, 11, 748, 328, 16, 71, 18, 15, 76, 1, 628, 2496, 2353, 13, 561, 4479, 889, 20021, 10539, 2663, 13, 6558, 728, 1716, 15865, 3712, 310, 1436, 1092, 279, 10562, 5324, 13, 198, 248069, 271, 248058, 198, 4754, 591, 763, 328, 5917, 10494, 487, 328, 15889, 763, 5046, 5454, 763, 328, 12513, 1433, 471, 79, 471, 77, 220, 16, 1137, 2212, 34093, 16799, 6971, 735, 1901, 471, 19, 15, 29958, 198, 248059, 248046, 198, 248045, 846, 198, 248066, 198, 16899, 220, 19, 68, 16, 65, 24, 66, 17, 69, 22, 64, 15, 67, 318, 50346, 8602, 359, 53380, 12, 25543, 837, 198, 262, 59176, 1386, 7836, 21621, 271, 67677, 32379, 283, 311, 18351, 2515, 1, 11189, 67, 10, 2299, 58, 3445, 15518, 2387, 871, 198, 94081, 32379, 283, 311, 18351, 2515, 83501, 11189, 67, 10, 2299, 58, 3445, 15518, 2387, 3, 871, 198, 18611, 687, 4563, 24647, 7008, 25, 592, 8, 1411, 514, 25, 198, 12, 262, 2702, 283, 220, 15, 198, 12, 262, 1097, 283, 220, 15, 198, 12, 262, 364, 295, 303, 696, 32379, 2571, 1947, 7008, 1590, 198, 12, 285, 413, 295, 4801, 363, 931, 1097, 25, 198, 12, 309, 4673, 14953, 925, 1, 11469, 7836, 25, 313, 1272, 0, 81, 54084, 198, 12, 285, 2702, 1373, 514, 1212, 5982, 7, 16, 578, 348, 696, 1797, 14059, 11722, 5982, 7, 17, 7025, 198, 12, 285, 1097, 283, 295, 4911, 363, 198, 12, 262, 413, 1097, 931, 2343, 7008, 1590, 198, 12, 285, 4673, 14953, 925, 1, 11469, 7836, 25, 313, 1272, 0, 81, 54084, 198, 12, 262, 460, 2702, 198, 10, 262, 295, 283, 696, 32379, 10749, 7008, 8, 198, 10, 262, 413, 295, 369, 2168, 25, 198, 10, 285, 4673, 14953, 925, 1, 11469, 7836, 25, 313, 1272, 0, 81, 54084, 198, 10, 262, 460, 514, 1212, 5982, 7, 16, 578, 348, 696, 1797, 14059, 11722, 5982, 7, 17, 7025, 198, 248067, 248046, 198, 248045, 74455, 198, 760, 20635, 10562, 12215, 279, 6136, 888, 660, 1324, 58533, 13139, 440, 264, 3074, 74483, 2353, 11, 748, 1132, 799, 4821, 791, 886, 369, 11330, 1381, 13, 353, 668, 14519, 279, 6136, 11, 864, 1048, 57409, 25053, 1881, 279, 5306, 13, 271, 248058, 198, 4754, 591, 763, 328, 3468, 2378, 487, 328, 15889, 763, 5046, 2267, 763, 328, 3431, 34093, 16799, 6971, 487, 328, 787, 763, 8728, 32379, 283, 311, 18351, 2515, 2037, 12883, 3312, 67, 10, 2299, 58, 3445, 15518, 2387, 56749, 871, 487, 328, 902, 763, 8728, 32379, 283, 311, 18351, 2515, 2037, 7, 3312, 67, 10, 2299, 58, 3445, 15518, 2387, 60592, 29958, 198, 248059, 248046, 198, 248045, 846, 198, 248066, 198, 3793, 25, 220, 16, 13329, 303, 2212, 34093, 16799, 6971, 198, 248067, 248046, 198, 248045, 74455, 198, 248058, 198, 4754, 591, 763, 328, 3468, 2378, 487, 328, 15889, 763, 5046, 2267, 763, 328, 3431, 34093, 16799, 6971, 487, 328, 787, 763, 328, 262, 295, 283, 696]}]} \ No newline at end of file +{"version": "v1", "seg_tokens": 1000, "segments": [{"name": "code", "ids": [12237, 85286, 54802, 19913, 343, 18, 4108, 6396, 471, 874, 50285, 980, 11, 874, 50285, 1644, 2195, 88, 13, 271, 34447, 1521, 279, 1510, 13985, 1644, 63, 5187, 1828, 6129, 4021, 1724, 411, 35426, 25, 1141, 1510, 21563, 2581, 42750, 1954, 900, 63, 198, 8746, 682, 874, 4314, 364, 913, 220, 19, 17, 27718, 48, 17, 62, 15, 63, 681, 864, 369, 24733, 1056, 279, 6522, 13, 2844, 369, 4924, 264, 387, 15, 782, 21, 198, 64919, 471, 279, 3443, 411, 4560, 369, 26208, 364, 369, 8047, 314, 279, 5129, 5224, 286, 13, 271, 1919, 369, 279, 5604, 7873, 364, 1510, 3431, 53683, 20123, 14, 63, 318, 77646, 16137, 16, 20, 25, 328, 21563, 19913, 343, 18, 6396, 318, 76, 2117, 11, 198, 2083, 50285, 980, 23285, 61406, 1049, 68434, 279, 4108, 1132, 26, 15167, 795, 369, 2496, 1301, 13, 198, 12237, 198, 1445, 1259, 20409, 548, 1120, 31144, 271, 464, 795, 8588, 198, 464, 51372, 198, 464, 1968, 198, 1445, 18904, 1120, 5586, 271, 21563, 19913, 47540, 283, 220, 15, 87, 19, 21, 20, 20, 19, 22, 19, 22, 220, 653, 328, 21563, 19913, 1, 2545, 12692, 1068, 271, 2, 50285, 980, 1756, 2663, 13, 220, 19, 17, 283, 1167, 17, 62, 15, 11, 279, 34100, 48, 12, 7146, 46, 69764, 17830, 517, 10845, 11, 26334, 494, 50285, 1644, 2195, 88, 430, 314, 220, 15, 13, 16, 24, 13, 198, 21563, 2581, 24973, 25, 6249, 18078, 11, 592, 60, 283, 313, 198, 262, 220, 15, 25, 328, 37, 18, 17, 487, 220, 16, 25, 328, 37, 16, 21, 487, 220, 17, 25, 328, 48, 19, 62, 15, 487, 220, 18, 25, 328, 48, 19, 62, 16, 487, 220, 19, 25, 328, 48, 19, 62, 17, 487, 220, 20, 25, 328, 48, 19, 62, 18, 487, 220, 21, 25, 328, 48, 20, 62, 15, 487, 220, 22, 25, 328, 48, 20, 62, 16, 487, 198, 262, 220, 23, 25, 328, 48, 23, 62, 15, 487, 220, 24, 25, 328, 48, 23, 62, 16, 487, 220, 16, 15, 25, 328, 48, 17, 9802, 487, 220, 16, 16, 25, 328, 48, 18, 9802, 487, 220, 16, 17, 25, 328, 48, 19, 9802, 487, 220, 16, 18, 25, 328, 48, 20, 9802, 487, 220, 16, 19, 25, 328, 48, 21, 9802, 487, 198, 262, 220, 16, 20, 25, 328, 48, 23, 9802, 487, 220, 16, 21, 25, 328, 59865, 17, 62, 5955, 50, 487, 220, 16, 22, 25, 328, 59865, 17, 6645, 50, 487, 220, 16, 23, 25, 328, 59865, 18, 62, 5955, 50, 487, 220, 16, 24, 25, 328, 59865, 16, 1063, 487, 220, 17, 15, 25, 328, 59865, 19, 1548, 43, 487, 198, 262, 220, 17, 16, 25, 328, 59865, 18, 1063, 487, 220, 17, 17, 25, 328, 59865, 17, 1063, 487, 220, 17, 18, 25, 328, 59865, 19, 6645, 50, 487, 220, 17, 19, 25, 328, 40, 23, 487, 220, 17, 20, 25, 328, 40, 16, 21, 487, 220, 17, 21, 25, 328, 40, 18, 17, 487, 220, 17, 22, 25, 328, 40, 21, 19, 487, 198, 262, 220, 17, 23, 25, 328, 37, 21, 19, 487, 220, 17, 24, 25, 328, 59865, 16, 1203, 487, 220, 18, 15, 25, 328, 19278, 16, 21, 487, 220, 18, 16, 25, 328, 48, 19, 62, 15, 62, 19, 62, 19, 487, 220, 18, 17, 25, 328, 48, 19, 62, 15, 62, 19, 62, 23, 487, 220, 18, 18, 25, 328, 48, 19, 62, 15, 62, 23, 62, 23, 487, 198, 262, 220, 18, 19, 25, 328, 51, 48, 16, 62, 15, 487, 220, 18, 20, 25, 328, 51, 48, 17, 62, 15, 487, 220, 18, 21, 25, 328, 59865, 19, 1548, 43, 62, 19, 62, 19, 487, 220, 18, 22, 25, 328, 59865, 19, 1548, 43, 62, 19, 62, 23, 487, 220, 18, 23, 25, 328, 59865, 19, 1548, 43, 62, 23, 62, 23, 487, 198, 262, 220, 18, 24, 25, 328, 28463, 11361, 19, 487, 220, 19, 15, 25, 328, 35125, 11361, 19, 487, 220, 19, 16, 25, 328, 48, 19, 62, 15, 62, 23, 62, 23, 487, 220, 19, 17, 25, 328, 48, 17, 62, 15, 487, 198, 92, 271, 2, 54802, 19913, 10836, 869, 913, 13746, 198, 21563, 19913, 57818, 283, 313, 198, 262, 220, 15, 25, 3376, 84, 23, 487, 220, 16, 681, 220, 16, 25, 3376, 72, 23, 487, 220, 16, 681, 220, 17, 25, 3376, 84, 16, 21, 487, 220, 17, 681, 220, 18, 25, 3376, 72, 16, 21, 487, 220, 17, 681, 220, 19, 25, 3376, 84, 18, 17, 487, 220, 19, 681, 220, 20, 25, 3376, 72, 18, 17, 487, 220, 19, 681, 198, 262, 220, 21, 25, 3376, 69, 18, 17, 487, 220, 19, 681, 220, 22, 25, 3376, 2554, 487, 220, 16, 681, 220, 23, 25, 3376, 889, 487, 2168, 681, 220, 24, 25, 3376, 1595, 487, 2168, 681, 198, 262, 220, 16, 15, 25, 3376, 84, 21, 19, 487, 220, 23, 681, 220, 16, 16, 25, 3376, 72, 21, 19, 487, 220, 23, 681, 220, 16, 17, 25, 3376, 69, 21, 19, 487, 220, 23, 681, 198, 92, 271, 2, 50285, 980, 2424, 16547, 364, 279, 3107, 56373, 411, 1558, 5533, 25, 318, 4425, 5255, 11, 5639, 791, 2424, 8, 198, 2, 1061, 369, 1092, 3520, 264, 791, 2308, 3677, 4763, 1698, 1716, 470, 11, 321, 424, 369, 279, 4954, 279, 61369, 4191, 13, 198, 38637, 57038, 83072, 25, 6249, 16793, 11, 13992, 18078, 11, 514, 4891, 283, 313, 198, 262, 328, 37, 18, 17, 763, 318, 16, 11, 220, 19, 681, 328, 37, 16, 21, 763, 318, 16, 11, 220, 17, 681, 328, 19278, 16, 21, 763, 318, 16, 11, 220, 17, 681, 328, 37, 21, 19, 763, 318, 16, 11, 220, 23, 681, 198, 262, 328, 48, 19, 62, 15, 763, 318, 18, 17, 11, 220, 16, 23, 681, 328, 48, 19, 62, 16, 763, 318, 18, 17, 11, 220, 17, 15, 681, 328, 48, 20, 62, 15, 763, 318, 18, 17, 11, 220, 17, 17, 681, 328, 48]}, {"name": "docs", "ids": [12, 2972, 48213, 318, 18232, 1354, 188912, 6326, 321, 24167, 6738, 6311, 382, 11, 279, 81787, 11255, 3231, 13914, 11, 57174, 11, 5904, 11312, 11, 2468, 1901, 11, 279, 380, 4100, 198, 220, 9665, 6000, 11, 279, 81726, 6297, 318, 1445, 220, 21, 19, 42, 25, 1132, 1141, 1379, 27864, 919, 11, 279, 2645, 22327, 494, 21360, 681, 321, 449, 2972, 71376, 6297, 332, 421, 39248, 279, 2645, 314, 18588, 1354, 440, 279, 1379, 66203, 11312, 318, 275, 37350, 2491, 310, 198, 220, 279, 10125, 1345, 488, 6040, 553, 198, 12, 2972, 32784, 64700, 660, 220, 17, 19, 11, 20, 22, 21, 11312, 11, 46120, 13, 561, 13540, 55288, 279, 11312, 421, 513, 524, 383, 279, 21966, 2972, 258, 1925, 93868, 506, 279, 1788, 854, 198, 220, 430, 279, 21966, 4138, 383, 279, 19954, 5981, 318, 7839, 55, 12, 20, 16, 17, 593, 11807, 55, 17, 61369, 11, 50285, 980, 579, 364, 279, 585, 71196, 1720, 553, 198, 12, 2972, 1178, 35, 64700, 279, 220, 17, 23, 13, 23, 18295, 307, 12, 1466, 1898, 11, 264, 2342, 6761, 791, 3817, 11, 1301, 632, 7333, 290, 3162, 279, 9787, 6297, 27718, 312, 675, 83290, 2050, 7561, 279, 198, 220, 1581, 11, 1801, 364, 35160, 82, 26, 383, 264, 88129, 12977, 1510, 312, 675, 83290, 17292, 63, 13205, 279, 1898, 303, 21360, 11, 653, 21, 15, 20, 553, 357, 1151, 5072, 21059, 46438, 948, 2050, 15339, 6420, 6103, 318, 7744, 220, 16, 20, 42, 6761, 2600, 8, 694, 2496, 30147, 539, 4924, 318, 10957, 220, 15, 13, 16, 13, 19, 18, 13, 16, 26, 1510, 6457, 4337, 1053, 841, 28066, 1203, 17092, 28, 16, 63, 12077, 1142, 279, 220, 15, 13, 16, 13, 19, 17, 3289, 310, 1510, 312, 675, 83290, 68463, 63, 1590, 279, 3498, 369, 82054, 364, 23683, 7575, 11, 264, 69726, 35160, 31352, 680, 506, 220, 16, 15, 12, 16, 19, 42, 6761, 2600, 440, 2050, 579, 220, 16, 21, 15339, 303, 10650, 11, 321, 383, 799, 318, 44780, 11, 220, 22, 22, 13, 23, 42, 33686, 9640, 8, 68463, 1301, 220, 17, 13, 16, 42, 6761, 2600, 2272, 2050, 579, 220, 16, 19, 13, 19, 42, 28133, 17, 15, 19, 15, 681, 1345, 383, 264, 5342, 3618, 440, 264, 6103, 23098, 999, 68463, 65552, 49219, 28133, 16, 24, 23, 19, 553, 561, 4560, 15339, 279, 1898, 198, 220, 303, 279, 3443, 279, 54802, 19913, 682, 424, 25, 35494, 19, 1548, 43, 318, 1719, 1581, 1898, 681, 1167, 19, 62, 15, 11, 1167, 20, 62, 15, 11, 1167, 20, 62, 16, 11, 1167, 23, 62, 15, 11, 32439, 23, 318, 36, 19, 44, 18, 440, 264, 5281, 8, 466, 48804, 16, 21, 13, 198, 220, 2088, 38012, 383, 220, 19, 11, 15, 15, 15, 4055, 6761, 2272, 279, 28336, 579, 1765, 48804, 16, 21, 1898, 11, 279, 2977, 791, 18950, 1412, 369, 1167, 23, 62, 15, 220, 15, 13, 20, 18, 12997, 32439, 23, 220, 17, 13, 21, 19, 12997, 198, 220, 1167, 20, 62, 16, 220, 18, 13, 22, 23, 12997, 1167, 20, 62, 15, 220, 19, 13, 17, 20, 12997, 35494, 19, 1548, 43, 220, 22, 13, 21, 15, 12997, 1167, 19, 62, 16, 220, 22, 13, 23, 15, 12997, 1167, 19, 62, 15, 220, 23, 13, 20, 20, 4, 318, 64, 1167, 23, 62, 15, 1898, 369, 220, 20, 19, 18295, 11, 449, 35494, 19, 1548, 43, 799, 220, 17, 23, 13, 23, 18295, 553, 1848, 279, 1167, 17, 62, 15, 1558, 11, 220, 22, 15, 15, 10756, 14751, 1931, 10885, 11, 279, 2977, 65427, 2272, 279, 48804, 16, 21, 1898, 369, 220, 15, 13, 15, 16, 16, 22, 318, 48, 20, 62, 15, 681, 220, 15, 13, 15, 16, 17, 16, 318, 48, 20, 62, 16, 681, 220, 15, 13, 15, 16, 17, 20, 198, 220, 318, 48, 23, 62, 15, 681, 220, 15, 13, 15, 16, 17, 23, 318, 11361, 23, 681, 220, 15, 13, 15, 16, 19, 15, 318, 48, 19, 62, 15, 8, 321, 220, 15, 13, 15, 16, 19, 21, 318, 59865, 19, 1548, 43, 681, 440, 279, 70665, 477, 2785, 220, 16, 4, 314, 48804, 16, 21, 579, 2892, 1560, 25, 866, 2222, 310, 279, 198, 220, 1898, 10480, 279, 220, 17, 14813, 1558, 539, 883, 220, 15, 13, 15, 16, 17, 11, 748, 279, 19251, 513, 2503, 310, 3184, 10431, 13, 18065, 1503, 524, 2915, 2361, 1898, 26, 198, 220, 424, 1132, 4203, 864, 54802, 19913, 279, 4560, 369, 2574, 13, 198, 12, 2972, 8068, 2820, 64700, 279, 1558, 579, 1765, 380, 4100, 6000, 61444, 685, 310, 220, 18, 10885, 26, 799, 1440, 888, 660, 220, 19, 23, 13224, 11988, 1070, 13, 220, 17, 13, 19, 12, 18, 13, 17, 198, 220, 10885, 791, 1440, 383, 5289, 13, 3095, 279, 9559, 42151, 279, 2193, 318, 1787, 49133, 11, 22513, 1414, 681, 2972, 39242, 18054, 332, 318, 8251, 198, 220, 220, 15, 13, 16, 13, 22, 8, 61444, 685, 310, 220, 20, 10885, 494, 279, 6575, 2880, 11, 694, 1132, 1332, 1141, 16384, 24693, 321, 2695, 1910, 424, 20589, 25, 198, 220, 1970, 49133, 220, 21, 12, 16, 16, 4, 10281, 11, 975, 1414, 33702, 13, 561, 61444, 513, 9770, 1040, 279, 380, 4100, 579, 11, 748, 279, 2468, 369, 279, 1788, 13, 198, 12, 2972, 34021, 12389, 332, 513, 14789, 303, 25903, 314, 685, 310, 220, 23, 11, 16, 24, 17, 10885, 27718, 312, 29554, 473, 3129, 74927, 220, 18, 17, 11, 22, 21, 23, 3278, 3309, 8, 440, 279, 11312, 198, 220, 70755, 310, 279, 21966, 888, 87567, 13, 8439, 4560, 220, 15, 13, 16, 13, 18, 24, 65, 28133, 20, 23, 18, 8, 279, 70755, 9761, 369, 28328, 303, 5639, 364, 279, 3641, 318, 64, 198, 220, 9575, 3641, 440, 10917, 74270, 5055, 16253, 9761, 15250, 321, 13205, 777, 6297, 15250, 8, 321, 1510, 312, 29554, 473, 3129, 63, 21220, 279, 198, 220, 7526, 11540, 421, 13205, 421, 9761, 2400, 13, 1049, 1132, 6486, 685, 9761, 15250, 1332, 220, 15, 13, 16, 13, 18, 24, 579, 5728]}, {"name": "english", "ids": [760, 324, 55965, 506, 279, 809, 314, 279, 17631, 1387, 12615, 995, 978, 26606, 364, 4409, 34227, 1578, 11, 694, 279, 6092, 1990, 3106, 4184, 680, 310, 424, 1396, 7548, 13, 35254, 25624, 852, 995, 1801, 279, 5728, 11, 321, 18022, 995, 3397, 5158, 424, 1441, 13, 1049, 557, 4777, 15513, 421, 383, 7548, 12967, 11, 948, 279, 49813, 995, 2047, 321, 279, 7397, 2981, 995, 978, 304, 18988, 4101, 11, 799, 1637, 1000, 4088, 279, 2984, 314, 279, 9506, 21268, 11, 14617, 279, 10679, 5817, 440, 279, 1328, 421, 17750, 4661, 279, 5373, 314, 279, 72103, 4998, 11, 24562, 279, 7493, 321, 28279, 7123, 11, 321, 1353, 13, 271, 2381, 1379, 314, 421, 854, 279, 1637, 995, 978, 36306, 63918, 11, 852, 10298, 279, 509, 429, 75, 702, 321, 11324, 1292, 4221, 279, 9856, 13, 3095, 1024, 29528, 5326, 6348, 680, 11, 279, 1328, 5642, 310, 1024, 63634, 84907, 11, 852, 557, 60163, 321, 6306, 279, 4220, 2478, 264, 11939, 314, 264, 13542, 1603, 12967, 13, 1216, 1018, 748, 11, 52039, 11, 364, 279, 1118, 2250, 5381, 13, 1848, 279, 11400, 7548, 264, 13071, 3571, 303, 3988, 494, 279, 9422, 11, 321, 551, 1669, 5387, 506, 279, 1844, 314, 279, 20618, 9799, 264, 2526, 48345, 14882, 4226, 1141, 1560, 3162, 279, 15484, 22317, 440, 264, 20496, 492, 564, 321, 18022, 70524, 506, 279, 32647, 13, 271, 1465, 1467, 1092, 780, 36919, 995, 3125, 1384, 310, 635, 11, 864, 557, 4161, 21100, 13, 1216, 5947, 1441, 279, 854, 11, 279, 4944, 314, 279, 9662, 11, 279, 803, 23235, 383, 279, 55618, 11, 321, 279, 2029, 421, 551, 1362, 1436, 1132, 799, 1637, 383, 9249, 13, 4844, 551, 14428, 1142, 1441, 279, 7493, 321, 28279, 7123, 11, 26164, 279, 21268, 440, 780, 26338, 12854, 9990, 11, 321, 3125, 279, 72103, 7110, 11, 852, 7712, 734, 19083, 279, 2199, 36516, 7803, 13, 561, 14882, 557, 15348, 290, 303, 449, 6258, 2843, 13, 11116, 10390, 692, 11, 264, 20922, 2812, 22635, 852, 995, 978, 48345, 7249, 494, 279, 28053, 11, 995, 10548, 780, 31111, 948, 279, 28774, 67490, 3808, 321, 995, 978, 31117, 440, 780, 7303, 13, 271, 760, 3255, 557, 303, 279, 2136, 5392, 279, 2614, 1936, 11, 321, 84907, 557, 46523, 539, 424, 11, 9712, 1521, 279, 9994, 8280, 1384, 303, 780, 2812, 13398, 13, 3437, 19013, 440, 1384, 557, 2414, 745, 13, 1848, 279, 4088, 1142, 3055, 279, 21268, 421, 11129, 11, 279, 72103, 7110, 995, 3125, 1384, 421, 279, 26606, 3000, 995, 6237, 13542, 279, 4220, 854, 13, 1049, 995, 6316, 11, 424, 995, 67474, 1141, 5215, 11, 321, 424, 995, 2725, 6681, 1092, 424, 557, 5617, 310, 635, 13, 561, 3000, 1362, 8543, 264, 14882, 3023, 494, 279, 22317, 11, 694, 424, 1362, 524, 5125, 421, 264, 14882, 557, 303, 11914, 13, 8020, 264, 1637, 1362, 635, 421, 11, 321, 1132, 413, 264, 1637, 557, 1017, 310, 1353, 13, 271, 15450, 303, 279, 6092, 6807, 2963, 883, 421, 5728, 1381, 430, 3384, 424, 995, 978, 264, 26802, 6365, 314, 8842, 11, 264, 43472, 46370, 2272, 279, 13001, 314, 12282, 13, 84907, 1503, 524, 1683, 424, 557, 3977, 314, 279, 2993, 13, 1216, 15028, 424, 5820, 1521, 17540, 13235, 279, 2235, 51624, 321, 1362, 524, 4847, 4300, 5402, 310, 2842, 2019, 680, 310, 279, 20618, 11, 321, 421, 424, 8308, 1521, 18022, 4674, 310, 381, 279, 799, 310, 809, 424, 13, 27934, 1159, 513, 1040, 421, 13, 2302, 680, 4119, 836, 7715, 11, 321, 1396, 748, 3428, 11, 1973, 5316, 8842, 424, 11, 781, 1423, 264, 491, 799, 13, 271, 1465, 369, 16570, 52616, 1381, 321, 4138, 430, 449, 22839, 383, 279, 37126, 9662, 8520, 421, 488, 628, 1436, 494, 279, 1844, 314, 279, 324, 55965, 383, 264, 2708, 1834, 25, 1330, 7493, 4022, 73090, 10932, 303, 6761, 383, 279, 32946, 11, 1754, 799, 14936, 539, 24296, 421, 1830, 46219, 11, 9039, 321, 24052, 9361, 310, 264, 2444, 2956, 303, 2361, 3046, 13, 561, 5757, 513, 1546, 1603, 13, 2302, 2213, 4409, 4156, 13, 1216, 1990, 5601, 680, 310, 279, 20618, 383, 79833, 13382, 948, 551, 369, 2046, 11, 73797, 279, 7123, 11, 321, 5686, 13, 271, 25337, 1384, 3069, 11, 321, 551, 668, 3184, 488, 264, 3255, 421, 682, 4161, 310, 635, 440, 324, 21204, 19154, 13, 733, 780, 2018, 1007, 383, 279, 9662, 8520, 11, 264, 16070, 1785, 59144, 264, 6103, 9704, 303, 279, 9039, 314, 264, 94496, 383, 799, 314, 279, 73090, 506, 279, 2944, 809, 314, 279, 1287, 13, 561, 9704, 557, 2526, 11, 1575, 4613, 279, 13001, 11, 321, 279, 3061, 20364, 424, 430, 3238, 10304, 13, 1049, 1000, 599, 978, 6749, 506, 2261, 279, 1727, 13145, 3814, 11, 4590, 5381, 2843, 13, 357, 51715, 383, 279, 13017, 14882, 11, 11946, 279, 62970, 383, 2361, 2531, 11, 13293, 421, 279, 164840, 6502, 33172, 2086, 11, 264, 36334, 16365, 5020, 1172, 279, 13087, 2696, 11, 321, 4439, 279, 10390, 692, 310, 2842, 13, 3095, 781, 8660, 279, 94496, 781, 1669, 264, 17504, 421, 557, 264, 2342, 2756, 494, 20738, 13, 997, 46333, 424, 2695, 264, 6353, 13, 997, 46333, 279, 94496, 1238, 279, 17504, 4490, 1000, 599, 2695, 264, 2182, 321, 279, 3240, 314, 264, 2526, 3626, 13, 271, 760, 51715, 557, 524, 26802, 261, 1056, 279, 3061, 13, 2838, 995, 4777, 978, 1017, 11, 3160, 3213, 310, 6512, 11, 321, 21520, 3213, 310, 2466, 364, 279, 14882, 310, 2842, 13, 561, 795, 995, 6625, 279, 1788, 3377, 26, 424, 995, 1066, 524, 6625, 424, 52039, 3213, 364, 5316, 310, 2373, 13, 84907, 15028, 883, 421, 3428, 13, 6983, 1785, 551, 682, 6237, 383, 682, 978, 1546, 1603, 506, 28043, 279, 2433, 1141, 24990, 3481, 310, 708, 4808, 11, 321, 1691, 2599, 1603, 506, 279, 2433, 18022, 3481, 13, 561, 12558, 1881, 279, 1330, 369, 1332, 1208, 1990, 7005, 836, 2426, 11, 524, 539, 3604, 279, 5484, 579, 2531, 2577, 1056, 279, 5484, 11, 694, 539, 59291, 948, 279, 2531, 4924, 682, 28599, 5324, 13, 271, 4272, 383, 79833, 13382, 11, 948, 551, 628, 11]}, {"name": "multilingual", "ids": [2495, 450, 71234, 176217, 48, 16451, 18, 13, 23, 12, 23129, 12, 5666, 9251, 2349, 1074, 71, 34098, 1509, 6616, 43765, 16451, 43765, 16451, 18, 13, 23, 12, 23129, 12, 5666, 31230, 6993, 16927, 224986, 6464, 197102, 13, 198, 31538, 5810, 4227, 198214, 11, 19611, 48473, 189164, 12, 4312, 598, 11, 6407, 91957, 161691, 15008, 8162, 182720, 13, 8963, 77477, 294, 11, 205397, 5927, 11, 229411, 198, 33, 20143, 1962, 192900, 5280, 188474, 31926, 1962, 46472, 6481, 70, 779, 51592, 13, 83430, 82, 2652, 41759, 49188, 188474, 6464, 13, 271, 550, 41201, 56104, 5810, 1477, 30, 271, 46445, 17104, 1477, 6993, 39124, 37521, 224986, 29469, 12, 4716, 82, 18184, 37525, 13, 17530, 9372, 5810, 74905, 220, 65121, 34913, 90283, 287, 13, 271, 12, 2972, 29140, 154563, 92503, 64700, 12981, 56104, 2659, 63538, 303, 16927, 211571, 12485, 200366, 13, 220, 21, 15, 56212, 453, 47584, 27923, 12223, 198, 220, 195937, 11, 10396, 3717, 72621, 63615, 13, 198, 12, 2972, 43, 13051, 188474, 57491, 64700, 12981, 56104, 1477, 6993, 54854, 2421, 11, 557, 3717, 5521, 839, 267, 318, 164509, 4227, 157536, 11, 5927, 12030, 12485, 415, 94388, 5280, 198, 220, 220, 18, 17, 42, 56212, 553, 271, 20353, 29, 198, 10948, 1721, 337, 29028, 28392, 25, 220497, 220, 20, 15, 22, 15, 318, 16, 17, 18295, 681, 89540, 220, 20, 220, 22, 21, 15, 15, 11, 220, 21, 19, 18295, 21360, 510, 337, 1721, 337, 29, 55771, 25, 27255, 220, 24, 15, 22, 15, 59781, 318, 16, 21, 18295, 681, 89540, 220, 24, 220, 18, 24, 15, 15, 55, 11, 220, 19, 22, 18295, 21360, 510, 337, 1419, 371, 29, 198, 10948, 1721, 1251, 29, 271, 91, 178297, 735, 238791, 92503, 735, 436, 13051, 188474, 57491, 735, 198, 91, 12091, 735, 12091, 25, 735, 12091, 25, 735, 198, 91, 2972, 48, 17, 62, 15, 332, 735, 220, 24, 19, 56212, 2600, 735, 220, 17, 11, 21, 20, 15, 56212, 2600, 735, 198, 91, 2972, 59865, 17, 6645, 50, 332, 735, 220, 22, 24, 56212, 2600, 735, 220, 17, 11, 15, 24, 15, 56212, 2600, 735, 198, 91, 2972, 59865, 18, 62, 5955, 50, 332, 735, 220, 21, 17, 56212, 2600, 735, 220, 16, 11, 22, 20, 15, 56212, 2600, 735, 198, 91, 2972, 59865, 18, 1063, 332, 735, 220, 20, 18, 56212, 2600, 735, 220, 16, 11, 21, 17, 15, 56212, 2600, 735, 198, 91, 2972, 34120, 332, 735, 220, 20, 20, 56212, 2600, 735, 220, 17, 11, 16, 23, 15, 56212, 2600, 735, 271, 510, 1251, 1721, 1251, 29, 271, 91, 178297, 735, 238791, 92503, 735, 436, 13051, 188474, 57491, 735, 198, 91, 12091, 735, 12091, 25, 735, 12091, 25, 735, 198, 91, 2972, 48, 17, 62, 15, 332, 735, 220, 21, 15, 56212, 2600, 735, 220, 16, 11, 16, 21, 15, 56212, 2600, 735, 198, 91, 2972, 59865, 17, 6645, 50, 332, 735, 220, 20, 17, 56212, 2600, 735, 220, 16, 11, 16, 16, 15, 56212, 2600, 735, 198, 91, 2972, 34120, 332, 735, 220, 19, 19, 56212, 2600, 735, 220, 16, 11, 19, 17, 15, 56212, 2600, 735, 271, 510, 1251, 1419, 371, 29, 198, 510, 1938, 29, 271, 45, 28392, 25, 1167, 17, 62, 15, 5280, 7943, 220, 15, 13, 16, 13, 18, 21, 11, 2659, 37402, 9625, 22126, 5280, 220, 15, 13, 16, 13, 17, 21, 318, 19, 42, 92503, 11, 220, 18, 17, 42, 17058, 12389, 553, 8148, 162946, 6231, 198, 8320, 15212, 62663, 303, 498, 82835, 50, 20668, 9251, 13715, 14, 82835, 50, 20668, 2, 19755, 32848, 38012, 553, 49584, 189202, 5280, 17966, 18588, 1354, 5810, 195937, 25, 49584, 198, 5182, 55, 220, 18, 15, 24, 15, 318, 17, 19, 18295, 8, 49214, 74905, 220, 16, 15, 15, 12, 16, 19, 15, 56212, 453, 47584, 27923, 181737, 13, 84697, 882, 1798, 632, 271, 2495, 450, 19092, 216660, 176217, 48, 16451, 18, 13, 23, 12, 23129, 12, 5666, 9251, 2349, 1074, 71, 34098, 1509, 6616, 43765, 16451, 43765, 16451, 18, 13, 23, 12, 23129, 12, 5666, 31230, 1668, 632, 6464, 4472, 13, 351, 16811, 198, 354, 6453, 79653, 293, 6, 5681, 11, 24111, 12506, 11, 7528, 59006, 293, 6, 221400, 632, 208450, 13, 7305, 243195, 11, 85388, 3717, 1970, 11, 12647, 909, 5167, 1778, 198, 20812, 399, 785, 9462, 24449, 8098, 1778, 24449, 12636, 401, 1970, 13, 215329, 808, 16343, 647, 14487, 6464, 13, 271, 550, 176591, 1725, 796, 183208, 907, 271, 63611, 324, 46450, 2305, 10529, 322, 7639, 1668, 24739, 6464, 401, 48852, 47021, 16766, 13, 1188, 3817, 7618, 3656, 47288, 220, 65121, 293, 21372, 3724, 13, 271, 12, 2972, 25933, 35768, 3424, 190724, 170592, 1147, 183208, 3656, 175873, 1147, 86333, 274, 78502, 11720, 6652, 5992, 71388, 9915, 13, 220, 21, 15, 10885, 1298, 184286, 11, 272, 16811, 5346, 72931, 1649, 14487, 29579, 13, 198, 12, 2972, 66621, 14487, 9640, 170592, 1147, 183208, 3656, 175873, 3687, 15110, 1323, 3633, 1649, 8740, 57519, 9825, 318, 3266, 632, 2128, 11, 3717, 1970, 5724, 632, 187282, 401, 9915, 401, 220, 18, 17, 42, 10885, 553, 271, 20353, 29, 198, 10948, 1721, 337, 29028, 28392, 535, 220497, 220, 20, 15, 22, 15, 318, 16, 17, 5805, 681, 89540, 220, 20, 220, 22, 21, 15, 15, 11, 220, 21, 19, 5805, 401, 21360, 510, 337, 1721, 337, 29, 55771, 535, 27255, 220, 24, 15, 22, 15, 59781, 318, 16, 21, 5805, 681, 89540, 220, 24, 220, 18, 24, 15, 15, 55, 11, 220, 19, 22, 5805, 401, 21360, 510, 337, 1419, 371, 29, 198, 10948, 1721, 1251, 29, 271, 91, 196281, 735, 240637, 3424, 190724, 735, 36983, 14487, 9640, 735, 198, 91, 12091, 735, 12091, 25, 735, 12091, 25, 735, 198, 91, 2972, 48, 17, 62, 15, 332, 735, 220, 24, 19, 10885, 2600, 735, 220, 17, 11, 21, 20, 15, 10885, 2600, 735, 198, 91, 2972, 59865, 17, 6645, 50, 332, 735, 220, 22, 24, 10885, 2600, 735, 220, 17, 11, 15, 24, 15, 10885, 2600, 735, 198, 91, 2972, 59865, 18, 62, 5955, 50, 332, 735, 220, 21, 17, 10885, 2600, 735, 220, 16, 11, 22, 20, 15]}, {"name": "agent", "ids": [248045, 8678, 198, 2523, 513, 264, 10505, 8057, 3133, 303, 279, 1156, 579, 12182, 13, 1394, 628, 1562, 7141, 310, 1301, 3425, 11, 2624, 279, 1970, 321, 1542, 10966, 13, 13262, 4203, 2526, 321, 10033, 1092, 488, 1467, 13, 248046, 198, 248045, 846, 198, 760, 1228, 1510, 1877, 20394, 24647, 63, 14107, 2394, 13278, 579, 10562, 13, 2885, 488, 1423, 680, 3069, 321, 4884, 424, 30, 248046, 198, 248045, 74455, 198, 248068, 198, 40, 1220, 1542, 279, 20738, 1228, 1118, 310, 1436, 279, 4880, 1412, 1876, 11, 1179, 1353, 506, 279, 709, 424, 6813, 13, 198, 248069, 271, 248058, 198, 4754, 591, 763, 328, 5917, 10494, 487, 328, 15889, 763, 5046, 5454, 763, 328, 12305, 471, 76, 26864, 6813, 12333, 2914, 16799, 6971, 471, 74, 1228, 20394, 24647, 471, 80, 29958, 198, 248059, 248046, 198, 248045, 846, 198, 248066, 198, 37, 21641, 498, 16, 15, 15, 4, 60, 198, 2960, 8442, 32009, 48956, 34589, 410, 198, 7443, 40015, 62, 1228, 20394, 24647, 42390, 40015, 271, 262, 687, 1228, 20394, 24647, 4406, 198, 285, 1992, 4563, 24647, 437, 24, 15, 82, 871, 606, 220, 24, 15, 198, 285, 1992, 4563, 24647, 437, 20, 76, 871, 606, 220, 18, 15, 15, 198, 29, 950, 1992, 4563, 24647, 437, 16, 71, 18, 15, 76, 871, 606, 220, 20, 19, 15, 15, 198, 36, 950, 14953, 25, 8060, 7836, 25, 359, 16, 71, 18, 15, 76, 6, 271, 23095, 12333, 2914, 16799, 6971, 25, 16, 19, 25, 14953, 198, 16, 4490, 11, 220, 16, 16, 48600, 2140, 303, 220, 15, 13, 17, 16, 82, 198, 248067, 248046, 198, 248045, 74455, 198, 248068, 198, 10570, 7896, 944, 694, 264, 10539, 869, 14107, 13, 6558, 728, 1301, 279, 6518, 13, 198, 248069, 271, 248058, 198, 4754, 591, 763, 328, 851, 2378, 487, 328, 15889, 763, 5046, 2267, 763, 328, 3431, 34093, 16799, 6971, 487, 328, 2388, 6325, 763, 220, 16, 11, 328, 400, 6325, 763, 220, 19, 15, 3307, 198, 248059, 248046, 198, 248045, 846, 198, 248066, 198, 464, 311, 271, 87662, 283, 5046, 82, 763, 220, 16, 11, 328, 76, 763, 220, 21, 15, 11, 328, 71, 763, 220, 18, 21, 15, 15, 11, 328, 67, 763, 220, 23, 21, 19, 15, 15, 92, 198, 27360, 283, 311, 18351, 2515, 83501, 11189, 67, 10, 2299, 58, 3445, 15518, 2387, 3, 871, 1358, 727, 4563, 24647, 7008, 25, 592, 8, 1411, 514, 25, 198, 262, 4071, 14048, 264, 7836, 1040, 359, 24, 15, 82, 504, 359, 20, 76, 6, 466, 359, 16, 71, 18, 15, 76, 6, 1083, 6283, 71483, 198, 262, 1414, 283, 1414, 16674, 973, 14344, 363, 198, 262, 413, 524, 1414, 25, 198, 285, 4673, 14953, 437, 3092, 7836, 871, 198, 262, 295, 283, 696, 32379, 10749, 7008, 8, 198, 262, 413, 295, 369, 2168, 25, 198, 285, 4673, 14953, 925, 1, 11469, 7836, 25, 313, 1272, 0, 81, 54084, 198, 262, 460, 514, 1212, 5982, 7, 16, 578, 348, 696, 1797, 14059, 11722, 5982, 7, 17, 7025, 198, 248067, 248046, 198, 248045, 74455, 198, 248068, 198, 760, 19567, 369, 74483, 506, 2107, 10029, 321, 8798, 6681, 799, 1324, 58533, 6505, 11, 748, 328, 16, 71, 18, 15, 76, 1, 628, 2496, 2353, 13, 561, 4479, 889, 20021, 10539, 2663, 13, 6558, 728, 1716, 15865, 3712, 310, 1436, 1092, 279, 10562, 5324, 13, 198, 248069, 271, 248058, 198, 4754, 591, 763, 328, 5917, 10494, 487, 328, 15889, 763, 5046, 5454, 763, 328, 12513, 1433, 471, 79, 471, 77, 220, 16, 1137, 2212, 34093, 16799, 6971, 735, 1901, 471, 19, 15, 29958, 198, 248059, 248046, 198, 248045, 846, 198, 248066, 198, 16899, 220, 19, 68, 16, 65, 24, 66, 17, 69, 22, 64, 15, 67, 318, 50346, 8602, 359, 53380, 12, 25543, 837, 198, 262, 59176, 1386, 7836, 21621, 271, 67677, 32379, 283, 311, 18351, 2515, 1, 11189, 67, 10, 2299, 58, 3445, 15518, 2387, 871, 198, 94081, 32379, 283, 311, 18351, 2515, 83501, 11189, 67, 10, 2299, 58, 3445, 15518, 2387, 3, 871, 198, 18611, 687, 4563, 24647, 7008, 25, 592, 8, 1411, 514, 25, 198, 12, 262, 2702, 283, 220, 15, 198, 12, 262, 1097, 283, 220, 15, 198, 12, 262, 364, 295, 303, 696, 32379, 2571, 1947, 7008, 1590, 198, 12, 285, 413, 295, 4801, 363, 931, 1097, 25, 198, 12, 309, 4673, 14953, 925, 1, 11469, 7836, 25, 313, 1272, 0, 81, 54084, 198, 12, 285, 2702, 1373, 514, 1212, 5982, 7, 16, 578, 348, 696, 1797, 14059, 11722, 5982, 7, 17, 7025, 198, 12, 285, 1097, 283, 295, 4911, 363, 198, 12, 262, 413, 1097, 931, 2343, 7008, 1590, 198, 12, 285, 4673, 14953, 925, 1, 11469, 7836, 25, 313, 1272, 0, 81, 54084, 198, 12, 262, 460, 2702, 198, 10, 262, 295, 283, 696, 32379, 10749, 7008, 8, 198, 10, 262, 413, 295, 369, 2168, 25, 198, 10, 285, 4673, 14953, 925, 1, 11469, 7836, 25, 313, 1272, 0, 81, 54084, 198, 10, 262, 460, 514, 1212, 5982, 7, 16, 578, 348, 696, 1797, 14059, 11722, 5982, 7, 17, 7025, 198, 248067, 248046, 198, 248045, 74455, 198, 760, 20635, 10562, 12215, 279, 6136, 888, 660, 1324, 58533, 13139, 440, 264, 3074, 74483, 2353, 11, 748, 1132, 799, 4821, 791, 886, 369, 11330, 1381, 13, 353, 668, 14519, 279, 6136, 11, 864, 1048, 57409, 25053, 1881, 279, 5306, 13, 271, 248058, 198, 4754, 591, 763, 328, 3468, 2378, 487, 328, 15889, 763, 5046, 2267, 763, 328, 3431, 34093, 16799, 6971, 487, 328, 787, 763, 8728, 32379, 283, 311, 18351, 2515, 2037, 12883, 3312, 67, 10, 2299, 58, 3445, 15518, 2387, 56749, 871, 487, 328, 902, 763, 8728, 32379, 283, 311, 18351, 2515, 2037, 7, 3312, 67, 10, 2299, 58, 3445, 15518, 2387, 60592, 29958, 198, 248059, 248046, 198, 248045, 846, 198, 248066, 198, 3793, 25, 220, 16, 13329, 303, 2212, 34093, 16799, 6971, 198, 248067, 248046, 198, 248045, 74455, 198, 248058, 198, 4754, 591, 763, 328, 3468, 2378, 487, 328, 15889, 763, 5046, 2267, 763, 328, 3431, 34093, 16799, 6971, 487, 328, 787, 763, 328, 262, 295, 283, 696]}]} \ No newline at end of file diff --git a/include/strata/prefill/mmq_kq_abi.hpp b/include/strata/prefill/mmq_kq_abi.hpp new file mode 100644 index 000000000..0aa833c5e --- /dev/null +++ b/include/strata/prefill/mmq_kq_abi.hpp @@ -0,0 +1,30 @@ +// include/strata/prefill/mmq_kq_abi.hpp - the interface between the engine (src/prefill/moe_mmq.cu) and strata_kq, the +// shared library with the K-quant MMQ prompt kernels (src/prefill/mmq_kq_lib.cu). Bump the version when a function's +// meaning changes; the argument block (ggml's mmq_args) is checked by its size. +#pragma once + +#define STRATA_KQ_ABI_VERSION 1 + +#if defined(_WIN32) +#define STRATA_KQ_LIB_NAME "strata_kq.dll" +#else +#define STRATA_KQ_LIB_NAME "libstrata_kq.so" +#endif + +extern "C" { +typedef int (*strata_kq_abi_fn)(); +typedef int (*strata_kq_init_fn)(int device); +typedef int (*strata_kq_run_fn)(int device, int ggml_type, const void* mmq_args, void* stream); +} + +// the engine's side (src/prefill/mmq_kq_loader.cpp) +namespace strata::prefill::mmq::kq { +/// STRATA_MMQ_KQ=1: the K-quant experts' prompt products through MMQ (opt-in: other rounding than the FP16 path). +bool wanted(); +/// wanted() and the library is open and started on every GPU (the first call opens it and says what happened). +bool available(); +/// One product on the current GPU: 0 = launched, else an error (-1: no library or not this type). +int run(int ggml_type, const void* mmq_args, void* stream); +/// sizeof(mmq_args) on the engine's side (moe_mmq.cu), checked against the library's. +int abi_args_size(); +} // namespace strata::prefill::mmq::kq diff --git a/src/prefill/mmq_kq_lib.cu b/src/prefill/mmq_kq_lib.cu new file mode 100644 index 000000000..d28b5ddfa --- /dev/null +++ b/src/prefill/mmq_kq_lib.cu @@ -0,0 +1,60 @@ +// src/prefill/mmq_kq_lib.cu - strata_kq, the shared library with llama.cpp's MMQ prompt kernels for the K-quants +// (Q4_K, Q5_K, Q5_1, Q6_K: the experts of Unsloth's UD-Q4/Q5/Q6_K_XL files). MODEL-UPDATE Phase 1: the engine loads +// every kernel of its own binary when it starts (CUDA_MODULE_LOADING=EAGER, src/program/generate.cpp), so kernels in +// the engine cost every model VRAM; these four instances were ~3.8 MB of sm_60 code (3 expert slots of IQ3_XXS on a +// P100). Here they load only when moe_mmq.cu opens this library - for a model with these experts, before the expert +// cache is sized - and never otherwise. The library has its own copy of ggml_cuda_host.cu's symbols and of the static +// CUDA runtime; it works in the engine's context on the stream it is given. +#include "common.cuh" +#include "mmq.cuh" + +#include "strata/prefill/mmq_kq_abi.hpp" + +#include + +#if defined(_WIN32) +#define STRATA_KQ_EXPORT extern "C" __declspec(dllexport) +#else +#define STRATA_KQ_EXPORT extern "C" __attribute__((visibility("default"))) +#endif + +namespace { +std::mutex g_mu; +ggml_backend_cuda_context* g_ctx[GGML_CUDA_MAX_DEVICES] = {}; + +ggml_backend_cuda_context* ctx_for(int dev) { + std::lock_guard lk(g_mu); + if (g_ctx[dev] == nullptr) g_ctx[dev] = new ggml_backend_cuda_context(dev); + return g_ctx[dev]; +} +} // namespace + +// the interface version and the argument block's size: both sides are compiled from the same mmq.cuh +STRATA_KQ_EXPORT int strata_kq_abi() { return STRATA_KQ_ABI_VERSION * 100000 + (int) sizeof(mmq_args); } + +// Starts this library's CUDA runtime in the engine's context on `device`: with CUDA_MODULE_LOADING=EAGER (the engine +// sets it before its first CUDA call) that loads every kernel here now, while the VRAM for it is still free. +// 0 = ready, else the cudaError_t. +STRATA_KQ_EXPORT int strata_kq_init(int device) { + cudaError_t e = cudaSetDevice(device); + if (e == cudaSuccess) e = cudaFree(nullptr); + return (int) e; +} + +// One MMQ product (moe_mmq.cu's Context::run for these types) on `device`, on the engine's `stream`. 0 = launched, -1 = +// a type this library does not have, else the cudaError_t of the launch. +STRATA_KQ_EXPORT int strata_kq_run(int device, int type, const void* args, void* stream) { + if (device < 0 || device >= GGML_CUDA_MAX_DEVICES) return -1; + if (cudaSetDevice(device) != cudaSuccess) return (int) cudaGetLastError(); + ggml_backend_cuda_context& ctx = *ctx_for(device); + const mmq_args& a = *(const mmq_args*) args; + const cudaStream_t s = (cudaStream_t) stream; + switch ((ggml_type) type) { + case GGML_TYPE_Q4_K: mul_mat_q_case(ctx, a, s); break; + case GGML_TYPE_Q5_K: mul_mat_q_case(ctx, a, s); break; + case GGML_TYPE_Q5_1: mul_mat_q_case(ctx, a, s); break; + case GGML_TYPE_Q6_K: mul_mat_q_case(ctx, a, s); break; + default: return -1; + } + return (int) cudaGetLastError(); +} diff --git a/src/prefill/mmq_kq_loader.cpp b/src/prefill/mmq_kq_loader.cpp new file mode 100644 index 000000000..b24985775 --- /dev/null +++ b/src/prefill/mmq_kq_loader.cpp @@ -0,0 +1,124 @@ +// src/prefill/mmq_kq_loader.cpp - opens strata_kq (src/prefill/mmq_kq_lib.cu), the library with the K-quant MMQ prompt +// kernels, from the engine's own folder - once, for the first model format that asks for it (moe_mmq.cu's supported()), +// which the engine does before it sizes the expert cache (src/program/generate.cpp). Without the library, or with an +// interface of another version, the K-quants keep the FP16 prompt path and the engine says so once. +#include "strata/prefill/mmq_kq_abi.hpp" + +#include + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#define WIN32_LEAN_AND_MEAN +#include +#else +#include +#include +#endif + +namespace strata::prefill::mmq::kq { +namespace { + +struct Lib { + bool ok = false; + strata_kq_run_fn run = nullptr; +}; + +std::string own_folder() { +#if defined(_WIN32) + char p[MAX_PATH] = {}; + const DWORD n = GetModuleFileNameA(nullptr, p, MAX_PATH); + std::string s(p, n); + const size_t k = s.find_last_of("\\/"); + return k == std::string::npos ? std::string() : s.substr(0, k + 1); +#else + char p[4096] = {}; + const ssize_t n = readlink("/proc/self/exe", p, sizeof(p) - 1); + std::string s(p, n > 0 ? (size_t) n : 0); + const size_t k = s.find_last_of('/'); + return k == std::string::npos ? std::string() : s.substr(0, k + 1); +#endif +} + +void* sym(void* h, const char* name) { +#if defined(_WIN32) + return reinterpret_cast(GetProcAddress((HMODULE) h, name)); +#else + return dlsym(h, name); +#endif +} + +Lib open_lib() { + Lib L; + const std::string path = own_folder() + STRATA_KQ_LIB_NAME; +#if defined(_WIN32) + void* h = (void*) LoadLibraryA(path.c_str()); +#else + void* h = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); +#endif + if (h == nullptr) { + std::fprintf(stderr, "strata: K-quant prompt kernels (STRATA_MMQ_KQ=1): %s not found next to the engine - " + "these experts take the FP16 prompt path\n", STRATA_KQ_LIB_NAME); + return L; + } + const auto abi = (strata_kq_abi_fn) sym(h, "strata_kq_abi"); + const auto init = (strata_kq_init_fn) sym(h, "strata_kq_init"); + L.run = (strata_kq_run_fn) sym(h, "strata_kq_run"); + if (abi == nullptr || init == nullptr || L.run == nullptr || abi() / 100000 != STRATA_KQ_ABI_VERSION || + abi() % 100000 != abi_args_size()) { + std::fprintf(stderr, "strata: K-quant prompt kernels: %s is from another engine version - these experts take " + "the FP16 prompt path\n", path.c_str()); + L.run = nullptr; + return L; + } + // every GPU this engine sees: each context loads the library's kernels now, while the VRAM is still free + int n = 0, cur = 0; + cudaGetDevice(&cur); + if (cudaGetDeviceCount(&n) != cudaSuccess) n = 0; + for (int d = 0; d < n; ++d) { + const int e = init(d); + if (e != 0) { + std::fprintf(stderr, "strata: K-quant prompt kernels: %s did not start on GPU %d (CUDA error %d) - these " + "experts take the FP16 prompt path\n", path.c_str(), d, e); + L.run = nullptr; + cudaSetDevice(cur); + return L; + } + } + cudaSetDevice(cur); + std::fprintf(stderr, "strata: K-quant prompt kernels from %s (STRATA_MMQ_KQ=1)\n", path.c_str()); + L.ok = true; + return L; +} + +const Lib& lib() { + static const Lib L = open_lib(); + return L; +} + +} // namespace + +bool wanted() { + static const bool on = [] { + const char* v = std::getenv("STRATA_MMQ_KQ"); + return v != nullptr && v[0] == '1'; + }(); + return on; +} + +bool available() { return wanted() && lib().ok; } + +int run(int type, const void* args, void* stream) { + int dev = 0; + cudaGetDevice(&dev); + return lib().ok ? lib().run(dev, type, args, stream) : -1; +} + +} // namespace strata::prefill::mmq::kq diff --git a/src/prefill/moe_mmq.cu b/src/prefill/moe_mmq.cu index b0446f8ee..8569e9102 100644 --- a/src/prefill/moe_mmq.cu +++ b/src/prefill/moe_mmq.cu @@ -7,11 +7,33 @@ #include "mmq.cuh" #include "quantize.cuh" +#include "strata/prefill/mmq_kq_abi.hpp" + #include #include namespace strata::prefill::mmq { +#if defined(STRATA_KQ_DSO) +namespace kq { +int abi_args_size() { return (int) sizeof(mmq_args); } +} // namespace kq +#endif namespace { +#if defined(STRATA_MMQ_KQUANTS) +// MODEL-UPDATE Phase 1: the K-quant products through MMQ are opt-in at run time (STRATA_MMQ_KQ=1); on CUDA their +// kernels are the strata_kq library (mmq_kq_loader.cpp), opened only then +bool kq_ok() { +#if defined(STRATA_KQ_DSO) + return kq::available(); +#else + static const bool on = [] { + const char* v = std::getenv("STRATA_MMQ_KQ"); + return v != nullptr && v[0] == '1'; + }(); + return on; +#endif +} +#endif void ck(cudaError_t e, const char* what) { if (e != cudaSuccess) { @@ -127,13 +149,12 @@ bool supported(int t) { case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_Q8_0: // the draft layer's dense matrices (E-9) -#ifdef STRATA_MMQ_KQUANTS - case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q5_1: // Unsloth's UD-Q4_K_XL experts (CUDA) -#if defined(__HIPCC__) || defined(STRATA_Q6K_EXPERTS) - case GGML_TYPE_Q6_K: // HIP: the dense GGUF projections (STRATA_DENSE_MMQ); CUDA: only the opt-in -DSTRATA_Q6K_EXPERTS=ON build -#endif -#endif return true; +#ifdef STRATA_MMQ_KQUANTS + // Unsloth's UD-Q4/Q5/Q6_K_XL experts; on HIP also the dense GGUF projections (STRATA_DENSE_MMQ) + case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q5_1: case GGML_TYPE_Q6_K: + return kq_ok(); +#endif default: return false; } @@ -213,12 +234,19 @@ void Context::run(const Product& p, void* stream) { case GGML_TYPE_IQ4_NL: mul_mat_q_case(ctx, a, s); break; case GGML_TYPE_IQ4_XS: mul_mat_q_case(ctx, a, s); break; case GGML_TYPE_Q8_0: mul_mat_q_case(ctx, a, s); break; -#ifdef STRATA_MMQ_KQUANTS +#if defined(STRATA_MMQ_KQUANTS) && defined(STRATA_KQ_DSO) + case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_Q5_1: { + const int e = kq::run((int) t, &a, s); + if (e != 0) { + std::fprintf(stderr, "prefill mmq: the K-quant library failed on type %d (error %d)\n", (int) t, e); + std::exit(1); + } + break; + } +#elif defined(STRATA_MMQ_KQUANTS) case GGML_TYPE_Q4_K: mul_mat_q_case(ctx, a, s); break; case GGML_TYPE_Q5_K: mul_mat_q_case(ctx, a, s); break; -#if defined(__HIPCC__) || defined(STRATA_Q6K_EXPERTS) case GGML_TYPE_Q6_K: mul_mat_q_case(ctx, a, s); break; -#endif case GGML_TYPE_Q5_1: mul_mat_q_case(ctx, a, s); break; #endif default: diff --git a/src/program/generate.cpp b/src/program/generate.cpp index 7a93c0a3e..68cef95ee 100644 --- a/src/program/generate.cpp +++ b/src/program/generate.cpp @@ -62,6 +62,9 @@ #include "strata/core/verify.hpp" #include "strata/core/mtp.hpp" #include "strata/prefill/prefill.hpp" +#if defined(STRATA_PREFILL_MMQ) +#include "strata/prefill/moe_mmq.hpp" +#endif #include "strata/core/native_dense.hpp" #include "strata/program/batch_read.hpp" #include "strata/program/logits_selection.hpp" @@ -2759,6 +2762,12 @@ int main(int argc, char** argv) { f.gu_type, f.d_type); return 1; } +#if defined(STRATA_PREFILL_MMQ) + // MODEL-UPDATE Phase 1: the K-quants' prompt kernels (STRATA_MMQ_KQ=1) are a library of their own on CUDA + // (strata_kq): asking for the formats opens it now, before the expert cache takes the free VRAM + (void) strata::prefill::mmq::supported(f.gu_type); + (void) strata::prefill::mmq::supported(f.d_type); +#endif } } const bool native_pack = strata::kernels::cpu::expert_layout().native; diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txt index 9c63332b8..15ef03dfc 100644 --- a/sycl/CMakeLists.txt +++ b/sycl/CMakeLists.txt @@ -149,6 +149,12 @@ find_package(Threads REQUIRED) target_link_libraries(strata_kernels_cpu PUBLIC strata_includes Threads::Threads) option(STRATA_NATIVE_EXPERTS "Native (IQ) expert support through ggml-cpu" ON) +# Q6_K gate/up in the grouped expert kernels (UD-Q6_K_XL, layer 2 of UD-Q5_K_XL): on by default, as in the CUDA / HIP +# build (MODEL-UPDATE Phase 1) +option(STRATA_Q6K_EXPERTS "Build the Q6_K gate/up grouped expert kernels (UD-Q6_K_XL, UD-Q5_K_XL)" ON) +if(STRATA_Q6K_EXPERTS) + add_compile_definitions(STRATA_Q6K_EXPERTS=1) +endif() set(STRATA_GGML_DIR "" CACHE PATH "llama.cpp checkout to take ggml from (empty: fetch the pinned commit)") if(STRATA_NATIVE_EXPERTS) set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)