feat(ext): let text providers declare their speculation defaults (#1047)

Co-authored-by: Claude Opus 5.5 <[email protected]>

CMakeLists.txt

@@ -372,7 +372,8 @@ add_subdirectory(src/eval)
 add_library(gufo_llm_cli STATIC
   src/cli/bench/bench.cpp
   src/cli/prompt/prompt.cpp
-  src/cli/provider_options.cpp)
+  src/cli/provider_options.cpp
+  src/cli/speculative_settings.cpp)
 target_link_libraries(gufo_llm_cli PUBLIC gufo_core gufo_compare gufo_ext_runtime)
 
 add_subdirectory(src/ext)

docs/extensions.md

@@ -279,6 +279,24 @@ rejects conflicting types. The Nix serving helper derives provider arguments and
 from metadata, including optional `serving` declarations. See
 [Nix serving](model-extensions/serving.md).
 
+Text providers declare the draft settings a speculative backend runs with in
+`TextProvider::speculative_defaults`, one entry per backend (`mtp`, `dflash2`,
+`dspark`). Core resolves each setting once the provider is selected, before any
+weights load, and logs the result in `load_completed`: an explicit
+`--draft-tokens`, `--min-draft-tokens` or `--draft-policy`, else the declared
+value, else the host's fallback of 7, 1 and no policy (DFlash2 accepts `fixed`
+and `adaptive`). `--draft-policy` must be in the entry's `draft_policies`. The
+resolved values reach `TextSpeculativeConfig`, `TextPromptOptions` and
+`TextBenchOptions`.
+
+```cpp
+.speculative_defaults = {{.backend = "mtp",
+                          .max_draft_tokens = 15,
+                          .min_draft_tokens = 1,
+                          .draft_policy = "calibrated",
+                          .draft_policies = {"calibrated", "fixed"}}},
+```
+
 Text providers receive their options. `serve`, `prompt`, `chat` and `bench`
 accept every option an available text provider declares for that command, list
 the ones core does not own under "Provider options" in `--help`, and parse values

include/gufo/ext/changes/0.2/text-speculative-defaults.md

@@ -0,0 +1,19 @@
+# Text speculative defaults
+
+Text (`src/ext/text.hpp`): a text provider declares the draft settings of each
+speculative backend.
+- `TextProvider::speculative_defaults` (`TextSpeculativeDefaults`): per backend,
+  the values used when `--draft-tokens`, `--min-draft-tokens` and
+  `--draft-policy` are absent, and the `--draft-policy` values it accepts.
+- `TextSpeculativeConfig::draft_policy`: the resolved policy name;
+  `dflash_policy` is derived from it for DFlash2.
+- `max_draft_tokens`, `min_draft_tokens` and `draft_policy` in
+  `TextSpeculativeConfig`, `TextPromptOptions` and `TextBenchOptions` hold the
+  resolved values: the explicit flag, else the provider's default, else the
+  host's 7, 1 and no policy.
+- A `--draft-policy` the selected provider does not accept fails after the
+  provider is selected, before weights load; one that no available provider
+  accepts fails while parsing.
+
+Compatibility: additive. A provider that declares nothing runs with the values
+it received before, and command lines accepted before are accepted unchanged.

src/cli/bench/bench.cpp

@@ -16,6 +16,7 @@
 #include "src/cli/arg_parser.hpp"
 #include "src/cli/provider_options.hpp"
 #include "src/cli/sampling_options.hpp"
+#include "src/cli/speculative_settings.hpp"
 #include "src/core/gguf_reader.hpp"
 #include "src/core/sampling.hpp"
 
@@ -219,7 +220,9 @@ namespace {
 
 std::optional<BenchOptions> ParseBenchOptions(std::span<const char* const> args,
                                               std::string* error_msg,
-                                              ProviderOptions& provider) {
+                                              const ext::Registry& registry,
+                                              ProviderOptions& provider,
+                                              SpeculativeFlags& speculative) {
   BenchOptions opt;
   bool explicit_p = false;
   bool explicit_n = false;
@@ -252,10 +255,9 @@ std::optional<BenchOptions> ParseBenchOptions(std::span<const char* const> args,
     return std::nullopt;
   }
 
-  if (opt.draft_tokens == 0 || opt.min_draft_tokens == 0 ||
-      opt.min_draft_tokens > opt.draft_tokens) {
+  if (opt.draft_tokens == 0 || opt.min_draft_tokens == 0) {
     if (error_msg != nullptr) {
-      *error_msg = "min-draft-tokens cannot exceed draft-tokens";
+      *error_msg = "draft-tokens and min-draft-tokens must be positive";
     }
     return std::nullopt;
   }
@@ -276,11 +278,14 @@ std::optional<BenchOptions> ParseBenchOptions(std::span<const char* const> args,
     opt.speculative_backend = "dspark";
   }
 
-  if (!opt.draft_policy.empty() &&
-      ((opt.draft_policy != "fixed" && opt.draft_policy != "adaptive") ||
-       opt.speculative_backend != "dflash2")) {
+  // Unset draft settings take the selected provider's defaults later.
+  speculative = SpeculativeFlags::Given(parser, opt.draft_tokens,
+                                        opt.min_draft_tokens, opt.draft_policy);
+  try {
+    CheckSpeculativeFlags(registry, opt.speculative_backend, speculative);
+  } catch (const std::invalid_argument& exception) {
     if (error_msg != nullptr)
-      *error_msg = "--draft-policy requires DFlash2 and fixed or adaptive";
+      *error_msg = exception.what();
     return std::nullopt;
   }
   if (opt.min_draft_tokens != 1 && opt.speculative_backend == "dflash2") {
@@ -301,7 +306,8 @@ std::optional<BenchOptions> ParseBenchOptions(std::span<const char* const> args,
   ext::Registry registry;
   ext::RegisterBuiltExtensions(registry);
   ProviderOptions provider(registry, "bench");
-  return ParseBenchOptions(args, error_msg, provider);
+  SpeculativeFlags speculative;
+  return ParseBenchOptions(args, error_msg, registry, provider, speculative);
 }
 
 int RunBench(std::span<const char* const> args) {
@@ -309,7 +315,9 @@ int RunBench(std::span<const char* const> args) {
   ext::Registry registry;
   ext::RegisterBuiltExtensions(registry);
   ProviderOptions provider_options(registry, "bench");
-  auto opt_res = ParseBenchOptions(args, &parse_err, provider_options);
+  SpeculativeFlags speculative;
+  auto opt_res = ParseBenchOptions(args, &parse_err, registry, provider_options,
+                                   speculative);
   if (!opt_res.has_value()) {
     if (!parse_err.empty()) {
       std::cerr << "Error: " << parse_err << "\n";
@@ -353,6 +361,11 @@ int RunBench(std::span<const char* const> args) {
   }
   try {
     opt.provider_options = provider_options.Select(provider->id);
+    const auto settings = ResolveSpeculativeSettings(
+        *provider, opt.speculative_backend, speculative);
+    opt.draft_tokens = settings.max_draft_tokens;
+    opt.min_draft_tokens = settings.min_draft_tokens;
+    opt.draft_policy = settings.draft_policy;
   } catch (const std::invalid_argument& exception) {
     std::cerr << "Error: " << exception.what() << "\n";
     PrintModelLoadTime(model_load_start, false);

src/cli/prompt/prompt.cpp

@@ -15,6 +15,7 @@
 #include "src/cli/arg_parser.hpp"
 #include "src/cli/provider_options.hpp"
 #include "src/cli/sampling_options.hpp"
+#include "src/cli/speculative_settings.hpp"
 #include "src/core/gguf_reader.hpp"
 #include "src/core/sampling.hpp"
 
@@ -187,7 +188,9 @@ void RegisterTextOptions(ArgParser& parser, PromptOptions& opt,
 std::optional<PromptOptions> ParseTextOptions(std::span<const char* const> args,
                                               std::string* error_msg,
                                               std::string_view command,
-                                              ProviderOptions& provider) {
+                                              const ext::Registry& registry,
+                                              ProviderOptions& provider,
+                                              SpeculativeFlags& speculative) {
   ArgParser parser("gufo " + std::string(command),
                    command == "chat" ? "Interactive conversation."
                                      : "Execute one prompt request and exit.");
@@ -240,19 +243,20 @@ std::optional<PromptOptions> ParseTextOptions(std::span<const char* const> args,
       *error_msg = "DSpark requires --dspark-model";
     return std::nullopt;
   }
-  if (opt.draft_tokens == 0 || opt.min_draft_tokens == 0 ||
-      opt.min_draft_tokens > opt.draft_tokens) {
+  if (opt.draft_tokens == 0 || opt.min_draft_tokens == 0) {
     if (error_msg != nullptr) {
-      *error_msg = "min-draft-tokens cannot exceed draft-tokens";
+      *error_msg = "draft-tokens and min-draft-tokens must be positive";
     }
     return std::nullopt;
   }
-
-  if (!opt.draft_policy.empty() &&
-      ((opt.draft_policy != "fixed" && opt.draft_policy != "adaptive") ||
-       opt.speculative_backend != "dflash2")) {
+  // Unset draft settings take the selected provider's defaults later.
+  speculative = SpeculativeFlags::Given(parser, opt.draft_tokens,
+                                        opt.min_draft_tokens, opt.draft_policy);
+  try {
+    CheckSpeculativeFlags(registry, opt.speculative_backend, speculative);
+  } catch (const std::invalid_argument& exception) {
     if (error_msg != nullptr)
-      *error_msg = "--draft-policy requires DFlash2 and fixed or adaptive";
+      *error_msg = exception.what();
     return std::nullopt;
   }
   if (opt.min_draft_tokens != 1 && opt.speculative_backend == "dflash2") {
@@ -296,12 +300,18 @@ std::optional<PromptOptions> ParseTextOptions(std::span<const char* const> args,
   return opt;
 }
 
-// Delivers the selected provider's options; false after reporting an error.
-bool SelectProviderOptions(const ProviderOptions& provider,
-                           const ext::TextProvider& selected,
-                           PromptOptions* opt) {
+// Delivers the selected provider's options and resolves its draft settings;
+// false after reporting an error.
+bool PrepareForProvider(const ProviderOptions& provider,
+                        const SpeculativeFlags& speculative,
+                        const ext::TextProvider& selected, PromptOptions* opt) {
   try {
     opt->provider_options = provider.Select(selected.id);
+    const auto settings = ResolveSpeculativeSettings(
+        selected, opt->speculative_backend, speculative);
+    opt->draft_tokens = settings.max_draft_tokens;
+    opt->min_draft_tokens = settings.min_draft_tokens;
+    opt->draft_policy = settings.draft_policy;
   } catch (const std::invalid_argument& exception) {
     std::cerr << "Error: " << exception.what() << "\n";
     return false;
@@ -359,7 +369,9 @@ std::optional<PromptOptions> ParsePromptOptions(
   ext::Registry registry;
   ext::RegisterBuiltExtensions(registry);
   ProviderOptions provider(registry, "prompt");
-  return ParseTextOptions(args, error_msg, "prompt", provider);
+  SpeculativeFlags speculative;
+  return ParseTextOptions(args, error_msg, "prompt", registry, provider,
+                          speculative);
 }
 
 int RunPrompt(std::span<const char* const> args) {
@@ -367,8 +379,9 @@ int RunPrompt(std::span<const char* const> args) {
   ext::Registry registry;
   ext::RegisterBuiltExtensions(registry);
   ProviderOptions provider_options(registry, "prompt");
-  const auto opt_res =
-      ParseTextOptions(args, &parse_err, "prompt", provider_options);
+  SpeculativeFlags speculative;
+  const auto opt_res = ParseTextOptions(args, &parse_err, "prompt", registry,
+                                        provider_options, speculative);
   if (!opt_res.has_value()) {
     if (!parse_err.empty()) {
       std::cerr << "Error: " << parse_err << "\n";
@@ -434,7 +447,7 @@ int RunPrompt(std::span<const char* const> args) {
     PrintModelLoadTime(model_load_start, false);
     return 1;
   }
-  if (!SelectProviderOptions(provider_options, *provider, &opt)) {
+  if (!PrepareForProvider(provider_options, speculative, *provider, &opt)) {
     PrintModelLoadTime(model_load_start, false);
     return 2;
   }
@@ -450,8 +463,9 @@ int RunChat(std::span<const char* const> args) {
   ext::Registry registry;
   ext::RegisterBuiltExtensions(registry);
   ProviderOptions provider_options(registry, "chat");
-  const auto opt_res =
-      ParseTextOptions(args, &parse_err, "chat", provider_options);
+  SpeculativeFlags speculative;
+  const auto opt_res = ParseTextOptions(args, &parse_err, "chat", registry,
+                                        provider_options, speculative);
   if (!opt_res.has_value()) {
     if (!parse_err.empty()) {
       std::cerr << "Error: " << parse_err << "\n";
@@ -494,7 +508,7 @@ int RunChat(std::span<const char* const> args) {
     PrintModelLoadTime(model_load_start, false);
     return 1;
   }
-  if (!SelectProviderOptions(provider_options, *provider, &opt)) {
+  if (!PrepareForProvider(provider_options, speculative, *provider, &opt)) {
     PrintModelLoadTime(model_load_start, false);
     return 2;
   }

src/cli/serve/serve.cpp

@@ -36,6 +36,7 @@
 #include "src/cli/serve/inference_backend.hpp"
 #include "src/cli/serve/logging.hpp"
 #include "src/cli/serve/sampling_request.hpp"
+#include "src/cli/speculative_settings.hpp"
 #include "src/core/diagnostics/gpu_queues.h"
 #include "src/core/platform/shutdown_signal.hpp"
 
@@ -1550,8 +1551,8 @@ int RunServe(std::span<const char* const> args) {
       return 2;
     }
     if (draft_tokens == 0 || min_draft_tokens == 0 ||
-        min_draft_tokens > draft_tokens ||
-        draft_tokens > std::numeric_limits<std::uint32_t>::max()) {
+        draft_tokens > std::numeric_limits<std::uint32_t>::max() ||
+        min_draft_tokens > std::numeric_limits<std::uint32_t>::max()) {
       std::cerr << "Error: speculative draft limits are invalid\n";
       return 2;
     }
@@ -1586,13 +1587,13 @@ int RunServe(std::span<const char* const> args) {
                 << "' is not supported by the HTTP server\n";
       return 2;
     }
+    // Unset draft settings take the selected provider's defaults during load.
+    const auto speculative_flags = SpeculativeFlags::Given(
+        llm_parser, static_cast<std::uint32_t>(draft_tokens),
+        static_cast<std::uint32_t>(min_draft_tokens), draft_policy);
     try {
       provider_options.Parse();
-      if (!draft_policy.empty() &&
-          speculative_config.backend != server::TextSpeculativeBackend::kDFlash)
-        throw std::invalid_argument("--draft-policy requires DFlash2");
-      speculative_config.dflash_policy =
-          server::speculative::ParseDFlashDraftPolicy(draft_policy);
+      CheckSpeculativeFlags(registry, speculative_backend, speculative_flags);
     } catch (const std::invalid_argument& exception) {
       std::cerr << "Error: " << exception.what() << '\n';
       return 2;
@@ -1609,16 +1610,13 @@ int RunServe(std::span<const char* const> args) {
         : speculative_config.backend == server::TextSpeculativeBackend::kMtp
             ? mtp_model_path
             : dflash_model_path;
-    speculative_config.max_draft_tokens =
-        static_cast<std::uint32_t>(draft_tokens);
-    speculative_config.min_draft_tokens =
-        static_cast<std::uint32_t>(min_draft_tokens);
     if (model.empty()) {
       std::cerr << "Error: --model <PATH> is required\n";
       return 2;
     }
     std::string err;
     ModelLoadLog load_log("text", model);
+    server::TextSpeculativeConfig loaded_speculative = speculative_config;
     backend = std::make_shared<server::InferenceBackend>();
     // The disk snapshot tier is walked during load, which can take minutes.
     // Answer 503 `Loading model` on the port meanwhile, as llama-server does,
@@ -1653,11 +1651,15 @@ int RunServe(std::span<const char* const> args) {
             .model_artifact_fingerprint = {},
         },
         vision_model_path, cache_ram, enable_yarn,
-        [&provider_options](
-            const ext::TextProvider& provider, const core::GgufReader&,
+        [&](const ext::TextProvider& provider, const core::GgufReader&,
             ext::TextRunnerOptions* options, std::string* error) {
           try {
             options->provider_options = provider_options.Select(provider.id);
+            ApplySpeculativeSettings(
+                ResolveSpeculativeSettings(provider, speculative_backend,
+                                           speculative_flags),
+                &options->speculative);
+            loaded_speculative = options->speculative;
             return true;
           } catch (const std::invalid_argument& exception) {
             *error = exception.what();
@@ -1721,7 +1723,10 @@ int RunServe(std::span<const char* const> args) {
         "model=" + backend->model_id() +
         " sessions=" + std::to_string(session_count) + " context_tokens=" +
         std::to_string(backend->max_context()) + " speculative=" + speculation +
-        " draft_limit=" + std::to_string(speculative_config.max_draft_tokens) +
+        " draft_limit=" + std::to_string(loaded_speculative.max_draft_tokens) +
+        (loaded_speculative.draft_policy.empty()
+             ? std::string()
+             : " draft_policy=" + loaded_speculative.draft_policy) +
         " disk_cache=" + (cache_disk_directory.empty() ? "off" : "enabled") +
         served_names);
   }

src/cli/speculative_settings.cpp

@@ -0,0 +1,119 @@
+#include "src/cli/speculative_settings.hpp"
+
+#include <algorithm>
+#include <stdexcept>
+#include <vector>
+
+namespace gufo::cli {
+namespace {
+
+/// The host's settings for a backend no provider declares.
+ext::TextSpeculativeDefaults HostDefaults(std::string_view backend) {
+  ext::TextSpeculativeDefaults defaults{.backend = std::string(backend)};
+  if (backend == "dflash2") {
+    defaults.draft_policies = {"fixed", "adaptive"};
+  }
+  return defaults;
+}
+
+ext::TextSpeculativeDefaults DefaultsFor(const ext::TextProvider& provider,
+                                         std::string_view backend) {
+  const auto found = std::ranges::find(provider.speculative_defaults, backend,
+                                       &ext::TextSpeculativeDefaults::backend);
+  return found != provider.speculative_defaults.end() ? *found
+                                                      : HostDefaults(backend);
+}
+
+bool Accepts(const ext::TextSpeculativeDefaults& defaults,
+             std::string_view policy) {
+  return std::ranges::find(defaults.draft_policies, policy) !=
+         defaults.draft_policies.end();
+}
+
+void CheckRange(std::uint32_t max_draft_tokens,
+                std::uint32_t min_draft_tokens) {
+  if (min_draft_tokens > max_draft_tokens) {
+    throw std::invalid_argument("min-draft-tokens cannot exceed draft-tokens");
+  }
+}
+
+}  // namespace
+
+SpeculativeFlags SpeculativeFlags::Given(const ArgParser& parser,
+                                         std::uint32_t max_draft_tokens,
+                                         std::uint32_t min_draft_tokens,
+                                         const std::string& draft_policy) {
+  SpeculativeFlags flags;
+  if (parser.WasSupplied("--draft-tokens")) {
+    flags.max_draft_tokens = max_draft_tokens;
+  }
+  if (parser.WasSupplied("--min-draft-tokens")) {
+    flags.min_draft_tokens = min_draft_tokens;
+  }
+  if (parser.WasSupplied("--draft-policy")) {
+    flags.draft_policy = draft_policy;
+  }
+  return flags;
+}
+
+void CheckSpeculativeFlags(const ext::Registry& registry,
+                           std::string_view backend,
+                           const SpeculativeFlags& flags) {
+  if (flags.max_draft_tokens && flags.min_draft_tokens) {
+    CheckRange(*flags.max_draft_tokens, *flags.min_draft_tokens);
+  }
+  if (!flags.draft_policy) {
+    return;
+  }
+  const auto& policy = *flags.draft_policy;
+  if (backend.empty()) {
+    throw std::invalid_argument("--draft-policy requires --speculative");
+  }
+  bool accepted = Accepts(HostDefaults(backend), policy);
+  for (const auto& info : registry.Providers()) {
+    const auto* provider = ext::FindTextProvider(info.id);
+    if (info.available && provider != nullptr) {
+      accepted = accepted || Accepts(DefaultsFor(*provider, backend), policy);
+    }
+  }
+  if (!accepted) {
+    throw std::invalid_argument("--draft-policy " + policy +
+                                " is not supported by --speculative " +
+                                std::string(backend));
+  }
+}
+
+SpeculativeSettings ResolveSpeculativeSettings(
+    const ext::TextProvider& provider, std::string_view backend,
+    const SpeculativeFlags& flags) {
+  const auto defaults =
+      backend.empty() ? HostDefaults(backend) : DefaultsFor(provider, backend);
+  SpeculativeSettings settings{
+      .max_draft_tokens =
+          flags.max_draft_tokens.value_or(defaults.max_draft_tokens),
+      .min_draft_tokens =
+          flags.min_draft_tokens.value_or(defaults.min_draft_tokens),
+      .draft_policy = flags.draft_policy.value_or(defaults.draft_policy),
+  };
+  if (flags.draft_policy &&
+      (backend.empty() || !Accepts(defaults, *flags.draft_policy))) {
+    throw std::invalid_argument(
+        "provider " + provider.id + " does not support --draft-policy " +
+        *flags.draft_policy + " for --speculative " + std::string(backend));
+  }
+  CheckRange(settings.max_draft_tokens, settings.min_draft_tokens);
+  return settings;
+}
+
+void ApplySpeculativeSettings(const SpeculativeSettings& settings,
+                              ext::TextSpeculativeConfig* config) {
+  config->max_draft_tokens = settings.max_draft_tokens;
+  config->min_draft_tokens = settings.min_draft_tokens;
+  config->draft_policy = settings.draft_policy;
+  if (config->backend == ext::TextSpeculativeBackend::kDFlash) {
+    config->dflash_policy =
+        ext::ParseTextDFlashDraftPolicy(settings.draft_policy);
+  }
+}
+
+}  // namespace gufo::cli

src/cli/speculative_settings.hpp

@@ -0,0 +1,56 @@
+#ifndef GUFO_CLI_SPECULATIVE_SETTINGS_HPP_
+#define GUFO_CLI_SPECULATIVE_SETTINGS_HPP_
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <string_view>
+
+#include "gufo/ext/registry.hpp"
+#include "gufo/ext/text.hpp"
+#include "src/cli/arg_parser.hpp"
+
+namespace gufo::cli {
+
+/// The draft settings given on the command line; unset ones take the selected
+/// provider's `TextSpeculativeDefaults`.
+struct SpeculativeFlags {
+  std::optional<std::uint32_t> max_draft_tokens;
+  std::optional<std::uint32_t> min_draft_tokens;
+  std::optional<std::string> draft_policy;
+
+  /// The values of `--draft-tokens`, `--min-draft-tokens` and
+  /// `--draft-policy` that `parser` was given.
+  static SpeculativeFlags Given(const ArgParser& parser,
+                                std::uint32_t max_draft_tokens,
+                                std::uint32_t min_draft_tokens,
+                                const std::string& draft_policy);
+};
+
+struct SpeculativeSettings {
+  std::uint32_t max_draft_tokens{7};
+  std::uint32_t min_draft_tokens{1};
+  std::string draft_policy;
+};
+
+/// Before a model is selected: throws std::invalid_argument when the flags
+/// cannot hold for any available provider (a policy without `backend`, or one
+/// no provider accepts for it; a minimum above an explicit maximum).
+void CheckSpeculativeFlags(const ext::Registry& registry,
+                           std::string_view backend,
+                           const SpeculativeFlags& flags);
+
+/// The settings `provider` runs `backend` with. Throws std::invalid_argument
+/// when it does not accept the given policy or the minimum exceeds the limit.
+/// An empty backend (no speculation) resolves to the host's values.
+SpeculativeSettings ResolveSpeculativeSettings(
+    const ext::TextProvider& provider, std::string_view backend,
+    const SpeculativeFlags& flags);
+
+/// Stores `settings` in `config`, including the DFlash2 policy they name.
+void ApplySpeculativeSettings(const SpeculativeSettings& settings,
+                              ext::TextSpeculativeConfig* config);
+
+}  // namespace gufo::cli
+
+#endif  // GUFO_CLI_SPECULATIVE_SETTINGS_HPP_

src/ext/text.hpp

@@ -654,6 +654,9 @@ struct TextSpeculativeConfig {
   std::uint32_t max_draft_tokens{7};
   std::uint32_t min_draft_tokens{1};
   TextDFlashDraftPolicy dflash_policy{TextDFlashDraftPolicy::kAdaptive};
+  /// The resolved `--draft-policy` (see `TextSpeculativeDefaults`); empty when
+  /// the backend has no policy.
+  std::string draft_policy{};
 };
 
 struct TextDiskCacheConfig {
@@ -849,6 +852,23 @@ using TextChatDriver =
     int (*)(const TextPromptOptions& options, const core::GgufReader& reader,
             std::chrono::steady_clock::time_point load_start);
 
+/// A provider's draft settings for one speculative backend. The commands
+/// resolve each setting as the explicit flag, else this default, else the
+/// host's (7 draft tokens, minimum 1, no policy; DFlash2 accepts `fixed` and
+/// `adaptive`) when the provider declares nothing for the backend.
+struct TextSpeculativeDefaults {
+  /// The `--speculative` value: `mtp`, `dflash2` or `dspark`.
+  std::string backend;
+  /// `--draft-tokens` when it is not given.
+  std::uint32_t max_draft_tokens{7};
+  /// `--min-draft-tokens` when it is not given.
+  std::uint32_t min_draft_tokens{1};
+  /// `--draft-policy` when it is not given; may be empty.
+  std::string draft_policy;
+  /// The `--draft-policy` values this backend accepts; empty accepts none.
+  std::vector<std::string> draft_policies;
+};
+
 struct TextProvider {
   std::string id;
   TextProbe probe = nullptr;
@@ -864,6 +884,8 @@ struct TextProvider {
   /// their `data` and reserved `size_bytes`; the others never probe such a
   /// file.
   bool accepts_unsized_tensors = false;
+  /// Draft settings per speculative backend the provider supports.
+  std::vector<TextSpeculativeDefaults> speculative_defaults{};
 };
 
 void RegisterTextProvider(TextProvider provider);

src/models/deepseek_v4_flash/provider.cpp

@@ -38,6 +38,9 @@ void deepseek_v4_flash_provider_register(gufo::ext::Registry& registry) {
       .chat_driver = &gufo::models::deepseek_v4_flash::RunDeepSeekChat,
       .sampling_defaults =
           &gufo::models::deepseek_v4_flash::DeepSeekSamplingDefaults,
+      .speculative_defaults = {{.backend = "dspark",
+                                .max_draft_tokens = 7,
+                                .min_draft_tokens = 1}},
   });
 #endif
 }

src/models/qwen/provider.cpp

@@ -36,6 +36,11 @@ void qwen_provider_register(gufo::ext::Registry& registry) {
       .prompt_driver = &gufo::models::qwen::RunQwenPrompt,
       .chat_driver = &gufo::models::qwen::RunQwenChat,
       .sampling_defaults = &gufo::models::qwen::QwenSamplingDefaults,
+      .speculative_defaults = {{.backend = "dflash2",
+                                .max_draft_tokens = 7,
+                                .min_draft_tokens = 1,
+                                .draft_policy = "adaptive",
+                                .draft_policies = {"fixed", "adaptive"}}},
   });
 #endif
 }

src/models/qwen38_flash_next/provider.cpp

@@ -39,6 +39,9 @@ void qwen38_flash_next_provider_register(gufo::ext::Registry& registry) {
       .chat_driver = &gufo::models::qwen38_flash_next::RunFlashNextChat,
       .sampling_defaults =
           &gufo::models::qwen38_flash_next::FlashNextSamplingDefaults,
+      .speculative_defaults = {{.backend = "mtp",
+                                .max_draft_tokens = 7,
+                                .min_draft_tokens = 1}},
   });
 #endif
 }

tests/cli/serve_provider_options_test.cpp

@@ -15,6 +15,7 @@
 #include "src/cli/arg_parser.hpp"
 #include "src/cli/provider_options.hpp"
 #include "src/cli/serve/inference_backend.hpp"
+#include "src/cli/speculative_settings.hpp"
 #include "src/core/platform/os_info.hpp"
 #include "tests/fixtures/gguf_builder.hpp"
 
@@ -22,11 +23,13 @@ namespace {
 
 int created = 0;
 gufo::ext::ParsedOptions received;
+gufo::ext::TextSpeculativeConfig received_speculative;
 
 std::shared_ptr<gufo::ext::TextModelRunner> Create(
     const gufo::ext::TextRunnerOptions& options, std::string* error) {
   ++created;
   received = options.provider_options;
+  received_speculative = options.speculative;
   *error = "fixture runner";
   return nullptr;
 }
@@ -54,6 +57,11 @@ void gufo::ext::RegisterBuiltExtensions(gufo::ext::Registry& registry) {
                    "serve-text";
           },
       .create = &Create,
+      .speculative_defaults = {{.backend = "dflash2",
+                                .max_draft_tokens = 15,
+                                .min_draft_tokens = 1,
+                                .draft_policy = "fixed",
+                                .draft_policies = {"fixed", "adaptive"}}},
   });
 }
 
@@ -70,25 +78,39 @@ bool Expect(bool condition, std::string_view what) {
 bool Load(const gufo::ext::Registry& registry, const std::string& model,
           std::vector<const char*> flags, std::string* error) {
   gufo::cli::ArgParser parser("gufo serve");
-  std::size_t draft_tokens = 7;
+  std::uint32_t draft_tokens = 7;
+  std::uint32_t min_draft_tokens = 1;
+  std::string draft_policy;
   parser.AddOption("-d", "--draft-tokens", "N", "", "Speculative",
                    &draft_tokens);
+  parser.AddOption("", "--min-draft-tokens", "N", "", "Speculative",
+                   &min_draft_tokens);
+  parser.AddOption("", "--draft-policy", "NAME", "", "Speculative",
+                   &draft_policy);
   gufo::cli::ProviderOptions provider_options(registry, "serve");
   provider_options.Register(parser);
   if (!parser.Parse(flags, error)) {
     return false;
   }
   provider_options.Parse();
+  const auto speculative = gufo::cli::SpeculativeFlags::Given(
+      parser, draft_tokens, min_draft_tokens, draft_policy);
   created = 0;
   received.clear();
   gufo::server::InferenceBackend backend;
   return backend.load(
-      model, error, 0, 1, {}, {}, {}, {}, {}, {}, false,
+      model, error, 0, 1, {}, {},
+      {.backend = gufo::ext::TextSpeculativeBackend::kDFlash}, {}, {}, {},
+      false,
       [&](const gufo::ext::TextProvider& provider,
           const gufo::core::GgufReader&, gufo::ext::TextRunnerOptions* options,
           std::string* message) {
         try {
           options->provider_options = provider_options.Select(provider.id);
+          gufo::cli::ApplySpeculativeSettings(
+              gufo::cli::ResolveSpeculativeSettings(provider, "dflash2",
+                                                    speculative),
+              &options->speculative);
           return true;
         } catch (const std::invalid_argument& exception) {
           *message = exception.what();
@@ -132,6 +154,24 @@ int main() {
                        "provider serve-text does not support --other-only for "
                        "serve",
                "another provider's option must fail before the runner");
+  ok &= Expect(!Load(registry, model, {}, &error) && created == 1 &&
+                   received_speculative.max_draft_tokens == 15 &&
+                   received_speculative.draft_policy == "fixed" &&
+                   received_speculative.dflash_policy ==
+                       gufo::ext::TextDFlashDraftPolicy::kFixed,
+               "the factory must receive the provider's draft defaults");
+  ok &= Expect(!Load(registry, model, {"-d", "9", "--draft-policy", "adaptive"},
+                     &error) &&
+                   created == 1 && received_speculative.max_draft_tokens == 9 &&
+                   received_speculative.draft_policy == "adaptive" &&
+                   received_speculative.dflash_policy ==
+                       gufo::ext::TextDFlashDraftPolicy::kAdaptive,
+               "explicit draft settings must win over the defaults");
+  ok &= Expect(!Load(registry, model, {"--draft-policy", "unknown"}, &error) &&
+                   created == 0 &&
+                   error.find("does not support --draft-policy unknown") !=
+                       std::string::npos,
+               "an unsupported policy must fail before the runner");
   std::filesystem::remove(path);
   return ok ? 0 : 1;
 }

tests/cli/text_cli_dispatch_test.cpp

@@ -11,6 +11,7 @@
 #include <sstream>
 #include <string>
 #include <string_view>
+#include <utility>
 #include <vector>
 
 #include "src/cli/bench/bench.hpp"
@@ -23,6 +24,9 @@ namespace {
 std::string dispatched;
 gufo::sampling::SamplingConfig dispatched_sampling;
 gufo::ext::ParsedOptions dispatched_options;
+std::size_t dispatched_draft_tokens = 0;
+std::size_t dispatched_min_draft_tokens = 0;
+std::string dispatched_draft_policy;
 bool invalid_defaults = false;
 
 std::optional<gufo::sampling::TextSamplingDefaults> SamplingDefaults(
@@ -37,8 +41,13 @@ std::optional<gufo::sampling::TextSamplingDefaults> SamplingDefaults(
   return defaults;
 }
 
+template<typename Options>
 int Dispatch(std::string_view command, const gufo::core::GgufReader& reader,
-             const gufo::ext::ParsedOptions& options) {
+             const Options& driver_options) {
+  const auto& options = driver_options.provider_options;
+  dispatched_draft_tokens = driver_options.draft_tokens;
+  dispatched_min_draft_tokens = driver_options.min_draft_tokens;
+  dispatched_draft_policy = driver_options.draft_policy;
   const std::string_view architecture =
       reader.GetMetadataString("general.architecture").value_or("");
   if (architecture != "test-text" && architecture != "other-text" &&
@@ -102,21 +111,26 @@ void gufo::ext::RegisterBuiltExtensions(gufo::ext::Registry& registry) {
           +[](const TextBenchOptions& options,
               const std::shared_ptr<const core::GgufReader>& reader,
               std::chrono::steady_clock::time_point) {
-            return Dispatch("bench", *reader, options.provider_options);
+            return Dispatch("bench", *reader, options);
           },
       .prompt_driver =
           +[](const TextPromptOptions& options, const core::GgufReader& reader,
               std::chrono::steady_clock::time_point) {
             dispatched_sampling = options.sampling;
-            return Dispatch("prompt", reader, options.provider_options);
+            return Dispatch("prompt", reader, options);
           },
       .chat_driver =
           +[](const TextPromptOptions& options, const core::GgufReader& reader,
               std::chrono::steady_clock::time_point) {
             dispatched_sampling = options.sampling;
-            return Dispatch("chat", reader, options.provider_options);
+            return Dispatch("chat", reader, options);
           },
       .sampling_defaults = &SamplingDefaults,
+      .speculative_defaults = {{.backend = "mtp",
+                                .max_draft_tokens = 15,
+                                .min_draft_tokens = 2,
+                                .draft_policy = "calibrated",
+                                .draft_policies = {"calibrated", "fixed"}}},
   });
   registry.AddTextProvider({
       .id = "other-text",
@@ -128,17 +142,17 @@ void gufo::ext::RegisterBuiltExtensions(gufo::ext::Registry& registry) {
           +[](const TextBenchOptions& options,
               const std::shared_ptr<const core::GgufReader>& reader,
               std::chrono::steady_clock::time_point) {
-            return Dispatch("bench", *reader, options.provider_options);
+            return Dispatch("bench", *reader, options);
           },
       .prompt_driver =
           +[](const TextPromptOptions& options, const core::GgufReader& reader,
               std::chrono::steady_clock::time_point) {
-            return Dispatch("prompt", reader, options.provider_options);
+            return Dispatch("prompt", reader, options);
           },
       .chat_driver =
           +[](const TextPromptOptions& options, const core::GgufReader& reader,
               std::chrono::steady_clock::time_point) {
-            return Dispatch("chat", reader, options.provider_options);
+            return Dispatch("chat", reader, options);
           },
   });
   // Owns a tensor layout core cannot size; the others never see such a file.
@@ -152,19 +166,19 @@ void gufo::ext::RegisterBuiltExtensions(gufo::ext::Registry& registry) {
           +[](const TextBenchOptions& options,
               const std::shared_ptr<const core::GgufReader>& reader,
               std::chrono::steady_clock::time_point) {
-            return Dispatch("bench", *reader, options.provider_options);
+            return Dispatch("bench", *reader, options);
           },
       .prompt_driver =
           +[](const TextPromptOptions& options, const core::GgufReader& reader,
               std::chrono::steady_clock::time_point) {
             dispatched_sampling = options.sampling;
-            return Dispatch("prompt", reader, options.provider_options);
+            return Dispatch("prompt", reader, options);
           },
       .chat_driver =
           +[](const TextPromptOptions& options, const core::GgufReader& reader,
               std::chrono::steady_clock::time_point) {
             dispatched_sampling = options.sampling;
-            return Dispatch("chat", reader, options.provider_options);
+            return Dispatch("chat", reader, options);
           },
       .sampling_defaults = &SamplingDefaults,
       .accepts_unsized_tensors = true,
@@ -247,6 +261,97 @@ bool CheckProviderOptions(std::string_view command, const std::string& model,
   return true;
 }
 
+// Draft settings: the explicit flag, else the provider's default for the
+// backend, else the host's 7, 1 and no policy.
+bool CheckSpeculativeDefaults(std::string_view command,
+                              const std::string& model,
+                              const std::string& other) {
+  struct Case {
+    const std::string* model;
+    std::vector<const char*> flags;
+    std::size_t max;
+    std::size_t min;
+    std::string_view policy;
+    const char* what;
+  };
+  const std::vector<Case> cases = {
+      {&model,
+       {"--speculative", "mtp", "--mtp-model", "x"},
+       15,
+       2,
+       "calibrated",
+       "the defaults"},
+      {&model,
+       {"--speculative", "mtp", "--mtp-model", "x", "--draft-tokens", "9"},
+       9,
+       2,
+       "calibrated",
+       "an explicit limit"},
+      {&model,
+       {"--speculative", "mtp", "--mtp-model", "x", "--min-draft-tokens", "1",
+        "--draft-policy", "fixed"},
+       15,
+       1,
+       "fixed",
+       "an explicit minimum and policy"},
+      {&model, {}, 7, 1, "", "no backend"},
+      {&model,
+       {"--speculative", "dspark", "--dspark-model", "x"},
+       7,
+       1,
+       "",
+       "a backend the provider does not declare"},
+      {&other,
+       {"--speculative", "mtp", "--mtp-model", "x"},
+       7,
+       1,
+       "",
+       "a provider declaring none"},
+      {&other,
+       {"--speculative", "dflash2", "--dflash-model", "x", "--draft-policy",
+        "fixed"},
+       7,
+       1,
+       "fixed",
+       "the host's DFlash2 policies"},
+  };
+  for (const auto& entry : cases) {
+    if (Run(command, *entry.model, entry.flags) != 42 ||
+        dispatched_draft_tokens != entry.max ||
+        dispatched_min_draft_tokens != entry.min ||
+        dispatched_draft_policy != entry.policy) {
+      return Fail(command,
+                  std::string("wrong draft settings for ") + entry.what);
+    }
+  }
+  const std::vector<std::pair<const std::string*, std::vector<const char*>>>
+      rejected = {
+          // No provider accepts these; rejected without a probe.
+          {&model, {"--draft-policy", "fixed"}},
+          {&model,
+           {"--speculative", "mtp", "--mtp-model", "x", "--draft-policy",
+            "unknown"}},
+          {&model,
+           {"--speculative", "mtp", "--mtp-model", "x", "--draft-tokens", "2",
+            "--min-draft-tokens", "3"}},
+          // Accepted by another provider or above the resolved limit.
+          {&other,
+           {"--speculative", "mtp", "--mtp-model", "x", "--draft-policy",
+            "fixed"}},
+          {&model,
+           {"--speculative", "mtp", "--mtp-model", "x", "--min-draft-tokens",
+            "16"}},
+          {&model,
+           {"--speculative", "mtp", "--mtp-model", "x", "--draft-tokens", "1"}},
+      };
+  for (const auto& [selected, flags] : rejected) {
+    if (Run(command, *selected, flags) != 2 || !dispatched.empty()) {
+      return Fail(command, "must reject impossible draft settings");
+    }
+  }
+  return true;
+}
+
 bool CheckHelp(std::string_view command) {
   std::ostringstream captured;
   auto* const previous = std::cout.rdbuf(captured.rdbuf());
@@ -377,6 +482,7 @@ int main(int argc, char** argv) {
       command == "bench" || CheckSamplingDefaults(command, model.c_str());
   const bool options_ok =
       CheckProviderOptions(command, model, other_path.string()) &&
+      CheckSpeculativeDefaults(command, model, other_path.string()) &&
       CheckHelp(command);
   const bool unsized_ok = CheckUnsizedTensors(command);
   std::filesystem::remove(path);