feat: configurable embedding model + degraded-mode reporting for knowledge search (fixes #234)

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

docs/features/knowledge-embeddings.md

@@ -0,0 +1,69 @@
+# Knowledge Embedding Model Configuration
+
+Knowledge / RAG search uses vector **embeddings**. The default embedding model is
+`text-embedding-3-small`. Many OpenAI keys or projects do not have access to that
+model (organization tier, model allowlist, or regional restrictions), which caused
+knowledge search to silently degrade to local substring matching.
+
+PraisonAIUI now makes this failure mode configurable and observable.
+
+## Configure the embedding model
+
+Set an accessible model via environment variable:
+
+```bash
+export AIUI_EMBEDDING_MODEL=text-embedding-ada-002
+```
+
+`OPENAI_EMBEDDING_MODEL` is also honoured as a fallback if `AIUI_EMBEDDING_MODEL`
+is not set.
+
+## Fallback chain
+
+The OpenAI-compatible `/v1/embeddings` endpoint tries the requested (or default)
+model first, then each model in the fallback chain if the primary returns a
+`model_not_found` / 403 access error:
+
+```bash
+export AIUI_EMBEDDING_FALLBACK_MODELS="text-embedding-ada-002,text-embedding-3-small"
+```
+
+If every candidate is inaccessible, the endpoint returns `503` with an actionable
+`model_not_found` error instead of a generic `500`.
+
+## Observing degraded search
+
+`GET /api/knowledge/status` reports embedding availability:
+
+```json
+{
+  "total": 12,
+  "backend": "SDKKnowledgeManager",
+  "status": "degraded",
+  "embedding": {"available": false, "error": "Project does not have access to model `text-embedding-3-small`"},
+  "warnings": ["Embedding model unavailable; knowledge search degraded to local text matching. Set AIUI_EMBEDDING_MODEL to an accessible model."]
+}
+```
+
+`POST /api/knowledge/search` indicates whether results came from vector search or a
+degraded local fallback:
+
+```json
+{
+  "results": [],
+  "count": 0,
+  "search_mode": "fallback",
+  "warning": "Vector search unavailable: Project does not have access to model text-embedding-3-small. Set AIUI_EMBEDDING_MODEL=text-embedding-ada-002 or enable embedding models in your provider project."
+}
+```
+
+A dashboard can show a warning banner whenever `search_mode == "fallback"` or
+`embedding.available == false`.
+
+## Model reference
+
+| Model | Notes |
+|-------|-------|
+| `text-embedding-3-small` | Default — best cost/quality when available |
+| `text-embedding-3-large` | Higher quality — may also be restricted |
+| `text-embedding-ada-002` | Legacy — widely available on older projects |

src/praisonaiui/features/knowledge.py

@@ -27,6 +27,17 @@ from ._base import BaseFeatureProtocol
 logger = logging.getLogger(__name__)
 
 
+def _is_embedding_unavailable(err: BaseException) -> bool:
+    """Detect an embedding model access / not-found failure."""
+    msg = str(err).lower()
+    return (
+        "does not have access" in msg
+        or "model_not_found" in msg
+        or "does not exist" in msg
+        or "notfounderror" in msg
+    )
+
+
 # ── Knowledge Protocol ───────────────────────────────────────────────
 
 
@@ -95,6 +106,10 @@ class KnowledgeProtocol(ABC):
         """Health check for this backend."""
         return {"status": "ok", "provider": self.__class__.__name__}
 
+    def last_search_meta(self) -> Dict[str, Any]:
+        """Metadata about the most recent search (mode, error)."""
+        return {"mode": "vector", "error": None}
+
 
 # ── Simple Knowledge Manager (Default, no deps) ─────────────────────
 
@@ -171,8 +186,12 @@ class SimpleKnowledgeManager(KnowledgeProtocol):
             "status": "ok",
             "provider": "SimpleKnowledgeManager",
             "total_entries": len(self._entries),
+            "embedding": {"available": False, "reason": "no_vector_backend"},
         }
 
+    def last_search_meta(self) -> Dict[str, Any]:
+        return {"mode": "local", "error": None}
+
 
 # ── SDK Knowledge Manager (wraps praisonaiagents.knowledge) ──────────
 
@@ -191,6 +210,9 @@ class SDKKnowledgeManager(KnowledgeProtocol):
         self._sdk_knowledge = None
         self._sdk_probed = False  # True once we've verified SDK actually works
         self._local_index: Dict[str, Dict[str, Any]] = {}
+        self._embedding_unavailable = False
+        self._embedding_error: Optional[str] = None
+        self._last_search_meta: Dict[str, Any] = {"mode": "vector", "error": None}
 
     def _get_sdk_knowledge(self) -> Any:
         """Lazy-init SDK Knowledge with health probe.
@@ -242,6 +264,10 @@ class SDKKnowledgeManager(KnowledgeProtocol):
                         "using local in-memory index"
                     )
                     self._sdk_knowledge = None
+                elif _is_embedding_unavailable(e):
+                    self._embedding_unavailable = True
+                    self._embedding_error = err_msg
+                    logger.warning("Embedding model unavailable: %s", e)
                 else:
                     # Other errors (e.g. empty DB) are fine — SDK works
                     pass
@@ -294,6 +320,7 @@ class SDKKnowledgeManager(KnowledgeProtocol):
             try:
                 results = sdk.search(query, limit=limit, user_id="praisonaiui")
                 if isinstance(results, list) and results:
+                    self._last_search_meta = {"mode": "vector", "error": None}
                     return [
                         {
                             "id": str(r.get("id", i)),
@@ -305,6 +332,9 @@ class SDKKnowledgeManager(KnowledgeProtocol):
                     ]
             except BaseException as e:
                 logger.warning("SDK knowledge search failed: %s; falling back to local", e)
+                if _is_embedding_unavailable(e):
+                    self._embedding_unavailable = True
+                    self._embedding_error = str(e)
 
         # Fallback to local text search
         query_lower = query.lower()
@@ -314,8 +344,26 @@ class SDKKnowledgeManager(KnowledgeProtocol):
                 results.append(e)
             if len(results) >= limit:
                 break
+
+        if self._embedding_unavailable:
+            self._last_search_meta = {
+                "mode": "fallback",
+                "error": (
+                    "Vector search unavailable: "
+                    f"{self._embedding_error or 'embedding model not accessible'}. "
+                    "Set AIUI_EMBEDDING_MODEL=text-embedding-ada-002 or enable embedding "
+                    "models in your provider project."
+                ),
+            }
+        elif sdk is not None:
+            self._last_search_meta = {"mode": "vector", "error": None}
+        else:
+            self._last_search_meta = {"mode": "local", "error": None}
         return results
 
+    def last_search_meta(self) -> Dict[str, Any]:
+        return dict(self._last_search_meta)
+
     def add_file(
         self,
         file_path: str,
@@ -453,11 +501,25 @@ class SDKKnowledgeManager(KnowledgeProtocol):
 
     def health(self) -> Dict[str, Any]:
         sdk = self._get_sdk_knowledge()
+        embedding_available = sdk is not None and not self._embedding_unavailable
+        warnings: List[str] = []
+        embedding: Dict[str, Any] = {"available": embedding_available}
+        if self._embedding_unavailable:
+            embedding["error"] = self._embedding_error
+            warnings.append(
+                "Embedding model unavailable; knowledge search degraded to local text "
+                "matching. Set AIUI_EMBEDDING_MODEL to an accessible model."
+            )
+        status = "ok"
+        if sdk is None or self._embedding_unavailable:
+            status = "degraded"
         return {
-            "status": "ok" if sdk is not None else "degraded",
+            "status": status,
             "provider": "SDKKnowledgeManager",
             "sdk_available": sdk is not None,
             "total_indexed": len(self._local_index),
+            "embedding": embedding,
+            "warnings": warnings,
         }
 
 
@@ -626,7 +688,15 @@ class KnowledgeFeature(BaseFeatureProtocol):
             query=query,
             limit=body.get("limit", 10),
         )
-        return JSONResponse({"results": results, "count": len(results)})
+        meta = mgr.last_search_meta()
+        return JSONResponse(
+            {
+                "results": results,
+                "count": len(results),
+                "search_mode": meta.get("mode", "unknown"),
+                "warning": meta.get("error"),
+            }
+        )
 
     async def _add_file(self, request: Request) -> JSONResponse:
         """POST /api/knowledge/add-file — ingest a file."""
@@ -675,6 +745,8 @@ class KnowledgeFeature(BaseFeatureProtocol):
                 "files": stats.get("files", 0),
                 "backend": h.get("provider", "unknown"),
                 "status": h.get("status", "ok"),
+                "embedding": h.get("embedding", {"available": True}),
+                "warnings": h.get("warnings", []),
             }
         )
 

src/praisonaiui/features/openai_api.py

@@ -6,6 +6,7 @@ Wraps praisonai.capabilities functions as HTTP endpoints.
 
 from __future__ import annotations
 
+import os
 import time
 import uuid
 from dataclasses import asdict
@@ -17,6 +18,35 @@ from starlette.routing import Route
 
 from ._base import BaseFeatureProtocol
 
+
+def _default_embedding_model() -> str:
+    """Resolve the default embedding model from environment."""
+    return os.environ.get(
+        "AIUI_EMBEDDING_MODEL",
+        os.environ.get("OPENAI_EMBEDDING_MODEL", "text-embedding-3-small"),
+    )
+
+
+def _embedding_fallback_models() -> list[str]:
+    """Resolve the embedding fallback chain from environment."""
+    raw = os.environ.get(
+        "AIUI_EMBEDDING_FALLBACK_MODELS",
+        "text-embedding-ada-002,text-embedding-3-small",
+    )
+    return [m.strip() for m in raw.split(",") if m.strip()]
+
+
+def _is_model_not_found(err: BaseException) -> bool:
+    """Heuristically detect embedding model access / not-found errors."""
+    msg = str(err).lower()
+    return (
+        "does not have access" in msg
+        or "model_not_found" in msg
+        or "does not exist" in msg
+        or "notfounderror" in msg
+    )
+
+
 # Lazy import capabilities
 _capabilities = None
 
@@ -280,29 +310,57 @@ class OpenAIAPIFeature(BaseFeatureProtocol):
 
         body = await request.json()
         input_text = body.get("input", "")
-        model = body.get("model", "text-embedding-3-small")
+        model = body.get("model") or _default_embedding_model()
 
-        try:
-            result = await caps.aembed(
-                input=input_text,
-                model=model,
-            )
+        candidates = [model]
+        for fallback in _embedding_fallback_models():
+            if fallback not in candidates:
+                candidates.append(fallback)
 
-            data = _result_to_dict(result)
+        last_err: BaseException | None = None
+        for candidate in candidates:
+            try:
+                result = await caps.aembed(
+                    input=input_text,
+                    model=candidate,
+                )
+                data = _result_to_dict(result)
+                return JSONResponse(
+                    {
+                        "object": "list",
+                        "data": [
+                            {
+                                "object": "embedding",
+                                "index": 0,
+                                "embedding": data.get("embedding", []),
+                            }
+                        ],
+                        "model": candidate,
+                        "usage": data.get("usage", {"prompt_tokens": 0, "total_tokens": 0}),
+                    }
+                )
+            except Exception as e:
+                last_err = e
+                if not _is_model_not_found(e):
+                    break
+
+        if last_err is not None and _is_model_not_found(last_err):
             return JSONResponse(
                 {
-                    "object": "list",
-                    "data": [
-                        {"object": "embedding", "index": 0, "embedding": data.get("embedding", [])}
-                    ],
-                    "model": model,
-                    "usage": data.get("usage", {"prompt_tokens": 0, "total_tokens": 0}),
-                }
-            )
-        except Exception as e:
-            return JSONResponse(
-                {"error": {"message": str(e), "type": "api_error"}}, status_code=500
+                    "error": {
+                        "message": (
+                            f"Embedding model unavailable: {last_err}. "
+                            "Set AIUI_EMBEDDING_MODEL to a model your project can access "
+                            "(e.g. text-embedding-ada-002)."
+                        ),
+                        "type": "model_not_found",
+                    }
+                },
+                status_code=503,
             )
+        return JSONResponse(
+            {"error": {"message": str(last_err), "type": "api_error"}}, status_code=500
+        )
 
     async def _image_generate(self, request: Request) -> JSONResponse:
         """POST /v1/images/generations — Generate images."""

tests/unit/test_knowledge_embeddings.py

@@ -0,0 +1,194 @@
+"""Unit tests for knowledge embedding fallback + degraded-mode reporting."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from starlette.testclient import TestClient
+
+from praisonaiui.features import _features, register_feature
+from praisonaiui.features import knowledge as knowledge_mod
+from praisonaiui.features.knowledge import (
+    KnowledgeFeature,
+    SDKKnowledgeManager,
+    SimpleKnowledgeManager,
+    _is_embedding_unavailable,
+)
+from praisonaiui.features.openai_api import (
+    OpenAIAPIFeature,
+    _default_embedding_model,
+    _embedding_fallback_models,
+    _is_model_not_found,
+)
+from praisonaiui.server import create_app
+
+
+class _FakeSDK:
+    def __init__(self, search_error: BaseException | None = None) -> None:
+        self._search_error = search_error
+
+    def search(self, query, limit=1, user_id=None):
+        if query == "__probe__":
+            return []
+        if self._search_error is not None:
+            raise self._search_error
+        return []
+
+
+def test_is_model_not_found_detects_403():
+    assert _is_model_not_found(Exception("Project does not have access to model x"))
+    assert _is_model_not_found(Exception("litellm.NotFoundError: nope"))
+    assert not _is_model_not_found(Exception("timeout"))
+
+
+def test_is_embedding_unavailable_detects_403():
+    assert _is_embedding_unavailable(Exception("model_not_found"))
+    assert not _is_embedding_unavailable(Exception("connection reset"))
+
+
+def test_default_embedding_model_env(monkeypatch):
+    monkeypatch.delenv("AIUI_EMBEDDING_MODEL", raising=False)
+    monkeypatch.delenv("OPENAI_EMBEDDING_MODEL", raising=False)
+    assert _default_embedding_model() == "text-embedding-3-small"
+    monkeypatch.setenv("AIUI_EMBEDDING_MODEL", "text-embedding-ada-002")
+    assert _default_embedding_model() == "text-embedding-ada-002"
+
+
+def test_fallback_models_env(monkeypatch):
+    monkeypatch.setenv("AIUI_EMBEDDING_FALLBACK_MODELS", "a, b ,,c")
+    assert _embedding_fallback_models() == ["a", "b", "c"]
+
+
+def test_simple_manager_reports_local_mode():
+    mgr = SimpleKnowledgeManager()
+    mgr.store("hello world")
+    mgr.search("hello")
+    assert mgr.last_search_meta()["mode"] == "local"
+    assert mgr.health()["embedding"]["available"] is False
+
+
+def test_sdk_search_reports_fallback_on_embedding_error():
+    mgr = SDKKnowledgeManager()
+    err = Exception("Project does not have access to model text-embedding-3-small")
+    mgr._sdk_knowledge = _FakeSDK(search_error=err)
+    mgr._sdk_probed = True
+
+    mgr.store("PraisonAIUI is a YAML-driven UI generator.")
+    mgr.search("documentation generator")
+
+    meta = mgr.last_search_meta()
+    assert meta["mode"] == "fallback"
+    assert "text-embedding" in meta["error"].lower()
+
+    health = mgr.health()
+    assert health["embedding"]["available"] is False
+    assert health["status"] == "degraded"
+    assert health["warnings"]
+
+
+def test_sdk_search_vector_mode_when_healthy():
+    class _OkSDK:
+        def search(self, query, limit=1, user_id=None):
+            if query == "__probe__":
+                return []
+            return [{"id": "1", "memory": "hit", "score": 0.9}]
+
+    mgr = SDKKnowledgeManager()
+    mgr._sdk_knowledge = _OkSDK()
+    mgr._sdk_probed = True
+    results = mgr.search("anything")
+    assert results
+    assert mgr.last_search_meta()["mode"] == "vector"
+
+
+def test_probe_flags_embedding_unavailable():
+    mgr = SDKKnowledgeManager()
+
+    class _Kn:
+        def search(self, query, limit=1):
+            raise Exception("model_not_found: no access")
+
+    with patch("praisonaiagents.knowledge.Knowledge", return_value=_Kn(), create=True):
+        mgr._get_sdk_knowledge()
+    assert mgr._embedding_unavailable is True
+
+
+@pytest.fixture
+def client():
+    saved = dict(_features)
+    saved_mgr = knowledge_mod._knowledge_manager
+    _features.clear()
+    register_feature(KnowledgeFeature())
+    register_feature(OpenAIAPIFeature())
+    yield TestClient(create_app())
+    _features.clear()
+    _features.update(saved)
+    knowledge_mod._knowledge_manager = saved_mgr
+
+
+def test_knowledge_search_reports_fallback(client):
+    mgr = SDKKnowledgeManager()
+    err = Exception("Project does not have access to model text-embedding-3-small")
+    mgr._sdk_knowledge = _FakeSDK(search_error=err)
+    mgr._sdk_probed = True
+    mgr.store("PraisonAIUI is a YAML-driven documentation site generator.")
+    knowledge_mod._knowledge_manager = mgr
+
+    r = client.post("/api/knowledge/search", json={"query": "documentation generator YAML"})
+    assert r.status_code == 200
+    body = r.json()
+    assert body["search_mode"] == "fallback"
+    assert body["warning"]
+    assert "embedding" in body["warning"].lower()
+
+
+def test_knowledge_status_reports_embedding(client):
+    mgr = SDKKnowledgeManager()
+    mgr._sdk_knowledge = _FakeSDK()
+    mgr._sdk_probed = True
+    mgr._embedding_unavailable = True
+    mgr._embedding_error = "no access"
+    knowledge_mod._knowledge_manager = mgr
+
+    r = client.get("/api/knowledge/status")
+    assert r.status_code == 200
+    body = r.json()
+    assert body["embedding"]["available"] is False
+    assert body["warnings"]
+
+
+def test_embeddings_endpoint_fallback_models(client):
+    call_models: list[str] = []
+
+    async def _aembed(input, model):
+        call_models.append(model)
+        if model == "text-embedding-3-small":
+            raise Exception("Project does not have access to model text-embedding-3-small")
+        return {"embedding": [0.1, 0.2], "usage": {"prompt_tokens": 1, "total_tokens": 1}}
+
+    fake_caps = type("Caps", (), {"aembed": staticmethod(AsyncMock(side_effect=_aembed))})
+    with patch(
+        "praisonaiui.features.openai_api._get_capabilities", return_value=fake_caps
+    ):
+        r = client.post(
+            "/v1/embeddings",
+            json={"input": "test", "model": "text-embedding-3-small"},
+        )
+    assert r.status_code == 200
+    body = r.json()
+    assert body["model"] == "text-embedding-ada-002"
+    assert call_models[0] == "text-embedding-3-small"
+
+
+def test_embeddings_endpoint_all_fail_returns_503(client):
+    async def _aembed(input, model):
+        raise Exception("model_not_found: no access to any embedding model")
+
+    fake_caps = type("Caps", (), {"aembed": staticmethod(AsyncMock(side_effect=_aembed))})
+    with patch(
+        "praisonaiui.features.openai_api._get_capabilities", return_value=fake_caps
+    ):
+        r = client.post("/v1/embeddings", json={"input": "test"})
+    assert r.status_code == 503
+    assert r.json()["error"]["type"] == "model_not_found"