langgenius/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-07-19 16:44:00 -04:00
Code Issues 775 Packages Projects Releases 10 Wiki Activity

Merge remote-tracking branch 'origin/main' into codex/upgrade-next-16-3-preview

Browse Source 
# Conflicts:
#	web/service/client.spec.ts
#	web/service/client.ts
This commit is contained in:
copilot-swe-agent[bot]
2026-07-02 08:19:07 +00:00
committed by GitHub
parent b243c000d9 451faa0e68
commit b995846d51
1136 changed files with 59117 additions and 17568 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.agents/skills/e2e-cucumber-playwright/SKILL.md
+4 -2
View File
@@ -5,7 +5,7 @@ description: Write, update, or review Dify end-to-end tests under `e2e/` that us
# Dify E2E Cucumber + Playwright
Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS.md`](../../../e2e/AGENTS.md) as the canonical guide for local architecture and conventions, then apply Playwright/Cucumber best practices only where they fit the current suite.
Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS.md`](../../../e2e/AGENTS.md) as the canonical package guide for local architecture and conventions, then read any feature-scoped `AGENTS.md` that owns the target area. Apply Playwright/Cucumber best practices only where they fit the current suite.
## Scope
@@ -25,6 +25,8 @@ Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS
4. Read [`references/cucumber-best-practices.md`](references/cucumber-best-practices.md) only when scenario wording, step granularity, tags, or expression design are involved.
5. Re-check official Playwright or Cucumber docs with the available documentation tools before introducing a new framework pattern.
Keep this skill focused on Cucumber, Playwright, and package-level E2E guidance. Put feature-specific conventions in the owning feature's `AGENTS.md` instead of adding them here.
## Local Rules
- `e2e/` uses Cucumber for scenarios and Playwright as the browser layer.
@@ -59,7 +61,7 @@ Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS
- Do not use `waitForTimeout`, manual polling, or raw visibility checks when a locator action or retrying assertion already expresses the behavior.
5. Validate narrowly.
- Run the narrowest tagged scenario or flow that exercises the change.
- Run `pnpm -C e2e check`.
- Run `vpr lint --fix --quiet` from the repository root and `pnpm -C e2e type-check`.
- Broaden verification only when the change affects hooks, tags, setup, or shared step semantics.
## Review Checklist
.github/CODEOWNERS
+3
View File
@@ -26,6 +26,9 @@
/cli/ @GareArc
/.github/workflows/cli-tests.yml @GareArc
# E2E
/e2e/ @lyzno1
# Backend (default owner, more specific rules below will override)
/api/ @QuantumGhost
.github/labeler.yml
+6
View File
@@ -7,3 +7,9 @@ web:
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- '.nvmrc'
e2e:
- changed-files:
- any-glob-to-any-file:
- 'e2e/**'
- '.github/workflows/web-e2e.yml'
.github/workflows/autofix.yml
+13 -1
View File
@@ -51,6 +51,18 @@ jobs:
with:
files: |
api/**
- name: Check frontend contract inputs
if: github.event_name != 'merge_group'
id: frontend-contract-changes
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
with:
files: |
api/**
packages/contracts/openapi-ts.api.config.ts
packages/contracts/package.json
packages/contracts/openapi/**
pnpm-lock.yaml
pnpm-workspace.yaml
- name: Check dify-agent inputs
if: github.event_name != 'merge_group'
id: dify-agent-changes
@@ -143,7 +155,7 @@ jobs:
uv run dev/generate_swagger_markdown_docs.py --swagger-dir ../packages/contracts/openapi --markdown-dir openapi/markdown --keep-swagger-json
- name: Generate frontend contracts
if: github.event_name != 'merge_group' && steps.api-changes.outputs.any_changed == 'true'
if: github.event_name != 'merge_group' && steps.frontend-contract-changes.outputs.any_changed == 'true'
run: pnpm --dir packages/contracts gen-api-contract-from-openapi
- name: ESLint autofix
.github/workflows/style.yml
+14
View File
@@ -29,6 +29,8 @@ jobs:
with:
files: |
api/**
scripts/check_no_new_getattr.py
scripts/ast_grep_rules/no_new_getattr.yml
.github/workflows/style.yml
- name: Setup UV and Python
@@ -51,6 +53,18 @@ jobs:
if: steps.changed-files.outputs.any_changed == 'true'
run: uv run --project api --dev python api/dev/lint_response_contracts.py --fail-on-mismatch
- name: Fetch merge target ref for getattr guard
if: steps.changed-files.outputs.any_changed == 'true'
run: git fetch --no-tags --depth=1 origin +refs/heads/main:refs/remotes/origin/main
- name: Bind merge target branch for getattr guard
if: steps.changed-files.outputs.any_changed == 'true'
run: git show-ref --verify --quiet refs/heads/main || git branch main origin/main
- name: Run No New Getattr Guard
if: steps.changed-files.outputs.any_changed == 'true'
run: uv run --project api python scripts/check_no_new_getattr.py --mode ci --merge-target main
- name: Run Type Checks
if: steps.changed-files.outputs.any_changed == 'true'
env:
api/clients/agent_backend/__init__.py
+4
View File
@@ -32,6 +32,8 @@ from clients.agent_backend.factory import create_agent_backend_run_client
from clients.agent_backend.fake_client import FakeAgentBackendRunClient, FakeAgentBackendScenario
from clients.agent_backend.request_builder import (
AGENT_SOUL_PROMPT_LAYER_ID,
DIFY_CONFIG_LAYER_ID,
DIFY_CORE_TOOLS_LAYER_ID,
DIFY_EXECUTION_CONTEXT_LAYER_ID,
DIFY_KNOWLEDGE_BASE_LAYER_ID,
DIFY_PLUGIN_TOOLS_LAYER_ID,
@@ -47,6 +49,8 @@ from clients.agent_backend.request_builder import (
__all__ = [
"AGENT_SOUL_PROMPT_LAYER_ID",
"DIFY_CONFIG_LAYER_ID",
"DIFY_CORE_TOOLS_LAYER_ID",
"DIFY_EXECUTION_CONTEXT_LAYER_ID",
"DIFY_KNOWLEDGE_BASE_LAYER_ID",
"DIFY_PLUGIN_TOOLS_LAYER_ID",
api/clients/agent_backend/request_builder.py
+73 -10
View File
@@ -20,6 +20,8 @@ from agenton.layers import ExitIntent
from agenton_collections.layers.plain import PLAIN_PROMPT_LAYER_TYPE_ID, PromptLayerConfig
from agenton_collections.layers.pydantic_ai import PYDANTIC_AI_HISTORY_LAYER_TYPE_ID
from dify_agent.layers.ask_human import DIFY_ASK_HUMAN_LAYER_TYPE_ID, DifyAskHumanLayerConfig
from dify_agent.layers.config import DIFY_CONFIG_LAYER_TYPE_ID, DifyConfigLayerConfig
from dify_agent.layers.dify_core_tools import DIFY_CORE_TOOLS_LAYER_TYPE_ID, DifyCoreToolsLayerConfig
from dify_agent.layers.dify_plugin import (
DIFY_PLUGIN_LLM_LAYER_TYPE_ID,
DIFY_PLUGIN_TOOLS_LAYER_TYPE_ID,
@@ -54,8 +56,10 @@ WORKFLOW_NODE_JOB_PROMPT_LAYER_ID = "workflow_node_job_prompt"
WORKFLOW_USER_PROMPT_LAYER_ID = "workflow_user_prompt"
AGENT_APP_USER_PROMPT_LAYER_ID = "agent_app_user_prompt"
DIFY_EXECUTION_CONTEXT_LAYER_ID = "execution_context"
DIFY_CONFIG_LAYER_ID = "config"
DIFY_DRIVE_LAYER_ID = "drive"
DIFY_PLUGIN_TOOLS_LAYER_ID = "tools"
DIFY_CORE_TOOLS_LAYER_ID = "core_tools"
DIFY_KNOWLEDGE_BASE_LAYER_ID = "knowledge"
DIFY_ASK_HUMAN_LAYER_ID = "ask_human"
DIFY_SHELL_LAYER_ID = "shell"
@@ -86,6 +90,10 @@ def _drive_layer_deps() -> dict[str, str]:
return {"shell": DIFY_SHELL_LAYER_ID}
def _config_layer_deps() -> dict[str, str]:
return {"shell": DIFY_SHELL_LAYER_ID}
def _shell_config_with_drive_ref(
shell_config: DifyShellLayerConfig | None,
drive_config: DifyDriveLayerConfig | None,
@@ -159,7 +167,9 @@ class AgentBackendWorkflowNodeRunInput(BaseModel):
idempotency_key: str | None = None
output: AgentBackendOutputConfig | None = None
tools: DifyPluginToolsLayerConfig | None = None
core_tools: DifyCoreToolsLayerConfig | None = None
knowledge: DifyKnowledgeBaseLayerConfig | None = None
config_layer_config: DifyConfigLayerConfig | None = None
# Drive Skills & Files declaration (dify.drive) — an index the agent pulls
# through the back proxy, never inline content; see AGENT_DRIVE_MANIFEST_ENABLED.
drive_config: DifyDriveLayerConfig | None = None
@@ -206,7 +216,9 @@ class AgentBackendAgentAppRunInput(BaseModel):
idempotency_key: str | None = None
output: AgentBackendOutputConfig | None = None
tools: DifyPluginToolsLayerConfig | None = None
core_tools: DifyCoreToolsLayerConfig | None = None
knowledge: DifyKnowledgeBaseLayerConfig | None = None
config_layer_config: DifyConfigLayerConfig | None = None
# Drive Skills & Files declaration (dify.drive) — an index the agent pulls
# through the back proxy, never inline content; see AGENT_DRIVE_MANIFEST_ENABLED.
drive_config: DifyDriveLayerConfig | None = None
@@ -243,8 +255,9 @@ class AgentBackendRunRequestBuilder:
Layer graph: optional Agent Soul system prompt → user prompt →
execution context → optional history (multi-turn) → LLM → optional
plugin tools / knowledge search → optional structured output. Mirrors the workflow-node
layer ordering minus the workflow-job / previous-node prompt.
plugin-direct tools / core-routed tools / knowledge search →
optional structured output. Mirrors the workflow-node layer ordering
minus the workflow-job / previous-node prompt.
"""
layers: list[RunLayerSpec] = []
if run_input.agent_soul_prompt:
@@ -274,11 +287,13 @@ class AgentBackendRunRequestBuilder:
]
)
include_shell = run_input.include_shell or run_input.drive_config is not None
include_shell = (
run_input.include_shell or run_input.config_layer_config is not None or run_input.drive_config is not None
)
if include_shell:
# Sandboxed bash workspace (dify.shell). It enters before drive so
# drive can materialize mentioned targets with `dify-agent drive pull`
# in the same shell-visible filesystem used by model commands.
# Sandboxed bash workspace (dify.shell). It enters before config/drive
# so eager pulls materialize content in the same filesystem used by
# model commands.
layers.append(
RunLayerSpec(
name=DIFY_SHELL_LAYER_ID,
@@ -289,6 +304,17 @@ class AgentBackendRunRequestBuilder:
)
)
if run_input.config_layer_config is not None:
layers.append(
RunLayerSpec(
name=DIFY_CONFIG_LAYER_ID,
type=DIFY_CONFIG_LAYER_TYPE_ID,
deps=_config_layer_deps(),
metadata=run_input.metadata,
config=run_input.config_layer_config,
)
)
if run_input.drive_config is not None:
# Drive Skills & Files declaration (dify.drive): the catalog plus
# prompt-mentioned entries eagerly pulled through the shell layer.
@@ -338,6 +364,17 @@ class AgentBackendRunRequestBuilder:
)
)
if run_input.core_tools is not None and run_input.core_tools.tools:
layers.append(
RunLayerSpec(
name=DIFY_CORE_TOOLS_LAYER_ID,
type=DIFY_CORE_TOOLS_LAYER_TYPE_ID,
deps={"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID},
metadata=run_input.metadata,
config=run_input.core_tools,
)
)
if run_input.knowledge is not None and run_input.knowledge.sets:
layers.append(
RunLayerSpec(
@@ -403,7 +440,8 @@ class AgentBackendRunRequestBuilder:
non-plugin layer graph that produced the snapshot. Plugin layers
(``dify.plugin.llm``, ``dify.plugin.tools``) are excluded from both the
composition and the snapshot before submission because their configs
require credentials that are not persisted between runs.
may carry credentials or runtime-only declarations that are not
persisted between runs.
"""
if not runtime_layer_specs:
raise ValueError(
@@ -436,8 +474,9 @@ class AgentBackendRunRequestBuilder:
"""Build a workflow Agent Node run request without defining another wire schema.
Layer graph mirrors the workflow surface: prompts → execution context →
optional drive/history → LLM → optional plugin tools / knowledge search
→ optional auxiliary layers such as ask_human, shell, and structured output.
optional drive/history → LLM → optional plugin-direct tools /
core-routed tools / knowledge search → optional auxiliary layers such
as ask_human, shell, and structured output.
"""
layers: list[RunLayerSpec] = []
if run_input.agent_soul_prompt:
@@ -473,7 +512,9 @@ class AgentBackendRunRequestBuilder:
]
)
include_shell = run_input.include_shell or run_input.drive_config is not None
include_shell = (
run_input.include_shell or run_input.config_layer_config is not None or run_input.drive_config is not None
)
if include_shell:
# Sandboxed bash workspace (dify.shell). It enters before drive so
# drive can materialize mentioned targets with `dify-agent drive pull`
@@ -488,6 +529,17 @@ class AgentBackendRunRequestBuilder:
)
)
if run_input.config_layer_config is not None:
layers.append(
RunLayerSpec(
name=DIFY_CONFIG_LAYER_ID,
type=DIFY_CONFIG_LAYER_TYPE_ID,
deps=_config_layer_deps(),
metadata=run_input.metadata,
config=run_input.config_layer_config,
)
)
if run_input.drive_config is not None:
# Drive Skills & Files declaration (dify.drive): the catalog plus
# prompt-mentioned entries eagerly pulled through the shell layer.
@@ -539,6 +591,17 @@ class AgentBackendRunRequestBuilder:
)
)
if run_input.core_tools is not None and run_input.core_tools.tools:
layers.append(
RunLayerSpec(
name=DIFY_CORE_TOOLS_LAYER_ID,
type=DIFY_CORE_TOOLS_LAYER_TYPE_ID,
deps={"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID},
metadata=run_input.metadata,
config=run_input.core_tools,
)
)
if run_input.knowledge is not None and run_input.knowledge.sets:
layers.append(
RunLayerSpec(
api/commands/__init__.py
+2 -1
View File
@@ -22,7 +22,7 @@ from .plugin import (
setup_system_trigger_oauth_client,
transform_datasource_credentials,
)
from .rbac import migrate_member_roles_to_rbac
from .rbac import migrate_dataset_permissions_to_rbac, migrate_member_roles_to_rbac
from .retention import (
archive_workflow_runs,
archive_workflow_runs_plan,
@@ -76,6 +76,7 @@ __all__ = [
"legacy_model_types",
"migrate_annotation_vector_database",
"migrate_data_for_plugin",
"migrate_dataset_permissions_to_rbac",
"migrate_knowledge_vector_database",
"migrate_member_roles_to_rbac",
"migrate_oss",
api/commands/data_migrate.py
+2
View File
@@ -7,6 +7,7 @@ from typing import cast
import click
from commands.rbac import migrate_dataset_permissions_to_rbac
from extensions.ext_database import db
from graphon.model_runtime.entities.model_entities import ModelType
from services.legacy_model_type_migration import (
@@ -177,3 +178,4 @@ def legacy_model_types(
data_migrate.add_command(legacy_model_types)
data_migrate.add_command(migrate_dataset_permissions_to_rbac)
api/commands/rbac.py
+226 -2
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import json
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -8,8 +9,11 @@ from sqlalchemy import select
from configs import dify_config
from core.db.session_factory import session_factory
from models import TenantAccountJoin, TenantAccountRole
from services.enterprise.rbac_service import ListOption, RBACService
from core.rbac import RBACResourceWhitelistScope
from models import Dataset, DatasetPermission, DatasetPermissionEnum, TenantAccountJoin, TenantAccountRole
from services.enterprise.rbac_service import ListOption, RBACService, ReplaceMemberBindings, ReplaceUserAccessPolicies
_RBAC_DEFAULT_ACCESS_POLICY_ID = "default"
_LEGACY_ROLE_TO_BUILTIN_TAG = {
TenantAccountRole.OWNER.value: "owner",
@@ -258,3 +262,223 @@ def migrate_member_roles_to_rbac(
fg="green",
)
)
def _dataset_permission_enum(permission: DatasetPermissionEnum | str | None) -> DatasetPermissionEnum:
if permission is None:
return DatasetPermissionEnum.ONLY_ME
try:
return DatasetPermissionEnum(permission)
except ValueError as exc:
raise ValueError(f"Unsupported legacy dataset permission: {permission}") from exc
def _rbac_dataset_scope_for_legacy_permission(permission: DatasetPermissionEnum) -> RBACResourceWhitelistScope:
if permission is DatasetPermissionEnum.ALL_TEAM:
return RBACResourceWhitelistScope.ALL
if permission in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.PARTIAL_TEAM}:
return RBACResourceWhitelistScope.SPECIFIC
raise ValueError(f"Unsupported legacy dataset permission: {permission}")
def _emit_dataset_permission_migration_event(payload: dict[str, object]) -> None:
click.echo(json.dumps(payload, sort_keys=True))
@click.command(
"rbac-migrate-dataset-permissions",
help=(
"Migrate legacy dataset permission scopes and partial members into RBAC dataset access bindings. "
"Side effect: replacing each dataset whitelist clears existing per-user policy bindings; "
"the command then recreates legacy partial-member default bindings."
),
)
@click.option("--tenant-id", help="Only migrate datasets in a single workspace.")
@click.option("--dataset-id", help="Only migrate a single dataset.")
@click.option("--batch-size", default=500, show_default=True, type=click.IntRange(min=1))
@click.option(
"--dry-run/--apply",
default=True,
show_default=True,
help="Preview the migration without writing RBAC bindings. Use --apply to write changes.",
)
def migrate_dataset_permissions_to_rbac(
tenant_id: str | None,
dataset_id: str | None,
batch_size: int,
dry_run: bool,
) -> None:
"""Backfill RBAC dataset access config from legacy `Dataset.permission`.
Legacy mapping:
- all_team_members -> RBAC dataset whitelist scope "all"
- partial_members -> RBAC dataset whitelist scope "specific" plus each partial member gets the
virtual default policy
- only_me -> RBAC dataset whitelist scope "specific" with no member policy bindings
The command replaces each dataset's RBAC whitelist scope first. RBAC clears
existing per-user policy bindings during that replace, then this command
recreates the legacy partial-member default bindings. Re-running it is
therefore idempotent for a dataset's current legacy configuration.
"""
click.echo(click.style("Starting RBAC dataset permission migration.", fg="green"))
scanned_count = 0
scope_migrated_count = 0
user_policy_migrated_count = 0
partial_dataset_count = 0
last_dataset_id: str | None = None
while True:
with session_factory.create_session() as session:
stmt = (
select(Dataset.id, Dataset.tenant_id, Dataset.permission, Dataset.created_by)
.order_by(Dataset.id.asc())
.limit(batch_size)
)
if tenant_id:
stmt = stmt.where(Dataset.tenant_id == tenant_id)
if dataset_id:
stmt = stmt.where(Dataset.id == dataset_id)
if last_dataset_id:
stmt = stmt.where(Dataset.id > last_dataset_id)
dataset_rows = list(session.execute(stmt).all())
if not dataset_rows:
break
dataset_ids = [str(row.id) for row in dataset_rows]
partial_members_by_dataset_id: dict[str, list[str]] = {item: [] for item in dataset_ids}
permission_rows = session.execute(
select(DatasetPermission.dataset_id, DatasetPermission.account_id).where(
DatasetPermission.dataset_id.in_(dataset_ids)
)
).all()
for row in permission_rows:
partial_members_by_dataset_id[str(row.dataset_id)].append(str(row.account_id))
for dataset in dataset_rows:
workspace_id = str(dataset.tenant_id)
current_dataset_id = str(dataset.id)
operator_account_id = str(dataset.created_by)
permission_value = _dataset_permission_enum(dataset.permission)
scope = _rbac_dataset_scope_for_legacy_permission(permission_value)
partial_member_ids = sorted(set(partial_members_by_dataset_id[current_dataset_id]))
should_bind_partial_members = permission_value is DatasetPermissionEnum.PARTIAL_TEAM
click.echo(
f"tenant={workspace_id} dataset={current_dataset_id} "
f"operator={operator_account_id} "
f"legacy_permission={permission_value} -> rbac_scope={scope} "
f"partial_members={len(partial_member_ids) if should_bind_partial_members else 0}"
)
scanned_count += 1
replace_whitelist_payload = ReplaceMemberBindings(scope=scope)
if dry_run:
_emit_dataset_permission_migration_event(
{
"event": "dataset_permission_migration_proposed_change",
"action": "replace_whitelist",
"dry_run": True,
"tenant_id": workspace_id,
"dataset_id": current_dataset_id,
"operator_account_id": operator_account_id,
"before": {
"legacy_dataset_permission": permission_value.value,
"legacy_partial_member_ids": partial_member_ids if should_bind_partial_members else [],
},
"after": {
"rbac_whitelist_scope": scope.value,
},
"call": {
"method": "RBACService.DatasetAccess.replace_whitelist",
"kwargs": {
"tenant_id": workspace_id,
"account_id": operator_account_id,
"dataset_id": current_dataset_id,
"payload": replace_whitelist_payload.model_dump(mode="json"),
},
},
}
)
if not dry_run:
RBACService.DatasetAccess.replace_whitelist(
tenant_id=workspace_id,
account_id=operator_account_id,
dataset_id=current_dataset_id,
payload=replace_whitelist_payload,
)
scope_migrated_count += 1
if should_bind_partial_members:
partial_dataset_count += 1
for member_account_id in partial_member_ids:
replace_user_access_policies_payload = ReplaceUserAccessPolicies(
access_policy_ids=[_RBAC_DEFAULT_ACCESS_POLICY_ID],
)
if dry_run:
_emit_dataset_permission_migration_event(
{
"event": "dataset_permission_migration_proposed_change",
"action": "replace_user_access_policies",
"dry_run": True,
"tenant_id": workspace_id,
"dataset_id": current_dataset_id,
"operator_account_id": operator_account_id,
"target_account_id": member_account_id,
"before": {
"legacy_dataset_permission": permission_value.value,
"legacy_partial_member_id": member_account_id,
},
"after": {
"rbac_user_access_policy_ids": [_RBAC_DEFAULT_ACCESS_POLICY_ID],
},
"call": {
"method": "RBACService.DatasetAccess.replace_user_access_policies",
"kwargs": {
"tenant_id": workspace_id,
"account_id": operator_account_id,
"dataset_id": current_dataset_id,
"target_account_id": member_account_id,
"payload": replace_user_access_policies_payload.model_dump(mode="json"),
},
},
}
)
continue
RBACService.DatasetAccess.replace_user_access_policies(
tenant_id=workspace_id,
account_id=operator_account_id,
dataset_id=current_dataset_id,
target_account_id=member_account_id,
payload=replace_user_access_policies_payload,
)
user_policy_migrated_count += 1
last_dataset_id = dataset_ids[-1]
if dataset_id:
break
if scanned_count == 0:
click.echo(click.style("No datasets found for migration.", fg="yellow"))
return
if dry_run:
click.echo(
click.style(
f"Dry run completed. Scanned {scanned_count} datasets; "
f"{partial_dataset_count} partial-member datasets would be migrated.",
fg="yellow",
)
)
else:
click.echo(
click.style(
"RBAC dataset permission migration completed. "
f"Scanned {scanned_count} datasets, migrated {scope_migrated_count} scopes, "
f"wrote {user_policy_migrated_count} user default-policy bindings.",
fg="green",
)
)
api/constants/recommended_apps.json
+2 -2
View File
File diff suppressed because one or more lines are too long
api/controllers/console/__init__.py
+2
View File
@@ -54,6 +54,7 @@ from .app import (
agent_app_access,
agent_app_feature,
agent_app_sandbox,
agent_config_inspector,
agent_drive_inspector,
annotation,
app,
@@ -157,6 +158,7 @@ __all__ = [
"agent_app_feature",
"agent_app_sandbox",
"agent_composer",
"agent_config_inspector",
"agent_drive_inspector",
"agent_providers",
"agent_roster",
api/controllers/console/agent/composer.py
+3 -1
View File
@@ -57,8 +57,9 @@ class WorkflowAgentComposerApi(Resource):
@login_required
@account_initialization_required
@get_app_model(mode=[AppMode.WORKFLOW, AppMode.ADVANCED_CHAT])
@with_current_user_id
@with_current_tenant_id
def get(self, tenant_id: str, app_model: App, node_id: str):
def get(self, tenant_id: str, account_id: str, app_model: App, node_id: str):
query = WorkflowAgentComposerQuery.model_validate(request.args.to_dict(flat=True))
return dump_response(
WorkflowAgentComposerResponse,
@@ -66,6 +67,7 @@ class WorkflowAgentComposerApi(Resource):
tenant_id=tenant_id,
app_id=app_model.id,
node_id=node_id,
account_id=account_id,
snapshot_id=query.snapshot_id,
),
)
api/controllers/console/agent/roster.py
+26 -10
View File
@@ -39,9 +39,11 @@ from controllers.console.wraps import (
)
from extensions.ext_database import db
from fields.agent_fields import (
AgentConfigDraftSummaryResponse,
AgentConfigSnapshotDetailResponse,
AgentConfigSnapshotListResponse,
AgentConfigSnapshotRestoreResponse,
AgentConfigSnapshotSummaryResponse,
AgentInviteOptionsResponse,
AgentLogListResponse,
AgentLogMessageListResponse,
@@ -50,11 +52,13 @@ from fields.agent_fields import (
AgentRosterListResponse,
AgentStatisticSummaryEnvelopeResponse,
)
from fields.base import ResponseModel
from libs.datetime_utils import parse_time_range
from libs.helper import dump_response
from libs.login import login_required
from models import Account
from models.agent import Agent, AgentStatus
from models.agent_config_entities import AgentSoulConfig
from models.enums import ApiTokenType
from models.model import ApiToken, App, IconType
from services.agent.composer_service import AgentComposerService
@@ -248,33 +252,37 @@ class AgentAppDetailWithSite(GenericAppDetailWithSite):
backing_app_id: str | None = None
hidden_app_backed: bool = False
debug_conversation_id: str | None = None
debug_conversation_has_messages: bool = False
debug_conversation_message_count: int = 0
role: str | None = None
active_config_is_published: bool = False
class AgentDebugConversationRefreshResponse(BaseModel):
debug_conversation_id: str
debug_conversation_has_messages: bool = False
debug_conversation_message_count: int = 0
class AgentPublishPayload(BaseModel):
version_note: str | None = Field(default=None, description="Optional note for this published Agent version")
class AgentPublishResponse(BaseModel):
class AgentPublishResponse(ResponseModel):
result: str
active_config_snapshot_id: str
active_config_snapshot: dict[str, object] | None = None
draft: dict[str, object] | None = None
active_config_snapshot: AgentConfigSnapshotSummaryResponse | None = None
draft: AgentConfigDraftSummaryResponse | None = None
class AgentBuildDraftCheckoutPayload(BaseModel):
force: bool = Field(default=False, description="Overwrite the existing current-user build draft")
class AgentBuildDraftResponse(BaseModel):
class AgentBuildDraftResponse(ResponseModel):
variant: str
draft: dict[str, object]
agent_soul: dict[str, object]
draft: AgentConfigDraftSummaryResponse
agent_soul: AgentSoulConfig
class AgentBuildDraftApplyResponse(BaseModel):
@@ -372,11 +380,17 @@ def _serialize_agent_app_detail(app_model, *, current_user: Account, agent_id: s
payload["backing_app_id"] = roster_service.runtime_backing_app_id(agent)
payload["hidden_app_backed"] = bool(agent.backing_app_id and agent.backing_app_id != agent.app_id)
payload["id"] = agent.id
payload["debug_conversation_id"] = roster_service.get_or_create_agent_app_debug_conversation_id(
debug_conversation_id = roster_service.get_or_create_agent_app_debug_conversation_id(
tenant_id=app_model.tenant_id,
agent_id=agent.id,
account_id=current_user.id,
)
message_count = roster_service.count_agent_app_debug_conversation_messages(
conversation_id=debug_conversation_id,
)
payload["debug_conversation_id"] = debug_conversation_id
payload["debug_conversation_has_messages"] = message_count > 0
payload["debug_conversation_message_count"] = message_count
payload["role"] = agent.role or ""
payload["active_config_is_published"] = roster_service.active_config_is_published(
tenant_id=app_model.tenant_id,
@@ -635,9 +649,11 @@ class AgentDebugConversationRefreshApi(Resource):
agent_id=str(agent_id),
account_id=current_user.id,
)
return AgentDebugConversationRefreshResponse(debug_conversation_id=debug_conversation_id).model_dump(
mode="json"
)
return AgentDebugConversationRefreshResponse(
debug_conversation_id=debug_conversation_id,
debug_conversation_has_messages=False,
debug_conversation_message_count=0,
).model_dump(mode="json")
@console_ns.route("/agent/<uuid:agent_id>/publish")
api/controllers/console/app/agent_app_feature.py
+1 -2
View File
@@ -30,7 +30,6 @@ from controllers.console.wraps import (
)
from events.app_event import app_model_config_was_updated
from extensions.ext_database import db
from libs.helper import dump_response
from libs.login import login_required
from models import Account
from models.agent_config_entities import (
@@ -99,4 +98,4 @@ class AgentAppFeatureConfigResource(Resource):
app_model_config_was_updated.send(app_model, app_model_config=new_app_model_config)
return dump_response(SimpleResultResponse, {"result": "success"})
return SimpleResultResponse(result="success").model_dump(mode="json")
api/controllers/console/app/agent_config_inspector.py
+1249
View File
File diff suppressed because it is too large Load Diff
api/controllers/console/app/annotation.py
+76 -59
View File
@@ -1,9 +1,11 @@
from typing import Any, Literal
from uuid import UUID
from flask import abort, make_response, request
from flask import abort, request
from flask_restx import Resource
from pydantic import BaseModel, Field, TypeAdapter, field_validator
from sqlalchemy import select
from werkzeug.exceptions import NotFound
from controllers.common.errors import NoFileUploadedError, TooManyFilesError
from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
@@ -26,11 +28,14 @@ from fields.annotation_fields import (
AnnotationExportList,
AnnotationHitHistory,
AnnotationHitHistoryList,
AnnotationJobStatusDetailResponse,
AnnotationJobStatusResponse,
AnnotationList,
)
from fields.base import ResponseModel
from libs.helper import uuid_value
from libs.login import login_required
from libs.helper import dump_response, uuid_value
from libs.login import current_account_with_tenant, login_required
from models.model import App
from services.annotation_service import (
AppAnnotationService,
EnableAnnotationArgs,
@@ -38,6 +43,17 @@ from services.annotation_service import (
UpdateAnnotationSettingArgs,
UpsertAnnotationArgs,
)
from services.app_ref_service import AppRef, AppRefService
def _get_app_ref(app_id: str) -> AppRef:
_, current_tenant_id = current_account_with_tenant()
app = db.session.scalar(
select(App).where(App.id == app_id, App.tenant_id == current_tenant_id, App.status == "normal").limit(1)
)
if app is None:
raise NotFound("App not found")
return AppRefService.create_app_ref(app)
class AnnotationReplyPayload(BaseModel):
@@ -99,23 +115,23 @@ class AnnotationFilePayload(BaseModel):
return uuid_value(value)
class AnnotationJobStatusResponse(ResponseModel):
job_id: str | None = None
job_status: str | None = None
error_msg: str | None = None
record_count: int | None = None
class AnnotationEmbeddingModelResponse(ResponseModel):
class AnnotationSettingEmbeddingModelResponse(ResponseModel):
embedding_provider_name: str | None = None
embedding_model_name: str | None = None
class AnnotationSettingResponse(ResponseModel):
id: str | None = None
enabled: bool
id: str | None = None
score_threshold: float | None = None
embedding_model: AnnotationEmbeddingModelResponse | None = None
embedding_model: AnnotationSettingEmbeddingModelResponse | None = None
class AnnotationBatchImportResponse(ResponseModel):
job_id: str | None = None
job_status: str | None = None
record_count: int | None = None
error_msg: str | None = None
register_schema_models(
@@ -142,7 +158,10 @@ register_response_schema_models(
AnnotationHitHistory,
AnnotationHitHistoryList,
AnnotationJobStatusResponse,
AnnotationJobStatusDetailResponse,
AnnotationSettingEmbeddingModelResponse,
AnnotationSettingResponse,
AnnotationBatchImportResponse,
)
@@ -172,7 +191,7 @@ class AnnotationReplyActionApi(Resource):
result = AppAnnotationService.enable_app_annotation(enable_args, str(app_id))
case "disable":
result = AppAnnotationService.disable_app_annotation(str(app_id))
return result, 200
return dump_response(AnnotationJobStatusResponse, result), 200
@console_ns.route("/apps/<uuid:app_id>/annotation-setting")
@@ -193,7 +212,7 @@ class AppAnnotationSettingDetailApi(Resource):
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_VIEW_LAYOUT)
def get(self, app_id: UUID):
result = AppAnnotationService.get_app_annotation_setting_by_app_id(str(app_id))
return result, 200
return dump_response(AnnotationSettingResponse, result), 200
@console_ns.route("/apps/<uuid:app_id>/annotation-settings/<uuid:annotation_setting_id>")
@@ -218,7 +237,7 @@ class AppAnnotationSettingUpdateApi(Resource):
result = AppAnnotationService.update_app_annotation_setting(
str(app_id), annotation_setting_id_str, setting_args
)
return result, 200
return dump_response(AnnotationSettingResponse, result), 200
@console_ns.route("/apps/<uuid:app_id>/annotation-reply/<string:action>/status/<uuid:job_id>")
@@ -227,9 +246,7 @@ class AnnotationReplyActionStatusApi(Resource):
@console_ns.doc(description="Get status of annotation reply action job")
@console_ns.doc(params={"app_id": "Application ID", "job_id": "Job ID", "action": "Action type"})
@console_ns.response(
200,
"Job status retrieved successfully",
console_ns.models[AnnotationJobStatusResponse.__name__],
200, "Job status retrieved successfully", console_ns.models[AnnotationJobStatusDetailResponse.__name__]
)
@console_ns.response(403, "Insufficient permissions")
@setup_required
@@ -251,7 +268,9 @@ class AnnotationReplyActionStatusApi(Resource):
app_annotation_error_key = f"{action}_app_annotation_error_{job_id_str}"
error_msg = redis_client.get(app_annotation_error_key).decode()
return {"job_id": job_id_str, "job_status": job_status, "error_msg": error_msg}, 200
return AnnotationJobStatusDetailResponse(
job_id=job_id_str, job_status=job_status, error_msg=error_msg
).model_dump(mode="json"), 200
@console_ns.route("/apps/<uuid:app_id>/annotations")
@@ -275,14 +294,9 @@ class AnnotationApi(Resource):
annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id(str(app_id), page, limit, keyword)
annotation_models = TypeAdapter(list[Annotation]).validate_python(annotation_list, from_attributes=True)
response = AnnotationList(
data=annotation_models,
has_more=len(annotation_list) == limit,
limit=limit,
total=total,
page=page,
)
return response.model_dump(mode="json"), 200
return AnnotationList(
data=annotation_models, has_more=len(annotation_list) == limit, limit=limit, total=total, page=page
).model_dump(mode="json"), 200
@console_ns.doc("create_annotation")
@console_ns.doc(description="Create a new annotation for an app")
@@ -308,7 +322,7 @@ class AnnotationApi(Resource):
if args.question is not None:
upsert_args["question"] = args.question
annotation = AppAnnotationService.up_insert_app_annotation_from_message(upsert_args, str(app_id))
return Annotation.model_validate(annotation, from_attributes=True).model_dump(mode="json")
return dump_response(Annotation, annotation), 201
@setup_required
@login_required
@@ -330,7 +344,8 @@ class AnnotationApi(Resource):
"message": "annotation_ids are required if the parameter is provided.",
}, 400
AppAnnotationService.delete_app_annotations_in_batch(str(app_id), annotation_ids)
app_ref = _get_app_ref(str(app_id))
AppAnnotationService.delete_app_annotations_in_batch(app_ref, annotation_ids)
return "", 204
# If no annotation_ids are provided, handle clearing all annotations
else:
@@ -357,14 +372,14 @@ class AnnotationExportApi(Resource):
def get(self, app_id: UUID):
annotation_list = AppAnnotationService.export_annotation_list_by_app_id(str(app_id))
annotation_models = TypeAdapter(list[Annotation]).validate_python(annotation_list, from_attributes=True)
response_data = AnnotationExportList(data=annotation_models).model_dump(mode="json")
# Create response with secure headers for CSV export
response = make_response(response_data, 200)
response.headers["Content-Type"] = "application/json; charset=utf-8"
response.headers["X-Content-Type-Options"] = "nosniff"
return response
return (
AnnotationExportList(data=annotation_models).model_dump(mode="json"),
200,
{
"Content-Type": "application/json; charset=utf-8",
"X-Content-Type-Options": "nosniff",
},
)
@console_ns.route("/apps/<uuid:app_id>/annotations/<uuid:annotation_id>")
@@ -389,9 +404,9 @@ class AnnotationUpdateDeleteApi(Resource):
update_args["answer"] = args.answer
if args.question is not None:
update_args["question"] = args.question
annotation = AppAnnotationService.update_app_annotation_directly(
update_args, str(app_id), str(annotation_id), db.session
)
app_ref = _get_app_ref(str(app_id))
annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
annotation = AppAnnotationService.update_app_annotation_directly(update_args, annotation_ref, db.session)
return Annotation.model_validate(annotation, from_attributes=True).model_dump(mode="json")
@setup_required
@@ -401,7 +416,9 @@ class AnnotationUpdateDeleteApi(Resource):
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
@console_ns.response(204, "Annotation deleted successfully")
def delete(self, app_id: UUID, annotation_id: UUID):
AppAnnotationService.delete_app_annotation(str(app_id), str(annotation_id), db.session)
app_ref = _get_app_ref(str(app_id))
annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
AppAnnotationService.delete_app_annotation(annotation_ref, db.session)
return "", 204
@@ -411,9 +428,7 @@ class AnnotationBatchImportApi(Resource):
@console_ns.doc(description="Batch import annotations from CSV file with rate limiting and security checks")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.response(
200,
"Batch import started successfully",
console_ns.models[AnnotationJobStatusResponse.__name__],
200, "Batch import started successfully", console_ns.models[AnnotationBatchImportResponse.__name__]
)
@console_ns.response(403, "Insufficient permissions")
@console_ns.response(400, "No file uploaded or too many files")
@@ -460,7 +475,10 @@ class AnnotationBatchImportApi(Resource):
if file_size == 0:
raise ValueError("The uploaded file is empty")
return AppAnnotationService.batch_import_app_annotations(str(app_id), file)
return dump_response(
AnnotationBatchImportResponse,
AppAnnotationService.batch_import_app_annotations(str(app_id), file),
)
@console_ns.route("/apps/<uuid:app_id>/annotations/batch-import-status/<uuid:job_id>")
@@ -469,9 +487,7 @@ class AnnotationBatchImportStatusApi(Resource):
@console_ns.doc(description="Get status of batch import job")
@console_ns.doc(params={"app_id": "Application ID", "job_id": "Job ID"})
@console_ns.response(
200,
"Job status retrieved successfully",
console_ns.models[AnnotationJobStatusResponse.__name__],
200, "Job status retrieved successfully", console_ns.models[AnnotationJobStatusDetailResponse.__name__]
)
@console_ns.response(403, "Insufficient permissions")
@setup_required
@@ -491,7 +507,9 @@ class AnnotationBatchImportStatusApi(Resource):
indexing_error_msg_key = f"app_annotation_batch_import_error_msg_{str(job_id)}"
error_msg = redis_client.get(indexing_error_msg_key).decode()
return {"job_id": job_id, "job_status": job_status, "error_msg": error_msg}, 200
return AnnotationJobStatusDetailResponse(
job_id=str(job_id), job_status=job_status, error_msg=error_msg
).model_dump(mode="json"), 200
@console_ns.route("/apps/<uuid:app_id>/annotations/<uuid:annotation_id>/hit-histories")
@@ -514,17 +532,16 @@ class AnnotationHitHistoryListApi(Resource):
def get(self, app_id: UUID, annotation_id: UUID):
page = request.args.get("page", default=1, type=int)
limit = request.args.get("limit", default=20, type=int)
app_ref = _get_app_ref(str(app_id))
annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
annotation_hit_history_list, total = AppAnnotationService.get_annotation_hit_histories(
str(app_id), str(annotation_id), page, limit
annotation_ref,
page,
limit,
)
history_models = TypeAdapter(list[AnnotationHitHistory]).validate_python(
annotation_hit_history_list, from_attributes=True
)
response = AnnotationHitHistoryList(
data=history_models,
has_more=len(annotation_hit_history_list) == limit,
limit=limit,
total=total,
page=page,
)
return response.model_dump(mode="json")
return AnnotationHitHistoryList(
data=history_models, has_more=len(annotation_hit_history_list) == limit, limit=limit, total=total, page=page
).model_dump(mode="json")
api/controllers/console/app/app.py
+50 -49
View File
@@ -230,13 +230,10 @@ class AppTracePayload(BaseModel):
class AppTraceResponse(ResponseModel):
enabled: bool
enabled: bool = False
tracing_provider: str | None = None
type JSONValue = Any
class Tag(ResponseModel):
id: str
name: str
@@ -257,7 +254,7 @@ class WorkflowPartial(ResponseModel):
class ModelConfigPartial(ResponseModel):
model: JSONValue | None = Field(default=None, validation_alias=AliasChoices("model_dict", "model"))
model: Any | None = Field(default=None, validation_alias=AliasChoices("model_dict", "model"))
pre_prompt: str | None = None
created_by: str | None = None
created_at: int | None = None
@@ -272,54 +269,52 @@ class ModelConfigPartial(ResponseModel):
class ModelConfig(ResponseModel):
opening_statement: str | None = None
suggested_questions: JSONValue | None = Field(
suggested_questions: Any | None = Field(
default=None, validation_alias=AliasChoices("suggested_questions_list", "suggested_questions")
)
suggested_questions_after_answer: JSONValue | None = Field(
suggested_questions_after_answer: Any | None = Field(
default=None,
validation_alias=AliasChoices("suggested_questions_after_answer_dict", "suggested_questions_after_answer"),
)
speech_to_text: JSONValue | None = Field(
speech_to_text: Any | None = Field(
default=None, validation_alias=AliasChoices("speech_to_text_dict", "speech_to_text")
)
text_to_speech: JSONValue | None = Field(
text_to_speech: Any | None = Field(
default=None, validation_alias=AliasChoices("text_to_speech_dict", "text_to_speech")
)
retriever_resource: JSONValue | None = Field(
retriever_resource: Any | None = Field(
default=None, validation_alias=AliasChoices("retriever_resource_dict", "retriever_resource")
)
annotation_reply: JSONValue | None = Field(
annotation_reply: Any | None = Field(
default=None, validation_alias=AliasChoices("annotation_reply_dict", "annotation_reply")
)
more_like_this: JSONValue | None = Field(
more_like_this: Any | None = Field(
default=None, validation_alias=AliasChoices("more_like_this_dict", "more_like_this")
)
sensitive_word_avoidance: JSONValue | None = Field(
sensitive_word_avoidance: Any | None = Field(
default=None, validation_alias=AliasChoices("sensitive_word_avoidance_dict", "sensitive_word_avoidance")
)
external_data_tools: JSONValue | None = Field(
external_data_tools: Any | None = Field(
default=None, validation_alias=AliasChoices("external_data_tools_list", "external_data_tools")
)
model: JSONValue | None = Field(default=None, validation_alias=AliasChoices("model_dict", "model"))
user_input_form: JSONValue | None = Field(
model: Any | None = Field(default=None, validation_alias=AliasChoices("model_dict", "model"))
user_input_form: Any | None = Field(
default=None, validation_alias=AliasChoices("user_input_form_list", "user_input_form")
)
dataset_query_variable: str | None = None
pre_prompt: str | None = None
agent_mode: JSONValue | None = Field(default=None, validation_alias=AliasChoices("agent_mode_dict", "agent_mode"))
agent_mode: Any | None = Field(default=None, validation_alias=AliasChoices("agent_mode_dict", "agent_mode"))
prompt_type: str | None = None
chat_prompt_config: JSONValue | None = Field(
chat_prompt_config: Any | None = Field(
default=None, validation_alias=AliasChoices("chat_prompt_config_dict", "chat_prompt_config")
)
completion_prompt_config: JSONValue | None = Field(
completion_prompt_config: Any | None = Field(
default=None, validation_alias=AliasChoices("completion_prompt_config_dict", "completion_prompt_config")
)
dataset_configs: JSONValue | None = Field(
dataset_configs: Any | None = Field(
default=None, validation_alias=AliasChoices("dataset_configs_dict", "dataset_configs")
)
file_upload: JSONValue | None = Field(
default=None, validation_alias=AliasChoices("file_upload_dict", "file_upload")
)
file_upload: Any | None = Field(default=None, validation_alias=AliasChoices("file_upload_dict", "file_upload"))
created_by: str | None = None
created_at: int | None = None
updated_by: str | None = None
@@ -440,7 +435,7 @@ class AppDetail(ResponseModel):
alias="model_config",
)
workflow: WorkflowPartial | None = None
tracing: JSONValue | None = None
tracing: Any | None = None
use_icon_as_answer_icon: bool | None = None
created_by: str | None = None
created_at: int | None = None
@@ -486,6 +481,16 @@ class AppExportResponse(ResponseModel):
data: str
class AppImportResponse(ResponseModel):
id: str
status: ImportStatus
app_id: str | None = None
app_mode: str | None = None
current_dsl_version: str
imported_dsl_version: str = ""
error: str = ""
def _enrich_app_list_items(session: Session, *, apps: Sequence[App], tenant_id: str) -> None:
if FeatureService.get_system_features().webapp_auth.enabled:
app_ids = [str(app.id) for app in apps]
@@ -528,7 +533,9 @@ def _enrich_app_list_items(session: Session, *, apps: Sequence[App], tenant_id:
register_enum_models(console_ns, RetrievalMethod, WorkflowExecutionStatus, DatasetPermissionEnum)
register_response_schema_models(console_ns, AppTraceResponse, RedirectUrlResponse, SimpleResultResponse)
register_response_schema_models(
console_ns, RedirectUrlResponse, SimpleResultResponse, AppImportResponse, AppTraceResponse
)
register_schema_models(
console_ns,
@@ -620,8 +627,8 @@ class AppListApi(Resource):
app_service = AppService()
app_pagination = app_service.get_paginate_apps(current_user_id, current_tenant_id, params, db.session)
if not app_pagination:
empty = AppPagination(page=args.page, limit=args.limit, total=0, has_more=False, data=[])
return empty.model_dump(mode="json"), 200
response = AppPagination(page=args.page, limit=args.limit, total=0, has_more=False, data=[])
return response.model_dump(mode="json"), 200
app_ids = [str(app.id) for app in app_pagination.items]
permission_keys_map = permissions.app.permission_keys_by_resource_ids(app_ids)
@@ -710,9 +717,7 @@ class StarredAppListApi(Resource):
return empty.model_dump(mode="json"), 200
_enrich_app_list_items(session, apps=app_pagination.items, tenant_id=current_tenant_id)
pagination_model = AppPagination.model_validate(app_pagination, from_attributes=True)
return pagination_model.model_dump(mode="json"), 200
return AppPagination.model_validate(app_pagination, from_attributes=True).model_dump(mode="json"), 200
@console_ns.route("/apps/<uuid:app_id>/star")
@@ -731,7 +736,7 @@ class AppStarApi(Resource):
@get_app_model(mode=None)
def post(self, session: Session, current_user_id: str, app_model: App):
AppService.star_app(session, app=app_model, account_id=current_user_id)
return dump_response(SimpleResultResponse, {"result": "success"})
return SimpleResultResponse(result="success").model_dump(mode="json")
@console_ns.doc("unstar_app")
@console_ns.doc(description="Remove the current account's star from an application")
@@ -747,7 +752,7 @@ class AppStarApi(Resource):
@get_app_model(mode=None)
def delete(self, session: Session, current_user_id: str, app_model: App):
AppService.unstar_app(session, app=app_model, account_id=current_user_id)
return dump_response(SimpleResultResponse, {"result": "success"})
return SimpleResultResponse(result="success").model_dump(mode="json")
@console_ns.route("/apps/<uuid:app_id>")
@@ -815,8 +820,7 @@ class AppApi(Resource):
"max_active_requests": args.max_active_requests or 0,
}
app_model = app_service.update_app(app_model, args_dict)
response_model = AppDetailWithSite.model_validate(app_model, from_attributes=True)
return response_model.model_dump(mode="json")
return dump_response(AppDetailWithSite, app_model)
@console_ns.doc("delete_app")
@console_ns.doc(description="Delete application")
@@ -844,6 +848,7 @@ class AppCopyApi(Resource):
@console_ns.doc(params={"app_id": "Application ID to copy"})
@console_ns.expect(console_ns.models[CopyAppPayload.__name__])
@console_ns.response(201, "App copied successfully", console_ns.models[AppDetailWithSite.__name__])
@console_ns.response(202, "App copy requires confirmation", console_ns.models[AppImportResponse.__name__])
@console_ns.response(403, "Insufficient permissions")
@setup_required
@login_required
@@ -873,10 +878,10 @@ class AppCopyApi(Resource):
)
if result.status == ImportStatus.FAILED:
session.rollback()
return result.model_dump(mode="json"), 400
return dump_response(AppImportResponse, result), 400
if result.status == ImportStatus.PENDING:
session.rollback()
return result.model_dump(mode="json"), 202
return dump_response(AppImportResponse, result), 202
session.commit()
# Inherit web app permission from original app
@@ -927,14 +932,14 @@ class AppExportApi(Resource):
"""Export app"""
args = AppExportQuery.model_validate(request.args.to_dict(flat=True))
payload = AppExportResponse(
response = AppExportResponse(
data=AppDslService.export_dsl(
app_model=app_model,
include_secret=args.include_secret,
workflow_id=args.workflow_id,
)
)
return payload.model_dump(mode="json")
return response.model_dump(mode="json")
@console_ns.route("/apps/<uuid:app_id>/publish-to-creators-platform")
@@ -960,7 +965,7 @@ class AppPublishToCreatorsPlatformApi(Resource):
claim_code = upload_dsl(dsl_bytes)
redirect_url = get_redirect_url(current_user_id, claim_code)
return {"redirect_url": redirect_url}
return RedirectUrlResponse(redirect_url=redirect_url).model_dump(mode="json")
@console_ns.route("/apps/<uuid:app_id>/name")
@@ -981,8 +986,7 @@ class AppNameApi(Resource):
app_service = AppService()
app_model = app_service.update_app_name(app_model, args.name)
response_model = AppDetail.model_validate(app_model, from_attributes=True)
return response_model.model_dump(mode="json")
return dump_response(AppDetail, app_model)
@console_ns.route("/apps/<uuid:app_id>/icon")
@@ -1009,8 +1013,7 @@ class AppIconApi(Resource):
args.icon_background or "",
args.icon_type,
)
response_model = AppDetail.model_validate(app_model, from_attributes=True)
return response_model.model_dump(mode="json")
return dump_response(AppDetail, app_model)
@console_ns.route("/apps/<uuid:app_id>/site-enable")
@@ -1032,8 +1035,7 @@ class AppSiteStatus(Resource):
app_service = AppService()
app_model = app_service.update_app_site_status(app_model, args.enable_site)
response_model = AppDetail.model_validate(app_model, from_attributes=True)
return response_model.model_dump(mode="json")
return dump_response(AppDetail, app_model)
@console_ns.route("/apps/<uuid:app_id>/api-enable")
@@ -1055,8 +1057,7 @@ class AppApiStatus(Resource):
app_service = AppService()
app_model = app_service.update_app_api_status(app_model, args.enable_api)
response_model = AppDetail.model_validate(app_model, from_attributes=True)
return response_model.model_dump(mode="json")
return dump_response(AppDetail, app_model)
@console_ns.route("/apps/<uuid:app_id>/trace")
@@ -1079,7 +1080,7 @@ class AppTraceApi(Resource):
"""Get app trace"""
app_trace_config = OpsTraceManager.get_app_tracing_config(app_model.id, session)
return app_trace_config
return dump_response(AppTraceResponse, app_trace_config)
@console_ns.doc("update_app_trace")
@console_ns.doc(description="Update app tracing configuration")
@@ -1107,4 +1108,4 @@ class AppTraceApi(Resource):
tracing_provider=args.tracing_provider,
)
return {"result": "success"}
return SimpleResultResponse(result="success").model_dump(mode="json")
api/controllers/console/app/audio.py
+36 -20
View File
@@ -1,5 +1,4 @@
import logging
from typing import Any
from flask import request
from flask_restx import Resource
@@ -7,7 +6,6 @@ from pydantic import BaseModel, Field, RootModel
from werkzeug.exceptions import InternalServerError
import services
from controllers.common.fields import AudioBinaryResponse
from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
from controllers.console import console_ns
from controllers.console.app.error import (
@@ -31,9 +29,12 @@ from controllers.console.wraps import (
)
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
from extensions.ext_database import db
from fields.base import ResponseModel
from graphon.model_runtime.errors.invoke import InvokeError
from libs.login import login_required
from libs.helper import dump_response
from libs.login import current_user, login_required
from models import App, AppMode
from services.app_ref_service import AppRefService
from services.audio_service import AudioService
from services.errors.audio import (
AudioTooLargeServiceError,
@@ -56,16 +57,27 @@ class TextToSpeechVoiceQuery(BaseModel):
language: str = Field(..., description="Language code")
class AudioTranscriptResponse(BaseModel):
class AudioTranscriptResponse(ResponseModel):
text: str = Field(description="Transcribed text from audio")
class TextToSpeechVoiceListResponse(RootModel[list[dict[str, Any]]]):
root: list[dict[str, Any]]
class TextToSpeechVoiceResponse(ResponseModel):
# see api/core/plugin/impl/model.py
name: str = Field(description="Voice display name")
value: str = Field(description="Voice identifier")
register_schema_models(console_ns, AudioTranscriptResponse, TextToSpeechPayload, TextToSpeechVoiceQuery)
register_response_schema_models(console_ns, AudioBinaryResponse, TextToSpeechVoiceListResponse)
class TextToSpeechVoiceListResponse(RootModel[list[TextToSpeechVoiceResponse]]):
root: list[TextToSpeechVoiceResponse] = Field(description="Available voices")
register_schema_models(console_ns, TextToSpeechPayload, TextToSpeechVoiceQuery)
register_response_schema_models(
console_ns,
AudioTranscriptResponse,
TextToSpeechVoiceResponse,
TextToSpeechVoiceListResponse,
)
@console_ns.route("/apps/<uuid:app_id>/audio-to-text")
@@ -94,7 +106,7 @@ class ChatMessageAudioApi(Resource):
end_user=None,
)
return response
return dump_response(AudioTranscriptResponse, response)
except services.errors.app_model_config.AppModelConfigBrokenError:
logger.exception("App model config broken.")
raise AppUnavailableError()
@@ -127,11 +139,8 @@ class ChatMessageTextApi(Resource):
@console_ns.doc(description="Convert text to speech for chat messages")
@console_ns.doc(params={"app_id": "App ID"})
@console_ns.expect(console_ns.models[TextToSpeechPayload.__name__])
@console_ns.response(
200,
"Text to speech conversion successful",
console_ns.models[AudioBinaryResponse.__name__],
)
# TTS returns provider audio bytes, so the success response is intentionally schema-less.
@console_ns.response(200, "Text to speech conversion successful")
@console_ns.response(400, "Bad request - Invalid parameters")
@setup_required
@login_required
@@ -140,16 +149,24 @@ class ChatMessageTextApi(Resource):
def post(self, app_model: App):
try:
payload = TextToSpeechPayload.model_validate(console_ns.payload)
message_ref = None
if payload.message_id:
app_ref = AppRefService.create_app_ref(app_model)
message_ref = AppRefService.create_message_ref(
app_ref,
payload.message_id,
account_id=current_user.id,
)
response = AudioService.transcript_tts(
# response-contract:ignore
return AudioService.transcript_tts(
app_model=app_model,
session=db.session,
text=payload.text,
voice=payload.voice,
message_id=payload.message_id,
message_ref=message_ref,
is_draft=True,
)
return response
except services.errors.app_model_config.AppModelConfigBrokenError:
logger.exception("App model config broken.")
raise AppUnavailableError()
@@ -180,8 +197,7 @@ class ChatMessageTextApi(Resource):
class TextModesApi(Resource):
@console_ns.doc("get_text_to_speech_voices")
@console_ns.doc(description="Get available TTS voices for a specific language")
@console_ns.doc(params={"app_id": "App ID"})
@console_ns.doc(params=query_params_from_model(TextToSpeechVoiceQuery))
@console_ns.doc(params={"app_id": "App ID", **query_params_from_model(TextToSpeechVoiceQuery)})
@console_ns.response(
200,
"TTS voices retrieved successfully",
@@ -202,7 +218,7 @@ class TextModesApi(Resource):
language=args.language,
)
return response
return dump_response(TextToSpeechVoiceListResponse, response)
except services.errors.audio.ProviderNotSupportTextToSpeechLanageServiceError:
raise AppUnavailableError("Text to audio voices language parameter loss.")
except NoAudioUploadedServiceError:
api/controllers/console/app/completion.py
+182 -13
View File
@@ -1,4 +1,6 @@
import json
import logging
from collections.abc import Generator
from typing import Any, Literal
from uuid import UUID
@@ -8,7 +10,7 @@ from pydantic import BaseModel, Field, field_validator
from werkzeug.exceptions import BadRequest, InternalServerError, NotFound
import services
from controllers.common.fields import GeneratedAppResponse, SimpleResultResponse
from controllers.common.fields import SimpleResultResponse
from controllers.common.schema import register_response_schema_models, register_schema_models
from controllers.console import console_ns
from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model
@@ -34,6 +36,7 @@ from controllers.console.wraps import (
)
from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
from core.app.entities.app_invoke_entities import InvokeFrom
from core.app.features.rate_limiting.rate_limit import RateLimitGenerator
from core.errors.error import (
ModelCurrentlyNotSupportError,
ProviderTokenNotInitError,
@@ -106,8 +109,34 @@ class ChatMessagePayload(BaseMessagePayload):
return uuid_value(value)
_BUILD_CHAT_FINALIZATION_QUERY = """Finalize this Build chat configuration for the agent.
This step is only for persisting Agent config changes discovered in the current Build chat. Do not install packages,
edit workspace files, run validation or debugging commands, make exploratory checks, or perform other work.
Use only the current Build chat message history to identify changes that need to be persisted. Do not inspect, test, or
validate old config unless the message history already shows that the old config is invalid.
Persist only the build-draft config resources that need to change, using the Agent config CLI usage provided in the
runtime prompt:
- config files for reusable artifacts that should be available later,
- config skills for reusable procedures or tools that should be available later,
- config env when environment keys or values need to be recorded,
- config note for concise durable context when useful.
When updating the config note, record only durable context needed by later runs, such as:
- what you installed or configured outside the workspace for this agent,
- where those external updates live, including CLI tools, packages, and persistent $HOME paths,
- how the agent should use it in later runs,
- any setup, authentication, or user action still required.
After config persistence completes, respond FINISHED."""
register_schema_models(console_ns, CompletionMessagePayload, ChatMessagePayload)
register_response_schema_models(console_ns, GeneratedAppResponse, SimpleResultResponse)
register_response_schema_models(console_ns, SimpleResultResponse)
# define completion message api for user
@@ -117,7 +146,7 @@ class CompletionMessageApi(Resource):
@console_ns.doc(description="Generate completion message for debugging")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.expect(console_ns.models[CompletionMessagePayload.__name__])
@console_ns.response(200, "Completion generated successfully", console_ns.models[GeneratedAppResponse.__name__])
@console_ns.response(200, "Completion generated successfully")
@console_ns.response(400, "Invalid request parameters")
@console_ns.response(404, "App not found")
@setup_required
@@ -138,6 +167,7 @@ class CompletionMessageApi(Resource):
app_model=app_model, user=current_user, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=streaming
)
# response-contract:ignore compact_generate_response
return helper.compact_generate_response(response)
except services.errors.conversation.ConversationNotExistsError:
raise NotFound("Conversation Not Exists.")
@@ -181,7 +211,7 @@ class CompletionMessageStopApi(Resource):
app_mode=AppMode.value_of(app_model.mode),
)
return {"result": "success"}, 200
return SimpleResultResponse(result="success").model_dump(mode="json"), 200
@console_ns.route("/apps/<uuid:app_id>/chat-messages")
@@ -190,7 +220,7 @@ class ChatMessageApi(Resource):
@console_ns.doc(description="Generate chat message for debugging")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.expect(console_ns.models[ChatMessagePayload.__name__])
@console_ns.response(200, "Chat message generated successfully", console_ns.models[GeneratedAppResponse.__name__])
@console_ns.response(200, "Chat message generated successfully")
@console_ns.response(400, "Invalid request parameters")
@console_ns.response(404, "App or conversation not found")
@setup_required
@@ -211,7 +241,7 @@ class AgentChatMessageApi(Resource):
@console_ns.doc(description="Generate an Agent App chat message for debugging")
@console_ns.doc(params={"agent_id": "Agent ID"})
@console_ns.expect(console_ns.models[ChatMessagePayload.__name__])
@console_ns.response(200, "Chat message generated successfully", console_ns.models[GeneratedAppResponse.__name__])
@console_ns.response(200, "Chat message generated successfully")
@console_ns.response(400, "Invalid request parameters")
@console_ns.response(404, "Agent or conversation not found")
@setup_required
@@ -231,6 +261,31 @@ class AgentChatMessageApi(Resource):
)
@console_ns.route("/agent/<uuid:agent_id>/build-chat/finalize")
class AgentBuildChatFinalizeApi(Resource):
@console_ns.doc("finalize_agent_build_chat")
@console_ns.doc(description="Run a build-draft Agent App turn that asks the agent to push config updates")
@console_ns.doc(params={"agent_id": "Agent ID"})
@console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])
@console_ns.response(400, "Invalid request parameters")
@console_ns.response(404, "Agent, build draft, or conversation not found")
@setup_required
@login_required
@account_initialization_required
@edit_permission_required
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_TEST_AND_RUN)
@with_current_user
@with_current_tenant_id
def post(self, current_tenant_id: str, current_user: Account, agent_id: UUID):
app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
return _create_build_chat_finalization_message(
current_tenant_id=current_tenant_id,
current_user=current_user,
app_model=app_model,
agent_id=str(agent_id),
)
@console_ns.route("/apps/<uuid:app_id>/chat-messages/<string:task_id>/stop")
class ChatMessageStopApi(Resource):
@console_ns.doc("stop_chat_message")
@@ -284,7 +339,11 @@ def _resolve_current_user_agent_debug_conversation_id(
def _create_chat_message(
*, current_user: Account, app_model: App, current_tenant_id: str | None = None, agent_id: str | None = None
*,
current_user: Account,
app_model: App,
current_tenant_id: str | None = None,
agent_id: str | None = None,
):
raw_payload = console_ns.payload or {}
args_model = ChatMessagePayload.model_validate(raw_payload)
@@ -314,12 +373,104 @@ def _create_chat_message(
if external_trace_id:
args["external_trace_id"] = external_trace_id
try:
response = AppGenerateService.generate(
app_model=app_model, user=current_user, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=streaming
)
return _generate_chat_message_response(
current_user=current_user,
app_model=app_model,
args=args,
streaming=streaming,
)
return helper.compact_generate_response(response)
def _create_build_chat_finalization_message(
*, current_user: Account, app_model: App, current_tenant_id: str, agent_id: str
):
debug_conversation_id = _resolve_current_user_agent_debug_conversation_id(
current_tenant_id=current_tenant_id,
current_user=current_user,
app_model=app_model,
agent_id=agent_id,
)
args: dict[str, Any] = {
"query": _BUILD_CHAT_FINALIZATION_QUERY,
"inputs": {},
"response_mode": "streaming",
"draft_type": "debug_build",
"conversation_id": debug_conversation_id,
"auto_generate_name": False,
}
external_trace_id = get_external_trace_id(request)
if external_trace_id:
args["external_trace_id"] = external_trace_id
response = _generate_chat_message(
current_user=current_user,
app_model=app_model,
args=args,
streaming=True,
)
_drain_streaming_generate_response(response)
return {"result": "success"}, 200
def _drain_streaming_generate_response(response: RateLimitGenerator | Generator[str, None, None]) -> None:
"""Consume a streamed app-generate response until a terminal message event arrives.
Finalize keeps the normal Agent App streaming path so the existing queue,
persistence, and runtime-session behavior stay intact. The console API only
changes the HTTP boundary: it drains the SSE stream server-side and returns
success after the generated build-chat message reaches ``message_end``.
"""
close = getattr(response, "close", None)
try:
for chunk in response:
for raw_event in chunk.split("\n\n"):
if not raw_event.strip():
continue
event_name: str | None = None
data_lines: list[str] = []
for line in raw_event.splitlines():
if line.startswith("event: "):
event_name = line.removeprefix("event: ").strip()
elif line.startswith("data: "):
data_lines.append(line.removeprefix("data: "))
if not data_lines:
if event_name == "ping":
continue
continue
payload = json.loads("\n".join(data_lines))
if not isinstance(payload, dict):
continue
payload_event = payload.get("event")
if payload_event == "message_end":
return
if payload_event == "error":
raise CompletionRequestError(str(payload.get("message") or "Build chat finalization failed."))
finally:
if callable(close):
close()
raise CompletionRequestError("Build chat finalization did not complete.")
def _generate_chat_message(
*,
current_user: Account,
app_model: App,
args: dict[str, Any],
streaming: bool,
):
try:
return AppGenerateService.generate(
app_model=app_model,
user=current_user,
args=args,
invoke_from=InvokeFrom.DEBUGGER,
streaming=streaming,
)
except services.errors.conversation.ConversationNotExistsError:
raise NotFound("Conversation Not Exists.")
except services.errors.conversation.ConversationCompletedError:
@@ -335,6 +486,8 @@ def _create_chat_message(
raise ProviderModelCurrentlyNotSupportError()
except InvokeRateLimitError as ex:
raise InvokeRateLimitHttpError(ex.description)
except CompletionRequestError:
raise
except InvokeError as e:
raise CompletionRequestError(e.description)
except ValueError as e:
@@ -344,6 +497,22 @@ def _create_chat_message(
raise InternalServerError()
def _generate_chat_message_response(
*,
current_user: Account,
app_model: App,
args: dict[str, Any],
streaming: bool,
):
response = _generate_chat_message(
current_user=current_user,
app_model=app_model,
args=args,
streaming=streaming,
)
return helper.compact_generate_response(response)
def _stop_chat_message(*, current_user_id: str, app_model: App, task_id: str):
AppTaskService.stop_task(
task_id=task_id,
@@ -352,4 +521,4 @@ def _stop_chat_message(*, current_user_id: str, app_model: App, task_id: str):
app_mode=AppMode.value_of(app_model.mode),
)
return {"result": "success"}, 200
return SimpleResultResponse(result="success").model_dump(mode="json"), 200
api/controllers/console/app/conversation.py
+15 -19
View File
@@ -9,7 +9,7 @@ from sqlalchemy import func, or_
from sqlalchemy.orm import selectinload
from werkzeug.exceptions import NotFound
from controllers.common.schema import query_params_from_model, register_schema_models
from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
from controllers.console import console_ns
from controllers.console.app.wraps import get_app_model
from controllers.console.wraps import (
@@ -39,6 +39,7 @@ from fields.conversation_fields import (
ConversationWithSummaryPagination as ConversationWithSummaryPaginationResponse,
)
from libs.datetime_utils import naive_utc_now, parse_time_range
from libs.helper import dump_response
from libs.login import login_required
from models import Conversation, EndUser, Message, MessageAnnotation
from models.account import Account
@@ -79,13 +80,14 @@ register_schema_models(
console_ns,
CompletionConversationQuery,
ChatConversationQuery,
)
register_response_schema_models(
console_ns,
ConversationResponse,
ConversationPaginationResponse,
ConversationMessageDetailResponse,
ConversationWithSummaryPaginationResponse,
ConversationDetailResponse,
CompletionConversationQuery,
ChatConversationQuery,
)
@@ -93,8 +95,7 @@ register_schema_models(
class CompletionConversationApi(Resource):
@console_ns.doc("list_completion_conversations")
@console_ns.doc(description="Get completion conversations with pagination and filtering")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(CompletionConversationQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(CompletionConversationQuery)})
@console_ns.response(200, "Success", console_ns.models[ConversationPaginationResponse.__name__])
@console_ns.response(403, "Insufficient permissions")
@setup_required
@@ -157,9 +158,7 @@ class CompletionConversationApi(Resource):
conversations = db.paginate(query, page=args.page, per_page=args.limit, error_out=False)
return ConversationPaginationResponse.model_validate(conversations, from_attributes=True).model_dump(
mode="json"
)
return dump_response(ConversationPaginationResponse, conversations)
@console_ns.route("/apps/<uuid:app_id>/completion-conversations/<uuid:conversation_id>")
@@ -179,9 +178,9 @@ class CompletionConversationDetailApi(Resource):
@get_app_model(mode=AppMode.COMPLETION)
def get(self, current_user: Account, app_model: App, conversation_id: UUID):
conversation_id_str = str(conversation_id)
return ConversationMessageDetailResponse.model_validate(
_get_conversation(current_user, app_model, conversation_id_str), from_attributes=True
).model_dump(mode="json")
return dump_response(
ConversationMessageDetailResponse, _get_conversation(current_user, app_model, conversation_id_str)
)
@console_ns.doc("delete_completion_conversation")
@console_ns.doc(description="Delete a completion conversation")
@@ -211,8 +210,7 @@ class CompletionConversationDetailApi(Resource):
class ChatConversationApi(Resource):
@console_ns.doc("list_chat_conversations")
@console_ns.doc(description="Get chat conversations with pagination, filtering and summary")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(ChatConversationQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(ChatConversationQuery)})
@console_ns.response(200, "Success", console_ns.models[ConversationWithSummaryPaginationResponse.__name__])
@console_ns.response(403, "Insufficient permissions")
@setup_required
@@ -314,9 +312,7 @@ class ChatConversationApi(Resource):
conversations = db.paginate(query, page=args.page, per_page=args.limit, error_out=False)
return ConversationWithSummaryPaginationResponse.model_validate(conversations, from_attributes=True).model_dump(
mode="json"
)
return dump_response(ConversationWithSummaryPaginationResponse, conversations)
@console_ns.route("/apps/<uuid:app_id>/chat-conversations/<uuid:conversation_id>")
@@ -336,9 +332,9 @@ class ChatConversationDetailApi(Resource):
@get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT, AppMode.AGENT])
def get(self, current_user: Account, app_model: App, conversation_id: UUID):
conversation_id_str = str(conversation_id)
return ConversationDetailResponse.model_validate(
_get_conversation(current_user, app_model, conversation_id_str), from_attributes=True
).model_dump(mode="json")
return dump_response(
ConversationDetailResponse, _get_conversation(current_user, app_model, conversation_id_str)
)
@console_ns.doc("delete_chat_conversation")
@console_ns.doc(description="Delete a chat conversation")
api/controllers/console/app/conversation_variables.py
+4 -4
View File
@@ -22,7 +22,7 @@ from controllers.console.wraps import (
from extensions.ext_database import db
from fields._value_type_serializer import serialize_value_type
from fields.base import ResponseModel
from libs.helper import to_timestamp
from libs.helper import dump_response, to_timestamp
from libs.login import login_required
from models import ConversationVariable
from models.model import App, AppMode
@@ -119,7 +119,8 @@ class ConversationVariablesApi(Resource):
with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
rows = session.scalars(stmt).all()
response = PaginatedConversationVariableResponse.model_validate(
return dump_response(
PaginatedConversationVariableResponse,
{
"page": page,
"limit": page_size,
@@ -135,6 +136,5 @@ class ConversationVariablesApi(Resource):
)
for row in rows
],
}
},
)
return response.model_dump(mode="json")
api/controllers/console/app/generator.py
+149 -28
View File
@@ -1,8 +1,10 @@
from collections.abc import Sequence
import json
from collections.abc import Generator, Sequence
from typing import Any, Literal
from flask_restx import Resource
from pydantic import BaseModel, Field, RootModel
from sqlalchemy import select
from sqlalchemy.orm import Session
from controllers.common.fields import SimpleDataResponse
@@ -23,8 +25,10 @@ from core.helper.code_executor.javascript.javascript_code_provider import Javasc
from core.helper.code_executor.python3.python3_code_provider import Python3CodeProvider
from core.llm_generator.entities import RuleCodeGeneratePayload, RuleGeneratePayload, RuleStructuredOutputPayload
from core.llm_generator.llm_generator import LLMGenerator
from core.workflow.generator.types import WorkflowGenerateErrorCode
from graphon.model_runtime.entities.llm_entities import LLMMode
from graphon.model_runtime.errors.invoke import InvokeError
from libs.helper import compact_generate_response
from libs.login import login_required
from models import App
from services.workflow_generator_service import WorkflowGeneratorService
@@ -64,7 +68,10 @@ class WorkflowGeneratePayload(BaseModel):
can reuse its existing handler.
"""
mode: Literal["workflow", "advanced-chat"] = Field(..., description="Target app mode for the generated graph")
mode: Literal["workflow", "advanced-chat", "auto"] = Field(
...,
description="Target app mode for the generated graph; 'auto' lets the backend classify the instruction",
)
instruction: str = Field(..., description="Natural-language workflow description")
ideal_output: str = Field(default="", description="Optional sample output for grounding")
model_config_data: ModelConfig = Field(
@@ -78,6 +85,19 @@ class WorkflowGeneratePayload(BaseModel):
)
class WorkflowInstructionSuggestionsPayload(BaseModel):
"""Payload for the workflow-generator instruction-suggestions endpoint.
Runs before the user picks a model, so the suggestions come from the
tenant's default model. The underlying generator never raises — an empty
``suggestions`` list is a valid 200 (soft-fail).
"""
mode: Literal["workflow", "advanced-chat"] = Field(..., description="Target app mode for the suggestions")
language: str | None = Field(default=None, description="Optional language to write the suggestions in")
count: int = Field(default=4, ge=1, le=6, description="Number of suggestions to return (1-6)")
class GeneratorResponse(RootModel[Any]):
root: Any
@@ -91,6 +111,7 @@ register_schema_models(
InstructionGeneratePayload,
InstructionTemplatePayload,
WorkflowGeneratePayload,
WorkflowInstructionSuggestionsPayload,
ModelConfig,
)
register_response_schema_models(console_ns, GeneratorResponse, SimpleDataResponse)
@@ -216,7 +237,9 @@ class InstructionGenerateApi(Resource):
try:
# Generate from nothing for a workflow node
if (args.current in (code_template, "")) and args.node_id != "":
app = session.get(App, args.flow_id)
app = session.scalar(
select(App).where(App.id == args.flow_id, App.tenant_id == current_tenant_id).limit(1)
)
if not app:
return {"error": f"app {args.flow_id} not found"}, 400
workflow = WorkflowService().get_draft_workflow(app_model=app, session=session)
@@ -313,6 +336,34 @@ class InstructionGenerationTemplateApi(Resource):
raise ValueError(f"Invalid type: {args.type}")
def _workflow_instruction_guard(args: WorkflowGeneratePayload) -> tuple[dict, int] | None:
"""Shared boundary guard for the workflow-generate endpoints.
Returns a ``(body, 400)`` tuple when the instruction is empty / whitespace
or either free-text field exceeds the cap, else ``None``. Pydantic only
validates the field is a str; a whitespace-only or pasted-document input
would otherwise waste a slow planner+builder roundtrip on a response the
validator rejects anyway. Both the blocking and streaming endpoints call
this so they reject identical inputs.
"""
if not args.instruction.strip():
return {
"error": "Instruction is required",
"errors": [{"code": WorkflowGenerateErrorCode.EMPTY_INSTRUCTION, "detail": "Instruction is required"}],
}, 400
if len(args.instruction) > _MAX_INSTRUCTION_LENGTH or len(args.ideal_output) > _MAX_INSTRUCTION_LENGTH:
return {
"error": "Instruction is too long",
"errors": [
{
"code": WorkflowGenerateErrorCode.INSTRUCTION_TOO_LONG,
"detail": f"Instruction and ideal output must each be at most {_MAX_INSTRUCTION_LENGTH} characters",
}
],
}, 400
return None
@console_ns.route("/workflow-generate")
class WorkflowGenerateApi(Resource):
"""Generate a Workflow / Chatflow draft graph from a natural-language description.
@@ -335,31 +386,11 @@ class WorkflowGenerateApi(Resource):
def post(self, current_tenant_id: str):
args = WorkflowGeneratePayload.model_validate(console_ns.payload)
# Reject obviously-empty instructions at the boundary — Pydantic only
# validates ``instruction`` is a str, but a whitespace-only string
# would still hit the LLM and waste a planner+builder roundtrip on a
# response that the postprocess validator would reject anyway.
if not args.instruction.strip():
return {
"error": "Instruction is required",
"errors": [{"code": "EMPTY_INSTRUCTION", "detail": "Instruction is required"}],
}, 400
# Bound the prompt at the boundary too: an arbitrarily long
# instruction (or pasted document) blows the planner/builder context
# window and fails with an opaque provider error after two slow LLM
# calls. The cap matches the frontend textarea's maxLength.
if len(args.instruction) > _MAX_INSTRUCTION_LENGTH or len(args.ideal_output) > _MAX_INSTRUCTION_LENGTH:
return {
"error": "Instruction is too long",
"errors": [
{
"code": "INSTRUCTION_TOO_LONG",
"detail": f"Instruction and ideal output must each be at most "
f"{_MAX_INSTRUCTION_LENGTH} characters",
}
],
}, 400
# Reject empty / over-length instructions at the boundary (shared with
# the streaming endpoint) before spending a planner+builder roundtrip.
guard = _workflow_instruction_guard(args)
if guard is not None:
return guard
try:
result = WorkflowGeneratorService.generate_workflow_graph(
@@ -380,3 +411,93 @@ class WorkflowGenerateApi(Resource):
raise CompletionRequestError(e.description)
return result
@console_ns.route("/workflow-generate/suggestions")
class WorkflowInstructionSuggestionsApi(Resource):
"""Suggest short, buildable example instructions for the cmd+k generator.
Runs before a model is selected (uses the tenant's default model). The
underlying generator never raises, so an empty list is a valid 200 — the
frontend renders "no suggestions" rather than an error, so no provider-error
mapping is needed here.
"""
@console_ns.doc("generate_workflow_instruction_suggestions")
@console_ns.doc(description="Suggest example workflow-generator instructions for the tenant")
@console_ns.expect(console_ns.models[WorkflowInstructionSuggestionsPayload.__name__])
@console_ns.response(200, "Suggestions generated successfully", console_ns.models[GeneratorResponse.__name__])
@console_ns.response(400, "Invalid request parameters")
@setup_required
@login_required
@account_initialization_required
@with_current_tenant_id
def post(self, current_tenant_id: str):
args = WorkflowInstructionSuggestionsPayload.model_validate(console_ns.payload)
suggestions = LLMGenerator.generate_workflow_instruction_suggestions(
tenant_id=current_tenant_id,
mode=args.mode,
language=args.language,
count=args.count,
)
return {"suggestions": suggestions}
@console_ns.route("/workflow-generate/stream")
class WorkflowGenerateStreamApi(Resource):
"""Plan-first streaming variant of ``/workflow-generate`` (Server-Sent Events).
Emits a ``plan`` event (high-level node list + app metadata) as soon as the
planner returns, then a final ``result`` event with the full graph — the
SAME envelope ``/workflow-generate`` returns. Provider-init / invoke errors
are surfaced as a single ``result`` event (code ``MODEL_ERROR``) so the
frontend's stream parser always receives a result rather than a non-SSE HTTP
error.
"""
@console_ns.doc("generate_workflow_graph_stream")
@console_ns.doc(description="Stream a Dify workflow graph (plan then result) via SSE")
@console_ns.expect(console_ns.models[WorkflowGeneratePayload.__name__])
@console_ns.response(200, "Server-Sent Events stream of plan/result events")
@console_ns.response(400, "Invalid request parameters")
@setup_required
@login_required
@account_initialization_required
@with_current_tenant_id
def post(self, current_tenant_id: str):
args = WorkflowGeneratePayload.model_validate(console_ns.payload)
# Same boundary guards as the blocking endpoint — return a normal 400
# JSON for these BEFORE opening the stream.
guard = _workflow_instruction_guard(args)
if guard is not None:
return guard
def generate() -> Generator[str, None, None]:
try:
for event_name, payload in WorkflowGeneratorService.generate_workflow_graph_stream(
tenant_id=current_tenant_id,
mode=args.mode,
instruction=args.instruction,
model_config=args.model_config_data,
ideal_output=args.ideal_output,
current_graph=args.current_graph,
):
body = {"event": event_name, **payload}
yield f"data: {json.dumps(body)}\n\n"
except (ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError, InvokeError) as e:
# The model instance is resolved inside the service (lazily, on
# first iteration), so a provider / init error surfaces here.
# Emit it as a single SSE result event rather than a non-SSE
# error response so the frontend's stream parser always gets a
# result it can render.
detail = getattr(e, "description", None) or str(e) or "Model invocation failed"
error_body = {
"event": "result",
"graph": {"nodes": [], "edges": [], "viewport": {"x": 0.0, "y": 0.0, "zoom": 0.7}},
"error": detail,
"errors": [{"code": WorkflowGenerateErrorCode.MODEL_ERROR, "detail": detail}],
}
yield f"data: {json.dumps(error_body)}\n\n"
return compact_generate_response(generate())
api/controllers/console/app/mcp_server.py
+17 -6
View File
@@ -22,10 +22,11 @@ from controllers.console.wraps import (
)
from extensions.ext_database import db
from fields.base import ResponseModel
from libs.helper import to_timestamp
from libs.helper import dump_response, to_timestamp
from libs.login import login_required
from models.enums import AppMCPServerStatus
from models.model import App, AppMCPServer
from services.app_ref_service import AppRefService
class MCPServerCreatePayload(BaseModel):
@@ -92,7 +93,7 @@ class AppMCPServerController(Resource):
server = db.session.scalar(select(AppMCPServer).where(AppMCPServer.app_id == app_model.id).limit(1))
if server is None:
return {}
return AppMCPServerResponse.model_validate(server, from_attributes=True).model_dump(mode="json")
return dump_response(AppMCPServerResponse, server)
@console_ns.doc("create_app_mcp_server")
@console_ns.doc(description="Create MCP server configuration for an application")
@@ -127,7 +128,7 @@ class AppMCPServerController(Resource):
)
db.session.add(server)
db.session.commit()
return AppMCPServerResponse.model_validate(server, from_attributes=True).model_dump(mode="json"), 201
return dump_response(AppMCPServerResponse, server), 201
@console_ns.doc("update_app_mcp_server")
@console_ns.doc(description="Update MCP server configuration for an application")
@@ -146,7 +147,17 @@ class AppMCPServerController(Resource):
@get_app_model
def put(self, app_model: App):
payload = MCPServerUpdatePayload.model_validate(console_ns.payload or {})
server = db.session.get(AppMCPServer, payload.id)
app_ref = AppRefService.create_app_ref(app_model)
server_ref = AppRefService.create_mcp_server_ref(app_ref, payload.id)
server = db.session.scalar(
select(AppMCPServer)
.where(
AppMCPServer.id == server_ref.server_id,
AppMCPServer.tenant_id == server_ref.tenant_id,
AppMCPServer.app_id == server_ref.app_id,
)
.limit(1)
)
if not server:
raise NotFound()
@@ -165,7 +176,7 @@ class AppMCPServerController(Resource):
except ValueError:
raise ValueError("Invalid status")
db.session.commit()
return AppMCPServerResponse.model_validate(server, from_attributes=True).model_dump(mode="json")
return dump_response(AppMCPServerResponse, server)
@console_ns.route("/apps/<uuid:server_id>/server/refresh")
@@ -192,4 +203,4 @@ class AppMCPServerRefreshController(Resource):
raise NotFound()
server.server_code = AppMCPServer.generate_server_code(16)
db.session.commit()
return AppMCPServerResponse.model_validate(server, from_attributes=True).model_dump(mode="json")
return dump_response(AppMCPServerResponse, server)
api/controllers/console/app/message.py
+15 -58
View File
@@ -1,5 +1,4 @@
import logging
from datetime import datetime
from typing import Literal
from uuid import UUID
@@ -38,16 +37,10 @@ from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotIni
from extensions.ext_database import db
from fields.base import ResponseModel
from fields.conversation_fields import (
AgentThought,
ConversationAnnotation,
ConversationAnnotationHitHistory,
Feedback,
JSONValue,
MessageFile,
format_files_contained,
MessageDetail as BaseMessageDetailResponse,
)
from graphon.model_runtime.errors.invoke import InvokeError
from libs.helper import to_timestamp, uuid_value
from libs.helper import dump_response, uuid_value
from libs.infinite_scroll_pagination import InfiniteScrollPagination
from libs.login import login_required
from models.account import Account
@@ -111,49 +104,16 @@ class FeedbackExportQuery(BaseModel):
raise ValueError("has_comment must be a boolean value")
class AnnotationCountResponse(BaseModel):
class AnnotationCountResponse(ResponseModel):
count: int = Field(description="Number of annotations")
class SuggestedQuestionsResponse(BaseModel):
class SuggestedQuestionsResponse(ResponseModel):
data: list[str] = Field(description="Suggested question")
class MessageDetailResponse(ResponseModel):
id: str
conversation_id: str
inputs: dict[str, JSONValue]
query: str
message: JSONValue | None = None
message_tokens: int | None = None
answer: str = Field(validation_alias="re_sign_file_url_answer")
answer_tokens: int | None = None
provider_response_latency: float | None = None
from_source: str
from_end_user_id: str | None = None
from_account_id: str | None = None
feedbacks: list[Feedback] = Field(default_factory=list)
workflow_run_id: str | None = None
annotation: ConversationAnnotation | None = None
annotation_hit_history: ConversationAnnotationHitHistory | None = None
created_at: int | None = None
agent_thoughts: list[AgentThought] = Field(default_factory=list)
message_files: list[MessageFile] = Field(default_factory=list)
class MessageDetailResponse(BaseMessageDetailResponse):
extra_contents: list[ExecutionExtraContentDomainModel] = Field(default_factory=list)
metadata: JSONValue | None = Field(default=None, validation_alias="message_metadata_dict")
status: str
error: str | None = None
parent_message_id: str | None = None
@field_validator("inputs", mode="before")
@classmethod
def _normalize_inputs(cls, value: JSONValue) -> JSONValue:
return format_files_contained(value)
@field_validator("created_at", mode="before")
@classmethod
def _normalize_created_at(cls, value: datetime | int | None) -> int | None:
return to_timestamp(value)
class MessageInfiniteScrollPaginationResponse(ResponseModel):
@@ -183,8 +143,7 @@ register_response_schema_models(
class ChatMessageListApi(Resource):
@console_ns.doc("list_chat_messages")
@console_ns.doc(description="Get chat messages for a conversation with pagination")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(ChatMessagesQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(ChatMessagesQuery)})
@console_ns.response(200, "Success", console_ns.models[MessageInfiniteScrollPaginationResponse.__name__])
@console_ns.response(404, "Conversation not found")
@login_required
@@ -274,7 +233,7 @@ class MessageAnnotationCountApi(Resource):
select(func.count(MessageAnnotation.id)).where(MessageAnnotation.app_id == app_model.id)
)
return {"count": count}
return AnnotationCountResponse(count=count or 0).model_dump(mode="json")
@console_ns.route("/apps/<uuid:app_id>/chat-messages/<uuid:message_id>/suggested-questions")
@@ -323,13 +282,12 @@ class AgentMessageSuggestedQuestionApi(Resource):
class MessageFeedbackExportApi(Resource):
@console_ns.doc("export_feedbacks")
@console_ns.doc(description="Export user feedback data for Google Sheets")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(FeedbackExportQuery))
@console_ns.response(
200,
"Feedback data exported successfully",
console_ns.models[TextFileResponse.__name__],
)
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(FeedbackExportQuery)})
@console_ns.response(400, "Invalid parameters")
@console_ns.response(500, "Internal server error")
@setup_required
@@ -354,7 +312,6 @@ class MessageFeedbackExportApi(Resource):
end_date=args.end_date,
format_type=args.format,
)
return export_data
except ValueError as e:
@@ -465,16 +422,16 @@ def _list_chat_messages(*, app_model: App, current_user: Account | None = None):
history_messages = list(reversed(history_messages))
attach_message_extra_contents(history_messages)
return MessageInfiniteScrollPaginationResponse.model_validate(
return dump_response(
MessageInfiniteScrollPaginationResponse,
InfiniteScrollPagination(data=history_messages, limit=args.limit, has_more=has_more),
from_attributes=True,
).model_dump(mode="json")
)
def _update_message_feedback(*, current_user: Account, app_model: App):
args = MessageFeedbackPayload.model_validate(console_ns.payload)
message_id = str(args.message_id)
message_id = args.message_id
message = db.session.scalar(
select(Message).where(Message.id == message_id, Message.app_id == app_model.id).limit(1)
@@ -509,7 +466,7 @@ def _update_message_feedback(*, current_user: Account, app_model: App):
db.session.commit()
return {"result": "success"}
return SimpleResultResponse(result="success").model_dump(mode="json")
def _get_message_suggested_questions(*, current_user: Account, app_model: App, message_id: UUID):
@@ -537,7 +494,7 @@ def _get_message_suggested_questions(*, current_user: Account, app_model: App, m
logger.exception("internal server error.")
raise InternalServerError()
return {"data": questions}
return dump_response(SuggestedQuestionsResponse, {"data": questions})
def _get_message_detail(*, app_model: App, message_id: UUID):
@@ -551,4 +508,4 @@ def _get_message_detail(*, app_model: App, message_id: UUID):
raise NotFound("Message Not Exists.")
attach_message_extra_contents([message])
return MessageDetailResponse.model_validate(message, from_attributes=True).model_dump(mode="json")
return dump_response(MessageDetailResponse, message)
api/controllers/console/app/site.py
+3 -2
View File
@@ -22,6 +22,7 @@ from controllers.console.wraps import (
from extensions.ext_database import db
from fields.base import ResponseModel
from libs.datetime_utils import naive_utc_now
from libs.helper import dump_response
from libs.login import login_required
from models import Site
from models.account import Account
@@ -127,7 +128,7 @@ class AppSite(Resource):
site.updated_at = naive_utc_now()
db.session.commit()
return AppSiteResponse.model_validate(site, from_attributes=True).model_dump(mode="json")
return dump_response(AppSiteResponse, site)
@console_ns.route("/apps/<uuid:app_id>/site/access-token-reset")
@@ -156,4 +157,4 @@ class AppSiteAccessTokenReset(Resource):
site.updated_at = naive_utc_now()
db.session.commit()
return AppSiteResponse.model_validate(site, from_attributes=True).model_dump(mode="json")
return dump_response(AppSiteResponse, site)
api/controllers/console/app/statistic.py
+52 -46
View File
@@ -1,7 +1,7 @@
from decimal import Decimal
import sqlalchemy as sa
from flask import abort, jsonify, request
from flask import abort, request
from flask_restx import Resource
from pydantic import BaseModel, Field, field_validator
@@ -20,7 +20,7 @@ from core.app.entities.app_invoke_entities import InvokeFrom
from extensions.ext_database import db
from fields.base import ResponseModel
from libs.datetime_utils import parse_time_range
from libs.helper import convert_datetime_to_date
from libs.helper import convert_datetime_to_date, dump_response
from libs.login import login_required
from models import AppMode
from models.account import Account
@@ -44,8 +44,15 @@ class DailyMessageStatisticItem(ResponseModel):
message_count: int
class DailyMessageStatisticResponse(ResponseModel):
data: list[DailyMessageStatisticItem]
register_schema_models(console_ns, StatisticTimeRangeQuery)
class StatisticDataResponse[T](ResponseModel):
data: list[T]
class DailyMessageStatisticResponse(StatisticDataResponse[DailyMessageStatisticItem]):
pass
class DailyConversationStatisticItem(ResponseModel):
@@ -53,8 +60,8 @@ class DailyConversationStatisticItem(ResponseModel):
conversation_count: int
class DailyConversationStatisticResponse(ResponseModel):
data: list[DailyConversationStatisticItem]
class DailyConversationStatisticResponse(StatisticDataResponse[DailyConversationStatisticItem]):
pass
class DailyTerminalStatisticItem(ResponseModel):
@@ -62,19 +69,19 @@ class DailyTerminalStatisticItem(ResponseModel):
terminal_count: int
class DailyTerminalStatisticResponse(ResponseModel):
data: list[DailyTerminalStatisticItem]
class DailyTerminalStatisticResponse(StatisticDataResponse[DailyTerminalStatisticItem]):
pass
class DailyTokenCostStatisticItem(ResponseModel):
date: str
token_count: int
total_price: str | float
currency: str
token_count: int | None = None
total_price: Decimal | None = None
currency: str | None = None
class DailyTokenCostStatisticResponse(ResponseModel):
data: list[DailyTokenCostStatisticItem]
class DailyTokenCostStatisticResponse(StatisticDataResponse[DailyTokenCostStatisticItem]):
pass
class AverageSessionInteractionStatisticItem(ResponseModel):
@@ -82,8 +89,8 @@ class AverageSessionInteractionStatisticItem(ResponseModel):
interactions: float
class AverageSessionInteractionStatisticResponse(ResponseModel):
data: list[AverageSessionInteractionStatisticItem]
class AverageSessionInteractionStatisticResponse(StatisticDataResponse[AverageSessionInteractionStatisticItem]):
pass
class UserSatisfactionRateStatisticItem(ResponseModel):
@@ -91,8 +98,8 @@ class UserSatisfactionRateStatisticItem(ResponseModel):
rate: float
class UserSatisfactionRateStatisticResponse(ResponseModel):
data: list[UserSatisfactionRateStatisticItem]
class UserSatisfactionRateStatisticResponse(StatisticDataResponse[UserSatisfactionRateStatisticItem]):
pass
class AverageResponseTimeStatisticItem(ResponseModel):
@@ -100,8 +107,8 @@ class AverageResponseTimeStatisticItem(ResponseModel):
latency: float
class AverageResponseTimeStatisticResponse(ResponseModel):
data: list[AverageResponseTimeStatisticItem]
class AverageResponseTimeStatisticResponse(StatisticDataResponse[AverageResponseTimeStatisticItem]):
pass
class TokensPerSecondStatisticItem(ResponseModel):
@@ -109,20 +116,27 @@ class TokensPerSecondStatisticItem(ResponseModel):
tps: float
class TokensPerSecondStatisticResponse(ResponseModel):
data: list[TokensPerSecondStatisticItem]
class TokensPerSecondStatisticResponse(StatisticDataResponse[TokensPerSecondStatisticItem]):
pass
register_schema_models(console_ns, StatisticTimeRangeQuery)
register_response_schema_models(
console_ns,
DailyMessageStatisticItem,
DailyMessageStatisticResponse,
DailyConversationStatisticItem,
DailyConversationStatisticResponse,
DailyTerminalStatisticItem,
DailyTerminalStatisticResponse,
DailyTokenCostStatisticItem,
DailyTokenCostStatisticResponse,
AverageSessionInteractionStatisticItem,
AverageSessionInteractionStatisticResponse,
UserSatisfactionRateStatisticItem,
UserSatisfactionRateStatisticResponse,
AverageResponseTimeStatisticItem,
AverageResponseTimeStatisticResponse,
TokensPerSecondStatisticItem,
TokensPerSecondStatisticResponse,
)
@@ -131,8 +145,7 @@ register_response_schema_models(
class DailyMessageStatistic(Resource):
@console_ns.doc("get_daily_message_statistics")
@console_ns.doc(description="Get daily message statistics for an application")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(StatisticTimeRangeQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(StatisticTimeRangeQuery)})
@console_ns.response(
200,
"Daily message statistics retrieved successfully",
@@ -185,15 +198,14 @@ WHERE
for i in rs:
response_data.append({"date": str(i.date), "message_count": i.message_count})
return jsonify({"data": response_data})
return dump_response(DailyMessageStatisticResponse, {"data": response_data})
@console_ns.route("/apps/<uuid:app_id>/statistics/daily-conversations")
class DailyConversationStatistic(Resource):
@console_ns.doc("get_daily_conversation_statistics")
@console_ns.doc(description="Get daily conversation statistics for an application")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(StatisticTimeRangeQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(StatisticTimeRangeQuery)})
@console_ns.response(
200,
"Daily conversation statistics retrieved successfully",
@@ -245,15 +257,14 @@ WHERE
for i in rs:
response_data.append({"date": str(i.date), "conversation_count": i.conversation_count})
return jsonify({"data": response_data})
return dump_response(DailyConversationStatisticResponse, {"data": response_data})
@console_ns.route("/apps/<uuid:app_id>/statistics/daily-end-users")
class DailyTerminalsStatistic(Resource):
@console_ns.doc("get_daily_terminals_statistics")
@console_ns.doc(description="Get daily terminal/end-user statistics for an application")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(StatisticTimeRangeQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(StatisticTimeRangeQuery)})
@console_ns.response(
200,
"Daily terminal statistics retrieved successfully",
@@ -306,15 +317,14 @@ WHERE
for i in rs:
response_data.append({"date": str(i.date), "terminal_count": i.terminal_count})
return jsonify({"data": response_data})
return dump_response(DailyTerminalStatisticResponse, {"data": response_data})
@console_ns.route("/apps/<uuid:app_id>/statistics/token-costs")
class DailyTokenCostStatistic(Resource):
@console_ns.doc("get_daily_token_cost_statistics")
@console_ns.doc(description="Get daily token cost statistics for an application")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(StatisticTimeRangeQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(StatisticTimeRangeQuery)})
@console_ns.response(
200,
"Daily token cost statistics retrieved successfully",
@@ -370,15 +380,14 @@ WHERE
{"date": str(i.date), "token_count": i.token_count, "total_price": i.total_price, "currency": "USD"}
)
return jsonify({"data": response_data})
return dump_response(DailyTokenCostStatisticResponse, {"data": response_data})
@console_ns.route("/apps/<uuid:app_id>/statistics/average-session-interactions")
class AverageSessionInteractionStatistic(Resource):
@console_ns.doc("get_average_session_interaction_statistics")
@console_ns.doc(description="Get average session interaction statistics for an application")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(StatisticTimeRangeQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(StatisticTimeRangeQuery)})
@console_ns.response(
200,
"Average session interaction statistics retrieved successfully",
@@ -450,15 +459,14 @@ ORDER BY
{"date": str(i.date), "interactions": float(i.interactions.quantize(Decimal("0.01")))}
)
return jsonify({"data": response_data})
return dump_response(AverageSessionInteractionStatisticResponse, {"data": response_data})
@console_ns.route("/apps/<uuid:app_id>/statistics/user-satisfaction-rate")
class UserSatisfactionRateStatistic(Resource):
@console_ns.doc("get_user_satisfaction_rate_statistics")
@console_ns.doc(description="Get user satisfaction rate statistics for an application")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(StatisticTimeRangeQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(StatisticTimeRangeQuery)})
@console_ns.response(
200,
"User satisfaction rate statistics retrieved successfully",
@@ -520,15 +528,14 @@ WHERE
}
)
return jsonify({"data": response_data})
return dump_response(UserSatisfactionRateStatisticResponse, {"data": response_data})
@console_ns.route("/apps/<uuid:app_id>/statistics/average-response-time")
class AverageResponseTimeStatistic(Resource):
@console_ns.doc("get_average_response_time_statistics")
@console_ns.doc(description="Get average response time statistics for an application")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(StatisticTimeRangeQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(StatisticTimeRangeQuery)})
@console_ns.response(
200,
"Average response time statistics retrieved successfully",
@@ -581,15 +588,14 @@ WHERE
for i in rs:
response_data.append({"date": str(i.date), "latency": round(i.latency * 1000, 4)})
return jsonify({"data": response_data})
return dump_response(AverageResponseTimeStatisticResponse, {"data": response_data})
@console_ns.route("/apps/<uuid:app_id>/statistics/tokens-per-second")
class TokensPerSecondStatistic(Resource):
@console_ns.doc("get_tokens_per_second_statistics")
@console_ns.doc(description="Get tokens per second statistics for an application")
@console_ns.doc(params={"app_id": "Application ID"})
@console_ns.doc(params=query_params_from_model(StatisticTimeRangeQuery))
@console_ns.doc(params={"app_id": "Application ID", **query_params_from_model(StatisticTimeRangeQuery)})
@console_ns.response(
200,
"Tokens per second statistics retrieved successfully",
@@ -645,4 +651,4 @@ WHERE
for i in rs:
response_data.append({"date": str(i.date), "tps": round(i.tokens_per_second, 4)})
return jsonify({"data": response_data})
return dump_response(TokensPerSecondStatisticResponse, {"data": response_data})
api/controllers/console/app/workflow.py
+81 -6
View File
@@ -6,7 +6,7 @@ from typing import Any, NotRequired, TypedDict, cast
from flask import abort, request
from flask_restx import Resource, fields
from pydantic import AliasChoices, BaseModel, Field, RootModel, ValidationError, field_validator
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, RootModel, ValidationError, field_validator
from sqlalchemy.orm import Session, sessionmaker
from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError, NotFound
@@ -78,6 +78,7 @@ from repositories.workflow_collaboration_repository import WORKFLOW_ONLINE_USERS
from services.app_generate_service import AppGenerateService
from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError
from services.errors.llm import InvokeRateLimitError
from services.workflow_ref_service import WorkflowRefService
from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService
logger = logging.getLogger(__name__)
@@ -158,8 +159,71 @@ class ConvertToWorkflowPayload(BaseModel):
icon_background: str | None = None
class WorkflowFeatureTogglePayload(BaseModel):
model_config = ConfigDict(extra="allow")
enabled: bool | None = None
class WorkflowSuggestedQuestionsAfterAnswerPayload(WorkflowFeatureTogglePayload):
model: dict[str, Any] | None = None
prompt: str | None = None
class WorkflowTextToSpeechPayload(WorkflowFeatureTogglePayload):
language: str | None = None
voice: str | None = None
autoPlay: str | None = None
class WorkflowSensitiveWordAvoidancePayload(WorkflowFeatureTogglePayload):
type: str | None = None
config: dict[str, Any] | None = None
class WorkflowFileUploadTransferPayload(WorkflowFeatureTogglePayload):
number_limits: int | None = None
transfer_methods: list[str] | None = None
class WorkflowFileUploadImagePayload(WorkflowFileUploadTransferPayload):
detail: str | None = None
class WorkflowFileUploadPreviewConfigPayload(BaseModel):
mode: str | None = None
file_type_list: list[str] | None = None
class WorkflowFileUploadPayload(WorkflowFeatureTogglePayload):
allowed_file_types: list[str] | None = None
allowed_file_extensions: list[str] | None = None
allowed_file_upload_methods: list[str] | None = None
number_limits: int | None = None
image: WorkflowFileUploadImagePayload | None = None
document: WorkflowFileUploadTransferPayload | None = None
audio: WorkflowFileUploadTransferPayload | None = None
video: WorkflowFileUploadTransferPayload | None = None
custom: WorkflowFileUploadTransferPayload | None = None
preview_config: WorkflowFileUploadPreviewConfigPayload | None = None
fileUploadConfig: dict[str, Any] | None = None
class WorkflowFeaturesConfigPayload(BaseModel):
model_config = ConfigDict(extra="allow")
opening_statement: str | None = None
suggested_questions: list[str] | None = None
suggested_questions_after_answer: WorkflowSuggestedQuestionsAfterAnswerPayload | None = None
text_to_speech: WorkflowTextToSpeechPayload | None = None
speech_to_text: WorkflowFeatureTogglePayload | None = None
retriever_resource: WorkflowFeatureTogglePayload | None = None
sensitive_word_avoidance: WorkflowSensitiveWordAvoidancePayload | None = None
file_upload: WorkflowFileUploadPayload | None = None
class WorkflowFeaturesPayload(BaseModel):
features: dict[str, Any] = Field(
features: WorkflowFeaturesConfigPayload = Field(
...,
description="Workflow feature configuration",
)
@@ -343,6 +407,15 @@ register_schema_models(
ConvertToWorkflowPayload,
WorkflowListQuery,
WorkflowUpdatePayload,
WorkflowFeatureTogglePayload,
WorkflowSuggestedQuestionsAfterAnswerPayload,
WorkflowTextToSpeechPayload,
WorkflowSensitiveWordAvoidancePayload,
WorkflowFileUploadTransferPayload,
WorkflowFileUploadImagePayload,
WorkflowFileUploadPreviewConfigPayload,
WorkflowFileUploadPayload,
WorkflowFeaturesConfigPayload,
WorkflowFeaturesPayload,
WorkflowOnlineUsersPayload,
DraftWorkflowTriggerRunPayload,
@@ -1274,7 +1347,7 @@ class WorkflowFeaturesApi(Resource):
def post(self, current_user: Account, app_model: App):
args = WorkflowFeaturesPayload.model_validate(console_ns.payload or {})
features = args.features
features = args.features.model_dump(mode="json", exclude_unset=True)
workflow_service = WorkflowService()
workflow_service.update_draft_workflow_features(app_model=app_model, features=features, account=current_user)
@@ -1406,15 +1479,15 @@ class WorkflowByIdApi(Resource):
return {"message": "No valid fields to update"}, 400
workflow_service = WorkflowService()
workflow_ref = WorkflowRefService.create_app_workflow_ref(app_model, workflow_id)
# Create a session and manage the transaction
with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
workflow = workflow_service.update_workflow(
session=session,
workflow_id=workflow_id,
tenant_id=app_model.tenant_id,
account_id=current_user.id,
data=update_data,
workflow_ref=workflow_ref,
)
if not workflow:
@@ -1434,12 +1507,14 @@ class WorkflowByIdApi(Resource):
Delete workflow
"""
workflow_service = WorkflowService()
workflow_ref = WorkflowRefService.create_app_workflow_ref(app_model, workflow_id)
# Create a session and manage the transaction
with sessionmaker(db.engine).begin() as session:
try:
workflow_service.delete_workflow(
session=session, workflow_id=workflow_id, tenant_id=app_model.tenant_id
session=session,
workflow_ref=workflow_ref,
)
except WorkflowInUseError as e:
abort(400, description=str(e))
api/controllers/console/app/workflow_comment.py
+2 -2
View File
@@ -189,14 +189,14 @@ class WorkflowCommentReplyUpdate(ResponseModel):
register_schema_models(
console_ns,
AccountWithRole,
WorkflowCommentMentionUsersPayload,
WorkflowCommentCreatePayload,
WorkflowCommentUpdatePayload,
WorkflowCommentReplyPayload,
)
register_response_schema_models(
console_ns,
AccountWithRole,
WorkflowCommentMentionUsersPayload,
WorkflowCommentAccount,
WorkflowCommentReply,
WorkflowCommentMention,
api/controllers/console/app/workflow_draft_variable.py
+29 -5
View File
@@ -6,7 +6,7 @@ from uuid import UUID
from flask import Response, request
from flask_restx import Resource, fields, marshal, marshal_with
from pydantic import BaseModel, Field
from pydantic import BaseModel, ConfigDict, Field
from sqlalchemy.orm import sessionmaker
from controllers.common.errors import InvalidArgumentError, NotFoundError
@@ -79,15 +79,33 @@ class WorkflowDraftVariableUpdatePayload(BaseModel):
value: Any | None = Field(default=None, description="Variable value")
class WorkflowVariableItemPayload(BaseModel):
model_config = ConfigDict(extra="allow")
id: str | None = None
name: str | None = None
value_type: str | None = None
value: Any | None = None
description: str | None = None
class ConversationVariableItemPayload(WorkflowVariableItemPayload):
pass
class EnvironmentVariableItemPayload(WorkflowVariableItemPayload):
pass
class ConversationVariableUpdatePayload(BaseModel):
conversation_variables: list[dict[str, Any]] = Field(
conversation_variables: list[ConversationVariableItemPayload] = Field(
...,
description="Conversation variables for the draft workflow",
)
class EnvironmentVariableUpdatePayload(BaseModel):
environment_variables: list[dict[str, Any]] = Field(
environment_variables: list[EnvironmentVariableItemPayload] = Field(
...,
description="Environment variables for the draft workflow",
)
@@ -114,7 +132,9 @@ register_schema_models(
console_ns,
WorkflowDraftVariableListQuery,
WorkflowDraftVariableUpdatePayload,
ConversationVariableItemPayload,
ConversationVariableUpdatePayload,
EnvironmentVariableItemPayload,
EnvironmentVariableUpdatePayload,
)
register_response_schema_models(console_ns, SimpleResultResponse, EnvironmentVariableListResponse)
@@ -615,7 +635,9 @@ class ConversationVariableCollectionApi(Resource):
workflow_service = WorkflowService()
conversation_variables_list = payload.conversation_variables
conversation_variables_list = [
variable.model_dump(mode="json", exclude_unset=True) for variable in payload.conversation_variables
]
conversation_variables = [
variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list
]
@@ -707,7 +729,9 @@ class EnvironmentVariableCollectionApi(Resource):
workflow_service = WorkflowService()
environment_variables_list = payload.environment_variables
environment_variables_list = [
variable.model_dump(mode="json", exclude_unset=True) for variable in payload.environment_variables
]
environment_variables = [
variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list
]
api/controllers/console/app/workflow_trigger.py
+4 -5
View File
@@ -12,6 +12,7 @@ from configs import dify_config
from controllers.common.schema import query_params_from_model, register_schema_models
from extensions.ext_database import db
from fields.base import ResponseModel
from libs.helper import dump_response
from libs.login import login_required
from models.enums import AppTriggerStatus
from models.model import App, AppMode
@@ -121,7 +122,7 @@ class WebhookTriggerApi(Resource):
if not webhook_trigger:
raise NotFound("Webhook trigger not found for this node")
return WebhookTriggerResponse.model_validate(webhook_trigger, from_attributes=True).model_dump(mode="json")
return dump_response(WebhookTriggerResponse, webhook_trigger)
@console_ns.route("/apps/<uuid:app_id>/triggers")
@@ -160,9 +161,7 @@ class AppTriggersApi(Resource):
else:
trigger.icon = "" # type: ignore
return WorkflowTriggerListResponse.model_validate({"data": triggers}, from_attributes=True).model_dump(
mode="json"
)
return dump_response(WorkflowTriggerListResponse, {"data": triggers})
@console_ns.route("/apps/<uuid:app_id>/trigger-enable")
@@ -204,4 +203,4 @@ class AppTriggerEnableApi(Resource):
else:
trigger.icon = "" # type: ignore
return WorkflowTriggerResponse.model_validate(trigger, from_attributes=True).model_dump(mode="json")
return dump_response(WorkflowTriggerResponse, trigger)
api/controllers/console/billing/billing.py
+7 -2
View File
@@ -17,6 +17,7 @@ from controllers.console.wraps import (
)
from enums.cloud_plan import CloudPlan
from extensions.ext_database import db
from fields.base import ResponseModel
from libs.login import login_required
from models import Account
from services.billing_service import BillingService
@@ -35,8 +36,12 @@ class BillingResponse(RootModel[dict[str, Any]]):
root: dict[str, Any]
class BillingInvoiceResponse(ResponseModel):
url: str
register_schema_models(console_ns, SubscriptionQuery, PartnerTenantsPayload)
register_response_schema_models(console_ns, BillingResponse)
register_response_schema_models(console_ns, BillingResponse, BillingInvoiceResponse)
@console_ns.route("/billing/subscription")
@@ -57,7 +62,7 @@ class Subscription(Resource):
@console_ns.route("/billing/invoices")
class Invoices(Resource):
@console_ns.response(200, "Success", console_ns.models[BillingResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[BillingInvoiceResponse.__name__])
@setup_required
@login_required
@account_initialization_required
api/controllers/console/datasets/data_source.py
+65 -66
View File
@@ -243,72 +243,71 @@ class DataSourceNotionListApi(Resource):
if not credential:
raise NotFound("Credential not found.")
exist_page_ids = []
with sessionmaker(db.engine).begin() as session:
# import notion in the exist dataset
if query.dataset_id:
dataset = DatasetService.get_dataset(query.dataset_id)
if not dataset:
raise NotFound("Dataset not found.")
if dataset.data_source_type != "notion_import":
raise ValueError("Dataset is not notion type.")
# import notion in the exist dataset
if query.dataset_id:
dataset = DatasetService.get_dataset(query.dataset_id, db.session)
if not dataset:
raise NotFound("Dataset not found.")
if dataset.data_source_type != "notion_import":
raise ValueError("Dataset is not notion type.")
documents = session.scalars(
select(Document).where(
Document.dataset_id == query.dataset_id,
Document.tenant_id == current_tenant_id,
Document.data_source_type == "notion_import",
Document.enabled.is_(True),
)
).all()
if documents:
for document in documents:
data_source_info = json.loads(document.data_source_info)
exist_page_ids.append(data_source_info["notion_page_id"])
# get all authorized pages
from core.datasource.datasource_manager import DatasourceManager
datasource_runtime = DatasourceManager.get_datasource_runtime(
provider_id="langgenius/notion_datasource/notion_datasource",
datasource_name="notion_datasource",
tenant_id=current_tenant_id,
datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,
)
datasource_provider_service = DatasourceProviderService()
if credential:
datasource_runtime.runtime.credentials = credential
datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime)
online_document_result: Generator[OnlineDocumentPagesMessage, None, None] = (
datasource_runtime.get_online_document_pages(
user_id=current_user.id,
datasource_parameters={},
provider_type=datasource_runtime.datasource_provider_type(),
documents = db.session.scalars(
select(Document).where(
Document.dataset_id == query.dataset_id,
Document.tenant_id == current_tenant_id,
Document.data_source_type == "notion_import",
Document.enabled.is_(True),
)
).all()
if documents:
for document in documents:
data_source_info = json.loads(document.data_source_info)
exist_page_ids.append(data_source_info["notion_page_id"])
# get all authorized pages
from core.datasource.datasource_manager import DatasourceManager
datasource_runtime = DatasourceManager.get_datasource_runtime(
provider_id="langgenius/notion_datasource/notion_datasource",
datasource_name="notion_datasource",
tenant_id=current_tenant_id,
datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,
)
datasource_provider_service = DatasourceProviderService()
if credential:
datasource_runtime.runtime.credentials = credential
datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime)
online_document_result: Generator[OnlineDocumentPagesMessage, None, None] = (
datasource_runtime.get_online_document_pages(
user_id=current_user.id,
datasource_parameters={},
provider_type=datasource_runtime.datasource_provider_type(),
)
try:
pages = []
workspace_info = {}
for message in online_document_result:
result = message.result
for info in result:
workspace_info = {
"workspace_id": info.workspace_id,
"workspace_name": info.workspace_name,
"workspace_icon": info.workspace_icon,
)
try:
pages = []
workspace_info = {}
for message in online_document_result:
result = message.result
for info in result:
workspace_info = {
"workspace_id": info.workspace_id,
"workspace_name": info.workspace_name,
"workspace_icon": info.workspace_icon,
}
for page in info.pages:
page_info = {
"page_id": page.page_id,
"page_name": page.page_name,
"type": page.type,
"parent_id": page.parent_id,
"is_bound": page.page_id in exist_page_ids,
"page_icon": page.page_icon,
}
for page in info.pages:
page_info = {
"page_id": page.page_id,
"page_name": page.page_name,
"type": page.type,
"parent_id": page.parent_id,
"is_bound": page.page_id in exist_page_ids,
"page_icon": page.page_icon,
}
pages.append(page_info)
except Exception as e:
raise e
notion_info = [{**workspace_info, "pages": pages}] if workspace_info else []
return dump_response(NotionIntegrateInfoListResponse, {"notion_info": notion_info}), 200
pages.append(page_info)
except Exception as e:
raise e
notion_info = [{**workspace_info, "pages": pages}] if workspace_info else []
return dump_response(NotionIntegrateInfoListResponse, {"notion_info": notion_info}), 200
@console_ns.route("/notion/pages/<uuid:page_id>/<string:page_type>/preview")
@@ -401,11 +400,11 @@ class DataSourceNotionDatasetSyncApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_CREATE_AND_MANAGEMENT)
def get(self, dataset_id: UUID) -> tuple[dict[str, str], int]:
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
documents = DocumentService.get_document_by_dataset_id(dataset_id_str)
documents = DocumentService.get_document_by_dataset_id(dataset_id_str, db.session)
for document in documents:
document_indexing_sync_task.delay(dataset_id_str, document.id)
return {"result": "success"}, 200
@@ -421,11 +420,11 @@ class DataSourceNotionDocumentSyncApi(Resource):
def get(self, dataset_id: UUID, document_id: UUID) -> tuple[dict[str, str], int]:
dataset_id_str = str(dataset_id)
document_id_str = str(document_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if document is None:
raise NotFound("Document not found.")
document_indexing_sync_task.delay(dataset_id_str, document_id_str)
api/controllers/console/datasets/datasets.py
+25 -21
View File
@@ -561,6 +561,7 @@ class DatasetListApi(Resource):
provider=payload.provider,
external_knowledge_api_id=payload.external_knowledge_api_id,
external_knowledge_id=payload.external_knowledge_id,
session=db.session,
)
except services.errors.dataset.DatasetNameDuplicateError:
raise DatasetNameDuplicateError()
@@ -598,7 +599,7 @@ class DatasetApi(Resource):
@with_current_tenant_id
def get(self, current_tenant_id: str, current_user: Account, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
try:
@@ -618,7 +619,7 @@ class DatasetApi(Resource):
provider_id = ModelProviderID(dataset.embedding_model_provider)
data["embedding_model_provider"] = str(provider_id)
if data.get("permission") == "partial_members":
part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session)
data.update({"partial_member_list": part_users_list})
# check embedding setting
@@ -661,7 +662,7 @@ class DatasetApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_EDIT)
def patch(self, current_tenant_id: str, current_user: Account, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
@@ -680,10 +681,10 @@ class DatasetApi(Resource):
# The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
if not dify_config.RBAC_ENABLED:
DatasetPermissionService.check_permission(
current_user, dataset, payload.permission, payload.partial_member_list
current_user, dataset, payload.permission, payload.partial_member_list, db.session
)
dataset = DatasetService.update_dataset(dataset_id_str, payload_data, current_user)
dataset = DatasetService.update_dataset(dataset_id_str, payload_data, current_user, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
@@ -698,12 +699,14 @@ class DatasetApi(Resource):
tenant_id = current_tenant_id
if payload.partial_member_list is not None and payload.permission == DatasetPermissionEnum.PARTIAL_TEAM:
DatasetPermissionService.update_partial_member_list(tenant_id, dataset_id_str, payload.partial_member_list)
DatasetPermissionService.update_partial_member_list(
tenant_id, dataset_id_str, payload.partial_member_list, db.session
)
# clear partial member list when permission is only_me or all_team_members
elif payload.permission in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM}:
DatasetPermissionService.clear_partial_member_list(dataset_id_str)
DatasetPermissionService.clear_partial_member_list(dataset_id_str, db.session)
partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session)
result_data.update({"partial_member_list": partial_member_list})
return result_data, 200
@@ -722,8 +725,8 @@ class DatasetApi(Resource):
raise Forbidden()
try:
if DatasetService.delete_dataset(dataset_id_str, current_user):
DatasetPermissionService.clear_partial_member_list(dataset_id_str)
if DatasetService.delete_dataset(dataset_id_str, current_user, db.session):
DatasetPermissionService.clear_partial_member_list(dataset_id_str, db.session)
return "", 204
else:
raise NotFound("Dataset not found.")
@@ -748,7 +751,7 @@ class DatasetUseCheckApi(Resource):
def get(self, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset_is_using = DatasetService.dataset_use_check(dataset_id_str)
dataset_is_using = DatasetService.dataset_use_check(dataset_id_str, db.session)
return {"is_using": dataset_is_using}, 200
@@ -769,7 +772,7 @@ class DatasetQueryApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_READONLY)
def get(self, current_user: Account, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
@@ -910,7 +913,7 @@ class DatasetRelatedAppListApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_READONLY)
def get(self, current_user: Account, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
@@ -919,7 +922,7 @@ class DatasetRelatedAppListApi(Resource):
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
app_dataset_joins = DatasetService.get_related_apps(dataset.id)
app_dataset_joins = DatasetService.get_related_apps(dataset.id, db.session)
related_apps = []
for app_dataset_join in app_dataset_joins:
@@ -1094,7 +1097,7 @@ class DatasetEnableApiApi(Resource):
def post(self, dataset_id: UUID, status: str):
dataset_id_str = str(dataset_id)
DatasetService.update_dataset_api_status(dataset_id_str, status == "enable")
DatasetService.update_dataset_api_status(dataset_id_str, status == "enable", db.session)
return {"result": "success"}, 200
@@ -1163,10 +1166,10 @@ class DatasetErrorDocs(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_READONLY)
def get(self, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
results = DocumentService.get_error_documents_by_dataset_id(dataset_id_str)
results = DocumentService.get_error_documents_by_dataset_id(dataset_id_str, db.session)
return dump_response(ErrorDocsResponse, {"data": results, "total": len(results)}), 200
@@ -1190,7 +1193,7 @@ class DatasetPermissionUserListApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_READONLY)
def get(self, current_user: Account, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
try:
@@ -1198,7 +1201,7 @@ class DatasetPermissionUserListApi(Resource):
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
partial_members_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
partial_members_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session)
return dump_response(PartialMemberListResponse, {"data": partial_members_list}), 200
@@ -1220,7 +1223,8 @@ class DatasetAutoDisableLogApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_READONLY)
def get(self, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
return dump_response(AutoDisableLogsResponse, DatasetService.get_dataset_auto_disable_logs(dataset_id_str)), 200
auto_disable_logs = DatasetService.get_dataset_auto_disable_logs(dataset_id_str, db.session)
return dump_response(AutoDisableLogsResponse, auto_disable_logs), 200
api/controllers/console/datasets/datasets_document.py
+46 -38
View File
@@ -49,6 +49,7 @@ from libs.login import login_required
from models import Account, DatasetProcessRule, Document, DocumentSegment, UploadFile
from models.dataset import DocumentPipelineExecutionLog
from models.enums import IndexingStatus, SegmentStatus
from services.dataset_ref_service import DatasetRefService
from services.dataset_service import DatasetService, DocumentService
from services.entities.knowledge_entities.knowledge_entities import KnowledgeConfig, ProcessRule, RetrievalModel
from services.file_service import FileService
@@ -181,7 +182,7 @@ class DocumentResource(Resource):
def get_document(
self, dataset_id: str, document_id: str, current_user: Account, current_tenant_id: str
) -> Document:
dataset = DatasetService.get_dataset(dataset_id)
dataset = DatasetService.get_dataset(dataset_id, db.session)
if not dataset:
raise NotFound("Dataset not found.")
@@ -190,7 +191,7 @@ class DocumentResource(Resource):
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
document = DocumentService.get_document(dataset_id, document_id)
document = DocumentService.get_document(dataset_id, document_id, session=db.session)
if not document:
raise NotFound("Document not found.")
@@ -201,7 +202,7 @@ class DocumentResource(Resource):
return document
def get_batch_documents(self, dataset_id: str, batch: str, current_user: Account) -> Sequence[Document]:
dataset = DatasetService.get_dataset(dataset_id)
dataset = DatasetService.get_dataset(dataset_id, db.session)
if not dataset:
raise NotFound("Dataset not found.")
@@ -210,7 +211,7 @@ class DocumentResource(Resource):
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
documents = DocumentService.get_batch_documents(dataset_id, batch)
documents = DocumentService.get_batch_documents(dataset_id, batch, db.session)
if not documents:
raise NotFound("Documents not found.")
@@ -241,7 +242,7 @@ class GetProcessRuleApi(Resource):
# get the latest process rule
document = db.get_or_404(Document, document_id)
dataset = DatasetService.get_dataset(document.dataset_id)
dataset = DatasetService.get_dataset(document.dataset_id, db.session)
if not dataset:
raise NotFound("Dataset not found.")
@@ -317,7 +318,7 @@ class DatasetDocumentListApi(Resource):
)
except (ArgumentTypeError, ValueError, Exception):
fetch = False
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
@@ -421,7 +422,7 @@ class DatasetDocumentListApi(Resource):
def post(self, current_user: Account, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
@@ -444,8 +445,10 @@ class DatasetDocumentListApi(Resource):
DocumentService.document_create_args_validate(knowledge_config)
try:
documents, batch = DocumentService.save_document_with_dataset_id(dataset, knowledge_config, current_user)
dataset = DatasetService.get_dataset(dataset_id_str)
documents, batch = DocumentService.save_document_with_dataset_id(
dataset, knowledge_config, current_user, session=db.session
)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
@@ -464,7 +467,7 @@ class DatasetDocumentListApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_EDIT)
def delete(self, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
# check user's model setting
@@ -472,7 +475,8 @@ class DatasetDocumentListApi(Resource):
try:
document_ids = request.args.getlist("document_id")
DocumentService.delete_documents(dataset, document_ids)
dataset_ref = DatasetRefService.create_dataset_ref(dataset)
DocumentService.delete_documents(dataset_ref, document_ids, dataset.doc_form, db.session)
except services.errors.document.DocumentIndexingError:
raise DocumentIndexingError("Cannot delete document during indexing.")
@@ -531,6 +535,7 @@ class DatasetInitApi(Resource):
tenant_id=current_tenant_id,
knowledge_config=knowledge_config,
account=current_user,
session=db.session,
)
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
@@ -867,7 +872,7 @@ class DocumentApi(DocumentResource):
if metadata == "only":
response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
elif metadata == "without":
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str)
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session)
document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
response = {
"id": document.id,
@@ -901,7 +906,7 @@ class DocumentApi(DocumentResource):
"need_summary": document.need_summary if document.need_summary is not None else False,
}
else:
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str)
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session)
document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
response = {
"id": document.id,
@@ -950,7 +955,7 @@ class DocumentApi(DocumentResource):
def delete(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):
dataset_id_str = str(dataset_id)
document_id_str = str(document_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
# check user's model setting
@@ -959,7 +964,7 @@ class DocumentApi(DocumentResource):
document = self.get_document(dataset_id_str, document_id_str, current_user, current_tenant_id)
try:
DocumentService.delete_document(document)
DocumentService.delete_document(document, db.session)
except services.errors.document.DocumentIndexingError:
raise DocumentIndexingError("Cannot delete document during indexing.")
@@ -983,7 +988,7 @@ class DocumentDownloadApi(DocumentResource):
def get(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID) -> dict[str, Any]:
# Reuse the shared permission/tenant checks implemented in DocumentResource.
document = self.get_document(str(dataset_id), str(document_id), current_user, current_tenant_id)
return {"url": DocumentService.get_document_download_url(document)}
return {"url": DocumentService.get_document_download_url(document, db.session)}
@console_ns.route("/datasets/<uuid:dataset_id>/documents/download-zip")
@@ -1013,6 +1018,7 @@ class DocumentBatchDownloadZipApi(DocumentResource):
document_ids=document_ids,
tenant_id=current_tenant_id,
current_user=current_user,
session=db.session,
)
# Delegate ZIP packing to FileService, but keep Flask response+cleanup in the route.
@@ -1161,7 +1167,7 @@ class DocumentStatusApi(DocumentResource):
self, current_user: Account, dataset_id: UUID, action: Literal["enable", "disable", "archive", "un_archive"]
):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
@@ -1178,7 +1184,7 @@ class DocumentStatusApi(DocumentResource):
document_ids = request.args.getlist("document_id")
try:
DocumentService.batch_update_document_status(dataset, document_ids, action, current_user)
DocumentService.batch_update_document_status(dataset, document_ids, action, current_user, db.session)
except services.errors.document.DocumentIndexingError as e:
raise InvalidActionError(str(e))
except ValueError as e:
@@ -1202,11 +1208,11 @@ class DocumentPauseApi(DocumentResource):
dataset_id_str = str(dataset_id)
document_id_str = str(document_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
# 404 if document not found
if document is None:
@@ -1218,7 +1224,7 @@ class DocumentPauseApi(DocumentResource):
try:
# pause document
DocumentService.pause_document(document)
DocumentService.pause_document(document, db.session)
except services.errors.document.DocumentIndexingError:
raise DocumentIndexingError("Cannot pause completed document.")
@@ -1237,10 +1243,10 @@ class DocumentRecoverApi(DocumentResource):
"""recover document."""
dataset_id_str = str(dataset_id)
document_id_str = str(document_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
# 404 if document not found
if document is None:
@@ -1251,7 +1257,7 @@ class DocumentRecoverApi(DocumentResource):
raise ArchivedDocumentImmutableError()
try:
# pause document
DocumentService.recover_document(document)
DocumentService.recover_document(document, db.session)
except services.errors.document.DocumentIndexingError:
raise DocumentIndexingError("Document is not in paused status.")
@@ -1271,13 +1277,13 @@ class DocumentRetryApi(DocumentResource):
"""retry document."""
payload = DocumentRetryPayload.model_validate(console_ns.payload or {})
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
retry_documents = []
if not dataset:
raise NotFound("Dataset not found.")
for document_id in payload.document_ids:
try:
document = DocumentService.get_document(dataset.id, document_id)
document = DocumentService.get_document(dataset.id, document_id, session=db.session)
# 404 if document not found
if document is None:
@@ -1295,7 +1301,7 @@ class DocumentRetryApi(DocumentResource):
logger.exception("Failed to retry document, document id: %s", document_id)
continue
# retry document
DocumentService.retry_document(dataset_id_str, retry_documents)
DocumentService.retry_document(dataset_id_str, retry_documents, db.session)
return "", 204
@@ -1313,14 +1319,14 @@ class DocumentRenameApi(DocumentResource):
# The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
if not current_user.is_dataset_editor:
raise Forbidden()
dataset = DatasetService.get_dataset(dataset_id)
dataset = DatasetService.get_dataset(dataset_id, db.session)
if not dataset:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_operator_permission(current_user, dataset)
DatasetService.check_dataset_operator_permission(current_user, dataset, session=db.session)
payload = DocumentRenamePayload.model_validate(console_ns.payload or {})
try:
document = DocumentService.rename_document(str(dataset_id), str(document_id), payload.name)
document = DocumentService.rename_document(str(dataset_id), str(document_id), payload.name, db.session)
except services.errors.document.DocumentIndexingError:
raise DocumentIndexingError("Cannot delete document during indexing.")
@@ -1338,11 +1344,11 @@ class WebsiteDocumentSyncApi(DocumentResource):
def get(self, current_tenant_id: str, dataset_id: UUID, document_id: UUID):
"""sync website document."""
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
document_id_str = str(document_id)
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
if document.tenant_id != current_tenant_id:
@@ -1353,7 +1359,7 @@ class WebsiteDocumentSyncApi(DocumentResource):
if DocumentService.check_archived(document):
raise ArchivedDocumentImmutableError()
# sync document
DocumentService.sync_website_document(dataset_id_str, document)
DocumentService.sync_website_document(dataset_id_str, document, db.session)
return {"result": "success"}, 200
@@ -1373,10 +1379,10 @@ class DocumentPipelineExecutionLogApi(DocumentResource):
dataset_id_str = str(dataset_id)
document_id_str = str(document_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
log = db.session.scalar(
@@ -1431,7 +1437,7 @@ class DocumentGenerateSummaryApi(Resource):
dataset_id_str = str(dataset_id)
# Get dataset
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
@@ -1465,7 +1471,7 @@ class DocumentGenerateSummaryApi(Resource):
raise ValueError("Summary index is not enabled for this dataset. Please enable it in the dataset settings.")
# Verify all documents exist and belong to the dataset
documents = DocumentService.get_documents_by_ids(dataset_id_str, document_list)
documents = DocumentService.get_documents_by_ids(dataset_id_str, document_list, db.session)
if len(documents) != len(document_list):
found_ids = {doc.id for doc in documents}
@@ -1481,6 +1487,7 @@ class DocumentGenerateSummaryApi(Resource):
DocumentService.update_documents_need_summary(
dataset_id=dataset_id_str,
document_ids=document_ids_to_update,
session=db.session,
need_summary=True,
)
@@ -1531,7 +1538,7 @@ class DocumentSummaryStatusApi(DocumentResource):
document_id_str = str(document_id)
# Get dataset
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
@@ -1547,6 +1554,7 @@ class DocumentSummaryStatusApi(DocumentResource):
result = SummaryIndexService.get_document_summary_status_detail(
document_id=document_id_str,
dataset_id=dataset_id_str,
session=db.session,
)
return result, 200
api/controllers/console/datasets/datasets_segments.py
+87 -134
View File
@@ -58,8 +58,9 @@ from graphon.model_runtime.entities.model_entities import ModelType
from libs.helper import dump_response, escape_like_pattern
from libs.login import login_required
from models import Account
from models.dataset import ChildChunk, DocumentSegment
from models.dataset import Dataset, Document, DocumentSegment
from models.model import UploadFile
from services.dataset_ref_service import DatasetRefService, SegmentRef
from services.dataset_service import DatasetService, DocumentService, SegmentService
from services.entities.knowledge_entities.knowledge_entities import ChildChunkUpdateArgs, SegmentUpdateArgs
from services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceError
@@ -162,6 +163,21 @@ register_response_schema_models(
)
def _get_segment_for_document(
dataset: Dataset, document: Document, segment_id: str
) -> tuple[SegmentRef, DocumentSegment]:
dataset_ref = DatasetRefService.create_dataset_ref(dataset)
document_ref = DatasetRefService.create_document_ref(dataset_ref, document)
if document_ref is None:
raise NotFound("Document not found.")
segment_ref = DatasetRefService.create_segment_ref(document_ref, segment_id)
segment = SegmentService.get_segment_by_ref(segment_ref)
if not segment:
raise NotFound("Segment not found.")
return segment_ref, segment
@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")
class DatasetDocumentSegmentListApi(Resource):
@console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT)
@@ -176,7 +192,7 @@ class DatasetDocumentSegmentListApi(Resource):
def get(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):
dataset_id_str = str(dataset_id)
document_id_str = str(document_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
@@ -185,7 +201,7 @@ class DatasetDocumentSegmentListApi(Resource):
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
@@ -286,14 +302,14 @@ class DatasetDocumentSegmentListApi(Resource):
def delete(self, current_user: Account, dataset_id: UUID, document_id: UUID):
# check dataset
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
# check user's model setting
DatasetService.check_dataset_model_setting(dataset)
# check document
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
segment_ids = request.args.getlist("segment_id")
@@ -305,7 +321,7 @@ class DatasetDocumentSegmentListApi(Resource):
DatasetService.check_dataset_permission(dataset, current_user, db.session)
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
SegmentService.delete_segments(segment_ids, document, dataset)
SegmentService.delete_segments(segment_ids, document, dataset, db.session)
return "", 204
@@ -331,11 +347,11 @@ class DatasetDocumentSegmentApi(Resource):
action: Literal["enable", "disable"],
):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
# check user's model setting
@@ -371,7 +387,7 @@ class DatasetDocumentSegmentApi(Resource):
if cache_result is not None:
raise InvalidActionError("Document is being indexed, please try again later")
try:
SegmentService.update_segments_status(segment_ids, action, dataset, document)
SegmentService.update_segments_status(segment_ids, action, dataset, document, db.session)
except Exception as e:
raise InvalidActionError(str(e))
return dump_response(SimpleResultResponse, {"result": "success"}), 200
@@ -394,12 +410,12 @@ class DatasetDocumentSegmentAddApi(Resource):
def post(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):
# check dataset
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
# check document
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
if not current_user.is_dataset_editor:
@@ -428,7 +444,7 @@ class DatasetDocumentSegmentAddApi(Resource):
payload = SegmentCreatePayload.model_validate(console_ns.payload or {})
payload_dict = payload.model_dump(exclude_none=True)
SegmentService.segment_create_args_validate(payload_dict, document)
segment = type_cast(DocumentSegment, SegmentService.create_segment(payload_dict, document, dataset))
segment = type_cast(DocumentSegment, SegmentService.create_segment(payload_dict, document, dataset, db.session))
summary = SummaryIndexService.get_segment_summary(segment_id=segment.id, dataset_id=dataset_id_str)
response = {
"data": segment_response_with_summary(segment, summary.summary_content if summary else None),
@@ -455,16 +471,23 @@ class DatasetDocumentSegmentUpdateApi(Resource):
):
# check dataset
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
# check user's model setting
DatasetService.check_dataset_model_setting(dataset)
# check document
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
# The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
if not current_user.is_dataset_editor:
raise Forbidden()
try:
DatasetService.check_dataset_permission(dataset, current_user, db.session)
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
# check embedding model setting
try:
@@ -481,22 +504,8 @@ class DatasetDocumentSegmentUpdateApi(Resource):
)
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
# check segment
segment_id_str = str(segment_id)
segment = db.session.scalar(
select(DocumentSegment)
.where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
.limit(1)
)
if not segment:
raise NotFound("Segment not found.")
# The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
if not current_user.is_dataset_editor:
raise Forbidden()
try:
DatasetService.check_dataset_permission(dataset, current_user, db.session)
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
_, segment = _get_segment_for_document(dataset, document, segment_id_str)
# validate args
payload = SegmentUpdatePayload.model_validate(console_ns.payload or {})
payload_dict = payload.model_dump(exclude_none=True)
@@ -504,7 +513,11 @@ class DatasetDocumentSegmentUpdateApi(Resource):
# Update segment (summary update with change detection is handled in SegmentService.update_segment)
segment = SegmentService.update_segment(
SegmentUpdateArgs.model_validate(payload.model_dump(exclude_none=True)), segment, document, dataset
SegmentUpdateArgs.model_validate(payload.model_dump(exclude_none=True)),
segment,
document,
dataset,
db.session,
)
summary = SummaryIndexService.get_segment_summary(segment_id=segment.id, dataset_id=dataset_id_str)
response = {
@@ -527,25 +540,16 @@ class DatasetDocumentSegmentUpdateApi(Resource):
):
# check dataset
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
# check user's model setting
DatasetService.check_dataset_model_setting(dataset)
# check document
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
# check segment
segment_id_str = str(segment_id)
segment = db.session.scalar(
select(DocumentSegment)
.where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
.limit(1)
)
if not segment:
raise NotFound("Segment not found.")
# The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
if not current_user.is_dataset_editor:
raise Forbidden()
@@ -553,7 +557,9 @@ class DatasetDocumentSegmentUpdateApi(Resource):
DatasetService.check_dataset_permission(dataset, current_user, db.session)
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
SegmentService.delete_segment(segment, document, dataset)
segment_id_str = str(segment_id)
_, segment = _get_segment_for_document(dataset, document, segment_id_str)
SegmentService.delete_segment(segment, document, dataset, db.session)
return "", 204
@@ -576,12 +582,12 @@ class DatasetDocumentSegmentBatchImportApi(Resource):
def post(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):
# check dataset
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
# check document
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
@@ -651,25 +657,20 @@ class ChildChunkAddApi(Resource):
):
# check dataset
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
# check document
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
# check segment
segment_id_str = str(segment_id)
segment = db.session.scalar(
select(DocumentSegment)
.where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
.limit(1)
)
if not segment:
raise NotFound("Segment not found.")
if not current_user.is_dataset_editor:
raise Forbidden()
try:
DatasetService.check_dataset_permission(dataset, current_user, db.session)
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
# check embedding model setting
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
try:
@@ -686,14 +687,12 @@ class ChildChunkAddApi(Resource):
)
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
try:
DatasetService.check_dataset_permission(dataset, current_user, db.session)
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
segment_id_str = str(segment_id)
_, segment = _get_segment_for_document(dataset, document, segment_id_str)
# validate args
try:
payload = ChildChunkCreatePayload.model_validate(console_ns.payload or {})
child_chunk = SegmentService.create_child_chunk(payload.content, segment, document, dataset)
child_chunk = SegmentService.create_child_chunk(payload.content, segment, document, dataset, db.session)
except ChildChunkIndexingServiceError as e:
raise ChildChunkIndexingError(str(e))
return dump_response(ChildChunkDetailResponse, {"data": child_chunk}), 200
@@ -709,25 +708,18 @@ class ChildChunkAddApi(Resource):
def get(self, current_tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
# check dataset
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
# check user's model setting
DatasetService.check_dataset_model_setting(dataset)
# check document
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
# check segment
segment_id_str = str(segment_id)
segment = db.session.scalar(
select(DocumentSegment)
.where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
.limit(1)
)
if not segment:
raise NotFound("Segment not found.")
_get_segment_for_document(dataset, document, segment_id_str)
args = query_params_from_request(ChildChunkListQuery, use_defaults_for_malformed_ints=True)
page = args.page
@@ -766,25 +758,16 @@ class ChildChunkAddApi(Resource):
):
# check dataset
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
# check user's model setting
DatasetService.check_dataset_model_setting(dataset)
# check document
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
# check segment
segment_id_str = str(segment_id)
segment = db.session.scalar(
select(DocumentSegment)
.where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
.limit(1)
)
if not segment:
raise NotFound("Segment not found.")
# The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
if not current_user.is_dataset_editor:
raise Forbidden()
@@ -792,10 +775,12 @@ class ChildChunkAddApi(Resource):
DatasetService.check_dataset_permission(dataset, current_user, db.session)
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
segment_id_str = str(segment_id)
_, segment = _get_segment_for_document(dataset, document, segment_id_str)
# validate args
payload = ChildChunkBatchUpdatePayload.model_validate(console_ns.payload or {})
try:
child_chunks = SegmentService.update_child_chunks(payload.chunks, segment, document, dataset)
child_chunks = SegmentService.update_child_chunks(payload.chunks, segment, document, dataset, db.session)
except ChildChunkIndexingServiceError as e:
raise ChildChunkIndexingError(str(e))
return dump_response(ChildChunkBatchUpdateResponse, {"data": child_chunks}), 200
@@ -825,39 +810,16 @@ class ChildChunkUpdateApi(Resource):
):
# check dataset
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
# check user's model setting
DatasetService.check_dataset_model_setting(dataset)
# check document
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
# check segment
segment_id_str = str(segment_id)
segment = db.session.scalar(
select(DocumentSegment)
.where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
.limit(1)
)
if not segment:
raise NotFound("Segment not found.")
# check child chunk
child_chunk_id_str = str(child_chunk_id)
child_chunk = db.session.scalar(
select(ChildChunk)
.where(
ChildChunk.id == child_chunk_id_str,
ChildChunk.tenant_id == current_tenant_id,
ChildChunk.segment_id == segment.id,
ChildChunk.document_id == document_id_str,
)
.limit(1)
)
if not child_chunk:
raise NotFound("Child chunk not found.")
# The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
if not current_user.is_dataset_editor:
raise Forbidden()
@@ -865,8 +827,14 @@ class ChildChunkUpdateApi(Resource):
DatasetService.check_dataset_permission(dataset, current_user, db.session)
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
segment_id_str = str(segment_id)
segment_ref, _ = _get_segment_for_document(dataset, document, segment_id_str)
child_chunk_id_str = str(child_chunk_id)
child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref)
if not child_chunk:
raise NotFound("Child chunk not found.")
try:
SegmentService.delete_child_chunk(child_chunk, dataset)
SegmentService.delete_child_chunk(child_chunk, dataset, db.session)
except ChildChunkDeleteIndexServiceError as e:
raise ChildChunkDeleteIndexError(str(e))
return "", 204
@@ -893,39 +861,16 @@ class ChildChunkUpdateApi(Resource):
):
# check dataset
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if not dataset:
raise NotFound("Dataset not found.")
# check user's model setting
DatasetService.check_dataset_model_setting(dataset)
# check document
document_id_str = str(document_id)
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
# check segment
segment_id_str = str(segment_id)
segment = db.session.scalar(
select(DocumentSegment)
.where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
.limit(1)
)
if not segment:
raise NotFound("Segment not found.")
# check child chunk
child_chunk_id_str = str(child_chunk_id)
child_chunk = db.session.scalar(
select(ChildChunk)
.where(
ChildChunk.id == child_chunk_id_str,
ChildChunk.tenant_id == current_tenant_id,
ChildChunk.segment_id == segment.id,
ChildChunk.document_id == document_id_str,
)
.limit(1)
)
if not child_chunk:
raise NotFound("Child chunk not found.")
# The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
if not current_user.is_dataset_editor:
raise Forbidden()
@@ -933,10 +878,18 @@ class ChildChunkUpdateApi(Resource):
DatasetService.check_dataset_permission(dataset, current_user, db.session)
except services.errors.account.NoPermissionError as e:
raise Forbidden(str(e))
segment_id_str = str(segment_id)
segment_ref, segment = _get_segment_for_document(dataset, document, segment_id_str)
child_chunk_id_str = str(child_chunk_id)
child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref)
if not child_chunk:
raise NotFound("Child chunk not found.")
# validate args
try:
payload = ChildChunkUpdatePayload.model_validate(console_ns.payload or {})
child_chunk = SegmentService.update_child_chunk(payload.content, child_chunk, segment, document, dataset)
child_chunk = SegmentService.update_child_chunk(
payload.content, child_chunk, segment, document, dataset, db.session
)
except ChildChunkIndexingServiceError as e:
raise ChildChunkIndexingError(str(e))
return dump_response(ChildChunkDetailResponse, {"data": child_chunk}), 200
api/controllers/console/datasets/external.py
+1 -1
View File
@@ -377,7 +377,7 @@ class ExternalKnowledgeHitTestingApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_PIPELINE_TEST)
def post(self, current_user: Account, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
api/controllers/console/datasets/hit_testing_base.py
+1 -1
View File
@@ -85,7 +85,7 @@ class DatasetsHitTestingBase:
dataset_id: str, current_user: Account | None = None, current_tenant_id: str | None = None
) -> Dataset:
current_user, _ = resolve_account_fallback(current_user, current_tenant_id)
dataset = DatasetService.get_dataset(dataset_id)
dataset = DatasetService.get_dataset(dataset_id, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
api/controllers/console/datasets/metadata.py
+6 -6
View File
@@ -61,7 +61,7 @@ class DatasetMetadataCreateApi(Resource):
metadata_args = MetadataArgs.model_validate(console_ns.payload or {})
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, db.session)
@@ -81,7 +81,7 @@ class DatasetMetadataCreateApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_CREATE_AND_MANAGEMENT)
def get(self, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
metadata = MetadataService.get_dataset_metadatas(db.session(), dataset)
@@ -105,7 +105,7 @@ class DatasetMetadataApi(Resource):
dataset_id_str = str(dataset_id)
metadata_id_str = str(metadata_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, db.session)
@@ -125,7 +125,7 @@ class DatasetMetadataApi(Resource):
def delete(self, current_user: Account, dataset_id: UUID, metadata_id: UUID):
dataset_id_str = str(dataset_id)
metadata_id_str = str(metadata_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, db.session)
@@ -162,7 +162,7 @@ class DatasetMetadataBuiltInFieldActionApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_EDIT)
def post(self, current_user: Account, dataset_id: UUID, action: Literal["enable", "disable"]):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, db.session)
@@ -191,7 +191,7 @@ class DocumentMetadataEditApi(Resource):
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_EDIT)
def post(self, current_user: Account, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, db.session)
api/controllers/console/datasets/rag_pipeline/rag_pipeline_datasets.py
+8 -8
View File
@@ -1,6 +1,5 @@
from flask_restx import Resource
from pydantic import BaseModel
from sqlalchemy.orm import Session
from werkzeug.exceptions import Forbidden
import services
@@ -66,19 +65,19 @@ class CreateRagPipelineDatasetApi(Resource):
yaml_content=payload.yaml_content,
)
try:
with Session(db.engine, expire_on_commit=False) as session:
rag_pipeline_dsl_service = RagPipelineDslService(session)
import_info = rag_pipeline_dsl_service.create_rag_pipeline_dataset(
tenant_id=current_tenant_id,
rag_pipeline_dataset_create_entity=rag_pipeline_dataset_create_entity,
)
session.commit()
rag_pipeline_dsl_service = RagPipelineDslService(db.session)
import_info = rag_pipeline_dsl_service.create_rag_pipeline_dataset(
tenant_id=current_tenant_id,
rag_pipeline_dataset_create_entity=rag_pipeline_dataset_create_entity,
)
if rag_pipeline_dataset_create_entity.permission == "partial_members":
DatasetPermissionService.update_partial_member_list(
current_tenant_id,
import_info["dataset_id"],
rag_pipeline_dataset_create_entity.partial_member_list,
db.session,
)
db.session.commit()
except services.errors.dataset.DatasetNameDuplicateError:
raise DatasetNameDuplicateError()
@@ -111,5 +110,6 @@ class CreateEmptyRagPipelineDatasetApi(Resource):
permission=DatasetPermissionEnum.ONLY_ME,
partial_member_list=None,
),
session=db.session,
)
return dump_response(DatasetDetailResponse, dataset), 201
api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py
+5 -4
View File
@@ -64,6 +64,7 @@ from services.rag_pipeline.pipeline_generate_service import PipelineGenerateServ
from services.rag_pipeline.rag_pipeline import RagPipelineService
from services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService
from services.rag_pipeline.rag_pipeline_transform_service import RagPipelineTransformService
from services.workflow_ref_service import WorkflowRefService
from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService
logger = logging.getLogger(__name__)
@@ -738,15 +739,15 @@ class RagPipelineByIdApi(Resource):
return {"message": "No valid fields to update"}, 400
rag_pipeline_service = RagPipelineService()
workflow_ref = WorkflowRefService.create_pipeline_workflow_ref(pipeline, workflow_id)
# Create a session and manage the transaction
with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
workflow = rag_pipeline_service.update_workflow(
session=session,
workflow_id=workflow_id,
tenant_id=pipeline.tenant_id,
account_id=current_user.id,
data=update_data,
workflow_ref=workflow_ref,
)
if not workflow:
@@ -769,13 +770,13 @@ class RagPipelineByIdApi(Resource):
abort(400, description=f"Cannot delete workflow that is currently in use by pipeline '{pipeline.id}'")
workflow_service = WorkflowService()
workflow_ref = WorkflowRefService.create_pipeline_workflow_ref(pipeline, workflow_id)
with sessionmaker(db.engine).begin() as session:
try:
workflow_service.delete_workflow(
session=session,
workflow_id=workflow_id,
tenant_id=pipeline.tenant_id,
workflow_ref=workflow_ref,
)
except WorkflowInUseError as e:
abort(400, description=str(e))
api/controllers/console/explore/audio.py
+12 -1
View File
@@ -22,7 +22,9 @@ from controllers.console.explore.wraps import InstalledAppResource
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
from extensions.ext_database import db
from graphon.model_runtime.errors.invoke import InvokeError
from libs.login import current_account_with_tenant
from models.model import InstalledApp
from services.app_ref_service import AppRefService
from services.audio_service import AudioService
from services.errors.audio import (
AudioTooLargeServiceError,
@@ -99,13 +101,22 @@ class ChatTextApi(InstalledAppResource):
message_id = payload.message_id
text = payload.text
voice = payload.voice
message_ref = None
if message_id:
current_user, _ = current_account_with_tenant()
app_ref = AppRefService.create_app_ref(app_model)
message_ref = AppRefService.create_message_ref(
app_ref,
message_id,
account_id=current_user.id,
)
response = AudioService.transcript_tts(
app_model=app_model,
session=db.session,
text=text,
voice=voice,
message_id=message_id,
message_ref=message_ref,
)
return response
except services.errors.app_model_config.AppModelConfigBrokenError:
api/controllers/console/explore/installed_app.py
+5 -1
View File
@@ -147,11 +147,15 @@ register_schema_models(
InstalledAppCreatePayload,
InstalledAppUpdatePayload,
InstalledAppsListQuery,
)
register_response_schema_models(
console_ns,
InstalledAppInfoResponse,
InstalledAppResponse,
InstalledAppListResponse,
SimpleMessageResponse,
SimpleResultMessageResponse,
)
register_response_schema_models(console_ns, SimpleMessageResponse, SimpleResultMessageResponse)
@console_ns.route("/installed-apps")
api/controllers/console/explore/parameter.py
+6 -1
View File
@@ -13,7 +13,12 @@ from services.app_service import AppService
class ExploreAppMetaResponse(BaseModel):
tool_icons: dict[str, Any] = Field(default_factory=dict)
"""Metadata consumed by the installed-app chat UI.
Built-in tool icons are URL strings; API-based tool icons are provider-defined payload objects.
"""
tool_icons: dict[str, str | dict[str, Any]] = Field(default_factory=dict)
register_response_schema_models(console_ns, fields.Parameters, ExploreAppMetaResponse)
api/controllers/console/explore/recommended_app.py
+18 -4
View File
@@ -70,19 +70,33 @@ class LearnDifyAppListResponse(ResponseModel):
recommended_apps: list[RecommendedAppResponse]
class RecommendedAppDetailResponse(RootModel[dict[str, Any]]):
root: dict[str, Any]
class RecommendedAppDetailResponse(ResponseModel):
id: str
name: str
icon: str | None = None
icon_background: str | None = None
mode: str
export_data: str
can_trial: bool | None = None
class RecommendedAppDetailNullableResponse(RootModel[RecommendedAppDetailResponse | None]):
pass
register_schema_models(
console_ns,
RecommendedAppsQuery,
)
register_response_schema_models(
console_ns,
RecommendedAppInfoResponse,
RecommendedAppResponse,
RecommendedAppListResponse,
LearnDifyAppListResponse,
RecommendedAppDetailResponse,
RecommendedAppDetailNullableResponse,
)
register_response_schema_models(console_ns, RecommendedAppDetailResponse)
def _resolve_language(language: str | None, user: Account) -> str:
@@ -130,7 +144,7 @@ class LearnDifyAppListApi(Resource):
@console_ns.route("/explore/apps/<uuid:app_id>")
class RecommendedAppApi(Resource):
@console_ns.response(200, "Success", console_ns.models[RecommendedAppDetailResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[RecommendedAppDetailNullableResponse.__name__])
@login_required
@account_initialization_required
def get(self, app_id: UUID):
api/controllers/console/explore/trial.py
+268 -6
View File
@@ -1,9 +1,10 @@
import logging
from datetime import datetime
from typing import Any, Literal, cast
from flask import request
from flask_restx import Resource, fields, marshal, marshal_with
from pydantic import BaseModel, Field
from pydantic import AliasChoices, BaseModel, Field, field_validator
from sqlalchemy import select
from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
@@ -63,6 +64,7 @@ from fields.app_fields import (
site_fields,
tag_fields,
)
from fields.base import ResponseModel
from fields.dataset_fields import dataset_fields
from fields.member_fields import simple_account_fields
from fields.message_fields import SuggestedQuestionsResponse
@@ -75,12 +77,13 @@ from fields.workflow_fields import (
from graphon.graph_engine.manager import GraphEngineManager
from graphon.model_runtime.errors.invoke import InvokeError
from libs import helper
from libs.helper import uuid_value
from libs.helper import to_timestamp, uuid_value
from models import Account
from models.account import TenantStatus
from models.model import AppMode, Site
from models.workflow import Workflow
from services.app_generate_service import AppGenerateService
from services.app_ref_service import AppRefService
from services.app_service import AppService
from services.audio_service import AudioService
from services.dataset_service import DatasetService
@@ -179,6 +182,254 @@ class TrialDatasetListQuery(BaseModel):
ids: list[str] = Field(default_factory=list, description="Dataset IDs")
type TrialAppMode = Literal["chat", "agent-chat", "advanced-chat", "workflow", "completion"]
type TrialIconType = Literal["emoji", "image", "link"]
type JsonObject = dict[str, Any]
class TrialAppModel(ResponseModel):
provider: str
name: str
mode: str | None = None
completion_params: JsonObject = Field(default_factory=dict)
class TrialAppAgentMode(ResponseModel):
enabled: bool | None = None
strategy: str | None = None
tools: list[JsonObject] = Field(default_factory=list)
class TrialAppModelConfigResponse(ResponseModel):
opening_statement: str | None = None
suggested_questions: list[str] = Field(
default_factory=list,
validation_alias=AliasChoices("suggested_questions_list", "suggested_questions"),
)
suggested_questions_after_answer: JsonObject | None = Field(
default=None,
validation_alias=AliasChoices("suggested_questions_after_answer_dict", "suggested_questions_after_answer"),
)
speech_to_text: JsonObject | None = Field(
default=None, validation_alias=AliasChoices("speech_to_text_dict", "speech_to_text")
)
text_to_speech: JsonObject | None = Field(
default=None, validation_alias=AliasChoices("text_to_speech_dict", "text_to_speech")
)
retriever_resource: JsonObject | None = Field(
default=None, validation_alias=AliasChoices("retriever_resource_dict", "retriever_resource")
)
annotation_reply: JsonObject | None = Field(
default=None, validation_alias=AliasChoices("annotation_reply_dict", "annotation_reply")
)
more_like_this: JsonObject | None = Field(
default=None, validation_alias=AliasChoices("more_like_this_dict", "more_like_this")
)
sensitive_word_avoidance: JsonObject | None = Field(
default=None, validation_alias=AliasChoices("sensitive_word_avoidance_dict", "sensitive_word_avoidance")
)
external_data_tools: list[JsonObject] = Field(
default_factory=list, validation_alias=AliasChoices("external_data_tools_list", "external_data_tools")
)
model: TrialAppModel | None = Field(default=None, validation_alias=AliasChoices("model_dict", "model"))
user_input_form: list[JsonObject] = Field(
default_factory=list, validation_alias=AliasChoices("user_input_form_list", "user_input_form")
)
dataset_query_variable: str | None = None
pre_prompt: str | None = None
agent_mode: TrialAppAgentMode | None = Field(
default=None,
validation_alias=AliasChoices("agent_mode_dict", "agent_mode"),
)
prompt_type: str | None = None
chat_prompt_config: JsonObject | None = Field(
default=None, validation_alias=AliasChoices("chat_prompt_config_dict", "chat_prompt_config")
)
completion_prompt_config: JsonObject | None = Field(
default=None, validation_alias=AliasChoices("completion_prompt_config_dict", "completion_prompt_config")
)
dataset_configs: JsonObject | None = Field(
default=None,
validation_alias=AliasChoices("dataset_configs_dict", "dataset_configs"),
)
file_upload: JsonObject | None = Field(
default=None,
validation_alias=AliasChoices("file_upload_dict", "file_upload"),
)
created_by: str | None = None
created_at: int | None = None
updated_by: str | None = None
updated_at: int | None = None
@field_validator("created_at", "updated_at", mode="before")
@classmethod
def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
return to_timestamp(value)
class TrialDeletedToolResponse(ResponseModel):
type: str
tool_name: str
provider_id: str
class TrialTagResponse(ResponseModel):
id: str
name: str
type: str
class TrialSiteResponse(ResponseModel):
access_token: str | None = Field(default=None, validation_alias="code")
code: str | None = None
title: str
icon_type: TrialIconType | None = None
icon: str | None = None
icon_background: str | None = None
description: str | None = None
default_language: str
chat_color_theme: str | None = None
chat_color_theme_inverted: bool | None = None
customize_domain: str | None = None
copyright: str | None = None
privacy_policy: str | None = None
input_placeholder: str | None = None
custom_disclaimer: str | None = None
customize_token_strategy: str | None = None
prompt_public: bool | None = None
app_base_url: str | None = None
show_workflow_steps: bool | None = None
use_icon_as_answer_icon: bool | None = None
created_by: str | None = None
created_at: int | None = None
updated_by: str | None = None
updated_at: int | None = None
icon_url: str | None = None
@field_validator("icon_type", mode="before")
@classmethod
def _normalize_icon_type(cls, value: Any) -> str | None:
if hasattr(value, "value"):
return value.value
return value
@field_validator("created_at", "updated_at", mode="before")
@classmethod
def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
return to_timestamp(value)
class TrialWorkflowPartialResponse(ResponseModel):
id: str
created_by: str | None = None
created_at: int | None = None
updated_by: str | None = None
updated_at: int | None = None
@field_validator("created_at", "updated_at", mode="before")
@classmethod
def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
return to_timestamp(value)
class TrialAppDetailResponse(ResponseModel):
id: str
name: str
description: str | None = None
mode: TrialAppMode = Field(validation_alias="mode_compatible_with_agent")
icon_type: TrialIconType | None = None
icon: str | None = None
icon_background: str | None = None
icon_url: str | None = None
enable_site: bool
enable_api: bool
model_config_: TrialAppModelConfigResponse | None = Field(
default=None,
validation_alias=AliasChoices("app_model_config", "model_config"),
alias="model_config",
)
workflow: TrialWorkflowPartialResponse | None = None
api_base_url: str | None = None
use_icon_as_answer_icon: bool | None = None
max_active_requests: int | None = None
created_by: str | None = None
created_at: int | None = None
updated_by: str | None = None
updated_at: int | None = None
deleted_tools: list[TrialDeletedToolResponse] = Field(default_factory=list)
access_mode: str | None = None
tags: list[TrialTagResponse] = Field(default_factory=list)
permission_keys: list[str] = Field(default_factory=list)
site: TrialSiteResponse
@field_validator("icon_type", mode="before")
@classmethod
def _normalize_icon_type(cls, value: Any) -> str | None:
if hasattr(value, "value"):
return value.value
return value
@field_validator("created_at", "updated_at", mode="before")
@classmethod
def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
return to_timestamp(value)
class TrialDatasetResponse(ResponseModel):
id: str
name: str
description: str | None = None
permission: str | None = None
data_source_type: str | None = None
indexing_technique: str | None = None
created_by: str | None = None
created_at: int | None = None
permission_keys: list[str] = Field(default_factory=list)
class TrialDatasetListResponse(ResponseModel):
data: list[TrialDatasetResponse]
has_more: bool
limit: int
total: int
page: int
class TrialWorkflowAccount(ResponseModel):
id: str
name: str | None = None
email: str | None = None
class TrialWorkflowResponse(ResponseModel):
id: str
graph: JsonObject = Field(validation_alias=AliasChoices("graph_dict", "graph"))
features: JsonObject = Field(default_factory=dict, validation_alias=AliasChoices("features_dict", "features"))
hash: str | None = Field(default=None, validation_alias=AliasChoices("unique_hash", "hash"))
version: str | None = None
marked_name: str | None = None
marked_comment: str | None = None
created_by: TrialWorkflowAccount | None = Field(
default=None,
validation_alias=AliasChoices("created_by_account", "created_by"),
)
created_at: int | None = None
updated_by: TrialWorkflowAccount | None = Field(
default=None,
validation_alias=AliasChoices("updated_by_account", "updated_by"),
)
updated_at: int | None = None
tool_published: bool | None = None
environment_variables: list[JsonObject] = Field(default_factory=list)
conversation_variables: list[JsonObject] = Field(default_factory=list)
rag_pipeline_variables: list[JsonObject] = Field(default_factory=list)
@field_validator("created_at", "updated_at", mode="before")
@classmethod
def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
return to_timestamp(value)
register_schema_models(
console_ns,
WorkflowRunRequest,
@@ -196,6 +447,9 @@ register_response_schema_models(
SimpleResultResponse,
SiteResponse,
SuggestedQuestionsResponse,
TrialAppDetailResponse,
TrialDatasetListResponse,
TrialWorkflowResponse,
)
@@ -414,6 +668,14 @@ class TrialChatTextApi(TrialAppResource):
message_id = request_data.message_id
text = request_data.text
voice = request_data.voice
message_ref = None
if message_id:
app_ref = AppRefService.create_app_ref(app_model)
message_ref = AppRefService.create_message_ref(
app_ref,
message_id,
account_id=current_user.id,
)
# Get IDs before they might be detached from session
app_id = app_model.id
@@ -424,7 +686,7 @@ class TrialChatTextApi(TrialAppResource):
session=db.session,
text=text,
voice=voice,
message_id=message_id,
message_ref=message_ref,
)
RecommendedAppService.add_trial_app_record(db.session, app_id, user_id)
return response
@@ -557,7 +819,7 @@ class TrialAppParameterApi(Resource):
class AppApi(Resource):
@console_ns.response(200, "Success", app_detail_with_site_model)
@console_ns.response(200, "Success", console_ns.models[TrialAppDetailResponse.__name__])
@get_app_model_with_trial(None)
@marshal_with(app_detail_with_site_model)
def get(self, app_model):
@@ -570,7 +832,7 @@ class AppApi(Resource):
class AppWorkflowApi(Resource):
@console_ns.response(200, "Success", workflow_model)
@console_ns.response(200, "Success", console_ns.models[TrialWorkflowResponse.__name__])
@get_app_model_with_trial(None)
@marshal_with(workflow_model)
def get(self, app_model):
@@ -584,7 +846,7 @@ class AppWorkflowApi(Resource):
class DatasetListApi(Resource):
@console_ns.doc(params=query_params_from_model(TrialDatasetListQuery))
@console_ns.response(200, "Success", dataset_list_model)
@console_ns.response(200, "Success", console_ns.models[TrialDatasetListResponse.__name__])
@get_app_model_with_trial(None)
def get(self, app_model):
page = request.args.get("page", default=1, type=int)
api/controllers/console/files.py
+17
View File
@@ -38,6 +38,22 @@ register_response_schema_models(console_ns, AllowedExtensionsResponse, TextConte
PREVIEW_WORDS_LIMIT = 3000
_FILE_UPLOAD_PARAMS = {
"file": {
"description": "File to upload",
"in": "formData",
"type": "file",
"required": True,
},
"source": {
"description": "Optional upload source",
"in": "formData",
"type": "string",
"enum": ["datasets"],
"required": False,
},
}
@console_ns.route("/files/upload")
class FileApi(Resource):
@@ -64,6 +80,7 @@ class FileApi(Resource):
@login_required
@account_initialization_required
@cloud_edition_billing_resource_check("documents")
@console_ns.doc(consumes=["multipart/form-data"], params=_FILE_UPLOAD_PARAMS)
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileResponse.__name__])
@with_current_user
def post(self, current_user: Account):
api/controllers/console/tag/tags.py
+2 -1
View File
@@ -117,7 +117,8 @@ def _enforce_snippet_tag_rbac_by_tag_id(tag_id: str) -> None:
if not dify_config.RBAC_ENABLED:
return
tag_type = db.session.scalar(select(Tag.type).where(Tag.id == tag_id).limit(1))
_, current_tenant_id = current_account_with_tenant()
tag_type = db.session.scalar(select(Tag.type).where(Tag.id == tag_id, Tag.tenant_id == current_tenant_id).limit(1))
_enforce_snippet_tag_rbac_if_needed(tag_type)
api/controllers/console/workspace/members.py
+2 -2
View File
@@ -84,8 +84,6 @@ class MemberActionTenantResponse(ResponseModel):
register_enum_models(console_ns, TenantAccountRole)
register_schema_models(
console_ns,
AccountWithRole,
AccountWithRoleList,
MemberInvitePayload,
MemberRoleUpdatePayload,
OwnerTransferEmailPayload,
@@ -94,6 +92,8 @@ register_schema_models(
)
register_response_schema_models(
console_ns,
AccountWithRole,
AccountWithRoleList,
SimpleResultDataResponse,
SimpleResultResponse,
VerificationTokenResponse,
api/controllers/console/workspace/plugin.py
+19 -2
View File
@@ -302,11 +302,28 @@ class PluginListResponse(ResponseModel):
class PluginVersionsResponse(ResponseModel):
versions: Any
versions: Mapping[str, PluginService.LatestPluginCache | None]
class PluginInstallationItemResponse(ResponseModel):
id: str
created_at: datetime
updated_at: datetime
tenant_id: str
endpoints_setups: int
endpoints_active: int
runtime_type: str
source: PluginInstallationSource
meta: Mapping[str, Any]
plugin_id: str
plugin_unique_identifier: str
version: str
checksum: str
declaration: PluginDeclarationResponse
class PluginInstallationsResponse(ResponseModel):
plugins: Any
plugins: list[PluginInstallationItemResponse]
class PluginManifestResponse(ResponseModel):
api/controllers/console/workspace/rbac.py
+34 -10
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
from enum import StrEnum
from typing import Any
from typing import Any, Literal
from flask import request
from flask_restx import Resource
@@ -10,10 +9,11 @@ from sqlalchemy import select
from werkzeug.exceptions import NotFound
from configs import dify_config
from controllers.common.schema import register_response_schema_models
from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
from controllers.console import console_ns
from controllers.console.wraps import RBACPermission, RBACResourceScope, rbac_permission_required
from core.db.session_factory import session_factory
from core.rbac import RBACResourceWhitelistScope
from libs.login import current_account_with_tenant, login_required
from models import Account
from services.enterprise import rbac_service as svc
@@ -511,14 +511,8 @@ class RBACAccessPolicyBindingUnlockApi(Resource):
# ---------------------------------------------------------------------------
class _AccessScope(StrEnum):
ALL = "all"
SPECIFIC = "specific"
ONLY_ME = "only_me"
class _ResourceAccessScopeRequest(BaseModel):
scope: _AccessScope
scope: RBACResourceWhitelistScope
class _ReplaceBindingsRequest(BaseModel):
@@ -544,6 +538,20 @@ class _DeleteMemberBindingsRequest(BaseModel):
return value
class _AccessControlLanguageQuery(BaseModel):
language: Literal["en", "ja", "zh"] | None = Field(default=None, description="Localized policy label language")
register_schema_models(
console_ns,
_ResourceAccessScopeRequest,
_ReplaceBindingsRequest,
_DeleteMemberBindingsRequest,
_AccessControlLanguageQuery,
svc.ReplaceUserAccessPolicies,
)
@console_ns.route("/workspaces/current/rbac/my-permissions")
class RBACMyPermissionsApi(Resource):
@login_required
@@ -563,6 +571,7 @@ class RBACMyPermissionsApi(Resource):
@console_ns.route("/workspaces/current/rbac/apps/<uuid:app_id>/access-policy")
class RBACAppMatrixApi(Resource):
@login_required
@console_ns.doc(params=query_params_from_model(_AccessControlLanguageQuery))
@console_ns.response(200, "Success", console_ns.models[svc.AppAccessMatrix.__name__])
def get(self, app_id):
tenant_id, account_id = _current_ids()
@@ -580,6 +589,7 @@ class RBACAppWhitelistApi(Resource):
return _dump(svc.RBACService.AppAccess.whitelist(tenant_id, account_id, str(app_id)))
@login_required
@console_ns.expect(console_ns.models[_ResourceAccessScopeRequest.__name__])
@console_ns.response(200, "Success", console_ns.models[svc.ResourceWhitelist.__name__])
def put(self, app_id):
tenant_id, account_id = _current_ids()
@@ -597,6 +607,7 @@ class RBACAppWhitelistApi(Resource):
@console_ns.route("/workspaces/current/rbac/apps/<uuid:app_id>/user-access-policies")
class RBACAppUserAccessPoliciesApi(Resource):
@login_required
@console_ns.doc(params=query_params_from_model(_AccessControlLanguageQuery))
@console_ns.response(200, "Success", console_ns.models[svc.ResourceUserAccessPoliciesResponse.__name__])
def get(self, app_id):
tenant_id, account_id = _current_ids()
@@ -608,6 +619,7 @@ class RBACAppUserAccessPoliciesApi(Resource):
@console_ns.route("/workspaces/current/rbac/apps/<uuid:app_id>/users/<uuid:target_account_id>/access-policies")
class RBACAppUserAccessPolicyAssignmentApi(Resource):
@login_required
@console_ns.expect(console_ns.models[svc.ReplaceUserAccessPolicies.__name__])
@console_ns.response(200, "Success", console_ns.models[svc.ReplaceUserAccessPoliciesResponse.__name__])
def put(self, app_id, target_account_id):
tenant_id, account_id = _current_ids()
@@ -641,6 +653,7 @@ class RBACAppMemberBindingsApi(Resource):
return _dump(svc.RBACService.AppAccess.list_member_bindings(tenant_id, account_id, str(app_id), str(policy_id)))
@login_required
@console_ns.expect(console_ns.models[_DeleteMemberBindingsRequest.__name__])
@console_ns.response(200, "Success", console_ns.models[svc.MemberBindingsResponse.__name__])
def delete(self, app_id, policy_id):
tenant_id, account_id = _current_ids()
@@ -663,6 +676,7 @@ class RBACAppMemberBindingsApi(Resource):
@console_ns.route("/workspaces/current/rbac/datasets/<uuid:dataset_id>/access-policy")
class RBACDatasetMatrixApi(Resource):
@login_required
@console_ns.doc(params=query_params_from_model(_AccessControlLanguageQuery))
@console_ns.response(200, "Success", console_ns.models[svc.DatasetAccessMatrix.__name__])
def get(self, dataset_id):
tenant_id, account_id = _current_ids()
@@ -680,6 +694,7 @@ class RBACDatasetWhitelistApi(Resource):
return _dump(svc.RBACService.DatasetAccess.whitelist(tenant_id, account_id, str(dataset_id)))
@login_required
@console_ns.expect(console_ns.models[_ResourceAccessScopeRequest.__name__])
@console_ns.response(200, "Success", console_ns.models[svc.ResourceWhitelist.__name__])
def put(self, dataset_id):
tenant_id, account_id = _current_ids()
@@ -697,6 +712,7 @@ class RBACDatasetWhitelistApi(Resource):
@console_ns.route("/workspaces/current/rbac/datasets/<uuid:dataset_id>/user-access-policies")
class RBACDatasetUserAccessPoliciesApi(Resource):
@login_required
@console_ns.doc(params=query_params_from_model(_AccessControlLanguageQuery))
@console_ns.response(200, "Success", console_ns.models[svc.ResourceUserAccessPoliciesResponse.__name__])
def get(self, dataset_id):
tenant_id, account_id = _current_ids()
@@ -708,6 +724,7 @@ class RBACDatasetUserAccessPoliciesApi(Resource):
@console_ns.route("/workspaces/current/rbac/datasets/<uuid:dataset_id>/users/<uuid:target_account_id>/access-policies")
class RBACDatasetUserAccessPolicyAssignmentApi(Resource):
@login_required
@console_ns.expect(console_ns.models[svc.ReplaceUserAccessPolicies.__name__])
@console_ns.response(200, "Success", console_ns.models[svc.ReplaceUserAccessPoliciesResponse.__name__])
def put(self, dataset_id, target_account_id):
tenant_id, account_id = _current_ids()
@@ -747,6 +764,7 @@ class RBACDatasetMemberBindingsApi(Resource):
)
@login_required
@console_ns.expect(console_ns.models[_DeleteMemberBindingsRequest.__name__])
@console_ns.response(200, "Success", console_ns.models[svc.MemberBindingsResponse.__name__])
def delete(self, dataset_id, policy_id):
tenant_id, account_id = _current_ids()
@@ -785,6 +803,7 @@ class RBACWorkspaceAppRoleBindingsApi(Resource):
@console_ns.route("/workspaces/current/rbac/workspace/apps/access-policies/<uuid:policy_id>/bindings")
class RBACWorkspaceAppBindingsApi(Resource):
@login_required
@console_ns.expect(console_ns.models[_ReplaceBindingsRequest.__name__])
@console_ns.response(200, "Success", console_ns.models[svc.AccessMatrixItem.__name__])
def put(self, policy_id):
tenant_id, account_id = _current_ids()
@@ -832,6 +851,7 @@ class RBACWorkspaceDatasetRoleBindingsApi(Resource):
@console_ns.route("/workspaces/current/rbac/workspace/datasets/access-policies/<uuid:policy_id>/bindings")
class RBACWorkspaceDatasetBindingsApi(Resource):
@login_required
@console_ns.expect(console_ns.models[_ReplaceBindingsRequest.__name__])
@console_ns.response(200, "Success", console_ns.models[svc.AccessMatrixItem.__name__])
def put(self, policy_id):
tenant_id, account_id = _current_ids()
@@ -873,6 +893,9 @@ class _ReplaceMemberRolesRequest(BaseModel):
return value
register_schema_models(console_ns, _ReplaceMemberRolesRequest)
@console_ns.route("/workspaces/current/rbac/members/<uuid:member_id>/rbac-roles")
class RBACMemberRolesApi(Resource):
@login_required
@@ -882,6 +905,7 @@ class RBACMemberRolesApi(Resource):
return _dump(svc.RBACService.MemberRoles.get(tenant_id, account_id, str(member_id)))
@login_required
@console_ns.expect(console_ns.models[_ReplaceMemberRolesRequest.__name__])
@console_ns.response(200, "Success", console_ns.models[svc.MemberRolesResponse.__name__])
def put(self, member_id):
tenant_id, account_id = _current_ids()
api/controllers/console/workspace/snippets.py
+104 -18
View File
@@ -1,11 +1,13 @@
import logging
import re
from datetime import datetime
from typing import Any
from urllib.parse import quote
from flask import Response, request
from flask_restx import Resource, marshal
from pydantic import RootModel
from pydantic import Field as PydanticField
from pydantic import field_validator
from sqlalchemy.orm import Session, sessionmaker
from werkzeug.datastructures import MultiDict
from werkzeug.exceptions import NotFound
@@ -30,27 +32,34 @@ from controllers.console.wraps import (
with_current_tenant_id,
with_current_user,
)
from core.plugin.entities.plugin import PluginDependency
from extensions.ext_database import db
from fields.base import ResponseModel
from fields.snippet_fields import snippet_fields, snippet_list_fields, snippet_pagination_fields
from fields.snippet_fields import snippet_fields, snippet_list_fields
from libs.helper import to_timestamp
from libs.login import login_required
from models import Account
from models.snippet import SnippetType
from services.app_dsl_service import ImportStatus
from services.snippet_dsl_service import SnippetDslService
from services.snippet_dsl_service import ImportStatus, SnippetDslService
from services.snippet_service import SnippetService
logger = logging.getLogger(__name__)
_TAG_IDS_BRACKET_PATTERN = re.compile(r"^tag_ids\[(\d+)\]$")
_CREATOR_IDS_BRACKET_PATTERN = re.compile(r"^creator_ids\[(\d+)\]$")
_CREATORS_BRACKET_PATTERN = re.compile(r"^creators\[(\d+)\]$")
class SnippetImportResponse(RootModel[dict[str, Any]]):
root: dict[str, Any]
class SnippetImportResponse(ResponseModel):
id: str
status: ImportStatus
snippet_id: str | None
current_dsl_version: str
imported_dsl_version: str
error: str
class SnippetDependencyCheckResponse(RootModel[dict[str, Any]]):
root: dict[str, Any]
class SnippetDependencyCheckResponse(ResponseModel):
leaked_dependencies: list[PluginDependency]
class SnippetUseCountResponse(ResponseModel):
@@ -58,6 +67,77 @@ class SnippetUseCountResponse(ResponseModel):
use_count: int
class SnippetTagResponse(ResponseModel):
id: str
name: str
type: str
class SnippetAccountResponse(ResponseModel):
id: str
name: str
email: str
class SnippetListItemResponse(ResponseModel):
id: str
name: str
description: str | None
type: SnippetType
version: int
use_count: int
is_published: bool
icon_info: dict[str, Any] | None
tags: list[SnippetTagResponse]
created_by: str | None
author_name: str | None
created_at: int
updated_by: str | None
updated_at: int
@field_validator("created_at", "updated_at", mode="before")
@classmethod
def _normalize_timestamp(cls, value: datetime | int | None) -> int:
timestamp = to_timestamp(value)
if timestamp is None:
raise ValueError("timestamp is required")
return timestamp
class SnippetResponse(ResponseModel):
id: str
name: str
description: str | None
type: SnippetType
version: int
use_count: int
is_published: bool
icon_info: dict[str, Any] | None
graph: dict[str, Any] = PydanticField(validation_alias="graph_dict")
input_fields: list[dict[str, Any]] = PydanticField(validation_alias="input_fields_list")
tags: list[SnippetTagResponse]
created_by: SnippetAccountResponse | None = PydanticField(validation_alias="created_by_account")
created_at: int
updated_by: SnippetAccountResponse | None = PydanticField(validation_alias="updated_by_account")
updated_at: int
@field_validator("created_at", "updated_at", mode="before")
@classmethod
def _normalize_timestamp(cls, value: datetime | int | None) -> int:
timestamp = to_timestamp(value)
if timestamp is None:
raise ValueError("timestamp is required")
return timestamp
class SnippetPaginationResponse(ResponseModel):
data: list[SnippetListItemResponse]
page: int
limit: int
total: int
has_more: bool
def _snippet_service() -> SnippetService:
return SnippetService(sessionmaker(bind=db.engine, expire_on_commit=False))
@@ -73,11 +153,19 @@ def _normalize_snippet_list_query_args(query_args: MultiDict[str, str]) -> dict[
indexed_tag_ids.extend((int(match.group(1)), value) for value in query_args.getlist(key))
continue
match = _CREATOR_IDS_BRACKET_PATTERN.fullmatch(key)
match = _CREATOR_IDS_BRACKET_PATTERN.fullmatch(key) or _CREATORS_BRACKET_PATTERN.fullmatch(key)
if match:
indexed_creator_ids.extend((int(match.group(1)), value) for value in query_args.getlist(key))
continue
if key in {"tag_ids", "creators", "creator_ids"}:
values = query_args.getlist(key)
if values:
normalized["creators" if key in {"creators", "creator_ids"} else key] = (
values if len(values) > 1 else values[0]
)
continue
value = query_args.get(key)
if value is not None:
normalized[key] = value
@@ -105,19 +193,17 @@ register_response_schema_models(
SnippetImportResponse,
SnippetDependencyCheckResponse,
SnippetUseCountResponse,
SnippetListItemResponse,
SnippetResponse,
SnippetPaginationResponse,
)
# Create namespace models for marshaling
snippet_model = console_ns.model("Snippet", snippet_fields)
snippet_list_model = console_ns.model("SnippetList", snippet_list_fields)
snippet_pagination_model = console_ns.model("SnippetPagination", snippet_pagination_fields)
@console_ns.route("/workspaces/current/customized-snippets")
class CustomizedSnippetsApi(Resource):
@console_ns.doc("list_customized_snippets")
@console_ns.doc(params=query_params_from_model(SnippetListQuery))
@console_ns.response(200, "Snippets retrieved successfully", snippet_pagination_model)
@console_ns.response(200, "Snippets retrieved successfully", console_ns.models[SnippetPaginationResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@@ -148,7 +234,7 @@ class CustomizedSnippetsApi(Resource):
@console_ns.doc("create_customized_snippet")
@console_ns.expect(console_ns.models.get(CreateSnippetPayload.__name__))
@console_ns.response(201, "Snippet created successfully", snippet_model)
@console_ns.response(201, "Snippet created successfully", console_ns.models[SnippetResponse.__name__])
@console_ns.response(400, "Invalid request")
@setup_required
@login_required
@@ -191,7 +277,7 @@ class CustomizedSnippetsApi(Resource):
@console_ns.route("/workspaces/current/customized-snippets/<uuid:snippet_id>")
class CustomizedSnippetDetailApi(Resource):
@console_ns.doc("get_customized_snippet")
@console_ns.response(200, "Snippet retrieved successfully", snippet_model)
@console_ns.response(200, "Snippet retrieved successfully", console_ns.models[SnippetResponse.__name__])
@console_ns.response(404, "Snippet not found")
@setup_required
@login_required
@@ -212,7 +298,7 @@ class CustomizedSnippetDetailApi(Resource):
@console_ns.doc("update_customized_snippet")
@console_ns.expect(console_ns.models.get(UpdateSnippetPayload.__name__))
@console_ns.response(200, "Snippet updated successfully", snippet_model)
@console_ns.response(200, "Snippet updated successfully", console_ns.models[SnippetResponse.__name__])
@console_ns.response(400, "Invalid request")
@console_ns.response(404, "Snippet not found")
@setup_required
api/controllers/console/workspace/trigger_providers.py
+77 -13
View File
@@ -1,5 +1,5 @@
import logging
from typing import Any
from typing import Any, Literal
from flask import make_response, redirect, request
from flask_restx import Resource
@@ -11,11 +11,19 @@ from configs import dify_config
from controllers.common.errors import NotFoundError
from controllers.common.fields import BinaryFileResponse, RedirectResponse, SimpleResultResponse
from controllers.common.schema import register_response_schema_models, register_schema_models
from core.entities.parameter_entities import AppSelectorScope, ModelSelectorScope, ToolSelectorScope
from core.plugin.entities.plugin_daemon import CredentialType
from core.plugin.impl.oauth import OAuthHandler
from core.trigger.entities.entities import SubscriptionBuilderUpdater
from core.tools.entities.common_entities import I18nObject
from core.trigger.entities.api_entities import (
SubscriptionBuilderApiEntity,
TriggerProviderApiEntity,
TriggerProviderSubscriptionApiEntity,
)
from core.trigger.entities.entities import RequestLog, SubscriptionBuilderUpdater
from core.trigger.trigger_manager import TriggerManager
from extensions.ext_database import db
from fields.base import ResponseModel
from graphon.model_runtime.utils.encoders import jsonable_encoder
from libs.login import login_required
from models.account import Account
@@ -70,14 +78,41 @@ class TriggerOAuthClientPayload(BaseModel):
class TriggerOAuthAuthorizeResponse(BaseModel):
authorization_url: str
subscription_builder_id: str
subscription_builder: Any
subscription_builder: SubscriptionBuilderApiEntity
class TriggerProviderConfigOptionResponse(BaseModel):
value: str = Field(..., description="The value of the option")
label: I18nObject = Field(..., description="The label of the option")
class TriggerProviderConfigResponse(BaseModel):
type: Literal[
"secret-input",
"text-input",
"select",
"boolean",
"app-selector",
"model-selector",
"array[tools]",
] = Field(..., description="The type of the credentials")
name: str = Field(..., description="The name of the credentials")
scope: AppSelectorScope | ModelSelectorScope | ToolSelectorScope | None = None
required: bool = False
default: int | str | float | bool | None = None
options: list[TriggerProviderConfigOptionResponse] | None = None
multiple: bool = False
label: I18nObject | None = None
help: I18nObject | None = None
url: str | None = None
placeholder: I18nObject | None = None
class TriggerOAuthClientResponse(BaseModel):
configured: bool
system_configured: bool
custom_configured: bool
oauth_client_schema: Any
oauth_client_schema: list[TriggerProviderConfigResponse]
custom_enabled: bool
redirect_uri: str
params: dict[str, Any]
@@ -87,6 +122,26 @@ class TriggerProviderOpaqueResponse(RootModel[Any]):
root: Any
class TriggerProviderListResponse(RootModel[list[TriggerProviderApiEntity]]):
root: list[TriggerProviderApiEntity]
class TriggerSubscriptionListResponse(RootModel[list[TriggerProviderSubscriptionApiEntity]]):
root: list[TriggerProviderSubscriptionApiEntity]
class TriggerSubscriptionBuilderCreateResponse(ResponseModel):
subscription_builder: SubscriptionBuilderApiEntity
class TriggerSubscriptionBuilderVerifyResponse(ResponseModel):
verified: bool
class TriggerSubscriptionBuilderLogsResponse(ResponseModel):
logs: list[RequestLog]
register_schema_models(
console_ns,
TriggerSubscriptionBuilderCreatePayload,
@@ -102,6 +157,15 @@ register_response_schema_models(
TriggerOAuthAuthorizeResponse,
TriggerOAuthClientResponse,
TriggerProviderOpaqueResponse,
TriggerProviderApiEntity,
TriggerProviderListResponse,
TriggerProviderSubscriptionApiEntity,
TriggerSubscriptionListResponse,
SubscriptionBuilderApiEntity,
TriggerSubscriptionBuilderCreateResponse,
TriggerSubscriptionBuilderVerifyResponse,
RequestLog,
TriggerSubscriptionBuilderLogsResponse,
)
@@ -118,7 +182,7 @@ class TriggerProviderIconApi(Resource):
@console_ns.route("/workspaces/current/triggers")
class TriggerProviderListApi(Resource):
@console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerProviderListResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@@ -130,7 +194,7 @@ class TriggerProviderListApi(Resource):
@console_ns.route("/workspaces/current/trigger-provider/<path:provider>/info")
class TriggerProviderInfoApi(Resource):
@console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerProviderApiEntity.__name__])
@setup_required
@login_required
@account_initialization_required
@@ -142,7 +206,7 @@ class TriggerProviderInfoApi(Resource):
@console_ns.route("/workspaces/current/trigger-provider/<path:provider>/subscriptions/list")
class TriggerSubscriptionListApi(Resource):
@console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerSubscriptionListResponse.__name__])
@setup_required
@login_required
@edit_permission_required
@@ -172,7 +236,7 @@ class TriggerSubscriptionListApi(Resource):
)
class TriggerSubscriptionBuilderCreateApi(Resource):
@console_ns.expect(console_ns.models[TriggerSubscriptionBuilderCreatePayload.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerSubscriptionBuilderCreateResponse.__name__])
@setup_required
@login_required
@edit_permission_required
@@ -202,7 +266,7 @@ class TriggerSubscriptionBuilderCreateApi(Resource):
"/workspaces/current/trigger-provider/<path:provider>/subscriptions/builder/<path:subscription_builder_id>",
)
class TriggerSubscriptionBuilderGetApi(Resource):
@console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[SubscriptionBuilderApiEntity.__name__])
@setup_required
@login_required
@edit_permission_required
@@ -220,7 +284,7 @@ class TriggerSubscriptionBuilderGetApi(Resource):
)
class TriggerSubscriptionBuilderVerifyApi(Resource):
@console_ns.expect(console_ns.models[TriggerSubscriptionBuilderVerifyPayload.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerSubscriptionBuilderVerifyResponse.__name__])
@setup_required
@login_required
@edit_permission_required
@@ -253,7 +317,7 @@ class TriggerSubscriptionBuilderVerifyApi(Resource):
)
class TriggerSubscriptionBuilderUpdateApi(Resource):
@console_ns.expect(console_ns.models[TriggerSubscriptionBuilderUpdatePayload.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[SubscriptionBuilderApiEntity.__name__])
@setup_required
@login_required
@edit_permission_required
@@ -286,7 +350,7 @@ class TriggerSubscriptionBuilderUpdateApi(Resource):
"/workspaces/current/trigger-provider/<path:provider>/subscriptions/builder/logs/<path:subscription_builder_id>",
)
class TriggerSubscriptionBuilderLogsApi(Resource):
@console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerSubscriptionBuilderLogsResponse.__name__])
@setup_required
@login_required
@edit_permission_required
@@ -689,7 +753,7 @@ class TriggerOAuthClientManageApi(Resource):
)
class TriggerSubscriptionVerifyApi(Resource):
@console_ns.expect(console_ns.models[TriggerSubscriptionBuilderVerifyPayload.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
@console_ns.response(200, "Success", console_ns.models[TriggerSubscriptionBuilderVerifyResponse.__name__])
@setup_required
@login_required
@edit_permission_required
api/controllers/inner_api/__init__.py
+4
View File
@@ -17,8 +17,10 @@ inner_api_ns = Namespace("inner_api", description="Internal API operations", pat
from . import mail as _mail
from . import runtime_credentials as _runtime_credentials
from .agent import tools as _agent_tools
from .app import dsl as _app_dsl
from .knowledge import retrieval as _knowledge_retrieval
from .plugin import agent_config as _agent_config
from .plugin import agent_drive as _agent_drive
from .plugin import plugin as _plugin
from .workspace import workspace as _workspace
@@ -26,7 +28,9 @@ from .workspace import workspace as _workspace
api.add_namespace(inner_api_ns)
__all__ = [
"_agent_config",
"_agent_drive",
"_agent_tools",
"_app_dsl",
"_knowledge_retrieval",
"_mail",
api/controllers/inner_api/agent/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Agent backend inner API controllers."""
api/controllers/inner_api/agent/tools.py
+68
View File
@@ -0,0 +1,68 @@
"""Inner API endpoint for Agent core tool invocation."""
from flask_restx import Resource
from pydantic import ValidationError
from controllers.common.schema import register_response_schema_models, register_schema_models
from controllers.inner_api import inner_api_ns
from controllers.inner_api.wraps import agent_inner_api_only
from extensions.ext_database import db
from libs.exception import BaseHTTPException
from services.agent_tool_inner_service import AgentToolInnerService
from services.entities.agent_tool_inner import AgentToolInvokeRequest, AgentToolInvokeResponse
from services.errors.agent_tool_inner import AgentToolInnerServiceError
class AgentToolInvokeHttpError(BaseHTTPException):
error_code = "agent_tool_invoke_failed"
description = "Agent tool invocation failed."
code = 500
def __init__(
self,
*,
error_code: str | None = None,
description: str | None = None,
status_code: int | None = None,
) -> None:
if error_code is not None:
self.error_code = error_code
if description is not None:
self.description = description
if status_code is not None:
self.code = status_code
super().__init__(self.description)
register_schema_models(inner_api_ns, AgentToolInvokeRequest)
register_response_schema_models(inner_api_ns, AgentToolInvokeResponse)
@inner_api_ns.route("/agent/tools/invoke")
class AgentToolInvokeApi(Resource):
"""Invoke one Agent tool through the API-owned core tool runtime path."""
@agent_inner_api_only
@inner_api_ns.doc("inner_agent_tool_invoke")
@inner_api_ns.expect(inner_api_ns.models[AgentToolInvokeRequest.__name__])
@inner_api_ns.response(200, "Tool invoked successfully", inner_api_ns.models[AgentToolInvokeResponse.__name__])
def post(self) -> dict[str, object]:
try:
payload = AgentToolInvokeRequest.model_validate(inner_api_ns.payload or {})
except ValidationError as exc:
raise AgentToolInvokeHttpError(
error_code="invalid_request",
description=str(exc),
status_code=400,
) from exc
try:
response = AgentToolInnerService().invoke(payload, session=db.session())
except AgentToolInnerServiceError as exc:
raise AgentToolInvokeHttpError(
error_code=exc.error_code,
description=exc.description,
status_code=exc.status_code,
) from exc
return response.model_dump(mode="json")
api/controllers/inner_api/plugin/agent_config.py
+243
View File
@@ -0,0 +1,243 @@
"""Inner API for Agent Soul-backed config assets.
These endpoints are called by the dify-agent server with the inner API key.
They resolve the requested Agent config version directly from Agent Soul JSON
and never expose signed download URLs or drive-owned metadata.
"""
from __future__ import annotations
import io
from flask import request, send_file
from flask_restx import Resource
from pydantic import BaseModel, ValidationError
from controllers.console.wraps import setup_required
from controllers.inner_api import inner_api_ns
from controllers.inner_api.wraps import plugin_inner_api_only
from services.agent_config_service import (
AgentConfigService,
AgentConfigServiceError,
AgentConfigVersionKind,
ConfigPushPayload,
)
class _ConfigTargetQuery(BaseModel):
tenant_id: str
user_id: str | None = None
config_version_id: str
config_version_kind: AgentConfigVersionKind
class _ConfigMutationRequest(BaseModel):
tenant_id: str
user_id: str
config_version_id: str
config_version_kind: AgentConfigVersionKind
class _ConfigPushRequest(_ConfigMutationRequest):
files: list[dict] = []
skills: list[dict] = []
env_text: str | None = None
note: str | None = None
def to_payload(self) -> ConfigPushPayload:
return ConfigPushPayload.model_validate(
{
"files": self.files,
"skills": self.skills,
"env_text": self.env_text,
"note": self.note,
}
)
class _ConfigEnvUpdateRequest(_ConfigMutationRequest):
env_text: str
class _ConfigNoteUpdateRequest(_ConfigMutationRequest):
note: str
def _target_query_from_request() -> _ConfigTargetQuery:
return _ConfigTargetQuery.model_validate(
{
"tenant_id": request.args.get("tenant_id"),
"user_id": request.args.get("user_id"),
"config_version_id": request.args.get("config_version_id"),
"config_version_kind": request.args.get("config_version_kind"),
}
)
def _error_response(exc: AgentConfigServiceError) -> tuple[dict[str, str], int]:
return {"code": exc.code, "message": exc.message}, exc.status_code
@inner_api_ns.route("/agent-config/<string:agent_id>/manifest")
class AgentConfigManifestApi(Resource):
@setup_required
@plugin_inner_api_only
@inner_api_ns.doc("agent_config_manifest")
def get(self, agent_id: str):
try:
query = _target_query_from_request()
return AgentConfigService().manifest(
tenant_id=query.tenant_id,
agent_id=agent_id,
user_id=query.user_id,
config_version_id=query.config_version_id,
config_version_kind=query.config_version_kind,
)
except ValidationError as exc:
return {"code": "invalid_request", "message": str(exc)}, 400
except AgentConfigServiceError as exc:
return _error_response(exc)
@inner_api_ns.route("/agent-config/<string:agent_id>/skills/<string:name>/pull")
class AgentConfigSkillPullApi(Resource):
@setup_required
@plugin_inner_api_only
@inner_api_ns.doc("agent_config_skill_pull")
def get(self, agent_id: str, name: str):
try:
query = _target_query_from_request()
result = AgentConfigService().pull_skill(
tenant_id=query.tenant_id,
agent_id=agent_id,
user_id=query.user_id,
config_version_id=query.config_version_id,
config_version_kind=query.config_version_kind,
name=name,
)
return send_file(
io.BytesIO(result.payload),
mimetype=result.mime_type,
as_attachment=True,
download_name=result.filename,
)
except ValidationError as exc:
return {"code": "invalid_request", "message": str(exc)}, 400
except AgentConfigServiceError as exc:
return _error_response(exc)
@inner_api_ns.route("/agent-config/<string:agent_id>/skills/<string:name>/inspect")
class AgentConfigSkillInspectApi(Resource):
@setup_required
@plugin_inner_api_only
@inner_api_ns.doc("agent_config_skill_inspect")
def get(self, agent_id: str, name: str):
try:
query = _target_query_from_request()
return AgentConfigService().inspect_skill(
tenant_id=query.tenant_id,
agent_id=agent_id,
user_id=query.user_id,
config_version_id=query.config_version_id,
config_version_kind=query.config_version_kind,
name=name,
)
except ValidationError as exc:
return {"code": "invalid_request", "message": str(exc)}, 400
except AgentConfigServiceError as exc:
return _error_response(exc)
@inner_api_ns.route("/agent-config/<string:agent_id>/files/<string:name>/pull")
class AgentConfigFilePullApi(Resource):
@setup_required
@plugin_inner_api_only
@inner_api_ns.doc("agent_config_file_pull")
def get(self, agent_id: str, name: str):
try:
query = _target_query_from_request()
result = AgentConfigService().pull_file(
tenant_id=query.tenant_id,
agent_id=agent_id,
user_id=query.user_id,
config_version_id=query.config_version_id,
config_version_kind=query.config_version_kind,
name=name,
)
return send_file(
io.BytesIO(result.payload),
mimetype=result.mime_type,
as_attachment=True,
download_name=result.filename,
)
except ValidationError as exc:
return {"code": "invalid_request", "message": str(exc)}, 400
except AgentConfigServiceError as exc:
return _error_response(exc)
@inner_api_ns.route("/agent-config/<string:agent_id>/push")
class AgentConfigPushApi(Resource):
@setup_required
@plugin_inner_api_only
@inner_api_ns.doc("agent_config_push")
def post(self, agent_id: str):
try:
body = _ConfigPushRequest.model_validate(request.get_json(silent=True) or {})
return AgentConfigService().push(
tenant_id=body.tenant_id,
agent_id=agent_id,
user_id=body.user_id,
config_version_id=body.config_version_id,
config_version_kind=body.config_version_kind,
payload=body.to_payload(),
)
except ValidationError as exc:
return {"code": "invalid_request", "message": str(exc)}, 400
except AgentConfigServiceError as exc:
return _error_response(exc)
@inner_api_ns.route("/agent-config/<string:agent_id>/env")
class AgentConfigEnvApi(Resource):
@setup_required
@plugin_inner_api_only
@inner_api_ns.doc("agent_config_env")
def patch(self, agent_id: str):
try:
body = _ConfigEnvUpdateRequest.model_validate(request.get_json(silent=True) or {})
return AgentConfigService().update_env(
tenant_id=body.tenant_id,
agent_id=agent_id,
user_id=body.user_id,
config_version_id=body.config_version_id,
config_version_kind=body.config_version_kind,
env_text=body.env_text,
)
except ValidationError as exc:
return {"code": "invalid_request", "message": str(exc)}, 400
except AgentConfigServiceError as exc:
return _error_response(exc)
@inner_api_ns.route("/agent-config/<string:agent_id>/note")
class AgentConfigNoteApi(Resource):
@setup_required
@plugin_inner_api_only
@inner_api_ns.doc("agent_config_note")
def put(self, agent_id: str):
try:
body = _ConfigNoteUpdateRequest.model_validate(request.get_json(silent=True) or {})
return AgentConfigService().update_note(
tenant_id=body.tenant_id,
agent_id=agent_id,
user_id=body.user_id,
config_version_id=body.config_version_id,
config_version_kind=body.config_version_kind,
note=body.note,
)
except ValidationError as exc:
return {"code": "invalid_request", "message": str(exc)}, 400
except AgentConfigServiceError as exc:
return _error_response(exc)
api/controllers/inner_api/wraps.py
+12
View File
@@ -95,3 +95,15 @@ def plugin_inner_api_only[**P, R](view: Callable[P, R]) -> Callable[P, R]:
return view(*args, **kwargs)
return decorated
def agent_inner_api_only[**P, R](view: Callable[P, R]) -> Callable[P, R]:
"""Temporary alias for agent-backend inner API callers.
Agent tool and knowledge calls currently share the same trusted
`dify-agent -> Dify API` transport credentials as the plugin inner bridge.
Keep the wrapper name agent-specific so the controller surface does not grow
more plugin-only semantics while auth settings stay shared.
"""
return plugin_inner_api_only(view)
api/controllers/service_api/app/annotation.py
+7 -4
View File
@@ -21,6 +21,7 @@ from services.annotation_service import (
InsertAnnotationArgs,
UpdateAnnotationArgs,
)
from services.app_ref_service import AppRefService
class AnnotationCreatePayload(BaseModel):
@@ -282,9 +283,9 @@ class AnnotationUpdateDeleteApi(Resource):
"""Update an existing annotation."""
payload = AnnotationCreatePayload.model_validate(service_api_ns.payload or {})
update_args: UpdateAnnotationArgs = {"question": payload.question, "answer": payload.answer}
annotation = AppAnnotationService.update_app_annotation_directly(
update_args, app_model.id, str(annotation_id), db.session
)
app_ref = AppRefService.create_app_ref(app_model)
annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
annotation = AppAnnotationService.update_app_annotation_directly(update_args, annotation_ref, db.session)
response = Annotation.model_validate(annotation, from_attributes=True)
return response.model_dump(mode="json")
@@ -313,5 +314,7 @@ class AnnotationUpdateDeleteApi(Resource):
@edit_permission_required
def delete(self, app_model: App, annotation_id: UUID):
"""Delete an annotation."""
AppAnnotationService.delete_app_annotation(app_model.id, str(annotation_id), db.session)
app_ref = AppRefService.create_app_ref(app_model)
annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
AppAnnotationService.delete_app_annotation(annotation_ref, db.session)
return "", 204
api/controllers/service_api/app/audio.py
+10 -1
View File
@@ -26,6 +26,7 @@ from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotIni
from extensions.ext_database import db
from graphon.model_runtime.errors.invoke import InvokeError
from models.model import App, EndUser
from services.app_ref_service import AppRefService
from services.audio_service import AudioService
from services.errors.audio import (
AudioTooLargeServiceError,
@@ -177,13 +178,21 @@ class TextApi(Resource):
message_id = payload.message_id
text = payload.text
voice = payload.voice
message_ref = None
if message_id:
app_ref = AppRefService.create_app_ref(app_model)
message_ref = AppRefService.create_message_ref(
app_ref,
message_id,
end_user_id=end_user.id,
)
response = AudioService.transcript_tts(
app_model=app_model,
session=db.session,
text=text,
voice=voice,
end_user=end_user.external_user_id,
message_id=message_id,
message_ref=message_ref,
)
return response
api/controllers/service_api/dataset/dataset.py
+20 -14
View File
@@ -519,6 +519,7 @@ class DatasetListApi(DatasetApiResource):
embedding_model_name=payload.embedding_model,
retrieval_model=payload.retrieval_model,
summary_index_setting=payload.summary_index_setting,
session=db.session,
)
except services.errors.dataset.DatasetNameDuplicateError:
raise DatasetNameDuplicateError()
@@ -561,7 +562,7 @@ class DatasetApi(DatasetApiResource):
)
def get(self, _, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
try:
@@ -597,7 +598,7 @@ class DatasetApi(DatasetApiResource):
retrieval_model_dict["search_method"] = "keyword_search"
if data.get("permission") == "partial_members":
part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session)
data.update({"partial_member_list": part_users_list})
return _dump_service_dataset_with_partial_members(data), 200
@@ -635,7 +636,7 @@ class DatasetApi(DatasetApiResource):
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def patch(self, _, dataset_id: UUID):
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
@@ -676,9 +677,10 @@ class DatasetApi(DatasetApiResource):
dataset,
str(payload.permission) if payload.permission else None,
payload.partial_member_list,
db.session,
)
dataset = DatasetService.update_dataset(dataset_id_str, update_data, current_user)
dataset = DatasetService.update_dataset(dataset_id_str, update_data, current_user, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
@@ -688,12 +690,14 @@ class DatasetApi(DatasetApiResource):
tenant_id = current_user.current_tenant_id
if payload.partial_member_list and payload.permission == DatasetPermissionEnum.PARTIAL_TEAM:
DatasetPermissionService.update_partial_member_list(tenant_id, dataset_id_str, payload.partial_member_list)
DatasetPermissionService.update_partial_member_list(
tenant_id, dataset_id_str, payload.partial_member_list, db.session
)
# clear partial member list when permission is only_me or all_team_members
elif payload.permission in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM}:
DatasetPermissionService.clear_partial_member_list(dataset_id_str)
DatasetPermissionService.clear_partial_member_list(dataset_id_str, db.session)
partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session)
result_data.update({"partial_member_list": partial_member_list})
return _dump_service_dataset_with_partial_members(result_data), 200
@@ -746,8 +750,8 @@ class DatasetApi(DatasetApiResource):
dataset_id_str = str(dataset_id)
try:
if DatasetService.delete_dataset(dataset_id_str, current_user):
DatasetPermissionService.clear_partial_member_list(dataset_id_str)
if DatasetService.delete_dataset(dataset_id_str, current_user, db.session):
DatasetPermissionService.clear_partial_member_list(dataset_id_str, db.session)
return "", 204
else:
raise NotFound("Dataset not found.")
@@ -812,7 +816,7 @@ class DocumentStatusApi(DatasetApiResource):
InvalidActionError: If the action is invalid or cannot be performed.
"""
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
@@ -831,7 +835,7 @@ class DocumentStatusApi(DatasetApiResource):
document_ids = data.get("document_ids", [])
try:
DocumentService.batch_update_document_status(dataset, document_ids, action, current_user)
DocumentService.batch_update_document_status(dataset, document_ids, action, current_user, db.session)
except services.errors.document.DocumentIndexingError as e:
raise InvalidActionError(str(e))
except ValueError as e:
@@ -939,9 +943,11 @@ class DatasetTagsApi(DatasetApiResource):
payload = TagUpdatePayload.model_validate(service_api_ns.payload or {})
tag_id = payload.tag_id
tag = TagService.update_tags(UpdateTagServicePayload(name=payload.name), tag_id, db.session)
tag = TagService.update_tags(
UpdateTagServicePayload(name=payload.name), tag_id, db.session, tag_type=TagType.KNOWLEDGE
)
binding_count = TagService.get_tag_binding_count(tag_id, db.session)
binding_count = TagService.get_tag_binding_count(tag_id, db.session, tag_type=TagType.KNOWLEDGE)
response = dump_response(
KnowledgeTagResponse,
@@ -971,7 +977,7 @@ class DatasetTagsApi(DatasetApiResource):
def delete(self, _):
"""Delete a knowledge type tag."""
payload = TagDeletePayload.model_validate(service_api_ns.payload or {})
TagService.delete_tag(payload.tag_id, db.session)
TagService.delete_tag(payload.tag_id, db.session, tag_type=TagType.KNOWLEDGE)
return "", 204
api/controllers/service_api/dataset/document.py
+13 -8
View File
@@ -400,6 +400,7 @@ def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Mapping[
account=current_user,
dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
created_from="api",
session=db.session,
)
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
@@ -459,6 +460,7 @@ def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID
account=current_user,
dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
created_from="api",
session=db.session,
)
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
@@ -756,6 +758,7 @@ class DocumentAddByFileApi(DatasetApiResource):
account=dataset.created_by_account,
dataset_process_rule=dataset_process_rule,
created_from="api",
session=db.session,
)
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
@@ -832,6 +835,7 @@ def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID
account=dataset.created_by_account,
dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
created_from="api",
session=db.session,
)
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
@@ -1002,6 +1006,7 @@ class DocumentBatchDownloadZipApi(DatasetApiResource):
document_ids=[str(document_id) for document_id in payload.document_ids],
tenant_id=str(tenant_id),
current_user=current_user,
session=db.session,
)
with ExitStack() as stack:
@@ -1058,7 +1063,7 @@ class DocumentIndexingStatusApi(DatasetApiResource):
if not dataset:
raise NotFound("Dataset not found.")
# get documents
documents = DocumentService.get_batch_documents(dataset_id_str, batch)
documents = DocumentService.get_batch_documents(dataset_id_str, batch, db.session)
if not documents:
raise NotFound("Documents not found.")
documents_status = []
@@ -1134,7 +1139,7 @@ class DocumentDownloadApi(DatasetApiResource):
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def get(self, tenant_id, dataset_id: UUID, document_id: UUID):
dataset = self.get_dataset(str(dataset_id), str(tenant_id))
document = DocumentService.get_document(dataset.id, str(document_id))
document = DocumentService.get_document(dataset.id, str(document_id), session=db.session)
if not document:
raise NotFound("Document not found.")
@@ -1142,7 +1147,7 @@ class DocumentDownloadApi(DatasetApiResource):
if document.tenant_id != str(tenant_id):
raise Forbidden("No permission.")
return {"url": DocumentService.get_document_download_url(document)}
return {"url": DocumentService.get_document_download_url(document, db.session)}
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
@@ -1190,7 +1195,7 @@ class DocumentApi(DatasetApiResource):
dataset = self.get_dataset(dataset_id_str, tenant_id)
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
@@ -1215,7 +1220,7 @@ class DocumentApi(DatasetApiResource):
if metadata == "only":
response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
elif metadata == "without":
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str)
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session)
document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
data_source_info = document.data_source_detail_dict
response = {
@@ -1250,7 +1255,7 @@ class DocumentApi(DatasetApiResource):
"need_summary": document.need_summary if document.need_summary is not None else False,
}
else:
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str)
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session)
document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
data_source_info = document.data_source_detail_dict
response = {
@@ -1345,7 +1350,7 @@ class DocumentApi(DatasetApiResource):
if not dataset:
raise ValueError("Dataset does not exist.")
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
# 404 if document not found
if document is None:
@@ -1357,7 +1362,7 @@ class DocumentApi(DatasetApiResource):
try:
# delete document
DocumentService.delete_document(document)
DocumentService.delete_document(document, db.session)
except services.errors.document.DocumentIndexingError:
raise DocumentIndexingError("Cannot delete document during indexing.")
api/controllers/service_api/dataset/metadata.py
+6 -6
View File
@@ -81,7 +81,7 @@ class DatasetMetadataCreateServiceApi(DatasetApiResource):
metadata_args = MetadataArgs.model_validate(service_api_ns.payload or {})
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, db.session)
@@ -116,7 +116,7 @@ class DatasetMetadataCreateServiceApi(DatasetApiResource):
def get(self, tenant_id, dataset_id: UUID):
"""Get all metadata for a dataset."""
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
metadata = MetadataService.get_dataset_metadatas(db.session(), dataset)
@@ -154,7 +154,7 @@ class DatasetMetadataServiceApi(DatasetApiResource):
dataset_id_str = str(dataset_id)
metadata_id_str = str(metadata_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, db.session)
@@ -189,7 +189,7 @@ class DatasetMetadataServiceApi(DatasetApiResource):
"""Delete metadata."""
dataset_id_str = str(dataset_id)
metadata_id_str = str(metadata_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, db.session)
@@ -257,7 +257,7 @@ class DatasetMetadataBuiltInFieldActionServiceApi(DatasetApiResource):
def post(self, tenant_id, dataset_id: UUID, action: Literal["enable", "disable"]):
"""Enable or disable built-in metadata field."""
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, db.session)
@@ -303,7 +303,7 @@ class DocumentMetadataEditServiceApi(DatasetApiResource):
def post(self, tenant_id, dataset_id: UUID):
"""Update metadata for multiple documents."""
dataset_id_str = str(dataset_id)
dataset = DatasetService.get_dataset(dataset_id_str)
dataset = DatasetService.get_dataset(dataset_id_str, db.session)
if dataset is None:
raise NotFound("Dataset not found.")
DatasetService.check_dataset_permission(dataset, current_user, db.session)
api/controllers/service_api/dataset/segment.py
+50 -71
View File
@@ -37,7 +37,8 @@ from fields.segment_fields import (
from graphon.model_runtime.entities.model_entities import ModelType
from libs.helper import dump_response
from libs.login import current_account_with_tenant
from models.dataset import Dataset, DocumentSegment
from models.dataset import Dataset, Document, DocumentSegment
from services.dataset_ref_service import DatasetRefService, SegmentRef
from services.dataset_service import DatasetService, DocumentService, SegmentService
from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs
from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError
@@ -127,6 +128,21 @@ register_response_schema_models(
)
def _get_segment_for_document(
dataset: Dataset, document: Document, segment_id: str
) -> tuple[SegmentRef, DocumentSegment]:
dataset_ref = DatasetRefService.create_dataset_ref(dataset)
document_ref = DatasetRefService.create_document_ref(dataset_ref, document)
if document_ref is None:
raise NotFound("Document not found.")
segment_ref = DatasetRefService.create_segment_ref(document_ref, segment_id)
segment = SegmentService.get_segment_by_ref(segment_ref)
if not segment:
raise NotFound("Segment not found.")
return segment_ref, segment
@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")
class SegmentApi(DatasetApiResource):
"""Resource for segments."""
@@ -175,7 +191,7 @@ class SegmentApi(DatasetApiResource):
raise NotFound("Dataset not found.")
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
if document.indexing_status != "completed":
@@ -210,7 +226,9 @@ class SegmentApi(DatasetApiResource):
for args_item in segment_items:
SegmentService.segment_create_args_validate(args_item, document)
segments = cast(list[DocumentSegment], SegmentService.multi_create_segment(segment_items, document, dataset))
segments = cast(
list[DocumentSegment], SegmentService.multi_create_segment(segment_items, document, dataset, db.session)
)
segment_ids = [segment.id for segment in segments]
summaries: dict[str, str | None] = {}
if segment_ids:
@@ -267,7 +285,7 @@ class SegmentApi(DatasetApiResource):
raise NotFound("Dataset not found.")
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
# check embedding model setting
@@ -337,7 +355,7 @@ class DatasetSegmentApi(DatasetApiResource):
)
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def delete(self, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
_, current_tenant_id = current_account_with_tenant()
current_account_with_tenant()
dataset_id_str = str(dataset_id)
# check dataset
dataset = db.session.scalar(
@@ -349,15 +367,12 @@ class DatasetSegmentApi(DatasetApiResource):
DatasetService.check_dataset_model_setting(dataset)
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
# check segment
segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
if not segment:
raise NotFound("Segment not found.")
SegmentService.delete_segment(segment, document, dataset)
_, segment = _get_segment_for_document(dataset, document, segment_id_str)
SegmentService.delete_segment(segment, document, dataset, db.session)
return "", 204
@service_api_ns.doc(
@@ -395,7 +410,7 @@ class DatasetSegmentApi(DatasetApiResource):
DatasetService.check_dataset_model_setting(dataset)
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
@@ -415,14 +430,11 @@ class DatasetSegmentApi(DatasetApiResource):
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
segment_id_str = str(segment_id)
# check segment
segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
if not segment:
raise NotFound("Segment not found.")
_, segment = _get_segment_for_document(dataset, document, segment_id_str)
payload = SegmentUpdatePayload.model_validate(service_api_ns.payload or {})
updated_segment = SegmentService.update_segment(payload.segment, segment, document, dataset)
updated_segment = SegmentService.update_segment(payload.segment, segment, document, dataset, db.session)
summary = SummaryIndexService.get_segment_summary(segment_id=updated_segment.id, dataset_id=dataset_id_str)
response = {
"data": segment_response_with_summary(updated_segment, summary.summary_content if summary else None),
@@ -457,7 +469,7 @@ class DatasetSegmentApi(DatasetApiResource):
service_api_ns.models[SegmentDetailResponse.__name__],
)
def get(self, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
_, current_tenant_id = current_account_with_tenant()
current_account_with_tenant()
dataset_id_str = str(dataset_id)
# check dataset
dataset = db.session.scalar(
@@ -469,14 +481,11 @@ class DatasetSegmentApi(DatasetApiResource):
DatasetService.check_dataset_model_setting(dataset)
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
# check segment
segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
if not segment:
raise NotFound("Segment not found.")
_, segment = _get_segment_for_document(dataset, document, segment_id_str)
summary = SummaryIndexService.get_segment_summary(segment_id=segment.id, dataset_id=dataset_id_str)
response = {
@@ -533,15 +542,12 @@ class ChildChunkApi(DatasetApiResource):
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
# check segment
segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
if not segment:
raise NotFound("Segment not found.")
_, segment = _get_segment_for_document(dataset, document, segment_id_str)
# check embedding model setting
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
@@ -564,7 +570,7 @@ class ChildChunkApi(DatasetApiResource):
payload = ChildChunkCreatePayload.model_validate(service_api_ns.payload or {})
try:
child_chunk = SegmentService.create_child_chunk(payload.content, segment, document, dataset)
child_chunk = SegmentService.create_child_chunk(payload.content, segment, document, dataset, db.session)
except ChildChunkIndexingServiceError as e:
raise ChildChunkIndexingError(str(e))
@@ -595,7 +601,7 @@ class ChildChunkApi(DatasetApiResource):
service_api_ns.models[ChildChunkListResponse.__name__],
)
def get(self, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
_, current_tenant_id = current_account_with_tenant()
current_account_with_tenant()
"""Get child chunks."""
dataset_id_str = str(dataset_id)
# check dataset
@@ -607,15 +613,12 @@ class ChildChunkApi(DatasetApiResource):
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
# check segment
segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
if not segment:
raise NotFound("Segment not found.")
_get_segment_for_document(dataset, document, segment_id_str)
args = query_params_from_request(ChildChunkListQuery, use_defaults_for_malformed_ints=True)
@@ -665,7 +668,7 @@ class DatasetChildChunkApi(DatasetApiResource):
@cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def delete(self, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID, child_chunk_id: UUID):
_, current_tenant_id = current_account_with_tenant()
current_account_with_tenant()
"""Delete child chunk."""
dataset_id_str = str(dataset_id)
# check dataset
@@ -677,34 +680,21 @@ class DatasetChildChunkApi(DatasetApiResource):
document_id_str = str(document_id)
# check document
document = DocumentService.get_document(dataset.id, document_id_str)
document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
# check segment
segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
if not segment:
raise NotFound("Segment not found.")
# validate segment belongs to the specified document
if segment.document_id != document_id_str:
raise NotFound("Document not found.")
segment_ref, _ = _get_segment_for_document(dataset, document, segment_id_str)
child_chunk_id_str = str(child_chunk_id)
# check child chunk
child_chunk = SegmentService.get_child_chunk_by_id(
child_chunk_id=child_chunk_id_str, tenant_id=current_tenant_id
)
child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref)
if not child_chunk:
raise NotFound("Child chunk not found.")
# validate child chunk belongs to the specified segment
if child_chunk.segment_id != segment.id:
raise NotFound("Child chunk not found.")
try:
SegmentService.delete_child_chunk(child_chunk, dataset)
SegmentService.delete_child_chunk(child_chunk, dataset, db.session)
except ChildChunkDeleteIndexServiceError as e:
raise ChildChunkDeleteIndexError(str(e))
@@ -739,7 +729,7 @@ class DatasetChildChunkApi(DatasetApiResource):
@cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
@cloud_edition_billing_rate_limit_check("knowledge", "dataset")
def patch(self, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID, child_chunk_id: UUID):
_, current_tenant_id = current_account_with_tenant()
current_account_with_tenant()
"""Update child chunk."""
dataset_id_str = str(dataset_id)
# check dataset
@@ -751,37 +741,26 @@ class DatasetChildChunkApi(DatasetApiResource):
document_id_str = str(document_id)
# get document
document = DocumentService.get_document(dataset_id_str, document_id_str)
document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
if not document:
raise NotFound("Document not found.")
segment_id_str = str(segment_id)
# get segment
segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
if not segment:
raise NotFound("Segment not found.")
# validate segment belongs to the specified document
if segment.document_id != document_id_str:
raise NotFound("Segment not found.")
segment_ref, segment = _get_segment_for_document(dataset, document, segment_id_str)
child_chunk_id_str = str(child_chunk_id)
# get child chunk
child_chunk = SegmentService.get_child_chunk_by_id(
child_chunk_id=child_chunk_id_str, tenant_id=current_tenant_id
)
child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref)
if not child_chunk:
raise NotFound("Child chunk not found.")
# validate child chunk belongs to the specified segment
if child_chunk.segment_id != segment.id:
raise NotFound("Child chunk not found.")
# validate args
payload = ChildChunkUpdatePayload.model_validate(service_api_ns.payload or {})
try:
child_chunk = SegmentService.update_child_chunk(payload.content, child_chunk, segment, document, dataset)
child_chunk = SegmentService.update_child_chunk(
payload.content, child_chunk, segment, document, dataset, db.session
)
except ChildChunkIndexingServiceError as e:
raise ChildChunkIndexingError(str(e))
api/controllers/web/audio.py
+10 -1
View File
@@ -26,6 +26,7 @@ from extensions.ext_database import db
from graphon.model_runtime.errors.invoke import InvokeError
from libs.helper import uuid_value
from models.model import App, EndUser
from services.app_ref_service import AppRefService
from services.audio_service import AudioService
from services.errors.audio import (
AudioTooLargeServiceError,
@@ -130,13 +131,21 @@ class TextApi(WebApiResource):
message_id = payload.message_id
text = payload.text
voice = payload.voice
message_ref = None
if message_id:
app_ref = AppRefService.create_app_ref(app_model)
message_ref = AppRefService.create_message_ref(
app_ref,
message_id,
end_user_id=end_user.id,
)
response = AudioService.transcript_tts(
app_model=app_model,
session=db.session,
text=text,
voice=voice,
end_user=end_user.external_user_id,
message_id=message_id,
message_ref=message_ref,
)
return response
api/core/app/app_config/common/sensitive_word_avoidance/manager.py
+80 -23
View File
@@ -1,10 +1,73 @@
from collections.abc import Mapping
from typing import Any
from typing import Annotated, Any, Literal
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from core.app.app_config.entities import SensitiveWordAvoidanceEntity
from core.moderation.factory import ModerationFactory
class SensitiveWordAvoidanceDisabledConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
enabled: Literal[False] = False
class SensitiveWordAvoidanceKeywordsConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
enabled: Literal[True] = True
type: Literal["keywords"]
config: dict[str, Any] = Field(default_factory=dict)
def run_provider_validation(self, tenant_id: str) -> None:
ModerationFactory.validate_config(name=self.type, tenant_id=tenant_id, config=self.config)
class SensitiveWordAvoidanceOpenAIConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
enabled: Literal[True] = True
type: Literal["openai_moderation"]
config: dict[str, Any] = Field(default_factory=dict)
def run_provider_validation(self, tenant_id: str) -> None:
ModerationFactory.validate_config(name=self.type, tenant_id=tenant_id, config=self.config)
class SensitiveWordAvoidanceAPIConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
enabled: Literal[True] = True
type: Literal["api"]
config: dict[str, Any] = Field(default_factory=dict)
def run_provider_validation(self, tenant_id: str) -> None:
ModerationFactory.validate_config(name=self.type, tenant_id=tenant_id, config=self.config)
EnabledSensitiveWordAvoidanceConfig = Annotated[
SensitiveWordAvoidanceKeywordsConfig | SensitiveWordAvoidanceOpenAIConfig | SensitiveWordAvoidanceAPIConfig,
Field(discriminator="type"),
]
SensitiveWordAvoidanceConfig = Annotated[
SensitiveWordAvoidanceDisabledConfig | EnabledSensitiveWordAvoidanceConfig,
Field(discriminator="enabled"),
]
_sensitive_word_avoidance_adapter: TypeAdapter[SensitiveWordAvoidanceConfig] = TypeAdapter(SensitiveWordAvoidanceConfig)
def _normalize_raw(raw: Any) -> Any:
if isinstance(raw, dict):
if raw.get("enabled") is None:
raw = {**raw, "enabled": False}
elif raw.get("enabled") is True and raw.get("config") is None:
raw = {**raw, "config": {}}
return raw
class SensitiveWordAvoidanceConfigManager:
@classmethod
def convert(cls, config: Mapping[str, Any]) -> SensitiveWordAvoidanceEntity | None:
@@ -24,30 +87,24 @@ class SensitiveWordAvoidanceConfigManager:
def validate_and_set_defaults(
cls, tenant_id: str, config: dict[str, Any], only_structure_validate: bool = False
) -> tuple[dict[str, Any], list[str]]:
if not config.get("sensitive_word_avoidance"):
config["sensitive_word_avoidance"] = {"enabled": False}
if not isinstance(config["sensitive_word_avoidance"], dict):
raw = config.get("sensitive_word_avoidance") or {"enabled": False}
if not isinstance(raw, dict):
raise ValueError("sensitive_word_avoidance must be of dict type")
if "enabled" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["enabled"]:
config["sensitive_word_avoidance"]["enabled"] = False
try:
validated = _sensitive_word_avoidance_adapter.validate_python(_normalize_raw(raw))
except ValidationError:
raise
if config["sensitive_word_avoidance"]["enabled"]:
if not config["sensitive_word_avoidance"].get("type"):
raise ValueError("sensitive_word_avoidance.type is required")
if not only_structure_validate:
typ = config["sensitive_word_avoidance"]["type"]
if not isinstance(typ, str):
raise ValueError("sensitive_word_avoidance.type must be a string")
sensitive_word_avoidance_config = config["sensitive_word_avoidance"].get("config")
if sensitive_word_avoidance_config is None:
sensitive_word_avoidance_config = {}
if not isinstance(sensitive_word_avoidance_config, dict):
raise ValueError("sensitive_word_avoidance.config must be a dict")
ModerationFactory.validate_config(name=typ, tenant_id=tenant_id, config=sensitive_word_avoidance_config)
if not only_structure_validate and isinstance(
validated,
(
SensitiveWordAvoidanceKeywordsConfig,
SensitiveWordAvoidanceOpenAIConfig,
SensitiveWordAvoidanceAPIConfig,
),
):
validated.run_provider_validation(tenant_id)
config["sensitive_word_avoidance"] = validated.model_dump()
return config, ["sensitive_word_avoidance"]
api/core/app/app_config/easy_ui_based_app/dataset/manager.py
+2 -1
View File
@@ -9,6 +9,7 @@ from core.app.app_config.entities import (
)
from core.entities.agent_entities import PlanningStrategy
from core.rag.data_post_processor.data_post_processor import RerankingModelDict, WeightsDict
from extensions.ext_database import db
from models.model import AppMode, AppModelConfigDict
from services.dataset_service import DatasetService
@@ -256,7 +257,7 @@ class DatasetConfigManager:
@classmethod
def is_dataset_exists(cls, tenant_id: str, dataset_id: str) -> bool:
# verify if the dataset ID exists
dataset = DatasetService.get_dataset(dataset_id)
dataset = DatasetService.get_dataset(dataset_id, db.session)
if not dataset:
return False
api/core/app/apps/agent_app/app_generator.py
+170 -32
View File
@@ -1,21 +1,28 @@
"""Agent App generator: orchestrate one conversation turn for an Agent App.
"""Agent App generator: orchestrate Agent App chat and finalize executions.
Mirrors the agent_chat generator (conversation + message + queue + streamed
response over the EasyUI chat pipeline), but the backing config comes from the
bound Agent Soul and the answer is produced by ``AgentAppRunner`` calling the
dify-agent backend rather than an in-process LLM/ReAct loop.
The primary mode mirrors the agent_chat generator (conversation + message +
queue + streamed response over the EasyUI chat pipeline), but the backing
config comes from the bound Agent Soul and the answer is produced by
``AgentAppRunner`` calling the dify-agent backend rather than an in-process
LLM/ReAct loop.
It also exposes a stateless build-finalize mode that reuses existing runtime
context from the bound debug conversation, triggers the Agent backend side
effect synchronously, and skips Dify-side chat/message persistence.
"""
from __future__ import annotations
import contextvars
import json
import logging
import threading
import uuid
from collections.abc import Generator, Mapping
from typing import Any
from collections.abc import Generator, Mapping, Sequence
from typing import Any, Literal
from flask import Flask, current_app
from pydantic import JsonValue
from sqlalchemy import and_, or_, select
from clients.agent_backend import AgentBackendRunEventAdapter
@@ -61,6 +68,13 @@ class AgentAppGeneratorError(ValueError):
"""Raised when an Agent App turn cannot be set up."""
def _append_prompt_file_mappings(query: str, prompt_file_mappings: Sequence[JsonValue]) -> str:
"""Append raw request file references to the backend user prompt."""
if not prompt_file_mappings:
return query
return f"{query}\n{json.dumps(list(prompt_file_mappings), ensure_ascii=False)}"
class AgentAppGenerator(MessageBasedAppGenerator):
def generate(
self,
@@ -74,14 +88,12 @@ class AgentAppGenerator(MessageBasedAppGenerator):
if not streaming:
raise AgentAppGeneratorError("Agent App only supports streaming mode")
query = args.get("query")
if not isinstance(query, str) or not query.strip():
raise AgentAppGeneratorError("query is required")
query = query.replace("\x00", "")
query = self._require_query(args)
inputs = args["inputs"]
prompt_file_mappings = args.get("files") or []
# Resolve the bound roster Agent + its current Agent Soul snapshot.
agent, agent_config_id, agent_soul = self._resolve_agent(
agent, agent_config_id, agent_config_version_kind, agent_soul = self._resolve_agent(
app_model,
invoke_from=invoke_from,
draft_type=args.get("draft_type"),
@@ -122,6 +134,7 @@ class AgentAppGenerator(MessageBasedAppGenerator):
),
query=query,
files=[],
prompt_file_mappings=prompt_file_mappings,
parent_message_id=(
args.get("parent_message_id")
if invoke_from not in {InvokeFrom.SERVICE_API, InvokeFrom.OPENAPI}
@@ -137,6 +150,7 @@ class AgentAppGenerator(MessageBasedAppGenerator):
trace_manager=trace_manager,
agent_id=agent.id,
agent_config_snapshot_id=agent_config_id,
agent_config_version_kind=agent_config_version_kind,
agent_runtime_session_snapshot_id=runtime_session_snapshot_id,
)
@@ -176,6 +190,86 @@ class AgentAppGenerator(MessageBasedAppGenerator):
)
return AgentAppGenerateResponseConverter.convert(response=response, invoke_from=invoke_from)
def generate_stateless(
self,
*,
app_model: App,
user: Account | EndUser,
args: Mapping[str, Any],
invoke_from: InvokeFrom,
) -> Mapping[str, Any]:
"""Run one Agent App turn without persisting Dify conversation messages."""
query = self._require_query(args)
conversation_id = args.get("conversation_id")
if not isinstance(conversation_id, str) or not conversation_id:
raise AgentAppGeneratorError("conversation_id is required")
agent, agent_config_id, agent_config_version_kind, agent_soul = self._resolve_agent(
app_model,
invoke_from=invoke_from,
draft_type=args.get("draft_type"),
user=user,
)
runtime_session_snapshot_id = self._runtime_session_snapshot_id(
invoke_from=invoke_from,
snapshot_id=agent_config_id,
)
return self._run_stateless(
app_model=app_model,
user=user,
invoke_from=invoke_from,
query=query,
conversation_id=conversation_id,
agent=agent,
agent_config_id=agent_config_id,
agent_config_version_kind=agent_config_version_kind,
agent_soul=agent_soul,
runtime_session_snapshot_id=runtime_session_snapshot_id,
)
def _run_stateless(
self,
*,
app_model: App,
user: Account | EndUser,
invoke_from: InvokeFrom,
query: str,
conversation_id: str,
agent: Agent,
agent_config_id: str,
agent_config_version_kind: Literal["snapshot", "draft", "build_draft"],
agent_soul: AgentSoulConfig,
runtime_session_snapshot_id: str | None,
) -> Mapping[str, Any]:
"""Run the Agent backend without creating or updating Dify chat records.
Build-chat finalization is an action against the Agent backend (for
example, ``dify-agent config push``). It may reuse the active build-chat
runtime snapshot for shell/config context, but the API side must not add
a synthetic user/assistant turn to the debug conversation.
"""
dify_context = DifyRunContext(
tenant_id=app_model.tenant_id,
app_id=app_model.id,
user_id=user.id,
user_from=UserFrom.ACCOUNT if isinstance(user, Account) else UserFrom.END_USER,
invoke_from=invoke_from,
)
self._build_runner(dify_context).run_stateless(
dify_context=dify_context,
agent_id=agent.id,
agent_config_snapshot_id=agent_config_id,
agent_config_version_kind=agent_config_version_kind,
agent_soul=agent_soul,
conversation_id=conversation_id,
query=query,
idempotency_key=str(uuid.uuid4()),
session_scope_snapshot_id=runtime_session_snapshot_id,
)
return {"result": "success"}
def resume_after_form_submission(
self,
*,
@@ -192,15 +286,15 @@ class AgentAppGenerator(MessageBasedAppGenerator):
persisted to the conversation. Live streaming to a reconnected client is
out of scope here — the message is persisted and can be re-fetched.
"""
agent, agent_config_id, agent_soul = self._resolve_agent(
app_model,
invoke_from=invoke_from,
draft_type="draft",
user=user,
)
conversation = ConversationService.get_conversation(
app_model=app_model, conversation_id=conversation_id, user=user
)
agent, agent_config_id, agent_config_version_kind, agent_soul = self._resolve_agent(
app_model,
invoke_from=invoke_from,
draft_type=self._resume_draft_type(app_model=app_model, conversation=conversation, user=user),
user=user,
)
app_config = AgentAppConfigManager.get_app_config(
app_model=app_model,
@@ -245,6 +339,7 @@ class AgentAppGenerator(MessageBasedAppGenerator):
trace_manager=trace_manager,
agent_id=agent.id,
agent_config_snapshot_id=agent_config_id,
agent_config_version_kind=agent_config_version_kind,
)
conversation, message = self._init_generate_records(application_generate_entity, conversation)
@@ -285,6 +380,30 @@ class AgentAppGenerator(MessageBasedAppGenerator):
stream=False,
)
@staticmethod
def _resume_draft_type(*, app_model: App, conversation: Any, user: Account | EndUser) -> str | None:
if conversation.invoke_from != InvokeFrom.DEBUGGER:
return None
active_session = AgentAppRuntimeSessionStore().load_active_session_for_conversation(
tenant_id=app_model.tenant_id,
app_id=app_model.id,
conversation_id=conversation.id,
)
snapshot_id = active_session.scope.agent_config_snapshot_id if active_session is not None else None
if snapshot_id and isinstance(user, Account):
draft = db.session.scalar(
select(AgentConfigDraft).where(
AgentConfigDraft.tenant_id == app_model.tenant_id,
AgentConfigDraft.id == snapshot_id,
)
)
if draft is not None:
if draft.draft_type == AgentConfigDraftType.DEBUG_BUILD and draft.account_id == user.id:
return AgentConfigDraftType.DEBUG_BUILD.value
if draft.draft_type == AgentConfigDraftType.DRAFT and draft.account_id is None:
return AgentConfigDraftType.DRAFT.value
return AgentConfigDraftType.DRAFT.value
def _generate_worker(
self,
*,
@@ -329,6 +448,10 @@ class AgentAppGenerator(MessageBasedAppGenerator):
)
if handled:
return
query = _append_prompt_file_mappings(
query=query,
prompt_file_mappings=application_generate_entity.prompt_file_mappings,
)
dify_context = DifyRunContext(
tenant_id=app_config.tenant_id,
@@ -337,27 +460,18 @@ class AgentAppGenerator(MessageBasedAppGenerator):
user_from=user_from,
invoke_from=application_generate_entity.invoke_from,
)
credentials_provider, _ = build_dify_model_access(dify_context)
_, _, agent_soul = self._resolve_agent_by_id(
tenant_id=app_config.tenant_id,
agent_id=application_generate_entity.agent_id,
snapshot_id=application_generate_entity.agent_config_snapshot_id,
)
runner = AgentAppRunner(
request_builder=AgentAppRuntimeRequestBuilder(credentials_provider=credentials_provider),
agent_backend_client=create_agent_backend_run_client(
base_url=dify_config.AGENT_BACKEND_BASE_URL,
use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
),
event_adapter=AgentBackendRunEventAdapter(),
session_store=AgentAppRuntimeSessionStore(),
)
runner = self._build_runner(dify_context)
runner.run(
dify_context=dify_context,
agent_id=application_generate_entity.agent_id,
agent_config_snapshot_id=application_generate_entity.agent_config_snapshot_id,
agent_config_version_kind=application_generate_entity.agent_config_version_kind,
agent_soul=agent_soul,
conversation_id=conversation.id,
query=query,
@@ -374,6 +488,27 @@ class AgentAppGenerator(MessageBasedAppGenerator):
finally:
db.session.close()
@staticmethod
def _require_query(args: Mapping[str, Any]) -> str:
query = args.get("query")
if not isinstance(query, str) or not query.strip():
raise AgentAppGeneratorError("query is required")
return query.replace("\x00", "")
@staticmethod
def _build_runner(dify_context: DifyRunContext) -> AgentAppRunner:
credentials_provider, _ = build_dify_model_access(dify_context)
return AgentAppRunner(
request_builder=AgentAppRuntimeRequestBuilder(credentials_provider=credentials_provider),
agent_backend_client=create_agent_backend_run_client(
base_url=dify_config.AGENT_BACKEND_BASE_URL,
use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
),
event_adapter=AgentBackendRunEventAdapter(),
session_store=AgentAppRuntimeSessionStore(),
)
def _run_input_guards(
self,
*,
@@ -446,7 +581,7 @@ class AgentAppGenerator(MessageBasedAppGenerator):
invoke_from: InvokeFrom,
draft_type: Any,
user: Account | EndUser,
) -> tuple[Agent, str, AgentSoulConfig]:
) -> tuple[Agent, str, Literal["snapshot", "draft", "build_draft"], AgentSoulConfig]:
agent = db.session.scalar(
select(Agent)
.where(
@@ -474,13 +609,16 @@ class AgentAppGenerator(MessageBasedAppGenerator):
account_id=user.id if isinstance(user, Account) else None,
)
agent_soul = AgentSoulConfig.model_validate(draft.config_snapshot_dict)
return agent, draft.id, agent_soul
config_version_kind: Literal["snapshot", "draft", "build_draft"] = (
"build_draft" if draft.draft_type == AgentConfigDraftType.DEBUG_BUILD else "draft"
)
return agent, draft.id, config_version_kind, agent_soul
_, snapshot, agent_soul = self._resolve_agent_by_id(
tenant_id=app_model.tenant_id,
agent_id=agent.id,
snapshot_id=agent.active_config_snapshot_id,
)
return agent, snapshot.id, agent_soul
return agent, snapshot.id, "snapshot", agent_soul
@staticmethod
def _runtime_session_snapshot_id(*, invoke_from: InvokeFrom, snapshot_id: str) -> str | None:
api/core/app/apps/agent_app/app_runner.py
+436 -34
View File
@@ -1,18 +1,23 @@
"""Agent App runner: drive one conversation turn through the dify-agent backend.
"""Agent App runner: drive Agent backend turns for both chat and finalize flows.
Unlike the legacy ``AgentChatAppRunner`` (which runs an in-process ReAct loop),
this runner delegates to the Agent backend: build the run request from the
Agent Soul + conversation, create the run, consume its event stream, and
republish the assistant answer as chat queue events so the existing
EasyUI chat task pipeline persists the message and streams SSE. The conversation
``session_snapshot`` is saved on success for multi-turn continuity (S3).
this runner delegates to the Agent backend and supports two execution modes.
- Normal chat turns build the run request from the Agent Soul + conversation,
consume backend stream events, republish the assistant answer through the
existing EasyUI chat task pipeline, and save the conversation
``session_snapshot`` on success for multi-turn continuity (S3).
- Stateless build-finalize turns reuse any prior conversation snapshot only to
construct the backend request, wait synchronously for backend completion, and
intentionally do not persist Dify-side chat records or runtime-session state.
"""
from __future__ import annotations
import json
import logging
from typing import Any
from decimal import Decimal
from typing import Any, Literal
from dify_agent.layers.ask_human import AskHumanToolArgs
from dify_agent.protocol import DeferredToolResultsPayload
@@ -28,6 +33,7 @@ from clients.agent_backend import (
AgentBackendStreamInternalEvent,
extract_runtime_layer_specs,
)
from configs import dify_config
from core.app.apps.agent_app.runtime_request_builder import (
AgentAppRuntimeBuildContext,
AgentAppRuntimeRequest,
@@ -41,13 +47,16 @@ from core.app.apps.agent_app.session_store import (
from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
from core.app.apps.exc import GenerateTaskStoppedError
from core.app.entities.app_invoke_entities import DifyRunContext
from core.app.entities.queue_entities import QueueLLMChunkEvent, QueueMessageEndEvent
from core.app.entities.queue_entities import QueueAgentThoughtEvent, QueueLLMChunkEvent, QueueMessageEndEvent
from core.repositories.human_input_repository import HumanInputFormRepository, HumanInputFormRepositoryImpl
from core.workflow.nodes.agent_v2.ask_human_hitl import AskHumanFormBuildError, create_ask_human_form
from core.workflow.nodes.agent_v2.ask_human_resume import build_deferred_tool_results, resolve_ask_human_form
from extensions.ext_database import db
from graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage
from models.agent_config_entities import AgentSoulConfig
from models.enums import CreatorUserRole
from models.model import MessageAgentThought
logger = logging.getLogger(__name__)
@@ -134,6 +143,283 @@ def publish_message_end(
)
class _AgentProcessRecorder:
"""Persist Agent v2 thinking/tool process events through the legacy thought model."""
def __init__(
self,
*,
dify_context: DifyRunContext,
message_id: str,
queue_manager: AppQueueManager,
) -> None:
self._dify_context = dify_context
self._message_id = message_id
self._queue_manager = queue_manager
self._next_position = 1
self._thinking_by_index: dict[int, str] = {}
self._tool_by_index: dict[int, str] = {}
self._tool_by_call_id: dict[str, str] = {}
self._open_tool_by_name: dict[str, set[str]] = {}
def handle_stream_event(self, event: AgentBackendStreamInternalEvent) -> None:
data = event.data
if not isinstance(data, dict):
return
event_kind = data.get("event_kind")
if event_kind == "part_delta":
self._handle_part_delta(data)
elif event_kind == "part_start":
self._handle_part(data)
elif event_kind in {"function_tool_call", "output_tool_call"}:
self._handle_tool_call_event(data)
elif event_kind in {"function_tool_result", "output_tool_result"}:
self._handle_tool_result_event(data)
def _handle_part_delta(self, data: dict[str, Any]) -> None:
delta = data.get("delta")
if not isinstance(delta, dict):
return
index = _event_index(data)
delta_kind = delta.get("part_delta_kind")
if delta_kind == "thinking":
content_delta = delta.get("content_delta")
if isinstance(content_delta, str) and content_delta:
self._append_thinking(index, content_delta)
return
if delta_kind == "tool_call":
self._record_tool_call_delta(index, delta)
def _handle_part(self, data: dict[str, Any]) -> None:
part = data.get("part")
if not isinstance(part, dict):
return
index = _event_index(data)
part_kind = part.get("part_kind")
if part_kind == "thinking":
content = part.get("content")
if isinstance(content, str) and content:
self._append_thinking(index, content)
return
if part_kind in {"tool-call", "builtin-tool-call"}:
self._record_tool_call_part(index, part)
return
if part_kind in {"tool-return", "builtin-tool-return"}:
self._record_tool_return_part(part)
def _handle_tool_call_event(self, data: dict[str, Any]) -> None:
part = data.get("part")
if isinstance(part, dict):
self._record_tool_call_part(_event_index(data), part)
def _handle_tool_result_event(self, data: dict[str, Any]) -> None:
part = data.get("part") or data.get("result")
if isinstance(part, dict):
self._record_tool_return_part(part)
return
content = data.get("content")
if content is not None:
self._record_tool_observation(
tool_call_id=_string_or_none(data.get("tool_call_id")),
tool_name=_string_or_none(data.get("tool_name")),
observation=content,
)
def _append_thinking(self, index: int, content_delta: str) -> None:
thought_id = self._thinking_by_index.get(index)
if thought_id is None:
thought_id = self._create_thought(thought=content_delta)
self._thinking_by_index[index] = thought_id
return
self._update_thought(thought_id, thought_delta=content_delta)
def _record_tool_call_delta(self, index: int, delta: dict[str, Any]) -> None:
tool_call_id = _string_or_none(delta.get("tool_call_id"))
tool_name = _string_or_none(delta.get("tool_name_delta"))
args_delta = delta.get("args_delta")
thought_id = self._lookup_tool_thought(index=index, tool_call_id=tool_call_id)
if thought_id is None:
thought_id = self._create_thought(tool=tool_name, tool_input=_json_or_text(args_delta))
self._remember_tool_thought(
index=index, tool_call_id=tool_call_id, tool_name=tool_name, thought_id=thought_id
)
return
self._update_thought(
thought_id,
tool=tool_name,
tool_input_delta=_json_or_text(args_delta),
)
def _record_tool_call_part(self, index: int, part: dict[str, Any]) -> None:
tool_call_id = _string_or_none(part.get("tool_call_id"))
tool_name = _string_or_none(part.get("tool_name"))
thought_id = self._lookup_tool_thought(index=index, tool_call_id=tool_call_id)
if thought_id is None:
thought_id = self._create_thought(tool=tool_name, tool_input=_json_or_text(part.get("args")))
self._remember_tool_thought(
index=index, tool_call_id=tool_call_id, tool_name=tool_name, thought_id=thought_id
)
return
self._update_thought(
thought_id,
tool=tool_name,
tool_input=_json_or_text(part.get("args")),
)
def _record_tool_return_part(self, part: dict[str, Any]) -> None:
tool_call_id = _string_or_none(part.get("tool_call_id"))
tool_name = _string_or_none(part.get("tool_name"))
content = part.get("content")
if content is None:
content = part
self._record_tool_observation(tool_call_id=tool_call_id, tool_name=tool_name, observation=content)
def _record_tool_observation(self, *, tool_call_id: str | None, tool_name: str | None, observation: Any) -> None:
thought_id = self._lookup_observation_thought(tool_call_id=tool_call_id, tool_name=tool_name)
if thought_id is None:
thought_id = self._create_thought(tool=tool_name)
else:
self._mark_tool_observed(thought_id)
self._update_thought(thought_id, observation=_json_or_text(observation))
def _lookup_tool_thought(self, *, index: int, tool_call_id: str | None) -> str | None:
if tool_call_id and tool_call_id in self._tool_by_call_id:
return self._tool_by_call_id[tool_call_id]
return self._tool_by_index.get(index)
def _remember_tool_thought(
self, *, index: int, tool_call_id: str | None, tool_name: str | None, thought_id: str
) -> None:
self._tool_by_index[index] = thought_id
if tool_call_id:
self._tool_by_call_id[tool_call_id] = thought_id
if tool_name:
self._open_tool_by_name.setdefault(tool_name, set()).add(thought_id)
def _lookup_observation_thought(self, *, tool_call_id: str | None, tool_name: str | None) -> str | None:
if tool_call_id:
return self._tool_by_call_id.get(tool_call_id)
if tool_name:
open_thought_ids = self._open_tool_by_name.get(tool_name, set())
if len(open_thought_ids) == 1:
return next(iter(open_thought_ids))
return None
def _mark_tool_observed(self, thought_id: str) -> None:
for open_thought_ids in self._open_tool_by_name.values():
open_thought_ids.discard(thought_id)
def _create_thought(
self, *, thought: str | None = None, tool: str | None = None, tool_input: str | None = None
) -> str:
row = MessageAgentThought(
message_id=self._message_id,
message_chain_id=None,
thought=thought,
tool=tool,
tool_labels_str=_tool_labels(tool),
tool_meta_str="{}",
tool_input=tool_input,
observation=None,
tool_process_data=None,
message=None,
message_token=0,
message_unit_price=Decimal(0),
message_price_unit=Decimal("0.001"),
message_files="",
answer="",
answer_token=0,
answer_unit_price=Decimal(0),
answer_price_unit=Decimal("0.001"),
tokens=0,
total_price=Decimal(0),
position=self._next_position,
currency="USD",
latency=0,
created_by_role=self._created_by_role(),
created_by=self._dify_context.user_id,
)
self._next_position += 1
db.session.add(row)
db.session.commit()
thought_id = str(row.id)
self._queue_manager.publish(
QueueAgentThoughtEvent(agent_thought_id=thought_id), PublishFrom.APPLICATION_MANAGER
)
return thought_id
def _update_thought(
self,
thought_id: str,
*,
thought_delta: str | None = None,
tool: str | None = None,
tool_input: str | None = None,
tool_input_delta: str | None = None,
observation: str | None = None,
) -> None:
row = db.session.get(MessageAgentThought, thought_id)
if row is None:
return
if thought_delta:
row.thought = f"{row.thought or ''}{thought_delta}"
if tool:
row.tool = tool
row.tool_labels_str = _tool_labels(tool)
if tool_input is not None:
row.tool_input = tool_input
if tool_input_delta:
row.tool_input = f"{row.tool_input or ''}{tool_input_delta}"
if observation is not None:
row.observation = observation
db.session.commit()
self._queue_manager.publish(
QueueAgentThoughtEvent(agent_thought_id=thought_id), PublishFrom.APPLICATION_MANAGER
)
def _created_by_role(self) -> CreatorUserRole:
if self._dify_context.invoke_from.runs_as_account():
return CreatorUserRole.ACCOUNT
return CreatorUserRole.END_USER
def _event_index(data: dict[str, Any]) -> int:
index = data.get("index")
return index if isinstance(index, int) else -1
def _string_or_none(value: Any) -> str | None:
return value if isinstance(value, str) and value else None
def _json_or_text(value: Any) -> str | None:
if value is None:
return None
if isinstance(value, str):
return value
try:
return json.dumps(value, ensure_ascii=False)
except Exception:
return str(value)
def _tool_labels(tool: str | None) -> str:
if not tool:
return "{}"
return json.dumps({tool: {"en_US": tool, "zh_Hans": tool}}, ensure_ascii=False)
class AgentAppRunner:
"""Runs one Agent App conversation turn against the Agent backend."""
@@ -156,6 +442,7 @@ class AgentAppRunner:
dify_context: DifyRunContext,
agent_id: str,
agent_config_snapshot_id: str,
agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot",
agent_soul: AgentSoulConfig,
conversation_id: str,
query: str,
@@ -164,42 +451,34 @@ class AgentAppRunner:
queue_manager: AppQueueManager,
session_scope_snapshot_id: str | None | _DefaultSessionScopeSnapshotId = _DEFAULT_SESSION_SCOPE_SNAPSHOT_ID,
) -> None:
if isinstance(session_scope_snapshot_id, _DefaultSessionScopeSnapshotId):
effective_session_scope_snapshot_id: str | None = agent_config_snapshot_id
else:
effective_session_scope_snapshot_id = session_scope_snapshot_id
scope = AgentAppSessionScope(
tenant_id=dify_context.tenant_id,
app_id=dify_context.app_id,
conversation_id=conversation_id,
scope = self._build_session_scope(
dify_context=dify_context,
agent_id=agent_id,
agent_config_snapshot_id=effective_session_scope_snapshot_id,
agent_config_snapshot_id=agent_config_snapshot_id,
conversation_id=conversation_id,
session_scope_snapshot_id=session_scope_snapshot_id,
)
# ENG-638: if a prior turn paused on ask_human and the form is now answered,
# resume by threading the human's reply into this run as deferred_tool_results.
stored = self._session_store.load_active_session(scope)
session_snapshot = stored.session_snapshot if stored is not None else None
deferred_tool_results = self._resolve_pending_ask_human(
stored=stored, dify_context=dify_context, message_id=message_id
)
runtime = self._request_builder.build(
AgentAppRuntimeBuildContext(
dify_context=dify_context,
agent_id=agent_id,
agent_config_snapshot_id=agent_config_snapshot_id,
agent_soul=agent_soul,
conversation_id=conversation_id,
user_query=query,
idempotency_key=message_id,
session_snapshot=session_snapshot,
deferred_tool_results=deferred_tool_results,
)
runtime = self._build_runtime(
dify_context=dify_context,
agent_id=agent_id,
agent_config_snapshot_id=agent_config_snapshot_id,
agent_config_version_kind=agent_config_version_kind,
agent_soul=agent_soul,
conversation_id=conversation_id,
query=query,
idempotency_key=message_id,
stored=stored,
message_id=message_id,
)
create_response = self._agent_backend_client.create_run(runtime.request)
terminal, streamed_answer = self._consume_stream(
create_response.run_id,
dify_context=dify_context,
message_id=message_id,
queue_manager=queue_manager,
model_name=model_name,
query=query,
@@ -241,6 +520,111 @@ class AgentAppRunner:
runtime_layer_specs=extract_runtime_layer_specs(runtime.request.composition),
)
def run_stateless(
self,
*,
dify_context: DifyRunContext,
agent_id: str,
agent_config_snapshot_id: str,
agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot",
agent_soul: AgentSoulConfig,
conversation_id: str,
query: str,
idempotency_key: str,
session_scope_snapshot_id: str | None | _DefaultSessionScopeSnapshotId = _DEFAULT_SESSION_SCOPE_SNAPSHOT_ID,
) -> None:
"""Run the Agent backend without creating Dify chat message records.
This path is used by build-chat finalization: the API must trigger the
backend side effects in the existing conversation session, but it must
not persist a synthetic user/assistant turn, update API-side runtime
session rows, or set up HITL state that depends on one.
"""
scope = self._build_session_scope(
dify_context=dify_context,
agent_id=agent_id,
agent_config_snapshot_id=agent_config_snapshot_id,
conversation_id=conversation_id,
session_scope_snapshot_id=session_scope_snapshot_id,
)
runtime = self._build_runtime(
dify_context=dify_context,
agent_id=agent_id,
agent_config_snapshot_id=agent_config_snapshot_id,
agent_config_version_kind=agent_config_version_kind,
agent_soul=agent_soul,
conversation_id=conversation_id,
query=query,
idempotency_key=idempotency_key,
stored=self._session_store.load_active_session(scope),
message_id=None,
)
create_response = self._agent_backend_client.create_run(runtime.request)
status = self._agent_backend_client.wait_run(
create_response.run_id,
timeout_seconds=dify_config.APP_MAX_EXECUTION_TIME,
)
if status.status != "succeeded":
error = getattr(status, "error", None) or f"Agent backend run ended with status {status.status}."
raise AgentBackendError(str(error))
def _build_session_scope(
self,
*,
dify_context: DifyRunContext,
agent_id: str,
agent_config_snapshot_id: str,
conversation_id: str,
session_scope_snapshot_id: str | None | _DefaultSessionScopeSnapshotId,
) -> AgentAppSessionScope:
if isinstance(session_scope_snapshot_id, _DefaultSessionScopeSnapshotId):
effective_session_scope_snapshot_id: str | None = agent_config_snapshot_id
else:
effective_session_scope_snapshot_id = session_scope_snapshot_id
return AgentAppSessionScope(
tenant_id=dify_context.tenant_id,
app_id=dify_context.app_id,
conversation_id=conversation_id,
agent_id=agent_id,
agent_config_snapshot_id=effective_session_scope_snapshot_id,
)
def _build_runtime(
self,
*,
dify_context: DifyRunContext,
agent_id: str,
agent_config_snapshot_id: str,
agent_config_version_kind: Literal["snapshot", "draft", "build_draft"],
agent_soul: AgentSoulConfig,
conversation_id: str,
query: str,
idempotency_key: str,
stored: StoredAgentAppSession | None,
message_id: str | None,
) -> AgentAppRuntimeRequest:
session_snapshot = stored.session_snapshot if stored is not None else None
deferred_tool_results = (
self._resolve_pending_ask_human(stored=stored, dify_context=dify_context, message_id=message_id)
if message_id is not None
else None
)
return self._request_builder.build(
AgentAppRuntimeBuildContext(
dify_context=dify_context,
agent_id=agent_id,
agent_config_snapshot_id=agent_config_snapshot_id,
agent_config_version_kind=agent_config_version_kind,
agent_soul=agent_soul,
conversation_id=conversation_id,
user_query=query,
idempotency_key=idempotency_key,
session_snapshot=session_snapshot,
deferred_tool_results=deferred_tool_results,
)
)
def _pause_for_ask_human(
self,
*,
@@ -334,12 +718,19 @@ class AgentAppRunner:
self,
run_id: str,
*,
dify_context: DifyRunContext,
message_id: str,
queue_manager: AppQueueManager,
model_name: str,
query: str | None,
):
terminal = None
streamed_answer_parts: list[str] = []
process_recorder = _AgentProcessRecorder(
dify_context=dify_context,
message_id=message_id,
queue_manager=queue_manager,
)
for public_event in self._agent_backend_client.stream_events(run_id):
if queue_manager.is_stopped():
self._cancel_run(run_id)
@@ -353,6 +744,17 @@ class AgentAppRunner:
AgentBackendInternalEventType.STREAM_EVENT,
):
if isinstance(internal_event, AgentBackendStreamInternalEvent):
try:
process_recorder.handle_stream_event(internal_event)
except Exception:
db.session.rollback()
logger.warning(
"Failed to persist Agent App process event: run_id=%s message_id=%s event_kind=%s",
run_id,
message_id,
internal_event.event_kind,
exc_info=True,
)
text_delta = self._extract_stream_text_delta(internal_event)
if text_delta:
streamed_answer_parts.append(text_delta)
api/core/app/apps/agent_app/runtime_request_builder.py
+48 -28
View File
@@ -12,7 +12,7 @@ from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any, Protocol, cast
from typing import Any, Literal, Protocol, cast
from agenton.compositor import CompositorSessionSnapshot
from dify_agent.layers.execution_context import (
@@ -29,20 +29,22 @@ from clients.agent_backend import (
redact_for_agent_backend_log,
)
from configs import dify_config
from core.app.entities.app_invoke_entities import DifyRunContext
from core.workflow.nodes.agent_v2.plugin_tools_builder import (
WorkflowAgentPluginToolsBuilder,
WorkflowAgentPluginToolsBuildError,
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
from core.workflow.nodes.agent_v2.dify_tools_builder import (
WorkflowAgentDifyToolLayersBuilder,
WorkflowAgentDifyToolsBuilder,
WorkflowAgentDifyToolsBuildError,
WorkflowAgentToolLayers,
)
from core.workflow.nodes.agent_v2.runtime_request_builder import (
append_runtime_warnings,
build_ask_human_layer_config,
build_drive_aware_soul_mention_resolver,
build_drive_layer_config,
build_config_aware_soul_mention_resolver,
build_config_layer_config,
build_knowledge_layer_config,
build_shell_layer_config,
)
from models.agent_config_entities import AgentSoulConfig
from models.agent_config_entities import AgentSoulConfig, AgentSoulToolsConfig
from models.provider_ids import ModelProviderID
from services.agent.prompt_mentions import build_soul_mention_resolver, expand_prompt_mentions
@@ -68,6 +70,7 @@ class AgentAppRuntimeBuildContext:
conversation_id: str
user_query: str
idempotency_key: str
agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot"
session_snapshot: CompositorSessionSnapshot | None = None
# ENG-638: set when resuming a chat turn after a submitted ask_human form.
deferred_tool_results: DeferredToolResultsPayload | None = None
@@ -88,11 +91,11 @@ class AgentAppRuntimeRequestBuilder:
*,
credentials_provider: CredentialsProvider,
request_builder: AgentBackendRunRequestBuilder | None = None,
plugin_tools_builder: WorkflowAgentPluginToolsBuilder | None = None,
dify_tools_builder: WorkflowAgentDifyToolLayersBuilder | None = None,
) -> None:
self._credentials_provider = credentials_provider
self._request_builder = request_builder or AgentBackendRunRequestBuilder()
self._plugin_tools_builder = plugin_tools_builder or WorkflowAgentPluginToolsBuilder()
self._dify_tools_builder = dify_tools_builder or WorkflowAgentDifyToolsBuilder()
def build(self, context: AgentAppRuntimeBuildContext) -> AgentAppRuntimeRequest:
agent_soul = context.agent_soul
@@ -105,38 +108,33 @@ class AgentAppRuntimeRequestBuilder:
metadata = self._build_metadata(context)
credentials = self._credentials_provider.fetch(agent_soul.model.model_provider, agent_soul.model.model)
try:
tools_layer = self._plugin_tools_builder.build(
tool_layers = self._build_tool_layers(
tenant_id=context.dify_context.tenant_id,
app_id=context.dify_context.app_id,
user_id=context.dify_context.user_id,
tools=agent_soul.tools,
invoke_from=context.dify_context.invoke_from,
)
except WorkflowAgentPluginToolsBuildError as error:
except WorkflowAgentDifyToolsBuildError as error:
raise AgentAppRuntimeRequestBuildError(error.error_code, str(error)) from error
if tools_layer is not None or agent_soul.tools.cli_tools:
if tool_layers.plugin_tools is not None or tool_layers.core_tools is not None or agent_soul.tools.cli_tools:
metadata["agent_tools"] = {
"dify_tool_count": len(tools_layer.tools) if tools_layer is not None else 0,
"dify_tool_names": [tool.name or tool.tool_name for tool in tools_layer.tools]
if tools_layer is not None
else [],
"dify_tool_count": len(tool_layers.exposed_tool_names()),
"dify_tool_names": tool_layers.exposed_tool_names(),
"cli_tool_count": len(agent_soul.tools.cli_tools),
}
drive_config = None
config_layer_config = None
soul_prompt_resolver = build_soul_mention_resolver(agent_soul)
if dify_config.AGENT_DRIVE_MANIFEST_ENABLED:
drive_config, drive_warnings = build_drive_layer_config(
config_layer_config, config_warnings = build_config_layer_config(
agent_soul,
tenant_id=context.dify_context.tenant_id,
agent_id=context.agent_id,
)
append_runtime_warnings(metadata, drive_warnings)
soul_prompt_resolver = build_drive_aware_soul_mention_resolver(
agent_soul,
tenant_id=context.dify_context.tenant_id,
agent_id=context.agent_id,
config_version_id=context.agent_config_snapshot_id,
config_version_kind=context.agent_config_version_kind,
)
append_runtime_warnings(metadata, config_warnings)
soul_prompt_resolver = build_config_aware_soul_mention_resolver(agent_soul)
knowledge_config = build_knowledge_layer_config(agent_soul)
request = self._request_builder.build_for_agent_app(
@@ -158,6 +156,7 @@ class AgentAppRuntimeRequestBuilder:
conversation_id=context.conversation_id,
agent_id=context.agent_id,
agent_config_version_id=context.agent_config_snapshot_id,
agent_config_version_kind=context.agent_config_version_kind,
# Agent Files §1.3: real Dify access context + agent run mode.
user_from=cast(DifyExecutionContextUserFrom, context.dify_context.user_from.value),
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
@@ -168,9 +167,10 @@ class AgentAppRuntimeRequestBuilder:
agent_soul_prompt=expand_prompt_mentions(agent_soul.prompt.system_prompt, soul_prompt_resolver).strip()
or None,
user_prompt=context.user_query,
tools=tools_layer,
tools=tool_layers.plugin_tools,
core_tools=tool_layers.core_tools,
knowledge=knowledge_config,
drive_config=drive_config,
config_layer_config=config_layer_config,
ask_human_config=build_ask_human_layer_config(agent_soul),
include_shell=dify_config.AGENT_SHELL_ENABLED,
shell_config=build_shell_layer_config(agent_soul),
@@ -183,6 +183,26 @@ class AgentAppRuntimeRequestBuilder:
redacted = cast(dict[str, Any], redact_for_agent_backend_log(request))
return AgentAppRuntimeRequest(request=request, redacted_request=redacted, metadata=metadata)
def _build_tool_layers(
self,
*,
tenant_id: str,
app_id: str,
user_id: str | None,
tools: AgentSoulToolsConfig,
invoke_from: InvokeFrom,
) -> WorkflowAgentToolLayers:
# Production Agent App runs intentionally keep existing plugin configs
# on the direct `dify.plugin.tools` route. This builder emits plugin
# tools directly and non-plugin Dify tools through `dify.core.tools`.
return self._dify_tools_builder.build_layers(
tenant_id=tenant_id,
app_id=app_id,
user_id=user_id,
tools=tools,
invoke_from=invoke_from,
)
@staticmethod
def _build_metadata(context: AgentAppRuntimeBuildContext) -> dict[str, Any]:
return {
api/core/app/apps/pipeline/pipeline_generator.py
+1 -1
View File
@@ -144,7 +144,7 @@ class PipelineGenerator(BaseAppGenerator):
DocumentService.check_document_creation_limits(len(datasource_info_list), features)
for datasource_info in datasource_info_list:
position = DocumentService.get_documents_position(dataset.id)
position = DocumentService.get_documents_position(dataset.id, session)
document = self._build_document(
tenant_id=pipeline.tenant_id,
dataset_id=dataset.id,
api/core/app/entities/app_invoke_entities.py
+15 -2
View File
@@ -1,8 +1,8 @@
from collections.abc import Mapping, Sequence
from enum import StrEnum
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Literal
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationInfo, field_validator
from constants import UUID_NIL
from core.app.app_config.entities import EasyUIBasedAppConfig, WorkflowUIBasedAppConfig
@@ -220,11 +220,24 @@ class AgentAppGenerateEntity(ChatAppGenerateEntity):
accepted-entity union. The answer is produced by the dify-agent backend
rather than an in-process LLM call; ``model_conf`` is synthesized from the
bound Agent Soul model so the chat task pipeline can persist usage.
``agent_config_version_kind`` selects which Agent config surface the
backend should read from: immutable snapshot, shared draft, or per-user
build draft.
``agent_runtime_session_snapshot_id`` carries the runtime session scope
used to resume or suspend within the same editable config surface.
``prompt_file_mappings`` preserves the raw request ``files`` array for the
Agent backend prompt. These references are appended to the backend prompt
text while the stored chat message keeps the user's original query.
"""
agent_id: str
agent_config_snapshot_id: str
agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot"
agent_runtime_session_snapshot_id: str | None = None
prompt_file_mappings: Sequence[JsonValue] = Field(default_factory=list)
class AdvancedChatAppGenerateEntity(ConversationAppGenerateEntity):
api/core/app/features/annotation_reply/annotation_reply.py
+1 -1
View File
@@ -45,7 +45,7 @@ class AnnotationReplyFeature:
embedding_model_name = collection_binding_detail.model_name
dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding(
embedding_provider_name, embedding_model_name, CollectionBindingType.ANNOTATION
embedding_provider_name, embedding_model_name, db.session, CollectionBindingType.ANNOTATION
)
dataset = Dataset(
api/core/callback_handler/index_tool_callback_handler.py
+10 -10
View File
@@ -2,6 +2,7 @@ import logging
from collections.abc import Sequence
from sqlalchemy import select, update
from sqlalchemy.orm import scoped_session
from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
from core.app.entities.app_invoke_entities import InvokeFrom
@@ -9,7 +10,6 @@ from core.app.entities.queue_entities import QueueRetrieverResourcesEvent
from core.rag.entities import RetrievalSourceMetadata
from core.rag.index_processor.constant.index_type import IndexStructureType
from core.rag.models.document import Document
from extensions.ext_database import db
from models.dataset import ChildChunk, DatasetQuery, DocumentSegment
from models.dataset import Document as DatasetDocument
from models.enums import CreatorUserRole, DatasetQuerySource
@@ -29,7 +29,7 @@ class DatasetIndexToolCallbackHandler:
self._user_id = user_id
self._invoke_from = invoke_from
def on_query(self, query: str, dataset_id: str):
def on_query(self, query: str, dataset_id: str, session: scoped_session):
"""
Handle query.
"""
@@ -46,16 +46,16 @@ class DatasetIndexToolCallbackHandler:
created_by=self._user_id,
)
db.session.add(dataset_query)
db.session.commit()
session.add(dataset_query)
session.commit()
def on_tool_end(self, documents: list[Document]):
def on_tool_end(self, documents: list[Document], session: scoped_session):
"""Handle tool end."""
for document in documents:
if document.metadata is not None:
document_id = document.metadata["document_id"]
dataset_document_stmt = select(DatasetDocument).where(DatasetDocument.id == document_id)
dataset_document = db.session.scalar(dataset_document_stmt)
dataset_document = session.scalar(dataset_document_stmt)
if not dataset_document:
_logger.warning(
"Expected DatasetDocument record to exist, but none was found, document_id=%s",
@@ -68,9 +68,9 @@ class DatasetIndexToolCallbackHandler:
ChildChunk.dataset_id == dataset_document.dataset_id,
ChildChunk.document_id == dataset_document.id,
)
child_chunk = db.session.scalar(child_chunk_stmt)
child_chunk = session.scalar(child_chunk_stmt)
if child_chunk:
db.session.execute(
session.execute(
update(DocumentSegment)
.where(DocumentSegment.id == child_chunk.segment_id)
.values(hit_count=DocumentSegment.hit_count + 1)
@@ -82,11 +82,11 @@ class DatasetIndexToolCallbackHandler:
conditions.append(DocumentSegment.dataset_id == document.metadata["dataset_id"])
# add hit count to document segment
db.session.execute(
session.execute(
update(DocumentSegment).where(*conditions).values(hit_count=DocumentSegment.hit_count + 1)
)
db.session.commit()
session.commit()
# TODO(-LAN-): Improve type check
def return_retriever_resource_info(self, resource: Sequence[RetrievalSourceMetadata]):
api/core/callback_handler/workflow_tool_callback_handler.py
+6 -4
View File
@@ -1,6 +1,7 @@
from collections.abc import Generator, Iterable, Mapping
from typing import Any
from configs import dify_config
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler, print_text
from core.ops.ops_trace_manager import TraceQueueManager
from core.tools.entities.tool_entities import ToolInvokeMessage
@@ -19,8 +20,9 @@ class DifyWorkflowCallbackHandler(DifyAgentCallbackHandler):
trace_manager: TraceQueueManager | None = None,
) -> Generator[ToolInvokeMessage, None, None]:
for tool_output in tool_outputs:
print_text("\n[on_tool_execution]\n", color=self.color)
print_text("Tool: " + tool_name + "\n", color=self.color)
print_text("Outputs: " + tool_output.model_dump_json()[:1000] + "\n", color=self.color)
print_text("\n")
if dify_config.DEBUG:
print_text("\n[on_tool_execution]\n", color=self.color)
print_text("Tool: " + tool_name + "\n", color=self.color)
print_text("Outputs: " + tool_output.model_dump_json()[:1000] + "\n", color=self.color)
print_text("\n")
yield tool_output
api/core/helper/credential_utils.py
+6 -3
View File
@@ -10,18 +10,21 @@ def is_credential_exists(credential_id: str, credential_type: "PluginCredentialT
"""
Check if the credential still exists in the database.
Uses the configured SQLAlchemy session factory instead of Flask-SQLAlchemy's
``db.engine`` because workflow graph node construction may run without an
active Flask application context.
:param credential_id: The credential ID to check
:param credential_type: The type of credential (MODEL or TOOL)
:return: True if credential exists, False otherwise
"""
from sqlalchemy import select
from sqlalchemy.orm import Session
from extensions.ext_database import db
from core.db import session_factory
from models.provider import ProviderCredential, ProviderModelCredential
from models.tools import BuiltinToolProvider
with Session(db.engine) as session:
with session_factory.create_session() as session:
if credential_type == PluginCredentialType.MODEL:
# Check both pre-defined and custom model credentials using a single UNION query
stmt = (
api/core/llm_generator/llm_generator.py
+218 -3
View File
@@ -2,7 +2,7 @@ import json
import logging
import re
from collections.abc import Sequence
from typing import Any, NotRequired, Protocol, TypedDict, cast
from typing import Any, Literal, NotRequired, Protocol, TypedDict, cast
import json_repair
from sqlalchemy import select
@@ -69,6 +69,53 @@ def _normalize_completion_params(completion_params: dict[str, object]) -> tuple[
return normalized_parameters, stop
# ── Workflow instruction-suggestion tuning ────────────────────────────────
# Suggestions are a soft, pre-model-pick enhancement: short, buildable example
# instructions proposed from the tenant's DEFAULT model. Every failure path
# degrades to an empty list, never an error.
_SUGGESTION_MIN_COUNT = 1
_SUGGESTION_MAX_COUNT = 6
_SUGGESTION_MAX_TOKENS = 512
_SUGGESTION_TEMPERATURE = 0.8
# Bound the grounding context so the prompt stays small regardless of how many
# knowledge bases / tools the tenant has installed.
_SUGGESTION_KB_LIMIT = 10
_SUGGESTION_TOOL_SAMPLE_LINES = 20
_SUGGESTION_SYSTEM_PROMPT = (
"You help a user start building a Dify app by proposing example build instructions. "
"Each suggestion must be a SHORT (at most 8 words), concrete, and BUILDABLE instruction "
"describing an app to generate for the given app type. Make the suggestions diverse — cover "
"different use cases. When the listed knowledge bases or installed tools fit a suggestion, "
"prefer them, but NEVER invent tools or knowledge bases that are not listed. "
"Reply with ONLY a JSON array of strings and nothing else."
)
def _parse_string_list(text: str) -> list[str]:
"""Extract a JSON array of strings from a (possibly noisy) LLM response.
Slices the first ``[...]`` span so surrounding prose / markdown fences are
tolerated, parses it with ``json`` and falls back to ``json_repair``, then
keeps only ``str`` items. Returns ``[]`` on any failure so callers can
treat parsing as best-effort.
"""
match = re.search(r"\[.*\]", text.strip(), re.DOTALL)
if not match:
return []
raw = match.group(0)
try:
parsed = json.loads(raw)
except Exception:
try:
parsed = json_repair.loads(raw)
except Exception:
return []
if not isinstance(parsed, list):
return []
return [item for item in parsed if isinstance(item, str)]
class WorkflowServiceInterface(Protocol):
def get_draft_workflow(self, app_model: App, workflow_id: str | None = None) -> Workflow | None:
pass
@@ -237,6 +284,170 @@ class LLMGenerator:
return questions
@classmethod
def generate_workflow_instruction_suggestions(
cls,
tenant_id: str,
*,
mode: Literal["workflow", "advanced-chat"],
language: str | None = None,
count: int = 4,
) -> list[str]:
"""Propose short, buildable example instructions for the workflow generator.
Runs BEFORE the user picks a model, so it uses the tenant's DEFAULT LLM
only. Suggestions are a soft enhancement, never a blocker: every failure
path (no default model, KB / tool lookup error, LLM error, unparseable
output) is swallowed and surfaced as an empty list — a valid result the
caller renders as "no suggestions". This method NEVER raises.
"""
count = max(_SUGGESTION_MIN_COUNT, min(count, _SUGGESTION_MAX_COUNT))
try:
model_instance = ModelManager.for_tenant(tenant_id=tenant_id).get_default_model_instance(
tenant_id=tenant_id,
model_type=ModelType.LLM,
)
except Exception:
logger.info("Workflow instruction suggestions: no default model for tenant %s", tenant_id)
return []
context_block = cls._build_suggestion_context(tenant_id)
app_type_label = (
"Workflow — single-shot automation" if mode == "workflow" else "Chatflow — conversational multi-turn"
)
user_lines = [
f"App type: {app_type_label}",
context_block,
f"Return exactly {count} distinct ideas as a JSON array of strings.",
]
if language:
user_lines.append(f"Write every idea in this language: {language}.")
user_prompt = "\n".join(line for line in user_lines if line)
prompt_messages: list[PromptMessage] = [
SystemPromptMessage(content=_SUGGESTION_SYSTEM_PROMPT),
UserPromptMessage(content=user_prompt),
]
try:
response: LLMResult = model_instance.invoke_llm(
prompt_messages=prompt_messages,
model_parameters={"max_tokens": _SUGGESTION_MAX_TOKENS, "temperature": _SUGGESTION_TEMPERATURE},
stream=False,
)
except Exception:
logger.exception("Workflow instruction suggestions: LLM invocation failed")
return []
raw_suggestions = _parse_string_list(response.message.get_text_content() or "")
# Strip whitespace + surrounding quotes, drop empties, dedupe
# case-insensitively (preserving first-seen casing), cap to ``count``.
cleaned: list[str] = []
seen: set[str] = set()
for item in raw_suggestions:
idea = item.strip().strip("\"'").strip()
if not idea:
continue
key = idea.casefold()
if key in seen:
continue
seen.add(key)
cleaned.append(idea)
if len(cleaned) >= count:
break
return cleaned
@staticmethod
def _build_suggestion_context(tenant_id: str) -> str:
"""Assemble an optional grounding block naming the tenant's KBs and tools.
Best-effort: each section is isolated in its own try/except so a failure
enumerating one (DB hiccup, plugin daemon down) never blocks the other
or the suggestion call itself. Returns "" when nothing is available.
"""
sections: list[str] = []
try:
from models.dataset import Dataset
names = db.session.scalars(
select(Dataset.name)
.where(Dataset.tenant_id == tenant_id)
.order_by(Dataset.created_at.desc())
.limit(_SUGGESTION_KB_LIMIT)
).all()
kb_names = [name for name in names if name]
if kb_names:
sections.append("Knowledge bases:\n" + "\n".join(f"- {name}" for name in kb_names))
except Exception:
logger.info("Workflow instruction suggestions: failed to load knowledge bases", exc_info=True)
try:
from core.workflow.generator.tool_catalogue import build_tool_catalogue, format_tool_catalogue
tool_text = format_tool_catalogue(build_tool_catalogue(tenant_id))
if tool_text:
sample = "\n".join(tool_text.splitlines()[:_SUGGESTION_TOOL_SAMPLE_LINES])
sections.append("Installed tools:\n" + sample)
except Exception:
logger.info("Workflow instruction suggestions: failed to load tool catalogue", exc_info=True)
if not sections:
return ""
return "\n\n".join(sections) + "\n\n"
@classmethod
def classify_workflow_mode(
cls,
tenant_id: str,
instruction: str,
model_config: ModelConfig,
) -> Literal["workflow", "advanced-chat"]:
"""Classify a free-text instruction into a concrete app mode.
One tiny LLM call using the model the user already picked (so no extra
provider setup is needed). Parsed leniently; defaults to
``advanced-chat`` on anything unexpected or any error, so a
``mode="auto"`` request never blocks generation. NEVER raises.
"""
default_mode: Literal["workflow", "advanced-chat"] = "advanced-chat"
try:
model_instance = ModelManager.for_tenant(tenant_id=tenant_id).get_model_instance(
tenant_id=tenant_id,
model_type=ModelType.LLM,
provider=model_config.provider,
model=model_config.name,
)
prompt_messages: list[PromptMessage] = [
UserPromptMessage(
content=(
"Reply with exactly one word: 'workflow' (one-shot automation, no chat) "
"or 'advanced-chat' (conversational multi-turn). "
f"Instruction: {instruction.strip()}"
)
),
]
response: LLMResult = model_instance.invoke_llm(
prompt_messages=prompt_messages,
model_parameters={"max_tokens": 4, "temperature": 0},
stream=False,
)
text = (response.message.get_text_content() or "").strip().lower()
except Exception:
logger.info("Workflow mode classification failed; defaulting to %s", default_mode, exc_info=True)
return default_mode
# Lenient parse: an affirmative "workflow" wins; everything else
# (including a truncated / empty / garbled reply) falls back to the
# conversational default. "advanced-chat" needs no positive match
# because it IS the default.
if "workflow" in text:
return "workflow"
return default_mode
@classmethod
def generate_rule_config(cls, tenant_id: str, args: RuleGeneratePayload):
output_parser = RuleConfigGeneratorOutputParser()
@@ -498,7 +709,11 @@ class LLMGenerator:
ideal_output: str | None,
):
last_run: Message | None = db.session.scalar(
select(Message).where(Message.app_id == flow_id).order_by(Message.created_at.desc()).limit(1)
select(Message)
.join(App, App.id == Message.app_id)
.where(Message.app_id == flow_id, App.tenant_id == tenant_id)
.order_by(Message.created_at.desc())
.limit(1)
)
if not last_run:
return LLMGenerator.__instruction_modify_common(
@@ -540,7 +755,7 @@ class LLMGenerator:
):
session = db.session()
app: App | None = session.scalar(select(App).where(App.id == flow_id).limit(1))
app: App | None = session.scalar(select(App).where(App.id == flow_id, App.tenant_id == tenant_id).limit(1))
if not app:
raise ValueError("App not found.")
workflow = workflow_service.get_draft_workflow(app_model=app)
api/core/plugin/entities/marketplace.py
+4 -2
View File
@@ -1,4 +1,4 @@
from typing import Any
from typing import Any, Literal
from pydantic import BaseModel, Field, computed_field, model_validator
@@ -32,7 +32,9 @@ class MarketplacePluginDeclaration(BaseModel):
latest_package_identifier: str = Field(
..., description="Unique identifier for the latest package release of the plugin"
)
status: str = Field(..., description="Indicate the status of marketplace plugin, enum from `active` `deleted`")
status: Literal["active", "deleted"] = Field(
..., description="Indicate the status of marketplace plugin, enum from `active` `deleted`"
)
deprecated_reason: str = Field(
..., description="Not empty when status='deleted', indicates the reason why this plugin is deleted(deprecated)"
)
api/core/plugin/plugin_service.py
+171 -79
View File
@@ -14,12 +14,14 @@ metadata.
import logging
import time
from collections.abc import Mapping, Sequence
from collections.abc import Iterator, Mapping, Sequence
from contextlib import contextmanager
from mimetypes import guess_type
from typing import ClassVar
from typing import Literal, Protocol
from pydantic import BaseModel, TypeAdapter, ValidationError
from redis import RedisError
from redis.exceptions import LockError
from sqlalchemy import delete, select, update
from sqlalchemy.orm import Session
from yarl import URL
@@ -67,14 +69,18 @@ logger = logging.getLogger(__name__)
_provider_entities_adapter: TypeAdapter[list[ProviderEntity]] = TypeAdapter(list[ProviderEntity])
class PluginService:
_plugin_model_providers_memory_cache: ClassVar[dict[str, tuple[int, float, tuple[ProviderEntity, ...]]]] = {}
class _RedisLock(Protocol):
def acquire(self, *, blocking: bool = True, blocking_timeout: float | None = None) -> bool: ...
def release(self) -> None: ...
class PluginService:
class LatestPluginCache(BaseModel):
plugin_id: str
version: str
unique_identifier: str
status: str
status: Literal["active", "deleted"]
deprecated_reason: str
alternative_plugin_id: str
@@ -82,6 +88,10 @@ class PluginService:
REDIS_TTL = 60 * 5 # 5 minutes
PLUGIN_MODEL_PROVIDERS_REDIS_KEY_PREFIX = "plugin_model_providers:tenant_id:"
PLUGIN_MODEL_PROVIDERS_GENERATION_REDIS_KEY_PREFIX = "plugin_model_providers_generation:tenant_id:"
PLUGIN_MODEL_PROVIDERS_LOCK_REDIS_KEY_PREFIX = "plugin_model_providers_refresh_lock:tenant_id:"
PLUGIN_MODEL_PROVIDERS_LOCK_TTL = 30
PLUGIN_MODEL_PROVIDERS_LOCK_WAIT_TIMEOUT = 2.0
PLUGIN_MODEL_PROVIDERS_LOCK_WAIT_INTERVAL = 0.05
PLUGIN_INSTALL_TASK_TERMINAL_STATUSES = (PluginInstallTaskStatus.Success, PluginInstallTaskStatus.Failed)
# Mirror the detail-panel endpoint query size so list reconciliation and
# the visible endpoint drawer exercise the same daemon pagination path.
@@ -98,6 +108,10 @@ class PluginService:
def _get_plugin_model_providers_generation_cache_key(cls, tenant_id: str) -> str:
return f"{cls.PLUGIN_MODEL_PROVIDERS_GENERATION_REDIS_KEY_PREFIX}{tenant_id}"
@classmethod
def _get_plugin_model_providers_lock_key(cls, tenant_id: str, generation: int) -> str:
return f"{cls.PLUGIN_MODEL_PROVIDERS_LOCK_REDIS_KEY_PREFIX}{tenant_id}:generation:{generation}"
@staticmethod
def _get_provider_short_name_alias(provider: PluginModelProviderEntity) -> str:
"""
@@ -128,10 +142,6 @@ class PluginService:
declaration.provider_name = cls._get_provider_short_name_alias(provider)
return declaration
@classmethod
def _copy_provider_entities(cls, providers: Sequence[ProviderEntity]) -> tuple[ProviderEntity, ...]:
return tuple(provider.model_copy(deep=True) for provider in providers)
@classmethod
def _load_plugin_model_providers_generation(cls, tenant_id: str) -> int | None:
cache_key = cls._get_plugin_model_providers_generation_cache_key(tenant_id)
@@ -163,66 +173,26 @@ class PluginService:
return None
@classmethod
def _load_in_memory_plugin_model_providers(
cls, memory_cache_key: str, generation: int
) -> tuple[ProviderEntity, ...] | None:
cached_entry = cls._plugin_model_providers_memory_cache.get(memory_cache_key)
if cached_entry is None:
return None
def _load_cached_plugin_model_providers_for_generation(
cls, tenant_id: str, generation: int | None
) -> tuple[tuple[ProviderEntity, ...] | None, bool]:
if generation is None:
return None, False
cached_generation, expires_at, providers = cached_entry
if cached_generation != generation or time.monotonic() >= expires_at:
cls._plugin_model_providers_memory_cache.pop(memory_cache_key, None)
return None
return cls._copy_provider_entities(providers)
@classmethod
def _store_in_memory_plugin_model_providers(
cls, memory_cache_key: str, generation: int, providers: Sequence[ProviderEntity]
) -> None:
ttl = dify_config.PLUGIN_MODEL_PROVIDERS_CACHE_TTL
if ttl <= 0:
cls._plugin_model_providers_memory_cache.pop(memory_cache_key, None)
return
cls._plugin_model_providers_memory_cache[memory_cache_key] = (
generation,
time.monotonic() + ttl,
cls._copy_provider_entities(providers),
)
@classmethod
def _load_cached_plugin_model_providers(
cls, tenant_id: str, *, client: PluginModelClient | None = None
) -> tuple[ProviderEntity, ...] | None:
generation = cls._load_plugin_model_providers_generation(tenant_id)
if generation is not None:
in_memory_cached_providers = cls._load_in_memory_plugin_model_providers(tenant_id, generation)
if in_memory_cached_providers is not None:
return in_memory_cached_providers
cache_keys = []
if generation is not None:
cache_keys.append(cls._get_plugin_model_providers_cache_key(tenant_id, generation))
if generation == 0:
cache_keys.append(cls._get_plugin_model_providers_cache_key(tenant_id))
if not cache_keys:
return None
cache_keys = [cls._get_plugin_model_providers_cache_key(tenant_id, generation)]
try:
cached_provider_entries = redis_client.mget(cache_keys)
except (RedisError, RuntimeError):
except (LockError, RedisError, RuntimeError):
logger.warning("Failed to read cached plugin model providers for tenant %s.", tenant_id, exc_info=True)
return None
return None, False
if len(cached_provider_entries) != len(cache_keys):
logger.warning(
"Unexpected cached plugin model providers response size for tenant %s.",
tenant_id,
)
return None
return None, False
for cache_key, cached_providers in zip(cache_keys, cached_provider_entries):
if not cached_providers:
@@ -230,9 +200,7 @@ class PluginService:
try:
providers = tuple(_provider_entities_adapter.validate_json(cached_providers))
if generation is not None:
cls._store_in_memory_plugin_model_providers(tenant_id, generation, providers)
return providers
return providers, True
except (TypeError, ValueError, ValidationError):
logger.warning(
"Invalid cached plugin model providers for tenant %s; deleting cache key %s.",
@@ -249,7 +217,7 @@ class PluginService:
exc_info=True,
)
return None
return None, True
@classmethod
def _store_cached_plugin_model_providers(
@@ -257,15 +225,92 @@ class PluginService:
) -> None:
cache_key = cls._get_plugin_model_providers_cache_key(tenant_id, generation)
try:
payload = _provider_entities_adapter.dump_json(list(providers)).decode("utf-8")
payload = _provider_entities_adapter.dump_json(list(providers))
redis_client.setex(cache_key, dify_config.PLUGIN_MODEL_PROVIDERS_CACHE_TTL, payload)
except (RedisError, RuntimeError):
logger.warning("Failed to cache plugin model providers for tenant %s.", tenant_id, exc_info=True)
@classmethod
@contextmanager
def _plugin_model_providers_refresh_lock(
cls, tenant_id: str, generation: int, *, wait_timeout: float
) -> Iterator[bool]:
lock_key = cls._get_plugin_model_providers_lock_key(tenant_id, generation)
try:
refresh_lock: _RedisLock = redis_client.lock(
lock_key,
timeout=cls.PLUGIN_MODEL_PROVIDERS_LOCK_TTL,
sleep=cls.PLUGIN_MODEL_PROVIDERS_LOCK_WAIT_INTERVAL,
)
except (RedisError, RuntimeError):
logger.warning(
"Failed to create plugin model providers refresh lock for tenant %s.",
tenant_id,
exc_info=True,
)
yield False
return
try:
lock_acquired = refresh_lock.acquire(blocking=True, blocking_timeout=wait_timeout)
except LockError:
logger.warning(
"Provider refresh lock timed out; direct daemon fallback. tenant_id=%s generation=%s",
tenant_id,
generation,
exc_info=True,
)
yield False
return
except (RedisError, RuntimeError):
# Redis failures should not block provider discovery; callers fetch directly from the daemon.
logger.warning(
"Failed to acquire plugin model providers refresh lock for tenant %s.",
tenant_id,
exc_info=True,
)
yield False
return
if not lock_acquired:
logger.warning(
"Provider refresh lock timed out; direct daemon fallback. tenant_id=%s generation=%s",
tenant_id,
generation,
)
yield False
return
try:
yield True
finally:
try:
refresh_lock.release()
except (LockError, RedisError, RuntimeError):
# Release failures must not hide the daemon result or the original exception.
logger.warning(
"Failed to release plugin model providers refresh lock for tenant %s generation %s.",
tenant_id,
generation,
exc_info=True,
)
@classmethod
def _fetch_and_cache_plugin_model_providers(
cls, tenant_id: str, client: PluginModelClient | None, *, refresh_generation: int | None
) -> tuple[ProviderEntity, ...]:
model_client = client or PluginModelClient()
providers = tuple(
cls._to_provider_entity(provider) for provider in model_client.fetch_model_providers(tenant_id)
)
generation = cls._load_plugin_model_providers_generation(tenant_id)
if generation is not None and generation == refresh_generation:
cls._store_cached_plugin_model_providers(tenant_id, generation, providers)
return providers
@classmethod
def invalidate_plugin_model_providers_cache(cls, tenant_id: str) -> None:
"""Invalidate tenant-scoped provider metadata across Redis and worker-local mirrors."""
cls._plugin_model_providers_memory_cache.pop(tenant_id, None)
"""Invalidate tenant-scoped provider metadata stored in Redis."""
cache_key = cls._get_plugin_model_providers_cache_key(tenant_id)
generation_key = cls._get_plugin_model_providers_generation_cache_key(tenant_id)
try:
@@ -287,21 +332,68 @@ class PluginService:
are intentionally owned by this service so tenant isolation and cache
expiry are handled in one place.
"""
cached_providers = cls._load_cached_plugin_model_providers(tenant_id, client=client)
if cached_providers is not None:
return cached_providers
deadline = time.monotonic() + cls.PLUGIN_MODEL_PROVIDERS_LOCK_WAIT_TIMEOUT
model_client = client or PluginModelClient()
providers = tuple(
cls._to_provider_entity(provider) for provider in model_client.fetch_model_providers(tenant_id)
)
if not providers:
return providers
generation = cls._load_plugin_model_providers_generation(tenant_id)
if generation is not None:
cls._store_in_memory_plugin_model_providers(tenant_id, generation, providers)
cls._store_cached_plugin_model_providers(tenant_id, generation, providers)
return providers
while True:
generation = cls._load_plugin_model_providers_generation(tenant_id)
cached_providers, cache_available = cls._load_cached_plugin_model_providers_for_generation(
tenant_id, generation
)
if cached_providers is not None:
return cached_providers
if generation is None or not cache_available:
return cls._fetch_and_cache_plugin_model_providers(
tenant_id,
client,
refresh_generation=generation,
)
wait_timeout = deadline - time.monotonic()
if wait_timeout < 0:
logger.warning(
"Provider refresh lock timed out; direct daemon fallback. tenant_id=%s generation=%s",
tenant_id,
generation,
)
return cls._fetch_and_cache_plugin_model_providers(
tenant_id,
client,
refresh_generation=generation,
)
with cls._plugin_model_providers_refresh_lock(
tenant_id,
generation,
wait_timeout=wait_timeout,
) as lock_acquired:
if not lock_acquired:
return cls._fetch_and_cache_plugin_model_providers(
tenant_id,
client,
refresh_generation=generation,
)
latest_generation = cls._load_plugin_model_providers_generation(tenant_id)
cached_providers, cache_available = cls._load_cached_plugin_model_providers_for_generation(
tenant_id, latest_generation
)
if cached_providers is not None:
return cached_providers
if latest_generation is None or not cache_available:
return cls._fetch_and_cache_plugin_model_providers(
tenant_id,
client,
refresh_generation=latest_generation,
)
if latest_generation != generation:
continue
return cls._fetch_and_cache_plugin_model_providers(
tenant_id,
client,
refresh_generation=generation,
)
@staticmethod
def fetch_latest_plugin_version(plugin_ids: Sequence[str]) -> Mapping[str, LatestPluginCache | None]:
api/core/rbac/__init__.py
+2 -2
View File
@@ -1,3 +1,3 @@
from core.rbac.entities import RBACPermission, RBACResourceScope
from core.rbac.entities import RBACPermission, RBACResourceScope, RBACResourceWhitelistScope
__all__ = ["RBACPermission", "RBACResourceScope"]
__all__ = ["RBACPermission", "RBACResourceScope", "RBACResourceWhitelistScope"]
api/core/rbac/entities.py
+8
View File
@@ -13,6 +13,14 @@ class RBACResourceScope(StrEnum):
WORKSPACE = "workspace"
class RBACResourceWhitelistScope(StrEnum):
"""Whitelist scopes accepted by RBAC app and dataset access config APIs."""
ALL = "all"
SPECIFIC = "specific"
ONLY_ME = "only_me"
class RBACPermission(StrEnum):
"""Permission points (RBAC scenes) checked by ``rbac_permission_required``.
api/core/tools/tool_engine.py
+3 -5
View File
@@ -111,7 +111,7 @@ class ToolEngine:
tool_messages=binary_files, agent_message=message, invoke_from=invoke_from, user_id=user_id
)
plain_text = ToolEngine._convert_tool_response_to_str(message_list)
plain_text = ToolEngine.tool_response_to_str(message_list)
meta = invocation_meta_dict["meta"]
@@ -234,10 +234,8 @@ class ToolEngine:
yield meta
@staticmethod
def _convert_tool_response_to_str(tool_response: list[ToolInvokeMessage]) -> str:
"""
Handle tool response
"""
def tool_response_to_str(tool_response: list[ToolInvokeMessage]) -> str:
"""Convert tool invoke messages into the plain-text observation shown to the model/user."""
parts: list[str] = []
json_parts: list[str] = []
api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.py
+2 -2
View File
@@ -79,7 +79,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool):
all_documents = rerank_runner.run(query, all_documents, self.score_threshold, self.top_k)
for hit_callback in self.hit_callbacks:
hit_callback.on_tool_end(all_documents)
hit_callback.on_tool_end(all_documents, db.session)
document_score_list = {}
for item in all_documents:
@@ -166,7 +166,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool):
return []
for hit_callback in hit_callbacks:
hit_callback.on_query(query, dataset.id)
hit_callback.on_query(query, dataset.id, db.session)
# get retrieval model , if the model is not setting , using default
retrieval_model = dataset.retrieval_model or default_retrieval_model
api/core/tools/utils/dataset_retriever/dataset_retriever_tool.py
+2 -2
View File
@@ -64,7 +64,7 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool):
if not dataset:
return ""
for hit_callback in self.hit_callbacks:
hit_callback.on_query(query, dataset.id)
hit_callback.on_query(query, dataset.id, db.session)
dataset_retrieval = DatasetRetrieval()
metadata_filter_document_ids, metadata_condition = dataset_retrieval.get_metadata_filter_condition(
[dataset.id],
@@ -159,7 +159,7 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool):
else:
documents = []
for hit_callback in self.hit_callbacks:
hit_callback.on_tool_end(documents)
hit_callback.on_tool_end(documents, db.session)
document_score_list = {}
if dataset.indexing_technique != IndexTechniqueType.ECONOMY:
for item in documents:
api/core/workflow/generator/runner.py
+157 -7
View File
@@ -27,6 +27,7 @@ import json
import logging
import re
import time
from collections.abc import Iterator
from typing import Any, ClassVar, cast
import json_repair
@@ -185,6 +186,48 @@ def _result_with_errors(
return base
def _with_mode(result: WorkflowGenerateResultDict, mode: WorkflowGenerationMode) -> WorkflowGenerateResultDict:
"""Stamp the resolved concrete ``mode`` onto a result envelope.
``mode="auto"`` requests are resolved to a concrete mode before planning;
echoing it back lets the frontend pick the right app type to create. It's
present for explicit modes too so the response shape stays uniform.
"""
result["mode"] = mode
return result
def _build_plan_event(
*,
plan: PlannerResultDict,
plan_nodes: list[dict[str, Any]],
start_inputs: list[dict[str, Any]],
mode: WorkflowGenerationMode,
) -> dict[str, Any]:
"""Shape the ``plan`` event emitted before the (slower) builder runs.
Node fields are pulled defensively: the planner schema only guarantees
``node_type`` is present, so ``label`` / ``purpose`` may be missing on a
terse plan and default to empty strings.
"""
return {
"title": str(plan.get("title") or ""),
"description": str(plan.get("description") or ""),
"app_name": str(plan.get("app_name") or "").strip(),
"icon": str(plan.get("icon") or "").strip(),
"mode": mode,
"nodes": [
{
"label": str(node.get("label") or ""),
"node_type": str(node.get("node_type") or ""),
"purpose": str(node.get("purpose") or ""),
}
for node in plan_nodes
],
"start_inputs": start_inputs,
}
def _stage_error_to_envelope_code(exc: Exception) -> str:
"""Map a stage-typed exception to the result envelope's error code."""
if isinstance(exc, _StageJSONError):
@@ -250,6 +293,100 @@ class WorkflowGenerator:
``errors`` and keep the previous version visible.
"""
# Consume the shared event generator and keep only the final result
# envelope — ``generate_workflow_graph_stream`` shares the exact same
# pipeline so the two stay behaviourally identical. The plan event is
# ignored here.
result: WorkflowGenerateResultDict | None = None
for event_name, payload in cls._iter_generation_events(
model_instance=model_instance,
model_parameters=model_parameters,
provider=provider,
model_name=model_name,
model_mode=model_mode,
mode=mode,
instruction=instruction,
ideal_output=ideal_output,
tool_catalogue_text=tool_catalogue_text,
installed_tools=installed_tools,
current_graph=current_graph,
):
if event_name == "result":
result = cast(WorkflowGenerateResultDict, payload)
# The event generator always emits exactly one result envelope; this
# fallback only guards against a future refactor that forgets to.
if result is None:
result = _with_mode(_empty_result(), mode)
return result
@classmethod
def generate_workflow_graph_stream(
cls,
*,
model_instance,
model_parameters: dict[str, Any],
provider: str,
model_name: str,
model_mode: str,
mode: WorkflowGenerationMode,
instruction: str,
ideal_output: str = "",
tool_catalogue_text: str = "",
installed_tools: set[tuple[str, str]] | None = None,
current_graph: dict[str, Any] | None = None,
) -> Iterator[tuple[str, dict[str, Any]]]:
"""
Streaming sibling of ``generate_workflow_graph``.
Yields a ``plan`` event (title / description / app_name / icon / mode /
high-level nodes / start_inputs) as soon as the planner returns, then a
final ``result`` event carrying the SAME envelope dict the non-streaming
method returns (graph / message / app_name / icon / error / errors /
mode, plus structural errors when any). On a planner / empty-plan /
builder failure only the ``result`` event is emitted — no ``plan``.
"""
yield from cls._iter_generation_events(
model_instance=model_instance,
model_parameters=model_parameters,
provider=provider,
model_name=model_name,
model_mode=model_mode,
mode=mode,
instruction=instruction,
ideal_output=ideal_output,
tool_catalogue_text=tool_catalogue_text,
installed_tools=installed_tools,
current_graph=current_graph,
)
@classmethod
def _iter_generation_events(
cls,
*,
model_instance,
model_parameters: dict[str, Any],
provider: str,
model_name: str,
model_mode: str,
mode: WorkflowGenerationMode,
instruction: str,
ideal_output: str = "",
tool_catalogue_text: str = "",
installed_tools: set[tuple[str, str]] | None = None,
current_graph: dict[str, Any] | None = None,
) -> Iterator[tuple[str, dict[str, Any]]]:
"""
Drive planner → builder → postprocess and yield generation events.
Shared core for both ``generate_workflow_graph`` (keeps only the final
``result``) and ``generate_workflow_graph_stream`` (streams every
event). Emits at most one ``plan`` event — only once the planner
produced a non-empty plan — followed by exactly one ``result`` event.
On a planner / empty-plan / builder failure it emits only the
``result`` event carrying the error envelope. Every result envelope is
stamped with the resolved concrete ``mode``.
"""
# ── 1. PLANNER ────────────────────────────────────────────────────
plan, plan_err = cls._run_stage(
stage="Planner",
@@ -265,16 +402,22 @@ class WorkflowGenerator:
),
)
if plan_err is not None:
return _result_with_errors(_empty_result(), [plan_err])
yield "result", cast(dict[str, Any], _with_mode(_result_with_errors(_empty_result(), [plan_err]), mode))
return
# The lambda return is non-None when no error fired — narrow it for type-checkers.
plan = cast(PlannerResultDict, plan)
plan_nodes: list[dict[str, Any]] = cast(list[dict[str, Any]], plan.get("nodes", []))
if not plan_nodes:
return _result_with_errors(
_empty_result(),
[_err(WorkflowGenerateErrorCode.EMPTY_PLAN, "Planner returned no nodes")],
empty_plan = _with_mode(
_result_with_errors(
_empty_result(),
[_err(WorkflowGenerateErrorCode.EMPTY_PLAN, "Planner returned no nodes")],
),
mode,
)
yield "result", cast(dict[str, Any], empty_plan)
return
# Planner-supplied user-input declarations. The builder uses these to
# populate ``start.data.variables`` so downstream ``{#start.<var>#}``
@@ -286,6 +429,10 @@ class WorkflowGenerator:
if isinstance(item, dict) and (item.get("variable") or "").strip()
]
# First event the stream sees: the high-level plan, before the slower
# builder call. Non-streaming callers ignore it.
yield "plan", _build_plan_event(plan=plan, plan_nodes=plan_nodes, start_inputs=start_inputs, mode=mode)
# ── 2. BUILDER ────────────────────────────────────────────────────
graph, build_err = cls._run_stage(
stage="Builder",
@@ -306,7 +453,8 @@ class WorkflowGenerator:
),
)
if build_err is not None:
return _result_with_errors(_empty_result(), [build_err])
yield "result", cast(dict[str, Any], _with_mode(_result_with_errors(_empty_result(), [build_err]), mode))
return
graph = cast(GraphDict, graph)
# ── 3. POSTPROC + VALIDATE ────────────────────────────────────────
@@ -322,6 +470,7 @@ class WorkflowGenerator:
"error": "",
"errors": [],
}
_with_mode(result, mode)
# Final structural sanity check — fail closed if start/end shape is
# wrong, container topology is broken, a tool was hallucinated, or a
@@ -330,8 +479,9 @@ class WorkflowGenerator:
structural_errors = cls._validate_structure(graph=graph, mode=mode, installed_tools=installed_tools)
if structural_errors:
logger.warning("Workflow generator: structural validation failed: %s", structural_errors)
return _result_with_errors(result, structural_errors)
return result
yield "result", cast(dict[str, Any], _result_with_errors(result, structural_errors))
return
yield "result", cast(dict[str, Any], result)
@classmethod
def _run_stage(
api/core/workflow/generator/types.py
+11
View File
@@ -11,6 +11,13 @@ from typing import Final, Literal, NotRequired, TypedDict
WorkflowGenerationMode = Literal["workflow", "advanced-chat"]
# The mode accepted at the API boundary. ``auto`` is a sentinel that asks the
# service to classify the instruction into a concrete ``WorkflowGenerationMode``
# (one tiny LLM call) BEFORE planning — see
# ``WorkflowGeneratorService._resolve_mode`` and
# ``LLMGenerator.classify_workflow_mode``.
WorkflowGenerationModeRequest = Literal["workflow", "advanced-chat", "auto"]
# Machine-readable error codes returned in ``WorkflowGenerateResultDict.errors``.
# Frontend maps these to localised copy via ``workflow.generator.errors.<code>``
@@ -148,3 +155,7 @@ class WorkflowGenerateResultDict(TypedDict):
icon: str
error: str
errors: list[WorkflowGenerateErrorDict]
# Resolved concrete generation mode ("workflow" / "advanced-chat"). Stamped
# onto every envelope so a ``mode="auto"`` request can tell the frontend
# which app type to create; present for explicit modes too for uniformity.
mode: NotRequired[str]
api/core/workflow/nodes/agent_v2/dify_tools_builder.py
+500
View File
@@ -0,0 +1,500 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any, Literal, Protocol, cast
from dify_agent.layers.dify_core_tools import DifyCoreToolConfig, DifyCoreToolProviderType, DifyCoreToolsLayerConfig
from dify_agent.layers.dify_plugin import (
DifyPluginCredentialValue,
DifyPluginToolConfig,
DifyPluginToolCredentialType,
DifyPluginToolParameter,
DifyPluginToolParameterForm,
DifyPluginToolsLayerConfig,
)
from sqlalchemy import select
from sqlalchemy.orm import Session
from core.agent.entities import AgentToolEntity
from core.app.entities.app_invoke_entities import InvokeFrom
from core.tools.__base.tool import Tool
from core.tools.entities.tool_entities import ToolProviderType
from core.tools.errors import ToolProviderCredentialValidationError, ToolProviderNotFoundError
from core.tools.tool_manager import ToolManager
from core.tools.workflow_as_tool.provider import WorkflowToolProviderController
from extensions.ext_database import db
from models.agent_config_entities import AgentSoulDifyToolConfig, AgentSoulToolsConfig
from models.provider_ids import ToolProviderID
from models.tools import WorkflowToolProvider
from services.tools.mcp_tools_manage_service import MCPToolManageService
class WorkflowAgentDifyToolsBuildError(ValueError):
"""Raised when Agent Soul tools cannot be prepared for Agent backend."""
def __init__(self, error_code: str, message: str) -> None:
self.error_code = error_code
super().__init__(message)
class AgentToolRuntimeProvider(Protocol):
def get_agent_tool_runtime(
self,
tenant_id: str,
app_id: str,
agent_tool: AgentToolEntity,
user_id: str | None = None,
invoke_from: InvokeFrom = InvokeFrom.DEBUGGER,
variable_pool: Any | None = None,
allow_file_parameters: bool = False,
use_default_for_missing_form_parameters: bool = False,
) -> Tool: ...
class ProviderToolsLister(Protocol):
def __call__(
self,
*,
tenant_id: str,
provider_type: ToolProviderType,
provider_id: str,
) -> list[str]: ...
class MCPProviderIDResolver(Protocol):
def __call__(self, *, tenant_id: str, provider_id: str) -> str: ...
@dataclass(frozen=True, slots=True)
class WorkflowAgentToolLayers:
plugin_tools: DifyPluginToolsLayerConfig | None = None
core_tools: DifyCoreToolsLayerConfig | None = None
def exposed_tool_names(self) -> list[str]:
names: list[str] = []
if self.plugin_tools is not None:
names.extend(tool.name or tool.tool_name for tool in self.plugin_tools.tools)
if self.core_tools is not None:
names.extend(tool.name or tool.tool_name for tool in self.core_tools.tools)
return names
class WorkflowAgentDifyToolLayersBuilder(Protocol):
def build_layers(
self,
*,
tenant_id: str,
app_id: str,
user_id: str | None,
tools: AgentSoulToolsConfig,
invoke_from: InvokeFrom,
) -> WorkflowAgentToolLayers: ...
def _list_provider_tool_names(
*,
tenant_id: str,
provider_type: ToolProviderType,
provider_id: str,
) -> list[str]:
"""Tool names a provider currently declares for provider-level Agent entries."""
match provider_type:
case ToolProviderType.PLUGIN:
plugin_provider = ToolManager.get_plugin_provider(provider_id, tenant_id)
return [tool.entity.identity.name for tool in plugin_provider.get_tools() or []]
case ToolProviderType.BUILT_IN:
builtin_provider = ToolManager.get_builtin_provider(provider_id, tenant_id)
return [tool.entity.identity.name for tool in builtin_provider.get_tools() or []]
case ToolProviderType.API:
api_provider, _ = ToolManager.get_api_provider_controller(tenant_id, provider_id)
return [tool.entity.identity.name for tool in api_provider.get_tools(tenant_id) or []]
case ToolProviderType.WORKFLOW:
db_provider = db.session.scalar(
select(WorkflowToolProvider)
.where(
WorkflowToolProvider.id == provider_id,
WorkflowToolProvider.tenant_id == tenant_id,
)
.limit(1)
)
if db_provider is None:
raise ToolProviderNotFoundError(f"workflow provider {provider_id} not found")
workflow_provider = WorkflowToolProviderController.from_db(db_provider)
return [tool.entity.identity.name for tool in workflow_provider.get_tools(tenant_id) or []]
case ToolProviderType.MCP:
mcp_provider = ToolManager.get_mcp_provider_controller(tenant_id, provider_id)
return [tool.entity.identity.name for tool in mcp_provider.get_tools() or []]
case _:
raise ToolProviderNotFoundError(f"provider type {provider_type.value} not found")
def _resolve_mcp_provider_id(*, tenant_id: str, provider_id: str) -> str:
"""Normalize MCP provider ids to the runtime-facing server identifier."""
service = MCPToolManageService(session=cast(Session, db.session))
try:
return service.get_provider_entity(provider_id, tenant_id, by_server_id=True).provider_id
except ValueError:
try:
return service.get_provider_entity(provider_id, tenant_id, by_server_id=False).provider_id
except ValueError as exc:
raise ToolProviderNotFoundError(f"mcp provider {provider_id} not found") from exc
class WorkflowAgentDifyToolsBuilder:
"""Prepare Agent Soul Dify tools for Agent backend run-layer configs.
Plugin tools keep their existing direct daemon path. Core-routed tools
(`builtin`/`api`/`workflow`/`mcp`) are emitted as `dify.core.tools`.
"""
def __init__(
self,
*,
tool_runtime_provider: AgentToolRuntimeProvider | None = None,
provider_tools_lister: ProviderToolsLister | None = None,
mcp_provider_id_resolver: MCPProviderIDResolver | None = None,
) -> None:
self._tool_runtime_provider = tool_runtime_provider or ToolManager
self._provider_tools_lister = provider_tools_lister or _list_provider_tool_names
self._mcp_provider_id_resolver = mcp_provider_id_resolver or _resolve_mcp_provider_id
def build_layers(
self,
*,
tenant_id: str,
app_id: str,
user_id: str | None,
tools: AgentSoulToolsConfig,
invoke_from: InvokeFrom,
) -> WorkflowAgentToolLayers:
"""Resolve user-selected Dify tools into direct/core Agent backend DTOs.
`invoke_from` is the real runtime caller category (DEBUGGER for a
Composer test run, SERVICE_API / WEB_APP for a published run). It must
be threaded through to `ToolManager` so credential quotas, rate limits,
and audit tags match the actual call site.
"""
enabled_tools = [tool for tool in tools.dify_tools if tool.enabled]
if not enabled_tools:
return WorkflowAgentToolLayers()
prepared_plugin: list[DifyPluginToolConfig] = []
prepared_core: list[DifyCoreToolConfig] = []
seen_names: set[str] = set()
for tool_config in self._expand_provider_entries(tenant_id=tenant_id, enabled_tools=enabled_tools):
normalized_tool_config = self._normalized_tool_config(tenant_id=tenant_id, tool_config=tool_config)
destination = self._tool_layer_destination(normalized_tool_config)
exposed_name = self._exposed_tool_name(normalized_tool_config)
if exposed_name in seen_names:
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_name_duplicated",
f"Duplicate Dify Tool name {exposed_name!r}.",
)
seen_names.add(exposed_name)
agent_tool = self._to_agent_tool_entity(normalized_tool_config)
tool_runtime = self._fetch_tool_runtime(
tenant_id=tenant_id,
app_id=app_id,
user_id=user_id,
agent_tool=agent_tool,
invoke_from=invoke_from,
tool_config=normalized_tool_config,
)
if destination == "plugin":
prepared_plugin.append(
self._to_plugin_backend_tool_config(normalized_tool_config, tool_runtime, exposed_name)
)
else:
prepared_core.append(
self._to_core_backend_tool_config(normalized_tool_config, tool_runtime, exposed_name)
)
return WorkflowAgentToolLayers(
plugin_tools=DifyPluginToolsLayerConfig(tools=prepared_plugin) if prepared_plugin else None,
core_tools=DifyCoreToolsLayerConfig(tools=prepared_core) if prepared_core else None,
)
def _expand_provider_entries(
self,
*,
tenant_id: str,
enabled_tools: list[AgentSoulDifyToolConfig],
) -> list[AgentSoulDifyToolConfig]:
"""Expand provider-level entries (`tool_name` omitted = all tools)."""
explicit_by_provider: dict[tuple[ToolProviderType, str], set[str]] = {}
for tool_config in enabled_tools:
if tool_config.tool_name is not None:
explicit_by_provider.setdefault(self._provider_key(tool_config), set()).add(tool_config.tool_name)
expanded: list[AgentSoulDifyToolConfig] = []
for tool_config in enabled_tools:
if tool_config.tool_name is not None:
expanded.append(tool_config)
continue
provider_type = ToolProviderType.value_of(tool_config.provider_type)
provider_id = self._provider_id(tool_config)
try:
tool_names = self._provider_declared_tool_names(
tenant_id=tenant_id,
provider_type=provider_type,
provider_id=provider_id,
)
except ToolProviderNotFoundError as exc:
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_declaration_not_found",
f"Dify Tool provider {provider_id!r} declaration not found: {exc}",
) from exc
if not tool_names:
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_declaration_not_found",
f"Dify Tool provider {provider_id!r} declares no tools.",
)
already_explicit = explicit_by_provider.get(self._provider_key(tool_config), set())
for tool_name in tool_names:
if tool_name in already_explicit:
continue
expanded.append(tool_config.model_copy(update={"tool_name": tool_name, "runtime_parameters": {}}))
return expanded
def _provider_declared_tool_names(
self,
*,
tenant_id: str,
provider_type: ToolProviderType,
provider_id: str,
) -> list[str]:
return self._provider_tools_lister(
tenant_id=tenant_id,
provider_type=provider_type,
provider_id=provider_id,
)
def _normalized_tool_config(
self,
*,
tenant_id: str,
tool_config: AgentSoulDifyToolConfig,
) -> AgentSoulDifyToolConfig:
if tool_config.provider_type != ToolProviderType.MCP.value:
return tool_config
provider_id = self._mcp_provider_id_resolver(tenant_id=tenant_id, provider_id=self._provider_id(tool_config))
return tool_config.model_copy(update={"provider_id": provider_id, "plugin_id": None, "provider": None})
def _fetch_tool_runtime(
self,
*,
tenant_id: str,
app_id: str,
user_id: str | None,
agent_tool: AgentToolEntity,
invoke_from: InvokeFrom,
tool_config: AgentSoulDifyToolConfig,
) -> Tool:
"""Resolve the API-side `Tool` runtime and map fetch errors to stable codes."""
try:
return self._tool_runtime_provider.get_agent_tool_runtime(
tenant_id=tenant_id,
app_id=app_id,
agent_tool=agent_tool,
user_id=user_id,
invoke_from=invoke_from,
variable_pool=None,
allow_file_parameters=True,
use_default_for_missing_form_parameters=True,
)
except ToolProviderNotFoundError as exc:
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_declaration_not_found",
f"Dify Tool {tool_config.tool_name!r} declaration not found: {exc}",
) from exc
except ToolProviderCredentialValidationError as exc:
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_credential_invalid",
f"Dify Tool {tool_config.tool_name!r} credential validation failed: {exc}",
) from exc
except ValueError as exc:
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_config_invalid",
f"Dify Tool {tool_config.tool_name!r} runtime construction failed: {exc}",
) from exc
@staticmethod
def _to_agent_tool_entity(tool_config: AgentSoulDifyToolConfig) -> AgentToolEntity:
assert tool_config.tool_name is not None
return AgentToolEntity(
provider_type=ToolProviderType.value_of(tool_config.provider_type),
provider_id=WorkflowAgentDifyToolsBuilder._provider_id(tool_config),
tool_name=tool_config.tool_name,
tool_parameters=dict(tool_config.runtime_parameters),
credential_id=tool_config.credential_ref.id if tool_config.credential_ref else None,
)
@staticmethod
def _provider_id(tool_config: AgentSoulDifyToolConfig) -> str:
if tool_config.provider_id:
return tool_config.provider_id
assert tool_config.plugin_id is not None
assert tool_config.provider is not None
return f"{tool_config.plugin_id}/{tool_config.provider}"
@staticmethod
def _provider_key(tool_config: AgentSoulDifyToolConfig) -> tuple[ToolProviderType, str]:
return (
ToolProviderType.value_of(tool_config.provider_type),
WorkflowAgentDifyToolsBuilder._provider_id(tool_config),
)
@staticmethod
def _tool_layer_destination(tool_config: AgentSoulDifyToolConfig) -> Literal["plugin", "core"]:
provider_type = ToolProviderType.value_of(tool_config.provider_type)
if provider_type is ToolProviderType.PLUGIN:
return "plugin"
if provider_type in {
ToolProviderType.BUILT_IN,
ToolProviderType.API,
ToolProviderType.WORKFLOW,
ToolProviderType.MCP,
}:
return "core"
if provider_type is ToolProviderType.DATASET_RETRIEVAL:
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_provider_not_supported",
"dataset-retrieval remains on the knowledge path and is not supported in Agent tool layers.",
)
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_provider_not_supported",
f"Dify Tool provider type {provider_type.value!r} is not supported in Agent tool layers.",
)
@staticmethod
def _exposed_tool_name(tool_config: AgentSoulDifyToolConfig) -> str:
assert tool_config.tool_name is not None
return tool_config.tool_name
def _to_plugin_backend_tool_config(
self,
tool_config: AgentSoulDifyToolConfig,
tool_runtime: Tool,
exposed_name: str,
) -> DifyPluginToolConfig:
runtime = tool_runtime.runtime
if runtime is None:
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_config_invalid",
f"Dify Tool {tool_config.tool_name!r} has no runtime.",
)
provider_id = self._provider_id(tool_config)
plugin_id, provider = self._plugin_provider(tool_config, provider_id)
parameters = self._prepared_parameters(tool_runtime)
runtime_parameters = self._runtime_parameters(tool_runtime, parameters)
description = self._description(tool_config, tool_runtime)
return DifyPluginToolConfig(
plugin_id=plugin_id,
provider=provider,
tool_name=exposed_name,
credential_type=self._credential_type(tool_config, runtime.credentials),
name=exposed_name,
description=description,
credentials=self._normalize_credentials(runtime.credentials, tool_name=exposed_name),
runtime_parameters=runtime_parameters,
parameters=parameters,
parameters_json_schema=tool_runtime.get_llm_parameters_json_schema(),
)
def _to_core_backend_tool_config(
self,
tool_config: AgentSoulDifyToolConfig,
tool_runtime: Tool,
exposed_name: str,
) -> DifyCoreToolConfig:
parameters = self._prepared_parameters(tool_runtime)
return DifyCoreToolConfig(
provider_type=cast(DifyCoreToolProviderType, tool_config.provider_type),
provider_id=self._provider_id(tool_config),
tool_name=tool_config.tool_name or exposed_name,
credential_id=tool_config.credential_ref.id if tool_config.credential_ref else None,
name=exposed_name,
description=self._description(tool_config, tool_runtime),
runtime_parameters=self._runtime_parameters(tool_runtime, parameters),
parameters=parameters,
parameters_json_schema=tool_runtime.get_llm_parameters_json_schema(),
)
@staticmethod
def _plugin_provider(tool_config: AgentSoulDifyToolConfig, provider_id: str) -> tuple[str, str]:
if tool_config.plugin_id and tool_config.provider:
return tool_config.plugin_id, tool_config.provider
provider_id_entity = ToolProviderID(provider_id)
return provider_id_entity.plugin_id, provider_id_entity.provider_name
@staticmethod
def _credential_type(
tool_config: AgentSoulDifyToolConfig,
credentials: Mapping[str, Any],
) -> DifyPluginToolCredentialType:
if not credentials and tool_config.credential_type == "unauthorized":
return "unauthorized"
return tool_config.credential_type
@staticmethod
def _prepared_parameters(tool_runtime: Tool) -> list[DifyPluginToolParameter]:
return [
DifyPluginToolParameter.model_validate(parameter.model_dump(mode="json"))
for parameter in tool_runtime.get_merged_runtime_parameters()
]
@staticmethod
def _description(tool_config: AgentSoulDifyToolConfig, tool_runtime: Tool) -> str | None:
description = tool_config.description
if description is None and tool_runtime.entity.description is not None:
description = tool_runtime.entity.description.llm
return description
@staticmethod
def _runtime_parameters(
tool_runtime: Tool,
parameters: list[DifyPluginToolParameter],
) -> dict[str, Any]:
runtime = tool_runtime.runtime
runtime_parameters = dict(runtime.runtime_parameters if runtime is not None else {})
missing = [
parameter.name
for parameter in parameters
if parameter.form is not DifyPluginToolParameterForm.LLM
and parameter.required
and parameter.default is None
and parameter.name not in runtime_parameters
]
if missing:
names = ", ".join(sorted(missing))
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_runtime_parameter_missing",
f"Dify Tool {tool_runtime.entity.identity.name!r} is missing runtime parameters: {names}.",
)
return runtime_parameters
@staticmethod
def _normalize_credentials(
credentials: Mapping[str, Any],
*,
tool_name: str,
) -> dict[str, DifyPluginCredentialValue]:
normalized: dict[str, DifyPluginCredentialValue] = {}
for key, value in credentials.items():
if isinstance(value, str | int | float | bool) or value is None:
normalized[key] = value
continue
raise WorkflowAgentDifyToolsBuildError(
"agent_tool_credential_shape_invalid",
(
f"Dify Plugin Tool {tool_name!r} credential {key!r} has a non-scalar value "
f"({type(value).__name__}); only str/int/float/bool/None are forwarded to the daemon."
),
)
return normalized
api/core/workflow/nodes/agent_v2/plugin_tools_builder.py
-334
View File
@@ -1,334 +0,0 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import Any, Protocol
from dify_agent.layers.dify_plugin import (
DifyPluginCredentialValue,
DifyPluginToolConfig,
DifyPluginToolCredentialType,
DifyPluginToolParameter,
DifyPluginToolParameterForm,
DifyPluginToolsLayerConfig,
)
from core.agent.entities import AgentToolEntity
from core.app.entities.app_invoke_entities import InvokeFrom
from core.tools.__base.tool import Tool
from core.tools.entities.tool_entities import ToolProviderType
from core.tools.errors import (
ToolProviderCredentialValidationError,
ToolProviderNotFoundError,
)
from core.tools.tool_manager import ToolManager
from models.agent_config_entities import AgentSoulDifyToolConfig, AgentSoulToolsConfig
from models.provider_ids import ToolProviderID
class WorkflowAgentPluginToolsBuildError(ValueError):
"""Raised when Agent Soul tools cannot be prepared for Agent backend."""
def __init__(self, error_code: str, message: str) -> None:
self.error_code = error_code
super().__init__(message)
class AgentToolRuntimeProvider(Protocol):
def get_agent_tool_runtime(
self,
tenant_id: str,
app_id: str,
agent_tool: AgentToolEntity,
user_id: str | None = None,
invoke_from: InvokeFrom = InvokeFrom.DEBUGGER,
variable_pool: Any | None = None,
allow_file_parameters: bool = False,
use_default_for_missing_form_parameters: bool = False,
) -> Tool: ...
class ProviderToolsLister(Protocol):
def __call__(self, *, tenant_id: str, provider_id: str) -> list[str]: ...
def _list_provider_tool_names(*, tenant_id: str, provider_id: str) -> list[str]:
"""Tool names a provider currently declares (provider-level config entries)."""
provider = ToolManager.get_builtin_provider(provider_id, tenant_id)
return [tool.entity.identity.name for tool in provider.get_tools() or []]
class WorkflowAgentPluginToolsBuilder:
"""Prepare Agent Soul Dify Plugin Tools for the public Agent backend DTO."""
def __init__(
self,
*,
tool_runtime_provider: AgentToolRuntimeProvider | None = None,
provider_tools_lister: ProviderToolsLister | None = None,
) -> None:
self._tool_runtime_provider = tool_runtime_provider or ToolManager
self._provider_tools_lister = provider_tools_lister or _list_provider_tool_names
def build(
self,
*,
tenant_id: str,
app_id: str,
user_id: str | None,
tools: AgentSoulToolsConfig,
invoke_from: InvokeFrom,
) -> DifyPluginToolsLayerConfig | None:
"""Resolve user-selected Dify Plugin Tools into the Agent backend DTO.
``invoke_from`` is the *real* runtime caller category (DEBUGGER for a
Composer test run, SERVICE_API / WEB_APP for a published run). It must
be threaded through to :class:`ToolManager` so credential quotas, rate
limits, and audit tags match the actual call site.
"""
enabled_tools = [tool for tool in tools.dify_tools if tool.enabled]
if not enabled_tools:
return None
prepared: list[DifyPluginToolConfig] = []
seen_names: set[str] = set()
for tool_config in self._expand_provider_entries(tenant_id=tenant_id, enabled_tools=enabled_tools):
agent_tool = self._to_agent_tool_entity(tool_config)
tool_runtime = self._fetch_tool_runtime(
tenant_id=tenant_id,
app_id=app_id,
user_id=user_id,
agent_tool=agent_tool,
invoke_from=invoke_from,
tool_config=tool_config,
)
exposed_name = self._exposed_tool_name(tool_config)
if exposed_name in seen_names:
raise WorkflowAgentPluginToolsBuildError(
"agent_tool_name_duplicated",
f"Duplicate Dify Plugin Tool name {exposed_name!r}.",
)
seen_names.add(exposed_name)
prepared.append(self._to_backend_tool_config(tool_config, tool_runtime, exposed_name))
return DifyPluginToolsLayerConfig(tools=prepared)
def _expand_provider_entries(
self,
*,
tenant_id: str,
enabled_tools: list[AgentSoulDifyToolConfig],
) -> list[AgentSoulDifyToolConfig]:
"""Expand provider-level entries (``tool_name`` omitted = all tools).
An explicit per-tool entry of the same provider wins over the expansion
(it may carry its own ``runtime_parameters``); expanded clones share the
provider entry's ``credential_ref`` and start with default parameters.
"""
explicit_by_provider: dict[str, set[str]] = {}
for tool_config in enabled_tools:
if tool_config.tool_name is not None:
explicit_by_provider.setdefault(self._provider_id(tool_config), set()).add(tool_config.tool_name)
expanded: list[AgentSoulDifyToolConfig] = []
for tool_config in enabled_tools:
if tool_config.tool_name is not None:
expanded.append(tool_config)
continue
provider_id = self._provider_id(tool_config)
try:
tool_names = self._provider_tools_lister(tenant_id=tenant_id, provider_id=provider_id)
except ToolProviderNotFoundError as exc:
raise WorkflowAgentPluginToolsBuildError(
"agent_tool_declaration_not_found",
f"Dify Plugin Tool provider {provider_id!r} declaration not found: {exc}",
) from exc
if not tool_names:
raise WorkflowAgentPluginToolsBuildError(
"agent_tool_declaration_not_found",
f"Dify Plugin Tool provider {provider_id!r} declares no tools.",
)
already_explicit = explicit_by_provider.get(provider_id, set())
for tool_name in tool_names:
if tool_name in already_explicit:
continue
expanded.append(tool_config.model_copy(update={"tool_name": tool_name, "runtime_parameters": {}}))
return expanded
def _fetch_tool_runtime(
self,
*,
tenant_id: str,
app_id: str,
user_id: str | None,
agent_tool: AgentToolEntity,
invoke_from: InvokeFrom,
tool_config: AgentSoulDifyToolConfig,
) -> Tool:
"""Resolve the API-side ``Tool`` runtime, mapping fetch errors to
Inspector-friendly error codes so callers can render distinct UX for
"tool definition gone" vs "credential failed".
"""
try:
return self._tool_runtime_provider.get_agent_tool_runtime(
tenant_id=tenant_id,
app_id=app_id,
agent_tool=agent_tool,
user_id=user_id,
invoke_from=invoke_from,
variable_pool=None,
allow_file_parameters=True,
use_default_for_missing_form_parameters=True,
)
except ToolProviderNotFoundError as exc:
raise WorkflowAgentPluginToolsBuildError(
"agent_tool_declaration_not_found",
f"Dify Plugin Tool {tool_config.tool_name!r} declaration not found: {exc}",
) from exc
except ToolProviderCredentialValidationError as exc:
raise WorkflowAgentPluginToolsBuildError(
"agent_tool_credential_invalid",
f"Dify Plugin Tool {tool_config.tool_name!r} credential validation failed: {exc}",
) from exc
except ValueError as exc:
# ToolManager raises bare ValueError when the agent tool's
# ``runtime`` / runtime parameters are missing. Surface it under a
# narrower error code than a generic "declaration not found" so
# frontend can render an actionable hint.
raise WorkflowAgentPluginToolsBuildError(
"agent_tool_config_invalid",
f"Dify Plugin Tool {tool_config.tool_name!r} runtime construction failed: {exc}",
) from exc
@staticmethod
def _to_agent_tool_entity(tool_config: AgentSoulDifyToolConfig) -> AgentToolEntity:
# Provider-level entries are expanded into per-tool clones before this point.
assert tool_config.tool_name is not None
return AgentToolEntity(
provider_type=ToolProviderType.value_of(tool_config.provider_type),
provider_id=WorkflowAgentPluginToolsBuilder._provider_id(tool_config),
tool_name=tool_config.tool_name,
tool_parameters=dict(tool_config.runtime_parameters),
credential_id=tool_config.credential_ref.id if tool_config.credential_ref else None,
)
@staticmethod
def _provider_id(tool_config: AgentSoulDifyToolConfig) -> str:
if tool_config.provider_id:
return tool_config.provider_id
assert tool_config.plugin_id is not None
assert tool_config.provider is not None
return f"{tool_config.plugin_id}/{tool_config.provider}"
@staticmethod
def _exposed_tool_name(tool_config: AgentSoulDifyToolConfig) -> str:
# Stage 3.1 decision: no user rename yet. Keep the model-visible tool
# name aligned with the plugin declaration identity. Provider-level
# entries are expanded into per-tool clones before this point.
assert tool_config.tool_name is not None
return tool_config.tool_name
def _to_backend_tool_config(
self,
tool_config: AgentSoulDifyToolConfig,
tool_runtime: Tool,
exposed_name: str,
) -> DifyPluginToolConfig:
runtime = tool_runtime.runtime
if runtime is None:
raise WorkflowAgentPluginToolsBuildError(
"agent_tool_config_invalid",
f"Dify Plugin Tool {tool_config.tool_name!r} has no runtime.",
)
provider_id = self._provider_id(tool_config)
plugin_id, provider = self._plugin_provider(tool_config, provider_id)
parameters = [
DifyPluginToolParameter.model_validate(parameter.model_dump(mode="json"))
for parameter in tool_runtime.get_merged_runtime_parameters()
]
runtime_parameters = self._runtime_parameters(tool_runtime, parameters)
description = tool_config.description
if description is None and tool_runtime.entity.description is not None:
description = tool_runtime.entity.description.llm
return DifyPluginToolConfig(
plugin_id=plugin_id,
provider=provider,
tool_name=exposed_name,
credential_type=self._credential_type(tool_config, runtime.credentials),
name=exposed_name,
description=description,
credentials=self._normalize_credentials(runtime.credentials, tool_name=exposed_name),
runtime_parameters=runtime_parameters,
parameters=parameters,
parameters_json_schema=tool_runtime.get_llm_parameters_json_schema(),
)
@staticmethod
def _plugin_provider(tool_config: AgentSoulDifyToolConfig, provider_id: str) -> tuple[str, str]:
if tool_config.plugin_id and tool_config.provider:
return tool_config.plugin_id, tool_config.provider
provider_id_entity = ToolProviderID(provider_id)
return provider_id_entity.plugin_id, provider_id_entity.provider_name
@staticmethod
def _credential_type(
tool_config: AgentSoulDifyToolConfig,
credentials: Mapping[str, Any],
) -> DifyPluginToolCredentialType:
if not credentials and tool_config.credential_type == "unauthorized":
return "unauthorized"
return tool_config.credential_type
@staticmethod
def _runtime_parameters(
tool_runtime: Tool,
parameters: list[DifyPluginToolParameter],
) -> dict[str, Any]:
runtime = tool_runtime.runtime
runtime_parameters = dict(runtime.runtime_parameters if runtime is not None else {})
missing = [
parameter.name
for parameter in parameters
if parameter.form is not DifyPluginToolParameterForm.LLM
and parameter.required
and parameter.default is None
and parameter.name not in runtime_parameters
]
if missing:
names = ", ".join(sorted(missing))
raise WorkflowAgentPluginToolsBuildError(
"agent_tool_runtime_parameter_missing",
f"Dify Plugin Tool {tool_runtime.entity.identity.name!r} is missing runtime parameters: {names}.",
)
return runtime_parameters
@staticmethod
def _normalize_credentials(
credentials: Mapping[str, Any],
*,
tool_name: str,
) -> dict[str, DifyPluginCredentialValue]:
"""Forward only scalar credential values to the Agent backend.
``DifyPluginCredentialValue`` is ``str | int | float | bool | None``.
Refusing non-scalar values (lists, dicts, custom objects) is safer than
``str(value)`` — stringifying a nested OAuth token blob produces a
Python ``repr`` that the plugin daemon cannot use, and we'd rather
surface a clear ``agent_tool_credential_shape_invalid`` than send junk.
"""
normalized: dict[str, DifyPluginCredentialValue] = {}
for key, value in credentials.items():
if isinstance(value, str | int | float | bool) or value is None:
normalized[key] = value
continue
raise WorkflowAgentPluginToolsBuildError(
"agent_tool_credential_shape_invalid",
(
f"Dify Plugin Tool {tool_name!r} credential {key!r} has a non-scalar value "
f"({type(value).__name__}); only str/int/float/bool/None are forwarded to the daemon."
),
)
return normalized
api/core/workflow/nodes/agent_v2/runtime_request_builder.py
+222 -137
View File
@@ -8,9 +8,11 @@ from typing import Any, Literal, Protocol, assert_never, cast
from agenton.compositor import CompositorSessionSnapshot
from dify_agent.agent_stub.protocol import AgentStubFileMapping
from dify_agent.layers.ask_human import DifyAskHumanLayerConfig
from dify_agent.layers.drive import (
DifyDriveLayerConfig,
DifyDriveSkillConfig,
from dify_agent.layers.config import (
DifyConfigFileConfig,
DifyConfigLayerConfig,
DifyConfigSkillConfig,
DifyConfigVersionConfig,
)
from dify_agent.layers.execution_context import (
DifyExecutionContextInvokeFrom,
@@ -55,6 +57,7 @@ from models.agent_config_entities import (
AgentKnowledgeModelConfig,
AgentKnowledgeRetrievalConfig,
AgentSoulConfig,
AgentSoulToolsConfig,
DeclaredArrayItem,
DeclaredOutputChildConfig,
DeclaredOutputConfig,
@@ -76,10 +79,14 @@ from services.agent.prompt_mentions import (
parse_prompt_mentions,
workflow_previous_node_output_refs_from_selectors,
)
from services.agent_drive_service import AgentDriveService, decode_drive_mention_ref
from .dify_tools_builder import (
WorkflowAgentDifyToolLayersBuilder,
WorkflowAgentDifyToolsBuilder,
WorkflowAgentDifyToolsBuildError,
WorkflowAgentToolLayers,
)
from .output_failure_orchestrator import retry_idempotency_key
from .plugin_tools_builder import WorkflowAgentPluginToolsBuilder, WorkflowAgentPluginToolsBuildError
from .runtime_feature_manifest import build_runtime_feature_manifest
_DENIED_PERMISSION_STATUSES = frozenset({"unauthorized", "denied", "forbidden", "invalid", "unavailable"})
@@ -97,6 +104,7 @@ _AGENT_STUB_FILE_TRANSFER_METHODS: Mapping[FileTransferMethod, AgentStubFileTran
FileTransferMethod.DATASOURCE_FILE: "datasource_file",
FileTransferMethod.REMOTE_URL: "remote_url",
}
_CANONICAL_DIFY_FILE_REFERENCE_PATTERN = r"^dify-file-ref:eyJyZWNvcmRfaWQiOi[A-Za-z0-9_-]+={0,2}$"
class WorkflowAgentRuntimeRequestBuildError(ValueError):
@@ -157,11 +165,11 @@ class WorkflowAgentRuntimeRequestBuilder:
*,
credentials_provider: CredentialsProvider,
request_builder: AgentBackendRunRequestBuilder | None = None,
plugin_tools_builder: WorkflowAgentPluginToolsBuilder | None = None,
dify_tools_builder: WorkflowAgentDifyToolLayersBuilder | None = None,
) -> None:
self._credentials_provider = credentials_provider
self._request_builder = request_builder or AgentBackendRunRequestBuilder()
self._plugin_tools_builder = plugin_tools_builder or WorkflowAgentPluginToolsBuilder()
self._dify_tools_builder = dify_tools_builder or WorkflowAgentDifyToolsBuilder()
def build(self, context: WorkflowAgentRuntimeBuildContext) -> WorkflowAgentRuntimeRequest:
agent_soul = AgentSoulConfig.model_validate(context.snapshot.config_snapshot_dict)
@@ -182,42 +190,33 @@ class WorkflowAgentRuntimeRequestBuilder:
user_prompt = workflow_context_prompt or self._WORKFLOW_USER_PROMPT_FALLBACK
credentials = self._credentials_provider.fetch(agent_soul.model.model_provider, agent_soul.model.model)
try:
tools_layer = self._plugin_tools_builder.build(
tool_layers = self._build_tool_layers(
tenant_id=context.dify_context.tenant_id,
app_id=context.dify_context.app_id,
user_id=context.dify_context.user_id,
tools=agent_soul.tools,
# Thread the *real* runtime invocation source through to
# ToolManager so credential quotas, rate limits, and audit
# trails match the actual call site (DEBUGGER for draft test
# run, SERVICE_API / WEB_APP for published run).
invoke_from=context.dify_context.invoke_from,
)
except WorkflowAgentPluginToolsBuildError as error:
except WorkflowAgentDifyToolsBuildError as error:
raise WorkflowAgentRuntimeRequestBuildError(error.error_code, str(error)) from error
if tools_layer is not None or agent_soul.tools.cli_tools:
if tool_layers.plugin_tools is not None or tool_layers.core_tools is not None or agent_soul.tools.cli_tools:
metadata["agent_tools"] = {
"dify_tool_count": len(tools_layer.tools) if tools_layer is not None else 0,
"dify_tool_names": [tool.name or tool.tool_name for tool in tools_layer.tools]
if tools_layer is not None
else [],
"dify_tool_count": len(tool_layers.exposed_tool_names()),
"dify_tool_names": tool_layers.exposed_tool_names(),
"cli_tool_count": len(agent_soul.tools.cli_tools),
}
drive_config: DifyDriveLayerConfig | None = None
config_layer_config: DifyConfigLayerConfig | None = None
soul_prompt_resolver = build_soul_mention_resolver(agent_soul)
if dify_config.AGENT_DRIVE_MANIFEST_ENABLED:
drive_config, drive_warnings = build_drive_layer_config(
config_layer_config, config_warnings = build_config_layer_config(
agent_soul,
tenant_id=context.dify_context.tenant_id,
agent_id=context.agent.id,
)
append_runtime_warnings(metadata, drive_warnings)
soul_prompt_resolver = build_drive_aware_soul_mention_resolver(
agent_soul,
tenant_id=context.dify_context.tenant_id,
agent_id=context.agent.id,
config_version_id=context.snapshot.id,
config_version_kind="snapshot",
)
append_runtime_warnings(metadata, config_warnings)
soul_prompt_resolver = build_config_aware_soul_mention_resolver(agent_soul)
soul_prompt = expand_prompt_mentions(agent_soul.prompt.system_prompt, soul_prompt_resolver).strip()
knowledge_config = build_knowledge_layer_config(agent_soul)
@@ -251,6 +250,7 @@ class WorkflowAgentRuntimeRequestBuilder:
conversation_id=get_system_text(context.variable_pool, SystemVariableKey.CONVERSATION_ID),
agent_id=context.agent.id,
agent_config_version_id=context.snapshot.id,
agent_config_version_kind="snapshot",
agent_mode=self._agent_backend_agent_mode(context.dify_context.invoke_from),
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
),
@@ -258,9 +258,10 @@ class WorkflowAgentRuntimeRequestBuilder:
workflow_node_job_prompt=workflow_job_prompt,
user_prompt=user_prompt,
output=self._build_output_config(node_job.declared_outputs),
tools=tools_layer,
tools=tool_layers.plugin_tools,
core_tools=tool_layers.core_tools,
knowledge=knowledge_config,
drive_config=drive_config,
config_layer_config=config_layer_config,
ask_human_config=build_ask_human_layer_config(agent_soul),
include_shell=dify_config.AGENT_SHELL_ENABLED,
shell_config=build_shell_layer_config(agent_soul),
@@ -279,6 +280,26 @@ class WorkflowAgentRuntimeRequestBuilder:
metadata=metadata,
)
def _build_tool_layers(
self,
*,
tenant_id: str,
app_id: str,
user_id: str | None,
tools: AgentSoulToolsConfig,
invoke_from: InvokeFrom,
) -> WorkflowAgentToolLayers:
# Production workflow runs intentionally keep existing plugin configs on
# the direct `dify.plugin.tools` route. This builder emits plugin tools
# directly and non-plugin Dify tools through `dify.core.tools`.
return self._dify_tools_builder.build_layers(
tenant_id=tenant_id,
app_id=app_id,
user_id=user_id,
tools=tools,
invoke_from=invoke_from,
)
@staticmethod
def _agent_backend_agent_mode(invoke_from: InvokeFrom) -> Literal["workflow_run", "single_step"]:
if invoke_from in {InvokeFrom.DEBUGGER, InvokeFrom.VALIDATION}:
@@ -494,7 +515,10 @@ class WorkflowAgentRuntimeRequestBuilder:
schema: dict[str, Any] = {"type": "object", "properties": properties}
if required:
schema["required"] = required
return AgentBackendOutputConfig(json_schema=schema)
return AgentBackendOutputConfig(
json_schema=schema,
description=WorkflowAgentRuntimeRequestBuilder._build_output_description(effective_outputs),
)
@staticmethod
def effective_declared_outputs(
@@ -551,48 +575,107 @@ class WorkflowAgentRuntimeRequestBuilder:
array_schema["items"]["description"] = array_item.description
return array_schema
case DeclaredOutputType.FILE:
return {
"oneOf": [
{
"type": "object",
"additionalProperties": False,
"properties": {
"transfer_method": {"const": FileTransferMethod.LOCAL_FILE.value},
"reference": {"type": "string"},
},
"required": ["transfer_method", "reference"],
},
{
"type": "object",
"additionalProperties": False,
"properties": {
"transfer_method": {"const": FileTransferMethod.TOOL_FILE.value},
"reference": {"type": "string"},
},
"required": ["transfer_method", "reference"],
},
{
"type": "object",
"additionalProperties": False,
"properties": {
"transfer_method": {"const": FileTransferMethod.DATASOURCE_FILE.value},
"reference": {"type": "string"},
},
"required": ["transfer_method", "reference"],
},
{
"type": "object",
"additionalProperties": False,
"properties": {
"transfer_method": {"const": FileTransferMethod.REMOTE_URL.value},
"url": {"type": "string"},
},
"required": ["transfer_method", "url"],
},
],
}
return WorkflowAgentRuntimeRequestBuilder._agent_stub_output_file_mapping_schema()
assert_never(output_type)
@staticmethod
def _agent_stub_output_file_mapping_schema() -> dict[str, Any]:
"""JSON Schema for Agent-produced output file mappings.
``AgentStubFileMapping.model_json_schema()`` cannot express the model's
``after`` validator: only ``remote_url`` may carry ``url``; every other
method must carry a canonical ``reference``. The structured-output model
needs that relationship in the schema, otherwise it may emit
``{"transfer_method": "local_file", "url": "..."}``, which passes the
broad generated schema but fails API-side output type checking.
For files produced inside an Agent run, the supported persisted shape is
narrower than every downloadable mapping: the sandbox must upload the
local artifact via ``dify-agent file upload <path>``, which returns a
``tool_file`` mapping. ``local_file`` and ``datasource_file`` are valid
for existing file references in workflow context, not for newly produced
Agent output files.
"""
return {
"title": "AgentStubFileMapping",
"description": (
"Agent output file mapping. Use `tool_file` with `reference` for files uploaded by "
"`dify-agent file upload <path>`; use `remote_url` only for files already reachable by URL."
),
"anyOf": [
{
"type": "object",
"additionalProperties": False,
"required": ["transfer_method", "reference"],
"properties": {
"transfer_method": {
"type": "string",
"enum": ["tool_file"],
},
"reference": {
"type": "string",
"minLength": 1,
"pattern": _CANONICAL_DIFY_FILE_REFERENCE_PATTERN,
"description": (
"Canonical Dify file reference returned by `dify-agent file upload <path>`. "
"Never use a local path, filename, URL, or synthesized dify-file-ref here."
),
},
},
},
{
"type": "object",
"additionalProperties": False,
"required": ["transfer_method", "url"],
"properties": {
"transfer_method": {
"type": "string",
"enum": ["remote_url"],
},
"url": {
"type": "string",
"minLength": 1,
"description": "Remote URL for a file that is already publicly reachable.",
},
},
},
],
}
@staticmethod
def _build_output_description(declared_outputs: Sequence[DeclaredOutputConfig]) -> str | None:
file_output_lines: list[str] = []
for output in declared_outputs:
if output.type == DeclaredOutputType.FILE:
file_output_lines.append(
f"- `{output.name}`: create the file in the sandbox, run `dify-agent file upload <path>`, "
f"and set `final_output.{output.name}` to the returned AgentStubFileMapping JSON object. "
"Do not call `final_output` before the upload command succeeds. Do not use the local path, "
"filename, URL, or a synthesized/base64-encoded value as the `reference`."
)
elif (
output.type == DeclaredOutputType.ARRAY
and output.array_item is not None
and output.array_item.type == DeclaredOutputType.FILE
):
file_output_lines.append(
f"- `{output.name}`: for every produced file, run `dify-agent file upload <path>` and set "
f"`final_output.{output.name}` to an array of the returned AgentStubFileMapping JSON objects. "
"Do not call `final_output` before all upload commands succeed. Do not use local paths, filenames, "
"URLs, or synthesized/base64-encoded values as `reference` values."
)
if not file_output_lines:
return None
return "\n".join(
[
"When filling file outputs, do not return a local filesystem path directly.",
"Upload each sandbox-local file through the Agent Stub CLI first. Copy the JSON printed by "
"`dify-agent file upload <path>` verbatim into the final output; never invent the `reference` value.",
*file_output_lines,
]
)
@staticmethod
def _apply_child_properties(schema: dict[str, Any], children: Sequence[DeclaredOutputChildConfig]) -> None:
if not children:
@@ -753,19 +836,12 @@ def append_runtime_warnings(metadata: dict[str, Any], warnings: list[dict[str, s
existing.extend(warnings)
def build_drive_aware_soul_mention_resolver(
agent_soul: AgentSoulConfig,
*,
tenant_id: str,
agent_id: str,
):
"""Resolve skill/file mentions against the agent drive and everything else via Agent Soul."""
def build_config_aware_soul_mention_resolver(agent_soul: AgentSoulConfig):
"""Resolve config skill/file mentions and delegate the rest to Agent Soul."""
base_resolver = build_soul_mention_resolver(agent_soul)
drive_service = AgentDriveService()
skill_catalog = drive_service.list_skills(tenant_id=tenant_id, agent_id=agent_id)
skill_names_by_key = {skill["skill_md_key"]: skill["name"] for skill in skill_catalog}
drive_keys = {item["key"] for item in drive_service.manifest(tenant_id=tenant_id, agent_id=agent_id)}
skill_names = {item.name for item in agent_soul.config_skills}
file_names = {item.name for item in agent_soul.config_files}
def _resolve(mention: object) -> str | None:
if not hasattr(mention, "kind") or not hasattr(mention, "ref_id"):
@@ -774,88 +850,97 @@ def build_drive_aware_soul_mention_resolver(
ref_id = cast(str, mention.ref_id)
label = cast(str | None, getattr(mention, "label", None))
if kind == MentionKind.SKILL:
decoded_key = decode_drive_mention_ref(ref_id)
return skill_names_by_key.get(decoded_key) or label or decoded_key
return ref_id if ref_id in skill_names else label or ref_id
if kind == MentionKind.FILE:
decoded_key = decode_drive_mention_ref(ref_id)
if decoded_key in drive_keys:
return decoded_key.rsplit("/", 1)[-1]
return label or decoded_key
return ref_id if ref_id in file_names else label or ref_id
return base_resolver(cast(Any, mention))
return _resolve
def build_drive_layer_config(
def build_config_layer_config(
agent_soul: AgentSoulConfig,
*,
tenant_id: str,
agent_id: str | None,
) -> tuple[DifyDriveLayerConfig | None, list[dict[str, str]]]:
"""Derive drive runtime catalog + prompt-mentioned eager-pull keys from the drive."""
agent_id: str | None = None,
config_version_id: str | None = None,
config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot",
) -> tuple[DifyConfigLayerConfig | None, list[dict[str, str]]]:
"""Derive prompt-mentioned eager-pull names from Agent Soul."""
mentioned_drive_refs = [
decode_drive_mention_ref(mention.ref_id)
for mention in parse_prompt_mentions(agent_soul.prompt.system_prompt)
if mention.kind in {MentionKind.SKILL, MentionKind.FILE}
]
ordered_mentions = list(dict.fromkeys(ref for ref in mentioned_drive_refs if ref))
if not agent_id:
if not ordered_mentions:
return None, []
return None, [
{
"section": "agent_soul.prompt.system_prompt",
"code": "drive_ref_dangling",
"message": "drive mentions are configured but the run has no bound agent to address a drive by.",
}
]
ordered_mentions = list(
dict.fromkeys(
mention.ref_id
for mention in parse_prompt_mentions(agent_soul.prompt.system_prompt)
if mention.kind in {MentionKind.SKILL, MentionKind.FILE} and mention.ref_id
)
)
if (
not agent_soul.config_skills
and not agent_soul.config_files
and not agent_soul.config_note
and not ordered_mentions
):
return None, []
drive_service = AgentDriveService()
skills_catalog = drive_service.list_skills(tenant_id=tenant_id, agent_id=agent_id)
manifest_items = drive_service.manifest(tenant_id=tenant_id, agent_id=agent_id)
manifest_by_key = {item["key"]: item for item in manifest_items}
skill_keys = {skill["skill_md_key"] for skill in skills_catalog}
skill_names = {skill.name for skill in agent_soul.config_skills}
file_names = {file_ref.name for file_ref in agent_soul.config_files}
warnings: list[dict[str, str]] = []
mentioned_skill_keys: list[str] = []
mentioned_file_keys: list[str] = []
for drive_key in ordered_mentions:
if drive_key in skill_keys:
mentioned_skill_keys.append(drive_key)
mentioned_skill_names: list[str] = []
mentioned_file_names: list[str] = []
for name in ordered_mentions:
if name in skill_names:
mentioned_skill_names.append(name)
continue
if drive_key in manifest_by_key:
mentioned_file_keys.append(drive_key)
if name in file_names:
mentioned_file_names.append(name)
continue
warnings.append(
{
"section": "agent_soul.prompt.system_prompt",
"code": "mention_target_missing",
"message": f"drive mention '{drive_key}' has no matching drive entry.",
"message": f"config mention '{name}' has no matching config asset.",
}
)
skills = [
DifyDriveSkillConfig(
path=skill["path"],
name=skill["name"],
description=skill["description"],
skill_md_key=skill["skill_md_key"],
archive_key=skill["archive_key"],
)
for skill in skills_catalog
]
return (
DifyDriveLayerConfig(
drive_ref=f"agent-{agent_id}",
skills=skills,
mentioned_skill_keys=mentioned_skill_keys,
mentioned_file_keys=mentioned_file_keys,
DifyConfigLayerConfig(
agent_id=agent_id,
config_version=DifyConfigVersionConfig(
id=config_version_id,
kind=config_version_kind,
writable=config_version_kind == "build_draft",
),
skills=[
DifyConfigSkillConfig(
name=skill.name,
description=skill.description,
size=skill.size,
mime_type=skill.mime_type,
)
for skill in agent_soul.config_skills
],
files=[
DifyConfigFileConfig(
name=file_ref.name,
size=file_ref.size,
mime_type=file_ref.mime_type,
)
for file_ref in agent_soul.config_files
],
env_keys=_agent_soul_config_env_keys(agent_soul),
note=agent_soul.config_note,
mentioned_skill_names=mentioned_skill_names,
mentioned_file_names=mentioned_file_names,
),
warnings,
)
def _agent_soul_config_env_keys(agent_soul: AgentSoulConfig) -> list[str]:
keys = [item.key or item.name or item.env_name or item.variable for item in agent_soul.env.variables]
return [key for key in keys if key]
def _cli_tool_enabled(item: object) -> bool:
"""A CLI tool is bootstrapped unless explicitly disabled (default is enabled)."""
data = _plain_mapping(item)
api/dev/lint_response_contracts.py
+7 -1
View File
@@ -595,9 +595,15 @@ def line_has_ignore_marker(line: str) -> bool:
return any(ignore_marker in normalized for ignore_marker in IGNORE_COMMENT_MARKERS)
def node_ignore_scan_end_lineno(node: ast.ClassDef | MethodNode) -> int:
if isinstance(node, ast.ClassDef):
return node.lineno
return node.end_lineno or node.lineno
def node_has_ignore_comment(lines: Sequence[str], node: ast.ClassDef | MethodNode) -> bool:
start = node_start_lineno(node)
end = node.end_lineno or node.lineno
end = node_ignore_scan_end_lineno(node)
if any(line_has_ignore_marker(line) for line in lines[start - 1 : end]):
return True
api/events/event_handlers/create_document_index.py
+9 -9
View File
@@ -1,4 +1,3 @@
import contextlib
import logging
import time
@@ -42,11 +41,12 @@ def handle(sender, **kwargs):
session.add(document)
session.commit()
with contextlib.suppress(Exception):
try:
indexing_runner = IndexingRunner()
indexing_runner.run(documents)
end_at = time.perf_counter()
logger.info(click.style(f"Processed dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))
except DocumentIsPausedError as ex:
logger.info(click.style(str(ex), fg="yellow"))
try:
indexing_runner = IndexingRunner()
indexing_runner.run(documents)
end_at = time.perf_counter()
logger.info(click.style(f"Processed dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))
except DocumentIsPausedError as ex:
logger.info(click.style(str(ex), fg="yellow"))
except Exception:
logger.exception("Document index event handler failed, dataset_id: %s", dataset_id)
api/extensions/ext_commands.py
+2
View File
@@ -27,6 +27,7 @@ def init_app(app: DifyApp):
install_plugins,
install_rag_pipeline_plugins,
migrate_data_for_plugin,
migrate_dataset_permissions_to_rbac,
migrate_member_roles_to_rbac,
migrate_oss,
migration_data_wizard,
@@ -56,6 +57,7 @@ def init_app(app: DifyApp):
upgrade_db,
fix_app_site_missing,
migrate_data_for_plugin,
migrate_dataset_permissions_to_rbac,
migrate_member_roles_to_rbac,
backfill_plugin_auto_upgrade,
extract_plugins,
api/fields/agent_fields.py
+3
View File
@@ -371,6 +371,9 @@ class WorkflowAgentComposerResponse(ResponseModel):
backing_app_id: str | None = None
hidden_app_backed: bool = False
chat_endpoint: str | None = None
debug_conversation_id: str | None = None
debug_conversation_has_messages: bool = False
debug_conversation_message_count: int = 0
workflow_id: str | None = None
node_id: str | None = None
api/fields/annotation_fields.py
+10
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from pydantic import Field, field_validator
@@ -29,6 +30,15 @@ class AnnotationList(ResponseModel):
page: int
class AnnotationJobStatusResponse(ResponseModel):
job_id: str
job_status: Literal["waiting", "processing", "completed", "error"] | str
class AnnotationJobStatusDetailResponse(AnnotationJobStatusResponse):
error_msg: str = ""
class AnnotationExportList(ResponseModel):
data: list[Annotation]
api/fields/conversation_variable_fields.py
+1 -12
View File
@@ -3,25 +3,14 @@ from __future__ import annotations
from datetime import datetime
from typing import Any
from flask_restx import fields
from pydantic import field_validator
from fields.base import ResponseModel
from graphon.variables.types import SegmentType
from libs.helper import TimestampField, to_timestamp
from libs.helper import to_timestamp
from ._value_type_serializer import serialize_value_type
conversation_variable_fields = {
"id": fields.String,
"name": fields.String,
"value_type": fields.String(attribute=serialize_value_type),
"value": fields.String,
"description": fields.String,
"created_at": TimestampField,
"updated_at": TimestampField,
}
class ConversationVariableResponse(ResponseModel):
id: str
api/models/agent_config_entities.py
+72 -8
View File
@@ -109,6 +109,7 @@ class OutputErrorStrategy(StrEnum):
# JSON-schema-friendly name pattern. Stage 4 §3.1 / §10.1.
_OUTPUT_NAME_PATTERN: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
_CONFIG_SKILL_NAME_PATTERN: Final[re.Pattern[str]] = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
JsonPrimitive = str | int | float | bool | None
RuntimeParameterValue = JsonPrimitive | list[str] | list[int] | list[float] | list[bool]
@@ -167,6 +168,63 @@ class AgentSoulFilesConfig(BaseModel):
files: list[AgentFileRefConfig] = Field(default_factory=list)
def validate_config_name(name: str) -> str:
normalized = name.strip()
if not normalized:
raise ValueError("config asset name must not be blank")
if normalized in {".", ".."}:
raise ValueError("config asset name must not be '.' or '..'")
if "/" in normalized or "\\" in normalized:
raise ValueError("config asset name must be a single path segment")
if "\x00" in normalized or any(ord(ch) < 0x20 for ch in normalized):
raise ValueError("config asset name must not contain control characters")
return normalized
def validate_config_skill_name(name: str) -> str:
normalized = validate_config_name(name)
if _CONFIG_SKILL_NAME_PATTERN.fullmatch(normalized) is None:
raise ValueError(f"config skill name {normalized!r} must match {_CONFIG_SKILL_NAME_PATTERN.pattern}")
return normalized
class AgentConfigFileRefConfig(BaseModel):
"""Stable Agent Soul reference to one config file payload."""
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=255)
file_kind: Literal["upload_file", "tool_file"]
file_id: str = Field(min_length=1, max_length=255)
size: int | None = None
hash: str | None = None
mime_type: str | None = None
@field_validator("name")
@classmethod
def _validate_name(cls, value: str) -> str:
return validate_config_name(value)
class AgentConfigSkillRefConfig(BaseModel):
"""Stable Agent Soul reference to one normalized skill archive."""
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=255)
description: str = ""
file_kind: Literal["tool_file"] = "tool_file"
file_id: str = Field(min_length=1, max_length=255)
size: int | None = None
hash: str | None = None
mime_type: str | None = "application/zip"
@field_validator("name")
@classmethod
def _validate_name(cls, value: str) -> str:
return validate_config_skill_name(value)
class AgentPermissionConfig(BaseModel):
model_config = ConfigDict(extra="ignore")
@@ -527,12 +585,15 @@ class AgentSoulDifyToolCredentialRef(BaseModel):
class AgentSoulDifyToolConfig(BaseModel):
"""One Dify Plugin Tool configured on Agent Soul.
"""One Dify tool configured on Agent Soul.
The API backend prepares this persisted product shape into
``DifyPluginToolConfig`` before sending a run request to Agent backend.
``provider_id`` keeps compatibility with existing Agent tool config payloads;
new callers should send ``plugin_id`` + ``provider`` when available.
either ``DifyPluginToolConfig`` or ``DifyCoreToolConfig`` before sending a
run request to Agent backend. ``plugin`` providers keep the direct
``dify.plugin.tools`` transport; ``builtin`` / ``api`` / ``workflow`` /
``mcp`` providers are prepared for ``dify.core.tools``. ``provider_id``
keeps compatibility with existing Agent tool config payloads; new callers
should send ``plugin_id`` + ``provider`` when available.
"""
# ``extra="ignore"`` (not ``"allow"``) so historical Agent Soul payloads
@@ -542,10 +603,10 @@ class AgentSoulDifyToolConfig(BaseModel):
model_config = ConfigDict(extra="ignore")
enabled: bool = True
# Dify Plugin Tools live behind the ``PLUGIN`` provider type. ``BUILT_IN`` /
# ``WORKFLOW`` / ``API`` providers are not exposed to the Agent backend in
# this layer — keep the default narrow so a missing field surfaces as
# ``agent_tool_declaration_not_found`` against the correct provider table.
# ``plugin`` remains the default for legacy Agent Soul payloads. The runtime
# now also accepts ``builtin`` / ``api`` / ``workflow`` / ``mcp`` here and
# routes them through ``dify.core.tools``; keeping the default narrow still
# makes a missing field resolve against the plugin provider table.
provider_type: str = "plugin"
provider_id: str | None = Field(default=None, max_length=255)
plugin_id: str | None = Field(default=None, max_length=255)
@@ -682,6 +743,9 @@ class AgentSoulConfig(BaseModel):
knowledge: AgentSoulKnowledgeConfig = Field(default_factory=AgentSoulKnowledgeConfig)
human: AgentSoulHumanConfig = Field(default_factory=AgentSoulHumanConfig)
env: AgentSoulEnvConfig = Field(default_factory=AgentSoulEnvConfig)
config_skills: list[AgentConfigSkillRefConfig] = Field(default_factory=list)
config_files: list[AgentConfigFileRefConfig] = Field(default_factory=list)
config_note: str = ""
files: AgentSoulFilesConfig = Field(default_factory=AgentSoulFilesConfig)
sandbox: AgentSoulSandboxConfig = Field(default_factory=AgentSoulSandboxConfig)
memory: AgentSoulMemoryConfig = Field(default_factory=AgentSoulMemoryConfig)
api/models/dataset.py
+2 -2
View File
@@ -14,7 +14,7 @@ from uuid import uuid4
import sqlalchemy as sa
from sqlalchemy import DateTime, String, func, select
from sqlalchemy.orm import Mapped, Session, mapped_column
from sqlalchemy.orm import Mapped, Session, mapped_column, scoped_session
from configs import dify_config
from core.rag.entities import ParentMode, Rule
@@ -1670,7 +1670,7 @@ class Pipeline(TypeBase):
init=False,
)
def retrieve_dataset(self, session: Session):
def retrieve_dataset(self, session: Session | scoped_session):
return session.scalar(select(Dataset).where(Dataset.pipeline_id == self.id))
api/models/enums.py
+12
View File
@@ -214,6 +214,18 @@ class EndUserType(StrEnum):
SERVICE_API = "service-api"
TRIGGER = "trigger"
@classmethod
@override
def _missing_(cls, value):
# Legacy rows persisted the service-api type with an underscore before it
# was normalized to the hyphenated value. The
# `4f7b2c8d9a10_normalize_legacy_end_user_type` migration rewrites those
# rows, but tolerate the old value here as well so an unmigrated end user
# keeps loading instead of failing enum validation on every request.
if value == "service_api":
return cls.SERVICE_API
return super()._missing_(value)
class DocumentDocType(StrEnum):
"""Document doc_type classification"""
api/openapi/markdown/console-openapi.md
+1659 -163
View File
File diff suppressed because it is too large Load Diff
api/providers/vdb/vdb-myscale/src/dify_vdb_myscale/myscale_vector.py
+10 -3
View File
@@ -132,9 +132,16 @@ class MyScaleVector(BaseVector):
@override
def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
return self._search(f"TextSearch('enable_nlq=false')(text, '{query}')", SortOrder.DESC, **kwargs)
return self._search(
"TextSearch('enable_nlq=false')(text, {query:String})",
SortOrder.DESC,
parameters={"query": query},
**kwargs,
)
def _search(self, dist: str, order: SortOrder, **kwargs: Any) -> list[Document]:
def _search(
self, dist: str, order: SortOrder, parameters: dict[str, Any] | None = None, **kwargs: Any
) -> list[Document]:
top_k = kwargs.get("top_k", 4)
if not isinstance(top_k, int) or top_k <= 0:
raise ValueError("top_k must be a positive integer")
@@ -159,7 +166,7 @@ class MyScaleVector(BaseVector):
vector=r["vector"],
metadata=r["metadata"],
)
for r in self._client.query(sql).named_results()
for r in self._client.query(sql, parameters=parameters).named_results()
]
except Exception:
logger.exception("Vector search operation failed")

Some files were not shown because too many files have changed in this diff Show More