diff --git a/docs/mcp-catalog-evaluation.md b/docs/mcp-catalog-evaluation.md index eff14206d8..1e2713a8f1 100644 --- a/docs/mcp-catalog-evaluation.md +++ b/docs/mcp-catalog-evaluation.md @@ -39,6 +39,12 @@ and a 192-task ceiling. Runs are sequential. Unsupported clients, resource stops failed startup, missing usage, and unverified cleanup remain failed measurements. They do not become zero-cost successes. +Trials disable account apps and plugins explicitly. A fresh `CODEX_HOME` alone +does not prevent account-connected tools from appearing in the client executor +catalog. For a deliberate mixed-catalog investigation, run `agent-trial.py` with +`--include-account-tools`; keep those results separate from REA-only trials. +The settings receipt records the same feature policy written to Codex config. + Each repetition compares no server, all REA tools, and a client-selected subset (`binary_session` by default), in the supported client-code host mode. Alternate repetitions reverse ordering. All variants receive the same developer instructions diff --git a/src/catalogDiscovery.ts b/src/catalogDiscovery.ts index 1b4b31a64e..6a5724f4ca 100644 --- a/src/catalogDiscovery.ts +++ b/src/catalogDiscovery.ts @@ -12,7 +12,13 @@ export const HOT_TOOL_NAMES: ReadonlySet = new Set([ // One page activates at most 20 schemas: callers can page a broad search instead // of inadvertently expanding the entire catalog into their client's context. -const pageSize = z.int().min(1).max(20).default(3); +const MAX_DISCOVERY_TOOLS = 20; +const DEFAULT_DISCOVERY_PAGE_SIZE = 3; +const pageSize = z + .int() + .min(1) + .max(MAX_DISCOVERY_TOOLS) + .default(DEFAULT_DISCOVERY_PAGE_SIZE); const inputSchema = z .strictObject({ query: z @@ -25,12 +31,14 @@ const inputSchema = z names: z .array(z.string().min(1)) .min(1) - .max(20) + .max(MAX_DISCOVERY_TOOLS) .optional() - .describe("Exact tool names to load instead of searching."), + .describe( + `At most ${MAX_DISCOVERY_TOOLS} exact tool names to load instead of searching.`, + ), offset: z.int().min(0).default(0), limit: pageSize.describe( - "Search page size; does not apply to exact names.", + `Search page size 1..${MAX_DISCOVERY_TOOLS} (default ${DEFAULT_DISCOVERY_PAGE_SIZE}); does not apply to exact names.`, ), }) .refine( diff --git a/tests/support/analysis-efficiency/agent-trial.py b/tests/support/analysis-efficiency/agent-trial.py index f5db80f9b9..8308c95c93 100644 --- a/tests/support/analysis-efficiency/agent-trial.py +++ b/tests/support/analysis-efficiency/agent-trial.py @@ -37,6 +37,8 @@ def main(): parser.add_argument('--profile-parent', type=Path, help='External disposable-profile parent (default: cache beside the source Codex home)') parser.add_argument('--authority', choices=['static', 'browser'], default='static') + parser.add_argument('--include-account-tools', action='store_true', + help='Allow account apps/plugins; default isolates REA from their catalogs') parser.add_argument('--capture-tool-context', action='store_true', help='Retain owned tool/usage items; excludes account/ambient state') parser.add_argument('--guide-enabled-tools', action='store_true', @@ -159,14 +161,20 @@ def main(): 'or delegate. Reply with exactly the text requested by the user.' ) quote = json.dumps + # A disposable CODEX_HOME does not disable account-connected apps. Keep + # the requested feature policy identical in the config and its receipt. + features = { + 'multi_agent': False, 'shell_snapshot': False, 'shell_tool': False, + 'unified_exec': False, 'code_mode_host': args.mode == 'client-code', + 'apps': args.include_account_tools, 'plugins': args.include_account_tools, + } config = '\n'.join([ 'approval_policy = "never"', 'sandbox_mode = "read-only"', 'web_search = "disabled"', 'model = ' + quote(args.model), 'model_reasoning_effort = ' + quote(args.reasoning), 'developer_instructions = ' + quote(instruction), - '[features]', 'multi_agent = false', 'shell_snapshot = false', - 'shell_tool = false', 'unified_exec = false', - 'code_mode_host = ' + ('true' if args.mode == 'client-code' else 'false'), + '[features]', + *[name + ' = ' + str(enabled).lower() for name, enabled in features.items()], '[history]', 'persistence = "none"', ]) + '\n' if not args.no_mcp: @@ -253,8 +261,7 @@ def main(): provider_env.get('PATH', os.environ.get('PATH', '')).encode()).hexdigest(), 'provider_environment_policy_sha256': hashlib.sha256( provider_policy_text.encode()).hexdigest(), - 'features': {'shell_tool': False, 'unified_exec': False, - 'multi_agent': False, 'code_mode_host': args.mode == 'client-code'}, + 'features': features, }, indent=2) + '\n') invocation = [args.codex, '--no-daemon', 'exec', '--strict-config', *([] if args.capture_tool_context else ['--ephemeral']), diff --git a/tests/support/analysis-efficiency/test_codex_context.py b/tests/support/analysis-efficiency/test_codex_context.py index 6b5c1350e7..2ebc6fb2a2 100644 --- a/tests/support/analysis-efficiency/test_codex_context.py +++ b/tests/support/analysis-efficiency/test_codex_context.py @@ -681,9 +681,12 @@ class ContextCaptureTests(unittest.TestCase): spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py')) agent = importlib.util.module_from_spec(spec) spec.loader.exec_module(agent) - for control, drift, mode in [(False, False, 'direct'), (True, False, 'direct'), - (False, True, 'direct'), (False, False, 'client-code')]: - with self.subTest(control=control, drift=drift, mode=mode), tempfile.TemporaryDirectory() as directory: + for control, drift, mode, account_tools in [ + (False, False, 'direct', False), (True, False, 'direct', False), + (False, True, 'direct', False), (False, False, 'client-code', False), + (False, False, 'client-code', True), (True, False, 'client-code', True), + ]: + with self.subTest(control=control, drift=drift, mode=mode, account_tools=account_tools), tempfile.TemporaryDirectory() as directory: root = Path(directory) runtime = root / 'runtime' (runtime / 'scripts').mkdir(parents=True) @@ -696,7 +699,8 @@ class ContextCaptureTests(unittest.TestCase): output = runtime / '.cache/trial' arguments = ['agent-trial.py', '--runtime-root', str(runtime), '--revision', 'fixture', '--prompt', str(question), '--output', str(output), '--catalog-load', '--mode', mode, - *(['--no-mcp'] if control else [])] + *(['--no-mcp'] if control else []), + *(['--include-account-tools'] if account_tools else [])] observed = {} def launch(_invocation, **options): @@ -723,6 +727,12 @@ class ContextCaptureTests(unittest.TestCase): self.assertEqual('required = true' in observed['config'], not control) self.assertNotIn('Use the registered REA MCP server', observed['config']) self.assertIn('code_mode_host = ' + ('true' if mode == 'client-code' else 'false'), observed['config']) + settings = json.loads((output / 'trial-settings.json').read_text()) + for feature in ['apps', 'plugins']: + self.assertIn(feature + ' = ' + str(account_tools).lower(), observed['config']) + self.assertIs(settings['features'][feature], account_tools) + for feature, enabled in settings['features'].items(): + self.assertIn(feature + ' = ' + str(enabled).lower(), observed['config']) self.assertNotIn('code_mode =', observed['config']) self.assertNotIn('code_mode_only =', observed['config']) self.assertTrue(json.loads((output / 'trial-settings.json').read_text())['workspace_isolated_from_runtime'])