Adds structured tool chat agent and DynamicStructuredTool class (#1103)

* [WIP] Structured tool input agent * Clean up typing * Update StructuredChatAgent prompt, adds integration test and validation preventing StructuredTools being passed into existing agents * Undo change to make existing agents less strictly typed, use overload signatures for initialize * Update parsing logic for retries * Update test cases * Progress on structured chat agent * Pass tool names into structured chat output fixing parser for more reliability * Revert output parser getFormatInstructions method signature change * Adds StructuredDynamicTool, docs * add renderTemplate call to format instructions, change StructuredDynamicTool to DynamicStructuredTool * Formatting --------- Co-authored-by: vowelparrot <130414180+vowelparrot@users.noreply.github.com> Co-authored-by: Jacob Lee <jacob@autocode.com> Co-authored-by: Nuno Campos <nuno@boringbits.io>
2026-07-19 12:21:48 -04:00 · 2023-05-19 07:48:09 -07:00
parent 7d09e9d395
commit c8d4e6bf7e
18 changed files with 656 additions and 30 deletions
@@ -0,0 +1,16 @@
+---
+hide_table_of_contents: true
+sidebar_position: 4
+---
+
+import CodeBlock from "@theme/CodeBlock";
+import Example from "@examples/agents/structured_chat.ts";
+
+# Structured Tool Chat Agent
+
+Structured Tool Chat Agents are designed to work with tools that take an input conforming to an arbitrary object schema, in contrast to other agents that only support tools that take a single string as input.
+
+This makes it easier to create and use tools that require multiple input values - rather than prompting for a stringified object or comma separated list, you can specify an object with multiple keys.
+Here's an example with a `DynamicStructuredTool`:
+
+<CodeBlock language="typescript">{Example}</CodeBlock>
@@ -0,0 +1,42 @@
+import { z } from "zod";
+import { ChatOpenAI } from "langchain/chat_models/openai";
+import { initializeAgentExecutorWithOptions } from "langchain/agents";
+import { Calculator } from "langchain/tools/calculator";
+import { DynamicStructuredTool } from "langchain/tools";
+
+export const run = async () => {
+  const model = new ChatOpenAI({ temperature: 0 });
+  const tools = [
+    new Calculator(), // Older existing single input tools will still work
+    new DynamicStructuredTool({
+      name: "random-number-generator",
+      description: "generates a random number between two input numbers",
+      schema: z.object({
+        low: z.number().describe("The lower bound of the generated number"),
+        high: z.number().describe("The upper bound of the generated number"),
+      }),
+      func: async ({ low, high }) =>
+        (Math.random() * (high - low) + low).toString(), // Outputs still must be strings
+    }),
+  ];
+
+  const executor = await initializeAgentExecutorWithOptions(tools, model, {
+    agentType: "structured-chat-zero-shot-react-description",
+    verbose: true,
+  });
+  console.log("Loaded agent.");
+
+  const input = `What is a random number between 5 and 10 raised to the second power?`;
+
+  console.log(`Executing with input "${input}"...`);
+
+  const result = await executor.call({ input });
+
+  console.log({ result });
+
+  /*
+    {
+      "output": "67.95299776074"
+    }
+  */
+};
@@ -9,7 +9,7 @@ import {
  BaseChatMessage,
  ChainValues,
 } from "../schema/index.js";
-import { Tool } from "../tools/base.js";
+import { StructuredTool, Tool } from "../tools/base.js";
 import {
  AgentActionOutputParser,
  AgentInput,
@@ -30,6 +30,8 @@ class ParseError extends Error {
 }

 export abstract class BaseAgent {
+  declare ToolType: StructuredTool;
+
  abstract get inputKeys(): string[];

  get returnValues(): string[] {
@@ -250,7 +252,7 @@ export abstract class Agent extends BaseSingleActionAgent {
   * @returns A PromptTemplate assembled from the given tools and fields.
   * */
  static createPrompt(
-    _tools: Tool[],
+    _tools: StructuredTool[],
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    _fields?: Record<string, any>
  ): BasePromptTemplate {
@@ -260,7 +262,7 @@ export abstract class Agent extends BaseSingleActionAgent {
  /** Construct an agent from an LLM and a list of tools */
  static fromLLMAndTools(
    _llm: BaseLanguageModel,
-    _tools: Tool[],
+    _tools: StructuredTool[],
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    _args?: AgentArgs
  ): Agent {
@@ -270,7 +272,7 @@ export abstract class Agent extends BaseSingleActionAgent {
  /**
   * Validate that appropriate tools are passed in
   */
-  static validateTools(_tools: Tool[]): void {}
+  static validateTools(_tools: StructuredTool[]): void {}

  _stop(): string[] {
    return [`\n${this.observationPrefix()}`];
@@ -33,6 +33,8 @@ export type ChatAgentInput = Optional<AgentInput, "outputParser">;
 * @augments Agent
 */
 export class ChatAgent extends Agent {
+  declare ToolType: Tool;
+
  constructor(input: ChatAgentInput) {
    const outputParser =
      input?.outputParser ?? ChatAgent.getDefaultOutputParser();
@@ -56,10 +58,10 @@ export class ChatAgent extends Agent {
  }

  static validateTools(tools: Tool[]) {
-    const invalidTool = tools.find((tool) => !tool.description);
-    if (invalidTool) {
+    const descriptionlessTool = tools.find((tool) => !tool.description);
+    if (descriptionlessTool) {
      const msg =
-        `Got a tool ${invalidTool.name} without a description.` +
+        `Got a tool ${descriptionlessTool.name} without a description.` +
        ` This agent requires descriptions for all tools.`;
      throw new Error(msg);
    }
@@ -43,6 +43,8 @@ export type ChatConversationalAgentInput = Optional<AgentInput, "outputParser">;
 * @augments Agent
 */
 export class ChatConversationalAgent extends Agent {
+  declare ToolType: Tool;
+
  constructor(input: ChatConversationalAgentInput) {
    const outputParser =
      input.outputParser ?? ChatConversationalAgent.getDefaultOutputParser();
@@ -66,10 +68,10 @@ export class ChatConversationalAgent extends Agent {
  }

  static validateTools(tools: Tool[]) {
-    const invalidTool = tools.find((tool) => !tool.description);
-    if (invalidTool) {
+    const descriptionlessTool = tools.find((tool) => !tool.description);
+    if (descriptionlessTool) {
      const msg =
-        `Got a tool ${invalidTool.name} without a description.` +
+        `Got a tool ${descriptionlessTool.name} without a description.` +
        ` This agent requires descriptions for all tools.`;
      throw new Error(msg);
    }
@@ -1,6 +1,5 @@
 import { BaseChain, ChainInputs } from "../chains/base.js";
 import { BaseMultiActionAgent, BaseSingleActionAgent } from "./agent.js";
-import { Tool } from "../tools/base.js";
 import { StoppingMethod } from "./types.js";
 import { SerializedLLMChain } from "../chains/serde.js";
 import {
@@ -13,7 +12,7 @@ import { CallbackManagerForChainRun } from "../callbacks/manager.js";

 export interface AgentExecutorInput extends ChainInputs {
  agent: BaseSingleActionAgent | BaseMultiActionAgent;
-  tools: Tool[];
+  tools: this["agent"]["ToolType"][];
  returnIntermediateSteps?: boolean;
  maxIterations?: number;
  earlyStoppingMethod?: StoppingMethod;
@@ -26,7 +25,7 @@ export interface AgentExecutorInput extends ChainInputs {
 export class AgentExecutor extends BaseChain {
  agent: BaseSingleActionAgent | BaseMultiActionAgent;

-  tools: Tool[];
+  tools: this["agent"]["ToolType"][];

  returnIntermediateSteps = false;

@@ -55,3 +55,12 @@ export {
  SerializedZeroShotAgent,
  StoppingMethod,
 } from "./types.js";
+export {
+  StructuredChatAgent,
+  StructuredChatAgentInput,
+  StructuredChatCreatePromptArgs,
+} from "./structured_chat/index.js";
+export {
+  StructuredChatOutputParser,
+  StructuredChatOutputParserWithRetries,
+} from "./structured_chat/outputParser.js";
@@ -1,9 +1,10 @@
 import { BaseLanguageModel } from "../base_language/index.js";
 import { CallbackManager } from "../callbacks/manager.js";
 import { BufferMemory } from "../memory/buffer_memory.js";
-import { Tool } from "../tools/base.js";
+import { StructuredTool, Tool } from "../tools/base.js";
 import { ChatAgent } from "./chat/index.js";
 import { ChatConversationalAgent } from "./chat_convo/index.js";
+import { StructuredChatAgent } from "./structured_chat/index.js";
 import { AgentExecutor, AgentExecutorInput } from "./executor.js";
 import { ZeroShotAgent } from "./mrkl/index.js";

@@ -73,6 +74,16 @@ export type InitializeAgentExecutorOptions =
      agentArgs?: Parameters<typeof ChatConversationalAgent.fromLLMAndTools>[2];
    } & Omit<AgentExecutorInput, "agent" | "tools">);

+/**
+ * @interface
+ */
+export type InitializeAgentExecutorOptionsStructured =
+  | {
+      agentType: "structured-chat-zero-shot-react-description";
+      agentArgs?: Parameters<typeof StructuredChatAgent.fromLLMAndTools>[2];
+      memory?: never;
+    } & Omit<AgentExecutorInput, "agent" | "tools">;
+
 /**
 * Initialize an agent executor with options
 * @param tools Array of tools to use in the agent
@@ -80,21 +91,36 @@ export type InitializeAgentExecutorOptions =
 * @param options Options for the agent, including agentType, agentArgs, and other options for AgentExecutor.fromAgentAndTools
 * @returns AgentExecutor
 */
-export const initializeAgentExecutorWithOptions = async (
+export async function initializeAgentExecutorWithOptions(
+  tools: StructuredTool[],
+  llm: BaseLanguageModel,
+  options: InitializeAgentExecutorOptionsStructured
+): Promise<AgentExecutor>;
+export async function initializeAgentExecutorWithOptions(
  tools: Tool[],
  llm: BaseLanguageModel,
-  options: InitializeAgentExecutorOptions = {
+  options?: InitializeAgentExecutorOptions
+): Promise<AgentExecutor>;
+export async function initializeAgentExecutorWithOptions(
+  tools: StructuredTool[] | Tool[],
+  llm: BaseLanguageModel,
+  options:
+    | InitializeAgentExecutorOptions
+    | InitializeAgentExecutorOptionsStructured = {
    agentType:
      llm._modelType() === "base_chat_model"
        ? "chat-zero-shot-react-description"
        : "zero-shot-react-description",
  }
-): Promise<AgentExecutor> => {
+): Promise<AgentExecutor> {
+  // Note this tools cast is safe as the overload signatures prevent
+  // the function from being called with a StructuredTool[] when
+  // the agentType is not in InitializeAgentExecutorOptionsStructured
  switch (options.agentType) {
    case "zero-shot-react-description": {
      const { agentArgs, ...rest } = options;
      return AgentExecutor.fromAgentAndTools({
-        agent: ZeroShotAgent.fromLLMAndTools(llm, tools, agentArgs),
+        agent: ZeroShotAgent.fromLLMAndTools(llm, tools as Tool[], agentArgs),
        tools,
        ...rest,
      });
@@ -102,7 +128,7 @@ export const initializeAgentExecutorWithOptions = async (
    case "chat-zero-shot-react-description": {
      const { agentArgs, ...rest } = options;
      return AgentExecutor.fromAgentAndTools({
-        agent: ChatAgent.fromLLMAndTools(llm, tools, agentArgs),
+        agent: ChatAgent.fromLLMAndTools(llm, tools as Tool[], agentArgs),
        tools,
        ...rest,
      });
@@ -110,7 +136,11 @@ export const initializeAgentExecutorWithOptions = async (
    case "chat-conversational-react-description": {
      const { agentArgs, memory, ...rest } = options;
      const executor = AgentExecutor.fromAgentAndTools({
-        agent: ChatConversationalAgent.fromLLMAndTools(llm, tools, agentArgs),
+        agent: ChatConversationalAgent.fromLLMAndTools(
+          llm,
+          tools as Tool[],
+          agentArgs
+        ),
        tools,
        memory:
          memory ??
@@ -123,8 +153,17 @@ export const initializeAgentExecutorWithOptions = async (
      });
      return executor;
    }
+    case "structured-chat-zero-shot-react-description": {
+      const { agentArgs, ...rest } = options;
+      const executor = AgentExecutor.fromAgentAndTools({
+        agent: StructuredChatAgent.fromLLMAndTools(llm, tools, agentArgs),
+        tools,
+        ...rest,
+      });
+      return executor;
+    }
    default: {
      throw new Error("Unknown agent type");
    }
  }
-};
+}
@@ -30,6 +30,8 @@ export type ZeroShotAgentInput = Optional<AgentInput, "outputParser">;
 * @augments Agent
 */
 export class ZeroShotAgent extends Agent {
+  declare ToolType: Tool;
+
  constructor(input: ZeroShotAgentInput) {
    const outputParser =
      input?.outputParser ?? ZeroShotAgent.getDefaultOutputParser();
@@ -53,10 +55,10 @@ export class ZeroShotAgent extends Agent {
  }

  static validateTools(tools: Tool[]) {
-    const invalidTool = tools.find((tool) => !tool.description);
-    if (invalidTool) {
+    const descriptionlessTool = tools.find((tool) => !tool.description);
+    if (descriptionlessTool) {
      const msg =
-        `Got a tool ${invalidTool.name} without a description.` +
+        `Got a tool ${descriptionlessTool.name} without a description.` +
        ` This agent requires descriptions for all tools.`;
      throw new Error(msg);
    }
@@ -0,0 +1,157 @@
+import { zodToJsonSchema } from "zod-to-json-schema";
+import { JsonSchema7ObjectType } from "zod-to-json-schema/src/parsers/object.js";
+
+import { BaseLanguageModel } from "../../base_language/index.js";
+import { LLMChain } from "../../chains/llm_chain.js";
+import { PromptTemplate } from "../../prompts/prompt.js";
+import {
+  ChatPromptTemplate,
+  HumanMessagePromptTemplate,
+  SystemMessagePromptTemplate,
+} from "../../prompts/chat.js";
+import { AgentStep } from "../../schema/index.js";
+import { StructuredTool } from "../../tools/base.js";
+import { Optional } from "../../types/type-utils.js";
+import { Agent, AgentArgs, OutputParserArgs } from "../agent.js";
+import { AgentInput } from "../types.js";
+import { StructuredChatOutputParserWithRetries } from "./outputParser.js";
+import { FORMAT_INSTRUCTIONS, PREFIX, SUFFIX } from "./prompt.js";
+
+export interface StructuredChatCreatePromptArgs {
+  /** String to put after the list of tools. */
+  suffix?: string;
+  /** String to put before the list of tools. */
+  prefix?: string;
+  /** List of input variables the final prompt will expect. */
+  inputVariables?: string[];
+}
+
+export type StructuredChatAgentInput = Optional<AgentInput, "outputParser">;
+
+/**
+ * Agent that interoperates with Structured Tools using React logic.
+ * @augments Agent
+ */
+export class StructuredChatAgent extends Agent {
+  constructor(input: StructuredChatAgentInput) {
+    const outputParser =
+      input?.outputParser ?? StructuredChatAgent.getDefaultOutputParser();
+    super({ ...input, outputParser });
+  }
+
+  _agentType() {
+    return "structured-chat-zero-shot-react-description" as const;
+  }
+
+  observationPrefix() {
+    return "Observation: ";
+  }
+
+  llmPrefix() {
+    return "Thought:";
+  }
+
+  _stop(): string[] {
+    return ["Observation:"];
+  }
+
+  static validateTools(tools: StructuredTool[]) {
+    const descriptionlessTool = tools.find((tool) => !tool.description);
+    if (descriptionlessTool) {
+      const msg =
+        `Got a tool ${descriptionlessTool.name} without a description.` +
+        ` This agent requires descriptions for all tools.`;
+      throw new Error(msg);
+    }
+  }
+
+  static getDefaultOutputParser(
+    fields?: OutputParserArgs & {
+      toolNames: string[];
+    }
+  ) {
+    if (fields?.llm) {
+      return StructuredChatOutputParserWithRetries.fromLLM(fields.llm, {
+        toolNames: fields.toolNames,
+      });
+    }
+    return new StructuredChatOutputParserWithRetries({
+      toolNames: fields?.toolNames,
+    });
+  }
+
+  async constructScratchPad(steps: AgentStep[]): Promise<string> {
+    const agentScratchpad = await super.constructScratchPad(steps);
+    if (agentScratchpad) {
+      return `This was your previous work (but I haven't seen any of it! I only see what you return as final answer):\n${agentScratchpad}`;
+    }
+    return agentScratchpad;
+  }
+
+  static createToolSchemasString(tools: StructuredTool[]) {
+    return tools
+      .map(
+        (tool) =>
+          `${tool.name}: ${tool.description}, args: ${JSON.stringify(
+            (zodToJsonSchema(tool.schema) as JsonSchema7ObjectType).properties
+          )}`
+      )
+      .join("\n");
+  }
+
+  /**
+   * Create prompt in the style of the agent.
+   *
+   * @param tools - List of tools the agent will have access to, used to format the prompt.
+   * @param args - Arguments to create the prompt with.
+   * @param args.suffix - String to put after the list of tools.
+   * @param args.prefix - String to put before the list of tools.
+   */
+  static createPrompt(
+    tools: StructuredTool[],
+    args?: StructuredChatCreatePromptArgs
+  ) {
+    const { prefix = PREFIX, suffix = SUFFIX } = args ?? {};
+    const template = [prefix, FORMAT_INSTRUCTIONS, suffix].join("\n\n");
+    const messages = [
+      new SystemMessagePromptTemplate(
+        new PromptTemplate({
+          template,
+          inputVariables: [],
+          partialVariables: {
+            tool_schemas: StructuredChatAgent.createToolSchemasString(tools),
+            tool_names: tools.map((tool) => tool.name).join(", "),
+          },
+        })
+      ),
+      HumanMessagePromptTemplate.fromTemplate("{input}\n\n{agent_scratchpad}"),
+    ];
+    return ChatPromptTemplate.fromPromptMessages(messages);
+  }
+
+  static fromLLMAndTools(
+    llm: BaseLanguageModel,
+    tools: StructuredTool[],
+    args?: StructuredChatCreatePromptArgs & AgentArgs
+  ) {
+    StructuredChatAgent.validateTools(tools);
+    const prompt = StructuredChatAgent.createPrompt(tools, args);
+    const outputParser =
+      args?.outputParser ??
+      StructuredChatAgent.getDefaultOutputParser({
+        llm,
+        toolNames: tools.map((tool) => tool.name),
+      });
+    const chain = new LLMChain({
+      prompt,
+      llm,
+      callbacks: args?.callbacks,
+    });
+
+    return new StructuredChatAgent({
+      llmChain: chain,
+      outputParser,
+      allowedTools: tools.map((t) => t.name),
+    });
+  }
+}
@@ -0,0 +1,95 @@
+import { AgentActionOutputParser } from "../types.js";
+import {
+  AGENT_ACTION_FORMAT_INSTRUCTIONS,
+  FORMAT_INSTRUCTIONS,
+} from "./prompt.js";
+import { OutputFixingParser } from "../../output_parsers/fix.js";
+import { BaseLanguageModel } from "../../base_language/index.js";
+import { AgentAction, AgentFinish } from "../../schema/index.js";
+import { OutputParserException } from "../../schema/output_parser.js";
+import { renderTemplate } from "../../prompts/index.js";
+
+export class StructuredChatOutputParser extends AgentActionOutputParser {
+  constructor(private toolNames: string[]) {
+    super();
+  }
+
+  async parse(text: string): Promise<AgentAction | AgentFinish> {
+    try {
+      const regex = /```(?:json)?(.*)(```)/gs;
+      const actionMatch = regex.exec(text);
+      if (actionMatch === null) {
+        throw new OutputParserException(
+          `Could not parse an action. The agent action must be within a markdown code block, and "action" must be a provided tool or "Final Answer"`
+        );
+      }
+      const response = JSON.parse(actionMatch[1].trim());
+      const { action, action_input } = response;
+
+      if (action === "Final Answer") {
+        return { returnValues: { output: action_input }, log: text };
+      }
+      return { tool: action, toolInput: action_input || {}, log: text };
+    } catch (e) {
+      throw new OutputParserException(
+        `Failed to parse. Text: "${text}". Error: ${e}`
+      );
+    }
+  }
+
+  getFormatInstructions(): string {
+    return renderTemplate(AGENT_ACTION_FORMAT_INSTRUCTIONS, "f-string", {
+      tool_names: this.toolNames.join(", "),
+    });
+  }
+}
+
+export interface StructuredChatOutputParserArgs {
+  baseParser?: StructuredChatOutputParser;
+  outputFixingParser?: OutputFixingParser<AgentAction | AgentFinish>;
+  toolNames?: string[];
+}
+
+export class StructuredChatOutputParserWithRetries extends AgentActionOutputParser {
+  private baseParser: StructuredChatOutputParser;
+
+  private outputFixingParser?: OutputFixingParser<AgentAction | AgentFinish>;
+
+  private toolNames: string[] = [];
+
+  constructor(fields: StructuredChatOutputParserArgs) {
+    super();
+    this.toolNames = fields.toolNames ?? this.toolNames;
+    this.baseParser =
+      fields?.baseParser ?? new StructuredChatOutputParser(this.toolNames);
+    this.outputFixingParser = fields?.outputFixingParser;
+  }
+
+  async parse(text: string): Promise<AgentAction | AgentFinish> {
+    if (this.outputFixingParser !== undefined) {
+      return this.outputFixingParser.parse(text);
+    }
+    return this.baseParser.parse(text);
+  }
+
+  getFormatInstructions(): string {
+    return renderTemplate(FORMAT_INSTRUCTIONS, "f-string", {
+      tool_names: this.toolNames.join(", "),
+    });
+  }
+
+  static fromLLM(
+    llm: BaseLanguageModel,
+    options: Omit<StructuredChatOutputParserArgs, "outputFixingParser">
+  ): StructuredChatOutputParserWithRetries {
+    const baseParser =
+      options.baseParser ??
+      new StructuredChatOutputParser(options.toolNames ?? []);
+    const outputFixingParser = OutputFixingParser.fromLLM(llm, baseParser);
+    return new StructuredChatOutputParserWithRetries({
+      baseParser,
+      outputFixingParser,
+      toolNames: options.toolNames,
+    });
+  }
+}
@@ -0,0 +1,59 @@
+export const PREFIX = `Answer the following questions truthfully and as best you can.`;
+export const AGENT_ACTION_FORMAT_INSTRUCTIONS = `Output a JSON markdown code snippet containing a valid JSON blob (denoted below by $JSON_BLOB).
+This $JSON_BLOB must have a "action" key (with the name of the tool to use) and an "action_input" key (tool input).
+
+Valid "action" values: "Final Answer" (which you must use when giving your final response to the user), or one of [{tool_names}].
+
+The $JSON_BLOB must be valid, parseable JSON and only contain a SINGLE action. Here is an example of an acceptable output:
+
+\`\`\`json
+{{
+  "action": $TOOL_NAME
+  "action_input": $INPUT
+}}
+\`\`\`
+
+Remember to include the surrounding markdown code snippet delimiters (begin with "\`\`\`" json and close with "\`\`\`")!
+`;
+export const FORMAT_INSTRUCTIONS = `You have access to the following tools.
+You must format your inputs to these tools to match their "JSON schema" definitions below.
+
+"JSON Schema" is a declarative language that allows you to annotate and validate JSON documents.
+
+For example, the example "JSON Schema" instance {{"properties": {{"foo": {{"description": "a list of test words", "type": "array", "items": {{"type": "string"}}}}}}, "required": ["foo"]}}}}
+would match an object with one required property, "foo". The "type" property specifies "foo" must be an "array", and the "description" property semantically describes it as "a list of test words". The items within "foo" must be strings.
+Thus, the object {{"foo": ["bar", "baz"]}} is a well-formatted instance of this example "JSON Schema". The object {{"properties": {{"foo": ["bar", "baz"]}}}} is not well-formatted.
+
+Here are the JSON Schema instances for the tools you have access to:
+
+{tool_schemas}
+
+The way you use the tools is as follows:
+
+------------------------
+
+${AGENT_ACTION_FORMAT_INSTRUCTIONS}
+
+If you are using a tool, "action_input" must adhere to the tool's input schema, given above.
+
+------------------------
+
+ALWAYS use the following format:
+
+Question: the input question you must answer
+Thought: you should always think about what to do
+Action:
+\`\`\`json
+$JSON_BLOB
+\`\`\`
+Observation: the result of the action
+... (this Thought/Action/Observation can repeat N times)
+Thought: I now know the final answer
+Action:
+\`\`\`json
+{{
+  "action": "Final Answer",
+  "action_input": "Final response to human"
+}}
+\`\`\``;
+export const SUFFIX = `Begin! Reminder to ALWAYS use the above format, and to use tools if appropriate.`;
@@ -26,6 +26,12 @@ const agents = [
    initializeAgentExecutorWithOptions(tools, new OpenAI({ temperature: 0 }), {
      agentType: "zero-shot-react-description",
    }),
+  (tools) =>
+    initializeAgentExecutorWithOptions(
+      tools,
+      new ChatOpenAI({ temperature: 0 }),
+      { agentType: "structured-chat-zero-shot-react-description" }
+    ),
 ] as ((
  tools: Tool[]
 ) => ReturnType<typeof initializeAgentExecutorWithOptions>)[];
@@ -0,0 +1,49 @@
+import { test } from "@jest/globals";
+import { z } from "zod";
+
+import { ChatOpenAI } from "../../chat_models/openai.js";
+import { Calculator } from "../../tools/calculator.js";
+import { StructuredTool } from "../../tools/base.js";
+import { initializeAgentExecutorWithOptions } from "../initialize.js";
+
+class FakeWebSearchTool extends StructuredTool {
+  schema = z.object({
+    query: z.string(),
+    max_results: z.number(),
+  });
+
+  name = "fake_web_search_tool";
+
+  description = "useful for when you need to search for up to date webpages.";
+
+  async _call({
+    query: _query,
+    max_results,
+  }: z.infer<this["schema"]>): Promise<string> {
+    return [...Array(max_results).keys()]
+      .map((n) => `https://langchain.com/tutorial/${n}`)
+      .join(", ");
+  }
+}
+
+test("Run structured chat agent", async () => {
+  const model = new ChatOpenAI({ modelName: "gpt-3.5-turbo", temperature: 0 });
+  const tools = [new Calculator(), new FakeWebSearchTool({})];
+
+  const executor = await initializeAgentExecutorWithOptions(tools, model, {
+    agentType: "structured-chat-zero-shot-react-description",
+  });
+  console.log("Loaded agent.");
+
+  const input0 = `what is 9 to the 2nd power?`;
+
+  const result0 = await executor.call({ input: input0 });
+
+  console.log(`Got output ${result0.output}`);
+
+  const input1 = `Give me some URLs for tutorial articles on LangChain as a comma separated list.`;
+
+  const result1 = await executor.call({ input: input1 });
+
+  console.log(`Got output ${result1.output}`);
+});
@@ -0,0 +1,44 @@
+import { test, expect } from "@jest/globals";
+import { StructuredChatOutputParser } from "../structured_chat/outputParser.js";
+import { AgentAction, AgentFinish } from "../../schema/index.js";
+
+test("Can parse JSON with text in front of it", async () => {
+  const testCases = [
+    {
+      input:
+        'Here we have some boilerplate nonsense```json\n{\n "action": "blogpost",\n  "action_input": "```sql\\nSELECT * FROM orders\\nJOIN users ON users.id = orders.user_id\\nWHERE users.email = \'bud\'```"\n}\n``` and at the end there is more nonsense',
+      output:
+        '{"action":"blogpost","action_input":"```sql\\nSELECT * FROM orders\\nJOIN users ON users.id = orders.user_id\\nWHERE users.email = \'bud\'```"}',
+      tool: "blogpost",
+      toolInput:
+        "```sql\nSELECT * FROM orders\nJOIN users ON users.id = orders.user_id\nWHERE users.email = 'bud'```",
+    },
+    {
+      input:
+        'Here we have some boilerplate nonsense```json\n{\n \t\r\n"action": "blogpost",\n\t\r  "action_input": "```sql\\nSELECT * FROM orders\\nJOIN users ON users.id = orders.user_id\\nWHERE users.email = \'bud\'```"\n\t\r}\n\n\n\t\r``` and at the end there is more nonsense',
+      output:
+        '{"action":"blogpost","action_input":"```sql\\nSELECT * FROM orders\\nJOIN users ON users.id = orders.user_id\\nWHERE users.email = \'bud\'```"}',
+      tool: "blogpost",
+      toolInput:
+        "```sql\nSELECT * FROM orders\nJOIN users ON users.id = orders.user_id\nWHERE users.email = 'bud'```",
+    },
+  ];
+
+  const p = new StructuredChatOutputParser(["blogpost"]);
+  for (const message of testCases) {
+    const parsed = await p.parse(message.input);
+    expect(parsed).toBeDefined();
+    if (message.tool === "Final Answer") {
+      expect((parsed as AgentFinish).returnValues).toBeDefined();
+    } else {
+      expect((parsed as AgentAction).tool).toEqual(message.tool);
+
+      if (typeof message.toolInput === "object") {
+        expect(message.toolInput).toEqual((parsed as AgentAction).toolInput);
+      }
+      if (typeof message.toolInput === "string") {
+        expect(message.toolInput).toContain((parsed as AgentAction).toolInput);
+      }
+    }
+  }
+});
@@ -0,0 +1,54 @@
+import { test, expect } from "@jest/globals";
+import { StructuredChatOutputParserWithRetries } from "../structured_chat/outputParser.js";
+import { ChatOpenAI } from "../../chat_models/openai.js";
+import { AgentAction, AgentFinish } from "../../schema/index.js";
+
+test("Can parse JSON with text in front of it", async () => {
+  const testCases = [
+    {
+      input:
+        'Here we have an invalid format (missing markdown block) that the parser should retry and fix: {\n \t\r\n"action": "blogpost",\n\t\r  "action_input": "```sql\\nSELECT * FROM orders\\nJOIN users ON users.id = orders.user_id\\nWHERE users.email = \'bud\'```"\n\t\r}\n\n\n\t\r and at the end there is more nonsense',
+      tool: "blogpost",
+      toolInput:
+        "```sql\nSELECT * FROM orders\nJOIN users ON users.id = orders.user_id\nWHERE users.email = 'bud'```",
+    },
+    {
+      input:
+        'Here we have an invalid format (missing markdown block) with a structured tool that the parser should retry and fix: {\n \t\r\n"action": "blogpost",\n\t\r  "action_input": {"query": "SELECT * FROM orders\\nJOIN users ON users.id = orders.user_id\\nWHERE users.email = $1",\n\t"parameters": ["bud"]\n\t}\n\t\r}\n\n\n\t\r and at the end there is more nonsense',
+      tool: "blogpost",
+      toolInput: {
+        query:
+          "SELECT * FROM orders\nJOIN users ON users.id = orders.user_id\nWHERE users.email = $1",
+        parameters: ["bud"],
+      },
+    },
+    {
+      input: `I don't know the answer.`,
+      tool: "Final Answer",
+      toolInput: "I don't know the answer.",
+    },
+  ];
+
+  const p = StructuredChatOutputParserWithRetries.fromLLM(
+    new ChatOpenAI({ temperature: 0, modelName: "gpt-3.5-turbo" }),
+    {
+      toolNames: ["blogpost"],
+    }
+  );
+  for (const message of testCases) {
+    const parsed = await p.parse(message.input);
+    expect(parsed).toBeDefined();
+    if (message.tool === "Final Answer") {
+      expect((parsed as AgentFinish).returnValues).toBeDefined();
+    } else {
+      expect((parsed as AgentAction).tool).toEqual(message.tool);
+
+      if (typeof message.toolInput === "object") {
+        expect(message.toolInput).toEqual((parsed as AgentAction).toolInput);
+      }
+      if (typeof message.toolInput === "string") {
+        expect(message.toolInput).toContain((parsed as AgentAction).toolInput);
+      }
+    }
+  }
+});
@@ -1,16 +1,31 @@
+import { z } from "zod";
 import { CallbackManagerForToolRun, Callbacks } from "../callbacks/manager.js";
-import { Tool } from "./base.js";
+import { StructuredTool, Tool } from "./base.js";

-export interface DynamicToolInput {
+export interface BaseDynamicToolInput {
  name: string;
  description: string;
+  returnDirect?: boolean;
+  verbose?: boolean;
+  callbacks?: Callbacks;
+}
+
+export interface DynamicToolInput extends BaseDynamicToolInput {
  func: (
    input: string,
    runManager?: CallbackManagerForToolRun
  ) => Promise<string>;
-  returnDirect?: boolean;
-  verbose?: boolean;
-  callbacks?: Callbacks;
+}
+
+export interface DynamicStructuredToolInput<
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  T extends z.ZodObject<any, any, any, any> = z.ZodObject<any, any, any, any>
+> extends BaseDynamicToolInput {
+  func: (
+    input: z.infer<T>,
+    runManager?: CallbackManagerForToolRun
+  ) => Promise<string>;
+  schema: T;
 }

 /**
@@ -39,3 +54,32 @@ export class DynamicTool extends Tool {
    return this.func(input, runManager);
  }
 }
+
+export class DynamicStructuredTool<
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  T extends z.ZodObject<any, any, any, any> = z.ZodObject<any, any, any, any>
+> extends StructuredTool {
+  name: string;
+
+  description: string;
+
+  func: DynamicStructuredToolInput["func"];
+
+  schema: T;
+
+  constructor(fields: DynamicStructuredToolInput<T>) {
+    super(fields);
+    this.name = fields.name;
+    this.description = fields.description;
+    this.func = fields.func;
+    this.returnDirect = fields.returnDirect ?? this.returnDirect;
+    this.schema = fields.schema;
+  }
+
+  protected _call(
+    arg: z.output<T>,
+    runManager?: CallbackManagerForToolRun
+  ): Promise<string> {
+    return this.func(arg, runManager);
+  }
+}
@@ -2,7 +2,12 @@ export { SerpAPI, SerpAPIParameters } from "./serpapi.js";
 export { DadJokeAPI } from "./dadjokeapi.js";
 export { BingSerpAPI } from "./bingserpapi.js";
 export { Tool, ToolParams, StructuredTool } from "./base.js";
-export { DynamicTool, DynamicToolInput } from "./dynamic.js";
+export {
+  DynamicTool,
+  DynamicToolInput,
+  DynamicStructuredTool,
+  DynamicStructuredToolInput,
+} from "./dynamic.js";
 export { IFTTTWebhook } from "./IFTTTWebhook.js";
 export { ChainTool, ChainToolInput } from "./chain.js";
 export {