custom schema

people_data_enrichment/README.md

@@ -177,4 +177,23 @@ You can pass the following parameters to customize the evaluation:
 
 ```shell
 python company_data_enrichment/run_eval.py --experiment-prefix "My custom prefix" --min-score 0.9
+```
+
+### Using different schema
+
+If your agent uses a schema that's different from the [example one above](#agent-schema), you can modify `make_agent_runner` in `run_eval.py` in the following way:
+
+```python
+def make_agent_runner(agent_id: str, agent_url: str):
+    agent_graph = RemoteGraph(agent_id, url=agent_url)
+
+    def run_agent(inputs: dict):
+        # transform the inputs (single LangSmith dataset record) to match the agent's schema
+        transformed_inputs = {"my_agent_key": inputs["Person"], ...}
+        response = agent_graph.invoke(transformed_inputs)
+        # transform the agent outputs to match expected eval schema
+        transformed_outputs = {"info": response["my_agent_output_key"]}
+        return transformed_outputs
+
+    return run_agent
 ```

people_data_enrichment/run_eval.py

@@ -15,11 +15,11 @@ FUZZY_MATCH_FIELDS = ("Role","Company")
 LIST_OF_STRING_FIELDS = ("Prior-Companies")
 DEFAULT_DATASET_NAME = "Person Researcher Dataset"
 DEFAULT_GRAPH_ID = "people_maistro"
-DEFAULT_AGENT_URL = "https://ht-abandoned-cynic-27-d4d35e0b052a570a9c5cb83f703881f4.default.us.langgraph.app"
+DEFAULT_AGENT_URL = "https://api.smith.langchain.com/marketplace/cc9aac58-f334-4545-80d9-59300faf8aa2"
 
-os.environ["OPENAI_API_KEY"] = "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"
-os.environ["LANGSMITH_API_KEY"] = "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"
-os.environ["TAVILY_API_KEY"] = "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"
+os.environ["OPENAI_API_KEY"] = "..."
+os.environ["LANGSMITH_API_KEY"] = "..."
+os.environ["TAVILY_API_KEY"] = "..."
 
 client = Client()
 
@@ -41,9 +41,6 @@ You have access to the following tools:
 
 Gather info for this person, company: {topic}"""
 
-
-graph = RemoteGraph("people_maistro", url="https://ht-abandoned-cynic-27-d4d35e0b052a570a9c5cb83f703881f4.default.us.langgraph.app", api_key=None)
-
 extraction_schema = {
     "type": "object",
     "required": [