cr

evals/e2e.int.test.ts

@@ -4,6 +4,51 @@ import { HumanMessage } from "@langchain/core/messages";
 import { Client } from "@langchain/langgraph-sdk";
 
 const inputs = [
+  {
+    inputs: {
+      messages: [
+        new HumanMessage(
+          "I'd like approval to explore integrating generative AI into our customer support chatbot. This could significantly improve response accuracy and customer satisfaction. I plan to start with a prototype to evaluate potential impacts.",
+        ),
+      ],
+    },
+  },
+  {
+    inputs: {
+      messages: [
+        new HumanMessage(
+          "Requesting permission to initiate a small-scale pilot project using generative AI to enhance our code review system. This could automate routine checks and free up engineers to focus on complex issues. I'll provide regular updates on the pilot’s progress.",
+        ),
+      ],
+    },
+  },
+  {
+    inputs: {
+      messages: [
+        new HumanMessage(
+          "I'm seeking approval to research and potentially implement generative AI for automatically generating documentation for our APIs. This initiative could greatly reduce manual documentation efforts and improve consistency across our products.",
+        ),
+      ],
+    },
+  },
+  {
+    inputs: {
+      messages: [
+        new HumanMessage(
+          "I propose we explore using generative AI to create synthetic datasets for testing our software products. This method could improve our test coverage and help detect bugs earlier. I'd like authorization to proceed with initial experiments.",
+        ),
+      ],
+    },
+  },
+  {
+    inputs: {
+      messages: [
+        new HumanMessage(
+          "Requesting approval to integrate generative AI into our existing software development pipeline for automatic code generation. This could improve developer productivity and reduce the time spent on repetitive coding tasks.",
+        ),
+      ],
+    },
+  },
   {
     inputs: {
       messages: [

scripts/generate-eval-data.ts

@@ -29,13 +29,15 @@ async function main() {
   const prompt = `You're an advanced AI assistant, helping AI researchers generate a dataset for their evaluation.
 You're helping them evaluate an agent who makes decisions on whether to approve/reject requests from employees at a company.
 
-These requests should be anything an employee could ask his manager for approval for at a software company like Rakuten.
-
 The employees you're emulating are all software engineers.
 
+These requests should be focused on project/product requests, specific to generative AI projects.
+
+For example, an employee could be requesting approval to start working on integrating AI into an existing product. Your examples should be focused on this type of request.
+
 Ensure each request is somewhat detailed, but not too long. These should be 2-3 sentences, and should emulate what real-life requests would look like.
 
-Generate 25 requests.`;
+Generate 5 requests.`;
 
   const response = await model.invoke([{ role: "user", content: prompt }]);