Copilot Chat: Scenario Tests (#1376)

### Motivation and Context We need a pipeline and tests to gate PRs coming into Copilot Chat. ### Description 1. Add a simple end to end test that simulates log in and sending a message to the bot. Will add more tests in the future. 2. Add a new GitHub action that will run against PRs that modify the Copilot sample app. 3. Fix a bug where the BackendProbe component doesn't automatically retry. Note: need to add a couple new GitHub secrets for the action to pass.
2026-10-12 02:19:06 +00:00 · 2023-06-09 20:14:31 +00:00
parent 073cd02111
commit 5c9903439d
9 changed files with 241 additions and 14 deletions
@@ -0,0 +1,86 @@
+name: Copilot Chat Tests
+on:
+  workflow_dispatch:
+  push:
+    branches: [ "main", "feature*" ]
+    paths: 
+      - 'samples/apps/copilot-chat-app/**'
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    defaults:
+      run:
+        working-directory: samples/apps/copilot-chat-app/webapp
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+        cache-dependency-path: samples/apps/copilot-chat-app/webapp/yarn.lock
+        cache: 'yarn'
+
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v3
+      with:
+        dotnet-version: 6.0.x
+
+    - name: Install dependencies
+      run: yarn install
+
+    - name: Install Playwright Browsers
+      run: yarn playwright install --with-deps
+
+    - name: Update AIService configuration
+      working-directory: samples/apps/copilot-chat-app/webapi
+      env:
+        AzureOpenAI__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }}
+        AzureOpenAI__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }}
+      run: |
+        dotnet dev-certs https
+        dotnet user-secrets set "AIService:Key" "$AzureOpenAI__ApiKey"
+        dotnet user-secrets set "AIService:Endpoint" "$AzureOpenAI__Endpoint"
+
+    - name: Start service in background
+      working-directory: samples/apps/copilot-chat-app/webapi
+      run: |
+        dotnet run > service-log.txt 2>&1 &
+        for attempt in {0..20}; do
+          jobs
+          echo 'Waiting for service to start...';
+          if curl -k https://localhost:40443/healthz; then
+            echo;
+            echo 'Service started';
+            break;
+          fi;
+
+          sleep 5;
+        done
+
+    - name: Run Playwright tests
+      env:
+        REACT_APP_BACKEND_URI: https://localhost:40443/
+        REACT_APP_AAD_CLIENT_ID: ${{ secrets.COPILOT_CHAT_REACT_APP_AAD_CLIENT_ID }}
+        REACT_APP_AAD_AUTHORITY: https://login.microsoftonline.com/common
+        REACT_APP_TEST_USER_ACCOUNT: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT }}
+        REACT_APP_TEST_USER_PASSWORD: ${{ secrets.COPILOT_CHAT_TEST_USER_PASSWORD }}
+      run: yarn playwright test
+
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: playwright-report
+        path: samples/apps/copilot-chat-app/webapp/playwright-report/
+        retention-days: 30
+
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: service-log
+        path: samples/apps/copilot-chat-app/webapi/service-log.txt
+        retention-days: 30
@@ -469,4 +469,7 @@ java/**/target
 java/.mvn/wrapper/maven-wrapper.jar

 # Java settings
-conf.properties
+conf.properties
+
+# Playwright
+playwright-report/
@@ -12,4 +12,8 @@ REACT_APP_SK_API_KEY=
 # Replace with your locally-trusted cert file
 # SSL_CRT_FILE=local-cert.crt
 # Replace with your locally-trusted cert key
-# SSL_KEY_FILE=local-cert.key
+# SSL_KEY_FILE=local-cert.key
+
+# For CI and testing purposes only
+REACT_APP_TEST_USER_ACCOUNT=
+REACT_APP_TEST_USER_PASSWORD=
@@ -22,6 +22,7 @@
    "@fluentui/react-components": "^9.13.0",
    "@fluentui/react-icons": "^2.0.193",
    "@fluentui/react-northstar": "^0.66.4",
+    "@playwright/test": "^1.34.3",
    "@reduxjs/toolkit": "^1.9.1",
    "debug": "^4.3.4",
    "microsoft-cognitiveservices-speech-sdk": "^1.27.0",
@@ -0,0 +1,59 @@
+import { defineConfig, devices } from '@playwright/test';
+import dotenv from 'dotenv';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+// Read the .env file if it exists. This is usually for testing locally.
+// Tests on CI should should get the environment variables from GitHub actions.
+const filename = fileURLToPath(import.meta.url);
+const dirname = path.dirname(filename);
+const envPath = path.resolve(dirname, '.env');
+if (fs.existsSync(envPath)) {
+  dotenv.config({ path: envPath });
+}
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+
+    /* Ignore certificate errors. */
+    ignoreHTTPSErrors: true,
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+    // Add more browsers here.
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: {
+    command: 'yarn start',
+    url: 'http://localhost:3000',
+    reuseExistingServer: !process.env.CI,
+    timeout: 120000,
+  },
+});
@@ -151,7 +151,11 @@ export const ChatHistoryItem: React.FC<ChatHistoryItemProps> = ({ message, getRe

    return (
        <>
-            <div className={isMe ? mergeClasses(classes.root, classes.alignEnd) : classes.root}>
+            <div
+                className={isMe ? mergeClasses(classes.root, classes.alignEnd) : classes.root}
+                data-testid={`chat-history-item-${messageIndex}`}
+                data-username={fullName}
+            >
                {!isMe && <Persona className={classes.persona} avatar={avatar} presence={{ status: 'available' }} />}
                <div className={isMe ? mergeClasses(classes.item, classes.me) : classes.item}>
                    <div className={classes.header}>
@@ -10,18 +10,22 @@ interface IData {

 const BackendProbe: FC<IData> = ({ uri, onBackendFound }) => {
    useEffect(() => {
-        const requestUrl = new URL('healthz', uri);
-        const fetchAsync = async () => {
-            try {
-                var result = await fetch(requestUrl);
+        const timer = setInterval(() => {
+            const requestUrl = new URL('healthz', uri);
+            const fetchAsync = async () => {
+                try {
+                    var result = await fetch(requestUrl);

-                if (result.ok) {
-                    onBackendFound();
-                }
-            } catch {}
-        };
+                    if (result.ok) {
+                        onBackendFound();
+                    }
+                } catch { }
+            };

-        fetchAsync();
+            fetchAsync();
+        }, 3000);
+
+        return () => clearInterval(timer);
    });

    return (
@@ -0,0 +1,51 @@
+import { expect, test } from '@playwright/test';
+
+/*
+This test is a simple end-to-end test that verifies the app can be
+loaded, and signed in. The user can send a message to the bot, and expect
+a response.
+*/
+test('get response from bot', async ({ page }) => {
+    // Make sure the server is running.
+    await page.goto('https://localhost:40443/healthz');
+    expect(page.getByText('Healthy')).toBeDefined();
+
+    await page.goto('/');
+    // Expect the page to contain a "Login" button.
+    await page.getByRole('button').click();
+    // Clicking the login button should redirect to the login page.
+    await expect(page).toHaveURL(new RegExp('^' + process.env.REACT_APP_AAD_AUTHORITY));
+    // Login with the test user.
+    await page.getByPlaceholder('Email, phone, or Skype').click();
+    await page.getByPlaceholder('Email, phone, or Skype').fill(process.env.REACT_APP_TEST_USER_ACCOUNT as string);
+    await page.getByRole('button', { name: 'Next' }).click();
+    await page.getByPlaceholder('Password').click();
+    await page.getByPlaceholder('Password').fill(process.env.REACT_APP_TEST_USER_PASSWORD as string);
+    await page.getByRole('button', { name: 'Sign in' }).click();
+
+    // Select No if asked to stay signed in.
+    const isAskingStaySignedIn = await page.$$("text='Stay signed in?'");
+    if (isAskingStaySignedIn) {
+        await page.getByRole('button', { name: 'No' }).click();
+    }
+
+    // After login, the page should redirect back to the app.
+    await expect(page).toHaveTitle('Copilot Chat');
+
+    // Send a message to the bot and wait for the response.
+    const responsePromise = page.waitForResponse('**/chat');
+    await page.locator('#chat-input').click();
+    await page.locator('#chat-input').fill('Hi!');
+    await page.locator('#chat-input').press('Enter');
+    await responsePromise;
+
+    // Expect the chat history to contain 3 messages.
+    // The first message is the welcome message from the bot.
+    // The second message is the user's message.
+    // The third message is the bot's response.
+    const chatHistoryItems = page.getByTestId(new RegExp('chat-history-item-*'));
+    expect((await chatHistoryItems.all()).length).toBe(3);
+
+    // Expect the third message to be the bot's response.
+    await expect(chatHistoryItems.nth(2)).toHaveAttribute('data-username', 'Copilot');
+});
@@ -2508,6 +2508,16 @@
    "@nodelib/fs.scandir" "2.1.5"
    fastq "^1.6.0"

+"@playwright/test@^1.34.3":
+  version "1.34.3"
+  resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.34.3.tgz#d9f1ac3f1a09633b5ca5351c50c308bf802bde53"
+  integrity sha512-zPLef6w9P6T/iT6XDYG3mvGOqOyb6eHaV9XtkunYs0+OzxBtrPAAaHotc0X+PJ00WPPnLfFBTl7mf45Mn8DBmw==
+  dependencies:
+    "@types/node" "*"
+    playwright-core "1.34.3"
+  optionalDependencies:
+    fsevents "2.3.2"
+
 "@pmmmwh/react-refresh-webpack-plugin@^0.5.3":
  version "0.5.10"
  resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.10.tgz#2eba163b8e7dbabb4ce3609ab5e32ab63dda3ef8"
@@ -6147,7 +6157,7 @@ fs.realpath@^1.0.0:
  resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
  integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==

-fsevents@^2.3.2, fsevents@~2.3.2:
+[email protected], fsevents@^2.3.2, fsevents@~2.3.2:
  version "2.3.2"
  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
@@ -8667,6 +8677,11 @@ pkg-up@^3.1.0:
  dependencies:
    find-up "^3.0.0"

+[email protected]:
+  version "1.34.3"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.34.3.tgz#bc906ea1b26bb66116ce329436ee59ba2e78fe9f"
+  integrity sha512-2pWd6G7OHKemc5x1r1rp8aQcpvDh7goMBZlJv6Co5vCNLVcQJdhxRL09SGaY6HcyHH9aT4tiynZabMofVasBYw==
+
 please-upgrade-node@^3.2.0:
  version "3.2.0"
  resolved "https://registry.yarnpkg.com/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz#aeddd3f994c933e4ad98b99d9a556efa0e2fe942"