Automate Bot tests with saved transcript files

I’ve recently worked on a testing framework with Ben Williams (@negativeeddy) that allows you to write your own chat scripts. However, I thought that saving transcripts of your chat from the Bot Emulator would be a really helpful way to produce automated testing. Unfortunately I could not find any out-of-the-box support for running transcript files. Thus the following was born;

Testing with Transcript files

In my example I have the following chat conversation with my Adventure Bot;


USER
Examine table

BOT
The table is old but in good condition.
On the table there is a key 'Room Key A'.
ACTION
Take Room Key A

USER
Take key

BOT
You Take key.
ACTION
look
inventory

When you save the transcript file from the Emulator it contains the serialized version of those activities. Here’s a snippet of the above;

{
“type”: “message”,
“text”: “examine table”,
“from”: {
“id”: “5c8620e2-ccd8-4624-9c91-9842610220a4”,
“name”: “User”
},
{
“type”: “message”,
“serviceUrl”: “http://localhost:50090”,
“channelId”: “emulator”,
“from”: {
“id”: “1”,
“name”: “Bot”,
“role”: “bot”
},
“conversation”: {
“id”: “b1bbf220-011b-11e9-aa8d-d94902c621f5|livechat”
},
“recipient”: {
“id”: “5c8620e2-ccd8-4624-9c91-9842610220a4”,
“role”: “user”
},
“text”: “The table is old but in good condition.\nOn the table there is a key ‘Room Key A’.\n”,
“inputHint”: “acceptingInput”,
“suggestedActions”: {
“actions”: [
{
“type”: “imBack”,
“title”: “take Room Key A”,
“value”: “take Room Key A”
}
]
},

Write a test

Now we have the transcript file we want to use it from a test. First create a Test project and get the contents of the file. Note, I’m using MSTest and Moq. I’m taking a hacky short cut and reading the whole file but it should be using streams;


[TestMethod]
public async Task TakeKeyFromTranscriptTest()
{
  var transcriptFile = await File.ReadAllTextAsync("Transcripts\\examineTable.transcript");
  var activityTranscript = JsonConvert.DeserializeObject(transcriptFile);
  await TestBotTranscriptAsync(activityTranscript);
}

Set-up the test

Now we configure the test and the service-under-test, or in our case – a dialog.


private async Task TestBotTranscriptAsync(List activityTranscript, object request = null)
{
  // create mock version
  var mockDependency = new Mock();
  RecognizerResult recognizerResult = FakeRecognizerResult(
   "examine table",
   "Examine",
   "table");

  mockDependency.Setup(x => x.RecognizeAsync(
   It.Is(i => i.Activity.Text.Equals("Examine table", StringComparison.InvariantCultureIgnoreCase)),
   It.IsAny()))
   .Returns(Task.FromResult(recognizerResult));

  recognizerResult = FakeRecognizerResult(
   "take Room Key A",
   "Take",
   "key");

  // set up mock method
  mockDependency.Setup(x => x.RecognizeAsync(
   It.Is(i => i.Activity.Text.Equals("take Room Key A", StringComparison.InvariantCultureIgnoreCase)),
   It.IsAny()))
   .Returns(Task.FromResult(recognizerResult));

  this.Recognizer = mockDependency.Object;

  await CreateDialogTest(
   nameof(AdventureDialog),
   (id, svc, log) => new AdventureDialog(id, svc, log),
   options: request)
   .AssertBotTranscript(activityTranscript)
   .StartTestAsync();
}

private static RecognizerResult FakeRecognizerResult(string query, string intent, string noun)
{
    var resultJson = "{\"text\":\"" + query +
        "\",\"alteredText\":null,\"intents\":{\"" + intent +
        "\":{\"score\":0.9696778}},\"entities\":{\"$instance\":{\"noun\":[{\"startIndex\":8,\"endIndex\":13,\"text\":\"" + noun +
        "\",\"type\":\"noun\",\"score\":0.997288644}]},\"noun\":[\"" + noun + "\"]}}";
    var recognizerResult = JsonConvert.DeserializeObject(resultJson);
    return recognizerResult;
}

The Botness of the test is inside the CreateDialogTest method;


public TestFlow CreateDialogTest(string dialogId,
Func createDialog,
object options = null)
where TDialog : Dialog
{
  // setup the bot state for the test
  var storage = new MemoryStorage();
  var conversationState = new ConversationState(storage);
  var userState = new UserState(storage);

  var accessors = new AdventureCBCBotAccessors(conversationState, userState)
  {
   AdventureState = conversationState.CreateProperty(AdventureCBCBotAccessors.AdventureStateName),
   DialogState = conversationState.CreateProperty(AdventureCBCBotAccessors.DialogStateName),
  };

  // create the test adapter with any required middleware
  var adapter = new TestAdapter(sendTraceActivity: true)
   .Use(new GameServicesMiddleware(accessors.AdventureState))
   .Use(new AutoSaveStateMiddleware(conversationState));

  // Create or Moq any services
  ILoggerFactory factory = new LoggerFactory()
   .AddConsole()
   .AddDebug();

  var services = new BotServices(
   new BotConfiguration(),
   TestEndpoint
  );

  if (this.Recognizer != null)
  {
   services.LuisServices.Add("LuisAdventureBot", this.Recognizer);
  }

  // Create the dialog under test
  var dialogState = conversationState.CreateProperty("dialogState");
  var dialogs = new DialogSet(dialogState);
  TDialog dialog = createDialog.Invoke(dialogId, services, factory);
   dialogs.Add(dialog);

  // Start the test flow-loop to execute the chat
  return new TestFlow(adapter, async (turnContext, cancellationToken) =>
  {
   var dc = await dialogs.CreateContextAsync(turnContext, cancellationToken);

   var result = await dc.ContinueDialogAsync(cancellationToken);

   if (result.Status == DialogTurnStatus.Complete)
   {
    if (result.Result != null)
    {
     await turnContext.SendActivityAsync(result.Result.ToString());
    }
    return;
   }

   if (!turnContext.Responded)
   {
    await dc.BeginDialogAsync(dialog.Id, options, cancellationToken);
   }
  });

public EndpointService TestEndpoint { get; set; } = new EndpointService() { Endpoint = "https://example.com/bot" };
}

Execute a test

You may have noticed the lines;


.AssertBotTranscript(activityTranscript)
.StartTestAsync();

These are were the execution and testing happen. StartTestAsync start the test adapter but the AssertBotTranscript activity extension is where the magic happens. Before we look at that we need to create a couple of helper classes. These classes are responsible for finding all the message activies and to identify which ones are from the user or the bot.


public enum StepActor
{
  User,
  Bot
}

public class BotScript
{

  public BotScript(List transcript)
  {
   this.Steps = this.ParseScript(transcript);
  }

  public string UserName { get; set; } = "User";

  public IEnumerable Steps { get; }

  private IEnumerable ParseScript(List transcript)
  {
   foreach (var activity in transcript
    .SkipWhile(t => t.Type != ActivityTypes.Message || t.From.Name != this.UserName)
    .Where(t => t.Type == ActivityTypes.Message))
   {
    if (activity.From.Name == this.UserName)
    {
     BotStep step = new BotStep
     {
       Actor = StepActor.User,
       Text = activity.Text,
     };

     yield return step;
    }
    else
    {
     BotStep step = new BotStep
     {
       Actor = StepActor.Bot,
       Text = activity.Text,
       Actions = activity.SuggestedActions?.Actions.Select(a => a.Title).ToList(),
     };

     yield return step;
    }
   }
 }
}

public class BotStep
{
  public StepActor Actor { get; internal set; }
  public string Text { get; internal set; }
  public List Actions { get; internal set; }
}

Now we have our User and Bot steps we can execute the test. This is achieved by enumerating all the steps from the transcripts. If it is a User step then we send it into the Flow. If it’s a Bot step then we assert that the Bot has responded with the expected value in the step.


public static TestFlow AssertBotTranscript(this TestFlow testFlow, List transcript)
{
  var botScript = new BotScript(transcript);
  return testFlow.AssertBotScript(botScript);
}

public static TestFlow AssertBotScript(this TestFlow testFlow, BotScript script)
{
  foreach (var step in script.Steps)
  {
   if (step.Actor == StepActor.Bot)
   {
     testFlow = testFlow.AssertBotStep(step);
   }
   else
   {
     testFlow = testFlow.Send(step.Text);
   }
  }
 return testFlow;
}

public static TestFlow AssertBotStep(this TestFlow testFlow, BotStep expectedResponse)
{
 return testFlow.AssertReply(activity =>
 {
  var messageActivity = activity.AsMessageActivity();

  // check main text of message
  if (!string.IsNullOrEmpty(expectedResponse.Text))
  {
   string fixedExpected = SanitizeText(expectedResponse.Text);
   string actualResponse = SanitizeText(messageActivity.Text);

   Assert.AreEqual(fixedExpected, actualResponse);
  }
}
}

private static string SanitizeText(string text)
{
  string fixedReply = text;

  if (!string.IsNullOrEmpty(fixedReply))
  {
   fixedReply = fixedReply.Replace("\r", string.Empty).Replace("\n", string.Empty);
  }

  return fixedReply;
}

All that’s left to is to run the test;

Capture

 

It is just a start

I’ve left out of lot of additional code you need, e.g. handling Adaptive cards, Actions, Hero Cards, etc. but it’s exactly the same idea. You need to flesh out the BotStep class and AssertReply to handle the different forms of message activities. Hopefully, this will just be included in the Framework of the Bot Community framework and you will never need to write this yourself. But for now, this should help you out.

Advertisements

Call overhead with large number of dialogs in Bot Framework

When you look at the samples for the Bot Framework you’ll no doubt see a pattern for the root dialog that looks like this;

public MyBot(MyBotAccessors accessors, ILoggerFactory loggerFactory)
{
    Dialogs = new DialogSet(accessors.DialogState);
    Dialogs.Add(new DialogA());
    Dialogs.Add(new DialogB());
    ...
    Dialogs.Add(new DialogN());
}

The great thing about the above pattern is that it is nice and clean, you know that the dialog set will be correctly initialized and therefore when you call var dialogResult = await dialogContext.ContinueDialogAsync or await dialogContext.BeginDialogAsync everything will just work.

The problem with the above is that the MyBot constructor will be called for every message that the bot receives. Therefore, every new DialogX will also be called. If you have a reasonably feature rich bot then this overhead might become a problem, especially if the constructors are doing anything…I mean you never write your constructor to do anything slow…right? Regardless I wanted to see if the overhead could be avoided…it can.

public MyBot(MyBotAccessors accessors, ILoggerFactory loggerFactory)
{
    // Initialize like before
    Dialogs = new DialogSet(accessors.DialogState);
}

public async Task OnTurnAsync(ITurnContext turnContext)
{
    ...
    // (skipping code to focus)
    var dialogContext = await Dialogs.CreateContextAsync(turnContext);

    // the context is loaded from state, regardless of our instances
    // so we can see what the Active Dialog Id and...
    if (dialogContext.ActiveDialog?.Id == "DialogB")
    {
        // create in JIT 
        Dialogs.Add(new DialogB());
    }
    ...

    // remember you'll also have to JIT to invoke the dialog
    // don't add it twice
    Dialogs.Add(new DialogB());
    await dialogContext.BeginDialogAsync(nameof(DialogB));
}

LUIS now understands people’s names

This one managed to slip past me, but recently Microsoft Language Understand Intelligent Service (LUIS) has been updated with a pre-built entity to match people’s names. Seems like an obvious requirement but is pretty tricky to implement. To match people’s names;

  1. Open your KB in the LUIS UI of your choice
  2. Go to the Entity menu
  3. select Add Prebuilt entity
  4. choose personName

Now you can create an Intent such as “Call” with “Call Jane” and “Jane” will be matched to personName. So far (and testing is difficult) I’ve thrown all my usual problematic name variants (Western, Nordic, Chinese, Indian) and it’s managed them all, even “Call peter phoxterd-hemmington-smythe” 🙂

Handling flow from Prompt Validators

Edit – warning this is about pre-release v4. Whilst it’s similar to the final release it’s not the same

Creating a basic flow with Botframework V4 is pretty straightforward. You define your steps and any other dialogs or prompts that you may wish to use. For example, consider this basic name capturing definition;

Add(Inputs.Text, new Microsoft.Bot.Builder.Dialogs.TextPrompt(TextValidator));
Add(Name, new WaterfallStep[]
{
    // Each step takes in a dialog context, arguments, and the next delegate.
    async (dc, args, next) =>
    {
        // Prompt for the user's first name.
        await dc.Prompt(Inputs.Text, "Hey, what's your first name?");
    },
    async (dc, args, next) =>
    {
        dc.ActiveDialog.State["1stName"] = args["Text"].ToString();

        // Prompt for the user's second name.
        await dc.Prompt(Inputs.Text, "what's your second name?");
    },
...
private Task TextValidator(ITurnContext context, TextResult toValidate)
{
    if (toValidate.Text == ".")
    {
        toValidate.Status = null;

    }

    return Task.CompletedTask;
}

The above code will reject the second name if the user types in a full-stop. But what will happen next needs to be considered. The current code will correctly reject the full-stop but it will restart the dialog and ask the user for their first name again. There are a couple of ways to handle this, and they can be implemented in unison.

Use a Retry Message

When a prompt has a Retry-Message associated with it then any failure from the Validator will keep the user on the same waterfall step.

...
 async (dc, args, next) =>
    {
        dc.ActiveDialog.State["1stName"] = args["Text"].ToString();
        // Prompt for the user's second name.
        await dc.Prompt(Inputs.Text, "what's your second name?",
            new PromptOptions
            {
                RetryPromptString = "what's your really your second name?"
            });
    },
...

The above code is probably all your need, but you may also want to equip your waterfall steps to be more…re-entrant.

Replay dialog

This is particularly useful if you provide a way to edit the user’s choices. The idea is that each step examines its arguments or state and if already fulfilled simply passes the flow onto the next step. In the final step you decide if everything has completed or if the flow should start again.
Example step;

async (dc, args, next) =>
    {
        if (args.ContainsKey("1stStep")
        {
          await next(args);
        }
        else
        {
          // Prompt for the user's first name.
          await dc.Prompt(Inputs.Text, "Hey, what's your first name?");
        }
    },
...

Final step;

async (dc, args, next) =>
    {
        if (AllConditionsMet(args))
        {
          // finish the dialog
          await dc.End(dc.ActiveDialog.State);
        }
        else
        {
          // replay the flow
          await dc.Replace("yourdialogName", dc.ActiveDialog.State);
        }
    },

Combining Retry and Replay

There is nothing stopping you using a combination of the methods. If you want to enable an editing scenario but want the framework to ensure a criteria is met for specific steps then just implement both.

Matching names in LUIS

I recently hit a thorny issue with allowing my Bot to understand names via LUIS. I tried to train the LUIS model with various utterances of, ‘how is ‘ but it really wasn’t getting anywhere. There are just so many variations of names that unless I retrieved a large name data set then this really wasn’t going to work. So I fell back to using a very specific match. Yes, that is not really in the spirit of LUIS but at least it keeps all the language models in one place. My trick is pretty simple;

  • Create your Intent; e.g. HowIs
  • Add your *specific* utterance; e.g. how is paul
  • Go to the Entities options and create a new Entity called ‘Name’…
  • Set the Entity Type to RegEx…
  • Add your RegEx, e.g. (?<=^.{7})(.*)
  • Hit train
  • Now when you visit your Intent you should see the name aspect labelled as ‘Name’. As I say, it’s pretty crude and frankly you don’t need LUIS to do such a match. But if you believe there is value in keeping all your language matching models in the same place then it’s the best solution I currently have.

    NB I tried using a training list and Pattern and neither seemed to help

    Unit Test Flow in Botframework V4

    In theory unit testing in Botframework V4 should easy enough as it’s based on .net core. But, as with many things, it might not be that obvious. So here is a quick snippet showing a test for a Waterfall flow;

    [TestMethod]
    public async Task Waterfall()
    {
        var convoState = new ConversationState<Dictionary>(new MemoryStorage());
    
        var adapter = new TestAdapter()
            .Use(convoState);
    
        //var dialogState = convoState.CreateProperty("dialogState");
        var dialogs = new DialogSet();
        dialogs.Add("textPrompt", new TextPrompt());
    
        dialogs.Add("test",
            new WaterfallStep[]
            {
                async (dc, args, next) =>
                {
                        await dc.Context.SendActivity("Welcome! We need to ask a few questions to get started.");
                        await dc.Prompt("textPrompt", "What's your name?");
                },
                async (dc, args, next) =>
                {
                    dc.ActiveDialog.State["name"] = args["Value"];
                    await dc.Context.SendActivity($"Thanks {args["Value"]}!");
                    await dc.End();
                }
            }
        );
    
        await new TestFlow(adapter, async (turnContext) =>
        {
            var state = turnContext.GetConversationState<Dictionary>();
            var dc = dialogs.CreateContext(turnContext, state);
            await dc.Continue();
            if (!turnContext.Responded)
            {
                await dc.Begin("test");
            }
        })
        .Send("Hello")
        .AssertReply("Welcome! We need to ask a few questions to get started.")
        .AssertReply("What's your name?")
        .Send("Paul")
        .AssertReply("Thanks Paul!")
        .StartTest();
    }
    

    A couple of points to note, you have to send a message to start the flow, and make sure you call turnContext.GetConversationState rather than using a local version of the state.

    Edit – you may want to also look at Automate Bot tests with saved transcript files , Testing using ConfigurationManager with .net core and How to write a unit test for Bot Framework v4