add chart handler

BotSharp.sln

@@ -143,6 +143,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.LLM.Tests", "tests
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.RealtimeVoice", "tests\BotSharp.Test.RealtimeVoice\BotSharp.Test.RealtimeVoice.csproj", "{B067B126-88CD-4282-BEEF-7369B64423EF}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ChartHandler", "src\Plugins\BotSharp.Plugin.ChartHandler\BotSharp.Plugin.ChartHandler.csproj", "{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -599,6 +601,14 @@ Global
 		{B067B126-88CD-4282-BEEF-7369B64423EF}.Release|Any CPU.Build.0 = Release|Any CPU
 		{B067B126-88CD-4282-BEEF-7369B64423EF}.Release|x64.ActiveCfg = Release|Any CPU
 		{B067B126-88CD-4282-BEEF-7369B64423EF}.Release|x64.Build.0 = Release|Any CPU
+		{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Debug|x64.Build.0 = Debug|Any CPU
+		{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.ActiveCfg = Release|Any CPU
+		{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -668,6 +678,7 @@ Global
 		{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
 		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
 		{B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
+		{0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

src/Infrastructure/BotSharp.Abstraction/Messaging/BotSharpMessageParser.cs

@@ -38,6 +38,10 @@ public static class BotSharpMessageParser
             {
                 targetType = typeof(TextMessage);
             }
+            else if (richType == RichTypeEnum.JsCode)
+            {
+                targetType = typeof(JsCodeTemplateMessage);
+            }
             else if (richType == RichTypeEnum.GenericTemplate)
             {
                 if (root.TryGetProperty("element_type", out element))

src/Infrastructure/BotSharp.Abstraction/Messaging/Enums/RichTypeEnum.cs

@@ -9,4 +9,6 @@ public static class RichTypeEnum
     public const string QuickReply = "quick_reply";
     public const string Text = "text";
     public const string Attachment = "attachment";
+
+    public const string JsCode = "js_code";
 }

src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/JsCodeTemplateMessage.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Abstraction.Messaging.Models.RichContent.Template;
+
+public class JsCodeTemplateMessage : IRichMessage
+{
+    [JsonPropertyName("rich_type")]
+    public string RichType => RichTypeEnum.JsCode;
+
+    [JsonPropertyName("text")]
+    public string Text { get; set; } = string.Empty;
+}

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -86,7 +86,6 @@
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\conversation.summary.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\response_with_function.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\translation_prompt.liquid" />
-    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_file_prompt.liquid" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-routing-fallback_to_router.fn.liquid" />
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\agent.json" />
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\instructions\instruction.liquid" />
@@ -97,6 +96,7 @@
     <None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
     <None Remove="data\plugins\config.json" />
 
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-select_file_instruction.liquid" />
     <None Remove="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\functions\get_weather.json" />
   </ItemGroup>
 
@@ -188,7 +188,7 @@
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\instructions\instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_file_prompt.liquid">
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-file-select_file_instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid">

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs

@@ -65,7 +65,7 @@ private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<Messag
             }).ToList();
 
             var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant;
-            var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "select_file_prompt";
+            var template = !string.IsNullOrWhiteSpace(options.Template) ? options.Template : "util-file-select_file_instruction";
 
             var foundAgent = db.GetAgent(agentId);
             var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, template);

src/Plugins/BotSharp.Plugin.ChartHandler/BotSharp.Plugin.ChartHandler.csproj

@@ -0,0 +1,30 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-chart-plot_chart.json" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-chart-plot_chart.fn.liquid" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-chart-plot_instruction.liquid" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-chart-plot_chart.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-chart-plot_chart.fn.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-chart-plot_instruction.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Infrastructure\BotSharp.Core\BotSharp.Core.csproj" />
+  </ItemGroup>
+</Project>

src/Plugins/BotSharp.Plugin.ChartHandler/ChartHandlerPlugin.cs

@@ -0,0 +1,18 @@
+using Microsoft.Extensions.Configuration;
+
+namespace BotSharp.Plugin.ChartHandler;
+
+public class ChartHandlerPlugin : IBotSharpPlugin
+{
+    public string Id => "9dacac1d-2e29-4f01-9d66-b0201f05a9fa";
+    public string Name => "Chart Plotter";
+    public string Description => "AI plots chart";
+    public string IconUrl => "https://cdn-icons-png.flaticon.com/512/423/423786.png";
+    public string[] AgentIds => [];
+
+    public void RegisterDI(IServiceCollection services, IConfiguration config)
+    {
+        services.AddScoped<IAgentUtilityHook, ChartHandlerUtilityHook>();
+    }
+
+}

src/Plugins/BotSharp.Plugin.ChartHandler/Enums/UtilityName.cs

@@ -0,0 +1,6 @@
+namespace BotSharp.Plugin.ChartHandler.Enums;
+
+public class UtilityName
+{
+    public const string ChartPlotter = "chart-plotter";
+}

src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs

@@ -0,0 +1,81 @@
+using BotSharp.Abstraction.Messaging.Models.RichContent.Template;
+
+namespace BotSharp.Plugin.ChartHandler.Functions;
+
+public class PlotChartFn : IFunctionCallback
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger<PlotChartFn> _logger;
+
+    public string Name => "util-chart-plot_chart";
+    public string Indication => "Plotting chart";
+
+
+    public PlotChartFn(
+        IServiceProvider services,
+        ILogger<PlotChartFn> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var convService = _services.GetRequiredService<IConversationService>();
+
+        var args = JsonSerializer.Deserialize<LlmContextIn>(message.FunctionArgs);
+        var agent = await agentService.GetAgent(message.CurrentAgentId);
+        var inst = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "util-chart-plot_instruction");
+        var innerAgent = new Agent
+        {
+            Id = agent.Id,
+            Name = agent.Name,
+            Instruction = inst,
+            TemplateDict = new Dictionary<string, object>
+            {
+                { "plotting_requirement", args?.PlottingRequirement ?? string.Empty },
+                { "chart_element_id", $"chart-{message.MessageId}" }
+            }
+        };
+
+        var response = await GetChatCompletion(innerAgent, [
+            new RoleDialogModel(AgentRole.User, "Please follow the instruction to generate the javascript code.")
+            {
+                CurrentAgentId = message.CurrentAgentId,
+                MessageId = message.MessageId
+            }
+        ]);
+
+        var obj = response.JsonContent<LlmContextOut>();
+        message.Content = obj?.GreetingMessage.IfNullOrEmptyAs("Here is the chart you ask for:");
+        message.RichContent = new RichContent<IRichMessage>
+        {
+            Recipient = new Recipient { Id = convService.ConversationId },
+            Message = new JsCodeTemplateMessage
+            {
+                Text = obj?.JsCode ?? string.Empty
+            }
+        };
+        message.StopCompletion = true;
+        return true;
+    }
+
+    private async Task<string> GetChatCompletion(Agent agent, List<RoleDialogModel> dialogs)
+    {
+        try
+        {
+            var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
+            var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", model: "gpt-4.1");
+            var response = await completion.GetChatCompletions(agent, dialogs);
+            return response.Content;
+        }
+        catch (Exception ex)
+        {
+            var error = $"Error when plotting chart. {ex.Message}";
+            _logger.LogWarning(ex, error);
+            return error;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.ChartHandler/Hooks/ChartHandlerUtilityHook.cs

@@ -0,0 +1,24 @@
+namespace BotSharp.Plugin.ChartHandler.Hooks;
+
+public class ChartHandlerUtilityHook : IAgentUtilityHook
+{
+    private const string PLOT_CHART_FN = "util-chart-plot_chart";
+
+    public void AddUtilities(List<AgentUtility> utilities)
+    {
+        var item = new AgentUtility
+        {
+            Category = "chart",
+            Name = UtilityName.ChartPlotter,
+            Items = [
+                new UtilityItem
+                {
+                    FunctionName = PLOT_CHART_FN,
+                    TemplateName = $"{PLOT_CHART_FN}.fn"
+                }
+            ]
+        };
+
+        utilities.Add(item);
+    }
+}

src/Plugins/BotSharp.Plugin.ChartHandler/LlmContext/LlmContextIn.cs

@@ -0,0 +1,10 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.ChartHandler.LlmContext;
+
+public class LlmContextIn
+{
+    [JsonPropertyName("plotting_requirement")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? PlottingRequirement { get; set; }
+}

src/Plugins/BotSharp.Plugin.ChartHandler/LlmContext/LlmContextOut.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.ChartHandler.LlmContext;
+
+public class LlmContextOut
+{
+    [JsonPropertyName("greeting_message")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? GreetingMessage { get; set; }
+
+    [JsonPropertyName("js_code")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? JsCode { get; set; }
+}

src/Plugins/BotSharp.Plugin.ChartHandler/Using.cs

@@ -0,0 +1,32 @@
+global using System;
+global using System.Collections.Generic;
+global using System.Text;
+global using System.Linq;
+global using System.Text.Json;
+global using System.Net.Mime;
+global using System.Threading.Tasks;
+global using Microsoft.Extensions.DependencyInjection;
+global using Microsoft.Extensions.Logging;
+global using BotSharp.Abstraction.Agents;
+global using BotSharp.Abstraction.Conversations;
+global using BotSharp.Abstraction.Plugins;
+global using BotSharp.Abstraction.Conversations.Models;
+global using BotSharp.Abstraction.Functions;
+global using BotSharp.Abstraction.Agents.Models;
+global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Files.Enums;
+global using BotSharp.Abstraction.Files.Models;
+global using BotSharp.Abstraction.Files;
+global using BotSharp.Abstraction.MLTasks;
+global using BotSharp.Abstraction.Utilities;
+global using BotSharp.Abstraction.Agents.Settings;
+global using BotSharp.Abstraction.Functions.Models;
+global using BotSharp.Abstraction.Repositories;
+global using BotSharp.Abstraction.Settings;
+global using BotSharp.Abstraction.Messaging;
+global using BotSharp.Abstraction.Messaging.Models.RichContent;
+global using BotSharp.Abstraction.Options;
+global using BotSharp.Core.Infrastructures;
+global using BotSharp.Plugin.ChartHandler.Enums;
+global using BotSharp.Plugin.ChartHandler.LlmContext;
+global using BotSharp.Plugin.ChartHandler.Hooks;

src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-chart-plot_chart.json

@@ -0,0 +1,14 @@
+{
+  "name": "util-chart-plot_chart",
+  "description": "If the user requests you plotting or generating charts, you can call this function to generate charts in any format that user requested.",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "plotting_requirement": {
+        "type": "string",
+        "description": "The requirement that user posted for plotting or generating charts."
+      }
+    },
+    "required": [ "plotting_requirement" ]
+  }
+}

src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_chart.fn.liquid

@@ -0,0 +1 @@
+Please call function util-chart-plot_chart if user wants to plot or generate charts.

src/Plugins/BotSharp.Plugin.ChartHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-chart-plot_instruction.liquid

@@ -0,0 +1,20 @@
+Please take a look at "Plotting Requirement" and generate a javascript code that can be used to render the charts on an html element.
+=== Plotting Requirement ===
+{{ plotting_requirement }}
+
+
+***** Important *****
+** Your output must be a single <script>...</script> block with everything needed inside.
+** You need to import Plotly.js to plot the charts. The script source should be "https://cdn.plot.ly/plotly-3.0.1.min.js".
+** You need to add the MODE bar for each chart you plot.
+** You must render the charts on the div html element with id {{ chart_element_id }}.
+** You must not create any new html element.
+** You must not apply any styles on any html element.
+** You must generate as less token as possible.
+
+*** Response Format ***
+You must output the response in the following JSON format:
+{
+    "greeting_message": "A short polite message that informs user that the charts have been generated.",
+    "js_code": "The javascript code that can generate the charts as requested."
+}

src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(TargetFramework)</TargetFramework>
@@ -9,10 +9,6 @@
     <GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
     <OutputPath>$(SolutionDir)packages</OutputPath>
   </PropertyGroup>
-
-  <ItemGroup>
-    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_edit_image_prompt.liquid" />
-  </ItemGroup>
 
   <ItemGroup>
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-file-generate_image.json" />

src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextIn.cs

@@ -12,10 +12,6 @@ public class LlmContextIn
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? ImageDescription { get; set; }
 
-    //[JsonPropertyName("image_url")]
-    //[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    //public string? ImageUrl { get; set; }
-
     [JsonPropertyName("image_urls")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public IEnumerable<string>? ImageUrls { get; set; }