Merge pull request #1242 from iceljc/features/refine-tokenizer

Features/refine tokenizer

Author: iceljc

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs

@@ -6,19 +6,31 @@ public class TokenStatsModel
     public string Model { get; set; }
     public string Prompt { get; set; }
 
-    #region Input
+    #region Text token
     public int TextInputTokens { get; set; }
     public int CachedTextInputTokens { get; set; }
+    public int TextOutputTokens { get; set; }
+    #endregion
+
+    #region Audio token
     public int AudioInputTokens { get; set; }
     public int CachedAudioInputTokens { get; set; }
+    public int AudioOutputTokens { get; set; }
     #endregion
 
-    #region Output
-    public int TextOutputTokens { get; set; }
-    public int AudioOutputTokens { get; set; }
+    #region Image token
+    public int ImageInputTokens { get; set; }
+    public int CachedImageInputTokens { get; set; }
+    public int ImageOutputTokens { get; set; }
     #endregion
 
+    #region Image
+    public int ImageGenerationCount { get; set; }
+    public float ImageGenerationUnitCost { get; set; }
+    #endregion
 
-    public int TotalInputTokens => TextInputTokens + CachedTextInputTokens + AudioInputTokens + CachedAudioInputTokens;
-    public int TotalOutputTokens => TextOutputTokens + AudioOutputTokens;
+    public int TotalInputTokens => TextInputTokens + CachedTextInputTokens 
+                                + AudioInputTokens + CachedAudioInputTokens
+                                + ImageInputTokens + CachedImageInputTokens;
+    public int TotalOutputTokens => TextOutputTokens + AudioOutputTokens + ImageOutputTokens;
 }

src/Infrastructure/BotSharp.Abstraction/Files/Proccessors/IFileProcessor.cs

@@ -1,7 +1,5 @@
 using BotSharp.Abstraction.Files.Options;
 using BotSharp.Abstraction.Files.Responses;
-using BotSharp.Abstraction.Knowledges.Options;
-using BotSharp.Abstraction.Knowledges.Responses;
 
 namespace BotSharp.Abstraction.Files.Proccessors;
 
@@ -11,7 +9,4 @@ public interface IFileProcessor
 
     Task<FileHandleResponse> HandleFilesAsync(Agent agent, string text, IEnumerable<InstructFileModel> files, FileHandleOptions? options = null)
         => throw new NotImplementedException();
-
-    Task<FileKnowledgeResponse> GetFileKnowledgeAsync(FileBinaryDataModel file, FileKnowledgeHandleOptions? options = null)
-        => throw new NotImplementedException();
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/IPhraseCollection.cs

@@ -0,0 +1,12 @@
+using BotSharp.Abstraction.Knowledges.Options;
+using BotSharp.Abstraction.Knowledges.Responses;
+
+namespace BotSharp.Abstraction.Knowledges.Processors;
+
+public interface IKnowledgeProcessor
+{
+    public string Provider { get; }
+
+    Task<FileKnowledgeResponse> GetFileKnowledgeAsync(FileBinaryDataModel file, FileKnowledgeHandleOptions? options = null)
+        => throw new NotImplementedException();
+}

src/Infrastructure/BotSharp.Abstraction/Knowledges/IPhraseService.cs

@@ -106,27 +106,37 @@ public class ImageSetting
     public ImageVariationSetting? Variation { get; set; }
 }
 
+/// <summary>
+/// Image generationg setting
+/// </summary>
 public class ImageGenerationSetting
 {
-    public ModelSettingBase? Style { get; set; }
-    public ModelSettingBase? Size { get; set; }
-    public ModelSettingBase? Quality { get; set; }
-    public ModelSettingBase? ResponseFormat { get; set; }
-    public ModelSettingBase? Background { get; set; }
+    /// <summary>
+    /// Parameters: quality, size, style, background, response format
+    /// </summary>
+    public Dictionary<string, ModelParamSetting>? Parameters { get; set; }
 }
 
+/// <summary>
+/// Image editing setting
+/// </summary>
 public class ImageEditSetting
 {
-    public ModelSettingBase? Size { get; set; }
-    public ModelSettingBase? Quality { get; set; }
-    public ModelSettingBase? ResponseFormat { get; set; }
-    public ModelSettingBase? Background { get; set; }
+    /// <summary>
+    /// Parameters: quality, size, background, response format
+    /// </summary>
+    public Dictionary<string, ModelParamSetting>? Parameters { get; set; }
 }
 
+/// <summary>
+/// Image variation setting
+/// </summary>
 public class ImageVariationSetting
 {
-    public ModelSettingBase? Size { get; set; }
-    public ModelSettingBase? ResponseFormat { get; set; }
+    /// <summary>
+    /// Parameters: size, response format
+    /// </summary>
+    public Dictionary<string, ModelParamSetting>? Parameters { get; set; }
 }
 #endregion
 
@@ -136,15 +146,21 @@ public class AudioSetting
     public AudioTranscriptionSetting? Transcription { get; set; }
 }
 
+/// <summary>
+/// Audio transcription setting
+/// </summary>
 public class AudioTranscriptionSetting
 {
     public float? Temperature { get; set; }
-    public ModelSettingBase? ResponseFormat { get; set; }
-    public ModelSettingBase? Granularity { get; set; }
+
+    /// <summary>
+    /// Parameters: granularity, response format
+    /// </summary>
+    public Dictionary<string, ModelParamSetting>? Parameters { get; set; }
 }
 #endregion
 
-public class ModelSettingBase
+public class ModelParamSetting
 {
     public string? Default { get; set; }
     public IEnumerable<string>? Options { get; set; }
@@ -156,15 +172,36 @@ public class ModelSettingBase
 /// </summary>
 public class LlmCostSetting
 {
-    // Input
+    #region Text token
     public float TextInputCost { get; set; } = 0f;
     public float CachedTextInputCost { get; set; } = 0f;
+    public float TextOutputCost { get; set; } = 0f;
+    #endregion
+
+    #region Audio token
     public float AudioInputCost { get; set; } = 0f;
     public float CachedAudioInputCost { get; set; } = 0f;
-
-    // Output
-    public float TextOutputCost { get; set; } = 0f;
     public float AudioOutputCost { get; set; } = 0f;
+    #endregion
+
+    #region Image token
+    public float ImageInputCost { get; set; } = 0f;
+    public float CachedImageInputCost { get; set; } = 0f;
+    public float ImageOutputCost { get; set; } = 0f;
+    #endregion
+
+    #region Image
+    public IList<LlmImageCost>? ImageCosts { get; set; }
+    #endregion
+}
+
+public class LlmImageCost
+{
+    /// <summary>
+    /// Attributes: e.g., [quality]: "medium", [size] = "1024x1024"
+    /// </summary>
+    public Dictionary<string, string> Attributes { get; set; } = [];
+    public float Cost { get; set; } = 0f;
 }
 
 public enum LlmModelType

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/SearchPhrasesResult.cs

@@ -1,14 +1,30 @@
+using BotSharp.Abstraction.MLTasks.Settings;
+
 namespace BotSharp.Abstraction.MLTasks.Utilities;
 
 public static class LlmUtility
 {
+    public static string? GetModelParameter(IDictionary<string, ModelParamSetting>? settings, string key, string curVal)
+    {
+        string? res = null;
+
+        if (settings != null
+            && settings.TryGetValue(key, out var value)
+            && value != null)
+        {
+            res = VerifyModelParameter(curVal, value.Default, value.Options);
+        }
+
+        return res;
+    }
+
     public static string? VerifyModelParameter(string? curVal, string? defaultVal, IEnumerable<string>? options = null)
     {
         if (options.IsNullOrEmpty())
         {
             return curVal.IfNullOrEmptyAs(defaultVal);
         }
 
-        return options.Contains(curVal) ? curVal : defaultVal;
+        return options!.Contains(curVal) ? curVal : defaultVal;
     }
 }

src/Infrastructure/BotSharp.Abstraction/Knowledges/Processors/IKnowledgeProcessor.cs

@@ -80,6 +80,9 @@ public class StatsCount
 {
     [JsonPropertyName("agent_call_count")]
     public long AgentCallCount { get; set; }
+
+    [JsonPropertyName("image_generation_total_count")]
+    public int ImageGenerationTotalCount { get; set; }
 }
 
 public class StatsLlmCost
@@ -95,4 +98,7 @@ public class StatsLlmCost
 
     [JsonPropertyName("completion_total_cost")]
     public float CompletionTotalCost { get; set; }
+
+    [JsonPropertyName("image_generation_total_cost")]
+    public float ImageGenerationTotalCost { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -29,6 +29,7 @@ public string Interval
 public class StatsCountDelta
 {
     public int AgentCallCountDelta { get; set; }
+    public int ImageGenerationTotalCountDelta { get; set; }
 }
 
 public class StatsLlmCostDelta
@@ -37,4 +38,5 @@ public class StatsLlmCostDelta
     public int CompletionTokensDelta { get; set; }
     public float PromptTotalCostDelta { get; set; }
     public float CompletionTotalCostDelta { get; set; }
+    public float ImageGenerationTotalCostDelta { get; set; }
 }