Skip to content

Commit fbd3c6f

Browse files
Oceania2018Oceania2018
committed
1 parent 7fbff8c commit fbd3c6f

5 files changed

Lines changed: 63 additions & 18 deletions

File tree

‎src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingArgs.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,12 @@ public class RoutingArgs
99
public string Reason { get; set; } = string.Empty;
1010

1111
[JsonPropertyName("answer")]
12-
public string Answer { get; set; } = string.Empty;
12+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
13+
public string? Answer { get; set; }
1314

1415
[JsonPropertyName("agent")]
15-
public string AgentName { get; set; } = string.Empty;
16+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
17+
public string? AgentName { get; set; }
1618

1719
public override string ToString()
1820
{

‎src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetNextInstruction.cs‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,23 @@ public partial class RoutingService
99
{
1010
public async Task<FunctionCallFromLlm> GetNextInstruction(string prompt)
1111
{
12-
var responseFormat = _settings.EnableReasoning ?
12+
var responseRouteToAgent = _settings.EnableReasoning ?
1313
JsonSerializer.Serialize(new FunctionCallFromLlm()) :
1414
JsonSerializer.Serialize(new RoutingArgs
1515
{
16-
Function = "route_to_agent"
16+
Function = "route_to_agent",
17+
AgentName = ""
1718
});
18-
var content = $"{prompt} Response must be in JSON format {responseFormat}";
19+
20+
var responseResponseToUser = _settings.EnableReasoning ?
21+
JsonSerializer.Serialize(new FunctionCallFromLlm()) :
22+
JsonSerializer.Serialize(new RoutingArgs
23+
{
24+
Function = "response_to_user",
25+
Answer = ""
26+
});
27+
28+
var content = $"{prompt}\r\nIf need to route to specific agent, output in JSON {responseRouteToAgent}.\r\nIf you can handle the request directly, output in JSON {responseResponseToUser}.";
1929

2030
var state = _services.GetRequiredService<IConversationStateService>();
2131

@@ -29,8 +39,34 @@ public async Task<FunctionCallFromLlm> GetNextInstruction(string prompt)
2939
model: _settings.Model);
3040

3141
content = _routerInstance.Router.Instruction + "\r\n\r\n" + content + "\r\nResponse: ";
32-
var text = await completion.GetCompletion(content);
33-
response = new RoleDialogModel(AgentRole.Assistant, text);
42+
43+
int retryCount = 0;
44+
45+
while (retryCount < 3)
46+
{
47+
try
48+
{
49+
var text = await completion.GetCompletion(content);
50+
response = new RoleDialogModel(AgentRole.Assistant, text);
51+
52+
var pattern = @"\{(?:[^{}]|(?<open>\{)|(?<-open>\}))+(?(open)(?!))\}";
53+
response.Content = Regex.Match(response.Content, pattern).Value;
54+
args = JsonSerializer.Deserialize<FunctionCallFromLlm>(response.Content);
55+
break;
56+
}
57+
catch (Exception ex)
58+
{
59+
_logger.LogError($"{ex.Message}: {response.Content}");
60+
args.Function = "response_to_user";
61+
args.Answer = ex.Message;
62+
args.AgentName = _settings.RouterName;
63+
content += "\r\nPlease response in JSON format.";
64+
}
65+
finally
66+
{
67+
retryCount++;
68+
}
69+
}
3470
}
3571
else
3672
{

‎src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@ public async Task<RoleDialogModel> InstructLoop()
7777
loopCount++;
7878

7979
var prompt = _settings.EnableReasoning ? "Tell me the next step?" : "Which agent is suitable to handle user's request based on the CONVERSATION?";
80-
prompt += " Or you can handle without asking specific agent.";
8180
var inst = await GetNextInstruction(prompt);
8281
inst.Question = inst.Question ?? message;
8382

‎src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,18 @@ public class ChatCompletionProvider : IChatCompletion
2323
private readonly IServiceProvider _services;
2424
private readonly ILogger _logger;
2525
private readonly LlamaSharpSettings _settings;
26+
private readonly ITokenStatistics _tokenStatistics;
2627
private string _model;
2728

2829
public ChatCompletionProvider(IServiceProvider services,
2930
ILogger<ChatCompletionProvider> logger,
30-
LlamaSharpSettings settings)
31+
LlamaSharpSettings settings,
32+
ITokenStatistics tokenStatistics)
3133
{
3234
_services = services;
3335
_logger = logger;
3436
_settings = settings;
37+
_tokenStatistics = tokenStatistics;
3538
}
3639

3740
public string Provider => "llama-sharp";
@@ -41,11 +44,8 @@ public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> con
4144
var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.Content}")).Trim();
4245
content += $"\r\n{AgentRole.Assistant}: ";
4346

44-
var state = _services.GetRequiredService<IConversationStateService>();
45-
var model = state.GetState("model", _settings.DefaultModel);
46-
4747
var llama = _services.GetRequiredService<LlamaAiModel>();
48-
llama.LoadModel(model);
48+
llama.LoadModel(_model);
4949
var executor = llama.GetStatelessExecutor();
5050

5151
var inferenceParams = new InferenceParams()
@@ -65,11 +65,13 @@ public RoleDialogModel GetChatCompletions(Agent agent, List<RoleDialogModel> con
6565
_logger.LogInformation(prompt);
6666
}
6767

68+
_tokenStatistics.StartTimer();
6869
foreach (var response in executor.Infer(prompt, inferenceParams))
6970
{
7071
Console.Write(response);
7172
totalResponse += response;
7273
}
74+
_tokenStatistics.StopTimer();
7375

7476
foreach (var anti in inferenceParams.AntiPrompts)
7577
{

‎src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using LLama;
66
using LLama.Common;
77
using Microsoft.Extensions.DependencyInjection;
8+
using Microsoft.Extensions.Logging;
89
using System;
910
using System.Threading.Tasks;
1011

@@ -13,34 +14,39 @@ namespace BotSharp.Plugin.LLamaSharp.Providers;
1314
public class TextCompletionProvider : ITextCompletion
1415
{
1516
private readonly IServiceProvider _services;
17+
private readonly ILogger _logger;
1618
private readonly LlamaSharpSettings _settings;
19+
private readonly ITokenStatistics _tokenStatistics;
1720
private string _model;
1821
public string Provider => "llama-sharp";
1922

2023
public TextCompletionProvider(IServiceProvider services,
21-
LlamaSharpSettings settings)
24+
ILogger<TextCompletionProvider> logger,
25+
LlamaSharpSettings settings,
26+
ITokenStatistics tokenStatistics)
2227
{
2328
_services = services;
29+
_logger = logger;
2430
_settings = settings;
31+
_tokenStatistics = tokenStatistics;
2532
}
2633

2734
public Task<string> GetCompletion(string text)
2835
{
29-
var state = _services.GetRequiredService<IConversationStateService>();
30-
var model = state.GetState("model", _settings.DefaultModel);
31-
3236
var llama = _services.GetRequiredService<LlamaAiModel>();
33-
llama.LoadModel(model);
37+
llama.LoadModel(_model);
3438

3539
var executor = new InstructExecutor(llama.Model.CreateContext(llama.Params));
3640
var inferenceParams = new InferenceParams() { Temperature = 0.5f, MaxTokens = 128 };
3741

42+
_tokenStatistics.StartTimer();
3843
string totalResponse = "";
3944
foreach (var response in executor.Infer(text, inferenceParams))
4045
{
4146
Console.Write(response);
4247
totalResponse += response;
4348
}
49+
_tokenStatistics.StopTimer();
4450

4551
return Task.FromResult(totalResponse);
4652
}

0 commit comments

Comments
 (0)