Repository navigation
Expand file tree
/
Copy pathBotTrainer.cs
More file actions
124 lines (105 loc) · 4.05 KB
/
Copy pathBotTrainer.cs
File metadata and controls
124 lines (105 loc) · 4.05 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Abstractions;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
namespace BotSharp.Core.Engines
{
public class BotTrainer
{
private readonly Database dc;
private readonly string agentId;
public BotTrainer()
{
}
public BotTrainer(string agentId, Database dc)
{
this.dc = dc;
this.agentId = agentId;
}
public async Task<ModelMetaData> Train(AgentBase agent, BotTrainOptions options)
{
var data = new NlpDoc();
// Get NLP Provider
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var platform = config.GetSection($"platform").Value;
var engine = config.GetSection($"{platform}:botEngine").Value;
string providerName = config.GetSection($"{engine}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpProvider;
provider.Configuration = config.GetSection(engine);
var pipeModel = new PipeModel
{
Name = providerName,
Class = provider.ToString(),
Meta = new JObject(),
Time = DateTime.UtcNow
};
await provider.Load(agent, pipeModel);
var settings = new PipeSettings
{
ProjectDir = options.AgentDir
};
settings.ModelDir = Path.Combine(options.AgentDir, options.Model);
if (!Directory.Exists(settings.ProjectDir))
{
Directory.CreateDirectory(settings.ProjectDir);
}
if (!Directory.Exists(settings.ModelDir))
{
Directory.CreateDirectory(settings.ModelDir);
}
var meta = new ModelMetaData
{
Platform = platform,
BotEngine = engine,
Language = agent.Language,
TrainingDate = DateTime.UtcNow,
Version = config.GetValue<String>($"Version"),
Pipeline = new List<PipeModel>() { pipeModel },
Model = settings.ModelDir
};
// pipe process
var pipelines = provider.Configuration.GetValue<String>($"pipe")
.Split(',')
.Select(x => x.Trim())
.ToList();
for (int pipeIdx = 0; pipeIdx < pipelines.Count; pipeIdx++)
{
var pipe = TypeHelper.GetInstance(pipelines[pipeIdx], assemblies) as INlpTrain;
// set configuration to current section
pipe.Configuration = provider.Configuration.GetSection(pipelines[pipeIdx]);
pipe.Settings = settings;
pipeModel = new PipeModel
{
Name = pipelines[pipeIdx],
Class = pipe.ToString(),
Time = DateTime.UtcNow
};
meta.Pipeline.Add(pipeModel);
await pipe.Train(agent, data, pipeModel);
}
// save model meta data
var metaJson = JsonConvert.SerializeObject(meta, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
File.WriteAllText(Path.Combine(settings.ModelDir, "model-meta.json"), metaJson);
Console.WriteLine(metaJson);
return meta;
}
}
}