Repository navigation
Expand file tree
/
Copy pathBotTrainer.cs
More file actions
134 lines (114 loc) · 4.59 KB
/
Copy pathBotTrainer.cs
File metadata and controls
134 lines (114 loc) · 4.59 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BotSharp.Core.Abstractions;
using BotSharp.Platform.Abstractions;
using BotSharp.Platform.Models;
using BotSharp.Platform.Models.MachineLearning;
using DotNetToolkit;
using EntityFrameworkCore.BootKit;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
namespace BotSharp.Core.Engines
{
public class BotTrainer
{
private readonly Database dc;
private readonly string agentId;
private readonly IPlatformSettings settings;
public BotTrainer(IPlatformSettings setting)
{
this.settings = setting;
}
public BotTrainer(string agentId, Database dc)
{
this.dc = dc;
this.agentId = agentId;
}
public async Task<ModelMetaData> Train(AgentBase agent, BotTrainOptions options)
{
var data = new NlpDoc();
// Get NLP Provider
var config = (IConfiguration)AppDomain.CurrentDomain.GetData("Configuration");
var assemblies = (string[])AppDomain.CurrentDomain.GetData("Assemblies");
var platform = config.GetSection($"platformModuleName").Value;
var engine = this.settings.BotEngine;
string providerName = config.GetSection($"{engine}_{agent.Language}:Provider").Value;
var provider = TypeHelper.GetInstance(providerName, assemblies) as INlpProvider;
provider.Configuration = config.GetSection($"{engine}_{agent.Language}");
var pipeModel = new PipeModel
{
Name = providerName,
Class = provider.ToString(),
Meta = new JObject(),
Time = DateTime.UtcNow
};
await provider.Load(agent, pipeModel);
var settings = new PipeSettings
{
ProjectDir = options.AgentDir
};
settings.ModelDir = Path.Combine(options.AgentDir, options.Model);
if (!Directory.Exists(settings.ProjectDir))
{
Directory.CreateDirectory(settings.ProjectDir);
}
if (!Directory.Exists(settings.ModelDir))
{
Directory.CreateDirectory(settings.ModelDir);
}
var meta = new ModelMetaData
{
Platform = platform,
BotEngine = engine,
Language = agent.Language,
TrainingDate = DateTime.UtcNow,
Version = config.GetValue<String>($"Version"),
Pipeline = new List<PipeModel>() { pipeModel },
Model = settings.ModelDir
};
// pipe process
var pipelines = provider.Configuration.GetValue<String>($"pipe")
.Split(',')
.Select(x => x.Trim())
.ToList();
for (int pipeIdx = 0; pipeIdx < pipelines.Count; pipeIdx++)
{
Console.WriteLine("");
Console.WriteLine($"Executing {pipeModel.Name}");
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
var pipe = TypeHelper.GetInstance(pipelines[pipeIdx], assemblies) as INlpTrain;
// set configuration to current section
pipe.Configuration = provider.Configuration.GetSection(pipelines[pipeIdx]);
pipe.Settings = settings;
pipeModel = new PipeModel
{
Name = pipelines[pipeIdx],
Class = pipe.ToString(),
Time = DateTime.UtcNow
};
meta.Pipeline.Add(pipeModel);
Console.WriteLine(JsonConvert.SerializeObject(pipeModel, Formatting.Indented));
await pipe.Train(agent, data, pipeModel);
stopwatch.Stop();
Console.WriteLine($"Executed pipe {pipeModel.Name} elapsed {stopwatch.Elapsed}");
}
// save model meta data
var metaJson = JsonConvert.SerializeObject(meta, new JsonSerializerSettings
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
File.WriteAllText(Path.Combine(settings.ModelDir, "model-meta.json"), metaJson);
return meta;
}
}
}