Repository navigation
Expand file tree
/
Copy pathFasttextDataReader.cs
More file actions
52 lines (46 loc) · 1.53 KB
/
Copy pathFasttextDataReader.cs
File metadata and controls
52 lines (46 loc) · 1.53 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace BotSharp.NLP.Corpus
{
/// <summary>
/// Fasttext labeled data reader
/// </summary>
public class FasttextDataReader
{
public List<Sentence> Read(ReaderOptions options)
{
if (String.IsNullOrEmpty(options.LabelPrefix))
{
options.LabelPrefix = "__label__";
}
var sentences = new List<Sentence>();
using (StreamReader reader = new StreamReader(Path.Combine(options.DataDir, options.FileName)))
{
while (!reader.EndOfStream)
{
string line = reader.ReadLine();
if (!String.IsNullOrEmpty(line))
{
var ms = Regex.Matches(line, options.LabelPrefix + @"\S+")
.Cast<Match>()
.ToList();
var text = line.Substring(ms.Last().Index + ms.Last().Length + 1);
ms.ForEach(m =>
{
sentences.Add(new Sentence
{
Label = m.Value.Substring(options.LabelPrefix.Length),
Text = text
});
});
}
}
}
return sentences;
}
}
}