Repository navigation
Expand file tree
/
Copy pathRegexTokenizer.cs
More file actions
124 lines (104 loc) · 3.87 KB
/
Copy pathRegexTokenizer.cs
File metadata and controls
124 lines (104 loc) · 3.87 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
/*
* BotSharp.NLP Library
* Copyright (C) 2018 Haiping Chen
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace BotSharp.NLP.Tokenize
{
/// <summary>
/// Regular-Expression Tokenizers
/// </summary>
public class RegexTokenizer : ITokenizer
{
/// <summary>
/// Tokenize a text into a sequence of alphabetic and non-alphabetic characters
/// </summary>
public const string WORD_PUNC = @"[^\w\s]+|\w+";
/// <summary>
/// Tokenize a string, treating any sequence of blank lines as a delimiter.
/// Blank lines are defined as lines containing no characters, except for space or tab characters.
/// options.IsGap = true
/// </summary>
public const string BLANK_LINE = @"\s*\n\s*\n\s*";
/// <summary>
/// Tokenize a string on whitespace (space, tab, newline).
/// In general, users should use the string ``split()`` method instead.
/// options.IsGap = true
/// </summary>
public const string WHITE_SPACE = @"\s+";
private Regex _regex;
public List<Token> Tokenize(string sentence, TokenizationOptions options)
{
string pattern = options.Pattern;
if (options.SpecialWords != null)
{
options.SpecialWords.ForEach(r =>
{
sentence = Regex.Replace(sentence, r, " " + r);
});
pattern = String.Join("|", options.SpecialWords) + "|" + pattern;
}
_regex = new Regex(pattern);
var matches = _regex.Matches(sentence).Cast<Match>().ToArray();
options.IsGap = new string[] { WHITE_SPACE, BLANK_LINE }.Contains(pattern);
if (options.IsGap)
{
int pos = 0;
var tokens = new Token[matches.Length + 1];
for (int span = 0; span <= matches.Length; span++)
{
var token = new Token
{
Text = (span == matches.Length) ? sentence.Substring(pos) : sentence.Substring(pos, matches[span].Index - pos),
Start = pos
};
token.Text = token.Text.Trim();
tokens[span] = token;
if (span < matches.Length)
{
pos = matches[span].Index + 1;
}
}
return tokens.ToList();
}
else
{
var m = matches.Select(x => new Token
{
Text = x.Value,
Start = x.Index
}).ToList();
if(options.SpecialWords != null)
{
int offset = 0;
m.ForEach(t =>
{
if (options.SpecialWords.Contains(t.Text))
{
offset++;
}
t.Start = t.Start - offset;
});
}
return m;
}
}
}
}