Repository navigation
Expand file tree
/
Copy pathSegDecoderTagger.cs
More file actions
131 lines (115 loc) · 3.72 KB
/
Copy pathSegDecoderTagger.cs
File metadata and controls
131 lines (115 loc) · 3.72 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
using Bigtree.Algorithm.CRFLite.Decoder;
using System;
using System.Collections.Generic;
using System.Text;
namespace Bigtree.Algorithm.CRFLite
{
public class SegDecoderTagger : DecoderTagger
{
public SegDecoderTagger(int nbest, int this_crf_max_word_num = BaseUtils.DEFAULT_CRF_MAX_WORD_NUM)
: base(nbest, this_crf_max_word_num)
{
crf_max_word_num = this_crf_max_word_num;
}
int seg_termbuf_build(CRFSegOut term_buf)
{
term_buf.Clear();
//build raw result at first
var iRet = termbuf_build(term_buf);
if (iRet != BaseUtils.RETURN_SUCCESS)
{
return iRet;
}
//Then build token result
var term_len = 0;
var weight = 0.0;
var num = 0;
for (var i = 0; i < x_.Count; i++)
{
//Adding the length of current token
var strTag = term_buf.result_[i];
term_len += x_[i][0].Length;
weight += term_buf.weight_[i];
num++;
//Check if current term is the end of a token
if ((strTag.StartsWith("B_") == false &&
strTag.StartsWith("M_") == false) ||
i == x_.Count - 1)
{
var tkn = new SegToken();
tkn.Length = term_len;
tkn.Offset = term_buf.termTotalLength;
var spos = strTag.IndexOf('_');
if (spos < 0)
{
if (strTag == "NOR")
{
tkn.Tag = "";
}
else
{
tkn.Tag = strTag;
}
}
else
{
tkn.Tag = strTag.Substring(spos + 1);
}
term_buf.termTotalLength += term_len;
//Calculate each token's weight
switch (vlevel_)
{
case 0:
tkn.Weight = 0.0;
break;
case 2:
tkn.Weight = weight / num;
weight = 0.0;
num = 0;
break;
}
term_buf.tokenList.Add(tkn);
term_len = 0;
}
}
return BaseUtils.RETURN_SUCCESS;
}
public int output(CRFSegOut[] pout)
{
var n = 0;
var ret = 0;
if (nbest_ == 1)
{
//If only best result and no need probability, "next" is not to be used
ret = seg_termbuf_build(pout[0]);
if (ret < 0)
{
return ret;
}
}
else
{
//Fill the n best result
var iNBest = nbest_;
if (pout.Length < iNBest)
{
iNBest = pout.Length;
}
for (n = 0; n < iNBest; ++n)
{
ret = next();
if (ret < 0)
{
break;
}
ret = seg_termbuf_build(pout[n]);
if (ret < 0)
{
return ret;
}
}
}
return BaseUtils.RETURN_SUCCESS;
}
}
}