forked from jeniyat/StackOverflowNER
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathloader_so.py
More file actions
173 lines (114 loc) · 4.51 KB
/
Copy pathloader_so.py
File metadata and controls
173 lines (114 loc) · 4.51 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
import json
import sys
def Merge_Label(inputFile):
merging_dict={}
merging_dict["Library_Function"]="Function"
merging_dict["Function_Name"]="Function"
merging_dict["Class_Name"]="Class"
merging_dict["Library_Class"]="Class"
merging_dict["Library_Variable"]="Variable"
merging_dict["Variable_Name"]="Variable"
merging_dict["Website"]="Website"
merging_dict["Organization"]="Website"
modified_file=inputFile[:-4]+"_merged_labels.txt"
Fout=open(modified_file,"w")
line_count=0
for line in open(inputFile):
line_count+=1
# print("line: in Merge_Label: utils_so: ",line)
# print(inputFile,":", line_count)
line_values=line.strip().split()
if len(line_values)<2:
opline=line
Fout.write(opline)
continue
gold_word=line_values[0]
gold_label=line_values[1]
raw_word=line_values[2]
raw_label=line_values[3]
#print(line_values)
if gold_word!=raw_word:
print("wrong mapping: ", line)
word=gold_word
label=gold_label
if label=="O":
opline=line
Fout.write(opline)
continue
# print(label)
label_split=label.split("-",1)
label_prefix=label_split[0]
label_name=label_split[1]
#print(label_name)
if label_name in merging_dict:
label_name=merging_dict[label_name]
#print(label_name)
new_label=label_prefix+"-"+label_name
#opline=word+" "+new_label+"\n"
opline=word+" "+new_label+" "+raw_word+" "+raw_label+"\n"
Fout.write(opline)
Fout.close()
return modified_file
return modified_file
def loader_so_text(path, merge_tag=True, replace_low_freq_tags=True):
if merge_tag:
path=Merge_Label(path)
set_of_selected_tags = []
if replace_low_freq_tags:
sorted_entity_list = ["Class","Class_Name", "Library_Class", "Application", "Library_Variable", "Variable_Name", "Variable", "User_Interface_Element", "Code_Block", "Library_Function","Function_Name", "Function", "Language", "Library", "Data_Structure", "Data_Type", "File_Type", "File_Name", "Version", "HTML_XML_Tag", "Device", "Operating_System", "User_Name", "Website", "Output_Block", "Error_Name", "Algorithm", "Organization", "Keyboard_IP", "Licence", "Organization"]
set_of_selected_tags.extend(sorted_entity_list[0:-6])
if 'Algorithm' not in set_of_selected_tags: set_of_selected_tags.append('Algorithm')
sentence = [] #list of words in the current sentence in formate each word list looks like [word, markdow tag name, mark down tag, NER tag]
sentences = [] #list of sentences
count_question=0
count_answer=0
max_len=0
for line in open(path):
if line.startswith("Question_ID"):
count_question+=1
if line.startswith("Answer_to_Question_ID"):
count_answer+=1
if line.strip()=="":
if len(sentence) > 0:
output_line = " ".join(w[0] for w in sentence)
if "code omitted for annotation" in output_line and "CODE_BLOCK :" in output_line:
sentence = []
continue
elif "omitted for annotation" in output_line and "OP_BLOCK :" in output_line:
sentence = []
continue
elif "Question_URL :" in output_line:
sentence = []
continue
elif "Question_ID :" in output_line:
sentence=[]
continue
else:
sentences.append(sentence)
if len(sentence)>max_len:
max_len=len(sentence)
sentence=[]
else:
line_values=line.strip().split()
gold_word=line_values[0]
gold_label=line_values[1]
raw_word=line_values[2]
raw_label=line_values[3]
gold_word=" ".join(gold_word.split('-----'))
gold_label_name= gold_label.replace("B-","").replace("I-","")
if gold_label_name not in set_of_selected_tags:
gold_label="O"
word_info=[gold_word, raw_label, gold_label]
sentence.append(word_info)
print("------------------------------------------------------------")
print("Number of questions in ", path, " : ", count_question)
print("Number of answers in ", path, " : ", count_answer)
print("Number of sentences in ", path, " : ", len(sentences))
print("Max len sentences has", max_len, "words")
print("------------------------------------------------------------")
return sentences
if __name__ == '__main__':
path_to_file = "../../resources/annotated_ner_data/StackOverflow/train.txt"
merge_tag= True
replace_low_freq_tags= True
all_sentneces = loader_so_text(path_to_file, merge_tag, replace_low_freq_tags)