-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathutils_preprocess.py
More file actions
140 lines (121 loc) · 4.33 KB
/
Copy pathutils_preprocess.py
File metadata and controls
140 lines (121 loc) · 4.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
import pandas as pd
import numpy as np
import nltk
def get_CorpusData(verbose=False):
with open("data/idwiki.txt", "r", encoding='UTF-8') as f:
data = f.read()
if verbose:
print("Data type:", type(data))
print("Number of letters:", len(data))
print("-------\nFirst 300 letters of the data\n-------")
display(data[0:300])
print("-------\nLast 300 letters of the data\n-------")
display(data[-300:])
return data
def split_to_sentences(data):
"""
Split data by linebreak "\n"
Args:
data: str
Returns:
A list of sentences
"""
sentences = data.split('\n')
sentences = [s.strip() for s in sentences]
sentences = [s for s in sentences if len(s) > 0]
return sentences
def tokenize_sentences(sentences):
"""
Tokenize sentences into tokens (words)
Args:
sentences: List of strings
Returns:
List of lists of tokens
"""
tokenized_sentences = []
for sentence in sentences:
sentence = sentence.lower()
tokenized = nltk.word_tokenize(sentence)
tokenized_sentences.append(tokenized)
return tokenized_sentences
def get_tokenized_data(data):
"""
Make a list of tokenized sentences
Args:
data: String
Returns:
List of lists of tokens
"""
sentences = split_to_sentences(data)
tokenized_sentences = tokenize_sentences(sentences)
return tokenized_sentences
def count_words(tokenized_sentences):
"""
Count the number of word appearence in the tokenized sentences
Args:
tokenized_sentences: List of lists of strings
Returns:
dict that maps word (str) to the frequency (int)
"""
word_counts = {}
for sentence in tokenized_sentences:
for token in sentence:
if token not in word_counts.keys():
word_counts[token] = 1
else:
word_counts[token] += 1
return word_counts
def get_words_with_nplus_frequency(tokenized_sentences, count_threshold):
"""
Find the words that appear N times or more
Args:
tokenized_sentences: List of lists of sentences
count_threshold: minimum number of occurrences for a word to be in the closed vocabulary.
Returns:
List of words that appear N times or more
"""
closed_vocab = []
word_counts = count_words(tokenized_sentences)
for word, cnt in word_counts.items():
if cnt >= count_threshold:
closed_vocab.append(word)
return closed_vocab
def replace_oov_words_by_unk(tokenized_sentences, vocabulary, unknown_token="<unk>"):
"""
Replace words not in the given vocabulary with '<unk>' token.
Args:
tokenized_sentences: List of lists of strings
vocabulary: List of strings that we will use
unknown_token: A string representing unknown (out-of-vocabulary) words
Returns:
List of lists of strings, with words not in the vocabulary replaced
"""
vocabulary = set(vocabulary)
replaced_tokenized_sentences = []
for sentence in tokenized_sentences:
replaced_sentence = []
for token in sentence:
if token in vocabulary:
replaced_sentence.append(token)
else:
replaced_sentence.append(unknown_token)
replaced_tokenized_sentences.append(replaced_sentence)
return replaced_tokenized_sentences
def preprocess_data(train_data, test_data, count_threshold):
"""
Preprocess data, i.e.,
- Find tokens that appear at least N times in the training data.
- Replace tokens that appear less than N times by "<unk>" both for training and test data.
Args:
train_data, test_data: List of lists of strings.
count_threshold: Words whose count is less than this are treated as unknown.
Returns:
Tuple of
- training data with low frequent words replaced by "<unk>"
- test data with low frequent words replaced by "<unk>"
- vocabulary of words that appear n times or more in the training data
"""
vocabulary = get_words_with_nplus_frequency(train_data, count_threshold)
train_data_replaced = replace_oov_words_by_unk(train_data, vocabulary)
test_data_replaced = replace_oov_words_by_unk(test_data, vocabulary)
return train_data_replaced, test_data_replaced, vocabulary