-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathutils_perplexity.py
More file actions
29 lines (25 loc) · 979 Bytes
/
Copy pathutils_perplexity.py
File metadata and controls
29 lines (25 loc) · 979 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
from libraries import utils_ngram as ung
def calculate_perplexity(sentence, n_gram_counts, n_plus1_gram_counts, vocabulary_size, k=1.0):
"""
Calculate perplexity for a list of sentences
Args:
sentence: List of strings
n_gram_counts: Dictionary of counts of (n+1)-grams
n_plus1_gram_counts: Dictionary of counts of (n+1)-grams
vocabulary_size: number of unique words in the vocabulary
k: Positive smoothing constant
Returns:
Perplexity score
"""
n = len(list(n_gram_counts.keys())[0])
sentence = ["<s>"] * n + sentence + ["<e>"]
sentence = tuple(sentence)
N = len(sentence)
product_pi = 1.0
for t in range(n, N):
n_gram = sentence[t-n:t]
word = sentence[t]
probability = ung.estimate_probability(word, n_gram, n_gram_counts, n_plus1_gram_counts, vocabulary_size, k=k)
product_pi *= 1/probability
perplexity = product_pi**(1/N)
return perplexity