forked from leejet/stable-diffusion.cpp
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathgpt_oss_tokenizer.cpp
More file actions
91 lines (82 loc) · 2.85 KB
/
Copy pathgpt_oss_tokenizer.cpp
File metadata and controls
91 lines (82 loc) · 2.85 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
#include "gpt_oss_tokenizer.h"
#include "core/util.h"
#include "json.hpp"
#include "vocab/vocab.h"
void GPTOSSTokenizer::load_from_merges(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
auto byte_unicode_pairs = bytes_to_unicode();
byte_encoder = std::map<int, std::u32string>(byte_unicode_pairs.begin(), byte_unicode_pairs.end());
for (auto& pair : byte_unicode_pairs) {
byte_decoder[pair.second] = pair.first;
}
nlohmann::json vocab;
try {
vocab = nlohmann::json::parse(vocab_utf8_str);
} catch (const nlohmann::json::parse_error&) {
GGML_ABORT("invalid vocab json str");
}
for (const auto& [key, value] : vocab.items()) {
std::u32string token = utf8_to_utf32(key);
int i = value;
encoder[token] = i;
decoder[i] = token;
}
encoder_len = static_cast<int>(encoder.size());
for (auto& special_token : special_tokens) {
auto token = utf8_to_utf32(special_token);
encoder[token] = encoder_len;
decoder[encoder_len] = token;
encoder_len++;
}
encoder_len = static_cast<int>(encoder.size());
LOG_DEBUG("vocab size: %d", encoder_len);
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
for (const auto& merge : merges) {
size_t space_pos = merge.find(' ');
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
}
LOG_DEBUG("merges size %zu", merge_pairs.size());
int rank = 0;
for (const auto& merge : merge_pairs) {
bpe_ranks[merge] = rank++;
}
bpe_len = rank;
}
GPTOSSTokenizer::GPTOSSTokenizer(const std::string& merges_utf8_str, const std::string& vocab_utf8_str) {
BOS_TOKEN = "<|startoftext|>";
UNK_TOKEN = "<|endoftext|>";
EOS_TOKEN = "<|endoftext|>";
PAD_TOKEN = "<|endoftext|>";
BOS_TOKEN_ID = 199998;
EOS_TOKEN_ID = 199999;
UNK_TOKEN_ID = 199999;
PAD_TOKEN_ID = 199999;
special_tokens = {
"<|startoftext|>",
"<|endoftext|>",
"<|reserved_200000|>",
"<|reserved_200001|>",
"<|return|>",
"<|constrain|>",
"<|reserved_200004|>",
"<|channel|>",
"<|start|>",
"<|end|>",
"<|message|>",
"<|reserved_200009|>",
"<|reserved_200010|>",
"<|reserved_200011|>",
"<|call|>",
"<|reserved_200013|>",
"<|reserved_200014|>",
"<|reserved_200015|>",
"<|reserved_200016|>",
"<|reserved_200017|>",
"<|endofprompt|>",
};
if (merges_utf8_str.size() > 0) {
load_from_merges(merges_utf8_str, vocab_utf8_str);
} else {
load_from_merges(load_gpt_oss_merges(), load_gpt_oss_vocab_json());
}
}