-
Notifications
You must be signed in to change notification settings - Fork 6
Expand file tree
/
Copy pathtest_main.cpp
More file actions
74 lines (69 loc) · 1.63 KB
/
test_main.cpp
File metadata and controls
74 lines (69 loc) · 1.63 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#include "llama-sb.h"
#include <cstdio>
#include <cstring>
static void print_usage(int, char ** argv) {
printf("\nexample usage:\n");
printf("\n %s -m model.gguf [-n n_predict] [-ngl n_gpu_layers] [prompt]\n", argv[0]);
printf("\n");
}
int main(int argc, char ** argv) {
// path to the model gguf file
std::string model_path;
// prompt to generate text from
std::string prompt = "Happy friday";
// number of tokens to predict
int n_predict = 32;
// parse command line arguments
int i = 1;
for (; i < argc; i++) {
if (strcmp(argv[i], "-m") == 0) {
if (i + 1 < argc) {
model_path = argv[++i];
} else {
print_usage(argc, argv);
return 1;
}
} else if (strcmp(argv[i], "-n") == 0) {
if (i + 1 < argc) {
try {
n_predict = std::stoi(argv[++i]);
} catch (...) {
print_usage(argc, argv);
return 1;
}
} else {
print_usage(argc, argv);
return 1;
}
} else {
// prompt starts here
break;
}
}
if (model_path.empty()) {
print_usage(argc, argv);
return 1;
}
if (i < argc) {
prompt = argv[i++];
for (; i < argc; i++) {
prompt += " ";
prompt += argv[i];
}
}
Llama llama;
if (llama.construct(model_path, 1024, 1024, -1)) {
LlamaIter iter;
llama.set_max_tokens(n_predict);
llama.generate(iter, prompt);
while (iter._has_next) {
auto out = llama.next(iter);
printf("\033[33m");
printf("%s\n", out.c_str());
printf("\n\033[0m");
}
} else {
fprintf(stderr, "ERR: %s\n", llama.last_error());
}
return 0;
}