Repository navigation
Expand file tree
/
Copy pathmain.cpp
More file actions
1945 lines (1825 loc) · 80.9 KB
/
Copy pathmain.cpp
File metadata and controls
1945 lines (1825 loc) · 80.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#include "chat.h"
#include "json-schema-to-grammar.h"
#include "llama.h"
#include "sampling.h"
#include <nlohmann/json.hpp>
#include <algorithm>
#include <chrono>
#include <cmath>
#include <cstdint>
#include <fstream>
#include <iostream>
#include <initializer_list>
#include <iterator>
#include <limits>
#include <memory>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
using json = nlohmann::ordered_json;
namespace {
constexpr int PROTOCOL_VERSION = 1;
constexpr size_t MAX_INPUT_LINE_BYTES = 16 * 1024 * 1024;
bool backend_initialized = false;
using sampler_ptr = std::unique_ptr<llama_sampler, decltype(&llama_sampler_free)>;
using model_ptr = std::unique_ptr<llama_model, decltype(&llama_model_free)>;
using context_ptr = std::unique_ptr<llama_context, decltype(&llama_free)>;
const std::vector<std::string> OPERATIONS = {
"protocol.describe",
"configuration.inspect",
"model.metadata",
"tokenizer.tokenize",
"tokenizer.token-to-piece",
"chat-template.render",
"grammar.json-schema-to-grammar",
"sampler.apply",
"decode.execute-plan",
"state.execute-script",
"sampler.bench",
"chat-template.bench",
"chat-parser.inspect",
"chat-parser.bench",
"reasoning-budget.inspect",
"decode.abort",
"decode.abort-recovery",
};
struct batch_entry {
llama_token token;
llama_pos position;
std::vector<llama_seq_id> sequence_ids;
bool request_logits;
};
// Owns every array referenced by the public llama_batch view. Keeping
// construction here makes the exact token/position/sequence/logits boundary
// explicit without relying on a higher-level helper.
class native_batch {
public:
explicit native_batch(std::vector<batch_entry> entries) : entries_(std::move(entries)) {
if (entries_.empty()) {
throw std::invalid_argument("batch must not be empty");
}
if (entries_.size() > static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
throw std::invalid_argument("batch contains too many entries");
}
tokens_.reserve(entries_.size());
positions_.reserve(entries_.size());
sequence_counts_.reserve(entries_.size());
sequence_pointers_.reserve(entries_.size());
logits_.reserve(entries_.size());
for (auto &entry : entries_) {
if (entry.sequence_ids.empty()) {
throw std::invalid_argument("each batch entry must have at least one sequence ID");
}
if (entry.sequence_ids.size() >
static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
throw std::invalid_argument("batch entry contains too many sequence IDs");
}
tokens_.push_back(entry.token);
positions_.push_back(entry.position);
sequence_counts_.push_back(static_cast<int32_t>(entry.sequence_ids.size()));
sequence_pointers_.push_back(entry.sequence_ids.data());
logits_.push_back(entry.request_logits ? 1 : 0);
}
}
native_batch(const native_batch &) = delete;
native_batch &operator=(const native_batch &) = delete;
llama_batch view() {
return {
static_cast<int32_t>(entries_.size()),
tokens_.data(),
nullptr,
positions_.data(),
sequence_counts_.data(),
sequence_pointers_.data(),
logits_.data(),
};
}
private:
std::vector<batch_entry> entries_;
std::vector<llama_token> tokens_;
std::vector<llama_pos> positions_;
std::vector<int32_t> sequence_counts_;
std::vector<llama_seq_id *> sequence_pointers_;
std::vector<int8_t> logits_;
};
template <typename T> T required(const json &value, const char *field) {
if (!value.contains(field)) {
throw std::invalid_argument(std::string("missing field: ") + field);
}
return value.at(field).get<T>();
}
template <typename T> T optional(const json &value, const char *field, T fallback) {
return value.contains(field) ? value.at(field).get<T>() : std::move(fallback);
}
void require_only_fields(const json &input, std::initializer_list<const char *> allowed) {
if (!input.is_object()) {
throw std::invalid_argument("operation input must be an object");
}
for (auto iterator = input.begin(); iterator != input.end(); ++iterator) {
const bool known = std::any_of(
allowed.begin(), allowed.end(),
[&](const char *field) { return iterator.key() == field; });
if (!known) {
throw std::invalid_argument("unknown input field: " + iterator.key());
}
}
}
json byte_array(const std::string &value) {
json result = json::array();
for (const unsigned char byte : value) {
result.push_back(byte);
}
return result;
}
json finite_number(float value) {
if (std::isfinite(value)) {
return json{{"value", value}, {"class", "finite"}};
}
if (std::isnan(value)) {
return json{{"value", nullptr}, {"class", "nan"}};
}
return json{
{"value", nullptr},
{"class", value > 0 ? "positive-infinity" : "negative-infinity"},
};
}
std::string trigger_type_name(common_grammar_trigger_type type) {
switch (type) {
case COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN:
return "token";
case COMMON_GRAMMAR_TRIGGER_TYPE_WORD:
return "word";
case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN:
return "pattern";
case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL:
return "pattern-full";
}
throw std::runtime_error("unknown grammar trigger type");
}
std::string vocab_type_name(enum llama_vocab_type type) {
switch (type) {
case LLAMA_VOCAB_TYPE_NONE:
return "none";
case LLAMA_VOCAB_TYPE_SPM:
return "spm";
case LLAMA_VOCAB_TYPE_BPE:
return "bpe";
case LLAMA_VOCAB_TYPE_WPM:
return "wpm";
case LLAMA_VOCAB_TYPE_UGM:
return "ugm";
case LLAMA_VOCAB_TYPE_RWKV:
return "rwkv";
case LLAMA_VOCAB_TYPE_PLAMO2:
return "plamo2";
}
throw std::runtime_error("unknown vocabulary type");
}
std::string rope_type_name(enum llama_rope_type type) {
switch (type) {
case LLAMA_ROPE_TYPE_NONE:
return "none";
case LLAMA_ROPE_TYPE_NORM:
return "norm";
case LLAMA_ROPE_TYPE_NEOX:
return "neox";
case LLAMA_ROPE_TYPE_MROPE:
return "mrope";
case LLAMA_ROPE_TYPE_VISION:
return "vision";
case LLAMA_ROPE_TYPE_IMROPE:
break;
}
throw std::runtime_error("model RoPE type is outside the approved shared projection");
}
void ensure_backend_initialized() {
if (!backend_initialized) {
llama_backend_init();
backend_initialized = true;
}
}
model_ptr load_model(const json &input, bool vocab_only_default) {
ensure_backend_initialized();
const std::string model_path = required<std::string>(input, "modelPath");
llama_model_params params = llama_model_default_params();
params.n_gpu_layers = optional<int32_t>(input, "nGpuLayers", 0);
params.vocab_only = optional<bool>(input, "vocabOnly", vocab_only_default);
params.use_mmap = optional<bool>(input, "useMmap", true);
params.use_mlock = optional<bool>(input, "useMlock", false);
params.check_tensors = optional<bool>(input, "checkTensors", false);
model_ptr model(llama_model_load_from_file(model_path.c_str(), params), llama_model_free);
if (!model) {
throw std::runtime_error("llama_model_load_from_file failed");
}
return model;
}
std::vector<llama_token> tokenize(const llama_vocab *vocab, const std::string &text,
bool add_special, bool parse_special) {
const size_t initial_size = text.size() + (add_special ? 2 : 0);
if (initial_size > static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
throw std::invalid_argument("text is too large to tokenize");
}
std::vector<llama_token> tokens(std::max<size_t>(initial_size, 1));
int32_t count =
llama_tokenize(vocab, text.data(), static_cast<int32_t>(text.size()), tokens.data(),
static_cast<int32_t>(tokens.size()), add_special, parse_special);
if (count == std::numeric_limits<int32_t>::min()) {
throw std::runtime_error("tokenization result exceeds int32_t");
}
if (count < 0) {
tokens.resize(static_cast<size_t>(-count));
count = llama_tokenize(vocab, text.data(), static_cast<int32_t>(text.size()), tokens.data(),
static_cast<int32_t>(tokens.size()), add_special, parse_special);
}
if (count < 0) {
throw std::runtime_error("llama_tokenize failed after resizing output");
}
tokens.resize(static_cast<size_t>(count));
return tokens;
}
std::string token_piece(const llama_vocab *vocab, llama_token token, bool special) {
std::vector<char> buffer(32);
int32_t count = llama_token_to_piece(vocab, token, buffer.data(), buffer.size(), 0, special);
if (count < 0) {
buffer.resize(static_cast<size_t>(-count));
count = llama_token_to_piece(vocab, token, buffer.data(), buffer.size(), 0, special);
}
if (count < 0) {
throw std::runtime_error("llama_token_to_piece failed after resizing output");
}
return std::string(buffer.data(), static_cast<size_t>(count));
}
json pieces_evidence(const llama_vocab *vocab, const std::vector<llama_token> &tokens,
bool special) {
json pieces = json::array();
for (const llama_token token : tokens) {
const std::string piece = token_piece(vocab, token, special);
pieces.push_back({{"token", token}, {"bytes", byte_array(piece)}});
}
return pieces;
}
json describe_protocol() {
json operations = json::array();
for (const auto &operation : OPERATIONS) {
operations.push_back(operation);
}
return {
{"protocolVersion", PROTOCOL_VERSION},
{"transport", "jsonl-stdin-stdout"},
{"operations", std::move(operations)},
};
}
context_ptr create_context(llama_model *model, llama_context_params params) {
context_ptr context(llama_init_from_model(model, params), llama_free);
if (!context) {
throw std::runtime_error("llama_init_from_model failed");
}
return context;
}
void configure_cpu_resident_context(llama_context_params ¶ms, const json &input) {
const int32_t threads = required<int32_t>(input, "threads");
const int32_t batch_threads = required<int32_t>(input, "batchThreads");
if (threads <= 0) {
throw std::invalid_argument("threads must be greater than zero");
}
if (batch_threads <= 0) {
throw std::invalid_argument("batchThreads must be greater than zero");
}
const bool offload_kqv = required<bool>(input, "offloadKqv");
const bool operation_offload = required<bool>(input, "operationOffload");
const std::string flash_attention = required<std::string>(input, "flashAttention");
if (offload_kqv || operation_offload || flash_attention != "off") {
throw std::invalid_argument("CPU-resident correctness requires offloadKqv=false, "
"operationOffload=false, and flashAttention=off");
}
params.n_threads = threads;
params.n_threads_batch = batch_threads;
params.offload_kqv = offload_kqv;
params.op_offload = operation_offload;
params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED;
}
void require_positive(uint32_t value, const char *field) {
if (value == 0) {
throw std::invalid_argument(std::string(field) + " must be greater than zero");
}
}
void validate_token(const llama_vocab *vocab, llama_token token, const char *field) {
const int32_t token_count = llama_vocab_n_tokens(vocab);
if (token < 0 || token >= token_count) {
throw std::invalid_argument(std::string(field) + " must be a valid model token in [0, " +
std::to_string(token_count) + ")");
}
}
std::vector<int32_t> validated_logit_indices(const json &input, const llama_vocab *vocab,
std::vector<int32_t> fallback) {
const auto indices =
optional<std::vector<int32_t>>(input, "logit_indices", std::move(fallback));
if (indices.empty()) {
throw std::invalid_argument("logit_indices must not be empty");
}
const int32_t token_count = llama_vocab_n_tokens(vocab);
for (const int32_t index : indices) {
if (index < 0 || index >= token_count) {
throw std::invalid_argument("logit index must be a valid model token in [0, " +
std::to_string(token_count) + ")");
}
}
return indices;
}
json selected_logits(const float *logits, const std::vector<int32_t> &indices) {
if (logits == nullptr) {
throw std::runtime_error("requested logits row is unavailable");
}
json values = json::array();
for (const int32_t index : indices) {
const json encoded = finite_number(logits[index]);
values.push_back({
{"index", index},
{"value", encoded.at("value")},
{"class", encoded.at("class")},
});
}
return values;
}
json decode_status(int32_t code) {
if (code != 0) {
throw std::runtime_error("llama_decode returned status " + std::to_string(code));
}
return {{"code", code}, {"class", "success"}};
}
json configuration_inspect(const json &input) {
model_ptr model = load_model(input, false);
const json context_input = required<json>(input, "context");
if (!context_input.is_object()) {
throw std::invalid_argument("context must be an object");
}
const uint32_t context_tokens = required<uint32_t>(context_input, "context_tokens");
const uint32_t batch_tokens = required<uint32_t>(context_input, "batch_tokens");
const uint32_t micro_batch_tokens = required<uint32_t>(context_input, "micro_batch_tokens");
const uint32_t sequences = required<uint32_t>(context_input, "sequences");
const int32_t threads = required<int32_t>(context_input, "threads");
const int32_t batch_threads = required<int32_t>(context_input, "batch_threads");
require_positive(context_tokens, "context.context_tokens");
require_positive(batch_tokens, "context.batch_tokens");
require_positive(micro_batch_tokens, "context.micro_batch_tokens");
require_positive(sequences, "context.sequences");
if (threads <= 0) {
throw std::invalid_argument("context.threads must be greater than zero");
}
if (batch_threads <= 0) {
throw std::invalid_argument("context.batch_threads must be greater than zero");
}
if (micro_batch_tokens > batch_tokens) {
throw std::invalid_argument(
"context.micro_batch_tokens must not exceed context.batch_tokens");
}
if (sequences > llama_max_parallel_sequences()) {
throw std::invalid_argument("context.sequences exceeds llama.cpp's maximum");
}
llama_context_params params = llama_context_default_params();
params.n_ctx = context_tokens;
params.n_batch = batch_tokens;
params.n_ubatch = micro_batch_tokens;
params.n_seq_max = sequences;
params.n_threads = threads;
params.n_threads_batch = batch_threads;
params.embeddings = required<bool>(context_input, "embeddings");
params.offload_kqv = required<bool>(context_input, "offload_kqv");
context_ptr context = create_context(model.get(), params);
return {
{"effective",
{
{"contextTokens", llama_n_ctx(context.get())},
{"contextTokensPerSequence", llama_n_ctx_seq(context.get())},
{"batchTokens", llama_n_batch(context.get())},
{"microBatchTokens", llama_n_ubatch(context.get())},
{"sequences", llama_n_seq_max(context.get())},
{"recurrentSequences", llama_n_rs_seq(context.get())},
{"threads", llama_n_threads(context.get())},
{"batchThreads", llama_n_threads_batch(context.get())},
}},
};
}
json decode_execute_plan(const json &input) {
model_ptr model = load_model(input, false);
const llama_vocab *vocab = llama_model_get_vocab(model.get());
if (vocab == nullptr) {
throw std::runtime_error("loaded model has no vocabulary");
}
const uint32_t context_tokens = required<uint32_t>(input, "context_tokens");
require_positive(context_tokens, "context_tokens");
const std::vector<json> batch_input = required<std::vector<json>>(input, "batch");
if (batch_input.empty()) {
throw std::invalid_argument("batch must not be empty");
}
const std::vector<int32_t> logit_indices = validated_logit_indices(input, vocab, {});
std::vector<batch_entry> entries;
entries.reserve(batch_input.size());
json plan = json::array();
llama_seq_id maximum_sequence_id = 0;
for (size_t batch_index = 0; batch_index < batch_input.size(); ++batch_index) {
const json &item = batch_input[batch_index];
if (!item.is_object()) {
throw std::invalid_argument("each batch entry must be an object");
}
const llama_token token = required<llama_token>(item, "token");
const llama_pos position = required<llama_pos>(item, "position");
const auto sequence_ids = required<std::vector<llama_seq_id>>(item, "sequence_ids");
const bool request_logits = required<bool>(item, "request_logits");
validate_token(vocab, token, "batch token");
if (position < 0 || static_cast<uint32_t>(position) >= context_tokens) {
throw std::invalid_argument("batch position must be within context_tokens");
}
if (sequence_ids.empty()) {
throw std::invalid_argument("each batch entry must have at least one sequence ID");
}
for (size_t index = 0; index < sequence_ids.size(); ++index) {
const llama_seq_id sequence_id = sequence_ids[index];
if (sequence_id < 0) {
throw std::invalid_argument("sequence IDs must be non-negative");
}
if (std::find(sequence_ids.begin(), sequence_ids.begin() + index, sequence_id) !=
sequence_ids.begin() + index) {
throw std::invalid_argument("sequence IDs within one batch entry must be unique");
}
maximum_sequence_id = std::max(maximum_sequence_id, sequence_id);
}
entries.push_back({token, position, sequence_ids, request_logits});
plan.push_back({
{"token", token},
{"position", position},
{"sequenceIds", sequence_ids},
{"requestLogits", request_logits},
});
}
const uint32_t sequences = static_cast<uint32_t>(maximum_sequence_id) + 1;
if (sequences > llama_max_parallel_sequences()) {
throw std::invalid_argument("batch sequence IDs exceed llama.cpp's maximum");
}
llama_context_params params = llama_context_default_params();
params.n_ctx = context_tokens;
params.n_seq_max = sequences;
configure_cpu_resident_context(params, input);
context_ptr context = create_context(model.get(), params);
native_batch batch(std::move(entries));
const int32_t status_code = llama_decode(context.get(), batch.view());
llama_synchronize(context.get());
const json status = decode_status(status_code);
json logits = json::array();
for (size_t batch_index = 0; batch_index < batch_input.size(); ++batch_index) {
if (!required<bool>(batch_input[batch_index], "request_logits")) {
continue;
}
if (batch_index > static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
throw std::runtime_error("batch logits index exceeds int32_t");
}
logits.push_back({
{"batchIndex", batch_index},
{"values",
selected_logits(llama_get_logits_ith(context.get(), static_cast<int32_t>(batch_index)),
logit_indices)},
});
}
return {
{"plan", std::move(plan)},
{"status", status},
{"logits", std::move(logits)},
};
}
json state_execute_script(const json &input) {
model_ptr model = load_model(input, false);
const llama_vocab *vocab = llama_model_get_vocab(model.get());
if (vocab == nullptr) {
throw std::runtime_error("loaded model has no vocabulary");
}
const std::vector<llama_token> prepare_tokens =
required<std::vector<llama_token>>(input, "prepare_tokens");
if (prepare_tokens.empty()) {
throw std::invalid_argument("prepare_tokens must not be empty");
}
if (prepare_tokens.size() > static_cast<size_t>(std::numeric_limits<llama_pos>::max())) {
throw std::invalid_argument("prepare_tokens is too large");
}
for (const llama_token token : prepare_tokens) {
validate_token(vocab, token, "prepare token");
}
const llama_seq_id sequence_id = required<llama_seq_id>(input, "sequence_id");
if (sequence_id < 0) {
throw std::invalid_argument("sequence_id must be non-negative");
}
const uint32_t sequences = static_cast<uint32_t>(sequence_id) + 1;
if (sequences > llama_max_parallel_sequences()) {
throw std::invalid_argument("sequence_id exceeds llama.cpp's maximum");
}
const std::vector<json> script = required<std::vector<json>>(input, "operations");
if (script.empty()) {
throw std::invalid_argument("operations must not be empty");
}
const std::vector<int32_t> logit_indices = validated_logit_indices(input, vocab, {0, 1, 2, 3});
uint32_t context_tokens = std::max<uint32_t>(32, static_cast<uint32_t>(prepare_tokens.size()));
for (const json &operation : script) {
if (!operation.is_object()) {
throw std::invalid_argument("each state operation must be an object");
}
const std::string type = required<std::string>(operation, "type");
if (type == "remove") {
const llama_pos start = required<llama_pos>(operation, "position_start");
const llama_pos end = required<llama_pos>(operation, "position_end");
if (start < 0 || end <= start) {
throw std::invalid_argument(
"remove requires a non-negative, non-empty position range");
}
} else if (type == "decode") {
const llama_token token = required<llama_token>(operation, "token");
const llama_pos position = required<llama_pos>(operation, "position");
validate_token(vocab, token, "decode token");
if (position < 0) {
throw std::invalid_argument("decode position must be non-negative");
}
if (position == std::numeric_limits<llama_pos>::max()) {
throw std::invalid_argument("decode position is too large");
}
context_tokens = std::max(context_tokens, static_cast<uint32_t>(position) + 1);
(void)required<bool>(operation, "request_logits");
} else {
throw std::invalid_argument("unsupported state operation: " + type);
}
}
llama_context_params params = llama_context_default_params();
params.n_ctx = context_tokens;
params.n_seq_max = sequences;
configure_cpu_resident_context(params, input);
context_ptr context = create_context(model.get(), params);
std::vector<batch_entry> prepare_entries;
prepare_entries.reserve(prepare_tokens.size());
for (size_t index = 0; index < prepare_tokens.size(); ++index) {
prepare_entries.push_back({
prepare_tokens[index],
static_cast<llama_pos>(index),
{sequence_id},
index + 1 == prepare_tokens.size(),
});
}
native_batch prepare_batch(std::move(prepare_entries));
decode_status(llama_decode(context.get(), prepare_batch.view()));
llama_synchronize(context.get());
llama_memory_t memory = llama_get_memory(context.get());
if (memory == nullptr) {
throw std::runtime_error("context has no sequence memory");
}
json operation_results = json::array();
json logits = json::array();
for (size_t operation_index = 0; operation_index < script.size(); ++operation_index) {
const json &operation = script[operation_index];
const std::string type = required<std::string>(operation, "type");
if (type == "remove") {
llama_synchronize(context.get());
const llama_pos start = required<llama_pos>(operation, "position_start");
const llama_pos end = required<llama_pos>(operation, "position_end");
const bool removed = llama_memory_seq_rm(memory, sequence_id, start, end);
operation_results.push_back({
{"index", operation_index},
{"type", type},
{"positionStart", start},
{"positionEnd", end},
{"success", removed},
});
continue;
}
const llama_token token = required<llama_token>(operation, "token");
const llama_pos position = required<llama_pos>(operation, "position");
const bool request_logits = required<bool>(operation, "request_logits");
native_batch decode_batch({{token, position, {sequence_id}, request_logits}});
const int32_t status_code = llama_decode(context.get(), decode_batch.view());
llama_synchronize(context.get());
const json status = decode_status(status_code);
operation_results.push_back({
{"index", operation_index},
{"type", type},
{"token", token},
{"position", position},
{"requestLogits", request_logits},
{"status", status},
});
if (request_logits) {
logits.push_back({
{"operationIndex", operation_index},
{"values", selected_logits(llama_get_logits_ith(context.get(), 0), logit_indices)},
});
}
}
llama_synchronize(context.get());
return {
{"operations", std::move(operation_results)},
{"state",
{
{"sequenceId", sequence_id},
{"positionMin", llama_memory_seq_pos_min(memory, sequence_id)},
{"positionMax", llama_memory_seq_pos_max(memory, sequence_id)},
}},
{"logits", std::move(logits)},
};
}
struct abort_probe_context {
model_ptr model;
context_ptr context;
llama_token token;
std::vector<int32_t> logit_indices;
};
abort_probe_context prepare_abort_probe(const json &input, bool require_logits) {
const int32_t n_gpu_layers = required<int32_t>(input, "nGpuLayers");
const bool use_mmap = required<bool>(input, "useMmap");
const bool use_mlock = required<bool>(input, "useMlock");
if (n_gpu_layers != 0 || !use_mmap || use_mlock) {
throw std::invalid_argument(
"CPU abort parity requires nGpuLayers=0, useMmap=true, and useMlock=false");
}
const uint32_t context_tokens = required<uint32_t>(input, "contextTokens");
require_positive(context_tokens, "contextTokens");
model_ptr model = load_model(input, false);
const llama_vocab *vocab = llama_model_get_vocab(model.get());
if (vocab == nullptr) {
throw std::runtime_error("loaded model has no vocabulary");
}
const llama_token token = required<llama_token>(input, "token");
validate_token(vocab, token, "token");
std::vector<int32_t> logit_indices;
if (require_logits) {
logit_indices = required<std::vector<int32_t>>(input, "logitIndices");
if (logit_indices.empty()) {
throw std::invalid_argument("logitIndices must not be empty");
}
const int32_t token_count = llama_vocab_n_tokens(vocab);
for (const int32_t index : logit_indices) {
if (index < 0 || index >= token_count) {
throw std::invalid_argument(
"logit index must be a valid model token in [0, " +
std::to_string(token_count) + ")");
}
}
}
llama_context_params params = llama_context_default_params();
params.n_ctx = context_tokens;
params.n_seq_max = 1;
configure_cpu_resident_context(params, input);
context_ptr context = create_context(model.get(), params);
return {std::move(model), std::move(context), token, std::move(logit_indices)};
}
bool abort_when_signaled(void *data) {
return *static_cast<const bool *>(data);
}
void observe_presignaled_abort(llama_context *context, llama_token token) {
bool signaled = true;
llama_set_abort_callback(context, abort_when_signaled, &signaled);
native_batch batch({{token, 0, {0}, true}});
const int32_t status_code = llama_decode(context, batch.view());
llama_synchronize(context);
llama_set_abort_callback(context, nullptr, nullptr);
if (status_code != 2) {
throw std::runtime_error("pre-signaled llama_decode returned status " +
std::to_string(status_code) + " instead of aborted status 2");
}
}
json decode_abort(const json &input) {
require_only_fields(input,
{"modelPath", "modelId", "model_id", "nGpuLayers", "useMmap",
"useMlock", "contextTokens", "threads", "batchThreads",
"offloadKqv", "operationOffload", "flashAttention", "token",
"fixturePaths", "engineConfiguration"});
abort_probe_context probe = prepare_abort_probe(input, false);
observe_presignaled_abort(probe.context.get(), probe.token);
return {{"class", "aborted"}};
}
json decode_abort_recovery(const json &input) {
require_only_fields(input,
{"modelPath", "modelId", "model_id", "nGpuLayers", "useMmap",
"useMlock", "contextTokens", "threads", "batchThreads",
"offloadKqv", "operationOffload", "flashAttention", "token",
"logitIndices", "fixturePaths", "engineConfiguration"});
abort_probe_context probe = prepare_abort_probe(input, true);
observe_presignaled_abort(probe.context.get(), probe.token);
llama_memory_t memory = llama_get_memory(probe.context.get());
if (memory == nullptr) {
throw std::runtime_error("context has no sequence memory");
}
// This is the production recovery boundary: clear all logical memory state
// without paying to zero-fill the backing storage.
llama_memory_clear(memory, false);
native_batch recovery_batch({{probe.token, 0, {0}, true}});
const int32_t status_code = llama_decode(probe.context.get(), recovery_batch.view());
llama_synchronize(probe.context.get());
const json recovery_status = decode_status(status_code);
return {
{"abort", {{"class", "aborted"}}},
{"recovery",
{
{"status", recovery_status},
{"logits",
selected_logits(llama_get_logits_ith(probe.context.get(), 0),
probe.logit_indices)},
}},
};
}
json model_metadata(const json &input) {
model_ptr model = load_model(input, false);
const llama_vocab *vocab = llama_model_get_vocab(model.get());
if (vocab == nullptr) {
throw std::runtime_error("loaded model has no vocabulary");
}
return {
{"sizeBytes", llama_model_size(model.get())},
{"parameterCount", llama_model_n_params(model.get())},
{"dimensions",
{
{"contextTrain", llama_model_n_ctx_train(model.get())},
{"embedding", llama_model_n_embd(model.get())},
{"layers", llama_model_n_layer(model.get())},
{"attentionHeads", llama_model_n_head(model.get())},
{"kvAttentionHeads", llama_model_n_head_kv(model.get())},
{"slidingWindow", llama_model_n_swa(model.get())},
}},
{"architecture",
{
{"ropeType", rope_type_name(llama_model_rope_type(model.get()))},
{"recurrent", llama_model_is_recurrent(model.get())},
{"hybrid", llama_model_is_hybrid(model.get())},
}},
{"vocabulary",
{
{"type", vocab_type_name(llama_vocab_type(vocab))},
{"tokenCount", llama_vocab_n_tokens(vocab)},
{"addBos", llama_vocab_get_add_bos(vocab)},
{"specialTokens",
{
{"bos", llama_vocab_bos(vocab)},
{"eos", llama_vocab_eos(vocab)},
{"newline", llama_vocab_nl(vocab)},
}},
}},
{"metadataCount", llama_model_meta_count(model.get())},
};
}
json tokenizer_tokenize(const json &input) {
model_ptr model = load_model(input, true);
const llama_vocab *vocab = llama_model_get_vocab(model.get());
if (vocab == nullptr) {
throw std::runtime_error("loaded model has no vocabulary");
}
const std::string text = required<std::string>(input, "text");
const bool add_special = optional<bool>(input, "addSpecial", false);
const bool parse_special = optional<bool>(input, "parseSpecial", false);
const bool include_pieces = optional<bool>(input, "includePieces", true);
const bool piece_special = optional<bool>(input, "pieceSpecial", true);
const std::vector<llama_token> tokens = tokenize(vocab, text, add_special, parse_special);
json evidence = {
{"inputBytes", byte_array(text)},
{"tokens", tokens},
{"addSpecial", add_special},
{"parseSpecial", parse_special},
};
if (include_pieces) {
evidence["pieces"] = pieces_evidence(vocab, tokens, piece_special);
evidence["pieceSpecial"] = piece_special;
}
return evidence;
}
json tokenizer_token_to_piece(const json &input) {
require_only_fields(input, {"modelPath", "modelId", "model_id", "tokens", "special",
"lstrip", "fixturePaths", "engineConfiguration"});
const bool special = required<bool>(input, "special");
if (!special) {
throw std::invalid_argument(
"the production token-to-piece boundary requires special=true");
}
const json lstrip = required<json>(input, "lstrip");
if (!lstrip.is_null()) {
throw std::invalid_argument(
"the production token-to-piece boundary requires lstrip=null");
}
model_ptr model = load_model(input, true);
const llama_vocab *vocab = llama_model_get_vocab(model.get());
if (vocab == nullptr) {
throw std::runtime_error("loaded model has no vocabulary");
}
const std::vector<llama_token> tokens = required<std::vector<llama_token>>(input, "tokens");
if (tokens.empty()) {
throw std::invalid_argument("tokens must not be empty");
}
json pieces = json::array();
json concatenated_bytes = json::array();
for (const llama_token token : tokens) {
validate_token(vocab, token, "token");
const std::string piece = token_piece(vocab, token, true);
const json bytes = byte_array(piece);
pieces.push_back(bytes);
for (const auto &byte : bytes) {
concatenated_bytes.push_back(byte);
}
}
return {
{"pieces", std::move(pieces)},
{"concatenatedBytes", std::move(concatenated_bytes)},
};
}
llama_sampler *make_sampler(const json &config) {
const std::string type = required<std::string>(config, "type");
if (type == "top-k") {
return llama_sampler_init_top_k(required<int32_t>(config, "k"));
}
if (type == "top-p") {
return llama_sampler_init_top_p(required<float>(config, "p"),
optional<size_t>(config, "minKeep", 1));
}
if (type == "min-p") {
return llama_sampler_init_min_p(required<float>(config, "p"),
optional<size_t>(config, "minKeep", 1));
}
if (type == "typical") {
return llama_sampler_init_typical(required<float>(config, "p"),
optional<size_t>(config, "minKeep", 1));
}
if (type == "temperature") {
return llama_sampler_init_temp(required<float>(config, "temperature"));
}
if (type == "temperature-ext") {
return llama_sampler_init_temp_ext(required<float>(config, "temperature"),
required<float>(config, "delta"),
required<float>(config, "exponent"));
}
if (type == "xtc") {
return llama_sampler_init_xtc(
required<float>(config, "probability"), required<float>(config, "threshold"),
optional<size_t>(config, "minKeep", 1), optional<uint32_t>(config, "seed", 0));
}
if (type == "top-n-sigma") {
return llama_sampler_init_top_n_sigma(required<float>(config, "n"));
}
if (type == "penalties") {
return llama_sampler_init_penalties(
required<int32_t>(config, "lastN"), required<float>(config, "repeat"),
required<float>(config, "frequency"), required<float>(config, "presence"));
}
throw std::invalid_argument("unsupported sampler type: " + type);
}
json sampler_apply(const json &input) {
const auto candidate_input = required<std::vector<json>>(input, "candidates");
if (candidate_input.empty()) {
throw std::invalid_argument("candidates must not be empty");
}
std::vector<llama_token_data> candidates;
candidates.reserve(candidate_input.size());
for (const auto &candidate : candidate_input) {
candidates.push_back({
required<llama_token>(candidate, "id"),
required<float>(candidate, "logit"),
optional<float>(candidate, "probability", 0.0F),
});
}
llama_token_data_array data = {candidates.data(), candidates.size(), -1, false};
llama_sampler_chain_params chain_params = llama_sampler_chain_default_params();
chain_params.no_perf = true;
sampler_ptr chain(llama_sampler_chain_init(chain_params), llama_sampler_free);
if (!chain) {
throw std::runtime_error("llama_sampler_chain_init failed");
}
for (const auto &config : optional<std::vector<json>>(input, "samplers", {})) {
llama_sampler *sampler = make_sampler(config);
if (sampler == nullptr) {
throw std::runtime_error("sampler constructor returned null");
}
llama_sampler_chain_add(chain.get(), sampler);
}
const json selection = input.value("selection", json{{"type", "distribution"}, {"seed", 0}});
const std::string selection_type = required<std::string>(selection, "type");
if (selection_type == "distribution") {
llama_sampler_chain_add(chain.get(),
llama_sampler_init_dist(optional<uint32_t>(selection, "seed", 0)));
} else if (selection_type == "greedy") {
llama_sampler_chain_add(chain.get(), llama_sampler_init_greedy());
} else if (selection_type != "none") {
throw std::invalid_argument("unsupported selection type: " + selection_type);
}
for (const llama_token token :
optional<std::vector<llama_token>>(input, "acceptedTokens", {})) {
llama_sampler_accept(chain.get(), token);
}
llama_sampler_apply(chain.get(), &data);
json transformed = json::array();
for (size_t index = 0; index < data.size; ++index) {
const auto logit = finite_number(data.data[index].logit);
const auto probability = finite_number(data.data[index].p);
transformed.push_back({
{"id", data.data[index].id},
{"logit", logit.at("value")},
{"logitClass", logit.at("class")},
{"probability", probability.at("value")},
{"probabilityClass", probability.at("class")},
});
}
const bool has_selection = data.selected >= 0 && static_cast<size_t>(data.selected) < data.size;
return {
{"candidates", std::move(transformed)},
{"sorted", data.sorted},
{"selectedIndex", has_selection ? json(data.selected) : json(nullptr)},
{"selectedToken", has_selection ? json(data.data[data.selected].id) : json(nullptr)},
{"selection", selection},
};
}
int64_t positive_elapsed_nanoseconds(const std::chrono::steady_clock::time_point &start,
const std::chrono::steady_clock::time_point &end) {