Skip to content

Commit 27fddd0

Browse files
JamePengJamePeng
committed
Update Submodule vendor/llama.cpp 4d19b28..cb30059
Signed-off-by: JamePeng <jame_peng@sina.com>
1 parent 4a973e3 commit 27fddd0

3 files changed

Lines changed: 23 additions & 5 deletions

File tree

‎llama_cpp/llama.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ def __init__(
121121
n_cpu_moe: int = 0,
122122
split_mode: int = llama_cpp_lib.llama_split_mode.LLAMA_SPLIT_MODE_LAYER,
123123
load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_AUTO,
124+
tensor_read_lazy: int = llama_cpp_lib.llama_tensor_read_lazy.LLAMA_TENSOR_READ_LAZY_AUTO,
124125
main_gpu: int = 0,
125126
tensor_split: Optional[List[float]] = None,
126127
kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None,
@@ -408,6 +409,7 @@ def __init__(
408409
self.model_params.n_gpu_layers = self._parse_n_gpu_layers(n_gpu_layers)
409410
self.model_params.split_mode = split_mode
410411
self.model_params.load_mode = load_mode
412+
self.model_params.tensor_read_lazy = tensor_read_lazy
411413
self.model_params.main_gpu = main_gpu
412414
self.tensor_split = tensor_split
413415
self._c_tensor_split = None
@@ -4391,6 +4393,7 @@ def __getstate__(self):
43914393
n_cpu_moe=self.n_cpu_moe,
43924394
split_mode=self.model_params.split_mode,
43934395
load_mode=self.model_params.load_mode,
4396+
tensor_read_lazy=self.model_params.tensor_read_lazy,
43944397
main_gpu=self.model_params.main_gpu,
43954398
tensor_split=self.tensor_split,
43964399
kv_overrides=self.kv_overrides,

‎llama_cpp/llama_cpp.py‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,13 +75,13 @@
7575

7676
# define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN
7777
LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN
78-
# define LLAMA_SESSION_VERSION 9
79-
LLAMA_SESSION_VERSION = 9
78+
# define LLAMA_SESSION_VERSION 10
79+
LLAMA_SESSION_VERSION = 10
8080

8181
# define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ
8282
LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ
83-
# define LLAMA_STATE_SEQ_VERSION 2
84-
LLAMA_STATE_SEQ_VERSION = 2
83+
# define LLAMA_STATE_SEQ_VERSION 3
84+
LLAMA_STATE_SEQ_VERSION = 3
8585

8686
# struct llama_vocab;
8787
llama_vocab_p = NewType("llama_vocab_p", int)
@@ -536,6 +536,16 @@ def llama_load_mode_name(load_mode: int) -> bytes:
536536
def llama_load_mode_from_str(str: ctypes.c_char_p) -> int:
537537
...
538538

539+
# enum llama_tensor_read_lazy {
540+
# LLAMA_TENSOR_READ_LAZY_OFF = 0, // always read the whole tensor up front
541+
# LLAMA_TENSOR_READ_LAZY_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)
542+
# LLAMA_TENSOR_READ_LAZY_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap)
543+
# };
544+
class llama_tensor_read_lazy(enum.IntEnum):
545+
LLAMA_TENSOR_READ_LAZY_OFF = 0 # always read the whole tensor up front
546+
LLAMA_TENSOR_READ_LAZY_AUTO = 1 # lazy only for marked tensors larger than 4 GiB (requires mmap)
547+
LLAMA_TENSOR_READ_LAZY_ON = 2 # read the rows of tensors marked by the arch on demand (requires mmap)
548+
539549
# enum llama_context_type {
540550
# LLAMA_CONTEXT_TYPE_DEFAULT = 0,
541551
# LLAMA_CONTEXT_TYPE_MTP = 1,
@@ -783,6 +793,8 @@ class llama_model_tensor_buft_override(ctypes.Structure):
783793
# enum llama_split_mode split_mode; // how to split the model across multiple GPUs
784794
# enum llama_load_mode load_mode; // how to load the model
785795

796+
# enum llama_tensor_read_lazy tensor_read_lazy; // on-demand reading of tensors marked by the arch
797+
786798
# // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
787799
# int32_t main_gpu;
788800

@@ -817,6 +829,7 @@ class llama_model_params(ctypes.Structure):
817829
n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers
818830
split_mode (int): how to split the model across multiple GPUs
819831
load_mode (int): how to load the model
832+
tensor_read_lazy (int): on-demand reading of tensors marked by the arch
820833
main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored
821834
tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
822835
progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted.
@@ -835,6 +848,7 @@ class llama_model_params(ctypes.Structure):
835848
n_gpu_layers: int
836849
split_mode: int
837850
load_mode: int
851+
tensor_read_lazy: int
838852
main_gpu: int
839853
tensor_split: CtypesArray[ctypes.c_float]
840854
progress_callback: Callable[[float, ctypes.c_void_p], bool]
@@ -853,6 +867,7 @@ class llama_model_params(ctypes.Structure):
853867
("n_gpu_layers", ctypes.c_int32),
854868
("split_mode", ctypes.c_int),
855869
("load_mode", ctypes.c_int),
870+
("tensor_read_lazy", ctypes.c_int),
856871
("main_gpu", ctypes.c_int32),
857872
("tensor_split", ctypes.POINTER(ctypes.c_float)),
858873
("progress_callback", llama_progress_callback),

‎vendor/llama.cpp‎

Submodule llama.cpp updated 173 files

0 commit comments

Comments
 (0)