Skip to content

Commit 4cb5068

Browse files
committed
Update Submodule vendor/llama.cpp 448147d..71ad059
Signed-off-by: JamePeng <jame_peng@sina.com>
1 parent 283cd1b commit 4cb5068

7 files changed

Lines changed: 31 additions & 4 deletions

File tree

‎llama_cpp/llama.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ def __init__(
157157
n_rs_seq: int = 0,
158158
n_outputs_max: int = 0,
159159
n_outputs_max_per_seq: int = 1,
160+
moe_cache_size: int = 0,
160161
n_threads: Optional[int] = None,
161162
n_threads_batch: Optional[int] = None,
162163
ctx_type: int = llama_cpp_lib.llama_context_type.LLAMA_CONTEXT_TYPE_DEFAULT,
@@ -307,6 +308,9 @@ def __init__(
307308
arguments, including draft length, probability threshold,
308309
draft model, cache types and backend sampling.
309310
tokenizer: Optional tokenizer to override the default tokenizer from llama.cpp.
311+
moe_cache_size: Device cache size in bytes for MoE experts kept in host memory,
312+
split among devices like layers. Requires a GPU and MoE model;
313+
tensor parallelism is unsupported. 0 disables caching. Experimental.
310314
type_k: KV cache data type for K (default: f16)
311315
type_v: KV cache data type for V (default: f16)
312316
spm_infill: Use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this.
@@ -341,6 +345,8 @@ def __init__(
341345
self.verbose = verbose
342346
self.verbosity = verbosity
343347
self._stack = contextlib.ExitStack()
348+
if moe_cache_size < 0:
349+
raise ValueError("moe_cache_size must be nonnegative")
344350
self.rpc_servers = normalize_rpc_servers(rpc_servers)
345351
if isinstance(rpc_local_devices, (str, bytes)):
346352
raise TypeError("rpc_local_devices must be a sequence of device names")
@@ -577,6 +583,7 @@ def __init__(
577583

578584
# Context Params
579585
self.context_params = llama_cpp_lib.llama_context_default_params()
586+
self.context_params.moe_cache_size = moe_cache_size
580587
self.context_params.n_ctx = n_ctx
581588
self.context_params.n_batch = self.n_batch
582589
self.context_params.n_ubatch = min(self.n_batch, n_ubatch)
@@ -4641,6 +4648,7 @@ def __getstate__(self):
46414648
n_rs_seq=self.context_params.n_rs_seq,
46424649
n_outputs_max=self.context_params.n_outputs_max,
46434650
n_outputs_max_per_seq=self.context_params.n_outputs_max_per_seq,
4651+
moe_cache_size=self.context_params.moe_cache_size,
46444652
n_threads=self.context_params.n_threads,
46454653
n_threads_batch=self.context_params.n_threads_batch,
46464654
ctx_type=self.context_params.ctx_type,

‎llama_cpp/llama_cpp.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -961,6 +961,8 @@ class llama_sampler_seq_config(ctypes.Structure):
961961
# enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]
962962
# enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]
963963

964+
# size_t moe_cache_size; // device cache bytes for host-memory MoE experts, 0 = disabled [EXPERIMENTAL]
965+
964966
# // Abort callback
965967
# // if it returns true, execution of llama_decode() will be aborted
966968
# // currently works only with CPU execution
@@ -1023,6 +1025,8 @@ class llama_context_params(ctypes.Structure):
10231025
type_k (int): data type for K cache
10241026
type_v (int): data type for V cache
10251027
1028+
moe_cache_size (int): Device cache bytes for host-memory MoE experts, split among devices like layers; 0 disables it. Experimental.
1029+
10261030
abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted
10271031
abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback
10281032
@@ -1066,6 +1070,7 @@ class llama_context_params(ctypes.Structure):
10661070
cb_eval_user_data: ctypes.c_void_p
10671071
type_k: int
10681072
type_v: int
1073+
moe_cache_size: int
10691074
abort_callback: Callable[[ctypes.c_void_p], bool]
10701075
abort_callback_data: ctypes.c_void_p
10711076
embeddings: bool
@@ -1105,6 +1110,7 @@ class llama_context_params(ctypes.Structure):
11051110
("cb_eval_user_data", ctypes.c_void_p),
11061111
("type_k", ctypes.c_int),
11071112
("type_v", ctypes.c_int),
1113+
("moe_cache_size", ctypes.c_size_t),
11081114
("abort_callback", ggml_abort_callback),
11091115
("abort_callback_data", ctypes.c_void_p),
11101116
("embeddings", ctypes.c_bool),

‎llama_cpp/llama_speculative.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1032,15 +1032,18 @@ def _validate_speculative_vocab_compatibility(
10321032
def _copy_draft_context_params(
10331033
context_params: Any, pooling_type_unspecified: int
10341034
) -> Any:
1035-
"""Copy target parameters without inheriting embedding or pooling mode.
1035+
"""Copy target parameters without embedding/pooling mode or MoE caching.
10361036
10371037
This mirrors ``common_base_params_to_speculative`` in llama.cpp. Draft
10381038
contexts configure hidden-state extraction themselves and must not
1039-
inherit a target embedding request or its pooling behavior.
1039+
inherit a target embedding request or its pooling behavior. MoE caching
1040+
is reserved for the target context.
10401041
"""
10411042
params = type(context_params).from_buffer_copy(context_params)
10421043
params.embeddings = False
10431044
params.pooling_type = pooling_type_unspecified
1045+
# llama.cpp reserves the host-expert device cache for the target context.
1046+
params.moe_cache_size = 0
10441047
return params
10451048

10461049
def _init_model_draft_engine(

‎llama_cpp/server/model.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
357357
# KV Cache Quantization
358358
type_k=settings.type_k,
359359
type_v=settings.type_v,
360+
moe_cache_size=settings.moe_cache_size,
360361
# Tokenizer
361362
tokenizer=tokenizer,
362363
# Misc

‎llama_cpp/server/settings.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,12 @@ class ModelSettings(BaseSettings):
199199
default=10,
200200
description="Number of tokens to predict using the draft model.",
201201
)
202+
# MoE Expert Cache
203+
moe_cache_size: int = Field(
204+
default=0,
205+
ge=0,
206+
description="Device cache size in bytes for host-memory MoE experts; 0 disables it. Experimental.",
207+
)
202208
# KV Cache Quantization
203209
type_k: Optional[int] = Field(
204210
default=None,

‎tests/test_speculation.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,19 +166,22 @@ def test_speculative_output_limits_match_llama_cpp():
166166
assert speculative_output_limits(3, 4, 8) == (3, 3)
167167

168168

169-
def test_draft_context_does_not_inherit_target_embedding_mode():
169+
def test_draft_context_does_not_inherit_target_embedding_mode_or_moe_cache():
170170
target = llama_cpp_lib.llama_context_default_params()
171171
target.embeddings = True
172172
target.pooling_type = llama_cpp_lib.LLAMA_POOLING_TYPE_MEAN
173+
target.moe_cache_size = 64 * 1024 * 1024
173174

174175
draft = LlamaMTPDecoding._copy_draft_context_params(
175176
target, llama_cpp_lib.LLAMA_POOLING_TYPE_UNSPECIFIED
176177
)
177178

178179
assert target.embeddings is True
179180
assert target.pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_MEAN
181+
assert target.moe_cache_size == 64 * 1024 * 1024
180182
assert draft.embeddings is False
181183
assert draft.pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_UNSPECIFIED
184+
assert draft.moe_cache_size == 0
182185

183186

184187
def test_spec_config_validates_draft_runtime_arguments():

‎vendor/llama.cpp‎

Submodule llama.cpp updated 130 files

0 commit comments

Comments
 (0)