@@ -157,6 +157,7 @@ def __init__(
157157 n_rs_seq : int = 0 ,
158158 n_outputs_max : int = 0 ,
159159 n_outputs_max_per_seq : int = 1 ,
160+ moe_cache_size : int = 0 ,
160161 n_threads : Optional [int ] = None ,
161162 n_threads_batch : Optional [int ] = None ,
162163 ctx_type : int = llama_cpp_lib .llama_context_type .LLAMA_CONTEXT_TYPE_DEFAULT ,
@@ -307,6 +308,9 @@ def __init__(
307308 arguments, including draft length, probability threshold,
308309 draft model, cache types and backend sampling.
309310 tokenizer: Optional tokenizer to override the default tokenizer from llama.cpp.
311+ moe_cache_size: Device cache size in bytes for MoE experts kept in host memory,
312+ split among devices like layers. Requires a GPU and MoE model;
313+ tensor parallelism is unsupported. 0 disables caching. Experimental.
310314 type_k: KV cache data type for K (default: f16)
311315 type_v: KV cache data type for V (default: f16)
312316 spm_infill: Use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this.
@@ -341,6 +345,8 @@ def __init__(
341345 self .verbose = verbose
342346 self .verbosity = verbosity
343347 self ._stack = contextlib .ExitStack ()
348+ if moe_cache_size < 0 :
349+ raise ValueError ("moe_cache_size must be nonnegative" )
344350 self .rpc_servers = normalize_rpc_servers (rpc_servers )
345351 if isinstance (rpc_local_devices , (str , bytes )):
346352 raise TypeError ("rpc_local_devices must be a sequence of device names" )
@@ -577,6 +583,7 @@ def __init__(
577583
578584 # Context Params
579585 self .context_params = llama_cpp_lib .llama_context_default_params ()
586+ self .context_params .moe_cache_size = moe_cache_size
580587 self .context_params .n_ctx = n_ctx
581588 self .context_params .n_batch = self .n_batch
582589 self .context_params .n_ubatch = min (self .n_batch , n_ubatch )
@@ -4641,6 +4648,7 @@ def __getstate__(self):
46414648 n_rs_seq = self .context_params .n_rs_seq ,
46424649 n_outputs_max = self .context_params .n_outputs_max ,
46434650 n_outputs_max_per_seq = self .context_params .n_outputs_max_per_seq ,
4651+ moe_cache_size = self .context_params .moe_cache_size ,
46444652 n_threads = self .context_params .n_threads ,
46454653 n_threads_batch = self .context_params .n_threads_batch ,
46464654 ctx_type = self .context_params .ctx_type ,
0 commit comments