Skip to content

Commit c58bd96

Browse files
committed
fix(Llama): use output embedding dimensions and respect non-causal microbatch limits
- Read embeddings using n_embd_out instead of the hidden dimension to avoid truncating projected outputs, such as EmbeddingGemma 2's 768-dimensional vectors. - Limit non-causal embedding batches to n_ubatch so each input fits in a single microbatch, preventing native assertions and preserving full-context attention. Signed-off-by: JamePeng <jame_peng@sina.com>
1 parent 6740fbe commit c58bd96

2 files changed

Lines changed: 6 additions & 2 deletions

File tree

‎llama_cpp/llama.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3138,6 +3138,9 @@ def embed(
31383138

31393139
ctx = self._ctx.ctx
31403140
n_batch = self.n_batch
3141+
# Bidirectional attention must see the entire input in one microbatch.
3142+
if not self._ctx.get_causal_attn():
3143+
n_batch = min(n_batch, self._ctx.n_ubatch())
31413144
n_ctx = self._n_ctx
31423145
n_seq_max = self.context_params.n_seq_max
31433146

@@ -3149,7 +3152,7 @@ def embed(
31493152
out_dim = (
31503153
llama_cpp_lib.llama_model_n_cls_out(self._model.model)
31513154
if is_rank
3152-
else self.n_embd()
3155+
else self.n_embd_out()
31533156
)
31543157

31553158
# Preserve the historical bool API while accepting llama.cpp's integer

‎tests/test_state.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,8 @@ def test_embedding_request_cleans_generation_state_and_preserves_result_order(mo
149149
llm.context_params = SimpleNamespace(embeddings=True, n_seq_max=2, no_perf=True)
150150
llm.n_batch = 8
151151
llm.pooling_type = lambda: lib.LLAMA_POOLING_TYPE_MEAN
152-
llm.n_embd = lambda: 2
152+
llm.n_embd_out = lambda: 2
153+
llm._ctx.get_causal_attn = lambda: True
153154
llm._model = SimpleNamespace(model=object())
154155
llm._batch = SimpleNamespace(reset=Mock(), n_tokens=lambda: 0, add_sequence=Mock())
155156
llm._ctx.decode = Mock(return_value=1 if failure == "decode" else 0)

0 commit comments

Comments
 (0)