Repository navigation
Expand file tree
/
Copy pathmagevl.py
More file actions
77 lines (57 loc) · 3.19 KB
/
Copy pathmagevl.py
File metadata and controls
77 lines (57 loc) · 3.19 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
from __future__ import annotations
from typing import Callable, Iterable, TYPE_CHECKING
import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import MmprojModel, ModelBase, gguf
@ModelBase.register("LlavaOnevision2ForConditionalGeneration")
@ModelBase.example("microsoft/Mage-VL-Cowork")
class MageVLVisionModel(MmprojModel):
# a rope pair uses 2 different angles: the graph runs 2 ropes on heads permuted to [evens | odds]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert self.hparams_vision is not None
self.n_embd = self.find_vparam(["hidden_size"])
self.n_head = self.find_vparam(["num_attention_heads"])
self.d_head = self.n_embd // self.n_head
d_half = self.d_head // 2
assert d_half % 16 == 0, "the 4:6:6 split of the rope requires d_head to be a multiple of 32"
# 4:6:6 split for t:h:w
n_t = 4 * (d_half // 16)
n_s = 6 * (d_half // 16)
theta = float(self.find_vparam(["rope_theta"]))
freq_t = 1.0 / theta ** (torch.arange(n_t, dtype=torch.float64) / n_t)
freq_s = 1.0 / theta ** (torch.arange(n_s, dtype=torch.float64) / n_s)
freq = torch.cat([freq_t, freq_s, freq_s])
self.rope_freq = torch.cat([freq, freq])
def pair_axis(i: int) -> int:
dim = (2 * i) % d_half
return 0 if dim < n_t else 1 if dim < n_t + n_s else 2
self.rope_pairs = sorted(range(d_half), key=pair_axis)
def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MAGEVL)
self.gguf_writer.add_vision_attention_layernorm_eps(self.find_vparam(["layer_norm_eps"]))
self.gguf_writer.add_vision_spatial_merge_size(self.find_vparam(["spatial_merge_size"]))
self.gguf_writer.add_vision_min_pixels(int(self.preprocessor_config["min_pixels"]))
self.gguf_writer.add_vision_max_pixels(int(self.preprocessor_config["max_pixels"]))
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, _ = item
if not name.startswith("visual."):
return None
return super().filter_tensors(item)
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
# ggml uses theta = pos / rope_freqs
pairs = torch.tensor(self.rope_pairs)
freq_a = self.rope_freq[2 * pairs]
freq_b = self.rope_freq[2 * pairs + 1]
yield ("visual.rope_freqs.weight", (1.0 / torch.cat([freq_a, freq_b])).to(torch.float32))
def permute_qk(self, data_torch: Tensor) -> Tensor:
rows = [2 * i for i in self.rope_pairs] + [2 * i + 1 for i in self.rope_pairs]
return data_torch.reshape(self.n_head, self.d_head, -1)[:, rows].reshape(data_torch.shape)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if ".self_attn.qkv." in name:
q, k, v = data_torch.split(self.n_embd, dim=0)
data_torch = torch.cat([self.permute_qk(q), self.permute_qk(k), v], dim=0)
yield from super().modify_tensors(data_torch, name, bid)