Repository navigation
Expand file tree
/
Copy pathinkling2.py
More file actions
331 lines (300 loc) · 18.8 KB
/
Copy pathinkling2.py
File metadata and controls
331 lines (300 loc) · 18.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
from __future__ import annotations
import json
import re
from pathlib import Path
from functools import partial
from typing import Callable, Iterable, TYPE_CHECKING, cast
import numpy as np
import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import LazyTorchTensor, ModelBase, MmprojModel, TextModel, gguf, logger
def _e4m3_table() -> np.ndarray:
codes = np.arange(256, dtype=np.uint8)
exponent = ((codes & 0x7F) >> 3).astype(np.int32)
mantissa = (codes & 7).astype(np.float32)
values = np.where(exponent == 0, np.ldexp(mantissa, -9), np.ldexp(1 + mantissa / 8, exponent - 7))
values = np.where(codes & 0x80, -values, values)
values[(codes & 0x7F) == 0x7F] = np.nan
return values.astype(np.float32)
_E4M3 = _e4m3_table()
_E2M1 = np.array([0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6], dtype=np.float32)
_EXPERT_CHUNK_ROWS = 128
def _dequant_expert_chunk(weight: Callable[[], Tensor], scale: Callable[[], Tensor], amax: Callable[[], Tensor] | None, denominator: Callable[[], Tensor] | None, name: str, expert: int, rows: slice) -> np.ndarray:
# Fresh lazy tensors release their mappings after each chunk instead of caching the layer.
packed = LazyTorchTensor.to_eager(weight().view(torch.uint8)[expert, rows]).numpy()
scales = LazyTorchTensor.to_eager(scale().view(torch.uint8)[expert, rows]).numpy()
if amax is not None:
block_scale = _E4M3[scales]
if np.isnan(block_scale).any():
raise ValueError(f"Inkling2: {name}_scale expert {expert}: E4M3FN NaN")
values = np.empty((packed.shape[0], packed.shape[1] * 2), dtype=np.float32)
values[:, 0::2] = _E2M1[packed & 15]
values[:, 1::2] = _E2M1[packed >> 4]
values.reshape(values.shape[0], -1, 16)[:] *= block_scale[:, :, None]
assert denominator is not None
a = LazyTorchTensor.to_eager(amax()[expert]).item()
d = LazyTorchTensor.to_eager(denominator()).item()
if not np.isfinite(a) or a < 0:
raise ValueError(f"Inkling2: {name}_amax expert {expert}: expected finite nonnegative value")
if not np.isfinite(d) or d <= 0:
raise ValueError(f"Inkling2: {name}_outer_scale_denominator expert {expert}: expected finite positive value")
with np.errstate(over="ignore", invalid="ignore"):
values *= np.float32(a) / np.float32(d)
else:
if np.any(scales == 255):
raise ValueError(f"Inkling2: {name}_scale expert {expert}: UE8M0 byte 255 is NaN")
values = _E4M3[packed]
if np.isnan(values).any():
raise ValueError(f"Inkling2: {name} expert {expert}: E4M3FN NaN")
block_scale = np.ldexp(np.ones(scales.shape, dtype=np.float32), scales.astype(np.int32) - 127)
with np.errstate(over="ignore", invalid="ignore"):
values.reshape(values.shape[0], -1, 32)[:] *= block_scale[:, :, None]
if not np.isfinite(values).all():
raise ValueError(f"Inkling2: {name} expert {expert}: non-finite dequantized weight")
return values
def _is_inkling2(dir_model: Path) -> bool:
if not (dir_model / "config.json").is_file():
return False
with open(dir_model / "config.json", encoding="utf-8") as f:
return json.load(f).get("model_type") in ("inkling2", "inkling2_text")
@ModelBase.register_hparams_loader(_is_inkling2)
def _load_inkling2_hparams(dir_model: Path):
# AutoConfig injects MXFP8 defaults and can replace the original layer schedule.
with open(dir_model / "config.json", encoding="utf-8") as f:
return json.load(f)
@ModelBase.register("Inkling2ForConditionalGeneration", "Inkling2ForCausalLM")
@ModelBase.example("thinkingmachines/Inkling-2-Small")
class Inkling2Model(TextModel):
model_arch = gguf.MODEL_ARCH.INKLING2
def __init__(self, dir_model: Path, *args, **kwargs):
self._expert_quant: dict[str, dict] = {}
self._packed_experts: dict[str, gguf.LazyChunkedTensor] = {}
quant_path = dir_model / "hf_quant_config.json"
if quant_path.is_file():
with open(quant_path, encoding="utf-8") as f:
groups = json.load(f).get("model", {}).get("parameter_groups", [])
for group in groups:
for name in group.get("tensors", []):
item = self.filter_tensors((name, lambda: torch.empty(0)))
if item is not None and item[0].endswith((".w13_weight", ".w2_weight")):
name = item[0]
if name in self._expert_quant:
raise ValueError(f"Inkling2: duplicate hf_quant_config.json scheme for {name}")
self._expert_quant[name] = group
super().__init__(dir_model, *args, **kwargs)
logger.warning("Inkling2: converting the text backbone only; vision and MTP tensors are skipped")
hp = self.hparams
for name, default in (("q_bias", False), ("o_bias", False), ("n_shared_experts", 0), ("use_value_norm_no_gamma", True), ("attn_res_normalize_values", True), ("scoring_func", "sigmoid")):
if hp.get(name, default) != default:
raise ValueError(f"Inkling2: unsupported {name}={hp[name]!r}")
types = hp.get("layer_types")
if types is None:
local = hp["local_layer_ids"]
types = ["hybrid_sliding" if i in local else "hybrid" for i in range(self.block_count)]
if len(types) != self.block_count or set(types) - {"hybrid", "hybrid_sliding"}:
raise ValueError("Inkling2: invalid layer_types")
self.is_swa = [t == "hybrid_sliding" for t in types]
def dequant_model(self):
for name in list(self.model_tensors):
if not name.endswith((".w13_weight", ".w2_weight")):
continue
weight = self.model_tensors[name]
tensor = weight()
companions = [name + suffix for suffix in ("_scale", "_amax", "_outer_scale_denominator")]
if tensor.dtype.is_floating_point and tensor.element_size() > 1:
if any(n in self.model_tensors for n in companions):
raise ValueError(f"Inkling2: floating-point {name} has packed quantization companions")
continue
config = self._expert_quant.get(name, {})
scheme = config.get("weight_transform")
if scheme not in ("nvfp4_nearest", "mxfp8_nearest"):
raise ValueError(f"Inkling2: {name}: missing or unsupported hf_quant_config.json scheme {scheme!r}")
nvfp4 = scheme == "nvfp4_nearest"
if nvfp4 and config.get("weight_scale_format") != "amax_denominator":
raise ValueError(f"Inkling2: {name}: expected NVFP4 amax_denominator scale format")
required = companions if nvfp4 else companions[:1]
for companion in required:
if companion not in self.model_tensors:
raise ValueError(f"Inkling2: {name}: missing companion {companion}")
if not nvfp4 and any(n in self.model_tensors for n in companions[1:]):
raise ValueError(f"Inkling2: {name}: MXFP8 must not have amax or outer_scale_denominator")
scale = self.model_tensors[companions[0]]
scale_tensor = scale()
weight_dtype, scale_dtype = (torch.uint8, torch.float8_e4m3fn) if nvfp4 else (torch.float8_e4m3fn, torch.uint8)
if tensor.dtype != weight_dtype or scale_tensor.dtype != scale_dtype:
raise ValueError(f"Inkling2: {name}: expected {weight_dtype} weights and {scale_dtype} scales, got {tensor.dtype} and {scale_tensor.dtype}")
experts = self.hparams["n_routed_experts"]
hidden = self.hparams["hidden_size"]
inter = self.find_hparam(["moe_intermediate_size", "intermediate_size"])
w13 = name.endswith(".w13_weight")
rows, cols = (2 * inter, hidden) if w13 else (hidden, inter)
shape = (experts, rows, cols)
if min(shape) <= 0 or cols % (16 if nvfp4 else 32) or tuple(tensor.shape) != (experts, rows, cols // (2 if nvfp4 else 1)):
raise ValueError(f"Inkling2: {name}: invalid packed shape {tuple(tensor.shape)} for logical shape {shape}")
ss = tuple(scale_tensor.shape)
if len(ss) != 3 or ss[0] != experts or min(ss) <= 0 or rows % ss[1] or cols % ss[2]:
raise ValueError(f"Inkling2: {name}_scale: shape {ss} does not divide logical shape {shape}")
block = (rows // ss[1], cols // ss[2])
if block != (1, 16 if nvfp4 else 32):
raise ValueError(f"Inkling2: {name}_scale: unsupported stored block shape {block}")
if list(block) != config.get("weight_block_shape"):
logger.warning("Inkling2: %s: hf_quant_config.json weight_block_shape %s conflicts with stored weight/scale shapes %s/%s; using %s", name, config.get("weight_block_shape"), tuple(tensor.shape), ss, list(block))
amax = denominator = None
if nvfp4:
amax, denominator = (self.model_tensors[n] for n in companions[1:])
for companion, gen, expected in ((companions[1], amax, (experts,)), (companions[2], denominator, (1,))):
t = gen()
if t.dtype != torch.float32 or tuple(t.shape) != expected:
raise ValueError(f"Inkling2: {companion}: expected F32 {expected}, got {t.dtype} {tuple(t.shape)}")
chunks: list[Callable[[], np.ndarray]] = []
for expert in range(experts):
for half in range(2 if w13 else 1):
for start in range(0, rows // 2 if w13 else rows, _EXPERT_CHUNK_ROWS):
stop = min(start + _EXPERT_CHUNK_ROWS, rows // 2 if w13 else rows)
# The same packed row slice selects the weight and its scale groups.
row_slice = slice(2 * start + half, 2 * stop + half, 2) if w13 else slice(start, stop)
chunks.append(partial(_dequant_expert_chunk, weight, scale, amax, denominator, name, expert, row_slice))
self._packed_experts[name] = gguf.LazyChunkedTensor(chunks, shape, np.float32)
self.model_tensors[name] = lambda shape=shape: torch.empty(shape, dtype=torch.float32, device="meta")
for companion in required:
del self.model_tensors[companion]
for name, gen in self.model_tensors.items():
t = gen()
if not t.dtype.is_floating_point or t.element_size() == 1 or name.endswith(("_weight_scale", "_weight_amax", "_weight_outer_scale_denominator")):
raise ValueError(f"Inkling2: unexpected packed tensor or orphan companion {name}")
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
if any(p in name for p in ("visual.", "vision_tower.", "mtp.", "markov_head", "nextn_predict")):
return None
name = name.replace("model.language_model.", "model.").replace("model.llm.", "model.")
# the mapping table cannot shift a block index: attn_queries.{i} feeds block i + 1
name = re.sub(r"attn_res\.attn_queries\.(\d+)", lambda m: f"layers.{int(m[1]) + 1}.attention_input.query", name)
# the table maps a module, not its bias: gate.weight is the router, gate.bias the score correction
name = name.replace("mlp.gate.bias", "mlp.gate.e_score_correction_bias")
return name, gen
def get_vocab_base(self):
tokens, types, pre = super().get_vocab_base()
n = self.hparams["unpadded_vocab_size"]
return tokens[:n], types[:n], pre
def set_vocab(self):
tokens, types, pre = self.get_vocab_base()
self.gguf_writer.add_tokenizer_model("gpt2")
self.gguf_writer.add_tokenizer_pre(pre)
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_types(types)
special = gguf.SpecialVocab(self.dir_model, load_merges=True)
eos = self.hparams["eos_token_id"]
special.special_token_ids["eos"] = eos
special.special_token_ids["eot"] = eos
special.add_special_token["bos"] = False
special.add_to_gguf(self.gguf_writer)
def set_gguf_parameters(self):
hp = self.hparams
hp["num_attention_heads"] = [hp.get("swa_num_attention_heads", hp["num_attention_heads"]) if s else hp["num_attention_heads"] for s in self.is_swa]
hp["num_key_value_heads"] = [hp.get("swa_num_key_value_heads", hp["num_key_value_heads"]) if s else hp["num_key_value_heads"] for s in self.is_swa]
for name in ("n_routed_experts", "num_experts_per_tok"):
if name not in hp:
raise KeyError(f"Inkling2: missing {name}")
super().set_gguf_parameters()
w = self.gguf_writer
w.add_vocab_size(hp["vocab_size"])
w.add_vocab_size_unpadded(hp["unpadded_vocab_size"])
w.add_expert_feed_forward_length(self.find_hparam(["moe_intermediate_size", "intermediate_size"]))
w.add_key_length_swa(hp.get("swa_head_dim", hp["head_dim"]))
w.add_value_length_swa(hp.get("swa_head_dim", hp["head_dim"]))
w.add_rope_dimension_count(0)
w.add_sliding_window(hp["sliding_window_size"])
w.add_sliding_window_pattern(self.is_swa)
w.add_shortconv_l_cache(self.find_hparam(["conv_kernel_size", "sconv_kernel_size"]))
w.add_expert_weights_scale(hp["route_scale"])
w.add_activation_situ_beta(hp["mlp_gate_softcap"])
w.add_activation_situ_linear_beta(hp["mlp_up_softcap"])
w.add_attn_res_block_size(hp["n_layers_per_attn_res_block"])
w.add_final_logit_softcapping(hp["final_logit_softcapping"])
w.add_relative_length(hp["d_rel"])
w.add_relative_extent(hp["rel_extent"])
w.add_qk_clip(hp["qk_l2_clip_threshold"])
w.add_relative_clip(hp["rel_l2_clip_threshold"])
w.add_log_scale_floor(hp["log_scaling_n_floor"])
w.add_log_scale_alpha(hp["log_scaling_alpha"])
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
if new_name.endswith(("_conv.weight", "_clip.weight", "_res_query.weight", "attn_rel_bias.weight")):
return gguf.GGMLQuantizationType.F32
return super().tensor_force_quant(name, new_name, bid, n_dims)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name in self._packed_experts:
target = name.replace(".w13_weight", ".gate_up_proj").replace(".w2_weight", ".down_proj")
yield self.map_tensor_name(target, try_suffixes=()) + ".weight", cast("Tensor", self._packed_experts[name])
return
if name.endswith(".w13_weight"):
shape = data_torch.shape
data_torch = data_torch.reshape(shape[0], shape[1] // 2, 2, shape[2]).transpose(1, 2).reshape(shape)
name = name.replace(".w13_weight", ".gate_up_proj")
name = name.replace(".w2_weight", ".down_proj")
for static, conv in ((".static_weight", ".conv1d.weight"), (".static_w", ".weight")):
if name.endswith(static):
data_torch = data_torch.flip(0).T.contiguous()
name = name.removesuffix(static) + conv
if name.endswith((".conv1d.weight", "sconv.weight")):
data_torch = data_torch.squeeze(1) if data_torch.ndim == 3 else data_torch
elif name.endswith(".rel_logits_proj.proj"):
data_torch = data_torch.T.contiguous()
# a name the table maps without a suffix is a bare parameter
new_name = self.tensor_map.get_name(name)
if new_name is not None:
suffix = ".bias" if name.endswith(".e_score_correction_bias") else ".weight"
yield new_name + suffix, data_torch
else:
yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("Inkling2ForConditionalGeneration")
class Inkling2VisionModel(MmprojModel):
def get_vision_config(self):
config = self.global_config.get("vision_config")
if not config:
return None
config = dict(config)
config["patch_size"] = config.get("native_patch_size", config.get("patch_size", 20))
config["spatial_merge_size"] = config.get("adapter_merge_size", config.get("spatial_merge_size", 2))
config["intermediate_size"] = config.get("intermediate_size") or int(config["hidden_size"] * config.get("mlp_ratio", 4.0))
config["image_size"] = config["posemb_grid_size"] * config["patch_size"]
if config.get("num_channels", 3) != 3 or config.get("hidden_act", "gelu") != "gelu":
raise ValueError("Inkling2 vision requires RGB patches and exact GELU")
return config
def set_gguf_parameters(self):
self.preprocessor_config.setdefault("image_mean", [0.5, 0.5, 0.5])
self.preprocessor_config.setdefault("image_std", [0.5, 0.5, 0.5])
super().set_gguf_parameters()
config = self.hparams
rope = config.get("rope_parameters") or {"rope_type": "axial", "rope_theta": 10000.0}
if rope["rope_type"] != "axial" or rope["rope_theta"] != 10000.0:
raise ValueError("Inkling2 vision requires axial RoPE with theta 10000")
w = self.gguf_writer
w.add_clip_vision_projector_type(gguf.VisionProjectorType.INKLING2V)
w.add_vision_attention_layernorm_eps(config["layer_norm_eps"])
w.add_vision_spatial_merge_size(config["spatial_merge_size"])
def dequant_model(self):
# vision tensors are BF16, only the text experts are quantized
return
@classmethod
def filter_tensors(cls, item):
name, gen = item
if not name.startswith(("model.visual.", "model.vision_tower.")):
return None
return name, gen
# mm.{bid} takes the projector layer index, the mapping table cannot derive it from these names
_projector_layers = {"model.visual.proj_in": 1, "model.visual.proj_out": 2, "model.vision_tower.merger.linear_1": 1, "model.vision_tower.merger.linear_2": 2}
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
module, _, suffix = name.rpartition(".")
if module in self._projector_layers:
yield self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, self._projector_layers[module], "." + suffix), data_torch
return
# lay out the linear patch embedding (HWC patches) as a conv2d for build_inp()
if name.endswith(".patch_embedding.weight"):
ps = self.hparams["patch_size"]
data_torch = data_torch.reshape(data_torch.shape[0], ps, ps, 3).permute(0, 3, 1, 2)
# these are bare parameters, they have no .weight
if name.endswith(("position_embedding", ".lambda_1", ".lambda_2", ".gamma")):
name = name.removesuffix(".gamma") + ".weight"
yield from super().modify_tensors(data_torch, name, bid)