Repository navigation
Expand file tree
/
Copy pathdrama3.py
More file actions
264 lines (221 loc) · 14 KB
/
Copy pathdrama3.py
File metadata and controls
264 lines (221 loc) · 14 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
from __future__ import annotations
from typing import Any, Callable, Iterable, TYPE_CHECKING
import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, MmprojModel, gguf
from .qwen import Qwen3_5MoeTextModel
# Fish Audio Drama 3 (FishOmniModel)
# text model: Qwen3.5-MoE backbone, samples the main token of each frame
# mmproj: codebook tower and Fish DAC decoder (codec.pth)
_TOWER_SHARED = "audio_decoder_mtp_shared.layers."
_TOWER_RESIDUAL = "audio_decoder_mtp.layers."
def load_codec(dir_model) -> dict[str, Tensor]:
sd = torch.load(dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
return sd.get("state_dict", sd)
def fold_conv(sd: dict[str, Tensor], prefix: str) -> Tensor:
# fold weight norm
if prefix + ".conv.weight" in sd:
return sd[prefix + ".conv.weight"].float()
g = sd[prefix + ".conv.parametrizations.weight.original0"].float()
v = sd[prefix + ".conv.parametrizations.weight.original1"].float()
return g * v / v.norm(dim=(1, 2), keepdim=True)
def fold_wn(g: Tensor, v: Tensor) -> Tensor:
# fold old-style weight norm of a 1x1 conv, returns [out, in]
return (g.float() * v.float() / v.float().norm(dim=(1, 2), keepdim=True)).squeeze(-1)
def fold_snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
alpha = alpha.float().flatten()
return alpha, 1.0 / (alpha + 1e-9)
@ModelBase.register("FishOmniModel")
@ModelBase.example("fishaudio/drama-3")
class Drama3TextModel(Qwen3_5MoeTextModel):
model_arch = gguf.MODEL_ARCH.QWEN35MOE
no_mtp = True # the checkpoint has no MTP head
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
if item[0].startswith("audio_decoder"):
return None
return super().filter_tensors(item)
@ModelBase.register("FishOmniModel")
@ModelBase.example("fishaudio/drama-3")
class Drama3MmprojModel(MmprojModel):
has_vision_encoder = False
has_audio_encoder = False
gen_projector_type = gguf.VisionProjectorType.DRAMA3_GEN
_LAYER_MAP = {
"enorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True),
"hnorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_HNORM, True),
"eh_proj.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_EH_PROJ, False),
"semantic_embedding.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, False),
"codebook_embedding.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, False),
"final_layernorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM, True),
"transformer_layer.attention_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM, True),
"transformer_layer.ffn_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM, True),
"transformer_layer.attention.q_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM, True),
"transformer_layer.attention.k_norm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM, True),
"transformer_layer.attention.wo.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT, False),
"transformer_layer.feed_forward.w1.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE, False),
"transformer_layer.feed_forward.w3.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP, False),
"transformer_layer.feed_forward.w2.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, False),
}
_heads: dict[int, Tensor]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._heads = {}
def get_audio_config(self) -> dict[str, Any] | None:
cfg = dict(self.global_config["audio_decoder_config"])
cfg["num_hidden_layers"] = cfg["shared_enrichment_layers"] + cfg["num_codebooks"] - 1
return cfg
def set_gguf_parameters(self):
cfg = self.hparams
self.gguf_writer.add_file_type(self.ftype)
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
self.gguf_writer.add_clip_gen_audio_projector_type(self.gen_projector_type)
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"])
self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"])
self.gguf_writer.add_gen_audio_block_count(cfg["num_hidden_layers"])
self.gguf_writer.add_gen_audio_head_count(cfg["n_head"])
self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"])
self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"])
# other tower and codec hparams are hardcoded in tools/mtmd
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name = item[0]
if name.startswith("audio_decoder") or name == "model.language_model.embed_tokens.weight":
return item
return None
def tensor_force_quant(self, name, new_name, bid, n_dims):
# conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
if new_name.endswith(".weight") and (
new_name in ("a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight")
or (".up.blk." in new_name and new_name.endswith(".dwconv.weight"))
or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
):
return gguf.GGMLQuantizationType.F16
# keep ConvTranspose1d kernels and folded codebook tables in F32
if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
return gguf.GGMLQuantizationType.F32
if new_name.startswith("a.gen.wav.quant."):
return gguf.GGMLQuantizationType.F32
return super().tensor_force_quant(name, new_name, bid, n_dims)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
T = gguf.MODEL_TENSOR
cfg = self.hparams
if name.startswith("a.gen.wav."):
yield (name, data_torch)
return
if name == "model.language_model.embed_tokens.weight":
# semantic band of the backbone embeddings
begin = self.global_config["semantic_start_token_id"]
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
return
if name == "audio_decoder.codebook_embeddings.weight":
n_cb = cfg["num_codebooks"]
yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(n_cb, -1, data_torch.shape[-1]))
return
if name.startswith("audio_decoder_mtp.heads."):
n_heads = cfg["num_codebooks"] - 1
self._heads[int(name.split(".")[2])] = data_torch
if len(self._heads) < n_heads:
return
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), torch.stack([self._heads.pop(i) for i in range(n_heads)]))
return
for prefix, offset in ((_TOWER_SHARED, 0), (_TOWER_RESIDUAL, cfg["shared_enrichment_layers"])):
if not name.startswith(prefix):
continue
idx, key = name[len(prefix):].split(".", 1)
il = int(idx) + offset
if key == "transformer_layer.attention.wqkv.weight":
# rows are [query | gate | key | value]
n_q = cfg["n_head"] * cfg["head_dim"]
n_kv = cfg["n_local_heads"] * cfg["head_dim"]
q, gate, k, v = data_torch.split([n_q, n_q, n_kv, n_kv], dim=0)
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, il), q)
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_GATE, il), gate)
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, il), k)
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, il), v)
return
tensor, zero_centered = self._LAYER_MAP[key]
if zero_centered:
data_torch = data_torch.float() + 1
yield (self.format_tensor_name(tensor, il), data_torch)
return
raise ValueError(f"unexpected tensor {name}")
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
# only the DAC decoder, the encoder is for voice cloning (not supported yet)
sd = load_codec(self.dir_model)
T = gguf.MODEL_TENSOR
def get(name: str) -> Tensor:
return sd[name].float()
def conv(prefix: str) -> Tensor:
return fold_conv(sd, prefix)
def quant_table(prefix: str) -> Tensor:
# fold out_proj into the codebook, so decoding is a row lookup
w = fold_wn(sd[prefix + ".out_proj.weight_g"], sd[prefix + ".out_proj.weight_v"])
return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias")
q = "quantizer."
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0"))
n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight"))
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB),
torch.stack([quant_table(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]))
# post_module uses interleaved RoPE, permute q/k rows so NEOX RoPE gives the same result
tfm = q + "post_module."
n_head, d_head = 16, 64
order = torch.cat([torch.arange(0, d_head, 2), torch.arange(1, d_head, 2)])
def neox_rows(w: Tensor) -> Tensor:
return w.reshape(n_head, d_head, -1)[:, order].reshape(w.shape)
n_tfm = sum(1 for k in sd if k.startswith(tfm + "layers.") and k.endswith(".ffn_norm.weight"))
for il in range(n_tfm):
p = f"{tfm}layers.{il}."
wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_Q, il), neox_rows(wq))
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_K, il), neox_rows(wk))
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_V, il), wv)
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_OUT, il), get(p + "attention.wo.weight"))
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_NORM, il), get(p + "attention_norm.weight"))
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_SCALE, il), get(p + "attention_layer_scale.gamma"))
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_NORM, il), get(p + "ffn_norm.weight"))
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_GATE, il), get(p + "feed_forward.w1.weight"))
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_UP, il), get(p + "feed_forward.w3.weight"))
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_DOWN, il), get(p + "feed_forward.w2.weight"))
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_SCALE, il), get(p + "ffn_layer_scale.gamma"))
yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), get(tfm + "norm.weight"))
# upsample: 2x (ConvTranspose1d + ConvNeXt)
for il in range(2):
p = f"{q}upsample.{il}."
yield (self.format_tensor_name(T.A_GEN_WAV_UP_CONV, il, suffix=".weight"), conv(p + "0"))
yield (self.format_tensor_name(T.A_GEN_WAV_UP_CONV, il, suffix=".bias"), get(p + "0.conv.bias"))
yield (self.format_tensor_name(T.A_GEN_WAV_UP_DWCONV, il, suffix=".weight"), conv(p + "1.dwconv"))
yield (self.format_tensor_name(T.A_GEN_WAV_UP_DWCONV, il, suffix=".bias"), get(p + "1.dwconv.conv.bias"))
for key, tensor in (("norm", T.A_GEN_WAV_UP_NORM), ("pwconv1", T.A_GEN_WAV_UP_PW1), ("pwconv2", T.A_GEN_WAV_UP_PW2)):
yield (self.format_tensor_name(tensor, il, suffix=".weight"), get(f"{p}1.{key}.weight"))
yield (self.format_tensor_name(tensor, il, suffix=".bias"), get(f"{p}1.{key}.bias"))
yield (self.format_tensor_name(T.A_GEN_WAV_UP_GAMMA, il, suffix=""), get(p + "1.gamma"))
# DAC: entry conv, 4x (Snake, ConvTranspose1d, 3 residual units), Snake, post conv
d = "decoder.model."
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".weight"), conv(d + "0"))
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias"))
for bid in range(4):
p = f"{d}{bid + 1}.block."
a, b = fold_snake(get(p + "0.alpha"))
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a)
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b)
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1"))
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".bias"), get(p + "1.conv.bias"))
for xid in range(3):
r = f"{p}{xid + 2}.block."
for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)):
a, b = fold_snake(get(r + act + ".alpha"))
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
yield (name + ".alpha", a)
yield (name + ".beta", b)
for cv, tensor in (("1", T.A_GEN_WAV_DAC_RES_CONV1), ("3", T.A_GEN_WAV_DAC_RES_CONV2)):
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
yield (name + ".weight", conv(r + cv))
yield (name + ".bias", get(r + cv + ".conv.bias"))
a, b = fold_snake(get(d + "5.alpha"))
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a)
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get(d + "6.conv.bias"))