JiRackDeltaNet_27b.py
36.1 KB · 759 lines · python Raw
1 # =============================================================================
2 # COPYRIGHT © 2026 Konstantin Vladimirovich Grabko. ALL RIGHTS RESERVED.
3 # JiRack Ultra Ternary Transformer
4 #
5 # CMS Manhattan JiRack Technology — PATENT PENDING
6 #
7 # This code is proprietary.
8 # Personal and non-commercial research use is allowed.
9 # Any commercial use, derivative works for profit, or distribution
10 # requires a paid license and 5% royalty.
11 #
12 # Unauthorized commercial use is strictly prohibited.
13 # Contact: grabko@cmsmanhattan.com
14 # =============================================================================
15 """
16 The custom ("hand-rolled") "JiRack Qwen3.8-27B" architecture follows the style of the earlier DS32B project: a single file, a custom `TransformerBlock`, no `AutoConfig`/`AutoModelForCausalLM` inside the architecture class, and no `model_id` in the constructor. All dimensions are hardcoded as constants below—verified via live inspection of the actual checkpoint (`inspect_qwen38_keys.py`, 851 tensors, cross-checked).
17
18 The mathematics for DeltaNet recurrence, convolution, gating, and partial-rotary mRoPE attention were **ported** directly from the actual `modeling_qwen3_5.py` (Apache 2.0, HuggingFace/Qwen) rather than reinvented—to avoid the risk of subtle numerical errors in the delicate recurrence formulas. While the logic was ported, the code was rewritten to match our naming conventions and style, and stripped down: no vision tower, and no `transformers`-style KV-cache classes (full sequence forward passes aren't needed for training; for generation, we use full recomputation at each step without an incremental cache—a deliberate simplification, though the cache could be added separately if desired).
19
20 Submodule names (`linear_attn` / `self_attn`, `in_proj_qkv`, `out_proj`, `mlp.gate_proj`, etc.) intentionally **match** those in the actual `Qwen3_5ForCausalLM` `state_dict`, allowing weights from the HF checkpoint to be loaded directly via `load_state_dict()` without manual key mapping. The module supports the following:
21 - building the model (JiRackQwen38ForCausalLM) from scratch with random weights;
22 - accepting a state_dict in the HF Qwen3_5ForCausalLM format (obtained via
23 `AutoModelForCausalLM.from_pretrained(...).state_dict()` in the converter)
24 and loading it directly;
25 - patching the necessary `nn.Linear` layers to `BitLinear` (`TERNARIZE_PATTERNS`);
26 - saving and loading its own JiRack checkpoints (`save_checkpoint`/`load_checkpoint`).
27
28 The converter (`convert_qwen38_to_jirack.py`) remains aware of the `model_id`
29 and the HF Hub—but this knowledge resides ONLY within the converter, not the architecture itself.
30 """
31
32 import os
33 import re
34 import torch
35 import torch.nn as nn
36 import torch.nn.functional as F
37
38
39 # =====================================================================
40 # 1. CONFIRMED АРХИТЕКТУРА -- жёстко зашитые константы (не AutoConfig)
41 # =====================================================================
42
43 HIDDEN_SIZE = 5120
44 VOCAB_SIZE = 248320
45 INTERMEDIATE_SIZE = 17408
46 NUM_LAYERS = 64
47 RMS_NORM_EPS = 1e-6
48
49 # Чередование слоёв: full_attention_interval = 4 ->
50 # 3x linear_attention, 1x full_attention, повторяется на все 64 слоя.
51 LAYER_TYPES = ["linear_attention"] * 3 + ["full_attention"]
52 LAYER_TYPES = (LAYER_TYPES * (NUM_LAYERS // len(LAYER_TYPES) + 1))[:NUM_LAYERS]
53
54 # --- Gated DeltaNet (linear_attention) ---
55 LINEAR_NUM_KEY_HEADS = 16
56 LINEAR_KEY_HEAD_DIM = 128
57 LINEAR_NUM_VALUE_HEADS = 48
58 LINEAR_VALUE_HEAD_DIM = 128
59 LINEAR_CONV_KERNEL_DIM = 4
60 DELTA_KEY_DIM = LINEAR_NUM_KEY_HEADS * LINEAR_KEY_HEAD_DIM # 2048
61 DELTA_VALUE_DIM = LINEAR_NUM_VALUE_HEADS * LINEAR_VALUE_HEAD_DIM # 6144
62 DELTA_CONV_DIM = DELTA_KEY_DIM * 2 + DELTA_VALUE_DIM # 10240
63 DELTA_CHUNK_SIZE = 64
64
65 # --- Gated Attention (full_attention) ---
66 NUM_ATTENTION_HEADS = 24
67 NUM_KEY_VALUE_HEADS = 4
68 HEAD_DIM = 256
69 PARTIAL_ROTARY_FACTOR = 0.25
70 ROPE_THETA = 1e7
71 MROPE_SECTION = [11, 11, 10]
72 ATTENTION_BIAS = False
73
74 ARCH_SUMMARY = f"""
75 Qwen3.8-27B (qwen3_5), hand-rolled JiRack TransformerBlock, text-only.
76 hidden_size={HIDDEN_SIZE}, vocab_size={VOCAB_SIZE},
77 intermediate_size={INTERMEDIATE_SIZE}, num_hidden_layers={NUM_LAYERS}
78 Чередование (interval=4): 3x linear_attention (DeltaNet), 1x full_attention
79 """
80
81 KEPT_FULL_PRECISION = """
82 The following always remain at full precision (not GEMM, or otherwise too sensitive):
83 - dt_bias, A_log (raw Parameters, DeltaNet decay)
84 - linear_attn.in_proj_b/in_proj_a (default behavior; see GATE_PROJ_PATTERNS)
85 - conv1d (depthwise grouped conv; not GEMM)
86 - all norms: RMSNormGated, q_norm/k_norm, model.norm
87 - embed_tokens, lm_head
88 - mtp.* (entire MTP/NextN head) (speculative decoding draft layer;
89 ternarization destroys acceptance rate;
90 the GGUF community keeps it at Q8_0)
91 """
92
93
94 # =====================================================================
95 # 2. BitLinear -- наш тернарный слой
96 # =====================================================================
97
98 class BitLinear(nn.Linear):
99 """b1.58-style per-tensor absmean ternary weights, per-token int8
100 activations, непрерывный lambda-warmup (STE)."""
101
102 def __init__(self, in_features, out_features, bias=False):
103 super().__init__(in_features, out_features, bias=bias)
104 self.eps = 1e-5
105 self.register_buffer("lambda_", torch.zeros(()), persistent=True)
106
107 @classmethod
108 def from_linear(cls, lin: nn.Linear) -> "BitLinear":
109 bl = cls(lin.in_features, lin.out_features, bias=lin.bias is not None)
110 bl = bl.to(dtype=lin.weight.dtype, device=lin.weight.device)
111 with torch.no_grad():
112 bl.weight.copy_(lin.weight)
113 if lin.bias is not None:
114 bl.bias.copy_(lin.bias)
115 return bl
116
117 def forward(self, x: torch.Tensor) -> torch.Tensor:
118 if not self.training and float(self.lambda_) < 1e-6:
119 return F.linear(x, self.weight, self.bias)
120
121 lam = self.lambda_.to(x.dtype)
122 w = self.weight
123 gamma = w.float().abs().mean().clamp(min=self.eps).to(w.dtype)
124 w_quant = torch.clamp(torch.round(w / gamma), -1.0, 1.0) * gamma
125 w_effective = w + lam * (w_quant - w).detach()
126
127 x_scale = 127.0 / x.abs().max(dim=-1, keepdim=True).values.clamp(min=self.eps)
128 x_quant = torch.clamp(torch.round(x * x_scale), -128.0, 127.0) / x_scale
129 x_effective = x + lam * (x_quant - x).detach()
130
131 return F.linear(x_effective, w_effective, self.bias)
132
133 def set_lambda(self, v: float):
134 self.lambda_.fill_(v)
135
136
137 # =====================================================================
138 # 3. Карта: что тернаризуем, что оставляем full precision
139 # =====================================================================
140
141 TERNARIZE_PATTERNS = [
142 r"linear_attn\.in_proj_qkv$",
143 r"linear_attn\.in_proj_z$",
144 r"linear_attn\.out_proj$",
145 r"self_attn\.q_proj$",
146 r"self_attn\.k_proj$",
147 r"self_attn\.v_proj$",
148 r"self_attn\.o_proj$",
149 r"mlp\.gate_proj$",
150 r"mlp\.up_proj$",
151 r"mlp\.down_proj$",
152 ]
153
154 GATE_PROJ_PATTERNS = [
155 r"linear_attn\.in_proj_b$",
156 r"linear_attn\.in_proj_a$",
157 ]
158
159
160 def apply_bitlinear_patch(model: nn.Module, include_gate_proj: bool = False):
161 """Точечная замена nn.Linear -> BitLinear по TERNARIZE_PATTERNS."""
162 patterns = list(TERNARIZE_PATTERNS)
163 if include_gate_proj:
164 patterns += GATE_PROJ_PATTERNS
165 compiled = [re.compile(p) for p in patterns]
166
167 patched = []
168 for name, module in list(model.named_modules()):
169 if not isinstance(module, nn.Linear):
170 continue
171 # MTP/NextN-голова НЕ тернаризуется -- всегда full precision.
172 # Суффиксные паттерны (self_attn.q_proj$ и т.п.) иначе зацепили бы
173 # mtp.layers.0.self_attn.q_proj.
174 if name.startswith("mtp"):
175 continue
176 if any(p.search(name) for p in compiled):
177 parent_name, _, child_name = name.rpartition(".")
178 parent = model.get_submodule(parent_name) if parent_name else model
179 setattr(parent, child_name, BitLinear.from_linear(module))
180 patched.append(name)
181 return patched
182
183
184 # =====================================================================
185 # 4. Общие блоки (порт математики из реального modeling_qwen3_5.py)
186 # =====================================================================
187
188 class RMSNorm(nn.Module):
189 """(x * w).to(dtype), w хранится как (weight) с offset +1 -- как в
190 оригинале Qwen3_5RMSNorm (инициализация нулями, эффективный вес 1+w)."""
191
192 def __init__(self, dim: int, eps: float = RMS_NORM_EPS):
193 super().__init__()
194 self.eps = eps
195 self.weight = nn.Parameter(torch.zeros(dim))
196
197 def forward(self, x: torch.Tensor) -> torch.Tensor:
198 dtype = x.dtype
199 x = x.float()
200 x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
201 x = x * (1.0 + self.weight.float())
202 return x.to(dtype)
203
204
205 class RMSNormGated(nn.Module):
206 """RMSNorm с гейтингом через SiLU(gate), применяется ПЕРЕД гейтингом
207 (норма считается по hidden_states, гейт применяется после)."""
208
209 def __init__(self, dim: int, eps: float = RMS_NORM_EPS):
210 super().__init__()
211 self.weight = nn.Parameter(torch.ones(dim))
212 self.eps = eps
213
214 def forward(self, x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:
215 dtype = x.dtype
216 x = x.float()
217 var = x.pow(2).mean(-1, keepdim=True)
218 x = x * torch.rsqrt(var + self.eps)
219 x = self.weight * x.to(dtype)
220 x = x * F.silu(gate.float())
221 return x.to(dtype)
222
223
224 def rotate_half(x: torch.Tensor) -> torch.Tensor:
225 x1 = x[..., : x.shape[-1] // 2]
226 x2 = x[..., x.shape[-1] // 2:]
227 return torch.cat((-x2, x1), dim=-1)
228
229
230 def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
231 """Partial rotary: только первые cos.shape[-1] каналов вращаются,
232 остальное (q_pass/k_pass) проходит без изменений."""
233 cos = cos.unsqueeze(unsqueeze_dim)
234 sin = sin.unsqueeze(unsqueeze_dim)
235 rotary_dim = cos.shape[-1]
236 q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
237 k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]
238 q_embed = torch.cat([(q_rot * cos) + (rotate_half(q_rot) * sin), q_pass], dim=-1)
239 k_embed = torch.cat([(k_rot * cos) + (rotate_half(k_rot) * sin), k_pass], dim=-1)
240 return q_embed, k_embed
241
242
243 class TextRotaryEmbedding(nn.Module):
244 """mRoPE с partial_rotary_factor. Для чистого текста (без vision)
245 3 позиционные "оси" (T,H,W) совпадают -> interleaved mRoPE
246 вырождается в обычный RoPE, но мы всё равно считаем честно тем же
247 путём, что и оригинал, для полной числовой идентичности."""
248
249 def __init__(self, head_dim=HEAD_DIM, partial_rotary_factor=PARTIAL_ROTARY_FACTOR,
250 theta=ROPE_THETA, mrope_section=MROPE_SECTION):
251 super().__init__()
252 self.rotary_dim = int(head_dim * partial_rotary_factor)
253 inv_freq = 1.0 / (theta ** (torch.arange(0, self.rotary_dim, 2, dtype=torch.float32) / self.rotary_dim))
254 self.register_buffer("inv_freq", inv_freq, persistent=False)
255 self.mrope_section = mrope_section
256
257 def _apply_interleaved_mrope(self, freqs: torch.Tensor) -> torch.Tensor:
258 # freqs: (3, bs, seq, rotary_dim // 2)
259 freqs_t = freqs[0].clone()
260 for dim, offset in enumerate((1, 2), start=1):
261 length = self.mrope_section[dim] * 3
262 idx = slice(offset, length, 3)
263 freqs_t[..., idx] = freqs[dim, ..., idx]
264 return freqs_t
265
266 @torch.no_grad()
267 def forward(self, x: torch.Tensor, position_ids: torch.LongTensor):
268 # position_ids: (bs, seq) -> реплицируем в 3 идентичные оси (text-only)
269 if position_ids.ndim == 2:
270 position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
271 inv_freq_exp = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1)
272 pos_exp = position_ids[:, :, None, :].float()
273 freqs = (inv_freq_exp @ pos_exp).transpose(2, 3) # (3, bs, seq, rotary_dim//2)
274 freqs = self._apply_interleaved_mrope(freqs) # (bs, seq, rotary_dim//2)
275 emb = torch.cat((freqs, freqs), dim=-1) # (bs, seq, rotary_dim)
276 return emb.cos().to(x.dtype), emb.sin().to(x.dtype)
277
278
279 def l2norm(x: torch.Tensor, dim: int = -1, eps: float = 1e-6):
280 return x * torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps)
281
282
283 def causal_conv1d(hidden_states: torch.Tensor, weight: torch.Tensor, activation: str = "silu"):
284 """hidden_states: (batch, channels, seq_len). Depthwise causal conv,
285 left-padded by kernel_size - 1, then activation."""
286 channels, seq_len = hidden_states.shape[1], hidden_states.shape[2]
287 padding = weight.shape[-1] - 1
288 out = F.conv1d(hidden_states, weight=weight.unsqueeze(1), bias=None, padding=padding, groups=channels)
289 out = out[:, :, :seq_len]
290 return F.silu(out) if activation == "silu" else out
291
292
293 def chunked_gated_delta_rule(query, key, value, g, beta, chunk_size=DELTA_CHUNK_SIZE):
294 """Port of `torch_chunk_gated_delta_rule` from `modeling_qwen3_5.py` — excluding
295 `initial_state`/`output_final_state` (training/full forward pass without
296 incremental cache). query/key/value: (b, h, t, d), g/beta: (b, h, t)."""
297 initial_dtype = query.dtype
298 query = l2norm(query, dim=-1, eps=1e-6)
299 key = l2norm(key, dim=-1, eps=1e-6)
300 query, key, value, beta, g = [x.contiguous().to(torch.float32) for x in (query, key, value, beta, g)]
301
302 b, h, t, k_dim = key.shape
303 v_dim = value.shape[-1]
304 pad = (chunk_size - t % chunk_size) % chunk_size
305 query = F.pad(query, (0, 0, 0, pad))
306 key = F.pad(key, (0, 0, 0, pad))
307 value = F.pad(value, (0, 0, 0, pad))
308 beta = F.pad(beta, (0, pad))
309 g = F.pad(g, (0, pad))
310 total_t = t + pad
311 scale = 1 / (query.shape[-1] ** 0.5)
312 query = query * scale
313
314 v_beta = value * beta.unsqueeze(-1)
315 k_beta = key * beta.unsqueeze(-1)
316 query, key, value, k_beta, v_beta = [
317 x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1]) for x in (query, key, value, k_beta, v_beta)
318 ]
319 g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)
320 mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0)
321
322 g = g.cumsum(dim=-1)
323 decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()
324 attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0)
325 for i in range(1, chunk_size):
326 row = attn[..., i, :i].clone()
327 sub = attn[..., :i, :i].clone()
328 attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2)
329 attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)
330 value = attn @ v_beta
331 k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))
332
333 last_state = torch.zeros(b, h, k_dim, v_dim, dtype=value.dtype, device=value.device)
334 core_out = torch.zeros_like(value)
335
336 for i in range(total_t // chunk_size):
337 q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]
338 attn_i = q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]
339 v_prime = k_cumdecay[:, :, i] @ last_state
340 v_new = v_i - v_prime
341 attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_state
342 core_out[:, :, i] = attn_inter + attn_i @ v_new
343 last_state = (
344 last_state * g[:, :, i, -1, None, None].exp()
345 + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new
346 )
347
348 core_out = core_out.reshape(core_out.shape[0], core_out.shape[1], -1, core_out.shape[-1])
349 core_out = core_out[:, :, :t]
350 core_out = core_out.transpose(1, 2).contiguous().to(initial_dtype)
351 return core_out
352
353
354 # =====================================================================
355 # 5. Наш собственный TransformerBlock (и подмодули linear_attn/self_attn)
356 # =====================================================================
357
358 class GatedDeltaNetBlock(nn.Module):
359 """Linear attention layer (Gated DeltaNet). Submodule names match the HF state_dict so that weights load directly."""
360
361 def __init__(self):
362 super().__init__()
363 self.conv1d = nn.Conv1d(
364 in_channels=DELTA_CONV_DIM, out_channels=DELTA_CONV_DIM, bias=False,
365 kernel_size=LINEAR_CONV_KERNEL_DIM, groups=DELTA_CONV_DIM,
366 padding=LINEAR_CONV_KERNEL_DIM - 1,
367 )
368 self.dt_bias = nn.Parameter(torch.ones(LINEAR_NUM_VALUE_HEADS))
369 A = torch.empty(LINEAR_NUM_VALUE_HEADS).uniform_(0.01, 16)
370 self.A_log = nn.Parameter(torch.log(A))
371 self.norm = RMSNormGated(LINEAR_VALUE_HEAD_DIM, eps=RMS_NORM_EPS)
372 self.out_proj = nn.Linear(DELTA_VALUE_DIM, HIDDEN_SIZE, bias=False)
373 self.in_proj_qkv = nn.Linear(HIDDEN_SIZE, DELTA_KEY_DIM * 2 + DELTA_VALUE_DIM, bias=False)
374 self.in_proj_z = nn.Linear(HIDDEN_SIZE, DELTA_VALUE_DIM, bias=False)
375 self.in_proj_b = nn.Linear(HIDDEN_SIZE, LINEAR_NUM_VALUE_HEADS, bias=False)
376 self.in_proj_a = nn.Linear(HIDDEN_SIZE, LINEAR_NUM_VALUE_HEADS, bias=False)
377
378 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
379 bsz, seq_len, _ = hidden_states.shape
380
381 mixed_qkv = self.in_proj_qkv(hidden_states).transpose(1, 2) # (b, conv_dim, t)
382 z = self.in_proj_z(hidden_states).reshape(bsz, seq_len, -1, LINEAR_VALUE_HEAD_DIM)
383 b = self.in_proj_b(hidden_states)
384 a = self.in_proj_a(hidden_states)
385
386 mixed_qkv = causal_conv1d(mixed_qkv, self.conv1d.weight.squeeze(1), activation="silu")
387 mixed_qkv = mixed_qkv.transpose(1, 2) # (b, t, conv_dim)
388
389 query, key, value = torch.split(mixed_qkv, [DELTA_KEY_DIM, DELTA_KEY_DIM, DELTA_VALUE_DIM], dim=-1)
390 query = query.reshape(bsz, seq_len, -1, LINEAR_KEY_HEAD_DIM)
391 key = key.reshape(bsz, seq_len, -1, LINEAR_KEY_HEAD_DIM)
392 value = value.reshape(bsz, seq_len, -1, LINEAR_VALUE_HEAD_DIM)
393
394 beta = b.sigmoid()
395 g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
396
397 rep = LINEAR_NUM_VALUE_HEADS // LINEAR_NUM_KEY_HEADS
398 if rep > 1:
399 query = query.repeat_interleave(rep, dim=2)
400 key = key.repeat_interleave(rep, dim=2)
401
402 # -> (b, h, t, d) for the chunked kernel
403 query_h = query.transpose(1, 2)
404 key_h = key.transpose(1, 2)
405 value_h = value.transpose(1, 2)
406 beta_h = beta.transpose(1, 2)
407 g_h = g.transpose(1, 2)
408
409 core_attn_out = chunked_gated_delta_rule(query_h, key_h, value_h, g_h, beta_h)
410
411 core_attn_out = core_attn_out.reshape(-1, LINEAR_VALUE_HEAD_DIM)
412 z_flat = z.reshape(-1, LINEAR_VALUE_HEAD_DIM)
413 core_attn_out = self.norm(core_attn_out, z_flat)
414 core_attn_out = core_attn_out.reshape(bsz, seq_len, -1)
415
416 return self.out_proj(core_attn_out)
417
418
419 class AttentionBlock(nn.Module):
420 """full_attention layer (Gated Attention + partial rotary mRoPE)."""
421
422 def __init__(self):
423 super().__init__()
424 self.num_key_value_groups = NUM_ATTENTION_HEADS // NUM_KEY_VALUE_HEADS
425 self.scaling = HEAD_DIM ** -0.5
426 self.q_proj = nn.Linear(HIDDEN_SIZE, NUM_ATTENTION_HEADS * HEAD_DIM * 2, bias=ATTENTION_BIAS)
427 self.k_proj = nn.Linear(HIDDEN_SIZE, NUM_KEY_VALUE_HEADS * HEAD_DIM, bias=ATTENTION_BIAS)
428 self.v_proj = nn.Linear(HIDDEN_SIZE, NUM_KEY_VALUE_HEADS * HEAD_DIM, bias=ATTENTION_BIAS)
429 self.o_proj = nn.Linear(NUM_ATTENTION_HEADS * HEAD_DIM, HIDDEN_SIZE, bias=ATTENTION_BIAS)
430 self.q_norm = RMSNorm(HEAD_DIM, eps=RMS_NORM_EPS)
431 self.k_norm = RMSNorm(HEAD_DIM, eps=RMS_NORM_EPS)
432
433 def forward(self, hidden_states, cos, sin, causal_mask):
434 bsz, seq_len, _ = hidden_states.shape
435 hidden_shape = (bsz, seq_len, -1, HEAD_DIM)
436
437 query_states, gate = torch.chunk(
438 self.q_proj(hidden_states).view(bsz, seq_len, -1, HEAD_DIM * 2), 2, dim=-1
439 )
440 gate = gate.reshape(bsz, seq_len, -1)
441
442 query_states = self.q_norm(query_states.view(hidden_shape)).transpose(1, 2)
443 key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
444 value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
445
446 query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
447
448 rep = self.num_key_value_groups
449 key_states = key_states.repeat_interleave(rep, dim=1)
450 value_states = value_states.repeat_interleave(rep, dim=1)
451
452 attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scaling
453 attn_weights = attn_weights + causal_mask
454 attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
455 attn_output = torch.matmul(attn_weights, value_states)
456 attn_output = attn_output.transpose(1, 2).contiguous().reshape(bsz, seq_len, -1)
457 attn_output = attn_output * torch.sigmoid(gate)
458
459 return self.o_proj(attn_output)
460
461
462 class MLPBlock(nn.Module):
463 def __init__(self):
464 super().__init__()
465 self.gate_proj = nn.Linear(HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False)
466 self.up_proj = nn.Linear(HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False)
467 self.down_proj = nn.Linear(INTERMEDIATE_SIZE, HIDDEN_SIZE, bias=False)
468
469 def forward(self, x):
470 return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
471
472
473 class TransformerBlock(nn.Module):
474 """Наш decoder layer -- linear_attn ИЛИ self_attn, в зависимости от
475 layer_type, + общий MLP. Имена атрибутов совпадают с HF state_dict."""
476
477 def __init__(self, layer_idx: int, layer_type: str = None):
478 super().__init__()
479 # layer_type задан явно -> используем его (нужно для MTP-слоя,
480 # который всегда full_attention и лежит ВНЕ LAYER_TYPES).
481 self.block_type = layer_type if layer_type is not None else LAYER_TYPES[layer_idx]
482 if self.block_type == "linear_attention":
483 self.linear_attn = GatedDeltaNetBlock()
484 else:
485 self.self_attn = AttentionBlock()
486 self.mlp = MLPBlock()
487 self.input_layernorm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
488 self.post_attention_layernorm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
489
490 def forward(self, hidden_states, cos, sin, causal_mask):
491 residual = hidden_states
492 hidden_states = self.input_layernorm(hidden_states)
493 if self.block_type == "linear_attention":
494 hidden_states = self.linear_attn(hidden_states)
495 else:
496 hidden_states = self.self_attn(hidden_states, cos, sin, causal_mask)
497 hidden_states = residual + hidden_states
498
499 residual = hidden_states
500 hidden_states = self.post_attention_layernorm(hidden_states)
501 hidden_states = self.mlp(hidden_states)
502 hidden_states = residual + hidden_states
503 return hidden_states
504
505
506 # =====================================================================
507 # 5b. MTP / NextN-голова (draft-слой спекулятивного декода)
508 # =====================================================================
509
510 class MTPBlock(nn.Module):
511 """Multi-Token Prediction голова -- ровно как в реальном чекпоинте
512 Qwen3.5/3.8 (mtp_num_hidden_layers=1, mtp_use_dedicated_embeddings=false).
513
514 Имена подмодулей СОВПАДАЮТ с ключами HF state_dict (top-level `mtp.`):
515 mtp.pre_fc_norm_embedding.weight
516 mtp.pre_fc_norm_hidden.weight
517 mtp.fc.weight [HIDDEN, 2*HIDDEN]
518 mtp.layers.0.* (обычный full_attention decoder)
519 mtp.norm.weight (final norm перед общим lm_head)
520
521 embed_tokens и lm_head ОБЩИЕ со стволом (dedicated embeddings = false),
522 здесь их нет. В GGUF это blk.{NUM_LAYERS}.nextn.eh_proj /
523 shared_head_norm + тензоры draft-слоя.
524
525 Механика (DeepSeek-style, порядок как в vLLM Qwen3NextMTP):
526 h = fc(cat(pre_fc_norm_embedding(emb_next), pre_fc_norm_hidden(h_prev)))
527 h = layers[0](h) # full attention
528 logits2 = lm_head(norm(h))
529 """
530
531 def __init__(self):
532 super().__init__()
533 self.pre_fc_norm_embedding = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
534 self.pre_fc_norm_hidden = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
535 self.fc = nn.Linear(2 * HIDDEN_SIZE, HIDDEN_SIZE, bias=False)
536 self.layers = nn.ModuleList([
537 TransformerBlock(layer_idx=NUM_LAYERS, layer_type="full_attention"),
538 ])
539 self.norm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
540
541 def forward(self, inputs_embeds, previous_hidden, cos, sin, causal_mask):
542 """inputs_embeds -- embed_tokens(токены, сдвинутые на +1)
543 previous_hidden -- hidden ствола ДО финального model.norm
544 (см. контракт: MTP сам применяет pre_fc_norm_hidden)."""
545 h = self.fc(torch.cat(
546 (self.pre_fc_norm_embedding(inputs_embeds),
547 self.pre_fc_norm_hidden(previous_hidden)),
548 dim=-1,
549 ))
550 h = self.layers[0](h, cos, sin, causal_mask)
551 return self.norm(h)
552
553
554 # =====================================================================
555 # 6. Полная модель
556 # =====================================================================
557
558 class JiRackQwen38Model(nn.Module):
559 def __init__(self):
560 super().__init__()
561 self.embed_tokens = nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE)
562 self.layers = nn.ModuleList([TransformerBlock(i) for i in range(NUM_LAYERS)])
563 self.norm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
564 self.rotary_emb = TextRotaryEmbedding()
565
566 def forward(self, input_ids: torch.LongTensor, return_prenorm: bool = False):
567 bsz, seq_len = input_ids.shape
568 hidden_states = self.embed_tokens(input_ids)
569
570 position_ids = torch.arange(seq_len, device=input_ids.device).unsqueeze(0).expand(bsz, -1)
571 cos, sin = self.rotary_emb(hidden_states, position_ids)
572
573 causal_mask = torch.full((seq_len, seq_len), float("-inf"), device=input_ids.device, dtype=torch.float32)
574 causal_mask = torch.triu(causal_mask, diagonal=1)[None, None, :, :].to(hidden_states.dtype)
575
576 for layer in self.layers:
577 hidden_states = layer(hidden_states, cos, sin, causal_mask)
578
579 if return_prenorm:
580 # Для MTP: голова сама применяет pre_fc_norm_hidden, финальный
581 # model.norm сюда НЕ входит (иначе двойная нормализация).
582 return self.norm(hidden_states), hidden_states, (cos, sin, causal_mask)
583 return self.norm(hidden_states)
584
585
586 class JiRackQwen38ForCausalLM(nn.Module):
587 """Верхнеуровневая модель. Имена атрибутов (`model`, `lm_head`)
588 совпадают с HF Qwen3_5ForCausalLM -- state_dict грузится напрямую."""
589
590 def __init__(self):
591 super().__init__()
592 self.model = JiRackQwen38Model()
593 self.lm_head = nn.Linear(HIDDEN_SIZE, VOCAB_SIZE, bias=False)
594 # MTP/NextN-голова: top-level атрибут `mtp` -- ключи state_dict
595 # получаются ровно mtp.*, как в реальном HF-чекпоинте.
596 self.mtp = MTPBlock()
597
598 def forward(self, input_ids: torch.LongTensor) -> torch.Tensor:
599 hidden_states = self.model(input_ids)
600 return self.lm_head(hidden_states)
601
602 def mtp_forward(self, input_ids: torch.LongTensor):
603 """Возвращает (logits_t1, logits_t2): обычные логиты ствола и
604 draft-логиты MTP-головы (предсказание токена t+2 из позиции t).
605 Для инференса/валидации MTP; в GGUF-конвертации не участвует."""
606 _, prenorm_hidden, (cos, sin, causal_mask) = self.model(
607 input_ids, return_prenorm=True)
608 logits_t1 = self.lm_head(self.model.norm(prenorm_hidden))
609
610 # Вход MTP -- эмбеддинги токенов, сдвинутых на +1 (следующий токен
611 # уже известен на шаге верификации). Последнюю позицию дублируем,
612 # чтобы сохранить форму (draft для неё не используется).
613 shifted = torch.cat([input_ids[:, 1:], input_ids[:, -1:]], dim=1)
614 emb_next = self.model.embed_tokens(shifted)
615 h2 = self.mtp(emb_next, prenorm_hidden, cos, sin, causal_mask)
616 logits_t2 = self.lm_head(h2)
617 return logits_t1, logits_t2
618
619 # -----------------------------------------------------------------
620 # Загрузка реальных весов (state_dict уже скачан конвертером через
621 # HF Hub -- сюда просто передаётся словарь тензоров)
622 # -----------------------------------------------------------------
623 def load_hf_state_dict(self, state_dict: dict, strict: bool = False):
624 """Грузит state_dict в формате оригинального Qwen3_5ForCausalLM.
625 mtp.* ключи теперь ГРУЗЯТСЯ в self.mtp (MTPBlock) -- фильтра
626 больше нет. strict=False оставлен из-за служебных lambda_-буферов
627 BitLinear, которых нет в реальном чекпоинте."""
628 missing, unexpected = self.load_state_dict(state_dict, strict=strict)
629 real_missing = [k for k in missing if not k.endswith("lambda_")]
630 if real_missing:
631 print(f"⚠️ {len(real_missing)} missing keys (первые 10): {real_missing[:10]}")
632 if unexpected:
633 print(f"⚠️ {len(unexpected)} unexpected keys (первые 10): {unexpected[:10]}")
634 return missing, unexpected
635
636 # -----------------------------------------------------------------
637 # Подсадка РЕАЛЬНЫХ mtp.* весов из оригинального HF-чекпоинта.
638 # Нужна, потому что старые JiRack .pt сохранены ДО появления MTPBlock
639 # и mtp-весов не содержат (там была бы случайная инициализация).
640 # -----------------------------------------------------------------
641 def load_mtp_from_hf_dir(self, hf_dir: str):
642 """Читает из папки оригинального HF-чекпоинта ТОЛЬКО mtp.* тензоры
643 (по index.json или перебором *.safetensors) и грузит их в self.mtp.
644 Возвращает число загруженных тензоров."""
645 import glob
646 import json as _json
647 from safetensors import safe_open
648
649 index_path = os.path.join(hf_dir, "model.safetensors.index.json")
650 mtp_map = {} # key -> shard file
651 if os.path.exists(index_path):
652 with open(index_path) as f:
653 weight_map = _json.load(f)["weight_map"]
654 for k, shard in weight_map.items():
655 if k.startswith("mtp."):
656 mtp_map[k] = os.path.join(hf_dir, shard)
657 else:
658 for shard in glob.glob(os.path.join(hf_dir, "*.safetensors")):
659 with safe_open(shard, framework="pt", device="cpu") as f:
660 for k in f.keys():
661 if k.startswith("mtp."):
662 mtp_map[k] = shard
663
664 if not mtp_map:
665 raise FileNotFoundError(
666 f"В {hf_dir} не найдено ни одного mtp.* тензора. Нужен "
667 f"ОРИГИНАЛЬНЫЙ чекпоинт Qwen3.8-27B (не JiRack-экспорт!). "
668 f"Если оригинальные шарды удалены -- перекачай с HF Hub "
669 f"только шарды с mtp.* по model.safetensors.index.json.")
670
671 own_dtype = next(self.mtp.parameters()).dtype
672 loaded = {}
673 by_shard = {}
674 for k, shard in mtp_map.items():
675 by_shard.setdefault(shard, []).append(k)
676 for shard, keys in by_shard.items():
677 with safe_open(shard, framework="pt", device="cpu") as f:
678 for k in keys:
679 loaded[k] = f.get_tensor(k).to(own_dtype)
680
681 missing, unexpected = self.load_state_dict(loaded, strict=False)
682 n_ok = len(loaded)
683 print(f"✅ MTP: загружено {n_ok} тензоров из {len(by_shard)} шардов ({hf_dir})")
684 if unexpected:
685 print(f"⚠️ MTP: unexpected keys: {unexpected[:10]}")
686 return n_ok
687
688 # -----------------------------------------------------------------
689 # JiRack-чекпоинт (свой формат, с lambda/patched_linear_names)
690 # -----------------------------------------------------------------
691 def save_checkpoint(self, path: str, lam: float, global_step: int = 0, patched_linear_names=None):
692 torch.save({
693 "model": self.state_dict(),
694 "lambda": lam,
695 "global_step": global_step,
696 "patched_linear_names": patched_linear_names or [],
697 }, path)
698
699 @classmethod
700 def load_checkpoint(cls, path: str, dtype=torch.bfloat16, include_gate_proj: bool = False):
701 ckpt = torch.load(path, map_location="cpu", weights_only=False)
702 model = cls()
703 patched_names = ckpt.get("patched_linear_names", [])
704 if patched_names:
705 for name in patched_names:
706 *parent_path, leaf = name.split(".")
707 parent = model
708 for p in parent_path:
709 parent = getattr(parent, p)
710 old = getattr(parent, leaf)
711 if not isinstance(old, BitLinear):
712 setattr(parent, leaf, BitLinear.from_linear(old))
713 else:
714 apply_bitlinear_patch(model, include_gate_proj=include_gate_proj)
715
716 has_mtp_in_ckpt = any(k.startswith("mtp.") for k in ckpt["model"])
717 model.load_state_dict(ckpt["model"], strict=False)
718 model = model.to(dtype)
719
720 if not has_mtp_in_ckpt:
721 print(
722 "⚠️ В этом .pt НЕТ mtp.* весов (чекпоинт сохранён до "
723 "появления MTPBlock). Голова сейчас со СЛУЧАЙНОЙ "
724 "инициализацией! Перед экспортом обязательно вызови "
725 "model.load_mtp_from_hf_dir('<папка оригинального HF-чекпоинта>')."
726 )
727
728 lam = ckpt.get("lambda", 0.0)
729 for m in model.modules():
730 if isinstance(m, BitLinear):
731 m.set_lambda(lam)
732
733 print(f"✅ Загружено. global_step={ckpt.get('global_step', 0)}, lambda={lam}")
734 return model
735
736
737 # =====================================================================
738 # 7. Демонстрация (без скачивания весов -- только структура)
739 # =====================================================================
740
741 if __name__ == "__main__":
742 print(ARCH_SUMMARY)
743 print(KEPT_FULL_PRECISION)
744
745 model = JiRackQwen38ForCausalLM()
746 n_params = sum(p.numel() for p in model.parameters())
747 print(f"✅ Модель собрана (случайные веса). Параметров: {n_params / 1e9:.2f}B")
748
749 patched = apply_bitlinear_patch(model)
750 print(f"✅ Заменено подмодулей на BitLinear: {len(patched)}")
751 print("Примеры первых 10:")
752 for n in patched[:10]:
753 print(f" {n}")
754
755 ids = torch.randint(0, VOCAB_SIZE, (1, 8))
756 with torch.no_grad():
757 logits = model(ids)
758 print(f"✅ Пробный forward (случайные веса) -> logits shape: {tuple(logits.shape)}")
759