config.json
4.3 KB · 169 lines · json Raw
1 {
2 "architectures": [
3 "Gemma4ForConditionalGeneration"
4 ],
5 "audio_token_id": 258881,
6 "boa_token_id": 256000,
7 "boi_token_id": 255999,
8 "dtype": "bfloat16",
9 "eoa_token_id": 258883,
10 "eoa_token_index": 258883,
11 "eoi_token_id": 258882,
12 "eos_token_id": [
13 1,
14 106
15 ],
16 "image_token_id": 258880,
17 "initializer_range": 0.02,
18 "model_type": "gemma4",
19 "text_config": {
20 "attention_bias": false,
21 "attention_dropout": 0.0,
22 "attention_k_eq_v": true,
23 "bos_token_id": 2,
24 "enable_moe_block": false,
25 "eos_token_id": 106,
26 "final_logit_softcapping": 30.0,
27 "global_head_dim": 512,
28 "head_dim": 256,
29 "hidden_activation": "gelu_pytorch_tanh",
30 "hidden_size": 5376,
31 "hidden_size_per_layer_input": 0,
32 "initializer_range": 0.02,
33 "intermediate_size": 21504,
34 "layer_types": [
35 "sliding_attention",
36 "sliding_attention",
37 "sliding_attention",
38 "sliding_attention",
39 "sliding_attention",
40 "full_attention",
41 "sliding_attention",
42 "sliding_attention",
43 "sliding_attention",
44 "sliding_attention",
45 "sliding_attention",
46 "full_attention",
47 "sliding_attention",
48 "sliding_attention",
49 "sliding_attention",
50 "sliding_attention",
51 "sliding_attention",
52 "full_attention",
53 "sliding_attention",
54 "sliding_attention",
55 "sliding_attention",
56 "sliding_attention",
57 "sliding_attention",
58 "full_attention",
59 "sliding_attention",
60 "sliding_attention",
61 "sliding_attention",
62 "sliding_attention",
63 "sliding_attention",
64 "full_attention",
65 "sliding_attention",
66 "sliding_attention",
67 "sliding_attention",
68 "sliding_attention",
69 "sliding_attention",
70 "full_attention",
71 "sliding_attention",
72 "sliding_attention",
73 "sliding_attention",
74 "sliding_attention",
75 "sliding_attention",
76 "full_attention",
77 "sliding_attention",
78 "sliding_attention",
79 "sliding_attention",
80 "sliding_attention",
81 "sliding_attention",
82 "full_attention",
83 "sliding_attention",
84 "sliding_attention",
85 "sliding_attention",
86 "sliding_attention",
87 "sliding_attention",
88 "full_attention",
89 "sliding_attention",
90 "sliding_attention",
91 "sliding_attention",
92 "sliding_attention",
93 "sliding_attention",
94 "full_attention"
95 ],
96 "max_position_embeddings": 262144,
97 "num_attention_heads": 32,
98 "num_global_key_value_heads": 4,
99 "num_hidden_layers": 60,
100 "num_key_value_heads": 16,
101 "num_kv_shared_layers": 0,
102 "pad_token_id": 0,
103 "rms_norm_eps": 1e-06,
104 "rope_parameters": {
105 "full_attention": {
106 "partial_rotary_factor": 0.25,
107 "rope_theta": 1000000.0,
108 "rope_type": "proportional"
109 },
110 "sliding_attention": {
111 "rope_theta": 10000.0,
112 "rope_type": "default"
113 }
114 },
115 "sliding_window": 1024,
116 "tie_word_embeddings": true,
117 "use_bidirectional_attention": "vision",
118 "use_cache": true,
119 "use_double_wide_mlp": false,
120 "vocab_size": 262144,
121 "vocab_size_per_layer_input": 262144
122 },
123 "tie_word_embeddings": true,
124 "transformers_version": "5.5.0.dev0",
125 "video_token_id": 258884,
126 "vision_config": {
127 "_name_or_path": "",
128 "attention_bias": false,
129 "attention_dropout": 0.0,
130 "chunk_size_feed_forward": 0,
131 "default_output_length": 280,
132 "dtype": "bfloat16",
133 "global_head_dim": 72,
134 "head_dim": 72,
135 "hidden_activation": "gelu_pytorch_tanh",
136 "hidden_size": 1152,
137 "id2label": {
138 "0": "LABEL_0",
139 "1": "LABEL_1"
140 },
141 "initializer_range": 0.02,
142 "intermediate_size": 4304,
143 "is_encoder_decoder": false,
144 "label2id": {
145 "LABEL_0": 0,
146 "LABEL_1": 1
147 },
148 "max_position_embeddings": 131072,
149 "model_type": "gemma4_vision",
150 "num_attention_heads": 16,
151 "num_hidden_layers": 27,
152 "num_key_value_heads": 16,
153 "output_attentions": false,
154 "output_hidden_states": false,
155 "patch_size": 16,
156 "pooling_kernel_size": 3,
157 "position_embedding_size": 10240,
158 "return_dict": true,
159 "rms_norm_eps": 1e-06,
160 "rope_parameters": {
161 "rope_theta": 100.0,
162 "rope_type": "default"
163 },
164 "standardize": true,
165 "use_clipped_linears": false
166 },
167 "vision_soft_tokens_per_image": 280
168 }
169