tokenizer_config.json
2.7 KB · 131 lines · json Raw
1 {
2 "add_prefix_space": null,
3 "backend": "tokenizers",
4 "bos_token": "<｜begin▁of▁sentence｜>",
5 "clean_up_tokenization_spaces": false,
6 "eos_token": "<｜end▁of▁sentence｜>",
7 "is_local": false,
8 "legacy": true,
9 "local_files_only": false,
10 "model_max_length": 16384,
11 "pad_token": "<｜end▁of▁sentence｜>",
12 "sp_model_kwargs": {},
13 "tokenizer_class": "Qwen2Tokenizer",
14 "unk_token": null,
15 "additional_special_tokens": [
16 "<|image|>",
17 "<|video|>",
18 "<|sound|>",
19 "<|voice|>",
20 "<|listening|>",
21 "<|vision|>",
22 "<|mood_happy|>",
23 "<|mood_sad|>",
24 "<|mood_angry|>",
25 "<|mood_neutral|>",
26 "<fim_prefix>",
27 "<fim_middle>",
28 "<fim_suffix>",
29 "<|action_start|>",
30 "<|action_end|>",
31 "<|trajectory_start|>",
32 "<|trajectory_end|>",
33 "<|joint_start|>",
34 "<|joint_end|>",
35 "<|sensor_start|>",
36 "<|sensor_end|>",
37 "<|command_start|>",
38 "<|command_end|>",
39 "<|state_start|>",
40 "<|state_end|>",
41 "<|pose|>",
42 "<|velocity|>",
43 "<|force|>",
44 "<|torque|>",
45 "<|gripper|>",
46 "<|navigation|>",
47 "<|obstacle|>",
48 "<|task_start|>",
49 "<|task_end|>",
50 "<|plan_start|>",
51 "<|plan_end|>",
52 "<|behavior_start|>",
53 "<|behavior_end|>",
54 "<|skill_start|>",
55 "<|skill_end|>",
56 "<|motor|>",
57 "<|servo|>",
58 "<|imu|>",
59 "<|lidar|>",
60 "<|camera|>",
61 "<|depth|>",
62 "<|waypoint|>",
63 "<|path|>",
64 "<|collision|>",
65 "<|grasp|>",
66 "<|release|>",
67 "<|homing|>",
68 "<|emergency_stop|>",
69 "<|calibration|>",
70 "<|manipulation|>",
71 "<|locomotion|>",
72 "<|feedback|>",
73 "<|control_loop|>",
74 "<|language|>",
75 "<|tool_call_start|>",
76 "<|tool_call_end|>",
77 "<|tool_result_start|>",
78 "<|tool_result_end|>",
79 "__SCIENCE__",
80 "__CODING__",
81 "__STOCK_EXCHANGE__",
82 "__MEDICINE__",
83 "__GOVERNMENT__",
84 "__NEWS__",
85 "__GENERAL__",
86 "__MATERIAL_SCIENCE__",
87 "__ELECTRONICS__",
88 "__MICROELECTRONICS__",
89 "__ENGINEERING__",
90 "__ROBOTICS__",
91 "__ENERGY__",
92 "__AUTOMOTIVE__",
93 "__AVIATION__",
94 "__MATH__",
95 "__PYTHON__",
96 "__C__",
97 "__CPP__",
98 "__C_SHARP__",
99 "__JAVA__",
100 "__JAVASCRIPT__",
101 "__TYPESCRIPT__",
102 "__RUST__",
103 "__GO__",
104 "__RUBY__",
105 "__PHP__",
106 "__SWIFT__",
107 "__KOTLIN__",
108 "__BASH__",
109 "__SQL__",
110 "__ASSEMBLY__",
111 "__PHILOSOPHY__",
112 "__LITERATURE__",
113 "__SOCIOLOGY__",
114 "__PSYCHOLOGY__",
115 "__POLITICAL_SCIENCE__",
116 "__CULTURAL_STUDIES__",
117 "__ETHNOGRAPHY__",
118 "__HUMAN_RIGHTS__",
119 "__COMPLIANCE__",
120 "__MILITARY__",
121 "__BANKING__",
122 "__OIL_INDUSTRY__",
123 "__LIGHT_INDUSTRY__",
124 "__NATURE__",
125 "__OCEAN__",
126 "__SPORT__",
127 "__CULINARY__",
128 "__TRAVEL__",
129 "__HOBBY__"
130 ]
131 }