Update tokenizer_config.json

tokenizer_config.json

@@ -3,33 +3,33 @@
   "add_eos_token": false,
   "add_prefix_space": null,
   "auto_map": {
-      "AutoConfig": [
-          "configuration_qwen2.Qwen2Config",
-          "qwen2_config.Qwen2Config"
-      ],
-      "AutoModel": [
-          "modeling_qwen2.Qwen2Model",
-          "qwen2.Qwen2ForCausalLM"
-      ],
-      "AutoModelForCausalLM": [
-          "modeling_qwen2.Qwen2ForCausalLM",
-          "qwen2.Qwen2ForCausalLM"
-      ],
-      "AutoModelForQuestionAnswering": [
-          "modeling_qwen2.Qwen2ForQuestionAnswering",
-          "qwen2.Qwen2ForQuestionAnswering"
-      ],
-      "AutoModelForSequenceClassification": [
-          "modeling_qwen2.Qwen2ForSequenceClassification",
-          "qwen2.Qwen2ForSequenceClassification"
-      ],
-      "AutoModelForTokenClassification": [
-          "modeling_qwen2.Qwen2ForTokenClassification",
-          "qwen2.Qwen2ForTokenClassification"
-      ]
+    "AutoConfig": [
+      "configuration_qwen2.Qwen2Config",
+      "qwen2_config.Qwen2Config"
+    ],
+    "AutoModel": [
+      "modeling_qwen2.Qwen2Model",
+      "qwen2.Qwen2ForCausalLM"
+    ],
+    "AutoModelForCausalLM": [
+      "modeling_qwen2.Qwen2ForCausalLM",
+      "qwen2.Qwen2ForCausalLM"
+    ],
+    "AutoModelForQuestionAnswering": [
+      "modeling_qwen2.Qwen2ForQuestionAnswering",
+      "qwen2.Qwen2ForQuestionAnswering"
+    ],
+    "AutoModelForSequenceClassification": [
+      "modeling_qwen2.Qwen2ForSequenceClassification",
+      "qwen2.Qwen2ForSequenceClassification"
+    ],
+    "AutoModelForTokenClassification": [
+      "modeling_qwen2.Qwen2ForTokenClassification",
+      "qwen2.Qwen2ForTokenClassification"
+    ]
   },
   "bos_token": "<|begin_of_text|>",
-     "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<|User|>:' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<|Assistant|>:<|tool_calls|>:<|tool_call|>:' + tool['type'] + '<|tool_sep|>:' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<|tool_call_end|>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<|tool_call|>:' + tool['type'] + '<|tool_sep|>:' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<|tool_call_end|>'}}{{'<|tool_calls_end|>:<|end_of_text|>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<|tool_outputs_end|>:' + message['content'] + '<|end_of_text|>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<|Assistant|>:' + content + '<|end_of_text|>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<|tool_outputs|>:<|tool_output|>:' + message['content'] + '<|tool_output_end|>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<|tool_output|>:' + message['content'] + '<|tool_output_end|>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<|tool_outputs_end|>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<|Assistant|>:'}}{% endif %}",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<|User|>:' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<|Assistant|>:<|tool_calls|>:<|tool_call|>:' + tool['type'] + '<|tool_sep|>:' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<|tool_call_end|>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<|tool_call|>:' + tool['type'] + '<|tool_sep|>:' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<|tool_call_end|>'}}{{'<|tool_calls_end|>:<|end_of_text|>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<|tool_outputs_end|>:' + message['content'] + '<|end_of_text|>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<|Assistant|>:' + content + '<|end_of_text|>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<|tool_outputs|>:<|tool_output|>:' + message['content'] + '<|tool_output_end|>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<|tool_output|>:' + message['content'] + '<|tool_output_end|>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<|tool_outputs_end|>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<|Assistant|>:'}}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|end_of_text|>",
   "legacy": true,
@@ -38,7 +38,7 @@
   "padding_side": "left",
   "sp_model_kwargs": {},
   "tokenizer_class": "Qwen2Tokenizer",
-   "transformers_version": "4.36.2",
+  "transformers_version": "4.36.2",
   "trust_remote_code": true,
   "unk_token": null,
   "use_default_system_prompt": false