Upload folder using huggingface_hub
Browse files- .DS_Store +0 -0
- .gitattributes +5 -0
- .ms_upload_cache +1 -0
- README.md +564 -0
- assets/intro.png +3 -0
- assets/logo.png +0 -0
- assets/overview.png +3 -0
- assets/perception.png +3 -0
- assets/planning.png +3 -0
- chat_template.jinja +154 -0
- chat_template.json +3 -0
- config.json +239 -0
- generation_config.json +13 -0
- merges.txt +0 -0
- model.safetensors +3 -0
- perception/config.json +101 -0
- perception/model.safetensors +3 -0
- planner-rl/config.json +42 -0
- planner-rl/model.safetensors +3 -0
- planner-sft/config.json +42 -0
- planner-sft/model.safetensors +3 -0
- preprocessor_config.json +21 -0
- tokenizer.json +3 -0
- tokenizer_config.json +305 -0
- vocab.json +0 -0
.DS_Store
ADDED
|
Binary file (8.2 kB). View file
|
|
|
.gitattributes
CHANGED
|
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
assets/intro.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
assets/overview.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
assets/perception.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
assets/planning.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
.ms_upload_cache
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"version": 3, "repo_id": "Qwen/Qwen-Drive-1.0-4B", "files": {"chat_template.jinja|1784816346.0|7756": {"hash": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "size": 7756, "status": "c"}, "assets/logo.png|1787737260.0|35572": {"hash": "dced19c1aebb227f094f1458d4980e6b75854fad6b933f1cf17f91aae5f9f2a2", "size": 35572, "status": "c"}, "README.md|1787738384.0|44209": {"hash": "faec64718442e416df89db5ca9f42ff38abc04dfde9e8404f27709b7b64e6d05", "size": 44209, "status": "c"}, "chat_template.json|1784816345.0|5143": {"hash": "539a99faa82b28397b7e8cc3c63473a42905438f07c1692de0c3f46e31202aec", "size": 5143, "status": "c"}, "assets/intro.png|1787737260.0|983596": {"hash": "2e09dce1266aac86ed85bcb072dce72e5118ee2951e54dbd330e35eb725a592b", "size": 983596, "status": "c"}, "generation_config.json|1784816346.0|244": {"hash": "303aba891d66ab63908a7b3cc9163bcb835fdf8b9f6301c73216f3f1eb3992dd", "size": 244, "status": "c"}, "config.json|1787154986.0|6091": {"hash": "91e765ca4a46e63b364b7e79f16bdb2b5d0e271684c566c03ae87d259c2d9ff8", "size": 6091, "status": "c"}, "perception/config.json|1787213337.0|1454": {"hash": "2a125ff844aa8798dfc1e66779174a31f163d15e453405fdc2f3c8eb3ad27d76", "size": 1454, "status": "c"}, "planner-rl/config.json|1787279627.0|913": {"hash": "4dbfc812ac2f4b5e6995220be6330f97f8f1c5827a8d5dbf061c50cf5e66d798", "size": 913, "status": "c"}, "assets/overview.png|1787737260.0|3198271": {"hash": "0fb61aa9d739ef960ff62ad5f88b5071e2a9168fb0271dde0fde4c5d92667d44", "size": 3198271, "status": "c"}, "planner-sft/config.json|1787279627.0|913": {"hash": "4dbfc812ac2f4b5e6995220be6330f97f8f1c5827a8d5dbf061c50cf5e66d798", "size": 913, "status": "c"}, "merges.txt|1784816345.0|3353259": {"hash": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", "size": 3353259, "status": "c"}, "assets/perception.png|1787737261.0|5679889": {"hash": "56eb16bc9c48da2acfede2605da27520be013bc8368a97d0290c3ca6a0ddae6d", "size": 5679889, "status": "c"}, "assets/planning.png|1787737261.0|8122708": {"hash": "80aeeb5baaaaf01b787b72084d99e0d4b04b063cf95bf8b0162630d6041844de", "size": 8122708, "status": "c"}, "perception/model.safetensors|1787213337.0|500368384": {"hash": "e964ca945f028bbbfadfdf9c1e47d31cfc3fe502d205eca0bb5a3d2c5ab450ae", "size": 500368384, "status": "c"}, "preprocessor_config.json|1784816345.0|390": {"hash": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", "size": 390, "status": "c"}, "tokenizer.json|1784816345.0|12807970": {"hash": "9825446f866dd1d1f6ab843f10af4014c168334d50d977f0afbd3ee304c4856a", "size": 12807970, "status": "c"}, "planner-sft/model.safetensors|1787235677.0|2079739550": {"hash": "dcf5989ed292799e77f539b21e5d8b701566676c1c5d13038135e97f96e2d7b4", "size": 2079739550, "status": "c"}, "planner-rl/model.safetensors|1787235679.0|2079739550": {"hash": "74478eb7ec5dea0d8144372ed7b60e6f508da7d9eb1522e567b391cad6dbaf29", "size": 2079739550, "status": "c"}, "model.safetensors|1787235675.0|9078630512": {"hash": "b9de4bf448f57485fdaa45c60b1eea8e41a4b6ae82ec0cee8855a1e0301caccc", "size": 9078630512, "status": "c"}, "tokenizer_config.json|1784816345.0|13815": {"hash": "28ed2feadb6891c10cd943b2b978f05ccca1f8b0093ec4e42809f43c7dc4a1a0", "size": 13815, "status": "c"}, "vocab.json|1784816345.0|6722759": {"hash": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003", "size": 6722759, "status": "c"}, ".DS_Store|1787858340.2658975|8196": {"hash": "ba352aaa94cae44c52bf3df1ee37853d5605dbff81be9a86e69d762532d290d6", "size": 8196, "status": "c"}}}
|
README.md
ADDED
|
@@ -0,0 +1,564 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
library_name: transformers
|
| 4 |
+
pipeline_tag: image-text-to-text
|
| 5 |
+
base_model:
|
| 6 |
+
- Qwen/Qwen3.5-4B
|
| 7 |
+
tags:
|
| 8 |
+
- autonomous-driving
|
| 9 |
+
- motion-planning
|
| 10 |
+
- 3d-perception
|
| 11 |
+
- visual-question-answering
|
| 12 |
+
- qwen
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
# Qwen-Drive-1.0-4B
|
| 16 |
+
|
| 17 |
+
<p align="center">
|
| 18 |
+
<img width="400px" src="assets/logo.png" alt="Qwen-Drive">
|
| 19 |
+
</p>
|
| 20 |
+
|
| 21 |
+
> [!Note]
|
| 22 |
+
> This repository contains the weights and configurations of Qwen-Drive-1.0 in the Hugging
|
| 23 |
+
> Face format. The accompanying code, demo data and documentation are released at
|
| 24 |
+
> [QwenLM/Qwen-Drive-1.0](https://github.com/QwenLM/Qwen-Drive-1.0).
|
| 25 |
+
|
| 26 |
+
Qwen-Drive-1.0 retains the architecture of the pretrained Qwen3.5 vision-language model and
|
| 27 |
+
integrates **3D perception**, **visual question answering**, and **motion planning** within a
|
| 28 |
+
unified framework. The natively multimodal Qwen3.5-4B serves as the shared VLM, with two
|
| 29 |
+
external modules attached: a **BEV perception head** jointly performing 3D object detection,
|
| 30 |
+
semantic occupancy prediction and BEV map segmentation, and a **Planning Expert** that
|
| 31 |
+
conditions on the shared VLM representations to generate future ego trajectories through flow
|
| 32 |
+
matching. The unchanged VLM answers free-form questions about driving scenes. A staged
|
| 33 |
+
training recipe combines driving supervision with general-purpose vision-language data, so
|
| 34 |
+
the model acquires driving-specific competence while preserving broad visual understanding
|
| 35 |
+
and instruction-following capability.
|
| 36 |
+
|
| 37 |
+
For more details, please refer to our blog post [Qwen-Drive-1.0](#).
|
| 38 |
+
|
| 39 |
+
## Highlights
|
| 40 |
+
|
| 41 |
+
- Qwen-Drive-1.0 is the first vision-language foundation model for autonomous driving that
|
| 42 |
+
unifies 3D perception and visual question answering at the pretraining stage and further
|
| 43 |
+
extends to motion planning, **while keeping the pretrained VLM architecture entirely
|
| 44 |
+
untouched**.
|
| 45 |
+
- An external BEV perception head serves as an explicit, inspectable 3D probe, jointly
|
| 46 |
+
learning 3D detection, semantic occupancy prediction, and BEV map segmentation, equipping
|
| 47 |
+
the same pretrained VLM with clear perception outputs while preserving highly competitive
|
| 48 |
+
vision-language performance.
|
| 49 |
+
- A staged training and data recipe unifies cross-dataset labels, rewrites responses, filters
|
| 50 |
+
samples for consistency, and combines driving data with general-purpose vision-language
|
| 51 |
+
supervision, supporting domain adaptation while mitigating catastrophic forgetting.
|
| 52 |
+
- A Planning Expert tailored to pretrained VLM representations generates future ego
|
| 53 |
+
trajectories with flow matching. Unified trajectory annotations enable joint training
|
| 54 |
+
across multiple public driving datasets and yield highly competitive results across
|
| 55 |
+
open-loop, pseudo-closed-loop, and closed-loop evaluations.
|
| 56 |
+
- Two Planning Experts are released: `planner-sft` supports both direct and reasoning
|
| 57 |
+
planning; `planner-rl` is further reward-optimized on NAVSIM PDMS, WOD-E2E RFS and a
|
| 58 |
+
displacement term, and runs best in the reasoning planning mode.
|
| 59 |
+
|
| 60 |
+
<p align="center">
|
| 61 |
+
<img src="assets/intro.png" alt="Qwen-Drive-1.0 performance overview" width="80%">
|
| 62 |
+
</p>
|
| 63 |
+
|
| 64 |
+
### Motion planning
|
| 65 |
+
|
| 66 |
+
<div style="font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;max-width:1000px;margin:0 auto;padding:16px 0">
|
| 67 |
+
<table style="width:100%;border-collapse:collapse;font-size:13px">
|
| 68 |
+
<thead><tr>
|
| 69 |
+
<th style="padding:10px 7px;text-align:left;font-weight:600;border-bottom:2px solid #7c3aed;color:#7c3aed"></th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">AutoVLA</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">SpanVLA</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">MindVLA-U1</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">Alpamayo-1.5</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">SimWAM<sub>IL</sub></th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">Qwen-Drive-1.0-SFT</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">Qwen-Drive-1.0-RL</th></tr></thead>
|
| 70 |
+
<tbody>
|
| 71 |
+
<tr><td colspan="8" style="padding:8px 12px;font-weight:600;color:#7c3aed;border-bottom:1px solid rgba(124, 58, 237, 0.2);background:rgba(124, 58, 237, 0.1)">Open-loop</td></tr>
|
| 72 |
+
<tr>
|
| 73 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">WOD-E2E (RFS val/test ↑)</td>
|
| 74 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--/7.56</td>
|
| 75 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 76 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>8.20/7.87</u></td>
|
| 77 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 78 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 79 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">7.95/7.78</td>
|
| 80 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>8.45/7.91</b></td>
|
| 81 |
+
</tr>
|
| 82 |
+
<tr>
|
| 83 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">WOD-E2E (ADE 5s val/test ↓)</td>
|
| 84 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--/2.96</td>
|
| 85 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 86 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>2.28/2.66</u></td>
|
| 87 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 88 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 89 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">2.31/<u>2.65</u></td>
|
| 90 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>1.27</b>/2.67</td>
|
| 91 |
+
</tr>
|
| 92 |
+
<tr>
|
| 93 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">PAI-AV (Avg. ADE 3s ↓)</td>
|
| 94 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 95 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 96 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 97 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>0.35</b></td>
|
| 98 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">0.41</td>
|
| 99 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>0.37</u></td>
|
| 100 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">0.42</td>
|
| 101 |
+
</tr>
|
| 102 |
+
<tr>
|
| 103 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">PAI-AV (Avg. ADE 5s ↓)</td>
|
| 104 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 105 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 106 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 107 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>1.05</b></td>
|
| 108 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 109 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>1.07</u></td>
|
| 110 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">1.11</td>
|
| 111 |
+
</tr>
|
| 112 |
+
<tr><td colspan="8" style="padding:8px 12px;font-weight:600;color:#7c3aed;border-bottom:1px solid rgba(124, 58, 237, 0.2);background:rgba(124, 58, 237, 0.1)">Pseudo-closed-loop</td></tr>
|
| 113 |
+
<tr>
|
| 114 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">NAVSIM (PDMS ↑)</td>
|
| 115 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">89.6</td>
|
| 116 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>90.3</u></td>
|
| 117 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 118 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 119 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>90.3</u></td>
|
| 120 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">88.2</td>
|
| 121 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>90.7</b></td>
|
| 122 |
+
</tr>
|
| 123 |
+
<tr>
|
| 124 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">NAVSIM best-of-6 (PDMS ↑)</td>
|
| 125 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 126 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 127 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 128 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 129 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 130 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>89.3</u></td>
|
| 131 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>91.4</b></td>
|
| 132 |
+
</tr>
|
| 133 |
+
<tr><td colspan="8" style="padding:8px 12px;font-weight:600;color:#7c3aed;border-bottom:1px solid rgba(124, 58, 237, 0.2);background:rgba(124, 58, 237, 0.1)">Closed-loop</td></tr>
|
| 134 |
+
<tr>
|
| 135 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">AlpaSim (at-fault score ↑)</td>
|
| 136 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 137 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 138 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">--</td>
|
| 139 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>0.45</b></td>
|
| 140 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">0.30</td>
|
| 141 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">0.27</td>
|
| 142 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>0.37</u></td>
|
| 143 |
+
</tr>
|
| 144 |
+
</tbody>
|
| 145 |
+
</table>
|
| 146 |
+
|
| 147 |
+
<p style="margin-top:12px;font-size:10px;opacity:0.7">
|
| 148 |
+
* AutoVLA and SimWAM train a separate model on each dataset.<br/>
|
| 149 |
+
* The SFT column reports Qwen-Drive-1.0-SFT conditioned on planning reasoning.<br/>
|
| 150 |
+
* IL denotes imitation learning.<br/>
|
| 151 |
+
* -- indicates that the method does not report a result on the corresponding benchmark.
|
| 152 |
+
</p>
|
| 153 |
+
|
| 154 |
+
</div>
|
| 155 |
+
|
| 156 |
+
With planning samples assembled purely from public sources, Qwen-Drive-1.0 unifies the
|
| 157 |
+
trajectory format across datasets and evaluates from open-loop prediction to closed-loop
|
| 158 |
+
driving. The SFT model is already competitive across all benchmarks. After reinforcement
|
| 159 |
+
learning, the model trades only a marginal open-loop displacement for comprehensive gains in
|
| 160 |
+
human-preference alignment and closed-loop safety.
|
| 161 |
+
|
| 162 |
+
<p align="center">
|
| 163 |
+
<img src="assets/planning.png" alt="Planning results" width="80%">
|
| 164 |
+
</p>
|
| 165 |
+
|
| 166 |
+
### Vision-language understanding
|
| 167 |
+
|
| 168 |
+
<div style="font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;max-width:1000px;margin:0 auto;padding:16px 0">
|
| 169 |
+
<table style="width:100%;border-collapse:collapse;font-size:13px">
|
| 170 |
+
<thead><tr>
|
| 171 |
+
<th style="padding:10px 7px;text-align:left;font-weight:600;border-bottom:2px solid #7c3aed;color:#7c3aed"></th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">InternVL3.5-8B-Instruct</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">LLaVA-OV2-8B</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">Qwen3.5-4B</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">Cosmos-Reason1-7B</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">Cosmos-Reason2-8B</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">Cosmos3-nano</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">MiMo-Embodied-7B</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">Alpamayo-1.5-10B</th><th style="padding:10px 7px;text-align:center;font-weight:500;border-bottom:2px solid #7c3aed;color:#7c3aed;font-size: 14px;">Qwen-Drive-1.0-SFT</th></tr></thead>
|
| 172 |
+
<tbody>
|
| 173 |
+
<tr><td colspan="10" style="padding:8px 12px;font-weight:600;color:#7c3aed;border-bottom:1px solid rgba(124, 58, 237, 0.2);background:rgba(124, 58, 237, 0.1)">Driving VQA</td></tr>
|
| 174 |
+
<tr>
|
| 175 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">LingoQA</td>
|
| 176 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">46.4</td>
|
| 177 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">41.2</td>
|
| 178 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">70.4</td>
|
| 179 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">45.2</td>
|
| 180 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">59.6</td>
|
| 181 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">65.0</td>
|
| 182 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>72.0</u></td>
|
| 183 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">64.0</td>
|
| 184 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>77.8</b></td>
|
| 185 |
+
</tr>
|
| 186 |
+
<tr>
|
| 187 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">Ego3D RMSE ↓</td>
|
| 188 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">23.01</td>
|
| 189 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">24.97</td>
|
| 190 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">13.17</td>
|
| 191 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">26.71</td>
|
| 192 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">12.62</td>
|
| 193 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">22.41</td>
|
| 194 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>9.85</u></td>
|
| 195 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">25.31</td>
|
| 196 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>7.78</b></td>
|
| 197 |
+
</tr>
|
| 198 |
+
<tr>
|
| 199 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">VLAD</td>
|
| 200 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">54.5</td>
|
| 201 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">58.7</td>
|
| 202 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>65.4</u></td>
|
| 203 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">33.6</td>
|
| 204 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">56.4</td>
|
| 205 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">57.7</td>
|
| 206 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">50.3</td>
|
| 207 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">9.1</td>
|
| 208 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>66.5</b></td>
|
| 209 |
+
</tr>
|
| 210 |
+
<tr>
|
| 211 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">SURDS</td>
|
| 212 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">32.8</td>
|
| 213 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">38.6</td>
|
| 214 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>53.0</u></td>
|
| 215 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">8.5</td>
|
| 216 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">19.5</td>
|
| 217 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">39.7</td>
|
| 218 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">43.1</td>
|
| 219 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">3.1</td>
|
| 220 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>66.1</b></td>
|
| 221 |
+
</tr>
|
| 222 |
+
<tr>
|
| 223 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">WaymoQA safety</td>
|
| 224 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">54.5</td>
|
| 225 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">49.7</td>
|
| 226 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">62.5</td>
|
| 227 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">39.5</td>
|
| 228 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">57.7</td>
|
| 229 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">56.9</td>
|
| 230 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>66.5</u></td>
|
| 231 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">42.6</td>
|
| 232 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>70.7</b></td>
|
| 233 |
+
</tr>
|
| 234 |
+
<tr>
|
| 235 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">WaymoQA all</td>
|
| 236 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">58.1</td>
|
| 237 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">55.2</td>
|
| 238 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">67.1</td>
|
| 239 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">43.9</td>
|
| 240 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">57.9</td>
|
| 241 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">58.4</td>
|
| 242 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>69.6</u></td>
|
| 243 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">44.4</td>
|
| 244 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>74.5</b></td>
|
| 245 |
+
</tr>
|
| 246 |
+
<tr>
|
| 247 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">CoC all</td>
|
| 248 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 249 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">0.6</td>
|
| 250 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">2.6</td>
|
| 251 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">3.2</td>
|
| 252 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">1.7</td>
|
| 253 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>4.0</u></td>
|
| 254 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 255 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">3.4</td>
|
| 256 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>41.3</b></td>
|
| 257 |
+
</tr>
|
| 258 |
+
<tr>
|
| 259 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">IH</td>
|
| 260 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">47.5</td>
|
| 261 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">54.0</td>
|
| 262 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">59.0</td>
|
| 263 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">30.5</td>
|
| 264 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">56.0</td>
|
| 265 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">2.0</td>
|
| 266 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>61.0</u></td>
|
| 267 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">3.0</td>
|
| 268 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>71.0</b></td>
|
| 269 |
+
</tr>
|
| 270 |
+
<tr><td colspan="10" style="padding:8px 12px;font-weight:600;color:#7c3aed;border-bottom:1px solid rgba(124, 58, 237, 0.2);background:rgba(124, 58, 237, 0.1)">Knowledge, Reasoning, and Recognition</td></tr>
|
| 271 |
+
<tr>
|
| 272 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">MMBench</td>
|
| 273 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">80.0</td>
|
| 274 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">82.7</td>
|
| 275 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>87.1</b></td>
|
| 276 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">80.0</td>
|
| 277 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">82.8</td>
|
| 278 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">79.6</td>
|
| 279 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 280 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">7.5</td>
|
| 281 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>85.5</u></td>
|
| 282 |
+
</tr>
|
| 283 |
+
<tr>
|
| 284 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">MMStar</td>
|
| 285 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">64.1</td>
|
| 286 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">64.9</td>
|
| 287 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>75.3</u></td>
|
| 288 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">63.5</td>
|
| 289 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">65.3</td>
|
| 290 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">66.7</td>
|
| 291 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">22.4</td>
|
| 292 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">26.1</td>
|
| 293 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>75.9</b></td>
|
| 294 |
+
</tr>
|
| 295 |
+
<tr>
|
| 296 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">MMMU</td>
|
| 297 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">62.0</td>
|
| 298 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">54.7</td>
|
| 299 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>73.4</b></td>
|
| 300 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">54.2</td>
|
| 301 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">59.1</td>
|
| 302 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">60.9</td>
|
| 303 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 304 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">27.4</td>
|
| 305 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>72.7</u></td>
|
| 306 |
+
</tr>
|
| 307 |
+
<tr>
|
| 308 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">MMMU-Pro std</td>
|
| 309 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">46.4</td>
|
| 310 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">36.3</td>
|
| 311 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>64.9</b></td>
|
| 312 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">38.4</td>
|
| 313 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">36.1</td>
|
| 314 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">46.4</td>
|
| 315 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">27.4</td>
|
| 316 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">15.6</td>
|
| 317 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>62.7</u></td>
|
| 318 |
+
</tr>
|
| 319 |
+
<tr>
|
| 320 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">MMMU-Pro vis</td>
|
| 321 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">42.3</td>
|
| 322 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">26.0</td>
|
| 323 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>61.3</b></td>
|
| 324 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">35.8</td>
|
| 325 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">43.5</td>
|
| 326 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">40.8</td>
|
| 327 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">28.1</td>
|
| 328 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">13.5</td>
|
| 329 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>59.7</u></td>
|
| 330 |
+
</tr>
|
| 331 |
+
<tr>
|
| 332 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">CharXiv</td>
|
| 333 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">41.7</td>
|
| 334 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">40.1</td>
|
| 335 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>65.1</b></td>
|
| 336 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">39.7</td>
|
| 337 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">42.5</td>
|
| 338 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">42.1</td>
|
| 339 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">57.5</td>
|
| 340 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">1.5</td>
|
| 341 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>64.4</u></td>
|
| 342 |
+
</tr>
|
| 343 |
+
<tr>
|
| 344 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">OCRBench</td>
|
| 345 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">83.2</td>
|
| 346 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">79.3</td>
|
| 347 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>86.9</u></td>
|
| 348 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">85.2</td>
|
| 349 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>87.0</b></td>
|
| 350 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">85.2</td>
|
| 351 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">78.8</td>
|
| 352 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">3.2</td>
|
| 353 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">86.4</td>
|
| 354 |
+
</tr>
|
| 355 |
+
<tr>
|
| 356 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">RealWorldQA</td>
|
| 357 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">66.9</td>
|
| 358 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">71.8</td>
|
| 359 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>76.3</u></td>
|
| 360 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">67.5</td>
|
| 361 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">67.5</td>
|
| 362 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">69.7</td>
|
| 363 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">28.5</td>
|
| 364 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">46.9</td>
|
| 365 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>79.0</b></td>
|
| 366 |
+
</tr>
|
| 367 |
+
<tr>
|
| 368 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">SimpleVQA</td>
|
| 369 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">40.8</td>
|
| 370 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">36.7</td>
|
| 371 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>47.8</b></td>
|
| 372 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">45.0</td>
|
| 373 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">45.3</td>
|
| 374 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">45.0</td>
|
| 375 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 376 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 377 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>46.1</u></td>
|
| 378 |
+
</tr>
|
| 379 |
+
<tr>
|
| 380 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">CountQA</td>
|
| 381 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">20.9</td>
|
| 382 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">22.6</td>
|
| 383 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>35.9</b></td>
|
| 384 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">18.5</td>
|
| 385 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">22.3</td>
|
| 386 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">23.6</td>
|
| 387 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">22.6</td>
|
| 388 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">4.7</td>
|
| 389 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>31.7</u></td>
|
| 390 |
+
</tr>
|
| 391 |
+
<tr><td colspan="10" style="padding:8px 12px;font-weight:600;color:#7c3aed;border-bottom:1px solid rgba(124, 58, 237, 0.2);background:rgba(124, 58, 237, 0.1)">Spatial Understanding and Grounding</td></tr>
|
| 392 |
+
<tr>
|
| 393 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">EmbSpatial</td>
|
| 394 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">74.2</td>
|
| 395 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>78.4</u></td>
|
| 396 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">76.0</td>
|
| 397 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">68.8</td>
|
| 398 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">77.6</td>
|
| 399 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">77.9</td>
|
| 400 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">45.1</td>
|
| 401 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">20.6</td>
|
| 402 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>78.9</b></td>
|
| 403 |
+
</tr>
|
| 404 |
+
<tr>
|
| 405 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">ERQA</td>
|
| 406 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">42.0</td>
|
| 407 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">42.3</td>
|
| 408 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>46.3</u></td>
|
| 409 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">38.5</td>
|
| 410 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">43.3</td>
|
| 411 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">41.3</td>
|
| 412 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">39.8</td>
|
| 413 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">27.5</td>
|
| 414 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>48.5</b></td>
|
| 415 |
+
</tr>
|
| 416 |
+
<tr>
|
| 417 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">RefSpatial</td>
|
| 418 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 419 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 420 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>54.5</b></td>
|
| 421 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">0.4</td>
|
| 422 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>51.8</u></td>
|
| 423 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 424 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">2.2</td>
|
| 425 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 426 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">50.8</td>
|
| 427 |
+
</tr>
|
| 428 |
+
<tr>
|
| 429 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">Omni3D</td>
|
| 430 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 431 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 432 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>47.4</b></td>
|
| 433 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 434 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">32.9</td>
|
| 435 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">32.3</td>
|
| 436 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 437 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 438 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>45.8</u></td>
|
| 439 |
+
</tr>
|
| 440 |
+
<tr>
|
| 441 |
+
<td style="padding:7px 7px;padding-left:20px;border-bottom:1px solid rgba(128, 128, 128, 0.15);">ODinW13</td>
|
| 442 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 443 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 444 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><u>40.8</u></td>
|
| 445 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">4.8</td>
|
| 446 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">40.2</td>
|
| 447 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">35.9</td>
|
| 448 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 449 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)">–</td>
|
| 450 |
+
<td style="padding:7px 7px;text-align:center;border-bottom:1px solid rgba(128, 128, 128, 0.15)"><b>45.9</b></td>
|
| 451 |
+
</tr>
|
| 452 |
+
</tbody>
|
| 453 |
+
</table>
|
| 454 |
+
|
| 455 |
+
<p style="margin-top:12px;font-size:10px;opacity:0.7">
|
| 456 |
+
* All benchmarks use the same high-certainty decoding settings (greedy=false, top-p=0.001, top-k=1, temperature=0.01, repetition_penalty=1.0, presence_penalty=0.0) to more directly reflect model capability.<br/>
|
| 457 |
+
* LingoQA is scored with Qwen-Plus as the judge instead of the official LingoJudge, which we found to score leniently and inconsistently across scenarios. Under the official LingoJudge protocol, Qwen-Drive-1.0-SFT obtains a LingoScore of 79.4.<br/>
|
| 458 |
+
* The same judge scores every method on each benchmark.<br/>
|
| 459 |
+
* -- marks an invalid or unparsable response.
|
| 460 |
+
</p>
|
| 461 |
+
|
| 462 |
+
</div>
|
| 463 |
+
|
| 464 |
+
**Driving VQA.** Qwen-Drive-1.0-SFT leads both general-purpose VLMs and driving or embodied
|
| 465 |
+
specialists on driving question answering, with the sharpest spatial understanding and a more
|
| 466 |
+
accurate sense of physical scale. The gain in causal reasoning is the most pronounced, and its
|
| 467 |
+
driving-decision capability generalizes from broad driving data rather than memorizing
|
| 468 |
+
specific scenarios.
|
| 469 |
+
|
| 470 |
+
**General vision-language understanding.** Large-scale driving training causes no evident
|
| 471 |
+
catastrophic forgetting. Qwen-Drive-1.0-SFT largely preserves its general vision-language
|
| 472 |
+
capability, performing on par with the base Qwen3.5-4B across the knowledge, reasoning,
|
| 473 |
+
recognition and spatial understanding benchmarks, while well preserving its
|
| 474 |
+
instruction-following capability.
|
| 475 |
+
|
| 476 |
+
### 3D perception
|
| 477 |
+
|
| 478 |
+
<p align="center">
|
| 479 |
+
<img src="assets/perception.png" alt="3D perception results" width="100%">
|
| 480 |
+
</p>
|
| 481 |
+
|
| 482 |
+
A single Qwen-Drive-1.0-SFT model produces coherent 3D detection, semantic occupancy, and BEV
|
| 483 |
+
map segmentation that reflect genuine 3D structure rather than inheriting label noise. The BEV
|
| 484 |
+
perception head is deliberately kept simple, so that it serves as an explicit, inspectable 3D
|
| 485 |
+
probe of the shared VLM representations rather than a specialist aimed at advanced perception
|
| 486 |
+
benchmarks.
|
| 487 |
+
|
| 488 |
+
## Repository Contents
|
| 489 |
+
|
| 490 |
+
Everything ships in one directory. The VLM sits at its root, shared by every task, and each
|
| 491 |
+
task head in a subfolder beside it.
|
| 492 |
+
|
| 493 |
+
```
|
| 494 |
+
Qwen-Drive-1.0-4B/ 9.1 GB the VLM, which on its own serves the VQA mode
|
| 495 |
+
├── planner-sft/ 2.1 GB Planning Expert, imitation-trained
|
| 496 |
+
├── planner-rl/ 2.1 GB Planning Expert after reward optimization
|
| 497 |
+
└── perception/ 0.5 GB BEV perception head
|
| 498 |
+
```
|
| 499 |
+
|
| 500 |
+
## Quickstart
|
| 501 |
+
|
| 502 |
+
Install the inference code from the GitHub repository:
|
| 503 |
+
|
| 504 |
+
```bash
|
| 505 |
+
git clone https://github.com/QwenLM/Qwen-Drive-1.0 qwen-drive && cd qwen-drive
|
| 506 |
+
pip install -e . --no-build-isolation
|
| 507 |
+
```
|
| 508 |
+
|
| 509 |
+
Download the weights (the VLM at the root plus every task head in its subfolder):
|
| 510 |
+
|
| 511 |
+
```bash
|
| 512 |
+
hf download Qwen/Qwen-Drive-1.0-4B --local-dir Qwen-Drive-1.0-4B
|
| 513 |
+
```
|
| 514 |
+
|
| 515 |
+
Load the VLM with a Planning Expert attached and predict trajectories:
|
| 516 |
+
|
| 517 |
+
```python
|
| 518 |
+
import torch
|
| 519 |
+
from qwen_drive import InferenceMode, QwenDriveForPlanning
|
| 520 |
+
from qwen_drive.benchmarks import read_scene_file
|
| 521 |
+
from qwen_drive.images import ImageArchive
|
| 522 |
+
|
| 523 |
+
model = QwenDriveForPlanning.from_pretrained(
|
| 524 |
+
"Qwen-Drive-1.0-4B",
|
| 525 |
+
planner="Qwen-Drive-1.0-4B/planner-rl",
|
| 526 |
+
dtype=torch.bfloat16,
|
| 527 |
+
attn_implementation="flash_attention_2",
|
| 528 |
+
).to("cuda").eval()
|
| 529 |
+
|
| 530 |
+
scene = next(
|
| 531 |
+
read_scene_file(
|
| 532 |
+
"data/demo/planning_scenes.jsonl",
|
| 533 |
+
image_archive=ImageArchive.open("data/demo/frames.parquet"),
|
| 534 |
+
)
|
| 535 |
+
).scene
|
| 536 |
+
|
| 537 |
+
result = model.run(InferenceMode.REASONING_PLANNING, scene=scene, num_samples=6)
|
| 538 |
+
print(result.reasoning)
|
| 539 |
+
print(result.trajectories.shape) # (6, 50, 3) -> (x, y, heading), 5 s at 10 Hz
|
| 540 |
+
```
|
| 541 |
+
|
| 542 |
+
- **VQA.** Load the root directory without a planner and the VLM answers free-form driving
|
| 543 |
+
questions on its own.
|
| 544 |
+
- **Perception.** Attach the BEV head with
|
| 545 |
+
`QwenDrivePerception.from_pretrained("Qwen-Drive-1.0-4B/perception")`; it binds the same
|
| 546 |
+
VLM and outputs 3D detections, occupancy and BEV map segmentation.
|
| 547 |
+
- `planner-rl` was reward-optimized only on reasoning-conditioned rollouts, so run it in the
|
| 548 |
+
reasoning planning mode. `planner-sft` covers both direct and reasoning planning.
|
| 549 |
+
|
| 550 |
+
`scripts/demo.py` in the GitHub repository runs four bundled planning scenes and six
|
| 551 |
+
perception frames end to end without any extra data.
|
| 552 |
+
|
| 553 |
+
## Citation
|
| 554 |
+
|
| 555 |
+
If you find our work helpful, feel free to give us a cite.
|
| 556 |
+
|
| 557 |
+
```bibtex
|
| 558 |
+
@article{qwendrive2026,
|
| 559 |
+
title = {Qwen-Drive-1.0: An Initial Step towards a Vision-Language Foundation Model for Autonomous Driving},
|
| 560 |
+
author = {Zhou, Xin and Zhao, Zongchuang and Yang, Zhibo and Li, Mingsheng and Zhong, Humen and Bai, Shuai and Chu, Du and Chen, Ruizhe and Li, Zhaohai and Tang, Jun and Wang, Qiuyue and Yang, Mingkun and Zhang, Jiazhao and Liu, Dayiheng and Liang, Dingkang and Bai, Xiang},
|
| 561 |
+
journal= {arXiv preprint arXiv:xxxx.xxxxx},
|
| 562 |
+
year = {2026}
|
| 563 |
+
}
|
| 564 |
+
```
|
assets/intro.png
ADDED
|
Git LFS Details
|
assets/logo.png
ADDED
|
assets/overview.png
ADDED
|
Git LFS Details
|
assets/perception.png
ADDED
|
Git LFS Details
|
assets/planning.png
ADDED
|
Git LFS Details
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- set image_count = namespace(value=0) %}
|
| 2 |
+
{%- set video_count = namespace(value=0) %}
|
| 3 |
+
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
| 4 |
+
{%- if content is string %}
|
| 5 |
+
{{- content }}
|
| 6 |
+
{%- elif content is iterable and content is not mapping %}
|
| 7 |
+
{%- for item in content %}
|
| 8 |
+
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
|
| 9 |
+
{%- if is_system_content %}
|
| 10 |
+
{{- raise_exception('System message cannot contain images.') }}
|
| 11 |
+
{%- endif %}
|
| 12 |
+
{%- if do_vision_count %}
|
| 13 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{%- if add_vision_id %}
|
| 16 |
+
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
| 19 |
+
{%- elif 'video' in item or item.type == 'video' %}
|
| 20 |
+
{%- if is_system_content %}
|
| 21 |
+
{{- raise_exception('System message cannot contain videos.') }}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
{%- if do_vision_count %}
|
| 24 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- if add_vision_id %}
|
| 27 |
+
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
| 30 |
+
{%- elif 'text' in item %}
|
| 31 |
+
{{- item.text }}
|
| 32 |
+
{%- else %}
|
| 33 |
+
{{- raise_exception('Unexpected item type in content.') }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{%- endfor %}
|
| 36 |
+
{%- elif content is none or content is undefined %}
|
| 37 |
+
{{- '' }}
|
| 38 |
+
{%- else %}
|
| 39 |
+
{{- raise_exception('Unexpected content type.') }}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{%- endmacro %}
|
| 42 |
+
{%- if not messages %}
|
| 43 |
+
{{- raise_exception('No messages provided.') }}
|
| 44 |
+
{%- endif %}
|
| 45 |
+
{%- if tools and tools is iterable and tools is not mapping %}
|
| 46 |
+
{{- '<|im_start|>system\n' }}
|
| 47 |
+
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
|
| 48 |
+
{%- for tool in tools %}
|
| 49 |
+
{{- "\n" }}
|
| 50 |
+
{{- tool | tojson }}
|
| 51 |
+
{%- endfor %}
|
| 52 |
+
{{- "\n</tools>" }}
|
| 53 |
+
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 54 |
+
{%- if messages[0].role == 'system' %}
|
| 55 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 56 |
+
{%- if content %}
|
| 57 |
+
{{- '\n\n' + content }}
|
| 58 |
+
{%- endif %}
|
| 59 |
+
{%- endif %}
|
| 60 |
+
{{- '<|im_end|>\n' }}
|
| 61 |
+
{%- else %}
|
| 62 |
+
{%- if messages[0].role == 'system' %}
|
| 63 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 64 |
+
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
| 65 |
+
{%- endif %}
|
| 66 |
+
{%- endif %}
|
| 67 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 68 |
+
{%- for message in messages[::-1] %}
|
| 69 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 70 |
+
{%- if ns.multi_step_tool and message.role == "user" %}
|
| 71 |
+
{%- set content = render_content(message.content, false)|trim %}
|
| 72 |
+
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
|
| 73 |
+
{%- set ns.multi_step_tool = false %}
|
| 74 |
+
{%- set ns.last_query_index = index %}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- if ns.multi_step_tool %}
|
| 79 |
+
{{- raise_exception('No user query found in messages.') }}
|
| 80 |
+
{%- endif %}
|
| 81 |
+
{%- for message in messages %}
|
| 82 |
+
{%- set content = render_content(message.content, true)|trim %}
|
| 83 |
+
{%- if message.role == "system" %}
|
| 84 |
+
{%- if not loop.first %}
|
| 85 |
+
{{- raise_exception('System message must be at the beginning.') }}
|
| 86 |
+
{%- endif %}
|
| 87 |
+
{%- elif message.role == "user" %}
|
| 88 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 89 |
+
{%- elif message.role == "assistant" %}
|
| 90 |
+
{%- set reasoning_content = '' %}
|
| 91 |
+
{%- if message.reasoning_content is string %}
|
| 92 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 93 |
+
{%- else %}
|
| 94 |
+
{%- if '</think>' in content %}
|
| 95 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 96 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 97 |
+
{%- endif %}
|
| 98 |
+
{%- endif %}
|
| 99 |
+
{%- set reasoning_content = reasoning_content|trim %}
|
| 100 |
+
{%- if loop.index0 > ns.last_query_index %}
|
| 101 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
| 102 |
+
{%- else %}
|
| 103 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 104 |
+
{%- endif %}
|
| 105 |
+
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
| 106 |
+
{%- for tool_call in message.tool_calls %}
|
| 107 |
+
{%- if tool_call.function is defined %}
|
| 108 |
+
{%- set tool_call = tool_call.function %}
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{%- if loop.first %}
|
| 111 |
+
{%- if content|trim %}
|
| 112 |
+
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 113 |
+
{%- else %}
|
| 114 |
+
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 115 |
+
{%- endif %}
|
| 116 |
+
{%- else %}
|
| 117 |
+
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 118 |
+
{%- endif %}
|
| 119 |
+
{%- if tool_call.arguments is defined %}
|
| 120 |
+
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 121 |
+
{{- '<parameter=' + args_name + '>\n' }}
|
| 122 |
+
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
| 123 |
+
{{- args_value }}
|
| 124 |
+
{{- '\n</parameter>\n' }}
|
| 125 |
+
{%- endfor %}
|
| 126 |
+
{%- endif %}
|
| 127 |
+
{{- '</function>\n</tool_call>' }}
|
| 128 |
+
{%- endfor %}
|
| 129 |
+
{%- endif %}
|
| 130 |
+
{{- '<|im_end|>\n' }}
|
| 131 |
+
{%- elif message.role == "tool" %}
|
| 132 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 133 |
+
{{- '<|im_start|>user' }}
|
| 134 |
+
{%- endif %}
|
| 135 |
+
{{- '\n<tool_response>\n' }}
|
| 136 |
+
{{- content }}
|
| 137 |
+
{{- '\n</tool_response>' }}
|
| 138 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 139 |
+
{{- '<|im_end|>\n' }}
|
| 140 |
+
{%- elif loop.last %}
|
| 141 |
+
{{- '<|im_end|>\n' }}
|
| 142 |
+
{%- endif %}
|
| 143 |
+
{%- else %}
|
| 144 |
+
{{- raise_exception('Unexpected message role.') }}
|
| 145 |
+
{%- endif %}
|
| 146 |
+
{%- endfor %}
|
| 147 |
+
{%- if add_generation_prompt %}
|
| 148 |
+
{{- '<|im_start|>assistant\n' }}
|
| 149 |
+
{%- if enable_thinking is defined and enable_thinking is false %}
|
| 150 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 151 |
+
{%- else %}
|
| 152 |
+
{{- '<think>\n' }}
|
| 153 |
+
{%- endif %}
|
| 154 |
+
{%- endif %}
|
chat_template.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {{- \"<|vision_start|><|image_pad|><|vision_end|>\"}}\n {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %}\n {{- \"<|audio_start|><|audio_pad|><|audio_end|>\"}}\n {%- elif content.type == 'video' or 'video' in content %}\n {{- \"<|vision_start|><|video_pad|><|vision_end|>\"}}\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content_item in message.content %}\n {%- if 'text' in content_item %}\n {{- content_item.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and message.content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {{- \"<|vision_start|><|image_pad|><|vision_end|>\"}}\n {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %}\n {{- \"<|audio_start|><|audio_pad|><|audio_end|>\"}}\n {%- elif content.type == 'video' or 'video' in content %}\n {{- \"<|vision_start|><|video_pad|><|vision_end|>\"}}\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}"
|
| 3 |
+
}
|
config.json
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_name_or_path": "",
|
| 3 |
+
"architectures": [
|
| 4 |
+
"QwenDriveForPlanning"
|
| 5 |
+
],
|
| 6 |
+
"chunk_size_feed_forward": 0,
|
| 7 |
+
"current_image_pixels": 921600,
|
| 8 |
+
"dtype": "bfloat16",
|
| 9 |
+
"expert_config": {
|
| 10 |
+
"_name_or_path": "",
|
| 11 |
+
"architectures": null,
|
| 12 |
+
"chunk_size_feed_forward": 0,
|
| 13 |
+
"dtype": null,
|
| 14 |
+
"ego_status_dim": 8,
|
| 15 |
+
"fourier_max_frequency": 16.0,
|
| 16 |
+
"fourier_num_features": 16,
|
| 17 |
+
"head_dim": 256,
|
| 18 |
+
"hidden_size": 1024,
|
| 19 |
+
"history_dynamics_dim": 2,
|
| 20 |
+
"id2label": {
|
| 21 |
+
"0": "LABEL_0",
|
| 22 |
+
"1": "LABEL_1"
|
| 23 |
+
},
|
| 24 |
+
"intermediate_size": 3584,
|
| 25 |
+
"is_encoder_decoder": false,
|
| 26 |
+
"label2id": {
|
| 27 |
+
"LABEL_0": 0,
|
| 28 |
+
"LABEL_1": 1
|
| 29 |
+
},
|
| 30 |
+
"layers_per_kv": 4,
|
| 31 |
+
"model_type": "qwen_drive_planning_expert",
|
| 32 |
+
"mrope_section": [
|
| 33 |
+
11,
|
| 34 |
+
11,
|
| 35 |
+
10
|
| 36 |
+
],
|
| 37 |
+
"nav_command_classes": 3,
|
| 38 |
+
"num_attention_heads": 16,
|
| 39 |
+
"num_hidden_layers": 32,
|
| 40 |
+
"num_key_value_heads": 4,
|
| 41 |
+
"output_attentions": false,
|
| 42 |
+
"output_hidden_states": false,
|
| 43 |
+
"partial_rotary_factor": 0.25,
|
| 44 |
+
"problem_type": null,
|
| 45 |
+
"return_dict": true,
|
| 46 |
+
"rms_norm_eps": 1e-05,
|
| 47 |
+
"rope_theta": 10000000.0,
|
| 48 |
+
"time_embed_dim": 128,
|
| 49 |
+
"time_embed_scale": 1000.0
|
| 50 |
+
},
|
| 51 |
+
"history_image_pixels": 174080,
|
| 52 |
+
"id2label": {
|
| 53 |
+
"0": "LABEL_0",
|
| 54 |
+
"1": "LABEL_1"
|
| 55 |
+
},
|
| 56 |
+
"image_patch_size": 16,
|
| 57 |
+
"image_spatial_merge_size": 2,
|
| 58 |
+
"image_temporal_patch_size": 2,
|
| 59 |
+
"is_encoder_decoder": false,
|
| 60 |
+
"label2id": {
|
| 61 |
+
"LABEL_0": 0,
|
| 62 |
+
"LABEL_1": 1
|
| 63 |
+
},
|
| 64 |
+
"max_reasoning_tokens": 256,
|
| 65 |
+
"min_one_minus_t": 0.1,
|
| 66 |
+
"model_type": "qwen_drive",
|
| 67 |
+
"noise_init_std": 1.0,
|
| 68 |
+
"noise_seed": 42,
|
| 69 |
+
"num_future_points": 50,
|
| 70 |
+
"num_history_points": 16,
|
| 71 |
+
"num_inference_steps": 10,
|
| 72 |
+
"output_attentions": false,
|
| 73 |
+
"output_hidden_states": false,
|
| 74 |
+
"problem_type": null,
|
| 75 |
+
"return_dict": true,
|
| 76 |
+
"trajectory_hz": 10.0,
|
| 77 |
+
"trajectory_point_dim": 3,
|
| 78 |
+
"trajectory_scale": [
|
| 79 |
+
165.0,
|
| 80 |
+
25.0,
|
| 81 |
+
1.5703125
|
| 82 |
+
],
|
| 83 |
+
"transformers_version": "5.15.0",
|
| 84 |
+
"vlm_config": {
|
| 85 |
+
"_name_or_path": "",
|
| 86 |
+
"architectures": [
|
| 87 |
+
"Qwen3_5ForConditionalGeneration"
|
| 88 |
+
],
|
| 89 |
+
"chunk_size_feed_forward": 0,
|
| 90 |
+
"dtype": null,
|
| 91 |
+
"id2label": {
|
| 92 |
+
"0": "LABEL_0",
|
| 93 |
+
"1": "LABEL_1"
|
| 94 |
+
},
|
| 95 |
+
"image_token_id": 248056,
|
| 96 |
+
"is_encoder_decoder": false,
|
| 97 |
+
"label2id": {
|
| 98 |
+
"LABEL_0": 0,
|
| 99 |
+
"LABEL_1": 1
|
| 100 |
+
},
|
| 101 |
+
"model_type": "qwen3_5",
|
| 102 |
+
"output_attentions": false,
|
| 103 |
+
"output_hidden_states": false,
|
| 104 |
+
"problem_type": null,
|
| 105 |
+
"return_dict": true,
|
| 106 |
+
"text_config": {
|
| 107 |
+
"_name_or_path": "",
|
| 108 |
+
"architectures": null,
|
| 109 |
+
"attention_bias": false,
|
| 110 |
+
"attention_dropout": 0.0,
|
| 111 |
+
"attn_output_gate": true,
|
| 112 |
+
"bos_token_id": null,
|
| 113 |
+
"chunk_size_feed_forward": 0,
|
| 114 |
+
"dtype": "bfloat16",
|
| 115 |
+
"eos_token_id": 248044,
|
| 116 |
+
"full_attention_interval": 4,
|
| 117 |
+
"head_dim": 256,
|
| 118 |
+
"hidden_act": "silu",
|
| 119 |
+
"hidden_size": 2560,
|
| 120 |
+
"id2label": {
|
| 121 |
+
"0": "LABEL_0",
|
| 122 |
+
"1": "LABEL_1"
|
| 123 |
+
},
|
| 124 |
+
"initializer_range": 0.02,
|
| 125 |
+
"intermediate_size": 9216,
|
| 126 |
+
"is_encoder_decoder": false,
|
| 127 |
+
"label2id": {
|
| 128 |
+
"LABEL_0": 0,
|
| 129 |
+
"LABEL_1": 1
|
| 130 |
+
},
|
| 131 |
+
"layer_types": [
|
| 132 |
+
"linear_attention",
|
| 133 |
+
"linear_attention",
|
| 134 |
+
"linear_attention",
|
| 135 |
+
"full_attention",
|
| 136 |
+
"linear_attention",
|
| 137 |
+
"linear_attention",
|
| 138 |
+
"linear_attention",
|
| 139 |
+
"full_attention",
|
| 140 |
+
"linear_attention",
|
| 141 |
+
"linear_attention",
|
| 142 |
+
"linear_attention",
|
| 143 |
+
"full_attention",
|
| 144 |
+
"linear_attention",
|
| 145 |
+
"linear_attention",
|
| 146 |
+
"linear_attention",
|
| 147 |
+
"full_attention",
|
| 148 |
+
"linear_attention",
|
| 149 |
+
"linear_attention",
|
| 150 |
+
"linear_attention",
|
| 151 |
+
"full_attention",
|
| 152 |
+
"linear_attention",
|
| 153 |
+
"linear_attention",
|
| 154 |
+
"linear_attention",
|
| 155 |
+
"full_attention",
|
| 156 |
+
"linear_attention",
|
| 157 |
+
"linear_attention",
|
| 158 |
+
"linear_attention",
|
| 159 |
+
"full_attention",
|
| 160 |
+
"linear_attention",
|
| 161 |
+
"linear_attention",
|
| 162 |
+
"linear_attention",
|
| 163 |
+
"full_attention"
|
| 164 |
+
],
|
| 165 |
+
"linear_conv_kernel_dim": 4,
|
| 166 |
+
"linear_key_head_dim": 128,
|
| 167 |
+
"linear_num_key_heads": 16,
|
| 168 |
+
"linear_num_value_heads": 32,
|
| 169 |
+
"linear_value_head_dim": 128,
|
| 170 |
+
"mamba_ssm_dtype": "float32",
|
| 171 |
+
"max_position_embeddings": 32768,
|
| 172 |
+
"mlp_only_layers": [],
|
| 173 |
+
"model_type": "qwen3_5_text",
|
| 174 |
+
"mtp_num_hidden_layers": 1,
|
| 175 |
+
"mtp_use_dedicated_embeddings": false,
|
| 176 |
+
"num_attention_heads": 16,
|
| 177 |
+
"num_hidden_layers": 32,
|
| 178 |
+
"num_key_value_heads": 4,
|
| 179 |
+
"output_attentions": false,
|
| 180 |
+
"output_hidden_states": false,
|
| 181 |
+
"pad_token_id": null,
|
| 182 |
+
"partial_rotary_factor": 0.25,
|
| 183 |
+
"problem_type": null,
|
| 184 |
+
"return_dict": true,
|
| 185 |
+
"rms_norm_eps": 1e-06,
|
| 186 |
+
"rope_parameters": {
|
| 187 |
+
"mrope_interleaved": true,
|
| 188 |
+
"mrope_section": [
|
| 189 |
+
11,
|
| 190 |
+
11,
|
| 191 |
+
10
|
| 192 |
+
],
|
| 193 |
+
"partial_rotary_factor": 0.25,
|
| 194 |
+
"rope_theta": 10000000,
|
| 195 |
+
"rope_type": "default"
|
| 196 |
+
},
|
| 197 |
+
"tie_word_embeddings": true,
|
| 198 |
+
"use_cache": true,
|
| 199 |
+
"vocab_size": 248320
|
| 200 |
+
},
|
| 201 |
+
"tie_word_embeddings": true,
|
| 202 |
+
"video_token_id": 248057,
|
| 203 |
+
"vision_config": {
|
| 204 |
+
"_name_or_path": "",
|
| 205 |
+
"architectures": null,
|
| 206 |
+
"chunk_size_feed_forward": 0,
|
| 207 |
+
"deepstack_visual_indexes": [],
|
| 208 |
+
"depth": 24,
|
| 209 |
+
"dtype": null,
|
| 210 |
+
"hidden_act": "gelu_pytorch_tanh",
|
| 211 |
+
"hidden_size": 1024,
|
| 212 |
+
"id2label": {
|
| 213 |
+
"0": "LABEL_0",
|
| 214 |
+
"1": "LABEL_1"
|
| 215 |
+
},
|
| 216 |
+
"in_channels": 3,
|
| 217 |
+
"initializer_range": 0.02,
|
| 218 |
+
"intermediate_size": 4096,
|
| 219 |
+
"is_encoder_decoder": false,
|
| 220 |
+
"label2id": {
|
| 221 |
+
"LABEL_0": 0,
|
| 222 |
+
"LABEL_1": 1
|
| 223 |
+
},
|
| 224 |
+
"model_type": "qwen3_5_vision",
|
| 225 |
+
"num_heads": 16,
|
| 226 |
+
"num_position_embeddings": 2304,
|
| 227 |
+
"out_hidden_size": 2560,
|
| 228 |
+
"output_attentions": false,
|
| 229 |
+
"output_hidden_states": false,
|
| 230 |
+
"patch_size": 16,
|
| 231 |
+
"problem_type": null,
|
| 232 |
+
"return_dict": true,
|
| 233 |
+
"spatial_merge_size": 2,
|
| 234 |
+
"temporal_patch_size": 2
|
| 235 |
+
},
|
| 236 |
+
"vision_end_token_id": 248054,
|
| 237 |
+
"vision_start_token_id": 248053
|
| 238 |
+
}
|
| 239 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token_id": 248044,
|
| 3 |
+
"do_sample": true,
|
| 4 |
+
"eos_token_id": [
|
| 5 |
+
248046,
|
| 6 |
+
248044
|
| 7 |
+
],
|
| 8 |
+
"pad_token_id": 248044,
|
| 9 |
+
"temperature": 0.6,
|
| 10 |
+
"top_k": 20,
|
| 11 |
+
"top_p": 0.95,
|
| 12 |
+
"transformers_version": "4.57.0.dev0"
|
| 13 |
+
}
|
merges.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b9de4bf448f57485fdaa45c60b1eea8e41a4b6ae82ec0cee8855a1e0301caccc
|
| 3 |
+
size 9078630512
|
perception/config.json
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"llm_dim": 2560,
|
| 3 |
+
"vit_dim": 1024,
|
| 4 |
+
"embed_dim": 256,
|
| 5 |
+
"det_pc_range": [
|
| 6 |
+
-51.2,
|
| 7 |
+
-51.2,
|
| 8 |
+
-5.0,
|
| 9 |
+
51.2,
|
| 10 |
+
51.2,
|
| 11 |
+
5.4
|
| 12 |
+
],
|
| 13 |
+
"det_voxel_size": [
|
| 14 |
+
0.512,
|
| 15 |
+
0.512,
|
| 16 |
+
10.4
|
| 17 |
+
],
|
| 18 |
+
"nuscenes_occ_pc_range": [
|
| 19 |
+
-40.0,
|
| 20 |
+
-40.0,
|
| 21 |
+
-1.0,
|
| 22 |
+
40.0,
|
| 23 |
+
40.0,
|
| 24 |
+
5.4
|
| 25 |
+
],
|
| 26 |
+
"nuscenes_occ_voxel_size": [
|
| 27 |
+
0.4,
|
| 28 |
+
0.4,
|
| 29 |
+
6.4
|
| 30 |
+
],
|
| 31 |
+
"nuplan_occ_pc_range": [
|
| 32 |
+
-50.0,
|
| 33 |
+
-50.0,
|
| 34 |
+
-4.0,
|
| 35 |
+
50.0,
|
| 36 |
+
50.0,
|
| 37 |
+
4.0
|
| 38 |
+
],
|
| 39 |
+
"nuplan_occ_voxel_size": [
|
| 40 |
+
0.5,
|
| 41 |
+
0.5,
|
| 42 |
+
0.5
|
| 43 |
+
],
|
| 44 |
+
"map_xbound": [
|
| 45 |
+
-30.0,
|
| 46 |
+
30.0,
|
| 47 |
+
0.15
|
| 48 |
+
],
|
| 49 |
+
"map_ybound": [
|
| 50 |
+
-15.0,
|
| 51 |
+
15.0,
|
| 52 |
+
0.15
|
| 53 |
+
],
|
| 54 |
+
"frustum_range": [
|
| 55 |
+
0,
|
| 56 |
+
0,
|
| 57 |
+
1.0,
|
| 58 |
+
896,
|
| 59 |
+
512,
|
| 60 |
+
60.0
|
| 61 |
+
],
|
| 62 |
+
"frustum_size": [
|
| 63 |
+
16.0,
|
| 64 |
+
16.0,
|
| 65 |
+
0.5
|
| 66 |
+
],
|
| 67 |
+
"bev_h": 200,
|
| 68 |
+
"bev_w": 200,
|
| 69 |
+
"occ_pillar_h": 16,
|
| 70 |
+
"occ_dim": 32,
|
| 71 |
+
"occ_num_classes": 10,
|
| 72 |
+
"det_num_classes": 7,
|
| 73 |
+
"map_num_classes": 6,
|
| 74 |
+
"num_query": 900,
|
| 75 |
+
"code_size": 10,
|
| 76 |
+
"num_encoder_layers": 6,
|
| 77 |
+
"num_decoder_layers": 6,
|
| 78 |
+
"image_size": [
|
| 79 |
+
896,
|
| 80 |
+
512
|
| 81 |
+
],
|
| 82 |
+
"transformers_version": "5.14.1",
|
| 83 |
+
"architectures": null,
|
| 84 |
+
"output_hidden_states": false,
|
| 85 |
+
"return_dict": true,
|
| 86 |
+
"dtype": null,
|
| 87 |
+
"chunk_size_feed_forward": 0,
|
| 88 |
+
"is_encoder_decoder": false,
|
| 89 |
+
"id2label": {
|
| 90 |
+
"0": "LABEL_0",
|
| 91 |
+
"1": "LABEL_1"
|
| 92 |
+
},
|
| 93 |
+
"label2id": {
|
| 94 |
+
"LABEL_0": 0,
|
| 95 |
+
"LABEL_1": 1
|
| 96 |
+
},
|
| 97 |
+
"problem_type": null,
|
| 98 |
+
"_name_or_path": "",
|
| 99 |
+
"model_type": "qwen_drive_perception",
|
| 100 |
+
"output_attentions": false
|
| 101 |
+
}
|
perception/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e964ca945f028bbbfadfdf9c1e47d31cfc3fe502d205eca0bb5a3d2c5ab450ae
|
| 3 |
+
size 500368384
|
planner-rl/config.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_name_or_path": "",
|
| 3 |
+
"architectures": null,
|
| 4 |
+
"chunk_size_feed_forward": 0,
|
| 5 |
+
"dtype": null,
|
| 6 |
+
"ego_status_dim": 8,
|
| 7 |
+
"fourier_max_frequency": 16.0,
|
| 8 |
+
"fourier_num_features": 16,
|
| 9 |
+
"head_dim": 256,
|
| 10 |
+
"hidden_size": 1024,
|
| 11 |
+
"history_dynamics_dim": 2,
|
| 12 |
+
"id2label": {
|
| 13 |
+
"0": "LABEL_0",
|
| 14 |
+
"1": "LABEL_1"
|
| 15 |
+
},
|
| 16 |
+
"intermediate_size": 3584,
|
| 17 |
+
"is_encoder_decoder": false,
|
| 18 |
+
"label2id": {
|
| 19 |
+
"LABEL_0": 0,
|
| 20 |
+
"LABEL_1": 1
|
| 21 |
+
},
|
| 22 |
+
"layers_per_kv": 4,
|
| 23 |
+
"model_type": "qwen_drive_planning_expert",
|
| 24 |
+
"mrope_section": [
|
| 25 |
+
11,
|
| 26 |
+
11,
|
| 27 |
+
10
|
| 28 |
+
],
|
| 29 |
+
"nav_command_classes": 3,
|
| 30 |
+
"num_attention_heads": 16,
|
| 31 |
+
"num_hidden_layers": 32,
|
| 32 |
+
"num_key_value_heads": 4,
|
| 33 |
+
"output_attentions": false,
|
| 34 |
+
"output_hidden_states": false,
|
| 35 |
+
"partial_rotary_factor": 0.25,
|
| 36 |
+
"problem_type": null,
|
| 37 |
+
"return_dict": true,
|
| 38 |
+
"rms_norm_eps": 1e-05,
|
| 39 |
+
"rope_theta": 10000000.0,
|
| 40 |
+
"time_embed_dim": 128,
|
| 41 |
+
"time_embed_scale": 1000.0
|
| 42 |
+
}
|
planner-rl/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:74478eb7ec5dea0d8144372ed7b60e6f508da7d9eb1522e567b391cad6dbaf29
|
| 3 |
+
size 2079739550
|
planner-sft/config.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_name_or_path": "",
|
| 3 |
+
"architectures": null,
|
| 4 |
+
"chunk_size_feed_forward": 0,
|
| 5 |
+
"dtype": null,
|
| 6 |
+
"ego_status_dim": 8,
|
| 7 |
+
"fourier_max_frequency": 16.0,
|
| 8 |
+
"fourier_num_features": 16,
|
| 9 |
+
"head_dim": 256,
|
| 10 |
+
"hidden_size": 1024,
|
| 11 |
+
"history_dynamics_dim": 2,
|
| 12 |
+
"id2label": {
|
| 13 |
+
"0": "LABEL_0",
|
| 14 |
+
"1": "LABEL_1"
|
| 15 |
+
},
|
| 16 |
+
"intermediate_size": 3584,
|
| 17 |
+
"is_encoder_decoder": false,
|
| 18 |
+
"label2id": {
|
| 19 |
+
"LABEL_0": 0,
|
| 20 |
+
"LABEL_1": 1
|
| 21 |
+
},
|
| 22 |
+
"layers_per_kv": 4,
|
| 23 |
+
"model_type": "qwen_drive_planning_expert",
|
| 24 |
+
"mrope_section": [
|
| 25 |
+
11,
|
| 26 |
+
11,
|
| 27 |
+
10
|
| 28 |
+
],
|
| 29 |
+
"nav_command_classes": 3,
|
| 30 |
+
"num_attention_heads": 16,
|
| 31 |
+
"num_hidden_layers": 32,
|
| 32 |
+
"num_key_value_heads": 4,
|
| 33 |
+
"output_attentions": false,
|
| 34 |
+
"output_hidden_states": false,
|
| 35 |
+
"partial_rotary_factor": 0.25,
|
| 36 |
+
"problem_type": null,
|
| 37 |
+
"return_dict": true,
|
| 38 |
+
"rms_norm_eps": 1e-05,
|
| 39 |
+
"rope_theta": 10000000.0,
|
| 40 |
+
"time_embed_dim": 128,
|
| 41 |
+
"time_embed_scale": 1000.0
|
| 42 |
+
}
|
planner-sft/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dcf5989ed292799e77f539b21e5d8b701566676c1c5d13038135e97f96e2d7b4
|
| 3 |
+
size 2079739550
|
preprocessor_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"size": {
|
| 3 |
+
"longest_edge": 16777216,
|
| 4 |
+
"shortest_edge": 65536
|
| 5 |
+
},
|
| 6 |
+
"patch_size": 16,
|
| 7 |
+
"temporal_patch_size": 2,
|
| 8 |
+
"merge_size": 2,
|
| 9 |
+
"image_mean": [
|
| 10 |
+
0.5,
|
| 11 |
+
0.5,
|
| 12 |
+
0.5
|
| 13 |
+
],
|
| 14 |
+
"image_std": [
|
| 15 |
+
0.5,
|
| 16 |
+
0.5,
|
| 17 |
+
0.5
|
| 18 |
+
],
|
| 19 |
+
"processor_class": "Qwen3VLProcessor",
|
| 20 |
+
"image_processor_type": "Qwen2VLImageProcessorFast"
|
| 21 |
+
}
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9825446f866dd1d1f6ab843f10af4014c168334d50d977f0afbd3ee304c4856a
|
| 3 |
+
size 12807970
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"added_tokens_decoder": {
|
| 4 |
+
"248044": {
|
| 5 |
+
"content": "<|endoftext|>",
|
| 6 |
+
"lstrip": false,
|
| 7 |
+
"normalized": false,
|
| 8 |
+
"rstrip": false,
|
| 9 |
+
"single_word": false,
|
| 10 |
+
"special": true
|
| 11 |
+
},
|
| 12 |
+
"248045": {
|
| 13 |
+
"content": "<|im_start|>",
|
| 14 |
+
"lstrip": false,
|
| 15 |
+
"normalized": false,
|
| 16 |
+
"rstrip": false,
|
| 17 |
+
"single_word": false,
|
| 18 |
+
"special": true
|
| 19 |
+
},
|
| 20 |
+
"248046": {
|
| 21 |
+
"content": "<|im_end|>",
|
| 22 |
+
"lstrip": false,
|
| 23 |
+
"normalized": false,
|
| 24 |
+
"rstrip": false,
|
| 25 |
+
"single_word": false,
|
| 26 |
+
"special": true
|
| 27 |
+
},
|
| 28 |
+
"248047": {
|
| 29 |
+
"content": "<|object_ref_start|>",
|
| 30 |
+
"lstrip": false,
|
| 31 |
+
"normalized": false,
|
| 32 |
+
"rstrip": false,
|
| 33 |
+
"single_word": false,
|
| 34 |
+
"special": true
|
| 35 |
+
},
|
| 36 |
+
"248048": {
|
| 37 |
+
"content": "<|object_ref_end|>",
|
| 38 |
+
"lstrip": false,
|
| 39 |
+
"normalized": false,
|
| 40 |
+
"rstrip": false,
|
| 41 |
+
"single_word": false,
|
| 42 |
+
"special": true
|
| 43 |
+
},
|
| 44 |
+
"248049": {
|
| 45 |
+
"content": "<|box_start|>",
|
| 46 |
+
"lstrip": false,
|
| 47 |
+
"normalized": false,
|
| 48 |
+
"rstrip": false,
|
| 49 |
+
"single_word": false,
|
| 50 |
+
"special": true
|
| 51 |
+
},
|
| 52 |
+
"248050": {
|
| 53 |
+
"content": "<|box_end|>",
|
| 54 |
+
"lstrip": false,
|
| 55 |
+
"normalized": false,
|
| 56 |
+
"rstrip": false,
|
| 57 |
+
"single_word": false,
|
| 58 |
+
"special": true
|
| 59 |
+
},
|
| 60 |
+
"248051": {
|
| 61 |
+
"content": "<|quad_start|>",
|
| 62 |
+
"lstrip": false,
|
| 63 |
+
"normalized": false,
|
| 64 |
+
"rstrip": false,
|
| 65 |
+
"single_word": false,
|
| 66 |
+
"special": true
|
| 67 |
+
},
|
| 68 |
+
"248052": {
|
| 69 |
+
"content": "<|quad_end|>",
|
| 70 |
+
"lstrip": false,
|
| 71 |
+
"normalized": false,
|
| 72 |
+
"rstrip": false,
|
| 73 |
+
"single_word": false,
|
| 74 |
+
"special": true
|
| 75 |
+
},
|
| 76 |
+
"248053": {
|
| 77 |
+
"content": "<|vision_start|>",
|
| 78 |
+
"lstrip": false,
|
| 79 |
+
"normalized": false,
|
| 80 |
+
"rstrip": false,
|
| 81 |
+
"single_word": false,
|
| 82 |
+
"special": true
|
| 83 |
+
},
|
| 84 |
+
"248054": {
|
| 85 |
+
"content": "<|vision_end|>",
|
| 86 |
+
"lstrip": false,
|
| 87 |
+
"normalized": false,
|
| 88 |
+
"rstrip": false,
|
| 89 |
+
"single_word": false,
|
| 90 |
+
"special": true
|
| 91 |
+
},
|
| 92 |
+
"248055": {
|
| 93 |
+
"content": "<|vision_pad|>",
|
| 94 |
+
"lstrip": false,
|
| 95 |
+
"normalized": false,
|
| 96 |
+
"rstrip": false,
|
| 97 |
+
"single_word": false,
|
| 98 |
+
"special": true
|
| 99 |
+
},
|
| 100 |
+
"248056": {
|
| 101 |
+
"content": "<|image_pad|>",
|
| 102 |
+
"lstrip": false,
|
| 103 |
+
"normalized": false,
|
| 104 |
+
"rstrip": false,
|
| 105 |
+
"single_word": false,
|
| 106 |
+
"special": true
|
| 107 |
+
},
|
| 108 |
+
"248057": {
|
| 109 |
+
"content": "<|video_pad|>",
|
| 110 |
+
"lstrip": false,
|
| 111 |
+
"normalized": false,
|
| 112 |
+
"rstrip": false,
|
| 113 |
+
"single_word": false,
|
| 114 |
+
"special": true
|
| 115 |
+
},
|
| 116 |
+
"248058": {
|
| 117 |
+
"content": "<tool_call>",
|
| 118 |
+
"lstrip": false,
|
| 119 |
+
"normalized": false,
|
| 120 |
+
"rstrip": false,
|
| 121 |
+
"single_word": false,
|
| 122 |
+
"special": true
|
| 123 |
+
},
|
| 124 |
+
"248059": {
|
| 125 |
+
"content": "</tool_call>",
|
| 126 |
+
"lstrip": false,
|
| 127 |
+
"normalized": false,
|
| 128 |
+
"rstrip": false,
|
| 129 |
+
"single_word": false,
|
| 130 |
+
"special": true
|
| 131 |
+
},
|
| 132 |
+
"248060": {
|
| 133 |
+
"content": "<|fim_prefix|>",
|
| 134 |
+
"lstrip": false,
|
| 135 |
+
"normalized": false,
|
| 136 |
+
"rstrip": false,
|
| 137 |
+
"single_word": false,
|
| 138 |
+
"special": true
|
| 139 |
+
},
|
| 140 |
+
"248061": {
|
| 141 |
+
"content": "<|fim_middle|>",
|
| 142 |
+
"lstrip": false,
|
| 143 |
+
"normalized": false,
|
| 144 |
+
"rstrip": false,
|
| 145 |
+
"single_word": false,
|
| 146 |
+
"special": true
|
| 147 |
+
},
|
| 148 |
+
"248062": {
|
| 149 |
+
"content": "<|fim_suffix|>",
|
| 150 |
+
"lstrip": false,
|
| 151 |
+
"normalized": false,
|
| 152 |
+
"rstrip": false,
|
| 153 |
+
"single_word": false,
|
| 154 |
+
"special": true
|
| 155 |
+
},
|
| 156 |
+
"248063": {
|
| 157 |
+
"content": "<|fim_pad|>",
|
| 158 |
+
"lstrip": false,
|
| 159 |
+
"normalized": false,
|
| 160 |
+
"rstrip": false,
|
| 161 |
+
"single_word": false,
|
| 162 |
+
"special": true
|
| 163 |
+
},
|
| 164 |
+
"248064": {
|
| 165 |
+
"content": "<|repo_name|>",
|
| 166 |
+
"lstrip": false,
|
| 167 |
+
"normalized": false,
|
| 168 |
+
"rstrip": false,
|
| 169 |
+
"single_word": false,
|
| 170 |
+
"special": true
|
| 171 |
+
},
|
| 172 |
+
"248065": {
|
| 173 |
+
"content": "<|file_sep|>",
|
| 174 |
+
"lstrip": false,
|
| 175 |
+
"normalized": false,
|
| 176 |
+
"rstrip": false,
|
| 177 |
+
"single_word": false,
|
| 178 |
+
"special": true
|
| 179 |
+
},
|
| 180 |
+
"248066": {
|
| 181 |
+
"content": "<tool_response>",
|
| 182 |
+
"lstrip": false,
|
| 183 |
+
"normalized": false,
|
| 184 |
+
"rstrip": false,
|
| 185 |
+
"single_word": false,
|
| 186 |
+
"special": true
|
| 187 |
+
},
|
| 188 |
+
"248067": {
|
| 189 |
+
"content": "</tool_response>",
|
| 190 |
+
"lstrip": false,
|
| 191 |
+
"normalized": false,
|
| 192 |
+
"rstrip": false,
|
| 193 |
+
"single_word": false,
|
| 194 |
+
"special": true
|
| 195 |
+
},
|
| 196 |
+
"248068": {
|
| 197 |
+
"content": "<think>",
|
| 198 |
+
"lstrip": false,
|
| 199 |
+
"normalized": false,
|
| 200 |
+
"rstrip": false,
|
| 201 |
+
"single_word": false,
|
| 202 |
+
"special": true
|
| 203 |
+
},
|
| 204 |
+
"248069": {
|
| 205 |
+
"content": "</think>",
|
| 206 |
+
"lstrip": false,
|
| 207 |
+
"normalized": false,
|
| 208 |
+
"rstrip": false,
|
| 209 |
+
"single_word": false,
|
| 210 |
+
"special": false
|
| 211 |
+
},
|
| 212 |
+
"248070": {
|
| 213 |
+
"content": "<|audio_start|>",
|
| 214 |
+
"lstrip": false,
|
| 215 |
+
"normalized": false,
|
| 216 |
+
"rstrip": false,
|
| 217 |
+
"single_word": false,
|
| 218 |
+
"special": true
|
| 219 |
+
},
|
| 220 |
+
"248071": {
|
| 221 |
+
"content": "<|audio_end|>",
|
| 222 |
+
"lstrip": false,
|
| 223 |
+
"normalized": false,
|
| 224 |
+
"rstrip": false,
|
| 225 |
+
"single_word": false,
|
| 226 |
+
"special": true
|
| 227 |
+
},
|
| 228 |
+
"248072": {
|
| 229 |
+
"content": "<tts_pad>",
|
| 230 |
+
"lstrip": false,
|
| 231 |
+
"normalized": false,
|
| 232 |
+
"rstrip": false,
|
| 233 |
+
"single_word": false,
|
| 234 |
+
"special": true
|
| 235 |
+
},
|
| 236 |
+
"248073": {
|
| 237 |
+
"content": "<tts_text_bos>",
|
| 238 |
+
"lstrip": false,
|
| 239 |
+
"normalized": false,
|
| 240 |
+
"rstrip": false,
|
| 241 |
+
"single_word": false,
|
| 242 |
+
"special": true
|
| 243 |
+
},
|
| 244 |
+
"248074": {
|
| 245 |
+
"content": "<tts_text_eod>",
|
| 246 |
+
"lstrip": false,
|
| 247 |
+
"normalized": false,
|
| 248 |
+
"rstrip": false,
|
| 249 |
+
"single_word": false,
|
| 250 |
+
"special": true
|
| 251 |
+
},
|
| 252 |
+
"248075": {
|
| 253 |
+
"content": "<tts_text_bos_single>",
|
| 254 |
+
"lstrip": false,
|
| 255 |
+
"normalized": false,
|
| 256 |
+
"rstrip": false,
|
| 257 |
+
"single_word": false,
|
| 258 |
+
"special": true
|
| 259 |
+
},
|
| 260 |
+
"248076": {
|
| 261 |
+
"content": "<|audio_pad|>",
|
| 262 |
+
"lstrip": false,
|
| 263 |
+
"normalized": false,
|
| 264 |
+
"rstrip": false,
|
| 265 |
+
"single_word": false,
|
| 266 |
+
"special": true
|
| 267 |
+
}
|
| 268 |
+
},
|
| 269 |
+
"additional_special_tokens": [
|
| 270 |
+
"<|im_start|>",
|
| 271 |
+
"<|im_end|>",
|
| 272 |
+
"<|object_ref_start|>",
|
| 273 |
+
"<|object_ref_end|>",
|
| 274 |
+
"<|box_start|>",
|
| 275 |
+
"<|box_end|>",
|
| 276 |
+
"<|quad_start|>",
|
| 277 |
+
"<|quad_end|>",
|
| 278 |
+
"<|vision_start|>",
|
| 279 |
+
"<|vision_end|>",
|
| 280 |
+
"<|vision_pad|>",
|
| 281 |
+
"<|image_pad|>",
|
| 282 |
+
"<|video_pad|>"
|
| 283 |
+
],
|
| 284 |
+
"bos_token": null,
|
| 285 |
+
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {{- \"<|vision_start|><|image_pad|><|vision_end|>\"}}\n {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %}\n {{- \"<|audio_start|><|audio_pad|><|audio_end|>\"}}\n {%- elif content.type == 'video' or 'video' in content %}\n {{- \"<|vision_start|><|video_pad|><|vision_end|>\"}}\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content_item in message.content %}\n {%- if 'text' in content_item %}\n {{- content_item.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and message.content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {{- \"<|vision_start|><|image_pad|><|vision_end|>\"}}\n {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %}\n {{- \"<|audio_start|><|audio_pad|><|audio_end|>\"}}\n {%- elif content.type == 'video' or 'video' in content %}\n {{- \"<|vision_start|><|video_pad|><|vision_end|>\"}}\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
|
| 286 |
+
"clean_up_tokenization_spaces": false,
|
| 287 |
+
"eos_token": "<|im_end|>",
|
| 288 |
+
"errors": "replace",
|
| 289 |
+
"model_max_length": 131072,
|
| 290 |
+
"pad_token": "<|endoftext|>",
|
| 291 |
+
"split_special_tokens": false,
|
| 292 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
| 293 |
+
"unk_token": null,
|
| 294 |
+
"add_bos_token": false,
|
| 295 |
+
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
| 296 |
+
"extra_special_tokens": {
|
| 297 |
+
"audio_bos_token": "<|audio_start|>",
|
| 298 |
+
"audio_eos_token": "<|audio_end|>",
|
| 299 |
+
"audio_token": "<|audio_pad|>",
|
| 300 |
+
"image_token": "<|image_pad|>",
|
| 301 |
+
"video_token": "<|video_pad|>",
|
| 302 |
+
"vision_bos_token": "<|vision_start|>",
|
| 303 |
+
"vision_eos_token": "<|vision_end|>"
|
| 304 |
+
}
|
| 305 |
+
}
|
vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|