Add files using upload-large-folder tool

config.json

@@ -1,44 +1,63 @@
 {
-    "architectures": [
-        "BailingMoeForCausalLM"
-    ],
-    "attention_dropout": 0.0,
-    "auto_map": {
-        "AutoConfig": "configuration_bailing_moe.BailingMoeConfig",
-        "AutoModel": "modeling_bailing_moe.BailingMoeModel",
-        "AutoModelForCausalLM": "modeling_bailing_moe.BailingMoeForCausalLM"
-    },
-    "eos_token_id": 126081,
-    "pad_token_id": 126081,
-    "first_k_dense_replace": 0,
-    "hidden_act": "silu",
-    "hidden_size": 2048,
-    "initializer_range": 0.006,
-    "intermediate_size": 1408,
-    "max_position_embeddings": 32768,
-    "model_type": "bailing_moe",
-    "moe_intermediate_size": 1408,
-    "num_experts": 64,
-    "num_shared_experts": 2,
-    "norm_topk_prob": true,
-    "num_attention_heads": 16,
-    "num_experts_per_tok": 6,
-    "num_hidden_layers": 28,
-    "num_key_value_heads": 4,
-    "pretraining_tp": 1,
-    "rms_norm_eps": 1e-06,
-    "rope_scaling": null,
-    "rope_theta": 600000,
-    "tie_word_embeddings": false,
-    "torch_dtype": "bfloat16",
-    "transformers_version": "4.40.0",
-    "use_cache": true,
-    "use_bias": false,
-    "use_qkv_bias": false,
-    "vocab_size": 126464,
-    "output_router_logits": false,
-    "embedding_dropout": 0.0,
-    "norm_head": false,
-    "norm_softmax": false,
-    "output_dropout": 0.0
-}
+  "all2all_overlap": true,
+  "architectures": [
+    "BailingMoeForCausalLM"
+  ],
+  "atorch_backend": "MegaBlocks",
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_bailing_moe.BailingMoeConfig",
+    "AutoModel": "modeling_bailing_moe.BailingMoeModel",
+    "AutoModelForCausalLM": "modeling_bailing_moe.BailingMoeForCausalLM"
+  },
+  "bos_token_id": 126080,
+  "dispatcher_type": "AllToAll",
+  "embedding_dropout": 0.0,
+  "eos_token_id": 126081,
+  "expert_model_parallelism": false,
+  "first_k_dense_replace": 0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.006,
+  "intermediate_size": 1408,
+  "is_scale_gradient": true,
+  "last_logits_l2_alpha": -1.0,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "merge_w1_v1": false,
+  "model_type": "bailing_moe",
+  "moe_impl": "raw",
+  "moe_intermediate_size": 1408,
+  "moe_mlp_prefix": false,
+  "norm_head": false,
+  "norm_softmax": false,
+  "norm_topk_prob": true,
+  "num_attention_heads": 16,
+  "num_experts": 64,
+  "num_experts_per_tok": 6,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "num_shared_experts": 2,
+  "output_dropout": 0.0,
+  "output_router_logits": false,
+  "pad_token_id": 126081,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 600000,
+  "router_balance_loss_alpha": 0,
+  "router_group_balance_loss_alpha": 0.0,
+  "router_z_loss_alpha": 0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.1",
+  "transpose_w1": true,
+  "use_bias": false,
+  "use_cache": true,
+  "use_qkv_bias": false,
+  "use_sliding_window": false,
+  "use_swiglu": false,
+  "vocab_size": 126464
+}

generation_config.json

@@ -1,5 +1,6 @@
 {
   "_from_model_config": true,
+  "bos_token_id": 126080,
   "eos_token_id": 126081,
   "pad_token_id": 126081,
   "transformers_version": "4.51.1"

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5092192b96ac711864b21fd6e772e28b7450db1cf921e723b5a74202df1e0d59
+oid sha256:1cab0b82ec17af068c667447d6b1f25ead0d84a641aaa8b867c3a96a70868110
 size 10000012352

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:533debde11b9f418a9525207ad3b1e7f31de1b21177213b58e3376935b8aab48
+oid sha256:919e7e62a34e498ffbad9ba1d636f445682d041cff7466a167a7f990517c8694
 size 9997403496

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2af3073fa0b56c8ae033d4d0e4cb43b5ba83f1cc6f189674d14005b14b96385a
+oid sha256:b5e803b5a48a21a25bf65b2972b0eedf77a091e3ad59825bb3f9b910de40bf73
 size 9995576736

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:caba3e3a08933b84ef7d1be48fea5308b1bd75dcb90e3d00b5a37e60f014110c
+oid sha256:ec9b071c31c1ee2ee113311bdb1ec917b895ea09089adef415abc39d015ff044
 size 3611653272

tokenizer_config.json

@@ -2129,6 +2129,22 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "126346": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126347": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
   "additional_special_tokens": [
@@ -2140,16 +2156,16 @@
     "<|number_end|>"
   ],
   "bos_token": "<|startoftext|>",
-  "chat_template": "{% set system_present = false %}{% set thinking_option = 'on' %}{% for message in messages %}{% set role = message['role'] | lower %}{% if role == 'system' %}{% set system_present = true %}{% endif %}{% endfor %}{% if not system_present %}{{ '<role>SYSTEM</role>detailed thinking ' + thinking_option }}{% endif %}{% for message in messages %}{% set role = message['role'] | lower %}{% if role == 'user' %}{% set role = 'HUMAN' %}{% endif %}{% set role = role | upper %}{{ '<role>' + role + '</role>' + message['content'] + ('\ndetailed thinking ' + thinking_option if role == 'SYSTEM' else '') }}{% endfor %}{% if add_generation_prompt %}{{ '<role>ASSISTANT</role><think>\n' }}{% endif %}",
+  "chat_template": "{% for message in messages %}{% set role = message['role'] | lower %}{% if role == 'user' %}{% set role = 'HUMAN' %}{% endif %}{% set role = role | upper %}{{ '<role>' + role + '</role>' + message['content'] }}{% endfor %}{% if add_generation_prompt %}{{ '<role>ASSISTANT</role>' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "cls_token": "[CLS]",
   "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
   "fast_tokenizer": true,
   "gmask_token": "[gMASK]",
   "merges_file": null,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<|endoftext|>",
-  "tokenizer_class": "PreTrainedTokenizerFast",
-  "trust_remote_code": true,
-  "vocab_file": null
+  "tokenizer_class": "PreTrainedTokenizer",
+  "trust_remote_code": true
 }