{
  "activation": "relu",
  "compile": false,
  "d_cond": 128,
  "d_model": 768,
  "dim_feedforward": 3072,
  "dropout": 0.1,
  "force_flash_attn": false,
  "inner_autocast": true,
  "layer_norm_eps": 1e-05,
  "mask_idx": 4,
  "max_length": 260,
  "max_rate": 100.0,
  "min_rate": 0.0,
  "nhead": 12,
  "num_embeddings": 1882,
  "num_layers": 12,
  "num_rate_bins": 1000,
  "padding_idx": 3,
  "rate_head_type": "scalar",
  "rate_head_use_mlp": true,
  "rotary_emb_dim": 64,
  "scalar_fn": "exp",
  "schedule_type": "simplified-kuma"
}