{
  "title": "Nex N2.5 local tokenizer comparison",
  "date": "2026-09-08",
  "method": "tokenizers.Tokenizer.encode(text, add_special_tokens=False); same ordered 50 probes as the existing YFarmX study",
  "scope": "Released local files; no completed hosted endpoint measurements; no claim of exact base weights or novel model identity",
  "files": [
    {
      "repo": "nex-agi/Nex-N2.5-mini",
      "files": {
        "tokenizer.json": {
          "url": "https://huggingface.co/nex-agi/Nex-N2.5-mini/resolve/main/tokenizer.json",
          "sha256": "87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4",
          "revision": null,
          "bytes": 19989343
        },
        "config.json": {
          "url": "https://huggingface.co/nex-agi/Nex-N2.5-mini/resolve/main/config.json",
          "sha256": "aed18afaa5edbc5cd9177aa3f8b2a095e0f539caedca9a3d68fe87ac838ee8d5",
          "revision": "87420286149d9cce9bd46cd335ef9bda33c37c1b",
          "bytes": 3236
        }
      },
      "count_vector": [
        14,
        16,
        3,
        3,
        3,
        4,
        5,
        9,
        12,
        30,
        12,
        22,
        9,
        10,
        18,
        7,
        10,
        13,
        12,
        10,
        14,
        15,
        8,
        13,
        9,
        30,
        18,
        54,
        24,
        29,
        29,
        28,
        5,
        13,
        40,
        36,
        6,
        26,
        18,
        25,
        16,
        44,
        5,
        6,
        31,
        22,
        36,
        17,
        19,
        36
      ],
      "vocab_size": 248077,
      "architecture": [
        "Qwen3_5MoeForConditionalGeneration"
      ],
      "model_type": "qwen3_5_moe"
    },
    {
      "repo": "Qwen/Qwen3.5-35B-A3B",
      "files": {
        "tokenizer.json": {
          "url": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B/resolve/main/tokenizer.json",
          "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42",
          "revision": null,
          "bytes": 12807982
        },
        "config.json": {
          "url": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B/resolve/main/config.json",
          "sha256": "5e4d7f74fec2f360eb9cfbfcd6ec0c4c76e684d3a11caaed259d9fd9bfbc7944",
          "revision": "59d61f3ce65a6d9863b86d2e96597125219dc754",
          "bytes": 3544
        }
      },
      "count_vector": [
        14,
        16,
        3,
        3,
        3,
        4,
        5,
        9,
        12,
        30,
        12,
        22,
        9,
        10,
        18,
        7,
        10,
        13,
        12,
        10,
        14,
        15,
        8,
        13,
        9,
        30,
        18,
        54,
        24,
        29,
        29,
        28,
        5,
        13,
        40,
        36,
        6,
        26,
        18,
        25,
        16,
        44,
        5,
        6,
        31,
        22,
        36,
        17,
        19,
        36
      ],
      "vocab_size": 248070,
      "architecture": [
        "Qwen3_5MoeForConditionalGeneration"
      ],
      "model_type": "qwen3_5_moe"
    },
    {
      "repo": "Qwen/Qwen3.6-35B-A3B",
      "files": {
        "tokenizer.json": {
          "url": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/resolve/main/tokenizer.json",
          "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42",
          "revision": null,
          "bytes": 12807982
        },
        "config.json": {
          "url": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/resolve/main/config.json",
          "sha256": "93a4693fa9d8392fbfccd4b3c9873f4bfdcb14fdede978b123d07d19675efe99",
          "revision": "995ad96eacd98c81ed38be0c5b274b04031597b0",
          "bytes": 3686
        }
      },
      "count_vector": [
        14,
        16,
        3,
        3,
        3,
        4,
        5,
        9,
        12,
        30,
        12,
        22,
        9,
        10,
        18,
        7,
        10,
        13,
        12,
        10,
        14,
        15,
        8,
        13,
        9,
        30,
        18,
        54,
        24,
        29,
        29,
        28,
        5,
        13,
        40,
        36,
        6,
        26,
        18,
        25,
        16,
        44,
        5,
        6,
        31,
        22,
        36,
        17,
        19,
        36
      ],
      "vocab_size": 248070,
      "architecture": [
        "Qwen3_5MoeForConditionalGeneration"
      ],
      "model_type": "qwen3_5_moe"
    },
    {
      "repo": "Qwen/Qwen3-30B-A3B",
      "files": {
        "tokenizer.json": {
          "url": "https://huggingface.co/Qwen/Qwen3-30B-A3B/resolve/main/tokenizer.json",
          "sha256": "aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4",
          "revision": null,
          "bytes": 11422654
        },
        "config.json": {
          "url": "https://huggingface.co/Qwen/Qwen3-30B-A3B/resolve/main/config.json",
          "sha256": "2850ddb3bf7aecad20b611e2d44f3077fc8193f4827c93beddd4c02ad63c2297",
          "revision": "ad44e777bcd18fa416d9da3bd8f70d33ebb85d39",
          "bytes": 963
        }
      },
      "count_vector": [
        14,
        16,
        3,
        3,
        3,
        4,
        5,
        9,
        12,
        30,
        12,
        22,
        10,
        12,
        17,
        8,
        13,
        17,
        16,
        8,
        9,
        29,
        12,
        28,
        5,
        10,
        10,
        30,
        8,
        14,
        26,
        17,
        5,
        12,
        22,
        36,
        7,
        25,
        15,
        25,
        16,
        44,
        5,
        6,
        31,
        22,
        36,
        17,
        19,
        36
      ],
      "vocab_size": 151669,
      "architecture": [
        "Qwen3MoeForCausalLM"
      ],
      "model_type": "qwen3_moe"
    }
  ],
  "structure": {
    "measured_at": "2026-09-08T22:04:26.557Z",
    "test_strings_sha256": "6be7ff70914a90b09890143a6e5d27471ea82dd3ef2458d67e16e18d53cfafc3",
    "comparisons": [
      {
        "control": "Qwen/Qwen3.5-35B-A3B",
        "nex_base_vocab": 248044,
        "control_base_vocab": 248044,
        "matching_vocab_entries": 248044,
        "nex_merge_count": 247587,
        "control_merge_count": 247587,
        "ordered_merges_equal": true,
        "added_tokens_nex": 33,
        "added_tokens_control": 26,
        "extra_tokens": [
          {
            "id": 248070,
            "content": "<|audio_start|>"
          },
          {
            "id": 248071,
            "content": "<|audio_end|>"
          },
          {
            "id": 248072,
            "content": "<tts_pad>"
          },
          {
            "id": 248073,
            "content": "<tts_text_bos>"
          },
          {
            "id": 248074,
            "content": "<tts_text_eod>"
          },
          {
            "id": 248075,
            "content": "<tts_text_bos_single>"
          },
          {
            "id": 248076,
            "content": "<|audio_pad|>"
          }
        ]
      },
      {
        "control": "Qwen/Qwen3.6-35B-A3B",
        "nex_base_vocab": 248044,
        "control_base_vocab": 248044,
        "matching_vocab_entries": 248044,
        "nex_merge_count": 247587,
        "control_merge_count": 247587,
        "ordered_merges_equal": true,
        "added_tokens_nex": 33,
        "added_tokens_control": 26,
        "extra_tokens": [
          {
            "id": 248070,
            "content": "<|audio_start|>"
          },
          {
            "id": 248071,
            "content": "<|audio_end|>"
          },
          {
            "id": 248072,
            "content": "<tts_pad>"
          },
          {
            "id": 248073,
            "content": "<tts_text_bos>"
          },
          {
            "id": 248074,
            "content": "<tts_text_eod>"
          },
          {
            "id": 248075,
            "content": "<tts_text_bos_single>"
          },
          {
            "id": 248076,
            "content": "<|audio_pad|>"
          }
        ]
      },
      {
        "control": "Qwen/Qwen3-30B-A3B",
        "nex_base_vocab": 248044,
        "control_base_vocab": 151643,
        "matching_vocab_entries": 281,
        "nex_merge_count": 247587,
        "control_merge_count": 151387,
        "ordered_merges_equal": false,
        "added_tokens_nex": 33,
        "added_tokens_control": 26,
        "extra_tokens": [
          {
            "id": 248044,
            "content": "<|endoftext|>"
          },
          {
            "id": 248045,
            "content": "<|im_start|>"
          },
          {
            "id": 248046,
            "content": "<|im_end|>"
          },
          {
            "id": 248047,
            "content": "<|object_ref_start|>"
          },
          {
            "id": 248048,
            "content": "<|object_ref_end|>"
          },
          {
            "id": 248049,
            "content": "<|box_start|>"
          },
          {
            "id": 248050,
            "content": "<|box_end|>"
          },
          {
            "id": 248051,
            "content": "<|quad_start|>"
          },
          {
            "id": 248052,
            "content": "<|quad_end|>"
          },
          {
            "id": 248053,
            "content": "<|vision_start|>"
          },
          {
            "id": 248054,
            "content": "<|vision_end|>"
          },
          {
            "id": 248055,
            "content": "<|vision_pad|>"
          },
          {
            "id": 248056,
            "content": "<|image_pad|>"
          },
          {
            "id": 248057,
            "content": "<|video_pad|>"
          },
          {
            "id": 248058,
            "content": "<tool_call>"
          },
          {
            "id": 248059,
            "content": "</tool_call>"
          },
          {
            "id": 248060,
            "content": "<|fim_prefix|>"
          },
          {
            "id": 248061,
            "content": "<|fim_middle|>"
          },
          {
            "id": 248062,
            "content": "<|fim_suffix|>"
          },
          {
            "id": 248063,
            "content": "<|fim_pad|>"
          },
          {
            "id": 248064,
            "content": "<|repo_name|>"
          },
          {
            "id": 248065,
            "content": "<|file_sep|>"
          },
          {
            "id": 248066,
            "content": "<tool_response>"
          },
          {
            "id": 248067,
            "content": "</tool_response>"
          },
          {
            "id": 248068,
            "content": "<think>"
          },
          {
            "id": 248069,
            "content": "</think>"
          },
          {
            "id": 248070,
            "content": "<|audio_start|>"
          },
          {
            "id": 248071,
            "content": "<|audio_end|>"
          },
          {
            "id": 248072,
            "content": "<tts_pad>"
          },
          {
            "id": 248073,
            "content": "<tts_text_bos>"
          },
          {
            "id": 248074,
            "content": "<tts_text_eod>"
          },
          {
            "id": 248075,
            "content": "<tts_text_bos_single>"
          },
          {
            "id": 248076,
            "content": "<|audio_pad|>"
          }
        ]
      }
    ]
  },
  "previous_release": {
    "url": "https://huggingface.co/nex-agi/Nex-N2-Mini/resolve/main/tokenizer.json",
    "sha256": "87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4",
    "bytes": 19989343,
    "semantically_identical_to_n25_mini": true,
    "components": {
      "version": true,
      "truncation": true,
      "padding": true,
      "added_tokens": true,
      "normalizer": true,
      "pre_tokenizer": true,
      "post_processor": true,
      "decoder": true,
      "model": true
    }
  },
  "endpoint_status": {
    "mini": "429 at baseline; no measurements",
    "pro": "Deferred by harness shared free-tier rule; no measurements"
  },
  "probe_text_published": false
}