diff --git a/.gitignore b/.gitignore index fe9534bc..c25e36a3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ DerivedData/ .idea .index-build *.out + +# Local venv for Tools/generate_tokenizer_baselines.py +.venv*/ diff --git a/Tests/TokenizersTests/MultilingualConformanceTests.swift b/Tests/TokenizersTests/MultilingualConformanceTests.swift new file mode 100644 index 00000000..da9ccd13 --- /dev/null +++ b/Tests/TokenizersTests/MultilingualConformanceTests.swift @@ -0,0 +1,330 @@ +// +// MultilingualConformanceTests.swift +// +// Byte-identical conformance tests: the Swift port's `tokenizer.encode(text:)` +// output must match the canonical HuggingFace Python `transformers` reference +// for every input in `Resources/MultilingualConformance/inputs.json` across +// the kernels in `BaselineKernel.all`. +// +// Inputs are categorised and stable-keyed so a divergence message points +// reviewers at the exact axis that broke (japanese-voiced-kana, emoji-keycap, +// thai-combining-marks, …). Baselines are regenerated by +// `Tools/generate_tokenizer_baselines.py` against the transformers version +// pinned in `Tools/requirements.txt`. +// +// `expectedDivergences` lets the test target ship green while bug fixes are +// in flight: each entry pairs a (model, input id) with the PR that will land +// the fix. An unexpected divergence is a hard failure (regression catch); +// an unexpected *match* prints a cleanup hint inviting removal of the entry +// but doesn't fail (so a freshly merged fix doesn't break CI on this file). +// +// Conformance design conceptually anchored in @apocryphx's +// ObjCTokenizer port (https://github.com/apocryphx/ObjCTokenizer) and the +// `expectedDivergences` + decoded-fields ideas from @john-rocky's closed +// #357. See issue #352 for the multilingual-divergence catalogue this +// corpus exercises. + +import Foundation +import Testing + +@testable import Hub +@testable import Models +@testable import Tokenizers + +// MARK: - Resource model + +private struct CorpusEntry: Decodable, Sendable { + let id: String + let category: String + let text: String +} + +private struct BaselineEntry: Decodable, Sendable { + let id: String + let inputIds: [Int] + let tokens: [String] + let decodedWithSpecial: String + let decodedSkipSpecial: String + + enum CodingKeys: String, CodingKey { + case id + case inputIds = "input_ids" + case tokens + case decodedWithSpecial = "decoded_with_special" + case decodedSkipSpecial = "decoded_skip_special" + } +} + +private struct BaselineMetadata: Decodable, Sendable { + let modelId: String + let transformersVersion: String + let generatedAt: String + let inputCount: Int + + enum CodingKeys: String, CodingKey { + case modelId = "model_id" + case transformersVersion = "transformers_version" + case generatedAt = "generated_at" + case inputCount = "input_count" + } +} + +private struct BaselineFile: Decodable, Sendable { + let metadata: BaselineMetadata + let entries: [BaselineEntry] +} + +// MARK: - Kernel matrix + +private struct BaselineKernel: Sendable, CustomStringConvertible { + /// Filename slug under `Resources/MultilingualConformance/baselines/_multilingual.json`. + let slug: String + /// `AutoTokenizer.from(pretrained:)` argument. + let modelId: String + + var description: String { modelId } + + static let all: [BaselineKernel] = [ + BaselineKernel(slug: "bge_small", modelId: "BAAI/bge-small-en-v1.5"), + BaselineKernel(slug: "t5_small", modelId: "google-t5/t5-small"), + BaselineKernel(slug: "gpt2", modelId: "openai-community/gpt2"), + BaselineKernel(slug: "roberta_base", modelId: "FacebookAI/roberta-base"), + BaselineKernel(slug: "qwen2_5", modelId: "Qwen/Qwen2.5-0.5B"), + BaselineKernel(slug: "tinyllama", modelId: "TinyLlama/TinyLlama-1.1B-Chat-v1.0"), + ] +} + +// MARK: - Divergences known to be in flight + +/// (modelId, inputId) pairs whose encode output is known to diverge from the +/// Python reference today, with a free-form note documenting the surface so +/// follow-up triage has a starting point. Cleanup-hint pattern inspired by +/// @john-rocky's closed #357: an unexpected match prints a hint inviting +/// entry removal, an unexpected divergence is a hard test failure. +private struct ExpectedDivergence: Sendable, Hashable { + let modelId: String + let inputId: String + let note: String +} + +private let expectedDivergences: Set = [ + // + // Two new bug clusters this corpus surfaces that aren't addressed by the + // initial fix wave (#354 / #355 / #356, all merged). Worth filing as + // separate follow-up issues under #352. + // + + // SentencePiece-BPE leading-whitespace runs collapse to single `▁` tokens + // instead of producing a single multi-space vocab entry (e.g. `▁▁▁▁`). + // Suggests the Metaspace pre-tokenizer or BPE merge step isn't recognising + // `▁▁▁▁` (id 268 in TinyLlama vocab) as a vocab-eligible merge target. + .init(modelId: "TinyLlama/TinyLlama-1.1B-Chat-v1.0", inputId: "code-python-if", note: "Metaspace leading-whitespace runs"), + .init(modelId: "TinyLlama/TinyLlama-1.1B-Chat-v1.0", inputId: "code-python-return", note: "Metaspace leading-whitespace runs"), + .init(modelId: "TinyLlama/TinyLlama-1.1B-Chat-v1.0", inputId: "code-python-recurse", note: "Metaspace leading-whitespace runs"), + .init(modelId: "TinyLlama/TinyLlama-1.1B-Chat-v1.0", inputId: "whitespace-runs", note: "Metaspace leading-whitespace runs"), + .init(modelId: "TinyLlama/TinyLlama-1.1B-Chat-v1.0", inputId: "whitespace-trailing-tabs", note: "Metaspace leading-whitespace runs"), + + // Qwen2.5 byte-level BPE picks a different merge ordering on Thai + // (and Thai-inside-multiscript) than HF Python. Byte-level encoding + // means there are no combining-mark traps; this is a merge-priority + // ordering issue in the BPE algorithm itself. Worth tracing once + // #355's merge-loop changes have settled. + .init(modelId: "Qwen/Qwen2.5-0.5B", inputId: "thai-combining-marks-greeting", note: "byte-level BPE merge-ordering on Thai"), + .init(modelId: "Qwen/Qwen2.5-0.5B", inputId: "thai-combining-marks-prose", note: "byte-level BPE merge-ordering on Thai"), + .init(modelId: "Qwen/Qwen2.5-0.5B", inputId: "multiscript-greetings", note: "byte-level BPE merge-ordering on Thai"), +] + +private func divergenceExpected(model: String, input: String) -> ExpectedDivergence? { + expectedDivergences.first { $0.modelId == model && $0.inputId == input } +} + +// MARK: - Resource loading + +private enum ConformanceError: Error, CustomStringConvertible { + case missingResource(String) + case decodeError(String, Error) + case unsupportedTokenizer(String) + + var description: String { + switch self { + case .missingResource(let name): "missing test resource: \(name)" + case .decodeError(let name, let err): "decode error in \(name): \(err)" + case .unsupportedTokenizer(let id): "tokenizer for \(id) was not a PreTrainedTokenizer" + } + } +} + +// SwiftPM's `.process("Resources")` flattens subdirectory structure into the +// test bundle root, so `subdirectory:` lookups don't apply here. Filenames +// (`inputs.json`, `_multilingual.json`) are unique within the test +// resources, so flat lookup is unambiguous. +private func loadCorpus() throws -> [CorpusEntry] { + guard let url = Bundle.module.url(forResource: "inputs", withExtension: "json") else { + throw ConformanceError.missingResource("inputs.json") + } + let data = try Data(contentsOf: url) + do { return try JSONDecoder().decode([CorpusEntry].self, from: data) } + catch { throw ConformanceError.decodeError("inputs.json", error) } +} + +private func loadBaseline(slug: String) throws -> BaselineFile { + let resource = "\(slug)_multilingual" + guard let url = Bundle.module.url(forResource: resource, withExtension: "json") else { + throw ConformanceError.missingResource("\(resource).json") + } + let data = try Data(contentsOf: url) + do { return try JSONDecoder().decode(BaselineFile.self, from: data) } + catch { throw ConformanceError.decodeError(resource, error) } +} + +// MARK: - Failure diagnostics + +/// Format a windowed diff around the first divergence point: `expected_window` +/// + `got_window` decoded to readable token strings, with the divergence +/// position underlined. Easier to triage than two long id arrays. +private func divergenceReport( + inputId: String, + category: String, + expectedIds: [Int], + expectedTokens: [String], + gotIds: [Int], + gotTokens: [String] +) -> String { + let commonLen = min(expectedIds.count, gotIds.count) + var divIdx = 0 + while divIdx < commonLen, expectedIds[divIdx] == gotIds[divIdx] { divIdx += 1 } + let windowLo = max(0, divIdx - 3) + let windowExpectedHi = min(expectedIds.count, divIdx + 5) + let windowGotHi = min(gotIds.count, divIdx + 5) + + func annotate(_ ids: [Int], _ tokens: [String], hi: Int) -> String { + var parts: [String] = [] + for i in windowLo..() + for entry in corpus { + #expect(!entry.id.isEmpty, "empty id at category=\(entry.category)") + #expect(!entry.category.isEmpty, "empty category at id=\(entry.id)") + // text may legitimately contain trailing whitespace etc.; + // empty string isn't a valid encode target though. + #expect(!entry.text.isEmpty, "empty text at id=\(entry.id)") + #expect(!seen.contains(entry.id), "duplicate id: \(entry.id)") + seen.insert(entry.id) + } + } + + @Test("Baselines cover the corpus exactly", arguments: BaselineKernel.all) + fileprivate func baselinesCoverCorpus(kernel: BaselineKernel) throws { + let corpus = try loadCorpus() + let baseline = try loadBaseline(slug: kernel.slug) + let corpusIds = Set(corpus.map(\.id)) + let baselineIds = Set(baseline.entries.map(\.id)) + + let missing = corpusIds.subtracting(baselineIds) + let extra = baselineIds.subtracting(corpusIds) + #expect( + missing.isEmpty, + "baseline for \(kernel.modelId) is missing entries: \(missing.sorted())" + ) + #expect( + extra.isEmpty, + "baseline for \(kernel.modelId) has stale entries (corpus shrunk?): \(extra.sorted())" + ) + #expect( + baseline.metadata.inputCount == baseline.entries.count, + "baseline metadata input_count mismatch for \(kernel.modelId): metadata=\(baseline.metadata.inputCount) entries=\(baseline.entries.count)" + ) + } + + @Test("Byte-identical token ids vs HF Python", arguments: BaselineKernel.all) + fileprivate func byteIdenticalTokenIds(kernel: BaselineKernel) async throws { + let corpus = try loadCorpus() + let baseline = try loadBaseline(slug: kernel.slug) + let entriesById = Dictionary(uniqueKeysWithValues: baseline.entries.map { ($0.id, $0) }) + + let tokenizerOpt = try await AutoTokenizer.from(pretrained: kernel.modelId) as? PreTrainedTokenizer + guard let tokenizer = tokenizerOpt else { + throw ConformanceError.unsupportedTokenizer(kernel.modelId) + } + + var unexpectedDivergences: [String] = [] + var unexpectedMatches: [ExpectedDivergence] = [] + + for input in corpus { + guard let expected = entriesById[input.id] else { continue } + let got = tokenizer.encode(text: input.text) + let knownDivergence = divergenceExpected(model: kernel.modelId, input: input.id) + + if got == expected.inputIds { + if let exp = knownDivergence { + // Test stays green but the table needs cleanup. + unexpectedMatches.append(exp) + } + } else { + if knownDivergence != nil { + // Listed in expectedDivergences — this is the in-flight + // state, not a regression. + continue + } + let gotTokens = got.map { tokenizer.convertIdToToken($0) ?? "?" } + unexpectedDivergences.append( + divergenceReport( + inputId: input.id, + category: input.category, + expectedIds: expected.inputIds, + expectedTokens: expected.tokens, + gotIds: got, + gotTokens: gotTokens + ) + ) + } + } + + // Cleanup hint — does NOT fail the test. A freshly merged improvement + // shouldn't break CI on this file; the hint just tells reviewers an + // expectedDivergences entry can be removed. + for match in unexpectedMatches { + print(""" + [\(kernel.modelId)] expectedDivergences entry no longer applies: + input id: \(match.inputId) + note: \(match.note) + hint: remove this entry from expectedDivergences in MultilingualConformanceTests.swift + """) + } + + // Regression catch — IS a failure. + #expect( + unexpectedDivergences.isEmpty, + """ + \(kernel.modelId): \(unexpectedDivergences.count) unexpected divergence(s) from HF Python reference. + If a divergence is being addressed in an open PR or is otherwise + known, add an ExpectedDivergence(modelId: …, inputId: …, note: …) entry. + + \(unexpectedDivergences.joined(separator: "\n\n")) + """ + ) + } +} diff --git a/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/bge_small_multilingual.json b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/bge_small_multilingual.json new file mode 100644 index 00000000..fe775922 --- /dev/null +++ b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/bge_small_multilingual.json @@ -0,0 +1,5087 @@ +{ + "metadata": { + "model_id": "BAAI/bge-small-en-v1.5", + "transformers_version": "4.57.1", + "generated_at": "2026-05-15T23:58:33+00:00", + "input_count": 83 + }, + "entries": [ + { + "id": "latin-diacritics-french", + "input_ids": [ + 101, + 7668, + 13746, + 15743, + 8508, + 1517, + 13675, + 21382, + 7987, + 9307, + 2063, + 2001, + 6581, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "cafe", + "resume", + "naive", + "facade", + "—", + "cr", + "##eme", + "br", + "##ule", + "##e", + "was", + "excellent", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] cafe resume naive facade — creme brulee was excellent. [SEP]", + "decoded_skip_special": "cafe resume naive facade — creme brulee was excellent." + }, + { + "id": "latin-diacritics-mixed", + "input_ids": [ + 101, + 4121, + 2140, + 19169, + 7987, + 12722, + 2063, + 1012, + 24951, + 2532, + 29536, + 2100, + 1037, + 2474, + 19409, + 9530, + 4560, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "huge", + "##l", + "uber", + "br", + "##uck", + "##e", + ".", + "mana", + "##na", + "vo", + "##y", + "a", + "la", + "pena", + "con", + "jose", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] hugel uber brucke. manana voy a la pena con jose. [SEP]", + "decoded_skip_special": "hugel uber brucke. manana voy a la pena con jose." + }, + { + "id": "latin-diacritics-portuguese", + "input_ids": [ + 101, + 7509, + 9094, + 1041, + 8529, + 2050, + 28744, + 9648, + 6970, + 7971, + 12956, + 1025, + 28306, + 8740, + 2818, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "sao", + "paulo", + "e", + "um", + "##a", + "cid", + "##ade", + "inter", + "##ess", + "##ante", + ";", + "koln", + "au", + "##ch", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] sao paulo e uma cidade interessante ; koln auch. [SEP]", + "decoded_skip_special": "sao paulo e uma cidade interessante ; koln auch." + }, + { + "id": "cyrillic-greeting", + "input_ids": [ + 101, + 1194, + 16856, + 10325, + 25529, + 15290, + 22919, + 1010, + 1191, + 10325, + 16856, + 999, + 1208, + 22919, + 14150, + 1197, + 15290, + 29747, + 22919, + 1197, + 14150, + 23925, + 15290, + 18947, + 10325, + 29744, + 10260, + 22919, + 14150, + 16856, + 10260, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "п", + "##р", + "##и", + "##в", + "##е", + "##т", + ",", + "м", + "##и", + "##р", + "!", + "э", + "##т", + "##о", + "т", + "##е", + "##с", + "##т", + "т", + "##о", + "##к", + "##е", + "##н", + "##и", + "##з", + "##а", + "##т", + "##о", + "##р", + "##а", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] привет, мир! это тест токенизатора. [SEP]", + "decoded_skip_special": "привет, мир! это тест токенизатора." + }, + { + "id": "cyrillic-prose", + "input_ids": [ + 101, + 1191, + 14150, + 29747, + 23925, + 25529, + 10260, + 1517, + 1196, + 22919, + 14150, + 29436, + 10325, + 29751, + 10260, + 1195, + 14150, + 29747, + 29747, + 15414, + 1012, + 1192, + 10260, + 29747, + 15290, + 29436, + 15290, + 18947, + 10325, + 15290, + 1181, + 14150, + 29436, + 15290, + 15290, + 2260, + 1191, + 10325, + 29436, + 29436, + 10325, + 14150, + 18947, + 19259, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "м", + "##о", + "##с", + "##к", + "##в", + "##а", + "—", + "с", + "##т", + "##о", + "##л", + "##и", + "##ц", + "##а", + "р", + "##о", + "##с", + "##с", + "##ии", + ".", + "н", + "##а", + "##с", + "##е", + "##л", + "##е", + "##н", + "##и", + "##е", + "б", + "##о", + "##л", + "##е", + "##е", + "12", + "м", + "##и", + "##л", + "##л", + "##и", + "##о", + "##н", + "##ов", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] москва — столица россии. население более 12 миллионов. [SEP]", + "decoded_skip_special": "москва — столица россии. население более 12 миллионов." + }, + { + "id": "greek-greeting", + "input_ids": [ + 101, + 1164, + 14608, + 29727, + 24824, + 29728, + 29723, + 29732, + 14608, + 1164, + 29730, + 29733, + 29728, + 29723, + 999, + 1161, + 1157, + 29727, + 29739, + 29733, + 29733, + 14608, + 1159, + 18199, + 16177, + 14608, + 18199, + 1169, + 29728, + 29730, + 29732, + 29736, + 24824, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "κ", + "##α", + "##λ", + "##η", + "##μ", + "##ε", + "##ρ", + "##α", + "κ", + "##ο", + "##σ", + "##μ", + "##ε", + "!", + "η", + "γ", + "##λ", + "##ω", + "##σ", + "##σ", + "##α", + "ε", + "##ι", + "##ν", + "##α", + "##ι", + "ο", + "##μ", + "##ο", + "##ρ", + "##φ", + "##η", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] καλημερα κοσμε! η γλωσσα ειναι ομορφη. [SEP]", + "decoded_skip_special": "καλημερα κοσμε! η γλωσσα ειναι ομορφη." + }, + { + "id": "cjk-simplified-greeting", + "input_ids": [ + 101, + 100, + 100, + 1010, + 1745, + 100, + 999, + 100, + 100, + 1775, + 100, + 100, + 100, + 100, + 1636, + 102 + ], + "tokens": [ + "[CLS]", + "[UNK]", + "[UNK]", + ",", + "世", + "[UNK]", + "!", + "[UNK]", + "[UNK]", + "分", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "。", + "[SEP]" + ], + "decoded_with_special": "[CLS] [UNK] [UNK], 世 [UNK]! [UNK] [UNK] 分 [UNK] [UNK] [UNK] [UNK] 。 [SEP]", + "decoded_skip_special": ", 世! 分 。" + }, + { + "id": "cjk-simplified-prose", + "input_ids": [ + 101, + 1746, + 1861, + 100, + 1745, + 100, + 1742, + 100, + 100, + 1756, + 1788, + 100, + 100, + 1916, + 100, + 100, + 1749, + 1740, + 1636, + 102 + ], + "tokens": [ + "[CLS]", + "中", + "文", + "[UNK]", + "世", + "[UNK]", + "上", + "[UNK]", + "[UNK]", + "人", + "口", + "[UNK]", + "[UNK]", + "的", + "[UNK]", + "[UNK]", + "之", + "一", + "。", + "[SEP]" + ], + "decoded_with_special": "[CLS] 中 文 [UNK] 世 [UNK] 上 [UNK] [UNK] 人 口 [UNK] [UNK] 的 [UNK] [UNK] 之 一 。 [SEP]", + "decoded_skip_special": "中 文 世 上 人 口 的 之 一 。" + }, + { + "id": "cjk-traditional-compare", + "input_ids": [ + 101, + 100, + 100, + 1746, + 1861, + 100, + 100, + 100, + 1746, + 1861, + 1873, + 100, + 1744, + 1794, + 1636, + 102 + ], + "tokens": [ + "[CLS]", + "[UNK]", + "[UNK]", + "中", + "文", + "[UNK]", + "[UNK]", + "[UNK]", + "中", + "文", + "有", + "[UNK]", + "不", + "同", + "。", + "[SEP]" + ], + "decoded_with_special": "[CLS] [UNK] [UNK] 中 文 [UNK] [UNK] [UNK] 中 文 有 [UNK] 不 同 。 [SEP]", + "decoded_skip_special": "中 文 中 文 有 不 同 。" + }, + { + "id": "cjk-traditional-hongkong", + "input_ids": [ + 101, + 1979, + 100, + 1916, + 100, + 100, + 1746, + 1861, + 1873, + 100, + 100, + 1916, + 100, + 100, + 1636, + 102 + ], + "tokens": [ + "[CLS]", + "香", + "[UNK]", + "的", + "[UNK]", + "[UNK]", + "中", + "文", + "有", + "[UNK]", + "[UNK]", + "的", + "[UNK]", + "[UNK]", + "。", + "[SEP]" + ], + "decoded_with_special": "[CLS] 香 [UNK] 的 [UNK] [UNK] 中 文 有 [UNK] [UNK] 的 [UNK] [UNK] 。 [SEP]", + "decoded_skip_special": "香 的 中 文 有 的 。" + }, + { + "id": "japanese-voiced-kana-greeting", + "input_ids": [ + 101, + 1655, + 30217, + 30194, + 30188, + 30198, + 1635, + 1745, + 100, + 1636, + 1714, + 30265, + 30228, + 30241, + 30221, + 30231, + 30265, + 30197, + 30239, + 30233, + 30240, + 30191, + 30184, + 1636, + 102 + ], + "tokens": [ + "[CLS]", + "こ", + "##ん", + "##に", + "##ち", + "##は", + "、", + "世", + "[UNK]", + "。", + "ト", + "##ー", + "##ク", + "##ナ", + "##イ", + "##サ", + "##ー", + "##の", + "##テ", + "##ス", + "##ト", + "##て", + "##す", + "。", + "[SEP]" + ], + "decoded_with_special": "[CLS] こんにちは 、 世 [UNK] 。 トークナイサーのテストてす 。 [SEP]", + "decoded_skip_special": "こんにちは 、 世 。 トークナイサーのテストてす 。" + }, + { + "id": "japanese-voiced-kana-prose", + "input_ids": [ + 101, + 1879, + 1755, + 1709, + 30262, + 30265, + 30198, + 30192, + 30191, + 30207, + 100, + 1682, + 30177, + 30193, + 100, + 1770, + 1802, + 1665, + 30184, + 1636, + 102 + ], + "tokens": [ + "[CLS]", + "東", + "京", + "タ", + "##ワ", + "##ー", + "##は", + "##と", + "##て", + "##も", + "[UNK]", + "や", + "##か", + "##な", + "[UNK]", + "光", + "地", + "て", + "##す", + "。", + "[SEP]" + ], + "decoded_with_special": "[CLS] 東 京 タワーはとても [UNK] やかな [UNK] 光 地 てす 。 [SEP]", + "decoded_skip_special": "東 京 タワーはとても やかな 光 地 てす 。" + }, + { + "id": "hangul-syllables-greeting", + "input_ids": [ + 101, + 1463, + 30006, + 30021, + 29992, + 30010, + 30025, + 30005, + 30006, + 29997, + 30009, + 29999, + 30013, + 1010, + 100, + 999, + 1467, + 30011, + 30002, + 30017, + 29992, + 30006, + 29999, + 30019, + 30000, + 30008, + 1467, + 30009, + 29997, + 30017, + 30003, + 30017, + 29999, + 30019, + 30024, + 29992, + 30019, + 29993, + 30006, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "ᄋ", + "##ᅡ", + "##ᆫ", + "##ᄂ", + "##ᅧ", + "##ᆼ", + "##ᄒ", + "##ᅡ", + "##ᄉ", + "##ᅦ", + "##ᄋ", + "##ᅭ", + ",", + "[UNK]", + "!", + "ᄐ", + "##ᅩ", + "##ᄏ", + "##ᅳ", + "##ᄂ", + "##ᅡ", + "##ᄋ", + "##ᅵ", + "##ᄌ", + "##ᅥ", + "ᄐ", + "##ᅦ", + "##ᄉ", + "##ᅳ", + "##ᄐ", + "##ᅳ", + "##ᄋ", + "##ᅵ", + "##ᆸ", + "##ᄂ", + "##ᅵ", + "##ᄃ", + "##ᅡ", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] 안녕하세요, [UNK]! 토크나이저 테스트입니다. [SEP]", + "decoded_skip_special": "안녕하세요,! 토크나이저 테스트입니다." + }, + { + "id": "hangul-syllables-prose", + "input_ids": [ + 101, + 1469, + 30006, + 30021, + 29991, + 30014, + 30020, + 29999, + 30008, + 29992, + 30017, + 30021, + 1469, + 30006, + 30021, + 29991, + 30017, + 30022, + 29994, + 30011, + 1468, + 30013, + 29991, + 30019, + 30005, + 30006, + 30024, + 29992, + 30019, + 29993, + 30006, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "ᄒ", + "##ᅡ", + "##ᆫ", + "##ᄀ", + "##ᅮ", + "##ᆨ", + "##ᄋ", + "##ᅥ", + "##ᄂ", + "##ᅳ", + "##ᆫ", + "ᄒ", + "##ᅡ", + "##ᆫ", + "##ᄀ", + "##ᅳ", + "##ᆯ", + "##ᄅ", + "##ᅩ", + "ᄑ", + "##ᅭ", + "##ᄀ", + "##ᅵ", + "##ᄒ", + "##ᅡ", + "##ᆸ", + "##ᄂ", + "##ᅵ", + "##ᄃ", + "##ᅡ", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] 한국어는 한글로 표기합니다. [SEP]", + "decoded_skip_special": "한국어는 한글로 표기합니다." + }, + { + "id": "arabic-greeting", + "input_ids": [ + 101, + 1295, + 17149, + 29820, + 29816, + 25573, + 1271, + 25573, + 23673, + 29830, + 25573, + 23673, + 22192, + 999, + 1297, + 29822, + 25573, + 1270, + 29821, + 29817, + 29816, + 25573, + 17149, + 1294, + 23673, + 22192, + 29820, + 23673, + 23673, + 1270, + 23673, + 23673, + 29831, + 29836, + 14498, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "م", + "##ر", + "##ح", + "##ب", + "##ا", + "ب", + "##ا", + "##ل", + "##ع", + "##ا", + "##ل", + "##م", + "!", + "ه", + "##ذ", + "##ا", + "ا", + "##خ", + "##ت", + "##ب", + "##ا", + "##ر", + "ل", + "##ل", + "##م", + "##ح", + "##ل", + "##ل", + "ا", + "##ل", + "##ل", + "##غ", + "##و", + "##ي", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] مرحبا بالعالم! هذا اختبار للمحلل اللغوي. [SEP]", + "decoded_skip_special": "مرحبا بالعالم! هذا اختبار للمحلل اللغوي." + }, + { + "id": "arabic-prose", + "input_ids": [ + 101, + 1270, + 23673, + 23673, + 29831, + 19433, + 1270, + 23673, + 29830, + 17149, + 29816, + 14498, + 19433, + 1294, + 29831, + 19433, + 1282, + 25573, + 22192, + 14498, + 19433, + 1273, + 29835, + 29817, + 29816, + 1295, + 15915, + 1270, + 23673, + 14498, + 22192, + 14498, + 15915, + 1270, + 23673, + 29837, + 1270, + 23673, + 14498, + 29824, + 25573, + 17149, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "ا", + "##ل", + "##ل", + "##غ", + "##ة", + "ا", + "##ل", + "##ع", + "##ر", + "##ب", + "##ي", + "##ة", + "ل", + "##غ", + "##ة", + "س", + "##ا", + "##م", + "##ي", + "##ة", + "ت", + "##ك", + "##ت", + "##ب", + "م", + "##ن", + "ا", + "##ل", + "##ي", + "##م", + "##ي", + "##ن", + "ا", + "##ل", + "##ى", + "ا", + "##ل", + "##ي", + "##س", + "##ا", + "##ر", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] اللغة العربية لغة سامية تكتب من اليمين الى اليسار. [SEP]", + "decoded_skip_special": "اللغة العربية لغة سامية تكتب من اليمين الى اليسار." + }, + { + "id": "hebrew-greeting", + "input_ids": [ + 101, + 1266, + 29799, + 29792, + 29800, + 1259, + 29792, + 29799, + 29800, + 999, + 1247, + 29128, + 1255, + 29789, + 29794, + 29802, + 1266, + 29799, + 1245, + 29795, + 29792, + 29810, + 29803, + 29796, + 29796, + 29793, + 29811, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "ש", + "##ל", + "##ו", + "##ם", + "ע", + "##ו", + "##ל", + "##ם", + "!", + "ז", + "##ה", + "מ", + "##ב", + "##ח", + "##ן", + "ש", + "##ל", + "ה", + "##ט", + "##ו", + "##ק", + "##נ", + "##י", + "##י", + "##ז", + "##ר", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] שלום עולם! זה מבחן של הטוקנייזר. [SEP]", + "decoded_skip_special": "שלום עולם! זה מבחן של הטוקנייזר." + }, + { + "id": "devanagari-hindi", + "input_ids": [ + 101, + 1327, + 29867, + 29874, + 29859, + 1325, + 29863, + 29877, + 29868, + 29876, + 999, + 1332, + 29875, + 100, + 1328, + 29869, + 29878, + 29851, + 29873, + 29858, + 1339, + 1344, + 102 + ], + "tokens": [ + "[CLS]", + "न", + "##म", + "##स", + "##त", + "द", + "##न", + "##ि", + "##य", + "##ा", + "!", + "य", + "##ह", + "[UNK]", + "प", + "##र", + "##ी", + "##क", + "##ष", + "##ण", + "ह", + "।", + "[SEP]" + ], + "decoded_with_special": "[CLS] नमसत दनिया! यह [UNK] परीकषण ह । [SEP]", + "decoded_skip_special": "नमसत दनिया! यह परीकषण ह ।" + }, + { + "id": "devanagari-sanskrit", + "input_ids": [ + 101, + 1338, + 29874, + 29851, + 29859, + 1330, + 29876, + 29869, + 29859, + 1315, + 29878, + 1314, + 29851, + 1328, + 29869, + 29876, + 29854, + 29878, + 29863, + 1330, + 29876, + 29873, + 29876, + 1339, + 1344, + 102 + ], + "tokens": [ + "[CLS]", + "स", + "##स", + "##क", + "##त", + "भ", + "##ा", + "##र", + "##त", + "क", + "##ी", + "ए", + "##क", + "प", + "##र", + "##ा", + "##च", + "##ी", + "##न", + "भ", + "##ा", + "##ष", + "##ा", + "ह", + "।", + "[SEP]" + ], + "decoded_with_special": "[CLS] ससकत भारत की एक पराचीन भाषा ह । [SEP]", + "decoded_skip_special": "ससकत भारत की एक पराचीन भाषा ह ।" + }, + { + "id": "thai-combining-marks-greeting", + "input_ids": [ + 101, + 100, + 999, + 100, + 102 + ], + "tokens": [ + "[CLS]", + "[UNK]", + "!", + "[UNK]", + "[SEP]" + ], + "decoded_with_special": "[CLS] [UNK]! [UNK] [SEP]", + "decoded_skip_special": "!" + }, + { + "id": "thai-combining-marks-prose", + "input_ids": [ + 101, + 100, + 102 + ], + "tokens": [ + "[CLS]", + "[UNK]", + "[SEP]" + ], + "decoded_with_special": "[CLS] [UNK] [SEP]", + "decoded_skip_special": "" + }, + { + "id": "mixed-script-cjk-latin", + "input_ids": [ + 101, + 1996, + 1781, + 1755, + 4440, + 1999, + 16798, + 2575, + 1517, + 24403, + 29316, + 999, + 102 + ], + "tokens": [ + "[CLS]", + "the", + "北", + "京", + "trip", + "in", + "202", + "##6", + "—", + "tres", + "bien", + "!", + "[SEP]" + ], + "decoded_with_special": "[CLS] the 北 京 trip in 2026 — tres bien! [SEP]", + "decoded_skip_special": "the 北 京 trip in 2026 — tres bien!" + }, + { + "id": "mixed-script-katakana-code", + "input_ids": [ + 101, + 1704, + 30265, + 30240, + 30260, + 30245, + 30255, + 30265, + 1024, + 11829, + 4964, + 1999, + 10797, + 1037, + 2509, + 2546, + 2683, + 2278, + 2475, + 2497, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "コ", + "##ー", + "##ト", + "##レ", + "##ヒ", + "##ュ", + "##ー", + ":", + "bug", + "fixed", + "in", + "commit", + "a", + "##3", + "##f", + "##9", + "##c", + "##2", + "##b", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] コートレヒュー : bug fixed in commit a3f9c2b. [SEP]", + "decoded_skip_special": "コートレヒュー : bug fixed in commit a3f9c2b." + }, + { + "id": "math-symbols-summation", + "input_ids": [ + 101, + 2765, + 1024, + 1173, + 1060, + 1035, + 1045, + 1027, + 4413, + 1006, + 2073, + 1060, + 1596, + 1579, + 1007, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "result", + ":", + "σ", + "x", + "_", + "i", + "=", + "42", + "(", + "where", + "x", + "∈", + "ℝ", + ")", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] result : σ x _ i = 42 ( where x ∈ ℝ ). [SEP]", + "decoded_skip_special": "result : σ x _ i = 42 ( where x ∈ ℝ )." + }, + { + "id": "math-symbols-epsilon-delta", + "input_ids": [ + 101, + 9872, + 1024, + 100, + 1028, + 1014, + 100, + 1028, + 1014, + 2107, + 2008, + 1064, + 1060, + 1011, + 1060, + 17110, + 1064, + 1026, + 1158, + 1591, + 1064, + 1042, + 1006, + 1060, + 1007, + 1011, + 1042, + 1006, + 1060, + 17110, + 1007, + 1064, + 1026, + 1159, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "theorem", + ":", + "[UNK]", + ">", + "0", + "[UNK]", + ">", + "0", + "such", + "that", + "|", + "x", + "-", + "x", + "##₀", + "|", + "<", + "δ", + "⇒", + "|", + "f", + "(", + "x", + ")", + "-", + "f", + "(", + "x", + "##₀", + ")", + "|", + "<", + "ε", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] theorem : [UNK] > 0 [UNK] > 0 such that | x - x₀ | < δ ⇒ | f ( x ) - f ( x₀ ) | < ε. [SEP]", + "decoded_skip_special": "theorem : > 0 > 0 such that | x - x₀ | < δ ⇒ | f ( x ) - f ( x₀ ) | < ε." + }, + { + "id": "currency-symbols-multi", + "input_ids": [ + 101, + 3465, + 1024, + 1574, + 12521, + 1012, + 2753, + 1585, + 1071, + 2487, + 1010, + 26667, + 1606, + 26812, + 1012, + 4293, + 1006, + 22480, + 1010, + 4606, + 100, + 1012, + 2199, + 2549, + 1007, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "cost", + ":", + "€", + "##12", + ".", + "50", + "→", + "¥", + "##1", + ",", + "820", + "≈", + "£10", + ".", + "75", + "(", + "approx", + ",", + "plus", + "[UNK]", + ".", + "000", + "##4", + ")", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] cost : €12. 50 → ¥1, 820 ≈ £10. 75 ( approx, plus [UNK]. 0004 ). [SEP]", + "decoded_skip_special": "cost : €12. 50 → ¥1, 820 ≈ £10. 75 ( approx, plus. 0004 )." + }, + { + "id": "box-drawing", + "input_ids": [ + 101, + 3482, + 1024, + 100, + 1616, + 7592, + 1616, + 100, + 1517, + 2589, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "box", + ":", + "[UNK]", + "│", + "hello", + "│", + "[UNK]", + "—", + "done", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] box : [UNK] │ hello │ [UNK] — done. [SEP]", + "decoded_skip_special": "box : │ hello │ — done." + }, + { + "id": "dingbats-chess", + "input_ids": [ + 101, + 4109, + 1024, + 1620, + 1621, + 1623, + 1624, + 1625, + 1626, + 100, + 100, + 100, + 100, + 100, + 1517, + 7433, + 2275, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "pieces", + ":", + "★", + "☆", + "♠", + "♣", + "♥", + "♦", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "—", + "chess", + "set", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] pieces : ★ ☆ ♠ ♣ ♥ ♦ [UNK] [UNK] [UNK] [UNK] [UNK] — chess set. [SEP]", + "decoded_skip_special": "pieces : ★ ☆ ♠ ♣ ♥ ♦ — chess set." + }, + { + "id": "math-astral-bold-script", + "input_ids": [ + 101, + 8785, + 7782, + 1024, + 100, + 100, + 1517, + 8785, + 5896, + 1024, + 100, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "math", + "bold", + ":", + "[UNK]", + "[UNK]", + "—", + "math", + "script", + ":", + "[UNK]", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] math bold : [UNK] [UNK] — math script : [UNK]. [SEP]", + "decoded_skip_special": "math bold : — math script :." + }, + { + "id": "astral-egyptian-hieroglyphs", + "input_ids": [ + 101, + 6811, + 7632, + 10624, + 25643, + 18757, + 1024, + 100, + 100, + 100, + 100, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "egyptian", + "hi", + "##ero", + "##gly", + "##phs", + ":", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] egyptian hieroglyphs : [UNK] [UNK] [UNK] [UNK]. [SEP]", + "decoded_skip_special": "egyptian hieroglyphs :." + }, + { + "id": "astral-cuneiform", + "input_ids": [ + 101, + 12731, + 2638, + 22631, + 1024, + 100, + 100, + 100, + 100, + 1517, + 3696, + 2862, + 8168, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "cu", + "##ne", + "##iform", + ":", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "—", + "sign", + "list", + "samples", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] cuneiform : [UNK] [UNK] [UNK] [UNK] — sign list samples. [SEP]", + "decoded_skip_special": "cuneiform : — sign list samples." + }, + { + "id": "astral-mahjong", + "input_ids": [ + 101, + 5003, + 2232, + 21958, + 13262, + 1024, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "ma", + "##h", + "##jong", + "tiles", + ":", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] mahjong tiles : [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK] [UNK]. [SEP]", + "decoded_skip_special": "mahjong tiles :." + }, + { + "id": "astral-cards", + "input_ids": [ + 101, + 5329, + 1024, + 100, + 100, + 100, + 100, + 100, + 100, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "cards", + ":", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] cards : [UNK] [UNK] [UNK] [UNK] [UNK] [UNK]. [SEP]", + "decoded_skip_special": "cards :." + }, + { + "id": "emoji-bmp-and-astral", + "input_ids": [ + 101, + 100, + 7592, + 999, + 100, + 2088, + 100, + 4888, + 100, + 2283, + 100, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "[UNK]", + "hello", + "!", + "[UNK]", + "world", + "[UNK]", + "launch", + "[UNK]", + "party", + "[UNK]", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] [UNK] hello! [UNK] world [UNK] launch [UNK] party [UNK]. [SEP]", + "decoded_skip_special": "hello! world launch party." + }, + { + "id": "emoji-zwj-family-pride-skin", + "input_ids": [ + 101, + 2155, + 1024, + 100, + 1517, + 6620, + 5210, + 1024, + 100, + 1517, + 3096, + 1011, + 4309, + 1024, + 100, + 100, + 100, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "family", + ":", + "[UNK]", + "—", + "pride", + "flag", + ":", + "[UNK]", + "—", + "skin", + "-", + "tone", + ":", + "[UNK]", + "[UNK]", + "[UNK]", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] family : [UNK] — pride flag : [UNK] — skin - tone : [UNK] [UNK] [UNK]. [SEP]", + "decoded_skip_special": "family : — pride flag : — skin - tone :." + }, + { + "id": "emoji-zwj-with-text-prefix", + "input_ids": [ + 101, + 100, + 1009, + 100, + 1009, + 100, + 1009, + 3231, + 102 + ], + "tokens": [ + "[CLS]", + "[UNK]", + "+", + "[UNK]", + "+", + "[UNK]", + "+", + "test", + "[SEP]" + ], + "decoded_with_special": "[CLS] [UNK] + [UNK] + [UNK] + test [SEP]", + "decoded_skip_special": "+ + + test" + }, + { + "id": "emoji-keycap-and-flags", + "input_ids": [ + 101, + 3145, + 17695, + 2015, + 1024, + 100, + 100, + 100, + 1517, + 5210, + 1024, + 100, + 100, + 100, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "key", + "##cap", + "##s", + ":", + "[UNK]", + "[UNK]", + "[UNK]", + "—", + "flag", + ":", + "[UNK]", + "[UNK]", + "[UNK]", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] keycaps : [UNK] [UNK] [UNK] — flag : [UNK] [UNK] [UNK]. [SEP]", + "decoded_skip_special": "keycaps : — flag :." + }, + { + "id": "url-with-query-fragment", + "input_ids": [ + 101, + 24471, + 2140, + 1027, + 16770, + 1024, + 1013, + 1013, + 2742, + 1012, + 4012, + 1013, + 4130, + 1013, + 2000, + 1013, + 7692, + 1029, + 1053, + 1027, + 29379, + 1004, + 3347, + 1027, + 8670, + 2480, + 1004, + 1050, + 1027, + 4413, + 1001, + 2930, + 1011, + 1017, + 102 + ], + "tokens": [ + "[CLS]", + "ur", + "##l", + "=", + "https", + ":", + "/", + "/", + "example", + ".", + "com", + "/", + "path", + "/", + "to", + "/", + "resource", + "?", + "q", + "=", + "foo", + "&", + "bar", + "=", + "ba", + "##z", + "&", + "n", + "=", + "42", + "#", + "section", + "-", + "3", + "[SEP]" + ], + "decoded_with_special": "[CLS] url = https : / / example. com / path / to / resource? q = foo & bar = baz & n = 42 # section - 3 [SEP]", + "decoded_skip_special": "url = https : / / example. com / path / to / resource? q = foo & bar = baz & n = 42 # section - 3" + }, + { + "id": "hex-and-base64", + "input_ids": [ + 101, + 2002, + 2595, + 1027, + 1014, + 2595, + 3207, + 4215, + 11306, + 11329, + 10354, + 15878, + 16336, + 2918, + 21084, + 1027, + 1061, + 2860, + 3501, + 3501, + 2480, + 2290, + 2615, + 2213, + 2480, + 2475, + 22269, + 3286, + 3215, + 2497, + 2860, + 2629, + 25465, + 2050, + 1027, + 1027, + 102 + ], + "tokens": [ + "[CLS]", + "he", + "##x", + "=", + "0", + "##x", + "##de", + "##ad", + "##bee", + "##fc", + "##af", + "##eb", + "##abe", + "base", + "##64", + "=", + "y", + "##w", + "##j", + "##j", + "##z", + "##g", + "##v", + "##m", + "##z", + "##2", + "##hp", + "##am", + "##ts", + "##b", + "##w", + "##5", + "##vc", + "##a", + "=", + "=", + "[SEP]" + ], + "decoded_with_special": "[CLS] hex = 0xdeadbeefcafebabe base64 = ywjjzgvmz2hpamtsbw5vca = = [SEP]", + "decoded_skip_special": "hex = 0xdeadbeefcafebabe base64 = ywjjzgvmz2hpamtsbw5vca = =" + }, + { + "id": "sha256-hash", + "input_ids": [ + 101, + 21146, + 17788, + 2575, + 1027, + 1041, + 2509, + 2497, + 2692, + 2278, + 22932, + 24594, + 2620, + 11329, + 2487, + 2278, + 16932, + 2683, + 10354, + 29292, + 2549, + 2278, + 2620, + 2683, + 2683, + 2575, + 26337, + 2683, + 18827, + 22907, + 6679, + 23632, + 2063, + 21472, + 26224, + 2497, + 2683, + 22022, + 3540, + 26224, + 28154, + 2683, + 2487, + 2497, + 2581, + 27531, + 2475, + 2497, + 27531, + 2629, + 102 + ], + "tokens": [ + "[CLS]", + "sha", + "##25", + "##6", + "=", + "e", + "##3", + "##b", + "##0", + "##c", + "##44", + "##29", + "##8", + "##fc", + "##1", + "##c", + "##14", + "##9", + "##af", + "##bf", + "##4", + "##c", + "##8", + "##9", + "##9", + "##6", + "##fb", + "##9", + "##24", + "##27", + "##ae", + "##41", + "##e", + "##46", + "##49", + "##b", + "##9", + "##34", + "##ca", + "##49", + "##59", + "##9", + "##1", + "##b", + "##7", + "##85", + "##2", + "##b", + "##85", + "##5", + "[SEP]" + ], + "decoded_with_special": "[CLS] sha256 = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 [SEP]", + "decoded_skip_special": "sha256 = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "id": "code-javascript-functional", + "input_ids": [ + 101, + 12098, + 2099, + 1012, + 4949, + 1006, + 1060, + 1027, + 1028, + 1060, + 1008, + 1016, + 1007, + 1012, + 11307, + 1006, + 1060, + 1027, + 1028, + 1060, + 1028, + 1014, + 1007, + 1012, + 5547, + 1006, + 1006, + 1037, + 1010, + 1038, + 1007, + 1027, + 1028, + 1037, + 1009, + 1038, + 1007, + 1025, + 102 + ], + "tokens": [ + "[CLS]", + "ar", + "##r", + ".", + "map", + "(", + "x", + "=", + ">", + "x", + "*", + "2", + ")", + ".", + "filter", + "(", + "x", + "=", + ">", + "x", + ">", + "0", + ")", + ".", + "reduce", + "(", + "(", + "a", + ",", + "b", + ")", + "=", + ">", + "a", + "+", + "b", + ")", + ";", + "[SEP]" + ], + "decoded_with_special": "[CLS] arr. map ( x = > x * 2 ). filter ( x = > x > 0 ). reduce ( ( a, b ) = > a + b ) ; [SEP]", + "decoded_skip_special": "arr. map ( x = > x * 2 ). filter ( x = > x > 0 ). reduce ( ( a, b ) = > a + b ) ;" + }, + { + "id": "code-cplusplus-template", + "input_ids": [ + 101, + 23561, + 1026, + 1056, + 1010, + 1057, + 1028, + 8285, + 1042, + 1006, + 1056, + 1004, + 1004, + 1056, + 1010, + 1057, + 1004, + 1004, + 1057, + 1007, + 1011, + 1028, + 11703, + 24228, + 5051, + 1006, + 1056, + 1009, + 1057, + 1007, + 1063, + 2709, + 1056, + 1009, + 1057, + 1025, + 1065, + 102 + ], + "tokens": [ + "[CLS]", + "template", + "<", + "t", + ",", + "u", + ">", + "auto", + "f", + "(", + "t", + "&", + "&", + "t", + ",", + "u", + "&", + "&", + "u", + ")", + "-", + ">", + "dec", + "##lty", + "##pe", + "(", + "t", + "+", + "u", + ")", + "{", + "return", + "t", + "+", + "u", + ";", + "}", + "[SEP]" + ], + "decoded_with_special": "[CLS] template < t, u > auto f ( t & & t, u & & u ) - > decltype ( t + u ) { return t + u ; } [SEP]", + "decoded_skip_special": "template < t, u > auto f ( t & & t, u & & u ) - > decltype ( t + u ) { return t + u ; }" + }, + { + "id": "code-bitops", + "input_ids": [ + 101, + 2765, + 1027, + 1006, + 1037, + 1028, + 1028, + 1016, + 1007, + 1064, + 1006, + 1038, + 1026, + 1026, + 1018, + 1007, + 1004, + 1014, + 2595, + 4246, + 1025, + 102 + ], + "tokens": [ + "[CLS]", + "result", + "=", + "(", + "a", + ">", + ">", + "2", + ")", + "|", + "(", + "b", + "<", + "<", + "4", + ")", + "&", + "0", + "##x", + "##ff", + ";", + "[SEP]" + ], + "decoded_with_special": "[CLS] result = ( a > > 2 ) | ( b < < 4 ) & 0xff ; [SEP]", + "decoded_skip_special": "result = ( a > > 2 ) | ( b < < 4 ) & 0xff ;" + }, + { + "id": "code-javascript-modern", + "input_ids": [ + 101, + 2065, + 1006, + 1060, + 999, + 1027, + 1027, + 19701, + 1004, + 1004, + 1061, + 1028, + 1027, + 1014, + 1007, + 1063, + 2709, + 1060, + 1029, + 1012, + 29379, + 1029, + 1029, + 12398, + 10175, + 5657, + 1025, + 1065, + 102 + ], + "tokens": [ + "[CLS]", + "if", + "(", + "x", + "!", + "=", + "=", + "null", + "&", + "&", + "y", + ">", + "=", + "0", + ")", + "{", + "return", + "x", + "?", + ".", + "foo", + "?", + "?", + "default", + "##val", + "##ue", + ";", + "}", + "[SEP]" + ], + "decoded_with_special": "[CLS] if ( x! = = null & & y > = 0 ) { return x?. foo?? defaultvalue ; } [SEP]", + "decoded_skip_special": "if ( x! = = null & & y > = 0 ) { return x?. foo?? defaultvalue ; }" + }, + { + "id": "code-python-def", + "input_ids": [ + 101, + 13366, + 10882, + 11735, + 6305, + 6895, + 1006, + 1050, + 1007, + 1024, + 102 + ], + "tokens": [ + "[CLS]", + "def", + "fi", + "##bon", + "##ac", + "##ci", + "(", + "n", + ")", + ":", + "[SEP]" + ], + "decoded_with_special": "[CLS] def fibonacci ( n ) : [SEP]", + "decoded_skip_special": "def fibonacci ( n ) :" + }, + { + "id": "code-python-if", + "input_ids": [ + 101, + 2065, + 1050, + 1026, + 1016, + 1024, + 102 + ], + "tokens": [ + "[CLS]", + "if", + "n", + "<", + "2", + ":", + "[SEP]" + ], + "decoded_with_special": "[CLS] if n < 2 : [SEP]", + "decoded_skip_special": "if n < 2 :" + }, + { + "id": "code-python-return", + "input_ids": [ + 101, + 2709, + 1050, + 102 + ], + "tokens": [ + "[CLS]", + "return", + "n", + "[SEP]" + ], + "decoded_with_special": "[CLS] return n [SEP]", + "decoded_skip_special": "return n" + }, + { + "id": "code-python-recurse", + "input_ids": [ + 101, + 2709, + 10882, + 11735, + 6305, + 6895, + 1006, + 1050, + 1011, + 1015, + 1007, + 1009, + 10882, + 11735, + 6305, + 6895, + 1006, + 1050, + 1011, + 1016, + 1007, + 102 + ], + "tokens": [ + "[CLS]", + "return", + "fi", + "##bon", + "##ac", + "##ci", + "(", + "n", + "-", + "1", + ")", + "+", + "fi", + "##bon", + "##ac", + "##ci", + "(", + "n", + "-", + "2", + ")", + "[SEP]" + ], + "decoded_with_special": "[CLS] return fibonacci ( n - 1 ) + fibonacci ( n - 2 ) [SEP]", + "decoded_skip_special": "return fibonacci ( n - 1 ) + fibonacci ( n - 2 )" + }, + { + "id": "code-go-func", + "input_ids": [ + 101, + 4569, + 2278, + 17021, + 1006, + 2171, + 5164, + 1007, + 1063, + 102 + ], + "tokens": [ + "[CLS]", + "fun", + "##c", + "greet", + "(", + "name", + "string", + ")", + "{", + "[SEP]" + ], + "decoded_with_special": "[CLS] func greet ( name string ) { [SEP]", + "decoded_skip_special": "func greet ( name string ) {" + }, + { + "id": "code-go-tab-if", + "input_ids": [ + 101, + 2065, + 2171, + 999, + 1027, + 1000, + 1000, + 1063, + 102 + ], + "tokens": [ + "[CLS]", + "if", + "name", + "!", + "=", + "\"", + "\"", + "{", + "[SEP]" + ], + "decoded_with_special": "[CLS] if name! = \" \" { [SEP]", + "decoded_skip_special": "if name! = \" \" {" + }, + { + "id": "code-go-tab-printf", + "input_ids": [ + 101, + 4718, + 2102, + 1012, + 6140, + 2546, + 1006, + 1000, + 7592, + 1010, + 1003, + 1055, + 999, + 1032, + 1050, + 1000, + 1010, + 2171, + 1007, + 102 + ], + "tokens": [ + "[CLS]", + "fm", + "##t", + ".", + "print", + "##f", + "(", + "\"", + "hello", + ",", + "%", + "s", + "!", + "\\", + "n", + "\"", + ",", + "name", + ")", + "[SEP]" + ], + "decoded_with_special": "[CLS] fmt. printf ( \" hello, % s! \\ n \", name ) [SEP]", + "decoded_skip_special": "fmt. printf ( \" hello, % s! \\ n \", name )" + }, + { + "id": "code-go-tab-close", + "input_ids": [ + 101, + 1065, + 102 + ], + "tokens": [ + "[CLS]", + "}", + "[SEP]" + ], + "decoded_with_special": "[CLS] } [SEP]", + "decoded_skip_special": "}" + }, + { + "id": "code-go-close", + "input_ids": [ + 101, + 1065, + 102 + ], + "tokens": [ + "[CLS]", + "}", + "[SEP]" + ], + "decoded_with_special": "[CLS] } [SEP]", + "decoded_skip_special": "}" + }, + { + "id": "programming-identifiers-mixed", + "input_ids": [ + 101, + 2131, + 1035, + 5310, + 1035, + 8909, + 4098, + 1035, + 17698, + 1035, + 2946, + 1056, + 1035, + 11687, + 1035, + 19204, + 1035, + 8909, + 24978, + 3126, + 4877, + 7971, + 3258, + 8663, + 8873, + 27390, + 3370, + 102 + ], + "tokens": [ + "[CLS]", + "get", + "_", + "user", + "_", + "id", + "max", + "_", + "buffer", + "_", + "size", + "t", + "_", + "pad", + "_", + "token", + "_", + "id", + "ns", + "##ur", + "##ls", + "##ess", + "##ion", + "##con", + "##fi", + "##gur", + "##ation", + "[SEP]" + ], + "decoded_with_special": "[CLS] get _ user _ id max _ buffer _ size t _ pad _ token _ id nsurlsessionconfiguration [SEP]", + "decoded_skip_special": "get _ user _ id max _ buffer _ size t _ pad _ token _ id nsurlsessionconfiguration" + }, + { + "id": "programming-identifiers-cases", + "input_ids": [ + 101, + 19130, + 18382, + 10755, + 17878, + 18382, + 7488, + 1035, + 2553, + 1035, + 13075, + 7491, + 1035, + 7488, + 1047, + 2620, + 2015, + 1035, + 17491, + 1035, + 4175, + 1035, + 1058, + 2475, + 102 + ], + "tokens": [ + "[CLS]", + "camel", + "##case", + "##var", + "pascal", + "##case", + "snake", + "_", + "case", + "_", + "var", + "screaming", + "_", + "snake", + "k", + "##8", + "##s", + "_", + "pod", + "_", + "count", + "_", + "v", + "##2", + "[SEP]" + ], + "decoded_with_special": "[CLS] camelcasevar pascalcase snake _ case _ var screaming _ snake k8s _ pod _ count _ v2 [SEP]", + "decoded_skip_special": "camelcasevar pascalcase snake _ case _ var screaming _ snake k8s _ pod _ count _ v2" + }, + { + "id": "code-string-multiscript", + "input_ids": [ + 101, + 14806, + 1027, + 1000, + 1194, + 16856, + 10325, + 25529, + 15290, + 22919, + 1010, + 1191, + 10325, + 16856, + 999, + 1000, + 1009, + 1000, + 1517, + 2013, + 1000, + 1009, + 1000, + 1879, + 1755, + 1000, + 102 + ], + "tokens": [ + "[CLS]", + "greeting", + "=", + "\"", + "п", + "##р", + "##и", + "##в", + "##е", + "##т", + ",", + "м", + "##и", + "##р", + "!", + "\"", + "+", + "\"", + "—", + "from", + "\"", + "+", + "\"", + "東", + "京", + "\"", + "[SEP]" + ], + "decoded_with_special": "[CLS] greeting = \" привет, мир! \" + \" — from \" + \" 東 京 \" [SEP]", + "decoded_skip_special": "greeting = \" привет, мир! \" + \" — from \" + \" 東 京 \"" + }, + { + "id": "code-comment-diacritics", + "input_ids": [ + 101, + 1013, + 1013, + 13746, + 1517, + 9170, + 7615, + 2007, + 22939, + 26775, + 18291, + 2015, + 102 + ], + "tokens": [ + "[CLS]", + "/", + "/", + "resume", + "—", + "swift", + "comment", + "with", + "dia", + "##cr", + "##itic", + "##s", + "[SEP]" + ], + "decoded_with_special": "[CLS] / / resume — swift comment with diacritics [SEP]", + "decoded_skip_special": "/ / resume — swift comment with diacritics" + }, + { + "id": "code-comment-greek-math", + "input_ids": [ + 101, + 1013, + 1008, + 1155, + 1009, + 1156, + 1027, + 1157, + 1517, + 3306, + 1011, + 3661, + 8785, + 7615, + 1008, + 1013, + 102 + ], + "tokens": [ + "[CLS]", + "/", + "*", + "α", + "+", + "β", + "=", + "γ", + "—", + "greek", + "-", + "letter", + "math", + "comment", + "*", + "/", + "[SEP]" + ], + "decoded_with_special": "[CLS] / * α + β = γ — greek - letter math comment * / [SEP]", + "decoded_skip_special": "/ * α + β = γ — greek - letter math comment * /" + }, + { + "id": "code-string-multiscript-emoji", + "input_ids": [ + 101, + 2292, + 2516, + 1027, + 1000, + 100, + 4888, + 1517, + 1194, + 29748, + 29747, + 23925, + 1517, + 100, + 100, + 1000, + 102 + ], + "tokens": [ + "[CLS]", + "let", + "title", + "=", + "\"", + "[UNK]", + "launch", + "—", + "п", + "##у", + "##с", + "##к", + "—", + "[UNK]", + "[UNK]", + "\"", + "[SEP]" + ], + "decoded_with_special": "[CLS] let title = \" [UNK] launch — пуск — [UNK] [UNK] \" [SEP]", + "decoded_skip_special": "let title = \" launch — пуск — \"" + }, + { + "id": "ipa-phonetic-transcription", + "input_ids": [ + 101, + 1996, + 24531, + 14193, + 1013, + 1042, + 29685, + 29696, + 1013, + 2005, + 1000, + 3869, + 1000, + 1517, + 2007, + 6911, + 1024, + 1149, + 2546, + 29685, + 29696, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "the", + "ipa", + "transcription", + "/", + "f", + "##ɪ", + "##ʃ", + "/", + "for", + "\"", + "fish", + "\"", + "—", + "with", + "stress", + ":", + "ˈ", + "##f", + "##ɪ", + "##ʃ", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] the ipa transcription / fɪʃ / for \" fish \" — with stress : ˈfɪʃ. [SEP]", + "decoded_skip_special": "the ipa transcription / fɪʃ / for \" fish \" — with stress : ˈfɪʃ." + }, + { + "id": "mandarin-tones", + "input_ids": [ + 101, + 12623, + 1024, + 5003, + 5003, + 5003, + 5003, + 1006, + 15831, + 12623, + 1517, + 2152, + 1010, + 4803, + 1010, + 23427, + 1010, + 4634, + 1007, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "tones", + ":", + "ma", + "ma", + "ma", + "ma", + "(", + "mandarin", + "tones", + "—", + "high", + ",", + "rising", + ",", + "dipping", + ",", + "falling", + ")", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] tones : ma ma ma ma ( mandarin tones — high, rising, dipping, falling ). [SEP]", + "decoded_skip_special": "tones : ma ma ma ma ( mandarin tones — high, rising, dipping, falling )." + }, + { + "id": "whitespace-runs", + "input_ids": [ + 101, + 7592, + 2088, + 1012, + 2203, + 102 + ], + "tokens": [ + "[CLS]", + "hello", + "world", + ".", + "end", + "[SEP]" + ], + "decoded_with_special": "[CLS] hello world. end [SEP]", + "decoded_skip_special": "hello world. end" + }, + { + "id": "whitespace-trailing-tabs", + "input_ids": [ + 101, + 12542, + 12461, + 15327, + 2077, + 2047, + 4179, + 1024, + 1006, + 21628, + 1007, + 1006, + 7258, + 1007, + 102 + ], + "tokens": [ + "[CLS]", + "trailing", + "whites", + "##pace", + "before", + "new", + "##line", + ":", + "(", + "tab", + ")", + "(", + "spaces", + ")", + "[SEP]" + ], + "decoded_with_special": "[CLS] trailing whitespace before newline : ( tab ) ( spaces ) [SEP]", + "decoded_skip_special": "trailing whitespace before newline : ( tab ) ( spaces )" + }, + { + "id": "punctuation-period", + "input_ids": [ + 101, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS]. [SEP]", + "decoded_skip_special": "." + }, + { + "id": "punctuation-comma", + "input_ids": [ + 101, + 1010, + 102 + ], + "tokens": [ + "[CLS]", + ",", + "[SEP]" + ], + "decoded_with_special": "[CLS], [SEP]", + "decoded_skip_special": "," + }, + { + "id": "punctuation-exclamation", + "input_ids": [ + 101, + 999, + 102 + ], + "tokens": [ + "[CLS]", + "!", + "[SEP]" + ], + "decoded_with_special": "[CLS]! [SEP]", + "decoded_skip_special": "!" + }, + { + "id": "punctuation-nested-quotes", + "input_ids": [ + 101, + 1000, + 2054, + 1029, + 1000, + 2016, + 2356, + 1517, + 1000, + 2428, + 1029, + 999, + 1000, + 1517, + 2748, + 1012, + 1012, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "\"", + "what", + "?", + "\"", + "she", + "asked", + "—", + "\"", + "really", + "?", + "!", + "\"", + "—", + "yes", + ".", + ".", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] \" what? \" she asked — \" really?! \" — yes... [SEP]", + "decoded_skip_special": "\" what? \" she asked — \" really?! \" — yes..." + }, + { + "id": "punctuation-dash-variants", + "input_ids": [ + 101, + 1012, + 1012, + 1012, + 1998, + 1515, + 2036, + 1516, + 1998, + 1517, + 1998, + 1518, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + ".", + ".", + ".", + "and", + "‒", + "also", + "–", + "and", + "—", + "and", + "―", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS]... and ‒ also – and — and ―. [SEP]", + "decoded_skip_special": "... and ‒ also – and — and ―." + }, + { + "id": "multiscript-greetings", + "input_ids": [ + 101, + 7592, + 100, + 100, + 1463, + 30006, + 30021, + 29992, + 30010, + 30025, + 1655, + 30217, + 30194, + 30188, + 30198, + 1327, + 29867, + 29874, + 29859, + 1295, + 17149, + 29820, + 29816, + 25573, + 1266, + 29799, + 29792, + 29800, + 100, + 1194, + 16856, + 10325, + 25529, + 15290, + 22919, + 1164, + 14608, + 29727, + 24824, + 29728, + 29723, + 29732, + 14608, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "hello", + "[UNK]", + "[UNK]", + "ᄋ", + "##ᅡ", + "##ᆫ", + "##ᄂ", + "##ᅧ", + "##ᆼ", + "こ", + "##ん", + "##に", + "##ち", + "##は", + "न", + "##म", + "##स", + "##त", + "م", + "##ر", + "##ح", + "##ب", + "##ا", + "ש", + "##ל", + "##ו", + "##ם", + "[UNK]", + "п", + "##р", + "##и", + "##в", + "##е", + "##т", + "κ", + "##α", + "##λ", + "##η", + "##μ", + "##ε", + "##ρ", + "##α", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] hello [UNK] [UNK] 안녕 こんにちは नमसत مرحبا שלום [UNK] привет καλημερα. [SEP]", + "decoded_skip_special": "hello 안녕 こんにちは नमसत مرحبا שלום привет καλημερα." + }, + { + "id": "multiscript-no-spaces", + "input_ids": [ + 101, + 1155, + 100, + 100, + 102 + ], + "tokens": [ + "[CLS]", + "α", + "[UNK]", + "[UNK]", + "[SEP]" + ], + "decoded_with_special": "[CLS] α [UNK] [UNK] [SEP]", + "decoded_skip_special": "α" + }, + { + "id": "multiscript-zwj-mixed", + "input_ids": [ + 101, + 1746, + 1861, + 1469, + 30006, + 30021, + 29991, + 30017, + 30022, + 1864, + 1876, + 1950, + 100, + 100, + 100, + 1792, + 102 + ], + "tokens": [ + "[CLS]", + "中", + "文", + "ᄒ", + "##ᅡ", + "##ᆫ", + "##ᄀ", + "##ᅳ", + "##ᆯ", + "日", + "本", + "語", + "[UNK]", + "[UNK]", + "[UNK]", + "合", + "[SEP]" + ], + "decoded_with_special": "[CLS] 中 文 한글 日 本 語 [UNK] [UNK] [UNK] 合 [SEP]", + "decoded_skip_special": "中 文 한글 日 本 語 合" + }, + { + "id": "multiscript-string-concat", + "input_ids": [ + 101, + 14806, + 1027, + 1000, + 100, + 100, + 1000, + 1009, + 100, + 1009, + 1000, + 1463, + 30006, + 30021, + 29992, + 30010, + 30025, + 1000, + 1009, + 1327, + 29867, + 29874, + 29859, + 1009, + 100, + 102 + ], + "tokens": [ + "[CLS]", + "greeting", + "=", + "\"", + "[UNK]", + "[UNK]", + "\"", + "+", + "[UNK]", + "+", + "\"", + "ᄋ", + "##ᅡ", + "##ᆫ", + "##ᄂ", + "##ᅧ", + "##ᆼ", + "\"", + "+", + "न", + "##म", + "##स", + "##त", + "+", + "[UNK]", + "[SEP]" + ], + "decoded_with_special": "[CLS] greeting = \" [UNK] [UNK] \" + [UNK] + \" 안녕 \" + नमसत + [UNK] [SEP]", + "decoded_skip_special": "greeting = \" \" + + \" 안녕 \" + नमसत +" + }, + { + "id": "multiscript-emoji-arrows", + "input_ids": [ + 101, + 1173, + 1060, + 1035, + 1045, + 100, + 1027, + 1781, + 1755, + 1585, + 25802, + 1087, + 5522, + 1087, + 1742, + 1902, + 100, + 2203, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "σ", + "x", + "_", + "i", + "[UNK]", + "=", + "北", + "京", + "→", + "munchen", + "·", + "tokyo", + "·", + "上", + "海", + "[UNK]", + "end", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] σ x _ i [UNK] = 北 京 → munchen · tokyo · 上 海 [UNK] end. [SEP]", + "decoded_skip_special": "σ x _ i = 北 京 → munchen · tokyo · 上 海 end." + }, + { + "id": "multiscript-arrow-chain", + "input_ids": [ + 101, + 100, + 100, + 100, + 100, + 102 + ], + "tokens": [ + "[CLS]", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "[SEP]" + ], + "decoded_with_special": "[CLS] [UNK] [UNK] [UNK] [UNK] [SEP]", + "decoded_skip_special": "" + }, + { + "id": "multiscript-function-call", + "input_ids": [ + 101, + 29379, + 1006, + 1746, + 1861, + 1010, + 1469, + 30006, + 30021, + 29991, + 30017, + 30022, + 1010, + 1864, + 1876, + 1950, + 1010, + 1339, + 29877, + 29863, + 29861, + 29878, + 1010, + 1195, + 29748, + 29747, + 29747, + 23925, + 15414, + 1007, + 1585, + 1063, + 100, + 1024, + 100, + 1010, + 1612, + 1024, + 1613, + 1065, + 102 + ], + "tokens": [ + "[CLS]", + "foo", + "(", + "中", + "文", + ",", + "ᄒ", + "##ᅡ", + "##ᆫ", + "##ᄀ", + "##ᅳ", + "##ᆯ", + ",", + "日", + "本", + "語", + ",", + "ह", + "##ि", + "##न", + "##द", + "##ी", + ",", + "р", + "##у", + "##с", + "##с", + "##к", + "##ии", + ")", + "→", + "{", + "[UNK]", + ":", + "[UNK]", + ",", + "⊕", + ":", + "⊗", + "}", + "[SEP]" + ], + "decoded_with_special": "[CLS] foo ( 中 文, 한글, 日 本 語, हिनदी, русскии ) → { [UNK] : [UNK], ⊕ : ⊗ } [SEP]", + "decoded_skip_special": "foo ( 中 文, 한글, 日 本 語, हिनदी, русскии ) → { :, ⊕ : ⊗ }" + }, + { + "id": "multiscript-legacy-symbols", + "input_ids": [ + 101, + 1578, + 20958, + 1580, + 1075, + 1079, + 1086, + 1073, + 100, + 100, + 100, + 100, + 1463, + 30006, + 30021, + 29992, + 30010, + 30025, + 1270, + 23673, + 102 + ], + "tokens": [ + "[CLS]", + "№", + "##42", + "™", + "©", + "®", + "¶", + "§", + "[UNK]", + "[UNK]", + "[UNK]", + "[UNK]", + "ᄋ", + "##ᅡ", + "##ᆫ", + "##ᄂ", + "##ᅧ", + "##ᆼ", + "ا", + "##ل", + "[SEP]" + ], + "decoded_with_special": "[CLS] №42 ™ © ® ¶ § [UNK] [UNK] [UNK] [UNK] 안녕 ال [SEP]", + "decoded_skip_special": "№42 ™ © ® ¶ § 안녕 ال" + }, + { + "id": "escape-sequences-with-zwj", + "input_ids": [ + 101, + 19701, + 1032, + 1014, + 5685, + 1032, + 23746, + 7875, + 1032, + 1050, + 2638, + 13668, + 3170, + 100, + 100, + 1062, + 2860, + 22578, + 12260, + 24817, + 102 + ], + "tokens": [ + "[CLS]", + "null", + "\\", + "0", + "##and", + "\\", + "tt", + "##ab", + "\\", + "n", + "##ne", + "##wl", + "##ine", + "[UNK]", + "[UNK]", + "z", + "##w", + "##js", + "##ele", + "##ctors", + "[SEP]" + ], + "decoded_with_special": "[CLS] null \\ 0and \\ ttab \\ nnewline [UNK] [UNK] zwjselectors [SEP]", + "decoded_skip_special": "null \\ 0and \\ ttab \\ nnewline zwjselectors" + }, + { + "id": "german-compound-short", + "input_ids": [ + 101, + 14085, + 3126, + 18337, + 1012, + 2139, + 16446, + 2818, + 1062, + 2666, + 11039, + 6892, + 2480, + 2102, + 7939, + 12849, + 8737, + 20049, + 2696, + 19742, + 17151, + 4315, + 13250, + 18900, + 5480, + 29548, + 4371, + 15916, + 11624, + 12083, + 27266, + 2063, + 1012, + 100, + 102 + ], + "tokens": [ + "[CLS]", + "nat", + "##ur", + "##lich", + ".", + "de", + "##uts", + "##ch", + "z", + "##ie", + "##ht", + "jet", + "##z", + "##t", + "den", + "ko", + "##mp", + "##osi", + "##ta", + "##hammer", + "aus", + "der", + "gram", + "##mat", + "##ik", + "##werk", + "##ze", + "##ug", + "##sch", + "##ub", + "##lad", + "##e", + ".", + "[UNK]", + "[SEP]" + ], + "decoded_with_special": "[CLS] naturlich. deutsch zieht jetzt den kompositahammer aus der grammatikwerkzeugschublade. [UNK] [SEP]", + "decoded_skip_special": "naturlich. deutsch zieht jetzt den kompositahammer aus der grammatikwerkzeugschublade." + }, + { + "id": "german-compound-long-1", + "input_ids": [ + 101, + 10047, + 22822, + 6914, + 2638, + 8671, + 5999, + 2319, + 7520, + 26061, + 13473, + 26896, + 5620, + 11191, + 3233, + 16417, + 10556, + 16020, + 12928, + 14416, + 13033, + 21850, + 10483, + 7361, + 4048, + 15878, + 8780, + 7693, + 3334, + 11265, + 10609, + 27665, + 2099, + 19723, + 6132, + 5428, + 10867, + 6299, + 8449, + 14540, + 7033, + 29501, + 3215, + 20100, + 6151, + 2025, + 3771, + 2618, + 1999, + 7367, + 2378, + 2365, + 12380, + 21693, + 21759, + 18595, + 26095, + 2078, + 15143, + 26745, + 3775, + 2480, + 25987, + 3280, + 9413, + 9153, + 4609, + 27412, + 5292, + 16093, + 8004, + 29501, + 2102, + 3854, + 21713, + 6238, + 9299, + 9153, + 12083, + 3385, + 10224, + 18337, + 7913, + 21031, + 9048, + 5643, + 2078, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "im", + "mor", + "##gen", + "##ne", + "##bel", + "##ged", + "##an", + "##ken", + "##vera", + "##ste", + "##lun", + "##gs", + "##wald", + "stand", + "ein", + "ka", + "##ffe", + "##eta", + "##ssen", + "##rand", + "##phi", + "##los", + "##op", + "##hi", + "##eb", + "##eo", + "##bach", + "##ter", + "ne", + "##ben", + "eine", + "##r", + "reg", + "##ens", + "##chi", + "##rm", + "##ver", + "##ges", + "##sl", + "##ich", + "##kei", + "##ts", + "##station", + "und", + "not", + "##ier", + "##te", + "in", + "se", + "##in", + "son", + "##nta", + "##gm", + "##org", + "##eni", + "##dee", + "##n", + "##fan", + "##gno", + "##ti", + "##z", + "##buch", + "die", + "er", + "##sta", + "##un", + "##liche", + "ha", + "##uf", + "##ig", + "##kei", + "##t", + "von", + "fen", + "##ster", + "##bank", + "##sta", + "##ub", + "##son", + "##nen", + "##lich", + "##tre", + "##fle", + "##xi", + "##one", + "##n", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] im morgennebelgedankenverastelungswald stand ein kaffeetassenrandphilosophiebeobachter neben einer regenschirmvergesslichkeitsstation und notierte in sein sonntagmorgenideenfangnotizbuch die erstaunliche haufigkeit von fensterbankstaubsonnenlichtreflexionen. [SEP]", + "decoded_skip_special": "im morgennebelgedankenverastelungswald stand ein kaffeetassenrandphilosophiebeobachter neben einer regenschirmvergesslichkeitsstation und notierte in sein sonntagmorgenideenfangnotizbuch die erstaunliche haufigkeit von fensterbankstaubsonnenlichtreflexionen." + }, + { + "id": "german-compound-long-2", + "input_ids": [ + 101, + 11265, + 10609, + 1045, + 14227, + 7680, + 20492, + 2063, + 27665, + 13970, + 7317, + 11624, + 26763, + 23111, + 2368, + 8671, + 13765, + 10143, + 5575, + 8043, + 23111, + 2121, + 5575, + 26212, + 11624, + 3170, + 1010, + 22894, + 7389, + 2094, + 16417, + 9388, + 10199, + 9648, + 3070, + 8523, + 26547, + 18349, + 4246, + 11231, + 3070, + 28793, + 3775, + 9856, + 15532, + 5620, + 26401, + 6199, + 29181, + 3334, + 10210, + 27665, + 2099, + 29461, + 21512, + 25311, + 20559, + 4817, + 19966, + 5480, + 21436, + 8180, + 20760, + 3070, + 8791, + 20766, + 3366, + 2022, + 26527, + 8004, + 2102, + 2162, + 1012, + 17151, + 4315, + 20863, + 2063, + 20976, + 8743, + 2063, + 14387, + 2232, + 16417, + 24185, + 28143, + 8950, + 8609, + 6290, + 7011, + 8093, + 3367, + 27225, + 13728, + 2271, + 5480, + 3683, + 8737, + 27296, + 2102, + 21200, + 27665, + 2213, + 16417, + 12173, + 6299, + 3489, + 8093, + 21338, + 24746, + 13728, + 21205, + 10143, + 5575, + 28745, + 6906, + 2102, + 6151, + 15544, + 12879, + 1024, + 1525, + 9353, + 11039, + 5575, + 1010, + 2079, + 5339, + 29536, + 12119, + 4682, + 13512, + 27665, + 16216, + 7847, + 7520, + 7011, + 4181, + 6299, + 2243, + 17048, + 5575, + 28745, + 12879, + 4430, + 7389, + 28032, + 14505, + 999, + 1523, + 102 + ], + "tokens": [ + "[CLS]", + "ne", + "##ben", + "i", + "##hm", + "sum", + "##mt", + "##e", + "eine", + "ku", + "##hl", + "##sch", + "##rank", + "##inn", + "##en", + "##bel", + "##eu", + "##cht", + "##ung", + "##ser", + "##inn", + "##er", + "##ung", + "##sma", + "##sch", + "##ine", + ",", + "wah", + "##ren", + "##d", + "ein", + "mar", + "##mel", + "##ade", + "##ng", + "##las", + "##deck", + "##elo", + "##ff", + "##nu", + "##ng", + "##sop", + "##ti", + "##mie", + "##run", + "##gs", + "##beau", + "##ft", + "##rag", + "##ter", + "mit", + "eine", + "##r", + "tre", + "##ppe", + "##nh", + "##aus", + "##ak", + "##ust", + "##ik", + "##uber", + "##ras", + "##chu", + "##ng", + "##san", + "##aly", + "##se", + "be", + "##schaft", + "##ig", + "##t", + "war", + ".", + "aus", + "der", + "fern", + "##e", + "nah", + "##ert", + "##e", + "sic", + "##h", + "ein", + "wo", + "##lke", + "##nk", + "##rat", + "##zer", + "##fa", + "##hr", + "##st", + "##uh", + "##lm", + "##us", + "##ik", + "##ko", + "##mp", + "##onis", + "##t", + "auf", + "eine", + "##m", + "ein", + "##rad", + "##ver", + "##ke", + "##hr", + "##sr", + "##ege", + "##lm", + "##issa", + "##cht", + "##ung", + "##sg", + "##era", + "##t", + "und", + "ri", + "##ef", + ":", + "„", + "ac", + "##ht", + "##ung", + ",", + "do", + "##rt", + "vo", + "##rne", + "lie", + "##gt", + "eine", + "ge", + "##dan", + "##ken", + "##fa", + "##den", + "##ver", + "##k", + "##not", + "##ung", + "##sg", + "##ef", + "##ah", + "##ren", + "##sit", + "##uation", + "!", + "“", + "[SEP]" + ], + "decoded_with_special": "[CLS] neben ihm summte eine kuhlschrankinnenbeleuchtungserinnerungsmaschine, wahrend ein marmeladenglasdeckeloffnungsoptimierungsbeauftragter mit einer treppenhausakustikuberraschungsanalyse beschaftigt war. aus der ferne naherte sich ein wolkenkratzerfahrstuhlmusikkomponist auf einem einradverkehrsregelmissachtungsgerat und rief : „ achtung, dort vorne liegt eine gedankenfadenverknotungsgefahrensituation! “ [SEP]", + "decoded_skip_special": "neben ihm summte eine kuhlschrankinnenbeleuchtungserinnerungsmaschine, wahrend ein marmeladenglasdeckeloffnungsoptimierungsbeauftragter mit einer treppenhausakustikuberraschungsanalyse beschaftigt war. aus der ferne naherte sich ein wolkenkratzerfahrstuhlmusikkomponist auf einem einradverkehrsregelmissachtungsgerat und rief : „ achtung, dort vorne liegt eine gedankenfadenverknotungsgefahrensituation! “" + }, + { + "id": "german-compound-long-3", + "input_ids": [ + 101, + 5436, + 2480, + 18337, + 2125, + 7159, + 2063, + 14387, + 2232, + 16417, + 27838, + 28813, + 11393, + 4221, + 19172, + 28522, + 12079, + 11101, + 16033, + 19310, + 6442, + 2389, + 1010, + 6151, + 2014, + 20559, + 16405, + 15378, + 20042, + 2368, + 2852, + 7416, + 20934, + 27923, + 10278, + 5017, + 29577, + 17579, + 8545, + 7096, + 28824, + 26527, + 13473, + 4014, + 2638, + 14227, + 23282, + 10224, + 1010, + 16417, + 11721, + 19731, + 14191, + 13777, + 21693, + 20267, + 2368, + 14971, + 10609, + 5677, + 24932, + 6151, + 16417, + 21929, + 25619, + 12173, + 8004, + 2310, + 2099, + 9148, + 12171, + 3334, + 10514, + 21512, + 20554, + 7245, + 2884, + 2890, + 21031, + 9048, + 5644, + 11368, + 9331, + 10536, + 5332, + 5484, + 1012, + 17070, + 12377, + 21559, + 24665, + 8630, + 12870, + 2078, + 9033, + 2063, + 7939, + 21122, + 6155, + 6299, + 12733, + 6519, + 6583, + 2818, + 22930, + 15900, + 14540, + 7033, + 3215, + 7507, + 25970, + 9126, + 6528, + 20915, + 22648, + 11039, + 5575, + 6151, + 17727, + 12298, + 17417, + 8743, + 2063, + 10682, + 14270, + 5054, + 4645, + 2368, + 16874, + 7416, + 4305, + 12734, + 5620, + 20528, + 2618, + 11239, + 2078, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "plot", + "##z", + "##lich", + "off", + "##net", + "##e", + "sic", + "##h", + "ein", + "ze", + "##itung", + "##sse", + "##ite", + "##num", + "##bla", + "##tter", + "##wind", + "##sto", + "##ß", + "##port", + "##al", + ",", + "und", + "her", + "##aus", + "pu", + "##rz", + "##elt", + "##en", + "dr", + "##ei", + "bu", + "##rok", + "##lam", + "##mer", + "##nsor", + "##tier", + "##we", + "##lt", + "##meister", + "##schaft", + "##ste", + "##il", + "##ne", + "##hm", + "##erin", + "##nen", + ",", + "ein", + "ga", + "##rte", + "##nz", + "##wer", + "##gm", + "##utz", + "##en", + "##far", + "##ben", + "##ber", + "##ater", + "und", + "ein", + "hoc", + "##hg", + "##rad", + "##ig", + "ve", + "##r", + "##wi", + "##rr", + "##ter", + "su", + "##ppe", + "##nl", + "##off", + "##el", + "##re", + "##fle", + "##xi", + "##ons", + "##met", + "##ap", + "##hy", + "##si", + "##ker", + ".", + "gem", + "##ein", + "##sam", + "gr", + "##und", + "##ete", + "##n", + "si", + "##e", + "den", + "bun", + "##des", + "##ver", + "##band", + "fur", + "na", + "##ch", + "##mit", + "##tag", + "##sl", + "##ich", + "##ts", + "##cha", + "##tten", + "##kan", + "##ten", + "##bet", + "##rac", + "##ht", + "##ung", + "und", + "imp", + "##rov", + "##isi", + "##ert", + "##e", + "sofa", + "##kis", + "##sen", + "##burg", + "##en", + "##vert", + "##ei", + "##di", + "##gun", + "##gs", + "##stra", + "##te", + "##gie", + "##n", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] plotzlich offnete sich ein zeitungsseitenumblatterwindstoßportal, und heraus purzelten drei buroklammernsortierweltmeisterschaftsteilnehmerinnen, ein gartenzwergmutzenfarbenberater und ein hochgradig verwirrter suppenloffelreflexionsmetaphysiker. gemeinsam grundeten sie den bundesverband fur nachmittagslichtschattenkantenbetrachtung und improvisierte sofakissenburgenverteidigungsstrategien. [SEP]", + "decoded_skip_special": "plotzlich offnete sich ein zeitungsseitenumblatterwindstoßportal, und heraus purzelten drei buroklammernsortierweltmeisterschaftsteilnehmerinnen, ein gartenzwergmutzenfarbenberater und ein hochgradig verwirrter suppenloffelreflexionsmetaphysiker. gemeinsam grundeten sie den bundesverband fur nachmittagslichtschattenkantenbetrachtung und improvisierte sofakissenburgenverteidigungsstrategien." + }, + { + "id": "german-compound-long-4", + "input_ids": [ + 101, + 2572, + 14863, + 4859, + 18601, + 29033, + 6528, + 14387, + 2232, + 2035, + 2063, + 10047, + 12256, + 5104, + 5403, + 2378, + 18940, + 6238, + 10404, + 3549, + 9153, + 12083, + 25394, + 29546, + 5831, + 15810, + 1010, + 24185, + 4315, + 10556, + 16020, + 12928, + 14416, + 13033, + 21850, + 10483, + 7361, + 4048, + 15878, + 8780, + 7693, + 3334, + 27665, + 24664, + 2121, + 27412, + 8040, + 7317, + 17854, + 5596, + 2063, + 19169, + 3280, + 15536, + 10143, + 8004, + 29501, + 2102, + 3854, + 4895, + 11493, + 3619, + 21572, + 28351, + 9285, + 19699, + 13765, + 3207, + 1010, + 11867, + 22648, + 14227, + 2271, + 11705, + 25383, + 11231, + 8569, + 25997, + 6151, + 24260, + 14066, + 8737, + 10343, + 5428, + 4246, + 7011, + 8093, + 3215, + 8449, + 5349, + 26527, + 8337, + 23270, + 6962, + 28120, + 10431, + 2243, + 3630, + 14376, + 2545, + 20501, + 2618, + 4014, + 12681, + 7507, + 4246, + 5575, + 13102, + 3217, + 3468, + 3549, + 7632, + 20042, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "am", + "abe", + "##nd", + "versa", + "##mmel", + "##ten", + "sic", + "##h", + "all", + "##e", + "im", + "mon", + "##ds", + "##che", + "##in", + "##fen", + "##ster", + "##rah", + "##men", + "##sta", + "##ub", + "##gli", + "##tzer", + "##zi", + "##mmer", + ",", + "wo", + "der", + "ka", + "##ffe", + "##eta", + "##ssen", + "##rand", + "##phi", + "##los", + "##op", + "##hi", + "##eb", + "##eo", + "##bach", + "##ter", + "eine", + "fei", + "##er", + "##liche", + "sc", + "##hl", + "##uss", + "##red", + "##e", + "uber", + "die", + "wi", + "##cht", + "##ig", + "##kei", + "##t", + "von", + "un", + "##sin", + "##ns", + "##pro", + "##duk", + "##tions", + "##fr", + "##eu", + "##de", + ",", + "sp", + "##rac", + "##hm", + "##us", + "##kel", + "##deh", + "##nu", + "##bu", + "##ngen", + "und", + "dona", + "##uda", + "##mp", + "##fs", + "##chi", + "##ff", + "##fa", + "##hr", + "##ts", + "##ges", + "##ell", + "##schaft", + "##ska", + "##pit", + "##ans", + "##mut", + "##zen", + "##k", + "##no", + "##pf", + "##ers", + "##atz", + "##te", + "##il", + "##bes", + "##cha", + "##ff", + "##ung", + "##sp", + "##ro", + "##ble", + "##men", + "hi", + "##elt", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] am abend versammelten sich alle im mondscheinfensterrahmenstaubglitzerzimmer, wo der kaffeetassenrandphilosophiebeobachter eine feierliche schlussrede uber die wichtigkeit von unsinnsproduktionsfreude, sprachmuskeldehnubungen und donaudampfschifffahrtsgesellschaftskapitansmutzenknopfersatzteilbeschaffungsproblemen hielt. [SEP]", + "decoded_skip_special": "am abend versammelten sich alle im mondscheinfensterrahmenstaubglitzerzimmer, wo der kaffeetassenrandphilosophiebeobachter eine feierliche schlussrede uber die wichtigkeit von unsinnsproduktionsfreude, sprachmuskeldehnubungen und donaudampfschifffahrtsgesellschaftskapitansmutzenknopfersatzteilbeschaffungsproblemen hielt." + }, + { + "id": "german-compound-long-5", + "input_ids": [ + 101, + 11271, + 2818, + 23506, + 11624, + 2618, + 2035, + 3351, + 26432, + 2638, + 16950, + 19699, + 6340, + 2368, + 26036, + 2102, + 1010, + 8740, + 17499, + 2099, + 21388, + 4315, + 13970, + 7317, + 11624, + 26763, + 23111, + 2368, + 8671, + 13765, + 10143, + 5575, + 8043, + 23111, + 2121, + 5575, + 26212, + 11624, + 3170, + 1010, + 3280, + 14387, + 2232, + 19169, + 2158, + 12439, + 13629, + 10722, + 3217, + 4246, + 11231, + 3070, + 23823, + 16715, + 2121, + 5705, + 3286, + 29501, + 3215, + 13777, + 3215, + 7507, + 5753, + 5575, + 2022, + 26086, + 13512, + 2063, + 1012, + 102 + ], + "tokens": [ + "[CLS]", + "dana", + "##ch", + "herr", + "##sch", + "##te", + "all", + "##ge", + "##mei", + "##ne", + "zu", + "##fr", + "##ied", + "##en", + "##hei", + "##t", + ",", + "au", + "##ße", + "##r", + "bei", + "der", + "ku", + "##hl", + "##sch", + "##rank", + "##inn", + "##en", + "##bel", + "##eu", + "##cht", + "##ung", + "##ser", + "##inn", + "##er", + "##ung", + "##sma", + "##sch", + "##ine", + ",", + "die", + "sic", + "##h", + "uber", + "man", + "##gel", + "##nde", + "tu", + "##ro", + "##ff", + "##nu", + "##ng", + "##sau", + "##fm", + "##er", + "##ks", + "##am", + "##kei", + "##ts", + "##wer", + "##ts", + "##cha", + "##tz", + "##ung", + "be", + "##kla", + "##gt", + "##e", + ".", + "[SEP]" + ], + "decoded_with_special": "[CLS] danach herrschte allgemeine zufriedenheit, außer bei der kuhlschrankinnenbeleuchtungserinnerungsmaschine, die sich uber mangelnde turoffnungsaufmerksamkeitswertschatzung beklagte. [SEP]", + "decoded_skip_special": "danach herrschte allgemeine zufriedenheit, außer bei der kuhlschrankinnenbeleuchtungserinnerungsmaschine, die sich uber mangelnde turoffnungsaufmerksamkeitswertschatzung beklagte." + } + ] +} diff --git a/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/gpt2_multilingual.json b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/gpt2_multilingual.json new file mode 100644 index 00000000..9dda743e --- /dev/null +++ b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/gpt2_multilingual.json @@ -0,0 +1,6571 @@ +{ + "metadata": { + "model_id": "openai-community/gpt2", + "transformers_version": "4.57.1", + "generated_at": "2026-05-15T23:58:34+00:00", + "input_count": 83 + }, + "entries": [ + { + "id": "latin-diacritics-french", + "input_ids": [ + 66, + 1878, + 2634, + 40560, + 16345, + 2634, + 41492, + 24685, + 16175, + 671, + 851, + 1067, + 14064, + 1326, + 865, + 42324, + 75, + 22161, + 373, + 6275, + 13 + ], + "tokens": [ + "c", + "af", + "é", + "Ġré", + "sum", + "é", + "Ġnaïve", + "Ġfa", + "ç", + "ade", + "ĠâĢĶ", + "Ġcr", + "è", + "me", + "Ġbr", + "û", + "l", + "ée", + "Ġwas", + "Ġexcellent", + "." + ], + "decoded_with_special": "café résumé naïve façade — crème brûlée was excellent.", + "decoded_skip_special": "café résumé naïve façade — crème brûlée was excellent." + }, + { + "id": "latin-diacritics-mixed", + "input_ids": [ + 39, + 9116, + 25280, + 6184, + 120, + 527, + 1709, + 9116, + 66, + 365, + 13, + 6669, + 12654, + 2271, + 23291, + 257, + 8591, + 613, + 30644, + 369, + 36997, + 13 + ], + "tokens": [ + "H", + "ü", + "gel", + "ĠÃ", + "¼", + "ber", + "ĠBr", + "ü", + "c", + "ke", + ".", + "ĠMa", + "ñ", + "ana", + "Ġvoy", + "Ġa", + "Ġla", + "Ġpe", + "ña", + "Ġcon", + "ĠJosé", + "." + ], + "decoded_with_special": "Hügel über Brücke. Mañana voy a la peña con José.", + "decoded_skip_special": "Hügel über Brücke. Mañana voy a la peña con José." + }, + { + "id": "latin-diacritics-portuguese", + "input_ids": [ + 50, + 28749, + 34410, + 38251, + 334, + 2611, + 269, + 312, + 671, + 493, + 68, + 601, + 12427, + 26, + 509, + 9101, + 18755, + 257, + 794, + 13 + ], + "tokens": [ + "S", + "ão", + "ĠPaulo", + "Ġé", + "Ġu", + "ma", + "Ġc", + "id", + "ade", + "Ġint", + "e", + "ress", + "ante", + ";", + "ĠK", + "ö", + "ln", + "Ġa", + "uch", + "." + ], + "decoded_with_special": "São Paulo é uma cidade interessante; Köln auch.", + "decoded_skip_special": "São Paulo é uma cidade interessante; Köln auch." + }, + { + "id": "cyrillic-greeting", + "input_ids": [ + 140, + 253, + 21169, + 18849, + 38857, + 16843, + 20375, + 11, + 12466, + 120, + 18849, + 21169, + 0, + 12466, + 255, + 20375, + 15166, + 220, + 20375, + 16843, + 21727, + 20375, + 220, + 20375, + 25443, + 118, + 16843, + 22177, + 18849, + 140, + 115, + 16142, + 20375, + 15166, + 21169, + 16142, + 13 + ], + "tokens": [ + "Ð", + "Ł", + "ÑĢ", + "и", + "в", + "е", + "ÑĤ", + ",", + "ĠÐ", + "¼", + "и", + "ÑĢ", + "!", + "ĠÐ", + "Ń", + "ÑĤ", + "о", + "Ġ", + "ÑĤ", + "е", + "Ñģ", + "ÑĤ", + "Ġ", + "ÑĤ", + "оÐ", + "º", + "е", + "н", + "и", + "Ð", + "·", + "а", + "ÑĤ", + "о", + "ÑĢ", + "а", + "." + ], + "decoded_with_special": "Привет, мир! Это тест токенизатора.", + "decoded_skip_special": "Привет, мир! Это тест токенизатора." + }, + { + "id": "cyrillic-prose", + "input_ids": [ + 140, + 250, + 15166, + 21727, + 31583, + 38857, + 16142, + 851, + 220, + 21727, + 20375, + 25443, + 119, + 18849, + 141, + 228, + 16142, + 12466, + 254, + 15166, + 21727, + 21727, + 18849, + 18849, + 13, + 12466, + 251, + 16142, + 21727, + 16843, + 30143, + 16843, + 22177, + 18849, + 16843, + 12466, + 109, + 25443, + 119, + 16843, + 16843, + 1105, + 12466, + 120, + 18849, + 30143, + 30143, + 18849, + 15166, + 22177, + 25443, + 110, + 13 + ], + "tokens": [ + "Ð", + "ľ", + "о", + "Ñģ", + "к", + "в", + "а", + "ĠâĢĶ", + "Ġ", + "Ñģ", + "ÑĤ", + "оÐ", + "»", + "и", + "Ñ", + "Ĩ", + "а", + "ĠÐ", + "ł", + "о", + "Ñģ", + "Ñģ", + "и", + "и", + ".", + "ĠÐ", + "Ŀ", + "а", + "Ñģ", + "е", + "л", + "е", + "н", + "и", + "е", + "ĠÐ", + "±", + "оÐ", + "»", + "е", + "е", + "Ġ12", + "ĠÐ", + "¼", + "и", + "л", + "л", + "и", + "о", + "н", + "оÐ", + "²", + "." + ], + "decoded_with_special": "Москва — столица России. Население более 12 миллионов.", + "decoded_skip_special": "Москва — столица России. Население более 12 миллионов." + }, + { + "id": "greek-greeting", + "input_ids": [ + 138, + 248, + 17394, + 39377, + 138, + 115, + 34703, + 138, + 255, + 33643, + 17394, + 7377, + 118, + 139, + 234, + 38392, + 34703, + 30950, + 0, + 7377, + 245, + 7377, + 111, + 39377, + 139, + 236, + 38392, + 38392, + 17394, + 7377, + 113, + 138, + 107, + 26180, + 17394, + 29945, + 18074, + 234, + 34703, + 26517, + 33643, + 139, + 228, + 138, + 115, + 13 + ], + "tokens": [ + "Î", + "ļ", + "α", + "λ", + "Î", + "·", + "μ", + "Î", + "Ń", + "Ïģ", + "α", + "ĠÎ", + "º", + "Ï", + "Į", + "Ïĥ", + "μ", + "ε", + "!", + "ĠÎ", + "Ĺ", + "ĠÎ", + "³", + "λ", + "Ï", + "İ", + "Ïĥ", + "Ïĥ", + "α", + "ĠÎ", + "µ", + "Î", + "¯", + "ν", + "α", + "ι", + "ĠÏ", + "Į", + "μ", + "ο", + "Ïģ", + "Ï", + "Ĩ", + "Î", + "·", + "." + ], + "decoded_with_special": "Καλημέρα κόσμε! Η γλώσσα είναι όμορφη.", + "decoded_skip_special": "Καλημέρα κόσμε! Η γλώσσα είναι όμορφη." + }, + { + "id": "cjk-simplified-greeting", + "input_ids": [ + 19526, + 254, + 25001, + 121, + 11, + 10310, + 244, + 45911, + 234, + 0, + 32573, + 247, + 42468, + 26344, + 228, + 46237, + 235, + 161, + 247, + 101, + 38184, + 233, + 46237, + 243, + 16764 + ], + "tokens": [ + "ä½", + "ł", + "å¥", + "½", + ",", + "ä¸", + "ĸ", + "çķ", + "Į", + "!", + "è¿", + "Ļ", + "æĺ¯", + "åĪ", + "Ĩ", + "è¯", + "į", + "å", + "Ļ", + "¨", + "æµ", + "ĭ", + "è¯", + "ķ", + "ãĢĤ" + ], + "decoded_with_special": "你好,世界!这是分词器测试。", + "decoded_skip_special": "你好,世界!这是分词器测试。" + }, + { + "id": "cjk-simplified-prose", + "input_ids": [ + 40792, + 23877, + 229, + 42468, + 10310, + 244, + 45911, + 234, + 41468, + 45635, + 18796, + 101, + 21689, + 20998, + 96, + 17312, + 222, + 13783, + 248, + 21410, + 46237, + 255, + 164, + 101, + 222, + 45298, + 31660, + 16764 + ], + "tokens": [ + "ä¸Ń", + "æĸ", + "ĩ", + "æĺ¯", + "ä¸", + "ĸ", + "çķ", + "Į", + "ä¸Ĭ", + "使", + "çĶ", + "¨", + "人", + "åı", + "£", + "æľ", + "Ģ", + "å¤", + "ļ", + "çļĦ", + "è¯", + "Ń", + "è", + "¨", + "Ģ", + "ä¹ĭ", + "ä¸Ģ", + "ãĢĤ" + ], + "decoded_with_special": "中文是世界上使用人口最多的语言之一。", + "decoded_skip_special": "中文是世界上使用人口最多的语言之一。" + }, + { + "id": "cjk-traditional-compare", + "input_ids": [ + 163, + 117, + 223, + 165, + 104, + 242, + 40792, + 23877, + 229, + 48958, + 229, + 163, + 108, + 94, + 165, + 104, + 242, + 40792, + 23877, + 229, + 17312, + 231, + 33699, + 222, + 38834, + 28938, + 234, + 16764 + ], + "tokens": [ + "ç", + "¹", + "ģ", + "é", + "«", + "Ķ", + "ä¸Ń", + "æĸ", + "ĩ", + "èĪ", + "ĩ", + "ç", + "°", + "¡", + "é", + "«", + "Ķ", + "ä¸Ń", + "æĸ", + "ĩ", + "æľ", + "ī", + "æī", + "Ģ", + "ä¸į", + "åIJ", + "Į", + "ãĢĤ" + ], + "decoded_with_special": "繁體中文與簡體中文有所不同。", + "decoded_skip_special": "繁體中文與簡體中文有所不同。" + }, + { + "id": "cjk-traditional-hongkong", + "input_ids": [ + 165, + 99, + 247, + 162, + 116, + 107, + 21410, + 163, + 117, + 223, + 165, + 104, + 242, + 40792, + 23877, + 229, + 17312, + 231, + 163, + 235, + 101, + 31965, + 117, + 21410, + 164, + 102, + 252, + 37605, + 247, + 16764 + ], + "tokens": [ + "é", + "¦", + "Ļ", + "æ", + "¸", + "¯", + "çļĦ", + "ç", + "¹", + "ģ", + "é", + "«", + "Ķ", + "ä¸Ń", + "æĸ", + "ĩ", + "æľ", + "ī", + "ç", + "į", + "¨", + "çī", + "¹", + "çļĦ", + "è", + "©", + "ŀ", + "å½", + "Ļ", + "ãĢĤ" + ], + "decoded_with_special": "香港的繁體中文有獨特的詞彙。", + "decoded_skip_special": "香港的繁體中文有獨特的詞彙。" + }, + { + "id": "japanese-voiced-kana-greeting", + "input_ids": [ + 46036, + 22174, + 28618, + 2515, + 94, + 31676, + 23513, + 10310, + 244, + 45911, + 234, + 16764, + 13298, + 42869, + 26229, + 11482, + 48458, + 6312, + 5641, + 24336, + 43302, + 30640, + 33623, + 16764 + ], + "tokens": [ + "ãģĵ", + "ãĤĵ", + "ãģ«", + "ãģ", + "¡", + "ãģ¯", + "ãĢģ", + "ä¸", + "ĸ", + "çķ", + "Į", + "ãĢĤ", + "ãĥĪ", + "ãĥ¼ãĤ¯", + "ãĥĬ", + "ãĤ¤", + "ãĤ¶", + "ãĥ¼", + "ãģ®", + "ãĥĨ", + "ãĤ¹ãĥĪ", + "ãģ§", + "ãģĻ", + "ãĢĤ" + ], + "decoded_with_special": "こんにちは、世界。トークナイザーのテストです。", + "decoded_skip_special": "こんにちは、世界。トークナイザーのテストです。" + }, + { + "id": "japanese-voiced-kana-prose", + "input_ids": [ + 30266, + 109, + 12859, + 105, + 23376, + 25589, + 6312, + 31676, + 30201, + 28134, + 43266, + 164, + 111, + 239, + 1792, + 226, + 27370, + 26945, + 17358, + 111, + 46268, + 28839, + 108, + 30640, + 33623, + 16764 + ], + "tokens": [ + "æĿ", + "±", + "äº", + "¬", + "ãĤ¿", + "ãĥ¯", + "ãĥ¼", + "ãģ¯", + "ãģ¨", + "ãģ¦", + "ãĤĤ", + "è", + "³", + "ij", + "ãĤ", + "Ħ", + "ãģĭ", + "ãģª", + "è¦", + "³", + "åħī", + "åľ", + "°", + "ãģ§", + "ãģĻ", + "ãĢĤ" + ], + "decoded_with_special": "東京タワーはとても賑やかな観光地です。", + "decoded_skip_special": "東京タワーはとても賑やかな観光地です。" + }, + { + "id": "hangul-syllables-greeting", + "input_ids": [ + 168, + 243, + 230, + 167, + 227, + 243, + 47991, + 246, + 168, + 226, + 116, + 168, + 248, + 242, + 11, + 23821, + 226, + 116, + 166, + 111, + 226, + 0, + 220, + 169, + 228, + 254, + 169, + 223, + 105, + 167, + 224, + 246, + 35975, + 112, + 168, + 254, + 222, + 220, + 169, + 227, + 234, + 168, + 232, + 97, + 169, + 232, + 116, + 168, + 252, + 227, + 46695, + 230, + 46695, + 97, + 13 + ], + "tokens": [ + "ì", + "ķ", + "Ī", + "ë", + "ħ", + "ķ", + "íķ", + "ĺ", + "ì", + "Ħ", + "¸", + "ì", + "ļ", + "Ķ", + ",", + "Ġì", + "Ħ", + "¸", + "ê", + "³", + "Ħ", + "!", + "Ġ", + "í", + "Ĩ", + "ł", + "í", + "ģ", + "¬", + "ë", + "Ĥ", + "ĺ", + "ìĿ", + "´", + "ì", + "ł", + "Ģ", + "Ġ", + "í", + "ħ", + "Į", + "ì", + "Ĭ", + "¤", + "í", + "Ĭ", + "¸", + "ì", + "ŀ", + "ħ", + "ëĭ", + "Ī", + "ëĭ", + "¤", + "." + ], + "decoded_with_special": "안녕하세요, 세계! 토크나이저 테스트입니다.", + "decoded_skip_special": "안녕하세요, 세계! 토크나이저 테스트입니다." + }, + { + "id": "hangul-syllables-prose", + "input_ids": [ + 47991, + 250, + 166, + 113, + 255, + 168, + 244, + 112, + 167, + 232, + 242, + 220, + 47991, + 250, + 166, + 116, + 222, + 167, + 94, + 250, + 220, + 169, + 239, + 250, + 166, + 116, + 108, + 47991, + 102, + 46695, + 230, + 46695, + 97, + 13 + ], + "tokens": [ + "íķ", + "ľ", + "ê", + "µ", + "Ń", + "ì", + "ĸ", + "´", + "ë", + "Ĭ", + "Ķ", + "Ġ", + "íķ", + "ľ", + "ê", + "¸", + "Ģ", + "ë", + "¡", + "ľ", + "Ġ", + "í", + "ij", + "ľ", + "ê", + "¸", + "°", + "íķ", + "©", + "ëĭ", + "Ī", + "ëĭ", + "¤", + "." + ], + "decoded_with_special": "한국어는 한글로 표기합니다.", + "decoded_skip_special": "한국어는 한글로 표기합니다." + }, + { + "id": "arabic-greeting", + "input_ids": [ + 25405, + 26897, + 148, + 255, + 39848, + 12919, + 17550, + 101, + 23525, + 44690, + 23525, + 25405, + 0, + 18923, + 229, + 148, + 108, + 12919, + 220, + 34247, + 106, + 41486, + 39848, + 12919, + 26897, + 220, + 13862, + 13862, + 25405, + 148, + 255, + 13862, + 13862, + 28981, + 13862, + 148, + 118, + 30335, + 22654, + 13 + ], + "tokens": [ + "Ùħ", + "ر", + "Ø", + "Ń", + "ب", + "ا", + "ĠØ", + "¨", + "اÙĦ", + "ع", + "اÙĦ", + "Ùħ", + "!", + "ĠÙ", + "ĩ", + "Ø", + "°", + "ا", + "Ġ", + "اØ", + "®", + "ت", + "ب", + "ا", + "ر", + "Ġ", + "ÙĦ", + "ÙĦ", + "Ùħ", + "Ø", + "Ń", + "ÙĦ", + "ÙĦ", + "ĠاÙĦ", + "ÙĦ", + "Ø", + "º", + "ÙĪ", + "ÙĬ", + "." + ], + "decoded_with_special": "مرحبا بالعالم! هذا اختبار للمحلل اللغوي.", + "decoded_skip_special": "مرحبا بالعالم! هذا اختبار للمحلل اللغوي." + }, + { + "id": "arabic-prose", + "input_ids": [ + 23525, + 13862, + 148, + 118, + 45632, + 28981, + 44690, + 26897, + 39848, + 22654, + 45632, + 220, + 13862, + 148, + 118, + 45632, + 17550, + 111, + 12919, + 25405, + 22654, + 45632, + 17550, + 103, + 149, + 225, + 41486, + 39848, + 47048, + 23338, + 28981, + 22654, + 25405, + 22654, + 23338, + 17550, + 98, + 13862, + 149, + 231, + 28981, + 22654, + 45692, + 12919, + 26897, + 13 + ], + "tokens": [ + "اÙĦ", + "ÙĦ", + "Ø", + "º", + "Ø©", + "ĠاÙĦ", + "ع", + "ر", + "ب", + "ÙĬ", + "Ø©", + "Ġ", + "ÙĦ", + "Ø", + "º", + "Ø©", + "ĠØ", + "³", + "ا", + "Ùħ", + "ÙĬ", + "Ø©", + "ĠØ", + "ª", + "Ù", + "ĥ", + "ت", + "ب", + "ĠÙħ", + "ÙĨ", + "ĠاÙĦ", + "ÙĬ", + "Ùħ", + "ÙĬ", + "ÙĨ", + "ĠØ", + "¥", + "ÙĦ", + "Ù", + "ī", + "ĠاÙĦ", + "ÙĬ", + "س", + "ا", + "ر", + "." + ], + "decoded_with_special": "اللغة العربية لغة سامية تكتب من اليمين إلى اليسار.", + "decoded_skip_special": "اللغة العربية لغة سامية تكتب من اليمين إلى اليسار." + }, + { + "id": "hebrew-greeting", + "input_ids": [ + 50227, + 40010, + 27072, + 147, + 251, + 14360, + 95, + 27072, + 40010, + 147, + 251, + 0, + 14360, + 244, + 38269, + 14360, + 252, + 49603, + 147, + 245, + 147, + 253, + 14360, + 102, + 40010, + 14360, + 242, + 147, + 246, + 27072, + 147, + 100, + 147, + 254, + 25529, + 33951, + 244, + 37778, + 13 + ], + "tokens": [ + "ש", + "׾", + "×ķ", + "×", + "Ŀ", + "Ġ×", + "¢", + "×ķ", + "׾", + "×", + "Ŀ", + "!", + "Ġ×", + "ĸ", + "×Ķ", + "Ġ×", + "ŀ", + "×ij", + "×", + "Ĺ", + "×", + "Ł", + "Ġ×", + "©", + "׾", + "Ġ×", + "Ķ", + "×", + "ĺ", + "×ķ", + "×", + "§", + "×", + "ł", + "×Ļ", + "×Ļ×", + "ĸ", + "ר", + "." + ], + "decoded_with_special": "שלום עולם! זה מבחן של הטוקנייזר.", + "decoded_skip_special": "שלום עולם! זה מבחן של הטוקנייזר." + }, + { + "id": "devanagari-hindi", + "input_ids": [ + 11976, + 101, + 11976, + 106, + 11976, + 116, + 24231, + 235, + 11976, + 97, + 24231, + 229, + 28225, + 99, + 24231, + 223, + 11976, + 101, + 11976, + 123, + 11976, + 107, + 48077, + 0, + 28225, + 107, + 11976, + 117, + 28225, + 253, + 24231, + 233, + 11976, + 243, + 11976, + 101, + 48077, + 11976, + 229, + 11976, + 250, + 11976, + 120, + 11976, + 108, + 28225, + 103, + 11976, + 108, + 24231, + 222, + 11976, + 243, + 24231, + 235, + 11976, + 115, + 11976, + 96, + 28225, + 117, + 24231, + 230, + 24231, + 97 + ], + "tokens": [ + "à¤", + "¨", + "à¤", + "®", + "à¤", + "¸", + "à¥", + "į", + "à¤", + "¤", + "à¥", + "ĩ", + "Ġà¤", + "¦", + "à¥", + "ģ", + "à¤", + "¨", + "à¤", + "¿", + "à¤", + "¯", + "ा", + "!", + "Ġà¤", + "¯", + "à¤", + "¹", + "Ġà¤", + "Ł", + "à¥", + "ĭ", + "à¤", + "ķ", + "à¤", + "¨", + "ा", + "à¤", + "ĩ", + "à¤", + "ľ", + "à¤", + "¼", + "à¤", + "°", + "Ġà¤", + "ª", + "à¤", + "°", + "à¥", + "Ģ", + "à¤", + "ķ", + "à¥", + "į", + "à¤", + "·", + "à¤", + "£", + "Ġà¤", + "¹", + "à¥", + "Ī", + "à¥", + "¤" + ], + "decoded_with_special": "नमस्ते दुनिया! यह टोकनाइज़र परीक्षण है।", + "decoded_skip_special": "नमस्ते दुनिया! यह टोकनाइज़र परीक्षण है।" + }, + { + "id": "devanagari-sanskrit", + "input_ids": [ + 11976, + 116, + 11976, + 224, + 11976, + 116, + 24231, + 235, + 11976, + 243, + 24231, + 225, + 11976, + 97, + 28225, + 255, + 48077, + 11976, + 108, + 11976, + 97, + 28225, + 243, + 24231, + 222, + 28225, + 237, + 11976, + 243, + 28225, + 103, + 24231, + 235, + 11976, + 108, + 48077, + 11976, + 248, + 24231, + 222, + 11976, + 101, + 28225, + 255, + 48077, + 11976, + 115, + 48077, + 28225, + 117, + 24231, + 230, + 24231, + 97 + ], + "tokens": [ + "à¤", + "¸", + "à¤", + "Ĥ", + "à¤", + "¸", + "à¥", + "į", + "à¤", + "ķ", + "à¥", + "ĥ", + "à¤", + "¤", + "Ġà¤", + "Ń", + "ा", + "à¤", + "°", + "à¤", + "¤", + "Ġà¤", + "ķ", + "à¥", + "Ģ", + "Ġà¤", + "ı", + "à¤", + "ķ", + "Ġà¤", + "ª", + "à¥", + "į", + "à¤", + "°", + "ा", + "à¤", + "ļ", + "à¥", + "Ģ", + "à¤", + "¨", + "Ġà¤", + "Ń", + "ा", + "à¤", + "·", + "ा", + "Ġà¤", + "¹", + "à¥", + "Ī", + "à¥", + "¤" + ], + "decoded_with_special": "संस्कृत भारत की एक प्राचीन भाषा है।", + "decoded_skip_special": "संस्कृत भारत की एक प्राचीन भाषा है।" + }, + { + "id": "thai-combining-marks-greeting", + "input_ids": [ + 19567, + 103, + 19567, + 100, + 19567, + 109, + 19567, + 103, + 19567, + 242, + 19567, + 113, + 19567, + 232, + 19567, + 110, + 19567, + 100, + 31479, + 224, + 19567, + 98, + 19567, + 223, + 0, + 220, + 19567, + 247, + 19567, + 113, + 31479, + 230, + 19567, + 226, + 19567, + 115, + 19567, + 255, + 19567, + 223, + 19567, + 110, + 19567, + 96, + 19567, + 245, + 19567, + 242, + 19567, + 103, + 19567, + 255, + 19567, + 248, + 31479, + 222, + 19567, + 226, + 19567, + 96, + 19567, + 115, + 31479, + 230, + 19567, + 255, + 19567, + 229, + 19567, + 243, + 19567, + 109, + 19567, + 242, + 19567, + 226, + 19567, + 111 + ], + "tokens": [ + "à¸", + "ª", + "à¸", + "§", + "à¸", + "±", + "à¸", + "ª", + "à¸", + "Ķ", + "à¸", + "µ", + "à¸", + "Ĭ", + "à¸", + "²", + "à¸", + "§", + "à¹", + "Ĥ", + "à¸", + "¥", + "à¸", + "ģ", + "!", + "Ġ", + "à¸", + "Ļ", + "à¸", + "µ", + "à¹", + "Ī", + "à¸", + "Ħ", + "à¸", + "·", + "à¸", + "Ń", + "à¸", + "ģ", + "à¸", + "²", + "à¸", + "£", + "à¸", + "Ĺ", + "à¸", + "Ķ", + "à¸", + "ª", + "à¸", + "Ń", + "à¸", + "ļ", + "à¹", + "Ģ", + "à¸", + "Ħ", + "à¸", + "£", + "à¸", + "·", + "à¹", + "Ī", + "à¸", + "Ń", + "à¸", + "ĩ", + "à¸", + "ķ", + "à¸", + "±", + "à¸", + "Ķ", + "à¸", + "Ħ", + "à¸", + "³" + ], + "decoded_with_special": "สวัสดีชาวโลก! นี่คือการทดสอบเครื่องตัดคำ", + "decoded_skip_special": "สวัสดีชาวโลก! นี่คือการทดสอบเครื่องตัดคำ" + }, + { + "id": "thai-combining-marks-prose", + "input_ids": [ + 19567, + 254, + 19567, + 110, + 19567, + 102, + 19567, + 110, + 31479, + 226, + 19567, + 245, + 19567, + 95, + 31479, + 226, + 19567, + 94, + 31479, + 230, + 31479, + 225, + 19567, + 232, + 31479, + 231, + 19567, + 232, + 31479, + 230, + 19567, + 255, + 19567, + 229, + 19567, + 100, + 31479, + 230, + 19567, + 110, + 19567, + 229, + 19567, + 96, + 19567, + 108, + 19567, + 104, + 19567, + 100, + 31479, + 230, + 19567, + 110, + 19567, + 229, + 19567, + 226, + 19567, + 111 + ], + "tokens": [ + "à¸", + "ł", + "à¸", + "²", + "à¸", + "©", + "à¸", + "²", + "à¹", + "Ħ", + "à¸", + "Ĺ", + "à¸", + "¢", + "à¹", + "Ħ", + "à¸", + "¡", + "à¹", + "Ī", + "à¹", + "ĥ", + "à¸", + "Ĭ", + "à¹", + "ī", + "à¸", + "Ĭ", + "à¹", + "Ī", + "à¸", + "Ń", + "à¸", + "ĩ", + "à¸", + "§", + "à¹", + "Ī", + "à¸", + "²", + "à¸", + "ĩ", + "à¸", + "£", + "à¸", + "°", + "à¸", + "«", + "à¸", + "§", + "à¹", + "Ī", + "à¸", + "²", + "à¸", + "ĩ", + "à¸", + "Ħ", + "à¸", + "³" + ], + "decoded_with_special": "ภาษาไทยไม่ใช้ช่องว่างระหว่างคำ", + "decoded_skip_special": "ภาษาไทยไม่ใช้ช่องว่างระหว่างคำ" + }, + { + "id": "mixed-script-cjk-latin", + "input_ids": [ + 464, + 10263, + 234, + 245, + 12859, + 105, + 5296, + 287, + 1160, + 2075, + 851, + 491, + 14064, + 82, + 275, + 2013, + 0 + ], + "tokens": [ + "The", + "Ġå", + "Į", + "Ĺ", + "äº", + "¬", + "Ġtrip", + "Ġin", + "Ġ20", + "26", + "ĠâĢĶ", + "Ġtr", + "è", + "s", + "Ġb", + "ien", + "!" + ], + "decoded_with_special": "The 北京 trip in 2026 — très bien!", + "decoded_skip_special": "The 北京 trip in 2026 — très bien!" + }, + { + "id": "mixed-script-katakana-code", + "input_ids": [ + 24679, + 12045, + 231, + 24186, + 36922, + 24440, + 6312, + 25, + 5434, + 5969, + 287, + 4589, + 257, + 18, + 69, + 24, + 66, + 17, + 65, + 13 + ], + "tokens": [ + "ãĤ³", + "ãĥ¼ãĥ", + "ī", + "ãĥ¬", + "ãĥĵ", + "ãĥ¥", + "ãĥ¼", + ":", + "Ġbug", + "Ġfixed", + "Ġin", + "Ġcommit", + "Ġa", + "3", + "f", + "9", + "c", + "2", + "b", + "." + ], + "decoded_with_special": "コードレビュー: bug fixed in commit a3f9c2b.", + "decoded_skip_special": "コードレビュー: bug fixed in commit a3f9c2b." + }, + { + "id": "math-symbols-summation", + "input_ids": [ + 23004, + 25, + 7377, + 96, + 2124, + 62, + 72, + 796, + 5433, + 357, + 3003, + 2124, + 18872, + 230, + 2343, + 226, + 251, + 737 + ], + "tokens": [ + "Result", + ":", + "ĠÎ", + "£", + "Ġx", + "_", + "i", + "Ġ=", + "Ġ42", + "Ġ(", + "where", + "Ġx", + "ĠâĪ", + "Ī", + "Ġâ", + "Ħ", + "Ŀ", + ")." + ], + "decoded_with_special": "Result: Σ x_i = 42 (where x ∈ ℝ).", + "decoded_skip_special": "Result: Σ x_i = 42 (where x ∈ ℝ)." + }, + { + "id": "math-symbols-epsilon-delta", + "input_ids": [ + 464, + 29625, + 25, + 18872, + 222, + 30950, + 29, + 15, + 18872, + 225, + 138, + 112, + 29, + 15, + 884, + 326, + 930, + 87, + 12, + 87, + 158, + 224, + 222, + 91, + 27, + 138, + 112, + 42527, + 240, + 930, + 69, + 7, + 87, + 13219, + 69, + 7, + 87, + 158, + 224, + 222, + 14726, + 27, + 30950, + 13 + ], + "tokens": [ + "The", + "orem", + ":", + "ĠâĪ", + "Ģ", + "ε", + ">", + "0", + "ĠâĪ", + "ĥ", + "Î", + "´", + ">", + "0", + "Ġsuch", + "Ġthat", + "Ġ|", + "x", + "-", + "x", + "â", + "Ĥ", + "Ģ", + "|", + "<", + "Î", + "´", + "Ġâĩ", + "Ĵ", + "Ġ|", + "f", + "(", + "x", + ")-", + "f", + "(", + "x", + "â", + "Ĥ", + "Ģ", + ")|", + "<", + "ε", + "." + ], + "decoded_with_special": "Theorem: ∀ε>0 ∃δ>0 such that |x-x₀|<δ ⇒ |f(x)-f(x₀)|<ε.", + "decoded_skip_special": "Theorem: ∀ε>0 ∃δ>0 such that |x-x₀|<δ ⇒ |f(x)-f(x₀)|<ε." + }, + { + "id": "currency-symbols-multi", + "input_ids": [ + 13729, + 25, + 10432, + 1065, + 13, + 1120, + 15168, + 38221, + 16, + 11, + 41739, + 15139, + 230, + 4248, + 940, + 13, + 2425, + 357, + 1324, + 13907, + 11, + 5556, + 2343, + 224, + 123, + 15, + 13, + 830, + 19, + 737 + ], + "tokens": [ + "Cost", + ":", + "ĠâĤ¬", + "12", + ".", + "50", + "ĠâĨĴ", + "ĠÂ¥", + "1", + ",", + "820", + "Ġâī", + "Ī", + "Ġ£", + "10", + ".", + "75", + "Ġ(", + "app", + "rox", + ",", + "Ġplus", + "Ġâ", + "Ĥ", + "¿", + "0", + ".", + "000", + "4", + ")." + ], + "decoded_with_special": "Cost: €12.50 → ¥1,820 ≈ £10.75 (approx, plus ₿0.0004).", + "decoded_skip_special": "Cost: €12.50 → ¥1,820 ≈ £10.75 (approx, plus ₿0.0004)." + }, + { + "id": "box-drawing", + "input_ids": [ + 14253, + 25, + 13305, + 234, + 16068, + 8418, + 7280, + 6552, + 238, + 19421, + 18435, + 19421, + 13305, + 242, + 16068, + 8418, + 7280, + 6552, + 246, + 851, + 1760, + 13 + ], + "tokens": [ + "Box", + ":", + "ĠâĶ", + "Į", + "âĶĢâĶĢâĶĢâĶĢ", + "âĶĢâĶĢ", + "âĶĢ", + "âĶ", + "IJ", + "ĠâĶĤ", + "ĠHello", + "ĠâĶĤ", + "ĠâĶ", + "Ķ", + "âĶĢâĶĢâĶĢâĶĢ", + "âĶĢâĶĢ", + "âĶĢ", + "âĶ", + "ĺ", + "ĠâĢĶ", + "Ġdone", + "." + ], + "decoded_with_special": "Box: ┌───────┐ │ Hello │ └───────┘ — done.", + "decoded_skip_special": "Box: ┌───────┐ │ Hello │ └───────┘ — done." + }, + { + "id": "dingbats-chess", + "input_ids": [ + 48223, + 728, + 25, + 23883, + 34719, + 228, + 20724, + 254, + 20724, + 96, + 20724, + 98, + 20724, + 99, + 20724, + 242, + 20724, + 243, + 20724, + 244, + 20724, + 246, + 20724, + 247, + 851, + 19780, + 900, + 13 + ], + "tokens": [ + "Pie", + "ces", + ":", + "Ġâĺħ", + "Ġâĺ", + "Ĩ", + "ĠâĻ", + "ł", + "ĠâĻ", + "£", + "ĠâĻ", + "¥", + "ĠâĻ", + "¦", + "ĠâĻ", + "Ķ", + "ĠâĻ", + "ķ", + "ĠâĻ", + "ĸ", + "ĠâĻ", + "ĺ", + "ĠâĻ", + "Ļ", + "ĠâĢĶ", + "Ġchess", + "Ġset", + "." + ], + "decoded_with_special": "Pieces: ★ ☆ ♠ ♣ ♥ ♦ ♔ ♕ ♖ ♘ ♙ — chess set.", + "decoded_skip_special": "Pieces: ★ ☆ ♠ ♣ ♥ ♦ ♔ ♕ ♖ ♘ ♙ — chess set." + }, + { + "id": "math-astral-bold-script", + "input_ids": [ + 37372, + 10758, + 25, + 220, + 47728, + 238, + 222, + 47728, + 238, + 223, + 47728, + 238, + 224, + 47728, + 238, + 225, + 220, + 47728, + 253, + 236, + 47728, + 253, + 237, + 47728, + 253, + 238, + 47728, + 253, + 239, + 851, + 10688, + 4226, + 25, + 220, + 47728, + 241, + 238, + 47728, + 241, + 239, + 47728, + 241, + 240, + 47728, + 241, + 241, + 13 + ], + "tokens": [ + "Math", + "Ġbold", + ":", + "Ġ", + "ðĿ", + "IJ", + "Ģ", + "ðĿ", + "IJ", + "ģ", + "ðĿ", + "IJ", + "Ĥ", + "ðĿ", + "IJ", + "ĥ", + "Ġ", + "ðĿ", + "Ł", + "İ", + "ðĿ", + "Ł", + "ı", + "ðĿ", + "Ł", + "IJ", + "ðĿ", + "Ł", + "ij", + "ĠâĢĶ", + "Ġmath", + "Ġscript", + ":", + "Ġ", + "ðĿ", + "ĵ", + "IJ", + "ðĿ", + "ĵ", + "ij", + "ðĿ", + "ĵ", + "Ĵ", + "ðĿ", + "ĵ", + "ĵ", + "." + ], + "decoded_with_special": "Math bold: 𝐀𝐁𝐂𝐃 𝟎𝟏𝟐𝟑 — math script: 𝓐𝓑𝓒𝓓.", + "decoded_skip_special": "Math bold: 𝐀𝐁𝐂𝐃 𝟎𝟏𝟐𝟑 — math script: 𝓐𝓑𝓒𝓓." + }, + { + "id": "astral-egyptian-hieroglyphs", + "input_ids": [ + 39299, + 666, + 13550, + 34619, + 746, + 82, + 25, + 220, + 172, + 241, + 222, + 222, + 220, + 172, + 241, + 224, + 222, + 220, + 172, + 241, + 225, + 222, + 220, + 172, + 241, + 228, + 236, + 172, + 241, + 227, + 241, + 172, + 241, + 237, + 237, + 172, + 241, + 232, + 244, + 13 + ], + "tokens": [ + "Egypt", + "ian", + "Ġhier", + "ogly", + "ph", + "s", + ":", + "Ġ", + "ð", + "ĵ", + "Ģ", + "Ģ", + "Ġ", + "ð", + "ĵ", + "Ĥ", + "Ģ", + "Ġ", + "ð", + "ĵ", + "ĥ", + "Ģ", + "Ġ", + "ð", + "ĵ", + "Ĩ", + "İ", + "ð", + "ĵ", + "ħ", + "ĵ", + "ð", + "ĵ", + "ı", + "ı", + "ð", + "ĵ", + "Ĭ", + "ĸ", + "." + ], + "decoded_with_special": "Egyptian hieroglyphs: 𓀀 𓂀 𓃀 𓆎𓅓𓏏𓊖.", + "decoded_skip_special": "Egyptian hieroglyphs: 𓀀 𓂀 𓃀 𓆎𓅓𓏏𓊖." + }, + { + "id": "astral-cuneiform", + "input_ids": [ + 34, + 1726, + 6933, + 25, + 220, + 172, + 240, + 222, + 255, + 220, + 172, + 240, + 230, + 254, + 220, + 172, + 240, + 224, + 245, + 220, + 172, + 240, + 228, + 254, + 851, + 1051, + 1351, + 8405, + 13 + ], + "tokens": [ + "C", + "une", + "iform", + ":", + "Ġ", + "ð", + "Ĵ", + "Ģ", + "Ń", + "Ġ", + "ð", + "Ĵ", + "Ī", + "ł", + "Ġ", + "ð", + "Ĵ", + "Ĥ", + "Ĺ", + "Ġ", + "ð", + "Ĵ", + "Ĩ", + "ł", + "ĠâĢĶ", + "Ġsign", + "Ġlist", + "Ġsamples", + "." + ], + "decoded_with_special": "Cuneiform: 𒀭 𒈠 𒂗 𒆠 — sign list samples.", + "decoded_skip_special": "Cuneiform: 𒀭 𒈠 𒂗 𒆠 — sign list samples." + }, + { + "id": "astral-mahjong", + "input_ids": [ + 40936, + 32428, + 19867, + 25, + 12520, + 222, + 222, + 12520, + 222, + 223, + 12520, + 222, + 224, + 12520, + 222, + 225, + 12520, + 222, + 226, + 12520, + 222, + 227, + 12520, + 222, + 228, + 12520, + 222, + 229, + 13 + ], + "tokens": [ + "Mah", + "jong", + "Ġtiles", + ":", + "ĠðŁ", + "Ģ", + "Ģ", + "ĠðŁ", + "Ģ", + "ģ", + "ĠðŁ", + "Ģ", + "Ĥ", + "ĠðŁ", + "Ģ", + "ĥ", + "ĠðŁ", + "Ģ", + "Ħ", + "ĠðŁ", + "Ģ", + "ħ", + "ĠðŁ", + "Ģ", + "Ĩ", + "ĠðŁ", + "Ģ", + "ĩ", + "." + ], + "decoded_with_special": "Mahjong tiles: 🀀 🀁 🀂 🀃 🀄 🀅 🀆 🀇.", + "decoded_skip_special": "Mahjong tiles: 🀀 🀁 🀂 🀃 🀄 🀅 🀆 🀇." + }, + { + "id": "astral-cards", + "input_ids": [ + 34, + 1371, + 25, + 12520, + 225, + 223, + 12520, + 225, + 224, + 12520, + 225, + 225, + 12520, + 224, + 94, + 12520, + 224, + 95, + 12520, + 224, + 96, + 13 + ], + "tokens": [ + "C", + "ards", + ":", + "ĠðŁ", + "ĥ", + "ģ", + "ĠðŁ", + "ĥ", + "Ĥ", + "ĠðŁ", + "ĥ", + "ĥ", + "ĠðŁ", + "Ĥ", + "¡", + "ĠðŁ", + "Ĥ", + "¢", + "ĠðŁ", + "Ĥ", + "£", + "." + ], + "decoded_with_special": "Cards: 🃁 🃂 🃃 🂡 🂢 🂣.", + "decoded_skip_special": "Cards: 🃁 🃂 🃃 🂡 🂢 🂣." + }, + { + "id": "emoji-bmp-and-astral", + "input_ids": [ + 41840, + 233, + 18435, + 0, + 12520, + 234, + 235, + 2159, + 12520, + 248, + 222, + 4219, + 12520, + 236, + 231, + 2151, + 12520, + 236, + 224, + 13 + ], + "tokens": [ + "ðŁij", + "ĭ", + "ĠHello", + "!", + "ĠðŁ", + "Į", + "į", + "ĠWorld", + "ĠðŁ", + "ļ", + "Ģ", + "Ġlaunch", + "ĠðŁ", + "İ", + "ī", + "Ġparty", + "ĠðŁ", + "İ", + "Ĥ", + "." + ], + "decoded_with_special": "👋 Hello! 🌍 World 🚀 launch 🎉 party 🎂.", + "decoded_skip_special": "👋 Hello! 🌍 World 🚀 launch 🎉 party 🎂." + }, + { + "id": "emoji-zwj-family-pride-skin", + "input_ids": [ + 24094, + 25, + 50169, + 101, + 447, + 235, + 41840, + 102, + 447, + 235, + 41840, + 100, + 447, + 235, + 41840, + 99, + 851, + 21735, + 6056, + 25, + 12520, + 237, + 111, + 37929, + 447, + 235, + 8582, + 234, + 230, + 851, + 17847, + 12, + 41527, + 25, + 50169, + 235, + 8582, + 237, + 121, + 14519, + 232, + 8582, + 237, + 123, + 50169, + 114, + 8582, + 237, + 119, + 13 + ], + "tokens": [ + "Family", + ":", + "ĠðŁij", + "¨", + "âĢ", + "į", + "ðŁij", + "©", + "âĢ", + "į", + "ðŁij", + "§", + "âĢ", + "į", + "ðŁij", + "¦", + "ĠâĢĶ", + "ĠPride", + "Ġflag", + ":", + "ĠðŁ", + "ı", + "³", + "ï¸ı", + "âĢ", + "į", + "ðŁ", + "Į", + "Ī", + "ĠâĢĶ", + "ĠSkin", + "-", + "tone", + ":", + "ĠðŁij", + "į", + "ðŁ", + "ı", + "½", + "Ġâľ", + "Ĭ", + "ðŁ", + "ı", + "¿", + "ĠðŁij", + "¶", + "ðŁ", + "ı", + "»", + "." + ], + "decoded_with_special": "Family: 👨‍👩‍👧‍👦 — Pride flag: 🏳️‍🌈 — Skin-tone: 👍🏽 ✊🏿 👶🏻.", + "decoded_skip_special": "Family: 👨‍👩‍👧‍👦 — Pride flag: 🏳️‍🌈 — Skin-tone: 👍🏽 ✊🏿 👶🏻." + }, + { + "id": "emoji-zwj-with-text-prefix", + "input_ids": [ + 8582, + 237, + 111, + 37929, + 447, + 235, + 8582, + 234, + 230, + 6836, + 485, + 10, + 41840, + 101, + 447, + 235, + 41840, + 102, + 447, + 235, + 41840, + 100, + 447, + 235, + 41840, + 99, + 24094, + 10, + 8582, + 229, + 102, + 8582, + 229, + 103, + 8582, + 229, + 107, + 8582, + 229, + 113, + 8582, + 229, + 118, + 8582, + 229, + 116, + 33152, + 10, + 9288 + ], + "tokens": [ + "ðŁ", + "ı", + "³", + "ï¸ı", + "âĢ", + "į", + "ðŁ", + "Į", + "Ī", + "Pr", + "ide", + "+", + "ðŁij", + "¨", + "âĢ", + "į", + "ðŁij", + "©", + "âĢ", + "į", + "ðŁij", + "§", + "âĢ", + "į", + "ðŁij", + "¦", + "Family", + "+", + "ðŁ", + "ĩ", + "©", + "ðŁ", + "ĩ", + "ª", + "ðŁ", + "ĩ", + "¯", + "ðŁ", + "ĩ", + "µ", + "ðŁ", + "ĩ", + "º", + "ðŁ", + "ĩ", + "¸", + "flags", + "+", + "test" + ], + "decoded_with_special": "🏳️‍🌈Pride+👨‍👩‍👧‍👦Family+🇩🇪🇯🇵🇺🇸flags+test", + "decoded_skip_special": "🏳️‍🌈Pride+👨‍👩‍👧‍👦Family+🇩🇪🇯🇵🇺🇸flags+test" + }, + { + "id": "emoji-keycap-and-flags", + "input_ids": [ + 9218, + 27979, + 25, + 352, + 37929, + 158, + 225, + 96, + 362, + 37929, + 158, + 225, + 96, + 513, + 37929, + 158, + 225, + 96, + 851, + 6056, + 25, + 12520, + 229, + 107, + 8582, + 229, + 113, + 12520, + 229, + 102, + 8582, + 229, + 103, + 12520, + 229, + 118, + 8582, + 229, + 116, + 13 + ], + "tokens": [ + "Key", + "caps", + ":", + "Ġ1", + "ï¸ı", + "â", + "ĥ", + "£", + "Ġ2", + "ï¸ı", + "â", + "ĥ", + "£", + "Ġ3", + "ï¸ı", + "â", + "ĥ", + "£", + "ĠâĢĶ", + "Ġflag", + ":", + "ĠðŁ", + "ĩ", + "¯", + "ðŁ", + "ĩ", + "µ", + "ĠðŁ", + "ĩ", + "©", + "ðŁ", + "ĩ", + "ª", + "ĠðŁ", + "ĩ", + "º", + "ðŁ", + "ĩ", + "¸", + "." + ], + "decoded_with_special": "Keycaps: 1️⃣ 2️⃣ 3️⃣ — flag: 🇯🇵 🇩🇪 🇺🇸.", + "decoded_skip_special": "Keycaps: 1️⃣ 2️⃣ 3️⃣ — flag: 🇯🇵 🇩🇪 🇺🇸." + }, + { + "id": "url-with-query-fragment", + "input_ids": [ + 6371, + 28, + 5450, + 1378, + 20688, + 13, + 785, + 14, + 6978, + 14, + 1462, + 14, + 31092, + 30, + 80, + 28, + 21943, + 5, + 5657, + 28, + 65, + 1031, + 5, + 77, + 28, + 3682, + 2, + 5458, + 12, + 18 + ], + "tokens": [ + "url", + "=", + "https", + "://", + "example", + ".", + "com", + "/", + "path", + "/", + "to", + "/", + "resource", + "?", + "q", + "=", + "foo", + "&", + "bar", + "=", + "b", + "az", + "&", + "n", + "=", + "42", + "#", + "section", + "-", + "3" + ], + "decoded_with_special": "url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3", + "decoded_skip_special": "url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3" + }, + { + "id": "hex-and-base64", + "input_ids": [ + 33095, + 28, + 15, + 87, + 7206, + 2885, + 33, + 6500, + 4851, + 8579, + 36, + 4339, + 12473, + 2779, + 2414, + 28, + 56, + 54, + 41, + 73, + 57, + 37094, + 76, + 57, + 17, + 24831, + 321, + 912, + 65, + 54, + 20, + 28435, + 32, + 855 + ], + "tokens": [ + "hex", + "=", + "0", + "x", + "DE", + "AD", + "B", + "EE", + "FC", + "AF", + "E", + "BA", + "BE", + "Ġbase", + "64", + "=", + "Y", + "W", + "J", + "j", + "Z", + "GV", + "m", + "Z", + "2", + "hp", + "am", + "ts", + "b", + "W", + "5", + "vc", + "A", + "==" + ], + "decoded_with_special": "hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==", + "decoded_skip_special": "hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==" + }, + { + "id": "sha256-hash", + "input_ids": [ + 26270, + 11645, + 28, + 68, + 18, + 65, + 15, + 66, + 2598, + 27728, + 16072, + 16, + 66, + 19442, + 1878, + 19881, + 19, + 66, + 23, + 38565, + 21855, + 24, + 1731, + 1983, + 3609, + 3901, + 68, + 19, + 33300, + 65, + 24, + 2682, + 6888, + 33781, + 2079, + 16, + 65, + 3695, + 4309, + 65, + 45432 + ], + "tokens": [ + "sha", + "256", + "=", + "e", + "3", + "b", + "0", + "c", + "44", + "298", + "fc", + "1", + "c", + "149", + "af", + "bf", + "4", + "c", + "8", + "996", + "fb", + "9", + "24", + "27", + "ae", + "41", + "e", + "4", + "649", + "b", + "9", + "34", + "ca", + "495", + "99", + "1", + "b", + "78", + "52", + "b", + "855" + ], + "decoded_with_special": "sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "decoded_skip_special": "sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "id": "code-javascript-functional", + "input_ids": [ + 3258, + 13, + 8899, + 7, + 87, + 5218, + 2124, + 1635, + 362, + 737, + 24455, + 7, + 87, + 5218, + 2124, + 1875, + 657, + 737, + 445, + 7234, + 19510, + 64, + 11, + 275, + 8, + 5218, + 257, + 1343, + 275, + 1776 + ], + "tokens": [ + "arr", + ".", + "map", + "(", + "x", + "Ġ=>", + "Ġx", + "Ġ*", + "Ġ2", + ").", + "filter", + "(", + "x", + "Ġ=>", + "Ġx", + "Ġ>", + "Ġ0", + ").", + "red", + "uce", + "((", + "a", + ",", + "Ġb", + ")", + "Ġ=>", + "Ġa", + "Ġ+", + "Ġb", + ");" + ], + "decoded_with_special": "arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);", + "decoded_skip_special": "arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);" + }, + { + "id": "code-cplusplus-template", + "input_ids": [ + 28243, + 27, + 51, + 11, + 52, + 29, + 8295, + 277, + 7, + 51, + 25226, + 83, + 11, + 471, + 25226, + 84, + 8, + 4613, + 2377, + 4906, + 7, + 83, + 10, + 84, + 8, + 1391, + 1441, + 256, + 1343, + 334, + 26, + 1782 + ], + "tokens": [ + "template", + "<", + "T", + ",", + "U", + ">", + "Ġauto", + "Ġf", + "(", + "T", + "&&", + "t", + ",", + "ĠU", + "&&", + "u", + ")", + "Ġ->", + "Ġdecl", + "type", + "(", + "t", + "+", + "u", + ")", + "Ġ{", + "Ġreturn", + "Ġt", + "Ġ+", + "Ġu", + ";", + "Ġ}" + ], + "decoded_with_special": "template auto f(T&&t, U&&u) -> decltype(t+u) { return t + u; }", + "decoded_skip_special": "template auto f(T&&t, U&&u) -> decltype(t+u) { return t + u; }" + }, + { + "id": "code-bitops", + "input_ids": [ + 20274, + 796, + 357, + 64, + 9609, + 362, + 8, + 930, + 357, + 65, + 9959, + 604, + 8, + 1222, + 657, + 87, + 5777, + 26 + ], + "tokens": [ + "result", + "Ġ=", + "Ġ(", + "a", + "Ġ>>", + "Ġ2", + ")", + "Ġ|", + "Ġ(", + "b", + "Ġ<<", + "Ġ4", + ")", + "Ġ&", + "Ġ0", + "x", + "FF", + ";" + ], + "decoded_with_special": "result = (a >> 2) | (b << 4) & 0xFF;", + "decoded_skip_special": "result = (a >> 2) | (b << 4) & 0xFF;" + }, + { + "id": "code-javascript-modern", + "input_ids": [ + 361, + 357, + 87, + 5145, + 855, + 9242, + 11405, + 331, + 18189, + 657, + 8, + 1391, + 1441, + 2124, + 30, + 13, + 21943, + 19153, + 4277, + 11395, + 26, + 1782 + ], + "tokens": [ + "if", + "Ġ(", + "x", + "Ġ!", + "==", + "Ġnull", + "Ġ&&", + "Ġy", + "Ġ>=", + "Ġ0", + ")", + "Ġ{", + "Ġreturn", + "Ġx", + "?", + ".", + "foo", + "Ġ??", + "Ġdefault", + "Value", + ";", + "Ġ}" + ], + "decoded_with_special": "if (x !== null && y >= 0) { return x?.foo ?? defaultValue; }", + "decoded_skip_special": "if (x !== null && y >= 0) { return x?.foo ?? defaultValue; }" + }, + { + "id": "code-python-def", + "input_ids": [ + 4299, + 12900, + 261, + 44456, + 7, + 77, + 2599 + ], + "tokens": [ + "def", + "Ġfib", + "on", + "acci", + "(", + "n", + "):" + ], + "decoded_with_special": "def fibonacci(n):", + "decoded_skip_special": "def fibonacci(n):" + }, + { + "id": "code-python-if", + "input_ids": [ + 220, + 220, + 220, + 611, + 299, + 1279, + 362, + 25 + ], + "tokens": [ + "Ġ", + "Ġ", + "Ġ", + "Ġif", + "Ġn", + "Ġ<", + "Ġ2", + ":" + ], + "decoded_with_special": " if n < 2:", + "decoded_skip_special": " if n < 2:" + }, + { + "id": "code-python-return", + "input_ids": [ + 220, + 220, + 220, + 220, + 220, + 220, + 220, + 1441, + 299 + ], + "tokens": [ + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġreturn", + "Ġn" + ], + "decoded_with_special": " return n", + "decoded_skip_special": " return n" + }, + { + "id": "code-python-recurse", + "input_ids": [ + 220, + 220, + 220, + 1441, + 12900, + 261, + 44456, + 7, + 77, + 532, + 352, + 8, + 1343, + 12900, + 261, + 44456, + 7, + 77, + 532, + 362, + 8 + ], + "tokens": [ + "Ġ", + "Ġ", + "Ġ", + "Ġreturn", + "Ġfib", + "on", + "acci", + "(", + "n", + "Ġ-", + "Ġ1", + ")", + "Ġ+", + "Ġfib", + "on", + "acci", + "(", + "n", + "Ġ-", + "Ġ2", + ")" + ], + "decoded_with_special": " return fibonacci(n - 1) + fibonacci(n - 2)", + "decoded_skip_special": " return fibonacci(n - 1) + fibonacci(n - 2)" + }, + { + "id": "code-go-func", + "input_ids": [ + 20786, + 12589, + 7, + 3672, + 4731, + 8, + 1391 + ], + "tokens": [ + "func", + "Ġgreet", + "(", + "name", + "Ġstring", + ")", + "Ġ{" + ], + "decoded_with_special": "func greet(name string) {", + "decoded_skip_special": "func greet(name string) {" + }, + { + "id": "code-go-tab-if", + "input_ids": [ + 197, + 361, + 1438, + 14512, + 13538, + 1391 + ], + "tokens": [ + "ĉ", + "if", + "Ġname", + "Ġ!=", + "Ġ\"\"", + "Ġ{" + ], + "decoded_with_special": "\tif name != \"\" {", + "decoded_skip_special": "\tif name != \"\" {" + }, + { + "id": "code-go-tab-printf", + "input_ids": [ + 197, + 197, + 69, + 16762, + 13, + 18557, + 69, + 7203, + 15496, + 11, + 4064, + 82, + 0, + 59, + 77, + 1600, + 1438, + 8 + ], + "tokens": [ + "ĉ", + "ĉ", + "f", + "mt", + ".", + "Print", + "f", + "(\"", + "Hello", + ",", + "Ġ%", + "s", + "!", + "\\", + "n", + "\",", + "Ġname", + ")" + ], + "decoded_with_special": "\t\tfmt.Printf(\"Hello, %s!\\n\", name)", + "decoded_skip_special": "\t\tfmt.Printf(\"Hello, %s!\\n\", name)" + }, + { + "id": "code-go-tab-close", + "input_ids": [ + 197, + 92 + ], + "tokens": [ + "ĉ", + "}" + ], + "decoded_with_special": "\t}", + "decoded_skip_special": "\t}" + }, + { + "id": "code-go-close", + "input_ids": [ + 92 + ], + "tokens": [ + "}" + ], + "decoded_with_special": "}", + "decoded_skip_special": "}" + }, + { + "id": "programming-identifiers-mixed", + "input_ids": [ + 1136, + 62, + 7220, + 62, + 312, + 25882, + 62, + 19499, + 45746, + 62, + 33489, + 309, + 62, + 15636, + 62, + 30001, + 62, + 312, + 10896, + 4261, + 6561, + 2521, + 38149 + ], + "tokens": [ + "get", + "_", + "user", + "_", + "id", + "ĠMAX", + "_", + "BU", + "FFER", + "_", + "SIZE", + "ĠT", + "_", + "pad", + "_", + "token", + "_", + "id", + "ĠNS", + "UR", + "LS", + "ession", + "Configuration" + ], + "decoded_with_special": "get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration", + "decoded_skip_special": "get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration" + }, + { + "id": "programming-identifiers-cases", + "input_ids": [ + 66, + 17983, + 20448, + 19852, + 35163, + 20448, + 17522, + 62, + 7442, + 62, + 7785, + 6374, + 32235, + 2751, + 62, + 50, + 4535, + 7336, + 479, + 23, + 82, + 62, + 33320, + 62, + 9127, + 62, + 85, + 17 + ], + "tokens": [ + "c", + "amel", + "Case", + "Var", + "ĠPascal", + "Case", + "Ġsnake", + "_", + "case", + "_", + "var", + "ĠSC", + "REAM", + "ING", + "_", + "S", + "NA", + "KE", + "Ġk", + "8", + "s", + "_", + "pod", + "_", + "count", + "_", + "v", + "2" + ], + "decoded_with_special": "camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2", + "decoded_skip_special": "camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2" + }, + { + "id": "code-string-multiscript", + "input_ids": [ + 70, + 2871, + 278, + 796, + 366, + 140, + 253, + 21169, + 18849, + 38857, + 16843, + 20375, + 11, + 12466, + 120, + 18849, + 21169, + 2474, + 1343, + 366, + 851, + 422, + 366, + 1343, + 366, + 30266, + 109, + 12859, + 105, + 1 + ], + "tokens": [ + "g", + "reet", + "ing", + "Ġ=", + "Ġ\"", + "Ð", + "Ł", + "ÑĢ", + "и", + "в", + "е", + "ÑĤ", + ",", + "ĠÐ", + "¼", + "и", + "ÑĢ", + "!\"", + "Ġ+", + "Ġ\"", + "ĠâĢĶ", + "Ġfrom", + "Ġ\"", + "Ġ+", + "Ġ\"", + "æĿ", + "±", + "äº", + "¬", + "\"" + ], + "decoded_with_special": "greeting = \"Привет, мир!\" + \" — from \" + \"東京\"", + "decoded_skip_special": "greeting = \"Привет, мир!\" + \" — from \" + \"東京\"" + }, + { + "id": "code-comment-diacritics", + "input_ids": [ + 1003, + 40560, + 16345, + 2634, + 851, + 15608, + 2912, + 351, + 2566, + 330, + 799, + 873 + ], + "tokens": [ + "//", + "Ġré", + "sum", + "é", + "ĠâĢĶ", + "ĠSwift", + "Ġcomment", + "Ġwith", + "Ġdi", + "ac", + "rit", + "ics" + ], + "decoded_with_special": "// résumé — Swift comment with diacritics", + "decoded_skip_special": "// résumé — Swift comment with diacritics" + }, + { + "id": "code-comment-greek-math", + "input_ids": [ + 15211, + 26367, + 1343, + 27169, + 796, + 7377, + 111, + 851, + 8312, + 12, + 9291, + 10688, + 2912, + 9466 + ], + "tokens": [ + "/*", + "Ġα", + "Ġ+", + "Ġβ", + "Ġ=", + "ĠÎ", + "³", + "ĠâĢĶ", + "ĠGreek", + "-", + "letter", + "Ġmath", + "Ġcomment", + "Ġ*/" + ], + "decoded_with_special": "/* α + β = γ — Greek-letter math comment */", + "decoded_skip_special": "/* α + β = γ — Greek-letter math comment */" + }, + { + "id": "code-string-multiscript-emoji", + "input_ids": [ + 1616, + 3670, + 796, + 366, + 8582, + 248, + 222, + 21225, + 851, + 12466, + 123, + 35072, + 21727, + 31583, + 851, + 5525, + 113, + 115, + 47947, + 243, + 1 + ], + "tokens": [ + "let", + "Ġtitle", + "Ġ=", + "Ġ\"", + "ðŁ", + "ļ", + "Ģ", + "ĠLaunch", + "ĠâĢĶ", + "ĠÐ", + "¿", + "Ñĥ", + "Ñģ", + "к", + "ĠâĢĶ", + "Ġè", + "µ", + "·", + "åĭ", + "ķ", + "\"" + ], + "decoded_with_special": "let title = \"🚀 Launch — пуск — 起動\"", + "decoded_skip_special": "let title = \"🚀 Launch — пуск — 起動\"" + }, + { + "id": "ipa-phonetic-transcription", + "input_ids": [ + 464, + 27966, + 26955, + 1220, + 69, + 133, + 103, + 134, + 225, + 14, + 329, + 366, + 11084, + 1, + 851, + 351, + 5503, + 25, + 220, + 45990, + 69, + 133, + 103, + 134, + 225, + 13 + ], + "tokens": [ + "The", + "ĠIPA", + "Ġtranscription", + "Ġ/", + "f", + "É", + "ª", + "Ê", + "ĥ", + "/", + "Ġfor", + "Ġ\"", + "fish", + "\"", + "ĠâĢĶ", + "Ġwith", + "Ġstress", + ":", + "Ġ", + "ËĪ", + "f", + "É", + "ª", + "Ê", + "ĥ", + "." + ], + "decoded_with_special": "The IPA transcription /fɪʃ/ for \"fish\" — with stress: ˈfɪʃ.", + "decoded_skip_special": "The IPA transcription /fɪʃ/ for \"fish\" — with stress: ˈfɪʃ." + }, + { + "id": "mandarin-tones", + "input_ids": [ + 51, + 1952, + 25, + 285, + 6557, + 285, + 131, + 236, + 285, + 24247, + 285, + 10235, + 357, + 49846, + 17714, + 23755, + 851, + 1029, + 11, + 7396, + 11, + 43144, + 11, + 7463, + 737 + ], + "tokens": [ + "T", + "ones", + ":", + "Ġm", + "á", + "Ġm", + "Ç", + "İ", + "Ġm", + "Ãł", + "Ġm", + "Äģ", + "Ġ(", + "Mand", + "arin", + "Ġtones", + "ĠâĢĶ", + "Ġhigh", + ",", + "Ġrising", + ",", + "Ġdipping", + ",", + "Ġfalling", + ")." + ], + "decoded_with_special": "Tones: má mǎ mà mā (Mandarin tones — high, rising, dipping, falling).", + "decoded_skip_special": "Tones: má mǎ mà mā (Mandarin tones — high, rising, dipping, falling)." + }, + { + "id": "whitespace-runs", + "input_ids": [ + 31373, + 220, + 220, + 220, + 995, + 220, + 220, + 220, + 220, + 220, + 220, + 220, + 220, + 764, + 220, + 220, + 220, + 220, + 220, + 220, + 886 + ], + "tokens": [ + "hello", + "Ġ", + "Ġ", + "Ġ", + "Ġworld", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ.", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġend" + ], + "decoded_with_special": "hello world . end", + "decoded_skip_special": "hello world . end" + }, + { + "id": "whitespace-trailing-tabs", + "input_ids": [ + 9535, + 4386, + 13216, + 10223, + 878, + 649, + 1370, + 25, + 197, + 7, + 8658, + 8, + 220, + 220, + 357, + 2777, + 2114, + 8 + ], + "tokens": [ + "tra", + "iling", + "Ġwhites", + "pace", + "Ġbefore", + "Ġnew", + "line", + ":", + "ĉ", + "(", + "tab", + ")", + "Ġ", + "Ġ", + "Ġ(", + "sp", + "aces", + ")" + ], + "decoded_with_special": "trailing whitespace before newline:\t(tab) (spaces)", + "decoded_skip_special": "trailing whitespace before newline:\t(tab) (spaces)" + }, + { + "id": "punctuation-period", + "input_ids": [ + 13 + ], + "tokens": [ + "." + ], + "decoded_with_special": ".", + "decoded_skip_special": "." + }, + { + "id": "punctuation-comma", + "input_ids": [ + 11 + ], + "tokens": [ + "," + ], + "decoded_with_special": ",", + "decoded_skip_special": "," + }, + { + "id": "punctuation-exclamation", + "input_ids": [ + 0 + ], + "tokens": [ + "!" + ], + "decoded_with_special": "!", + "decoded_skip_special": "!" + }, + { + "id": "punctuation-nested-quotes", + "input_ids": [ + 1, + 2061, + 1701, + 673, + 1965, + 851, + 366, + 27485, + 30823, + 851, + 3763, + 986 + ], + "tokens": [ + "\"", + "What", + "?\"", + "Ġshe", + "Ġasked", + "ĠâĢĶ", + "Ġ\"", + "really", + "?!\"", + "ĠâĢĶ", + "Ġyes", + "..." + ], + "decoded_with_special": "\"What?\" she asked — \"really?!\" — yes...", + "decoded_skip_special": "\"What?\" she asked — \"really?!\" — yes..." + }, + { + "id": "punctuation-dash-variants", + "input_ids": [ + 986, + 290, + 564, + 240, + 635, + 784, + 290, + 851, + 290, + 21096, + 13 + ], + "tokens": [ + "...", + "Ġand", + "ĠâĢ", + "Ĵ", + "Ġalso", + "ĠâĢĵ", + "Ġand", + "ĠâĢĶ", + "Ġand", + "ĠâĢķ", + "." + ], + "decoded_with_special": "... and ‒ also – and — and ―.", + "decoded_skip_special": "... and ‒ also – and — and ―." + }, + { + "id": "multiscript-greetings", + "input_ids": [ + 15496, + 220, + 19526, + 254, + 25001, + 121, + 23821, + 243, + 230, + 167, + 227, + 243, + 23294, + 241, + 22174, + 28618, + 2515, + 94, + 31676, + 28225, + 101, + 11976, + 106, + 11976, + 116, + 24231, + 235, + 11976, + 97, + 24231, + 229, + 47048, + 26897, + 148, + 255, + 39848, + 12919, + 14360, + 102, + 40010, + 27072, + 147, + 251, + 220, + 19567, + 103, + 19567, + 100, + 19567, + 109, + 19567, + 103, + 19567, + 242, + 19567, + 113, + 12466, + 253, + 21169, + 18849, + 38857, + 16843, + 20375, + 7377, + 248, + 17394, + 39377, + 138, + 115, + 34703, + 138, + 255, + 33643, + 17394, + 13 + ], + "tokens": [ + "Hello", + "Ġ", + "ä½", + "ł", + "å¥", + "½", + "Ġì", + "ķ", + "Ī", + "ë", + "ħ", + "ķ", + "Ġãģ", + "ĵ", + "ãĤĵ", + "ãģ«", + "ãģ", + "¡", + "ãģ¯", + "Ġà¤", + "¨", + "à¤", + "®", + "à¤", + "¸", + "à¥", + "į", + "à¤", + "¤", + "à¥", + "ĩ", + "ĠÙħ", + "ر", + "Ø", + "Ń", + "ب", + "ا", + "Ġ×", + "©", + "׾", + "×ķ", + "×", + "Ŀ", + "Ġ", + "à¸", + "ª", + "à¸", + "§", + "à¸", + "±", + "à¸", + "ª", + "à¸", + "Ķ", + "à¸", + "µ", + "ĠÐ", + "Ł", + "ÑĢ", + "и", + "в", + "е", + "ÑĤ", + "ĠÎ", + "ļ", + "α", + "λ", + "Î", + "·", + "μ", + "Î", + "Ń", + "Ïģ", + "α", + "." + ], + "decoded_with_special": "Hello 你好 안녕 こんにちは नमस्ते مرحبا שלום สวัสดี Привет Καλημέρα.", + "decoded_skip_special": "Hello 你好 안녕 こんにちは नमस्ते مرحبا שלום สวัสดี Привет Καλημέρα." + }, + { + "id": "multiscript-no-spaces", + "input_ids": [ + 17394, + 19526, + 254, + 26638, + 168, + 243, + 230, + 42063, + 11976, + 101, + 138, + 112, + 11976, + 116, + 8582, + 240, + 104, + 44690, + 41840, + 233, + 50227, + 8582, + 248, + 222, + 41486, + 24861, + 222, + 24861, + 225, + 24861, + 230, + 158, + 229, + 240 + ], + "tokens": [ + "α", + "ä½", + "ł", + "β", + "ì", + "ķ", + "Ī", + "γ", + "à¤", + "¨", + "Î", + "´", + "à¤", + "¸", + "ðŁ", + "Ĵ", + "«", + "ع", + "ðŁij", + "ĭ", + "ש", + "ðŁ", + "ļ", + "Ģ", + "ت", + "âĪ", + "Ģ", + "âĪ", + "ĥ", + "âĪ", + "Ī", + "â", + "ĩ", + "Ĵ" + ], + "decoded_with_special": "α你β안γनδस💫ع👋ש🚀ت∀∃∈⇒", + "decoded_skip_special": "α你β안γनδस💫ع👋ש🚀ت∀∃∈⇒" + }, + { + "id": "multiscript-zwj-mixed", + "input_ids": [ + 40792, + 23877, + 229, + 47991, + 250, + 166, + 116, + 222, + 33768, + 98, + 17312, + 105, + 45739, + 252, + 162, + 115, + 115, + 41840, + 233, + 47991, + 102, + 46695, + 230, + 46695, + 97, + 8582, + 236, + 231, + 5239, + 162, + 115, + 115, + 28938, + 230 + ], + "tokens": [ + "ä¸Ń", + "æĸ", + "ĩ", + "íķ", + "ľ", + "ê", + "¸", + "Ģ", + "æĹ", + "¥", + "æľ", + "¬", + "èª", + "ŀ", + "æ", + "·", + "·", + "ðŁij", + "ĭ", + "íķ", + "©", + "ëĭ", + "Ī", + "ëĭ", + "¤", + "ðŁ", + "İ", + "ī", + "text", + "æ", + "·", + "·", + "åIJ", + "Ī" + ], + "decoded_with_special": "中文한글日本語混👋합니다🎉text混合", + "decoded_skip_special": "中文한글日本語混👋합니다🎉text混合" + }, + { + "id": "multiscript-string-concat", + "input_ids": [ + 70, + 2871, + 278, + 2625, + 19526, + 254, + 25001, + 121, + 1, + 10, + 8582, + 234, + 235, + 10, + 1, + 168, + 243, + 230, + 167, + 227, + 243, + 1, + 10, + 11976, + 101, + 11976, + 106, + 11976, + 116, + 24231, + 235, + 11976, + 97, + 24231, + 229, + 10, + 41840, + 101, + 447, + 235, + 41840, + 102, + 447, + 235, + 41840, + 100 + ], + "tokens": [ + "g", + "reet", + "ing", + "=\"", + "ä½", + "ł", + "å¥", + "½", + "\"", + "+", + "ðŁ", + "Į", + "į", + "+", + "\"", + "ì", + "ķ", + "Ī", + "ë", + "ħ", + "ķ", + "\"", + "+", + "à¤", + "¨", + "à¤", + "®", + "à¤", + "¸", + "à¥", + "į", + "à¤", + "¤", + "à¥", + "ĩ", + "+", + "ðŁij", + "¨", + "âĢ", + "į", + "ðŁij", + "©", + "âĢ", + "į", + "ðŁij", + "§" + ], + "decoded_with_special": "greeting=\"你好\"+🌍+\"안녕\"+नमस्ते+👨‍👩‍👧", + "decoded_skip_special": "greeting=\"你好\"+🌍+\"안녕\"+नमस्ते+👨‍👩‍👧" + }, + { + "id": "multiscript-emoji-arrows", + "input_ids": [ + 138, + 96, + 2124, + 62, + 72, + 12520, + 229, + 107, + 8582, + 229, + 113, + 15139, + 254, + 10263, + 234, + 245, + 12859, + 105, + 15168, + 40790, + 77, + 6607, + 14128, + 309, + 13090, + 2584, + 13090, + 14128, + 220, + 41468, + 38184, + 115, + 14519, + 101, + 886, + 13 + ], + "tokens": [ + "Î", + "£", + "Ġx", + "_", + "i", + "ĠðŁ", + "ĩ", + "¯", + "ðŁ", + "ĩ", + "µ", + "Ġâī", + "ł", + "Ġå", + "Į", + "Ĺ", + "äº", + "¬", + "ĠâĨĴ", + "ĠMü", + "n", + "chen", + "Ġ·", + "ĠT", + "Åį", + "ky", + "Åį", + "Ġ·", + "Ġ", + "ä¸Ĭ", + "æµ", + "·", + "Ġâľ", + "¨", + "Ġend", + "." + ], + "decoded_with_special": "Σ x_i 🇯🇵 ≠ 北京 → München · Tōkyō · 上海 ✨ end.", + "decoded_skip_special": "Σ x_i 🇯🇵 ≠ 北京 → München · Tōkyō · 上海 ✨ end." + }, + { + "id": "multiscript-arrow-chain", + "input_ids": [ + 8582, + 248, + 222, + 11976, + 101, + 11976, + 106, + 11976, + 116, + 24231, + 235, + 11976, + 97, + 24231, + 229, + 39310, + 140, + 253, + 21169, + 18849, + 38857, + 16843, + 20375, + 39310, + 19526, + 254, + 25001, + 121, + 39310, + 168, + 243, + 230, + 167, + 227, + 243, + 39310, + 25405, + 26897, + 148, + 255, + 39848, + 12919, + 39310, + 32391, + 97, + 37929 + ], + "tokens": [ + "ðŁ", + "ļ", + "Ģ", + "à¤", + "¨", + "à¤", + "®", + "à¤", + "¸", + "à¥", + "į", + "à¤", + "¤", + "à¥", + "ĩ", + "âĨĴ", + "Ð", + "Ł", + "ÑĢ", + "и", + "в", + "е", + "ÑĤ", + "âĨĴ", + "ä½", + "ł", + "å¥", + "½", + "âĨĴ", + "ì", + "ķ", + "Ī", + "ë", + "ħ", + "ķ", + "âĨĴ", + "Ùħ", + "ر", + "Ø", + "Ń", + "ب", + "ا", + "âĨĴ", + "âĿ", + "¤", + "ï¸ı" + ], + "decoded_with_special": "🚀नमस्ते→Привет→你好→안녕→مرحبا→❤️", + "decoded_skip_special": "🚀नमस्ते→Привет→你好→안녕→مرحبا→❤️" + }, + { + "id": "multiscript-function-call", + "input_ids": [ + 21943, + 7, + 40792, + 23877, + 229, + 11, + 47991, + 250, + 166, + 116, + 222, + 11, + 33768, + 98, + 17312, + 105, + 45739, + 252, + 11, + 11976, + 117, + 11976, + 123, + 11976, + 101, + 24231, + 235, + 11976, + 99, + 24231, + 222, + 11, + 21169, + 35072, + 21727, + 21727, + 31583, + 18849, + 140, + 117, + 8, + 39310, + 90, + 24861, + 222, + 25, + 24861, + 225, + 11, + 158, + 232, + 243, + 25, + 158, + 232, + 245, + 92 + ], + "tokens": [ + "foo", + "(", + "ä¸Ń", + "æĸ", + "ĩ", + ",", + "íķ", + "ľ", + "ê", + "¸", + "Ģ", + ",", + "æĹ", + "¥", + "æľ", + "¬", + "èª", + "ŀ", + ",", + "à¤", + "¹", + "à¤", + "¿", + "à¤", + "¨", + "à¥", + "į", + "à¤", + "¦", + "à¥", + "Ģ", + ",", + "ÑĢ", + "Ñĥ", + "Ñģ", + "Ñģ", + "к", + "и", + "Ð", + "¹", + ")", + "âĨĴ", + "{", + "âĪ", + "Ģ", + ":", + "âĪ", + "ĥ", + ",", + "â", + "Ĭ", + "ķ", + ":", + "â", + "Ĭ", + "Ĺ", + "}" + ], + "decoded_with_special": "foo(中文,한글,日本語,हिन्दी,русский)→{∀:∃,⊕:⊗}", + "decoded_skip_special": "foo(中文,한글,日本語,हिन्दी,русский)→{∀:∃,⊕:⊗}" + }, + { + "id": "multiscript-legacy-symbols", + "input_ids": [ + 158, + 226, + 244, + 3682, + 2343, + 8008, + 37929, + 10673, + 38852, + 30581, + 8460, + 10432, + 126, + 98, + 14988, + 158, + 224, + 123, + 7377, + 96, + 24861, + 222, + 220, + 19526, + 254, + 25001, + 121, + 23821, + 243, + 230, + 167, + 227, + 243, + 28981 + ], + "tokens": [ + "â", + "Ħ", + "ĸ", + "42", + "Ġâ", + "Ħ¢", + "ï¸ı", + "Ġ©", + "Ġ®", + "Ġ¶", + "Ġ§", + "ĠâĤ¬", + "Â", + "¥", + "£", + "â", + "Ĥ", + "¿", + "ĠÎ", + "£", + "âĪ", + "Ģ", + "Ġ", + "ä½", + "ł", + "å¥", + "½", + "Ġì", + "ķ", + "Ī", + "ë", + "ħ", + "ķ", + "ĠاÙĦ" + ], + "decoded_with_special": "№42 ™️ © ® ¶ § €¥£₿ Σ∀ 你好 안녕 ال", + "decoded_skip_special": "№42 ™️ © ® ¶ § €¥£₿ Σ∀ 你好 안녕 ال" + }, + { + "id": "escape-sequences-with-zwj", + "input_ids": [ + 33991, + 59, + 15, + 392, + 59, + 926, + 397, + 59, + 77, + 3605, + 1370, + 162, + 115, + 115, + 17739, + 98, + 447, + 235, + 57, + 54, + 41, + 37929, + 19738, + 669 + ], + "tokens": [ + "NULL", + "\\", + "0", + "and", + "\\", + "tt", + "ab", + "\\", + "n", + "new", + "line", + "æ", + "·", + "·", + "åħ", + "¥", + "âĢ", + "į", + "Z", + "W", + "J", + "ï¸ı", + "select", + "ors" + ], + "decoded_with_special": "NULL\\0and\\ttab\\nnewline混入‍ZWJ️selectors", + "decoded_skip_special": "NULL\\0and\\ttab\\nnewline混入‍ZWJ️selectors" + }, + { + "id": "german-compound-short", + "input_ids": [ + 47849, + 25151, + 33467, + 13, + 1024, + 40768, + 1976, + 494, + 4352, + 12644, + 89, + 83, + 2853, + 32364, + 1930, + 270, + 993, + 321, + 647, + 257, + 385, + 4587, + 20159, + 6759, + 1134, + 86, + 9587, + 2736, + 10339, + 354, + 549, + 75, + 671, + 13, + 30325, + 226 + ], + "tokens": [ + "Nat", + "ür", + "lich", + ".", + "ĠDe", + "utsch", + "Ġz", + "ie", + "ht", + "Ġjet", + "z", + "t", + "Ġden", + "ĠKom", + "pos", + "it", + "ah", + "am", + "mer", + "Ġa", + "us", + "Ġder", + "ĠGram", + "mat", + "ik", + "w", + "erk", + "ze", + "ugs", + "ch", + "ub", + "l", + "ade", + ".", + "ĠðŁĺ", + "Ħ" + ], + "decoded_with_special": "Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. 😄", + "decoded_skip_special": "Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. 😄" + }, + { + "id": "german-compound-long-1", + "input_ids": [ + 3546, + 37417, + 29727, + 6667, + 2004, + 962, + 268, + 332, + 11033, + 301, + 417, + 2150, + 2032, + 1940, + 1302, + 304, + 259, + 509, + 21223, + 316, + 562, + 268, + 25192, + 28864, + 418, + 2522, + 494, + 1350, + 672, + 620, + 353, + 497, + 11722, + 304, + 7274, + 3310, + 641, + 354, + 2533, + 332, + 70, + 408, + 33467, + 365, + 896, + 17529, + 3318, + 407, + 959, + 660, + 287, + 384, + 259, + 6295, + 429, + 363, + 76, + 2398, + 268, + 485, + 33701, + 648, + 1662, + 528, + 65, + 794, + 4656, + 1931, + 301, + 1942, + 677, + 258, + 367, + 11033, + 3046, + 328, + 365, + 270, + 18042, + 19426, + 1706, + 17796, + 38031, + 549, + 1559, + 38572, + 75, + 30830, + 5420, + 2588, + 295, + 268, + 13 + ], + "tokens": [ + "Im", + "ĠMorg", + "enne", + "bel", + "ged", + "ank", + "en", + "ver", + "ä", + "st", + "el", + "ung", + "sw", + "ald", + "Ġstand", + "Ġe", + "in", + "ĠK", + "affe", + "et", + "ass", + "en", + "rand", + "phil", + "os", + "oph", + "ie", + "be", + "ob", + "ach", + "ter", + "Ġne", + "ben", + "Ġe", + "iner", + "ĠReg", + "ens", + "ch", + "irm", + "ver", + "g", + "ess", + "lich", + "ke", + "its", + "station", + "Ġund", + "Ġnot", + "ier", + "te", + "Ġin", + "Ġse", + "in", + "ĠSon", + "nt", + "ag", + "m", + "org", + "en", + "ide", + "enf", + "ang", + "not", + "iz", + "b", + "uch", + "Ġdie", + "Ġer", + "st", + "aun", + "lic", + "he", + "ĠH", + "ä", + "uf", + "ig", + "ke", + "it", + "Ġvon", + "ĠFen", + "ster", + "bank", + "sta", + "ub", + "son", + "nen", + "l", + "icht", + "ref", + "lex", + "ion", + "en", + "." + ], + "decoded_with_special": "Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen.", + "decoded_skip_special": "Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen." + }, + { + "id": "german-compound-long-2", + "input_ids": [ + 45, + 1765, + 268, + 1312, + 23940, + 30114, + 660, + 304, + 500, + 509, + 9116, + 71, + 7278, + 354, + 43027, + 3732, + 268, + 1350, + 293, + 794, + 83, + 2150, + 2655, + 5083, + 2150, + 82, + 5356, + 354, + 500, + 11, + 266, + 11033, + 71, + 10920, + 304, + 259, + 1526, + 17694, + 324, + 1516, + 21921, + 12501, + 7750, + 9101, + 487, + 77, + 2150, + 82, + 40085, + 959, + 2150, + 82, + 1350, + 559, + 701, + 22562, + 353, + 10255, + 304, + 7274, + 4700, + 381, + 268, + 30404, + 461, + 436, + 1134, + 9116, + 527, + 8847, + 354, + 2150, + 12807, + 3400, + 325, + 7284, + 354, + 11033, + 701, + 328, + 83, + 1175, + 13, + 27545, + 4587, + 12880, + 710, + 299, + 11033, + 372, + 660, + 264, + 488, + 304, + 259, + 27094, + 3464, + 74, + 10366, + 9107, + 69, + 993, + 81, + 301, + 7456, + 75, + 14664, + 36073, + 3361, + 261, + 396, + 257, + 3046, + 304, + 7749, + 412, + 259, + 6335, + 332, + 365, + 71, + 3808, + 2301, + 417, + 3927, + 19725, + 2150, + 82, + 1362, + 11033, + 83, + 3318, + 374, + 2086, + 25, + 564, + 252, + 32, + 21474, + 2150, + 11, + 288, + 419, + 410, + 8553, + 6486, + 13655, + 304, + 500, + 402, + 276, + 962, + 33701, + 40780, + 332, + 74, + 1662, + 2150, + 82, + 469, + 69, + 993, + 918, + 48937, + 2288, + 0, + 447, + 250 + ], + "tokens": [ + "N", + "eb", + "en", + "Ġi", + "hm", + "Ġsumm", + "te", + "Ġe", + "ine", + "ĠK", + "ü", + "h", + "ls", + "ch", + "rank", + "inn", + "en", + "be", + "le", + "uch", + "t", + "ung", + "ser", + "inner", + "ung", + "s", + "mas", + "ch", + "ine", + ",", + "Ġw", + "ä", + "h", + "rend", + "Ġe", + "in", + "ĠMar", + "mel", + "ad", + "eng", + "las", + "dec", + "kel", + "ö", + "ff", + "n", + "ung", + "s", + "optim", + "ier", + "ung", + "s", + "be", + "au", + "ft", + "rag", + "ter", + "Ġmit", + "Ġe", + "iner", + "ĠTre", + "pp", + "en", + "haus", + "ak", + "ust", + "ik", + "ü", + "ber", + "ras", + "ch", + "ung", + "san", + "aly", + "se", + "Ġbes", + "ch", + "ä", + "ft", + "ig", + "t", + "Ġwar", + ".", + "ĠAus", + "Ġder", + "ĠFer", + "ne", + "Ġn", + "ä", + "her", + "te", + "Ġs", + "ich", + "Ġe", + "in", + "ĠWol", + "ken", + "k", + "rat", + "zer", + "f", + "ah", + "r", + "st", + "uh", + "l", + "mus", + "ikk", + "omp", + "on", + "ist", + "Ġa", + "uf", + "Ġe", + "inem", + "ĠE", + "in", + "rad", + "ver", + "ke", + "h", + "rs", + "reg", + "el", + "miss", + "acht", + "ung", + "s", + "ger", + "ä", + "t", + "Ġund", + "Ġr", + "ief", + ":", + "ĠâĢ", + "ŀ", + "A", + "cht", + "ung", + ",", + "Ġd", + "ort", + "Ġv", + "orne", + "Ġlie", + "gt", + "Ġe", + "ine", + "ĠG", + "ed", + "ank", + "enf", + "aden", + "ver", + "k", + "not", + "ung", + "s", + "ge", + "f", + "ah", + "ren", + "sit", + "uation", + "!", + "âĢ", + "ľ" + ], + "decoded_with_special": "Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“", + "decoded_skip_special": "Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“" + }, + { + "id": "german-compound-long-3", + "input_ids": [ + 3646, + 9101, + 22877, + 33467, + 6184, + 114, + 487, + 3262, + 68, + 264, + 488, + 304, + 259, + 47447, + 2150, + 82, + 325, + 270, + 268, + 2178, + 75, + 11033, + 83, + 353, + 7972, + 301, + 78, + 39683, + 634, + 282, + 11, + 3318, + 339, + 430, + 385, + 1308, + 17396, + 1452, + 288, + 260, + 72, + 347, + 9116, + 305, + 74, + 2543, + 647, + 5907, + 419, + 959, + 86, + 2120, + 1326, + 6223, + 11693, + 701, + 4169, + 346, + 710, + 71, + 647, + 3732, + 268, + 11, + 304, + 259, + 402, + 433, + 19471, + 86, + 6422, + 76, + 9116, + 83, + 4801, + 16370, + 11722, + 527, + 729, + 3318, + 304, + 259, + 289, + 5374, + 9744, + 328, + 3326, + 86, + 343, + 81, + 353, + 8105, + 268, + 75, + 9101, + 487, + 417, + 5420, + 2588, + 507, + 4164, + 6570, + 893, + 18320, + 13, + 15669, + 68, + 1040, + 321, + 1036, + 9116, + 358, + 316, + 268, + 264, + 494, + 2853, + 33816, + 332, + 3903, + 277, + 25151, + 399, + 620, + 20124, + 3775, + 33467, + 912, + 354, + 41769, + 74, + 415, + 268, + 11181, + 81, + 19725, + 2150, + 3318, + 47833, + 959, + 660, + 41427, + 461, + 747, + 37036, + 268, + 332, + 660, + 312, + 328, + 2150, + 82, + 2536, + 2397, + 2013, + 13 + ], + "tokens": [ + "Pl", + "ö", + "tz", + "lich", + "ĠÃ", + "¶", + "ff", + "net", + "e", + "Ġs", + "ich", + "Ġe", + "in", + "ĠZeit", + "ung", + "s", + "se", + "it", + "en", + "umb", + "l", + "ä", + "t", + "ter", + "wind", + "st", + "o", + "ÃŁ", + "port", + "al", + ",", + "Ġund", + "Ġhe", + "ra", + "us", + "Ġpur", + "zel", + "ten", + "Ġd", + "re", + "i", + "ĠB", + "ü", + "ro", + "k", + "lam", + "mer", + "ns", + "ort", + "ier", + "w", + "elt", + "me", + "isters", + "cha", + "ft", + "ste", + "il", + "ne", + "h", + "mer", + "inn", + "en", + ",", + "Ġe", + "in", + "ĠG", + "art", + "enz", + "w", + "erg", + "m", + "ü", + "t", + "zen", + "far", + "ben", + "ber", + "ater", + "Ġund", + "Ġe", + "in", + "Ġh", + "och", + "grad", + "ig", + "Ġver", + "w", + "ir", + "r", + "ter", + "ĠSupp", + "en", + "l", + "ö", + "ff", + "el", + "ref", + "lex", + "ions", + "met", + "aph", + "ys", + "iker", + ".", + "ĠGem", + "e", + "ins", + "am", + "Ġgr", + "ü", + "nd", + "et", + "en", + "Ġs", + "ie", + "Ġden", + "ĠBundes", + "ver", + "band", + "Ġf", + "ür", + "ĠN", + "ach", + "mitt", + "ags", + "lich", + "ts", + "ch", + "atten", + "k", + "ant", + "en", + "bet", + "r", + "acht", + "ung", + "Ġund", + "Ġimprovis", + "ier", + "te", + "ĠSof", + "ak", + "iss", + "enburg", + "en", + "ver", + "te", + "id", + "ig", + "ung", + "s", + "str", + "ateg", + "ien", + "." + ], + "decoded_with_special": "Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien.", + "decoded_skip_special": "Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien." + }, + { + "id": "german-compound-long-4", + "input_ids": [ + 5840, + 2275, + 437, + 1646, + 6475, + 417, + 1452, + 264, + 488, + 28654, + 545, + 27328, + 15952, + 10745, + 268, + 1706, + 11392, + 3653, + 38031, + 549, + 4743, + 10557, + 89, + 10957, + 11, + 24486, + 4587, + 509, + 21223, + 316, + 562, + 268, + 25192, + 28864, + 418, + 2522, + 494, + 1350, + 672, + 620, + 353, + 304, + 500, + 730, + 959, + 677, + 258, + 48467, + 1046, + 445, + 68, + 6184, + 120, + 527, + 4656, + 370, + 30830, + 328, + 365, + 270, + 18042, + 791, + 82, + 3732, + 82, + 18230, + 21841, + 507, + 19503, + 2507, + 11, + 5522, + 620, + 14664, + 365, + 335, + 17231, + 77, + 9116, + 65, + 2150, + 268, + 3318, + 2094, + 3885, + 696, + 9501, + 354, + 361, + 487, + 993, + 81, + 912, + 3212, + 19187, + 11693, + 701, + 8135, + 499, + 270, + 11033, + 77, + 5796, + 9116, + 83, + 4801, + 15418, + 404, + 69, + 364, + 27906, + 660, + 346, + 12636, + 354, + 2001, + 2150, + 82, + 1676, + 903, + 3653, + 23105, + 2120, + 13 + ], + "tokens": [ + "Am", + "ĠAb", + "end", + "Ġvers", + "amm", + "el", + "ten", + "Ġs", + "ich", + "Ġalle", + "Ġim", + "ĠMond", + "sche", + "inf", + "en", + "ster", + "rah", + "men", + "sta", + "ub", + "gl", + "itzer", + "z", + "immer", + ",", + "Ġwo", + "Ġder", + "ĠK", + "affe", + "et", + "ass", + "en", + "rand", + "phil", + "os", + "oph", + "ie", + "be", + "ob", + "ach", + "ter", + "Ġe", + "ine", + "Ġfe", + "ier", + "lic", + "he", + "ĠSchl", + "uss", + "red", + "e", + "ĠÃ", + "¼", + "ber", + "Ġdie", + "ĠW", + "icht", + "ig", + "ke", + "it", + "Ġvon", + "ĠUn", + "s", + "inn", + "s", + "produ", + "kt", + "ions", + "fre", + "ude", + ",", + "ĠSpr", + "ach", + "mus", + "ke", + "ld", + "eh", + "n", + "ü", + "b", + "ung", + "en", + "Ġund", + "ĠDon", + "aud", + "amp", + "fs", + "ch", + "if", + "ff", + "ah", + "r", + "ts", + "ges", + "ells", + "cha", + "ft", + "sk", + "ap", + "it", + "ä", + "n", + "sm", + "ü", + "t", + "zen", + "kn", + "op", + "f", + "ers", + "atz", + "te", + "il", + "bes", + "ch", + "aff", + "ung", + "s", + "pro", + "ble", + "men", + "Ġhi", + "elt", + "." + ], + "decoded_with_special": "Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt.", + "decoded_skip_special": "Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt." + }, + { + "id": "german-compound-long-5", + "input_ids": [ + 21174, + 620, + 607, + 3808, + 354, + 660, + 477, + 24090, + 68, + 500, + 1168, + 3046, + 2228, + 268, + 29361, + 11, + 35851, + 39683, + 263, + 307, + 72, + 4587, + 509, + 9116, + 71, + 7278, + 354, + 43027, + 3732, + 268, + 1350, + 293, + 794, + 83, + 2150, + 2655, + 5083, + 2150, + 82, + 5356, + 354, + 500, + 11, + 4656, + 264, + 488, + 6184, + 120, + 527, + 582, + 25280, + 358, + 68, + 309, + 25151, + 9101, + 487, + 77, + 2150, + 82, + 559, + 69, + 647, + 591, + 321, + 365, + 896, + 86, + 861, + 20601, + 11033, + 22877, + 2150, + 307, + 74, + 30909, + 660, + 13 + ], + "tokens": [ + "Dan", + "ach", + "Ġher", + "rs", + "ch", + "te", + "Ġall", + "gem", + "e", + "ine", + "ĠZ", + "uf", + "ried", + "en", + "heit", + ",", + "Ġau", + "ÃŁ", + "er", + "Ġbe", + "i", + "Ġder", + "ĠK", + "ü", + "h", + "ls", + "ch", + "rank", + "inn", + "en", + "be", + "le", + "uch", + "t", + "ung", + "ser", + "inner", + "ung", + "s", + "mas", + "ch", + "ine", + ",", + "Ġdie", + "Ġs", + "ich", + "ĠÃ", + "¼", + "ber", + "Ġman", + "gel", + "nd", + "e", + "ĠT", + "ür", + "ö", + "ff", + "n", + "ung", + "s", + "au", + "f", + "mer", + "ks", + "am", + "ke", + "its", + "w", + "ert", + "sch", + "ä", + "tz", + "ung", + "Ġbe", + "k", + "lag", + "te", + "." + ], + "decoded_with_special": "Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte.", + "decoded_skip_special": "Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte." + } + ] +} diff --git a/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/qwen2_5_multilingual.json b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/qwen2_5_multilingual.json new file mode 100644 index 00000000..b0c6d5fb --- /dev/null +++ b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/qwen2_5_multilingual.json @@ -0,0 +1,4689 @@ +{ + "metadata": { + "model_id": "Qwen/Qwen2.5-0.5B", + "transformers_version": "4.57.1", + "generated_at": "2026-05-15T23:58:36+00:00", + "input_count": 83 + }, + "entries": [ + { + "id": "latin-diacritics-french", + "input_ids": [ + 924, + 58858, + 9333, + 1242, + 963, + 94880, + 586, + 94872, + 1021, + 1959, + 1560, + 24267, + 1411, + 29772, + 133607, + 572, + 9073, + 13 + ], + "tokens": [ + "ca", + "fé", + "Ġré", + "sum", + "é", + "Ġnaï", + "ve", + "Ġfaç", + "ade", + "ĠâĢĶ", + "Ġcr", + "ème", + "Ġbr", + "û", + "lée", + "Ġwas", + "Ġexcellent", + "." + ], + "decoded_with_special": "café résumé naïve façade — crème brûlée was excellent.", + "decoded_skip_special": "café résumé naïve façade — crème brûlée was excellent." + }, + { + "id": "latin-diacritics-mixed", + "input_ids": [ + 39, + 2391, + 28852, + 13785, + 3240, + 19358, + 68, + 13, + 11331, + 87084, + 23063, + 264, + 1187, + 1051, + 18324, + 390, + 43265, + 13 + ], + "tokens": [ + "H", + "ü", + "gel", + "Ġüber", + "ĠBr", + "ück", + "e", + ".", + "ĠMa", + "ñana", + "Ġvoy", + "Ġa", + "Ġla", + "Ġpe", + "ña", + "Ġcon", + "ĠJosé", + "." + ], + "decoded_with_special": "Hügel über Brücke. Mañana voy a la peña con José.", + "decoded_skip_special": "Hügel über Brücke. Mañana voy a la peña con José." + }, + { + "id": "latin-diacritics-portuguese", + "input_ids": [ + 50, + 3413, + 36926, + 3958, + 10608, + 55084, + 39353, + 4942, + 26, + 141377, + 10928, + 13 + ], + "tokens": [ + "S", + "ão", + "ĠPaulo", + "Ġé", + "Ġuma", + "Ġcidade", + "Ġinteress", + "ante", + ";", + "ĠKöln", + "Ġauch", + "." + ], + "decoded_with_special": "São Paulo é uma cidade interessante; Köln auch.", + "decoded_skip_special": "São Paulo é uma cidade interessante; Köln auch." + }, + { + "id": "cyrillic-greeting", + "input_ids": [ + 53645, + 26991, + 8178, + 11, + 137144, + 0, + 128868, + 137901, + 10813, + 14746, + 9684, + 8959, + 57306, + 1478, + 13 + ], + "tokens": [ + "ÐŁÑĢ", + "ив", + "еÑĤ", + ",", + "ĠмиÑĢ", + "!", + "ĠÐŃÑĤо", + "ĠÑĤеÑģÑĤ", + "ĠÑĤ", + "ок", + "ени", + "з", + "аÑĤоÑĢ", + "а", + "." + ], + "decoded_with_special": "Привет, мир! Это тест токенизатора.", + "decoded_skip_special": "Привет, мир! Это тест токенизатора." + }, + { + "id": "cyrillic-prose", + "input_ids": [ + 37991, + 22787, + 137303, + 1959, + 130297, + 1802, + 86894, + 129069, + 13, + 34789, + 17953, + 12121, + 17175, + 128714, + 220, + 16, + 17, + 139642, + 6715, + 13 + ], + "tokens": [ + "Ðľ", + "оÑģ", + "ква", + "ĠâĢĶ", + "ĠÑģÑĤол", + "и", + "ÑĨа", + "ĠÐłÐ¾ÑģÑģии", + ".", + "ĠÐĿ", + "аÑģ", + "ел", + "ение", + "Ġболее", + "Ġ", + "1", + "2", + "Ġмиллион", + "ов", + "." + ], + "decoded_with_special": "Москва — столица России. Население более 12 миллионов.", + "decoded_skip_special": "Москва — столица России. Население более 12 миллионов." + }, + { + "id": "greek-greeting", + "input_ids": [ + 144996, + 18945, + 33486, + 41424, + 43123, + 79431, + 38079, + 18945, + 71638, + 75195, + 43928, + 43123, + 30143, + 0, + 7851, + 245, + 62027, + 33486, + 145240, + 43928, + 43928, + 18945, + 59147, + 54141, + 33269, + 88902, + 17383, + 234, + 43123, + 27554, + 38079, + 85034, + 41424, + 13 + ], + "tokens": [ + "Îļ", + "α", + "λ", + "η", + "μ", + "ÎŃ", + "Ïģ", + "α", + "Ġκ", + "ÏĮ", + "Ïĥ", + "μ", + "ε", + "!", + "ĠÎ", + "Ĺ", + "Ġγ", + "λ", + "Ïİ", + "Ïĥ", + "Ïĥ", + "α", + "Ġε", + "ί", + "ν", + "αι", + "ĠÏ", + "Į", + "μ", + "ο", + "Ïģ", + "ÏĨ", + "η", + "." + ], + "decoded_with_special": "Καλημέρα κόσμε! Η γλώσσα είναι όμορφη.", + "decoded_skip_special": "Καλημέρα κόσμε! Η γλώσσα είναι όμορφη." + }, + { + "id": "cjk-simplified-greeting", + "input_ids": [ + 108386, + 11, + 99489, + 0, + 100346, + 17177, + 99689, + 31548, + 81705, + 1773 + ], + "tokens": [ + "ä½łå¥½", + ",", + "ä¸ĸçķĮ", + "!", + "è¿Ļæĺ¯", + "åĪĨ", + "è¯į", + "åĻ¨", + "æµĭè¯ķ", + "ãĢĤ" + ], + "decoded_with_special": "你好,世界!这是分词器测试。", + "decoded_skip_special": "你好,世界!这是分词器测试。" + }, + { + "id": "cjk-simplified-prose", + "input_ids": [ + 104811, + 20412, + 102506, + 37029, + 102055, + 106654, + 102064, + 100653, + 1773 + ], + "tokens": [ + "ä¸Ńæĸĩ", + "æĺ¯", + "ä¸ĸçķĮä¸Ĭ", + "使çĶ¨", + "人åı£", + "æľĢå¤ļçļĦ", + "è¯Ńè¨Ģ", + "ä¹ĭä¸Ģ", + "ãĢĤ" + ], + "decoded_with_special": "中文是世界上使用人口最多的语言之一。", + "decoded_skip_special": "中文是世界上使用人口最多的语言之一。" + }, + { + "id": "cjk-traditional-compare", + "input_ids": [ + 100112, + 100480, + 104811, + 101905, + 103271, + 100480, + 104811, + 115742, + 1773 + ], + "tokens": [ + "ç¹ģ", + "é«Ķ", + "ä¸Ńæĸĩ", + "èĪĩ", + "ç°¡", + "é«Ķ", + "ä¸Ńæĸĩ", + "æľīæīĢä¸įåIJĮ", + "ãĢĤ" + ], + "decoded_with_special": "繁體中文與簡體中文有所不同。", + "decoded_skip_special": "繁體中文與簡體中文有所不同。" + }, + { + "id": "cjk-traditional-hongkong", + "input_ids": [ + 101926, + 9370, + 100112, + 100480, + 104811, + 18830, + 103093, + 65278, + 9370, + 112924, + 14273, + 247, + 1773 + ], + "tokens": [ + "é¦Ļ港", + "çļĦ", + "ç¹ģ", + "é«Ķ", + "ä¸Ńæĸĩ", + "æľī", + "çį¨", + "çī¹", + "çļĦ", + "è©ŀ", + "å½", + "Ļ", + "ãĢĤ" + ], + "decoded_with_special": "香港的繁體中文有獨特的詞彙。", + "decoded_skip_special": "香港的繁體中文有獨特的詞彙。" + }, + { + "id": "japanese-voiced-kana-greeting", + "input_ids": [ + 89015, + 5373, + 99489, + 1773, + 137084, + 28120, + 136766, + 133659, + 15767, + 140592, + 37541, + 1773 + ], + "tokens": [ + "ãģĵãĤĵãģ«ãģ¡ãģ¯", + "ãĢģ", + "ä¸ĸçķĮ", + "ãĢĤ", + "ãĥĪãĥ¼", + "ãĤ¯", + "ãĥĬãĤ¤", + "ãĤ¶ãĥ¼", + "ãģ®", + "ãĥĨãĤ¹ãĥĪ", + "ãģ§ãģĻ", + "ãĢĤ" + ], + "decoded_with_special": "こんにちは、世界。トークナイザーのテストです。", + "decoded_skip_special": "こんにちは、世界。トークナイザーのテストです。" + }, + { + "id": "japanese-voiced-kana-prose", + "input_ids": [ + 102356, + 46553, + 46207, + 125980, + 15322, + 129235, + 82712, + 239, + 70189, + 128777, + 24071, + 111, + 99225, + 29490, + 37541, + 1773 + ], + "tokens": [ + "æĿ±", + "京", + "ãĤ¿", + "ãĥ¯ãĥ¼", + "ãģ¯", + "ãģ¨ãģ¦ãĤĤ", + "è³", + "ij", + "ãĤĦ", + "ãģĭãģª", + "è¦", + "³", + "åħī", + "åľ°", + "ãģ§ãģĻ", + "ãĢĤ" + ], + "decoded_with_special": "東京タワーはとても賑やかな観光地です。", + "decoded_skip_special": "東京タワーはとても賑やかな観光地です。" + }, + { + "id": "hangul-syllables-greeting", + "input_ids": [ + 126246, + 144370, + 91145, + 11, + 133196, + 0, + 10764, + 228, + 57160, + 223, + 105, + 60315, + 12802, + 126781, + 10764, + 72509, + 53189, + 78952, + 13 + ], + "tokens": [ + "ìķĪ", + "ëħķ", + "íķĺìĦ¸ìļĶ", + ",", + "ĠìĦ¸ê³Ħ", + "!", + "Ġí", + "Ĩ", + "łí", + "ģ", + "¬", + "ëĤĺ", + "ìĿ´", + "ìłĢ", + "Ġí", + "ħĮ", + "ìĬ¤íĬ¸", + "ìŀħëĭĪëĭ¤", + "." + ], + "decoded_with_special": "안녕하세요, 세계! 토크나이저 테스트입니다.", + "decoded_skip_special": "안녕하세요, 세계! 토크나이저 테스트입니다." + }, + { + "id": "hangul-syllables-prose", + "input_ids": [ + 23573, + 124785, + 31079, + 16560, + 61298, + 83291, + 17380, + 139052, + 20487, + 60838, + 13 + ], + "tokens": [ + "íķľ", + "êµŃ", + "ìĸ´", + "ëĬĶ", + "Ġíķľ", + "ê¸Ģ", + "ë¡ľ", + "Ġíijľ", + "기", + "íķ©ëĭĪëĭ¤", + "." + ], + "decoded_with_special": "한국어는 한글로 표기합니다.", + "decoded_skip_special": "한국어는 한글로 표기합니다." + }, + { + "id": "arabic-greeting", + "input_ids": [ + 124122, + 29825, + 124671, + 124476, + 129634, + 0, + 128280, + 136837, + 124422, + 128883, + 124552, + 8532, + 137056, + 124623, + 13 + ], + "tokens": [ + "Ùħر", + "ØŃ", + "با", + "ĠباÙĦ", + "عاÙĦÙħ", + "!", + "ĠÙĩذا", + "Ġاخت", + "بار", + "ĠÙĦÙĦÙħ", + "ØŃÙĦ", + "ÙĦ", + "ĠاÙĦÙĦغ", + "ÙĪÙĬ", + "." + ], + "decoded_with_special": "مرحبا بالعالم! هذا اختبار للمحلل اللغوي.", + "decoded_skip_special": "مرحبا بالعالم! هذا اختبار للمحلل اللغوي." + }, + { + "id": "arabic-prose", + "input_ids": [ + 31382, + 130353, + 25871, + 129071, + 56794, + 81778, + 25871, + 59842, + 135507, + 39434, + 128667, + 63237, + 17166, + 124176, + 123860, + 128252, + 17166, + 124392, + 35038, + 13 + ], + "tokens": [ + "اÙĦ", + "ÙĦغ", + "Ø©", + "ĠاÙĦعربÙĬØ©", + "ĠÙĦ", + "غ", + "Ø©", + "Ġس", + "اÙħÙĬØ©", + "Ġت", + "Ùĥتب", + "ĠÙħÙĨ", + "ĠاÙĦ", + "ÙĬÙħ", + "ÙĬÙĨ", + "ĠØ¥ÙĦÙī", + "ĠاÙĦ", + "ÙĬس", + "ار", + "." + ], + "decoded_with_special": "اللغة العربية لغة سامية تكتب من اليمين إلى اليسار.", + "decoded_skip_special": "اللغة العربية لغة سامية تكتب من اليمين إلى اليسار." + }, + { + "id": "hebrew-greeting", + "input_ids": [ + 126654, + 123855, + 124907, + 0, + 126197, + 91511, + 136330, + 124756, + 125182, + 123914, + 123868, + 125117, + 50226, + 13 + ], + "tokens": [ + "ש׾×ķ×Ŀ", + "Ġ×¢", + "×ķ׾×Ŀ", + "!", + "Ġ×ĸ×Ķ", + "Ġ×ŀ", + "×ij×Ĺף", + "Ġש׾", + "Ġ×Ķ×ĺ", + "×ķק", + "׳×Ļ", + "×Ļ×ĸ", + "ר", + "." + ], + "decoded_with_special": "שלום עולם! זה מבחן של הטוקנייזר.", + "decoded_skip_special": "שלום עולם! זה מבחן של הטוקנייזר." + }, + { + "id": "devanagari-hindi", + "input_ids": [ + 60096, + 87244, + 78368, + 30484, + 97, + 34370, + 14925, + 99, + 72653, + 60096, + 42311, + 107, + 23868, + 0, + 14925, + 107, + 93948, + 14925, + 253, + 54575, + 64704, + 60096, + 31411, + 229, + 146114, + 5502, + 120, + 44179, + 83636, + 44179, + 43647, + 64704, + 30484, + 115, + 146548, + 84310, + 12619, + 230, + 146031 + ], + "tokens": [ + "न", + "म", + "स", + "à¥įà¤", + "¤", + "à¥ĩ", + "Ġà¤", + "¦", + "à¥ģ", + "न", + "िà¤", + "¯", + "ा", + "!", + "Ġà¤", + "¯", + "ह", + "Ġà¤", + "Ł", + "à¥ĭ", + "à¤ķ", + "न", + "ाà¤", + "ĩ", + "à¤ľ", + "à¤", + "¼", + "र", + "Ġप", + "र", + "à¥Ģ", + "à¤ķ", + "à¥įà¤", + "·", + "ण", + "Ġह", + "à¥", + "Ī", + "।" + ], + "decoded_with_special": "नमस्ते दुनिया! यह टोकनाइज़र परीक्षण है।", + "decoded_skip_special": "नमस्ते दुनिया! यह टोकनाइज़र परीक्षण है।" + }, + { + "id": "devanagari-sanskrit", + "input_ids": [ + 78368, + 72314, + 78368, + 30484, + 243, + 12619, + 225, + 79238, + 14925, + 255, + 31411, + 108, + 79238, + 47809, + 43647, + 14925, + 237, + 64704, + 83636, + 85033, + 31411, + 248, + 43647, + 60096, + 14925, + 255, + 31411, + 115, + 23868, + 84310, + 12619, + 230, + 146031 + ], + "tokens": [ + "स", + "à¤Ĥ", + "स", + "à¥įà¤", + "ķ", + "à¥", + "ĥ", + "त", + "Ġà¤", + "Ń", + "ाà¤", + "°", + "त", + "Ġà¤ķ", + "à¥Ģ", + "Ġà¤", + "ı", + "à¤ķ", + "Ġप", + "à¥įर", + "ाà¤", + "ļ", + "à¥Ģ", + "न", + "Ġà¤", + "Ń", + "ाà¤", + "·", + "ा", + "Ġह", + "à¥", + "Ī", + "।" + ], + "decoded_with_special": "संस्कृत भारत की एक प्राचीन भाषा है।", + "decoded_skip_special": "संस्कृत भारत की एक प्राचीन भाषा है।" + }, + { + "id": "thai-combining-marks-greeting", + "input_ids": [ + 143126, + 23271, + 125136, + 28319, + 126660, + 126397, + 0, + 220, + 20184, + 47171, + 40327, + 83581, + 93874, + 139678, + 124648, + 64684, + 123824, + 39855, + 124090, + 125506 + ], + "tokens": [ + "สว", + "ั", + "สà¸Ķ", + "ี", + "à¸Ĭาว", + "à¹Ĥลà¸ģ", + "!", + "Ġ", + "à¸Ļ", + "ีà¹Ī", + "à¸Ħ", + "ืà¸Ń", + "à¸ģาร", + "à¸Ĺà¸Ķสà¸Ńà¸ļ", + "à¹Ģà¸Ħร", + "ืà¹Ī", + "à¸Ńà¸ĩ", + "à¸ķ", + "ัà¸Ķ", + "à¸Ħำ" + ], + "decoded_with_special": "สวัสดีชาวโลก! นี่คือการทดสอบเครื่องตัดคำ", + "decoded_skip_special": "สวัสดีชาวโลก! นี่คือการทดสอบเครื่องตัดคำ" + }, + { + "id": "thai-combining-marks-prose", + "input_ids": [ + 127382, + 125451, + 86032, + 18625, + 124207, + 19841, + 48120, + 124961, + 37213, + 123958, + 129778, + 123958, + 125506 + ], + "tokens": [ + "à¸łà¸²à¸©à¸²", + "à¹Ħà¸Ĺย", + "à¹Ħม", + "à¹Ī", + "à¹ĥà¸Ĭ", + "à¹ī", + "à¸Ĭ", + "à¹Īà¸Ńà¸ĩ", + "ว", + "à¹Īาà¸ĩ", + "ระหว", + "à¹Īาà¸ĩ", + "à¸Ħำ" + ], + "decoded_with_special": "ภาษาไทยไม่ใช้ช่องว่างระหว่างคำ", + "decoded_skip_special": "ภาษาไทยไม่ใช้ช่องว่างระหว่างคำ" + }, + { + "id": "mixed-script-cjk-latin", + "input_ids": [ + 785, + 94305, + 245, + 46553, + 8411, + 304, + 220, + 17, + 15, + 17, + 21, + 1959, + 24901, + 14370, + 0 + ], + "tokens": [ + "The", + "ĠåĮ", + "Ĺ", + "京", + "Ġtrip", + "Ġin", + "Ġ", + "2", + "0", + "2", + "6", + "ĠâĢĶ", + "Ġtrès", + "Ġbien", + "!" + ], + "decoded_with_special": "The 北京 trip in 2026 — très bien!", + "decoded_skip_special": "The 北京 trip in 2026 — très bien!" + }, + { + "id": "mixed-script-katakana-code", + "input_ids": [ + 140653, + 137267, + 25, + 9876, + 8356, + 304, + 5266, + 264, + 18, + 69, + 24, + 66, + 17, + 65, + 13 + ], + "tokens": [ + "ãĤ³ãĥ¼ãĥī", + "ãĥ¬ãĥĵãĥ¥ãĥ¼", + ":", + "Ġbug", + "Ġfixed", + "Ġin", + "Ġcommit", + "Ġa", + "3", + "f", + "9", + "c", + "2", + "b", + "." + ], + "decoded_with_special": "コードレビュー: bug fixed in commit a3f9c2b.", + "decoded_skip_special": "コードレビュー: bug fixed in commit a3f9c2b." + }, + { + "id": "math-symbols-summation", + "input_ids": [ + 2077, + 25, + 7851, + 96, + 856, + 5318, + 284, + 220, + 19, + 17, + 320, + 2870, + 856, + 48335, + 86023, + 568 + ], + "tokens": [ + "Result", + ":", + "ĠÎ", + "£", + "Ġx", + "_i", + "Ġ=", + "Ġ", + "4", + "2", + "Ġ(", + "where", + "Ġx", + "ĠâĪĪ", + "ĠâĦĿ", + ")." + ], + "decoded_with_special": "Result: Σ x_i = 42 (where x ∈ ℝ).", + "decoded_skip_special": "Result: Σ x_i = 42 (where x ∈ ℝ)." + }, + { + "id": "math-symbols-epsilon-delta", + "input_ids": [ + 785, + 13173, + 25, + 54700, + 30143, + 29, + 15, + 11995, + 225, + 85386, + 29, + 15, + 1741, + 429, + 760, + 87, + 6558, + 89669, + 91, + 27, + 85386, + 58703, + 760, + 69, + 2075, + 7287, + 69, + 2075, + 89669, + 17935, + 27, + 30143, + 13 + ], + "tokens": [ + "The", + "orem", + ":", + "ĠâĪĢ", + "ε", + ">", + "0", + "ĠâĪ", + "ĥ", + "δ", + ">", + "0", + "Ġsuch", + "Ġthat", + "Ġ|", + "x", + "-x", + "âĤĢ", + "|", + "<", + "δ", + "ĠâĩĴ", + "Ġ|", + "f", + "(x", + ")-", + "f", + "(x", + "âĤĢ", + ")|", + "<", + "ε", + "." + ], + "decoded_with_special": "Theorem: ∀ε>0 ∃δ>0 such that |x-x₀|<δ ⇒ |f(x)-f(x₀)|<ε.", + "decoded_skip_special": "Theorem: ∀ε>0 ∃δ>0 such that |x-x₀|<δ ⇒ |f(x)-f(x₀)|<ε." + }, + { + "id": "currency-symbols-multi", + "input_ids": [ + 14940, + 25, + 12984, + 16, + 17, + 13, + 20, + 15, + 11397, + 71488, + 16, + 11, + 23, + 17, + 15, + 21103, + 230, + 6938, + 16, + 15, + 13, + 22, + 20, + 320, + 48053, + 11, + 5519, + 2858, + 224, + 123, + 15, + 13, + 15, + 15, + 15, + 19, + 568 + ], + "tokens": [ + "Cost", + ":", + "ĠâĤ¬", + "1", + "2", + ".", + "5", + "0", + "ĠâĨĴ", + "ĠÂ¥", + "1", + ",", + "8", + "2", + "0", + "Ġâī", + "Ī", + "Ġ£", + "1", + "0", + ".", + "7", + "5", + "Ġ(", + "approx", + ",", + "Ġplus", + "Ġâ", + "Ĥ", + "¿", + "0", + ".", + "0", + "0", + "0", + "4", + ")." + ], + "decoded_with_special": "Cost: €12.50 → ¥1,820 ≈ £10.75 (approx, plus ₿0.0004).", + "decoded_skip_special": "Cost: €12.50 → ¥1,820 ≈ £10.75 (approx, plus ₿0.0004)." + }, + { + "id": "box-drawing", + "input_ids": [ + 1611, + 25, + 22612, + 234, + 32323, + 16991, + 14555, + 145624, + 33391, + 21927, + 33391, + 22612, + 242, + 32323, + 16991, + 14555, + 145474, + 1959, + 2814, + 13 + ], + "tokens": [ + "Box", + ":", + "ĠâĶ", + "Į", + "âĶĢâĶĢâĶĢâĶĢ", + "âĶĢâĶĢ", + "âĶĢ", + "âĶIJ", + "ĠâĶĤ", + "ĠHello", + "ĠâĶĤ", + "ĠâĶ", + "Ķ", + "âĶĢâĶĢâĶĢâĶĢ", + "âĶĢâĶĢ", + "âĶĢ", + "âĶĺ", + "ĠâĢĶ", + "Ġdone", + "." + ], + "decoded_with_special": "Box: ┌───────┐ │ Hello │ └───────┘ — done.", + "decoded_skip_special": "Box: ┌───────┐ │ Hello │ └───────┘ — done." + }, + { + "id": "dingbats-chess", + "input_ids": [ + 91225, + 25, + 37234, + 92848, + 35358, + 254, + 35358, + 96, + 67579, + 35358, + 99, + 35358, + 242, + 35358, + 243, + 35358, + 244, + 35358, + 246, + 35358, + 247, + 1959, + 32719, + 738, + 13 + ], + "tokens": [ + "Pieces", + ":", + "Ġâĺħ", + "ĠâĺĨ", + "ĠâĻ", + "ł", + "ĠâĻ", + "£", + "ĠâĻ¥", + "ĠâĻ", + "¦", + "ĠâĻ", + "Ķ", + "ĠâĻ", + "ķ", + "ĠâĻ", + "ĸ", + "ĠâĻ", + "ĺ", + "ĠâĻ", + "Ļ", + "ĠâĢĶ", + "Ġchess", + "Ġset", + "." + ], + "decoded_with_special": "Pieces: ★ ☆ ♠ ♣ ♥ ♦ ♔ ♕ ♖ ♘ ♙ — chess set.", + "decoded_skip_special": "Pieces: ★ ☆ ♠ ♣ ♥ ♦ ♔ ♕ ♖ ♘ ♙ — chess set." + }, + { + "id": "math-astral-bold-script", + "input_ids": [ + 8815, + 13939, + 25, + 81250, + 238, + 222, + 147822, + 147382, + 147823, + 220, + 56252, + 253, + 236, + 56252, + 253, + 237, + 56252, + 253, + 238, + 56252, + 253, + 239, + 1959, + 6888, + 5316, + 25, + 81250, + 241, + 238, + 123871, + 239, + 151316, + 123871, + 241, + 13 + ], + "tokens": [ + "Math", + "Ġbold", + ":", + "ĠðĿ", + "IJ", + "Ģ", + "ðĿIJģ", + "ðĿIJĤ", + "ðĿIJĥ", + "Ġ", + "ðĿ", + "Ł", + "İ", + "ðĿ", + "Ł", + "ı", + "ðĿ", + "Ł", + "IJ", + "ðĿ", + "Ł", + "ij", + "ĠâĢĶ", + "Ġmath", + "Ġscript", + ":", + "ĠðĿ", + "ĵ", + "IJ", + "ðĿĵ", + "ij", + "ðĿĵĴ", + "ðĿĵ", + "ĵ", + "." + ], + "decoded_with_special": "Math bold: 𝐀𝐁𝐂𝐃 𝟎𝟏𝟐𝟑 — math script: 𝓐𝓑𝓒𝓓.", + "decoded_skip_special": "Math bold: 𝐀𝐁𝐂𝐃 𝟎𝟏𝟐𝟑 — math script: 𝓐𝓑𝓒𝓓." + }, + { + "id": "astral-egyptian-hieroglyphs", + "input_ids": [ + 85667, + 1103, + 12412, + 538, + 9525, + 82, + 25, + 220, + 124373, + 222, + 222, + 220, + 148864, + 222, + 220, + 128241, + 222, + 220, + 124373, + 228, + 236, + 150195, + 241, + 128234, + 237, + 124373, + 232, + 244, + 13 + ], + "tokens": [ + "Egypt", + "ian", + "Ġhier", + "og", + "lyph", + "s", + ":", + "Ġ", + "ðĵ", + "Ģ", + "Ģ", + "Ġ", + "ðĵĤ", + "Ģ", + "Ġ", + "ðĵĥ", + "Ģ", + "Ġ", + "ðĵ", + "Ĩ", + "İ", + "ðĵħ", + "ĵ", + "ðĵı", + "ı", + "ðĵ", + "Ĭ", + "ĸ", + "." + ], + "decoded_with_special": "Egyptian hieroglyphs: 𓀀 𓂀 𓃀 𓆎𓅓𓏏𓊖.", + "decoded_skip_special": "Egyptian hieroglyphs: 𓀀 𓂀 𓃀 𓆎𓅓𓏏𓊖." + }, + { + "id": "astral-cuneiform", + "input_ids": [ + 34, + 2886, + 7250, + 25, + 220, + 151268, + 255, + 220, + 125003, + 230, + 254, + 220, + 125003, + 224, + 245, + 220, + 125003, + 228, + 254, + 1959, + 1841, + 1140, + 10469, + 13 + ], + "tokens": [ + "C", + "une", + "iform", + ":", + "Ġ", + "ðĴĢ", + "Ń", + "Ġ", + "ðĴ", + "Ī", + "ł", + "Ġ", + "ðĴ", + "Ĥ", + "Ĺ", + "Ġ", + "ðĴ", + "Ĩ", + "ł", + "ĠâĢĶ", + "Ġsign", + "Ġlist", + "Ġsamples", + "." + ], + "decoded_with_special": "Cuneiform: 𒀭 𒈠 𒂗 𒆠 — sign list samples.", + "decoded_skip_special": "Cuneiform: 𒀭 𒈠 𒂗 𒆠 — sign list samples." + }, + { + "id": "astral-mahjong", + "input_ids": [ + 46208, + 98460, + 20493, + 25, + 11162, + 222, + 222, + 11162, + 222, + 223, + 11162, + 222, + 224, + 11162, + 222, + 225, + 11162, + 222, + 226, + 11162, + 222, + 227, + 11162, + 222, + 228, + 11162, + 222, + 229, + 13 + ], + "tokens": [ + "Mah", + "jong", + "Ġtiles", + ":", + "ĠðŁ", + "Ģ", + "Ģ", + "ĠðŁ", + "Ģ", + "ģ", + "ĠðŁ", + "Ģ", + "Ĥ", + "ĠðŁ", + "Ģ", + "ĥ", + "ĠðŁ", + "Ģ", + "Ħ", + "ĠðŁ", + "Ģ", + "ħ", + "ĠðŁ", + "Ģ", + "Ĩ", + "ĠðŁ", + "Ģ", + "ĩ", + "." + ], + "decoded_with_special": "Mahjong tiles: 🀀 🀁 🀂 🀃 🀄 🀅 🀆 🀇.", + "decoded_skip_special": "Mahjong tiles: 🀀 🀁 🀂 🀃 🀄 🀅 🀆 🀇." + }, + { + "id": "astral-cards", + "input_ids": [ + 28448, + 25, + 11162, + 36109, + 11162, + 225, + 224, + 11162, + 225, + 225, + 11162, + 224, + 94, + 11162, + 224, + 95, + 11162, + 224, + 96, + 13 + ], + "tokens": [ + "Cards", + ":", + "ĠðŁ", + "ĥģ", + "ĠðŁ", + "ĥ", + "Ĥ", + "ĠðŁ", + "ĥ", + "ĥ", + "ĠðŁ", + "Ĥ", + "¡", + "ĠðŁ", + "Ĥ", + "¢", + "ĠðŁ", + "Ĥ", + "£", + "." + ], + "decoded_with_special": "Cards: 🃁 🃂 🃃 🂡 🂢 🂣.", + "decoded_skip_special": "Cards: 🃁 🃂 🃃 🂡 🂢 🂣." + }, + { + "id": "emoji-bmp-and-astral", + "input_ids": [ + 145707, + 21927, + 0, + 11162, + 234, + 235, + 4337, + 11162, + 248, + 222, + 7050, + 11162, + 236, + 231, + 4614, + 11162, + 236, + 224, + 13 + ], + "tokens": [ + "ðŁijĭ", + "ĠHello", + "!", + "ĠðŁ", + "Į", + "į", + "ĠWorld", + "ĠðŁ", + "ļ", + "Ģ", + "Ġlaunch", + "ĠðŁ", + "İ", + "ī", + "Ġparty", + "ĠðŁ", + "İ", + "Ĥ", + "." + ], + "decoded_with_special": "👋 Hello! 🌍 World 🚀 launch 🎉 party 🎂.", + "decoded_skip_special": "👋 Hello! 🌍 World 🚀 launch 🎉 party 🎂." + }, + { + "id": "emoji-zwj-family-pride-skin", + "input_ids": [ + 15192, + 25, + 61804, + 101, + 378, + 235, + 145233, + 378, + 235, + 145665, + 378, + 235, + 145988, + 1959, + 42146, + 5181, + 25, + 11162, + 237, + 111, + 30543, + 378, + 235, + 145354, + 1959, + 26951, + 97936, + 25, + 61804, + 235, + 145375, + 25521, + 232, + 146530, + 61804, + 114, + 144321, + 13 + ], + "tokens": [ + "Family", + ":", + "ĠðŁij", + "¨", + "âĢ", + "į", + "ðŁij©", + "âĢ", + "į", + "ðŁij§", + "âĢ", + "į", + "ðŁij¦", + "ĠâĢĶ", + "ĠPride", + "Ġflag", + ":", + "ĠðŁ", + "ı", + "³", + "ï¸ı", + "âĢ", + "į", + "ðŁĮĪ", + "ĠâĢĶ", + "ĠSkin", + "-tone", + ":", + "ĠðŁij", + "į", + "ðŁı½", + "Ġâľ", + "Ĭ", + "ðŁı¿", + "ĠðŁij", + "¶", + "ðŁı»", + "." + ], + "decoded_with_special": "Family: 👨‍👩‍👧‍👦 — Pride flag: 🏳️‍🌈 — Skin-tone: 👍🏽 ✊🏿 👶🏻.", + "decoded_skip_special": "Family: 👨‍👩‍👧‍👦 — Pride flag: 🏳️‍🌈 — Skin-tone: 👍🏽 ✊🏿 👶🏻." + }, + { + "id": "emoji-zwj-with-text-prefix", + "input_ids": [ + 147233, + 30543, + 378, + 235, + 145354, + 47, + 1399, + 10, + 145367, + 378, + 235, + 145233, + 378, + 235, + 145665, + 378, + 235, + 145988, + 15192, + 10, + 145862, + 145336, + 146131, + 145516, + 145526, + 145383, + 11161, + 10, + 1944 + ], + "tokens": [ + "ðŁı³", + "ï¸ı", + "âĢ", + "į", + "ðŁĮĪ", + "P", + "ride", + "+", + "ðŁij¨", + "âĢ", + "į", + "ðŁij©", + "âĢ", + "į", + "ðŁij§", + "âĢ", + "į", + "ðŁij¦", + "Family", + "+", + "ðŁĩ©", + "ðŁĩª", + "ðŁĩ¯", + "ðŁĩµ", + "ðŁĩº", + "ðŁĩ¸", + "flags", + "+", + "test" + ], + "decoded_with_special": "🏳️‍🌈Pride+👨‍👩‍👧‍👦Family+🇩🇪🇯🇵🇺🇸flags+test", + "decoded_skip_special": "🏳️‍🌈Pride+👨‍👩‍👧‍👦Family+🇩🇪🇯🇵🇺🇸flags+test" + }, + { + "id": "emoji-keycap-and-flags", + "input_ids": [ + 1592, + 48233, + 25, + 220, + 16, + 30543, + 158, + 225, + 96, + 220, + 17, + 30543, + 158, + 225, + 96, + 220, + 18, + 30543, + 158, + 225, + 96, + 1959, + 5181, + 25, + 11162, + 229, + 107, + 145516, + 11162, + 229, + 102, + 145336, + 11162, + 229, + 118, + 145383, + 13 + ], + "tokens": [ + "Key", + "caps", + ":", + "Ġ", + "1", + "ï¸ı", + "â", + "ĥ", + "£", + "Ġ", + "2", + "ï¸ı", + "â", + "ĥ", + "£", + "Ġ", + "3", + "ï¸ı", + "â", + "ĥ", + "£", + "ĠâĢĶ", + "Ġflag", + ":", + "ĠðŁ", + "ĩ", + "¯", + "ðŁĩµ", + "ĠðŁ", + "ĩ", + "©", + "ðŁĩª", + "ĠðŁ", + "ĩ", + "º", + "ðŁĩ¸", + "." + ], + "decoded_with_special": "Keycaps: 1️⃣ 2️⃣ 3️⃣ — flag: 🇯🇵 🇩🇪 🇺🇸.", + "decoded_skip_special": "Keycaps: 1️⃣ 2️⃣ 3️⃣ — flag: 🇯🇵 🇩🇪 🇺🇸." + }, + { + "id": "url-with-query-fragment", + "input_ids": [ + 1085, + 51999, + 1110, + 8687, + 905, + 50976, + 32429, + 53100, + 43782, + 28, + 7975, + 5, + 2257, + 22086, + 1370, + 5, + 77, + 28, + 19, + 17, + 2, + 2809, + 12, + 18 + ], + "tokens": [ + "url", + "=https", + "://", + "example", + ".com", + "/path", + "/to", + "/resource", + "?q", + "=", + "foo", + "&", + "bar", + "=b", + "az", + "&", + "n", + "=", + "4", + "2", + "#", + "section", + "-", + "3" + ], + "decoded_with_special": "url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3", + "decoded_skip_special": "url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3" + }, + { + "id": "hex-and-base64", + "input_ids": [ + 17308, + 28, + 15, + 78979, + 94373, + 7099, + 37, + 81318, + 8264, + 1867, + 36, + 2331, + 21, + 19, + 91994, + 54, + 41, + 73, + 57, + 39718, + 76, + 57, + 17, + 21197, + 309, + 2576, + 65, + 54, + 20, + 7362, + 32, + 418 + ], + "tokens": [ + "hex", + "=", + "0", + "xDE", + "ADB", + "EE", + "F", + "CAF", + "EB", + "AB", + "E", + "Ġbase", + "6", + "4", + "=Y", + "W", + "J", + "j", + "Z", + "GV", + "m", + "Z", + "2", + "hp", + "am", + "ts", + "b", + "W", + "5", + "vc", + "A", + "==" + ], + "decoded_with_special": "hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==", + "decoded_skip_special": "hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==" + }, + { + "id": "sha256-hash", + "input_ids": [ + 15247, + 17, + 20, + 21, + 40391, + 18, + 65, + 15, + 66, + 19, + 19, + 17, + 24, + 23, + 8316, + 16, + 66, + 16, + 19, + 24, + 2577, + 13233, + 19, + 66, + 23, + 24, + 24, + 21, + 10798, + 24, + 17, + 19, + 17, + 22, + 5918, + 19, + 16, + 68, + 19, + 21, + 19, + 24, + 65, + 24, + 18, + 19, + 924, + 19, + 24, + 20, + 24, + 24, + 16, + 65, + 22, + 23, + 20, + 17, + 65, + 23, + 20, + 20 + ], + "tokens": [ + "sha", + "2", + "5", + "6", + "=e", + "3", + "b", + "0", + "c", + "4", + "4", + "2", + "9", + "8", + "fc", + "1", + "c", + "1", + "4", + "9", + "af", + "bf", + "4", + "c", + "8", + "9", + "9", + "6", + "fb", + "9", + "2", + "4", + "2", + "7", + "ae", + "4", + "1", + "e", + "4", + "6", + "4", + "9", + "b", + "9", + "3", + "4", + "ca", + "4", + "9", + "5", + "9", + "9", + "1", + "b", + "7", + "8", + "5", + "2", + "b", + "8", + "5", + "5" + ], + "decoded_with_special": "sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "decoded_skip_special": "sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "id": "code-javascript-functional", + "input_ids": [ + 1118, + 4770, + 2075, + 589, + 856, + 353, + 220, + 17, + 568, + 5315, + 2075, + 589, + 856, + 861, + 220, + 15, + 568, + 26273, + 1188, + 64, + 11, + 293, + 8, + 589, + 264, + 488, + 293, + 1215 + ], + "tokens": [ + "arr", + ".map", + "(x", + "Ġ=>", + "Ġx", + "Ġ*", + "Ġ", + "2", + ").", + "filter", + "(x", + "Ġ=>", + "Ġx", + "Ġ>", + "Ġ", + "0", + ").", + "reduce", + "((", + "a", + ",", + "Ġb", + ")", + "Ġ=>", + "Ġa", + "Ġ+", + "Ġb", + ");" + ], + "decoded_with_special": "arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);", + "decoded_skip_special": "arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);" + }, + { + "id": "code-cplusplus-template", + "input_ids": [ + 4214, + 3125, + 50481, + 29, + 3233, + 282, + 4140, + 7672, + 83, + 11, + 547, + 7672, + 84, + 8, + 1464, + 75853, + 1155, + 10, + 84, + 8, + 314, + 470, + 259, + 488, + 575, + 26, + 335 + ], + "tokens": [ + "template", + "", + "Ġauto", + "Ġf", + "(T", + "&&", + "t", + ",", + "ĠU", + "&&", + "u", + ")", + "Ġ->", + "Ġdecltype", + "(t", + "+", + "u", + ")", + "Ġ{", + "Ġreturn", + "Ġt", + "Ġ+", + "Ġu", + ";", + "Ġ}" + ], + "decoded_with_special": "template auto f(T&&t, U&&u) -> decltype(t+u) { return t + u; }", + "decoded_skip_special": "template auto f(T&&t, U&&u) -> decltype(t+u) { return t + u; }" + }, + { + "id": "code-bitops", + "input_ids": [ + 1382, + 284, + 320, + 64, + 3578, + 220, + 17, + 8, + 760, + 320, + 65, + 1115, + 220, + 19, + 8, + 609, + 220, + 15, + 9264, + 26 + ], + "tokens": [ + "result", + "Ġ=", + "Ġ(", + "a", + "Ġ>>", + "Ġ", + "2", + ")", + "Ġ|", + "Ġ(", + "b", + "Ġ<<", + "Ġ", + "4", + ")", + "Ġ&", + "Ġ", + "0", + "xFF", + ";" + ], + "decoded_with_special": "result = (a >> 2) | (b << 4) & 0xFF;", + "decoded_skip_special": "result = (a >> 2) | (b << 4) & 0xFF;" + }, + { + "id": "code-javascript-modern", + "input_ids": [ + 333, + 320, + 87, + 4376, + 845, + 1009, + 379, + 2604, + 220, + 15, + 8, + 314, + 470, + 856, + 4607, + 7975, + 9415, + 20163, + 26, + 335 + ], + "tokens": [ + "if", + "Ġ(", + "x", + "Ġ!==", + "Ġnull", + "Ġ&&", + "Ġy", + "Ġ>=", + "Ġ", + "0", + ")", + "Ġ{", + "Ġreturn", + "Ġx", + "?.", + "foo", + "Ġ??", + "ĠdefaultValue", + ";", + "Ġ}" + ], + "decoded_with_special": "if (x !== null && y >= 0) { return x?.foo ?? defaultValue; }", + "decoded_skip_special": "if (x !== null && y >= 0) { return x?.foo ?? defaultValue; }" + }, + { + "id": "code-python-def", + "input_ids": [ + 750, + 75698, + 1445, + 1648 + ], + "tokens": [ + "def", + "Ġfibonacci", + "(n", + "):" + ], + "decoded_with_special": "def fibonacci(n):", + "decoded_skip_special": "def fibonacci(n):" + }, + { + "id": "code-python-if", + "input_ids": [ + 262, + 421, + 308, + 366, + 220, + 17, + 25 + ], + "tokens": [ + "ĠĠĠ", + "Ġif", + "Ġn", + "Ġ<", + "Ġ", + "2", + ":" + ], + "decoded_with_special": " if n < 2:", + "decoded_skip_special": " if n < 2:" + }, + { + "id": "code-python-return", + "input_ids": [ + 286, + 470, + 308 + ], + "tokens": [ + "ĠĠĠĠĠĠĠ", + "Ġreturn", + "Ġn" + ], + "decoded_with_special": " return n", + "decoded_skip_special": " return n" + }, + { + "id": "code-python-recurse", + "input_ids": [ + 262, + 470, + 75698, + 1445, + 481, + 220, + 16, + 8, + 488, + 75698, + 1445, + 481, + 220, + 17, + 8 + ], + "tokens": [ + "ĠĠĠ", + "Ġreturn", + "Ġfibonacci", + "(n", + "Ġ-", + "Ġ", + "1", + ")", + "Ġ+", + "Ġfibonacci", + "(n", + "Ġ-", + "Ġ", + "2", + ")" + ], + "decoded_with_special": " return fibonacci(n - 1) + fibonacci(n - 2)", + "decoded_skip_special": " return fibonacci(n - 1) + fibonacci(n - 2)" + }, + { + "id": "code-go-func", + "input_ids": [ + 2830, + 40786, + 3153, + 914, + 8, + 314 + ], + "tokens": [ + "func", + "Ġgreet", + "(name", + "Ġstring", + ")", + "Ġ{" + ], + "decoded_with_special": "func greet(name string) {", + "decoded_skip_special": "func greet(name string) {" + }, + { + "id": "code-go-tab-if", + "input_ids": [ + 743, + 829, + 961, + 1591, + 314 + ], + "tokens": [ + "ĉif", + "Ġname", + "Ġ!=", + "Ġ\"\"", + "Ġ{" + ], + "decoded_with_special": "\tif name != \"\" {", + "decoded_skip_special": "\tif name != \"\" {" + }, + { + "id": "code-go-tab-printf", + "input_ids": [ + 197, + 11009, + 19367, + 445, + 9707, + 11, + 1018, + 82, + 14771, + 77, + 497, + 829, + 8 + ], + "tokens": [ + "ĉ", + "ĉfmt", + ".Printf", + "(\"", + "Hello", + ",", + "Ġ%", + "s", + "!\\", + "n", + "\",", + "Ġname", + ")" + ], + "decoded_with_special": "\t\tfmt.Printf(\"Hello, %s!\\n\", name)", + "decoded_skip_special": "\t\tfmt.Printf(\"Hello, %s!\\n\", name)" + }, + { + "id": "code-go-tab-close", + "input_ids": [ + 197, + 92 + ], + "tokens": [ + "ĉ", + "}" + ], + "decoded_with_special": "\t}", + "decoded_skip_special": "\t}" + }, + { + "id": "code-go-close", + "input_ids": [ + 92 + ], + "tokens": [ + "}" + ], + "decoded_with_special": "}", + "decoded_skip_special": "}" + }, + { + "id": "programming-identifiers-mixed", + "input_ids": [ + 455, + 3317, + 842, + 8334, + 14394, + 4098, + 350, + 30290, + 6458, + 842, + 39863, + 5283, + 7688 + ], + "tokens": [ + "get", + "_user", + "_id", + "ĠMAX", + "_BUFFER", + "_SIZE", + "ĠT", + "_pad", + "_token", + "_id", + "ĠNSURL", + "Session", + "Configuration" + ], + "decoded_with_special": "get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration", + "decoded_skip_special": "get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration" + }, + { + "id": "programming-identifiers-cases", + "input_ids": [ + 93321, + 4207, + 3962, + 57359, + 4207, + 25265, + 19096, + 4612, + 7531, + 15915, + 1718, + 1098, + 7326, + 3390, + 595, + 23, + 82, + 85337, + 3180, + 2273, + 17 + ], + "tokens": [ + "camel", + "Case", + "Var", + "ĠPascal", + "Case", + "Ġsnake", + "_case", + "_var", + "ĠSC", + "REAM", + "ING", + "_S", + "NA", + "KE", + "Ġk", + "8", + "s", + "_pod", + "_count", + "_v", + "2" + ], + "decoded_with_special": "camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2", + "decoded_skip_special": "camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2" + }, + { + "id": "code-string-multiscript", + "input_ids": [ + 70, + 43632, + 284, + 330, + 53645, + 26991, + 8178, + 11, + 137144, + 8958, + 488, + 330, + 1959, + 504, + 330, + 488, + 330, + 102356, + 46553, + 1 + ], + "tokens": [ + "g", + "reeting", + "Ġ=", + "Ġ\"", + "ÐŁÑĢ", + "ив", + "еÑĤ", + ",", + "ĠмиÑĢ", + "!\"", + "Ġ+", + "Ġ\"", + "ĠâĢĶ", + "Ġfrom", + "Ġ\"", + "Ġ+", + "Ġ\"", + "æĿ±", + "京", + "\"" + ], + "decoded_with_special": "greeting = \"Привет, мир!\" + \" — from \" + \"東京\"", + "decoded_skip_special": "greeting = \"Привет, мир!\" + \" — from \" + \"東京\"" + }, + { + "id": "code-comment-diacritics", + "input_ids": [ + 322, + 9333, + 1242, + 963, + 1959, + 23670, + 3980, + 448, + 1853, + 580, + 1003, + 1211 + ], + "tokens": [ + "//", + "Ġré", + "sum", + "é", + "ĠâĢĶ", + "ĠSwift", + "Ġcomment", + "Ġwith", + "Ġdi", + "ac", + "rit", + "ics" + ], + "decoded_with_special": "// résumé — Swift comment with diacritics", + "decoded_skip_special": "// résumé — Swift comment with diacritics" + }, + { + "id": "code-comment-greek-math", + "input_ids": [ + 1057, + 19043, + 488, + 33218, + 284, + 62027, + 1959, + 17860, + 79368, + 6888, + 3980, + 639 + ], + "tokens": [ + "/*", + "Ġα", + "Ġ+", + "Ġβ", + "Ġ=", + "Ġγ", + "ĠâĢĶ", + "ĠGreek", + "-letter", + "Ġmath", + "Ġcomment", + "Ġ*/" + ], + "decoded_with_special": "/* α + β = γ — Greek-letter math comment */", + "decoded_skip_special": "/* α + β = γ — Greek-letter math comment */" + }, + { + "id": "code-string-multiscript-emoji", + "input_ids": [ + 1149, + 2265, + 284, + 330, + 145836, + 23205, + 1959, + 5063, + 43055, + 4793, + 1959, + 8908, + 113, + 115, + 90775, + 1 + ], + "tokens": [ + "let", + "Ġtitle", + "Ġ=", + "Ġ\"", + "ðŁļĢ", + "ĠLaunch", + "ĠâĢĶ", + "Ġп", + "ÑĥÑģ", + "к", + "ĠâĢĶ", + "Ġè", + "µ", + "·", + "åĭķ", + "\"" + ], + "decoded_with_special": "let title = \"🚀 Launch — пуск — 起動\"", + "decoded_skip_special": "let title = \"🚀 Launch — пуск — 起動\"" + }, + { + "id": "ipa-phonetic-transcription", + "input_ids": [ + 785, + 55747, + 45840, + 608, + 69, + 145076, + 145388, + 14, + 369, + 330, + 18170, + 1, + 1959, + 448, + 8464, + 25, + 220, + 144936, + 69, + 145076, + 145388, + 13 + ], + "tokens": [ + "The", + "ĠIPA", + "Ġtranscription", + "Ġ/", + "f", + "ɪ", + "Êĥ", + "/", + "Ġfor", + "Ġ\"", + "fish", + "\"", + "ĠâĢĶ", + "Ġwith", + "Ġstress", + ":", + "Ġ", + "ËĪ", + "f", + "ɪ", + "Êĥ", + "." + ], + "decoded_with_special": "The IPA transcription /fɪʃ/ for \"fish\" — with stress: ˈfɪʃ.", + "decoded_skip_special": "The IPA transcription /fɪʃ/ for \"fish\" — with stress: ˈfɪʃ." + }, + { + "id": "mandarin-tones", + "input_ids": [ + 51, + 3154, + 25, + 28730, + 296, + 144824, + 126199, + 296, + 30657, + 320, + 44, + 437, + 42740, + 41976, + 1959, + 1550, + 11, + 16062, + 11, + 85821, + 11, + 15679, + 568 + ], + "tokens": [ + "T", + "ones", + ":", + "Ġmá", + "Ġm", + "Çİ", + "ĠmÃł", + "Ġm", + "Äģ", + "Ġ(", + "M", + "and", + "arin", + "Ġtones", + "ĠâĢĶ", + "Ġhigh", + ",", + "Ġrising", + ",", + "Ġdipping", + ",", + "Ġfalling", + ")." + ], + "decoded_with_special": "Tones: má mǎ mà mā (Mandarin tones — high, rising, dipping, falling).", + "decoded_skip_special": "Tones: má mǎ mà mā (Mandarin tones — high, rising, dipping, falling)." + }, + { + "id": "whitespace-runs", + "input_ids": [ + 14990, + 262, + 1879, + 260, + 659, + 981, + 835 + ], + "tokens": [ + "hello", + "ĠĠĠ", + "Ġworld", + "ĠĠĠĠĠĠĠĠ", + "Ġ.", + "ĠĠĠĠĠĠ", + "Ġend" + ], + "decoded_with_special": "hello world . end", + "decoded_skip_special": "hello world . end" + }, + { + "id": "whitespace-trailing-tabs", + "input_ids": [ + 376, + 14277, + 36372, + 1573, + 39027, + 25, + 197, + 27845, + 8, + 256, + 320, + 44285, + 8 + ], + "tokens": [ + "tr", + "ailing", + "Ġwhitespace", + "Ġbefore", + "Ġnewline", + ":", + "ĉ", + "(tab", + ")", + "ĠĠ", + "Ġ(", + "spaces", + ")" + ], + "decoded_with_special": "trailing whitespace before newline:\t(tab) (spaces)", + "decoded_skip_special": "trailing whitespace before newline:\t(tab) (spaces)" + }, + { + "id": "punctuation-period", + "input_ids": [ + 13 + ], + "tokens": [ + "." + ], + "decoded_with_special": ".", + "decoded_skip_special": "." + }, + { + "id": "punctuation-comma", + "input_ids": [ + 11 + ], + "tokens": [ + "," + ], + "decoded_with_special": ",", + "decoded_skip_special": "," + }, + { + "id": "punctuation-exclamation", + "input_ids": [ + 0 + ], + "tokens": [ + "!" + ], + "decoded_with_special": "!", + "decoded_skip_special": "!" + }, + { + "id": "punctuation-nested-quotes", + "input_ids": [ + 30337, + 7521, + 1340, + 4588, + 1959, + 330, + 53660, + 30, + 8958, + 1959, + 9834, + 1112 + ], + "tokens": [ + "\"What", + "?\"", + "Ġshe", + "Ġasked", + "ĠâĢĶ", + "Ġ\"", + "really", + "?", + "!\"", + "ĠâĢĶ", + "Ġyes", + "..." + ], + "decoded_with_special": "\"What?\" she asked — \"really?!\" — yes...", + "decoded_skip_special": "\"What?\" she asked — \"really?!\" — yes..." + }, + { + "id": "punctuation-dash-variants", + "input_ids": [ + 1112, + 323, + 636, + 240, + 1083, + 1365, + 323, + 1959, + 323, + 50653, + 13 + ], + "tokens": [ + "...", + "Ġand", + "ĠâĢ", + "Ĵ", + "Ġalso", + "ĠâĢĵ", + "Ġand", + "ĠâĢĶ", + "Ġand", + "ĠâĢķ", + "." + ], + "decoded_with_special": "... and ‒ also – and — and ―.", + "decoded_skip_special": "... and ‒ also – and — and ―." + }, + { + "id": "multiscript-greetings", + "input_ids": [ + 9707, + 220, + 108386, + 95170, + 144370, + 220, + 89015, + 14925, + 101, + 87244, + 78368, + 30484, + 97, + 34370, + 23364, + 126860, + 124671, + 124756, + 123881, + 129328, + 37213, + 23271, + 125136, + 28319, + 79484, + 26991, + 8178, + 7851, + 248, + 18945, + 33486, + 41424, + 43123, + 79431, + 38079, + 18945, + 13 + ], + "tokens": [ + "Hello", + "Ġ", + "ä½łå¥½", + "ĠìķĪ", + "ëħķ", + "Ġ", + "ãģĵãĤĵãģ«ãģ¡ãģ¯", + "Ġà¤", + "¨", + "म", + "स", + "à¥įà¤", + "¤", + "à¥ĩ", + "ĠÙħ", + "رØŃ", + "با", + "Ġש׾", + "×ķ×Ŀ", + "Ġส", + "ว", + "ั", + "สà¸Ķ", + "ี", + "ĠÐŁÑĢ", + "ив", + "еÑĤ", + "ĠÎ", + "ļ", + "α", + "λ", + "η", + "μ", + "ÎŃ", + "Ïģ", + "α", + "." + ], + "decoded_with_special": "Hello 你好 안녕 こんにちは नमस्ते مرحبا שלום สวัสดี Привет Καλημέρα.", + "decoded_skip_special": "Hello 你好 안녕 こんにちは नमस्ते مرحبا שלום สวัสดี Привет Καλημέρα." + }, + { + "id": "multiscript-no-spaces", + "input_ids": [ + 18945, + 56568, + 51255, + 126246, + 59374, + 60096, + 85386, + 78368, + 145207, + 23224, + 145707, + 58411, + 145836, + 14293, + 144192, + 149026, + 145690, + 144016 + ], + "tokens": [ + "α", + "ä½ł", + "β", + "ìķĪ", + "γ", + "न", + "δ", + "स", + "ðŁĴ«", + "ع", + "ðŁijĭ", + "ש", + "ðŁļĢ", + "ت", + "âĪĢ", + "âĪĥ", + "âĪĪ", + "âĩĴ" + ], + "decoded_with_special": "α你β안γनδस💫ع👋ש🚀ت∀∃∈⇒", + "decoded_skip_special": "α你β안γनδस💫ع👋ש🚀ت∀∃∈⇒" + }, + { + "id": "multiscript-zwj-mixed", + "input_ids": [ + 104811, + 23573, + 83291, + 101059, + 102819, + 100235, + 145707, + 60838, + 144841, + 1318, + 105063 + ], + "tokens": [ + "ä¸Ńæĸĩ", + "íķľ", + "ê¸Ģ", + "æĹ¥æľ¬", + "èªŀ", + "æ··", + "ðŁijĭ", + "íķ©ëĭĪëĭ¤", + "ðŁİī", + "text", + "æ··åIJĪ" + ], + "decoded_with_special": "中文한글日本語混👋합니다🎉text混合", + "decoded_skip_special": "中文한글日本語混👋합니다🎉text混合" + }, + { + "id": "multiscript-string-concat", + "input_ids": [ + 70, + 43632, + 428, + 108386, + 5572, + 145691, + 5172, + 126246, + 144370, + 5572, + 60096, + 87244, + 78368, + 30484, + 97, + 34370, + 10, + 145367, + 378, + 235, + 145233, + 378, + 235, + 145665 + ], + "tokens": [ + "g", + "reeting", + "=\"", + "ä½łå¥½", + "\"+", + "ðŁĮį", + "+\"", + "ìķĪ", + "ëħķ", + "\"+", + "न", + "म", + "स", + "à¥įà¤", + "¤", + "à¥ĩ", + "+", + "ðŁij¨", + "âĢ", + "į", + "ðŁij©", + "âĢ", + "į", + "ðŁij§" + ], + "decoded_with_special": "greeting=\"你好\"+🌍+\"안녕\"+नमस्ते+👨‍👩‍👧", + "decoded_skip_special": "greeting=\"你好\"+🌍+\"안녕\"+नमस्ते+👨‍👩‍👧" + }, + { + "id": "multiscript-emoji-arrows", + "input_ids": [ + 144624, + 856, + 5318, + 11162, + 229, + 107, + 145516, + 94304, + 94305, + 245, + 46553, + 11397, + 89266, + 9592, + 350, + 55661, + 7891, + 55661, + 9592, + 64118, + 55135, + 25521, + 101, + 835, + 13 + ], + "tokens": [ + "Σ", + "Ġx", + "_i", + "ĠðŁ", + "ĩ", + "¯", + "ðŁĩµ", + "Ġâīł", + "ĠåĮ", + "Ĺ", + "京", + "ĠâĨĴ", + "ĠMünchen", + "Ġ·", + "ĠT", + "Åį", + "ky", + "Åį", + "Ġ·", + "Ġä¸Ĭ", + "æµ·", + "Ġâľ", + "¨", + "Ġend", + "." + ], + "decoded_with_special": "Σ x_i 🇯🇵 ≠ 北京 → München · Tōkyō · 上海 ✨ end.", + "decoded_skip_special": "Σ x_i 🇯🇵 ≠ 北京 → München · Tōkyō · 上海 ✨ end." + }, + { + "id": "multiscript-arrow-chain", + "input_ids": [ + 145836, + 60096, + 87244, + 78368, + 30484, + 97, + 34370, + 51018, + 53645, + 26991, + 8178, + 51018, + 108386, + 51018, + 126246, + 144370, + 51018, + 124122, + 29825, + 124671, + 51018, + 141390, + 30543 + ], + "tokens": [ + "ðŁļĢ", + "न", + "म", + "स", + "à¥įà¤", + "¤", + "à¥ĩ", + "âĨĴ", + "ÐŁÑĢ", + "ив", + "еÑĤ", + "âĨĴ", + "ä½łå¥½", + "âĨĴ", + "ìķĪ", + "ëħķ", + "âĨĴ", + "Ùħر", + "ØŃ", + "با", + "âĨĴ", + "âĿ¤", + "ï¸ı" + ], + "decoded_with_special": "🚀नमस्ते→Привет→你好→안녕→مرحبا→❤️", + "decoded_skip_special": "🚀नमस्ते→Привет→你好→안녕→مرحبا→❤️" + }, + { + "id": "multiscript-function-call", + "input_ids": [ + 7975, + 7, + 104811, + 11, + 23573, + 83291, + 11, + 101059, + 102819, + 11, + 93948, + 42311, + 101, + 30484, + 99, + 43647, + 11, + 129089, + 126302, + 8, + 51018, + 90, + 144192, + 25, + 149026, + 11, + 146193, + 25, + 146549, + 92 + ], + "tokens": [ + "foo", + "(", + "ä¸Ńæĸĩ", + ",", + "íķľ", + "ê¸Ģ", + ",", + "æĹ¥æľ¬", + "èªŀ", + ",", + "ह", + "िà¤", + "¨", + "à¥įà¤", + "¦", + "à¥Ģ", + ",", + "ÑĢÑĥÑģ", + "Ñģкий", + ")", + "âĨĴ", + "{", + "âĪĢ", + ":", + "âĪĥ", + ",", + "âĬķ", + ":", + "âĬĹ", + "}" + ], + "decoded_with_special": "foo(中文,한글,日本語,हिन्दी,русский)→{∀:∃,⊕:⊗}", + "decoded_skip_special": "foo(中文,한글,日本語,हिन्दी,русский)→{∀:∃,⊕:⊗}" + }, + { + "id": "multiscript-legacy-symbols", + "input_ids": [ + 144287, + 19, + 17, + 28653, + 95, + 30543, + 7240, + 60479, + 78846, + 16625, + 12984, + 81093, + 21651, + 149632, + 7851, + 96, + 144192, + 220, + 108386, + 95170, + 144370, + 17166 + ], + "tokens": [ + "âĦĸ", + "4", + "2", + "ĠâĦ", + "¢", + "ï¸ı", + "Ġ©", + "Ġ®", + "Ġ¶", + "Ġ§", + "ĠâĤ¬", + "Â¥", + "£", + "âĤ¿", + "ĠÎ", + "£", + "âĪĢ", + "Ġ", + "ä½łå¥½", + "ĠìķĪ", + "ëħķ", + "ĠاÙĦ" + ], + "decoded_with_special": "№42 ™️ © ® ¶ § €¥£₿ Σ∀ 你好 안녕 ال", + "decoded_skip_special": "№42 ™️ © ® ¶ § €¥£₿ Σ∀ 你好 안녕 ال" + }, + { + "id": "escape-sequences-with-zwj", + "input_ids": [ + 4576, + 59, + 15, + 437, + 4955, + 6192, + 1699, + 89202, + 100235, + 17254, + 378, + 235, + 81756, + 41, + 30543, + 95264 + ], + "tokens": [ + "NULL", + "\\", + "0", + "and", + "\\t", + "tab", + "\\n", + "newline", + "æ··", + "åħ¥", + "âĢ", + "į", + "ZW", + "J", + "ï¸ı", + "selectors" + ], + "decoded_with_special": "NULL\\0and\\ttab\\nnewline混入‍ZWJ️selectors", + "decoded_skip_special": "NULL\\0and\\ttab\\nnewline混入‍ZWJ️selectors" + }, + { + "id": "german-compound-short", + "input_ids": [ + 65214, + 56530, + 13, + 33315, + 74247, + 426, + 37699, + 3371, + 26692, + 2724, + 1466, + 66599, + 9421, + 2694, + 28837, + 8470, + 1579, + 58864, + 2986, + 13299, + 331, + 392, + 75, + 1021, + 13, + 26525, + 226 + ], + "tokens": [ + "Nat", + "ürlich", + ".", + "ĠDeutsch", + "Ġzie", + "ht", + "Ġjetzt", + "Ġden", + "ĠKom", + "posit", + "ah", + "ammer", + "Ġaus", + "Ġder", + "ĠGram", + "mat", + "ik", + "werk", + "ze", + "ugs", + "ch", + "ub", + "l", + "ade", + ".", + "ĠðŁĺ", + "Ħ" + ], + "decoded_with_special": "Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. 😄", + "decoded_skip_special": "Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. 😄" + }, + { + "id": "german-compound-long-1", + "input_ids": [ + 1427, + 78060, + 25136, + 9779, + 3556, + 89289, + 423, + 2305, + 267, + 301, + 2185, + 2280, + 4747, + 2498, + 4368, + 730, + 37780, + 295, + 27277, + 11335, + 45085, + 11343, + 645, + 1371, + 674, + 610, + 465, + 84732, + 17058, + 3184, + 46745, + 2853, + 423, + 70, + 433, + 6772, + 440, + 1199, + 20155, + 2030, + 537, + 65161, + 304, + 19349, + 11840, + 94905, + 76, + 39563, + 577, + 268, + 42477, + 1921, + 449, + 72043, + 2746, + 2714, + 20491, + 359, + 20005, + 90593, + 81548, + 26612, + 6538, + 44306, + 3667, + 17033, + 20491, + 392, + 930, + 12495, + 37633, + 1097, + 2571, + 290, + 268, + 13 + ], + "tokens": [ + "Im", + "ĠMorg", + "enne", + "bel", + "ged", + "anken", + "ver", + "ä", + "st", + "el", + "ung", + "sw", + "ald", + "Ġstand", + "Ġein", + "ĠK", + "affe", + "et", + "assen", + "rand", + "phil", + "osoph", + "ie", + "be", + "ob", + "ach", + "ter", + "Ġneben", + "Ġeiner", + "ĠReg", + "ensch", + "irm", + "ver", + "g", + "ess", + "lich", + "ke", + "its", + "station", + "Ġund", + "Ġnot", + "ierte", + "Ġin", + "Ġsein", + "ĠSon", + "ntag", + "m", + "orgen", + "ide", + "en", + "fang", + "not", + "iz", + "buch", + "Ġdie", + "Ġer", + "sta", + "un", + "liche", + "ĠHä", + "ufig", + "keit", + "Ġvon", + "ĠFen", + "ster", + "bank", + "sta", + "ub", + "son", + "nen", + "licht", + "ref", + "lex", + "ion", + "en", + "." + ], + "decoded_with_special": "Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen.", + "decoded_skip_special": "Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen." + }, + { + "id": "german-compound-long-2", + "input_ids": [ + 45, + 3065, + 268, + 58442, + 34259, + 665, + 9820, + 730, + 21558, + 4730, + 331, + 13129, + 34711, + 1371, + 273, + 32030, + 2185, + 799, + 4382, + 2185, + 3563, + 300, + 87895, + 11, + 66983, + 4368, + 386, + 2178, + 301, + 329, + 826, + 14493, + 33425, + 301, + 51176, + 46621, + 704, + 417, + 318, + 1268, + 28122, + 1371, + 50528, + 52015, + 465, + 5451, + 17058, + 12475, + 602, + 268, + 31551, + 585, + 590, + 1579, + 130977, + 84384, + 331, + 28122, + 43703, + 325, + 41852, + 56539, + 17917, + 4116, + 13, + 20004, + 2694, + 28462, + 811, + 308, + 21519, + 13820, + 9089, + 4368, + 29543, + 2712, + 9855, + 266, + 7070, + 69, + 15113, + 267, + 12540, + 17771, + 355, + 1579, + 22097, + 618, + 380, + 7219, + 17443, + 18069, + 13281, + 423, + 440, + 65262, + 1580, + 301, + 1831, + 15938, + 28122, + 137143, + 2030, + 435, + 4738, + 25, + 14835, + 32, + 13920, + 2185, + 11, + 52434, + 13906, + 811, + 63940, + 9820, + 64477, + 89289, + 69, + 21140, + 423, + 74, + 1921, + 28122, + 709, + 96922, + 46865, + 4002, + 0, + 2073 + ], + "tokens": [ + "N", + "eb", + "en", + "Ġihm", + "Ġsumm", + "te", + "Ġeine", + "ĠK", + "üh", + "ls", + "ch", + "rank", + "innen", + "be", + "le", + "ucht", + "ung", + "ser", + "inner", + "ung", + "sm", + "as", + "chine", + ",", + "Ġwährend", + "Ġein", + "ĠM", + "arm", + "el", + "ad", + "eng", + "las", + "deck", + "el", + "öff", + "nung", + "so", + "pt", + "im", + "ier", + "ungs", + "be", + "auf", + "trag", + "ter", + "Ġmit", + "Ġeiner", + "ĠTre", + "pp", + "en", + "haus", + "ak", + "ust", + "ik", + "üb", + "erras", + "ch", + "ungs", + "analy", + "se", + "Ġbesch", + "äft", + "igt", + "Ġwar", + ".", + "ĠAus", + "Ġder", + "ĠFer", + "ne", + "Ġn", + "äh", + "erte", + "Ġsich", + "Ġein", + "ĠWol", + "ken", + "kr", + "at", + "zer", + "f", + "ahr", + "st", + "uh", + "lm", + "us", + "ik", + "kom", + "pon", + "ist", + "Ġauf", + "Ġeinem", + "ĠEin", + "rad", + "ver", + "ke", + "hrs", + "reg", + "el", + "miss", + "acht", + "ungs", + "gerät", + "Ġund", + "Ġr", + "ief", + ":", + "ĠâĢŀ", + "A", + "cht", + "ung", + ",", + "Ġdort", + "Ġvor", + "ne", + "Ġliegt", + "Ġeine", + "ĠGed", + "anken", + "f", + "aden", + "ver", + "k", + "not", + "ungs", + "ge", + "fahren", + "sit", + "uation", + "!", + "âĢľ" + ], + "decoded_with_special": "Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“", + "decoded_skip_special": "Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“" + }, + { + "id": "german-compound-long-3", + "input_ids": [ + 2120, + 2956, + 37592, + 6772, + 63318, + 4711, + 68, + 9089, + 4368, + 28831, + 28122, + 325, + 275, + 9018, + 2024, + 14033, + 465, + 19154, + 33052, + 7997, + 37953, + 11, + 2030, + 83380, + 3999, + 27393, + 1960, + 57519, + 141506, + 10561, + 8666, + 932, + 6860, + 1268, + 86, + 3818, + 2660, + 9303, + 20542, + 5342, + 321, + 88487, + 34711, + 11, + 4368, + 479, + 471, + 16597, + 86, + 2375, + 76, + 28658, + 5679, + 23559, + 7964, + 652, + 962, + 2030, + 4368, + 60021, + 6759, + 343, + 31100, + 404, + 81, + 465, + 6299, + 2752, + 75, + 51176, + 301, + 1097, + 2571, + 908, + 4059, + 1342, + 1047, + 24803, + 13, + 77775, + 1330, + 309, + 1081, + 34271, + 33748, + 9911, + 3371, + 46307, + 423, + 7053, + 7180, + 28275, + 23692, + 2032, + 37633, + 21062, + 14456, + 74, + 54834, + 65, + 17366, + 15938, + 2185, + 2030, + 79063, + 65161, + 35547, + 585, + 36964, + 11240, + 4370, + 64832, + 307, + 343, + 28122, + 495, + 1328, + 3591, + 13 + ], + "tokens": [ + "Pl", + "ö", + "tz", + "lich", + "Ġöff", + "net", + "e", + "Ġsich", + "Ġein", + "ĠZeit", + "ungs", + "se", + "it", + "enum", + "bl", + "ät", + "ter", + "wind", + "sto", + "ÃŁ", + "portal", + ",", + "Ġund", + "Ġheraus", + "Ġpur", + "zel", + "ten", + "Ġdrei", + "ĠBüro", + "kl", + "amm", + "ern", + "sort", + "ier", + "w", + "elt", + "me", + "isters", + "chaft", + "ste", + "il", + "nehmer", + "innen", + ",", + "Ġein", + "ĠG", + "art", + "enz", + "w", + "erg", + "m", + "üt", + "zen", + "far", + "ben", + "ber", + "ater", + "Ġund", + "Ġein", + "Ġhoch", + "grad", + "ig", + "Ġverw", + "ir", + "r", + "ter", + "ĠSup", + "pen", + "l", + "öff", + "el", + "ref", + "lex", + "ions", + "met", + "aph", + "ys", + "iker", + ".", + "ĠGeme", + "ins", + "am", + "Ġgr", + "ünd", + "eten", + "Ġsie", + "Ġden", + "ĠBundes", + "ver", + "band", + "Ġfür", + "ĠNach", + "mitt", + "ags", + "licht", + "sch", + "atten", + "k", + "anten", + "b", + "etr", + "acht", + "ung", + "Ġund", + "Ġimprovis", + "ierte", + "ĠSof", + "ak", + "issen", + "bur", + "gen", + "verte", + "id", + "ig", + "ungs", + "str", + "ateg", + "ien", + "." + ], + "decoded_with_special": "Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien.", + "decoded_skip_special": "Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien." + }, + { + "id": "german-compound-long-4", + "input_ids": [ + 6091, + 3680, + 408, + 5436, + 8666, + 301, + 1960, + 9089, + 12304, + 732, + 386, + 49958, + 1528, + 13573, + 268, + 267, + 615, + 1466, + 5676, + 20491, + 392, + 6072, + 20480, + 64474, + 11, + 23738, + 2694, + 730, + 37780, + 295, + 27277, + 11335, + 45085, + 11343, + 645, + 1371, + 674, + 610, + 465, + 9820, + 1152, + 1268, + 20005, + 49279, + 1854, + 91661, + 13785, + 2746, + 467, + 34966, + 26612, + 6538, + 1230, + 15940, + 4412, + 8555, + 5840, + 908, + 24716, + 793, + 11, + 15515, + 610, + 37727, + 17659, + 450, + 24134, + 130977, + 11595, + 2030, + 4320, + 7880, + 1121, + 3848, + 331, + 333, + 542, + 1466, + 3342, + 1991, + 288, + 69701, + 4886, + 391, + 275, + 15141, + 3563, + 28658, + 5679, + 19095, + 453, + 49793, + 20175, + 55632, + 143729, + 2649, + 28122, + 776, + 891, + 5676, + 15588, + 3818, + 13 + ], + "tokens": [ + "Am", + "ĠAb", + "end", + "Ġvers", + "amm", + "el", + "ten", + "Ġsich", + "Ġalle", + "Ġim", + "ĠM", + "onds", + "che", + "inf", + "en", + "st", + "err", + "ah", + "men", + "sta", + "ub", + "gl", + "itzer", + "zimmer", + ",", + "Ġwo", + "Ġder", + "ĠK", + "affe", + "et", + "assen", + "rand", + "phil", + "osoph", + "ie", + "be", + "ob", + "ach", + "ter", + "Ġeine", + "Ġfe", + "ier", + "liche", + "ĠSchl", + "uss", + "rede", + "Ġüber", + "Ġdie", + "ĠW", + "ichtig", + "keit", + "Ġvon", + "ĠUn", + "sin", + "ns", + "produ", + "kt", + "ions", + "fre", + "ude", + ",", + "ĠSpr", + "ach", + "mus", + "kel", + "de", + "hn", + "üb", + "ungen", + "Ġund", + "ĠDon", + "aud", + "amp", + "fs", + "ch", + "if", + "ff", + "ah", + "rt", + "sg", + "es", + "ellschaft", + "sk", + "ap", + "it", + "än", + "sm", + "üt", + "zen", + "kn", + "op", + "fers", + "atz", + "teil", + "besch", + "aff", + "ungs", + "pro", + "ble", + "men", + "Ġhi", + "elt", + "." + ], + "decoded_with_special": "Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt.", + "decoded_skip_special": "Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt." + }, + { + "id": "german-compound-long-5", + "input_ids": [ + 38025, + 610, + 54739, + 21062, + 665, + 678, + 64649, + 482, + 1863, + 1704, + 4487, + 268, + 22385, + 11, + 7906, + 52662, + 13279, + 2694, + 730, + 21558, + 4730, + 331, + 13129, + 34711, + 1371, + 273, + 32030, + 2185, + 799, + 4382, + 2185, + 3563, + 300, + 87895, + 11, + 2746, + 9089, + 13785, + 296, + 17281, + 42341, + 57358, + 51176, + 46621, + 9081, + 1704, + 1174, + 91647, + 440, + 1199, + 86, + 15546, + 331, + 14033, + 38132, + 387, + 10561, + 351, + 665, + 13 + ], + "tokens": [ + "Dan", + "ach", + "Ġherr", + "sch", + "te", + "Ġall", + "geme", + "ine", + "ĠZ", + "uf", + "ried", + "en", + "heit", + ",", + "Ġau", + "ÃŁer", + "Ġbei", + "Ġder", + "ĠK", + "üh", + "ls", + "ch", + "rank", + "innen", + "be", + "le", + "ucht", + "ung", + "ser", + "inner", + "ung", + "sm", + "as", + "chine", + ",", + "Ġdie", + "Ġsich", + "Ġüber", + "Ġm", + "angel", + "nde", + "ĠTür", + "öff", + "nung", + "sa", + "uf", + "mer", + "ksam", + "ke", + "its", + "w", + "erts", + "ch", + "ät", + "zung", + "Ġbe", + "kl", + "ag", + "te", + "." + ], + "decoded_with_special": "Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte.", + "decoded_skip_special": "Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte." + } + ] +} diff --git a/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/roberta_base_multilingual.json b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/roberta_base_multilingual.json new file mode 100644 index 00000000..c18776ab --- /dev/null +++ b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/roberta_base_multilingual.json @@ -0,0 +1,6903 @@ +{ + "metadata": { + "model_id": "FacebookAI/roberta-base", + "transformers_version": "4.57.1", + "generated_at": "2026-05-15T23:58:34+00:00", + "input_count": 83 + }, + "entries": [ + { + "id": "latin-diacritics-french", + "input_ids": [ + 0, + 438, + 2001, + 1140, + 15367, + 18581, + 1140, + 39300, + 18363, + 3381, + 1829, + 93, + 3977, + 8025, + 1794, + 5378, + 27564, + 462, + 9703, + 21, + 4206, + 4, + 2 + ], + "tokens": [ + "", + "c", + "af", + "é", + "Ġré", + "sum", + "é", + "Ġnaïve", + "Ġfa", + "ç", + "ade", + "ĠâĢĶ", + "Ġcr", + "è", + "me", + "Ġbr", + "û", + "l", + "ée", + "Ġwas", + "Ġexcellent", + ".", + "" + ], + "decoded_with_special": "café résumé naïve façade — crème brûlée was excellent.", + "decoded_skip_special": "café résumé naïve façade — crème brûlée was excellent." + }, + { + "id": "latin-diacritics-mixed", + "input_ids": [ + 0, + 725, + 2768, + 24612, + 952, + 4394, + 1943, + 2265, + 2768, + 438, + 1071, + 4, + 3066, + 6303, + 1113, + 32098, + 10, + 897, + 3723, + 14379, + 2764, + 23087, + 4, + 2 + ], + "tokens": [ + "", + "H", + "ü", + "gel", + "ĠÃ", + "¼", + "ber", + "ĠBr", + "ü", + "c", + "ke", + ".", + "ĠMa", + "ñ", + "ana", + "Ġvoy", + "Ġa", + "Ġla", + "Ġpe", + "ña", + "Ġcon", + "ĠJosé", + ".", + "" + ], + "decoded_with_special": "Hügel über Brücke. Mañana voy a la peña con José.", + "decoded_skip_special": "Hügel über Brücke. Mañana voy a la peña con José." + }, + { + "id": "latin-diacritics-portuguese", + "input_ids": [ + 0, + 104, + 4214, + 15333, + 7935, + 1717, + 1916, + 740, + 808, + 1829, + 6979, + 242, + 5224, + 5285, + 131, + 229, + 3671, + 8502, + 10, + 4272, + 4, + 2 + ], + "tokens": [ + "", + "S", + "ão", + "ĠPaulo", + "Ġé", + "Ġu", + "ma", + "Ġc", + "id", + "ade", + "Ġint", + "e", + "ress", + "ante", + ";", + "ĠK", + "ö", + "ln", + "Ġa", + "uch", + ".", + "" + ], + "decoded_with_special": "São Paulo é uma cidade interessante; Köln auch.", + "decoded_skip_special": "São Paulo é uma cidade interessante; Köln auch." + }, + { + "id": "cyrillic-greeting", + "input_ids": [ + 0, + 25417, + 4333, + 24269, + 35328, + 42907, + 25482, + 35555, + 6, + 18697, + 4394, + 35328, + 24269, + 328, + 18697, + 12410, + 35555, + 34251, + 1437, + 35555, + 25482, + 36709, + 35555, + 1437, + 35555, + 41613, + 3070, + 25482, + 36765, + 35328, + 25417, + 18400, + 26161, + 35555, + 34251, + 24269, + 26161, + 4, + 2 + ], + "tokens": [ + "", + "Ð", + "Ł", + "ÑĢ", + "и", + "в", + "е", + "ÑĤ", + ",", + "ĠÐ", + "¼", + "и", + "ÑĢ", + "!", + "ĠÐ", + "Ń", + "ÑĤ", + "о", + "Ġ", + "ÑĤ", + "е", + "Ñģ", + "ÑĤ", + "Ġ", + "ÑĤ", + "оÐ", + "º", + "е", + "н", + "и", + "Ð", + "·", + "а", + "ÑĤ", + "о", + "ÑĢ", + "а", + ".", + "" + ], + "decoded_with_special": "Привет, мир! Это тест токенизатора.", + "decoded_skip_special": "Привет, мир! Это тест токенизатора." + }, + { + "id": "cyrillic-prose", + "input_ids": [ + 0, + 25417, + 48, + 34251, + 36709, + 41171, + 42907, + 26161, + 93, + 1437, + 36709, + 35555, + 41613, + 2023, + 35328, + 22063, + 27819, + 26161, + 18697, + 21402, + 34251, + 36709, + 36709, + 35328, + 35328, + 4, + 18697, + 46, + 26161, + 36709, + 25482, + 40966, + 25482, + 36765, + 35328, + 25482, + 18697, + 15389, + 41613, + 2023, + 25482, + 25482, + 316, + 18697, + 4394, + 35328, + 40966, + 40966, + 35328, + 34251, + 36765, + 41613, + 14292, + 4, + 2 + ], + "tokens": [ + "", + "Ð", + "ľ", + "о", + "Ñģ", + "к", + "в", + "а", + "ĠâĢĶ", + "Ġ", + "Ñģ", + "ÑĤ", + "оÐ", + "»", + "и", + "Ñ", + "Ĩ", + "а", + "ĠÐ", + "ł", + "о", + "Ñģ", + "Ñģ", + "и", + "и", + ".", + "ĠÐ", + "Ŀ", + "а", + "Ñģ", + "е", + "л", + "е", + "н", + "и", + "е", + "ĠÐ", + "±", + "оÐ", + "»", + "е", + "е", + "Ġ12", + "ĠÐ", + "¼", + "и", + "л", + "л", + "и", + "о", + "н", + "оÐ", + "²", + ".", + "" + ], + "decoded_with_special": "Москва — столица России. Население более 12 миллионов.", + "decoded_skip_special": "Москва — столица России. Население более 12 миллионов." + }, + { + "id": "greek-greeting", + "input_ids": [ + 0, + 41335, + 15113, + 44721, + 48191, + 41335, + 18400, + 47049, + 41335, + 12410, + 47630, + 44721, + 43662, + 3070, + 45704, + 14285, + 47721, + 47049, + 47016, + 328, + 43662, + 6800, + 43662, + 15264, + 48191, + 45704, + 12736, + 47721, + 47721, + 44721, + 43662, + 8906, + 41335, + 10965, + 47118, + 44721, + 46399, + 46927, + 14285, + 47049, + 46122, + 47630, + 45704, + 27819, + 41335, + 18400, + 4, + 2 + ], + "tokens": [ + "", + "Î", + "ļ", + "α", + "λ", + "Î", + "·", + "μ", + "Î", + "Ń", + "Ïģ", + "α", + "ĠÎ", + "º", + "Ï", + "Į", + "Ïĥ", + "μ", + "ε", + "!", + "ĠÎ", + "Ĺ", + "ĠÎ", + "³", + "λ", + "Ï", + "İ", + "Ïĥ", + "Ïĥ", + "α", + "ĠÎ", + "µ", + "Î", + "¯", + "ν", + "α", + "ι", + "ĠÏ", + "Į", + "μ", + "ο", + "Ïģ", + "Ï", + "Ĩ", + "Î", + "·", + ".", + "" + ], + "decoded_with_special": "Καλημέρα κόσμε! Η γλώσσα είναι όμορφη.", + "decoded_skip_special": "Καλημέρα κόσμε! Η γλώσσα είναι όμορφη." + }, + { + "id": "cjk-simplified-greeting", + "input_ids": [ + 0, + 47856, + 21402, + 48975, + 10809, + 6, + 46015, + 25448, + 49127, + 14285, + 328, + 48145, + 27, + 48569, + 47166, + 27819, + 48128, + 8384, + 42393, + 27, + 11423, + 48818, + 13859, + 48128, + 15722, + 45682, + 2 + ], + "tokens": [ + "", + "ä½", + "ł", + "å¥", + "½", + ",", + "ä¸", + "ĸ", + "çķ", + "Į", + "!", + "è¿", + "Ļ", + "æĺ¯", + "åĪ", + "Ĩ", + "è¯", + "į", + "å", + "Ļ", + "¨", + "æµ", + "ĭ", + "è¯", + "ķ", + "ãĢĤ", + "" + ], + "decoded_with_special": "你好,世界!这是分词器测试。", + "decoded_skip_special": "你好,世界!这是分词器测试。" + }, + { + "id": "cjk-simplified-prose", + "input_ids": [ + 0, + 47643, + 47240, + 6382, + 48569, + 46015, + 25448, + 49127, + 14285, + 48827, + 49535, + 47873, + 11423, + 47973, + 45262, + 2469, + 44636, + 7471, + 47983, + 15113, + 44574, + 48128, + 12410, + 36484, + 11423, + 7471, + 49065, + 48105, + 45682, + 2 + ], + "tokens": [ + "", + "ä¸Ń", + "æĸ", + "ĩ", + "æĺ¯", + "ä¸", + "ĸ", + "çķ", + "Į", + "ä¸Ĭ", + "使", + "çĶ", + "¨", + "人", + "åı", + "£", + "æľ", + "Ģ", + "å¤", + "ļ", + "çļĦ", + "è¯", + "Ń", + "è", + "¨", + "Ģ", + "ä¹ĭ", + "ä¸Ģ", + "ãĢĤ", + "" + ], + "decoded_with_special": "中文是世界上使用人口最多的语言之一。", + "decoded_skip_special": "中文是世界上使用人口最多的语言之一。" + }, + { + "id": "cjk-traditional-compare", + "input_ids": [ + 0, + 36714, + 9253, + 10172, + 41907, + 4958, + 10674, + 47643, + 47240, + 6382, + 48967, + 6382, + 36714, + 7487, + 5543, + 41907, + 4958, + 10674, + 47643, + 47240, + 6382, + 44636, + 23171, + 48548, + 7471, + 48664, + 47504, + 14285, + 45682, + 2 + ], + "tokens": [ + "", + "ç", + "¹", + "ģ", + "é", + "«", + "Ķ", + "ä¸Ń", + "æĸ", + "ĩ", + "èĪ", + "ĩ", + "ç", + "°", + "¡", + "é", + "«", + "Ķ", + "ä¸Ń", + "æĸ", + "ĩ", + "æľ", + "ī", + "æī", + "Ģ", + "ä¸į", + "åIJ", + "Į", + "ãĢĤ", + "" + ], + "decoded_with_special": "繁體中文與簡體中文有所不同。", + "decoded_skip_special": "繁體中文與簡體中文有所不同。" + }, + { + "id": "cjk-traditional-hongkong", + "input_ids": [ + 0, + 41907, + 18164, + 27, + 37127, + 18537, + 10965, + 44574, + 36714, + 9253, + 10172, + 41907, + 4958, + 10674, + 47643, + 47240, + 6382, + 44636, + 23171, + 36714, + 8384, + 11423, + 48964, + 9253, + 44574, + 36484, + 15375, + 17772, + 48944, + 27, + 45682, + 2 + ], + "tokens": [ + "", + "é", + "¦", + "Ļ", + "æ", + "¸", + "¯", + "çļĦ", + "ç", + "¹", + "ģ", + "é", + "«", + "Ķ", + "ä¸Ń", + "æĸ", + "ĩ", + "æľ", + "ī", + "ç", + "į", + "¨", + "çī", + "¹", + "çļĦ", + "è", + "©", + "ŀ", + "å½", + "Ļ", + "ãĢĤ", + "" + ], + "decoded_with_special": "香港的繁體中文有獨特的詞彙。", + "decoded_skip_special": "香港的繁體中文有獨特的詞彙。" + }, + { + "id": "japanese-voiced-kana-greeting", + "input_ids": [ + 0, + 49047, + 48913, + 48016, + 46311, + 5543, + 48549, + 47341, + 46015, + 25448, + 49127, + 14285, + 45682, + 49280, + 49850, + 49689, + 48611, + 49751, + 48438, + 48018, + 49312, + 49615, + 48190, + 48735, + 45682, + 2 + ], + "tokens": [ + "", + "ãģĵ", + "ãĤĵ", + "ãģ«", + "ãģ", + "¡", + "ãģ¯", + "ãĢģ", + "ä¸", + "ĸ", + "çķ", + "Į", + "ãĢĤ", + "ãĥĪ", + "ãĥ¼ãĤ¯", + "ãĥĬ", + "ãĤ¤", + "ãĤ¶", + "ãĥ¼", + "ãģ®", + "ãĥĨ", + "ãĤ¹ãĥĪ", + "ãģ§", + "ãģĻ", + "ãĢĤ", + "" + ], + "decoded_with_special": "こんにちは、世界。トークナイザーのテストです。", + "decoded_skip_special": "こんにちは、世界。トークナイザーのテストです。" + }, + { + "id": "japanese-voiced-kana-prose", + "input_ids": [ + 0, + 48414, + 15389, + 46499, + 11582, + 49343, + 49770, + 48438, + 48549, + 48281, + 48241, + 48885, + 36484, + 15264, + 3602, + 47783, + 11936, + 48757, + 48247, + 48334, + 15264, + 49500, + 46537, + 7487, + 48190, + 48735, + 45682, + 2 + ], + "tokens": [ + "", + "æĿ", + "±", + "äº", + "¬", + "ãĤ¿", + "ãĥ¯", + "ãĥ¼", + "ãģ¯", + "ãģ¨", + "ãģ¦", + "ãĤĤ", + "è", + "³", + "ij", + "ãĤ", + "Ħ", + "ãģĭ", + "ãģª", + "è¦", + "³", + "åħī", + "åľ", + "°", + "ãģ§", + "ãģĻ", + "ãĢĤ", + "" + ], + "decoded_with_special": "東京タワーはとても賑やかな観光地です。", + "decoded_skip_special": "東京タワーはとても賑やかな観光地です。" + }, + { + "id": "hangul-syllables-greeting", + "input_ids": [ + 0, + 43998, + 15722, + 23133, + 45209, + 5782, + 15722, + 48589, + 711, + 43998, + 11936, + 18537, + 43998, + 15113, + 10674, + 6, + 46747, + 11936, + 18537, + 46873, + 15264, + 11936, + 328, + 1437, + 47649, + 27819, + 21402, + 47649, + 10172, + 11582, + 45209, + 9264, + 711, + 48280, + 20024, + 43998, + 21402, + 7471, + 1437, + 47649, + 5782, + 14285, + 43998, + 27969, + 10470, + 47649, + 27969, + 18537, + 43998, + 17772, + 5782, + 48444, + 23133, + 48444, + 10470, + 4, + 2 + ], + "tokens": [ + "", + "ì", + "ķ", + "Ī", + "ë", + "ħ", + "ķ", + "íķ", + "ĺ", + "ì", + "Ħ", + "¸", + "ì", + "ļ", + "Ķ", + ",", + "Ġì", + "Ħ", + "¸", + "ê", + "³", + "Ħ", + "!", + "Ġ", + "í", + "Ĩ", + "ł", + "í", + "ģ", + "¬", + "ë", + "Ĥ", + "ĺ", + "ìĿ", + "´", + "ì", + "ł", + "Ģ", + "Ġ", + "í", + "ħ", + "Į", + "ì", + "Ĭ", + "¤", + "í", + "Ĭ", + "¸", + "ì", + "ŀ", + "ħ", + "ëĭ", + "Ī", + "ëĭ", + "¤", + ".", + "" + ], + "decoded_with_special": "안녕하세요, 세계! 토크나이저 테스트입니다.", + "decoded_skip_special": "안녕하세요, 세계! 토크나이저 테스트입니다." + }, + { + "id": "hangul-syllables-prose", + "input_ids": [ + 0, + 48589, + 48, + 46873, + 8906, + 12410, + 43998, + 25448, + 20024, + 45209, + 27969, + 10674, + 1437, + 48589, + 48, + 46873, + 18537, + 7471, + 45209, + 5543, + 48, + 1437, + 47649, + 3602, + 48, + 46873, + 18537, + 7487, + 48589, + 15375, + 48444, + 23133, + 48444, + 10470, + 4, + 2 + ], + "tokens": [ + "", + "íķ", + "ľ", + "ê", + "µ", + "Ń", + "ì", + "ĸ", + "´", + "ë", + "Ĭ", + "Ķ", + "Ġ", + "íķ", + "ľ", + "ê", + "¸", + "Ģ", + "ë", + "¡", + "ľ", + "Ġ", + "í", + "ij", + "ľ", + "ê", + "¸", + "°", + "íķ", + "©", + "ëĭ", + "Ī", + "ëĭ", + "¤", + ".", + "" + ], + "decoded_with_special": "한국어는 한글로 표기합니다.", + "decoded_skip_special": "한국어는 한글로 표기합니다." + }, + { + "id": "arabic-greeting", + "input_ids": [ + 0, + 33070, + 29434, + 21593, + 12410, + 41117, + 29438, + 25790, + 11423, + 43321, + 41688, + 43321, + 33070, + 328, + 36963, + 6382, + 21593, + 7487, + 29438, + 1437, + 34955, + 2840, + 38605, + 41117, + 29438, + 29434, + 1437, + 30992, + 30992, + 33070, + 21593, + 12410, + 30992, + 30992, + 32681, + 30992, + 21593, + 3070, + 33866, + 26174, + 4, + 2 + ], + "tokens": [ + "", + "Ùħ", + "ر", + "Ø", + "Ń", + "ب", + "ا", + "ĠØ", + "¨", + "اÙĦ", + "ع", + "اÙĦ", + "Ùħ", + "!", + "ĠÙ", + "ĩ", + "Ø", + "°", + "ا", + "Ġ", + "اØ", + "®", + "ت", + "ب", + "ا", + "ر", + "Ġ", + "ÙĦ", + "ÙĦ", + "Ùħ", + "Ø", + "Ń", + "ÙĦ", + "ÙĦ", + "ĠاÙĦ", + "ÙĦ", + "Ø", + "º", + "ÙĪ", + "ÙĬ", + ".", + "" + ], + "decoded_with_special": "مرحبا بالعالم! هذا اختبار للمحلل اللغوي.", + "decoded_skip_special": "مرحبا بالعالم! هذا اختبار للمحلل اللغوي." + }, + { + "id": "arabic-prose", + "input_ids": [ + 0, + 43321, + 30992, + 21593, + 3070, + 38419, + 32681, + 41688, + 29434, + 41117, + 26174, + 38419, + 1437, + 30992, + 21593, + 3070, + 38419, + 25790, + 15264, + 29438, + 33070, + 26174, + 38419, + 25790, + 10278, + 26068, + 862, + 38605, + 41117, + 42620, + 31746, + 32681, + 26174, + 33070, + 26174, + 31746, + 25790, + 8210, + 30992, + 26068, + 23171, + 32681, + 26174, + 40637, + 29438, + 29434, + 4, + 2 + ], + "tokens": [ + "", + "اÙĦ", + "ÙĦ", + "Ø", + "º", + "Ø©", + "ĠاÙĦ", + "ع", + "ر", + "ب", + "ÙĬ", + "Ø©", + "Ġ", + "ÙĦ", + "Ø", + "º", + "Ø©", + "ĠØ", + "³", + "ا", + "Ùħ", + "ÙĬ", + "Ø©", + "ĠØ", + "ª", + "Ù", + "ĥ", + "ت", + "ب", + "ĠÙħ", + "ÙĨ", + "ĠاÙĦ", + "ÙĬ", + "Ùħ", + "ÙĬ", + "ÙĨ", + "ĠØ", + "¥", + "ÙĦ", + "Ù", + "ī", + "ĠاÙĦ", + "ÙĬ", + "س", + "ا", + "ر", + ".", + "" + ], + "decoded_with_special": "اللغة العربية لغة سامية تكتب من اليمين إلى اليسار.", + "decoded_skip_special": "اللغة العربية لغة سامية تكتب من اليمين إلى اليسار." + }, + { + "id": "hebrew-greeting", + "input_ids": [ + 0, + 48352, + 47759, + 46174, + 42358, + 46, + 42455, + 7258, + 46174, + 47759, + 42358, + 46, + 328, + 42455, + 25448, + 47824, + 42455, + 17772, + 48405, + 42358, + 6800, + 42358, + 4333, + 42455, + 15375, + 47759, + 42455, + 10674, + 42358, + 711, + 46174, + 42358, + 6248, + 42358, + 21402, + 48177, + 46774, + 25448, + 47581, + 4, + 2 + ], + "tokens": [ + "", + "ש", + "׾", + "×ķ", + "×", + "Ŀ", + "Ġ×", + "¢", + "×ķ", + "׾", + "×", + "Ŀ", + "!", + "Ġ×", + "ĸ", + "×Ķ", + "Ġ×", + "ŀ", + "×ij", + "×", + "Ĺ", + "×", + "Ł", + "Ġ×", + "©", + "׾", + "Ġ×", + "Ķ", + "×", + "ĺ", + "×ķ", + "×", + "§", + "×", + "ł", + "×Ļ", + "×Ļ×", + "ĸ", + "ר", + ".", + "" + ], + "decoded_with_special": "שלום עולם! זה מבחן של הטוקנייזר.", + "decoded_skip_special": "שלום עולם! זה מבחן של הטוקנייזר." + }, + { + "id": "devanagari-hindi", + "input_ids": [ + 0, + 14238, + 11423, + 14238, + 2840, + 14238, + 18537, + 22919, + 8384, + 14238, + 10470, + 22919, + 6382, + 22214, + 18164, + 22919, + 10172, + 14238, + 11423, + 14238, + 9470, + 14238, + 10965, + 35636, + 328, + 22214, + 10965, + 14238, + 9253, + 22214, + 4333, + 22919, + 13859, + 14238, + 15722, + 14238, + 11423, + 35636, + 14238, + 6382, + 14238, + 48, + 14238, + 4394, + 14238, + 7487, + 22214, + 10278, + 14238, + 7487, + 22919, + 7471, + 14238, + 15722, + 22919, + 8384, + 14238, + 18400, + 14238, + 2469, + 22214, + 9253, + 22919, + 23133, + 22919, + 10470, + 2 + ], + "tokens": [ + "", + "à¤", + "¨", + "à¤", + "®", + "à¤", + "¸", + "à¥", + "į", + "à¤", + "¤", + "à¥", + "ĩ", + "Ġà¤", + "¦", + "à¥", + "ģ", + "à¤", + "¨", + "à¤", + "¿", + "à¤", + "¯", + "ा", + "!", + "Ġà¤", + "¯", + "à¤", + "¹", + "Ġà¤", + "Ł", + "à¥", + "ĭ", + "à¤", + "ķ", + "à¤", + "¨", + "ा", + "à¤", + "ĩ", + "à¤", + "ľ", + "à¤", + "¼", + "à¤", + "°", + "Ġà¤", + "ª", + "à¤", + "°", + "à¥", + "Ģ", + "à¤", + "ķ", + "à¥", + "į", + "à¤", + "·", + "à¤", + "£", + "Ġà¤", + "¹", + "à¥", + "Ī", + "à¥", + "¤", + "" + ], + "decoded_with_special": "नमस्ते दुनिया! यह टोकनाइज़र परीक्षण है।", + "decoded_skip_special": "नमस्ते दुनिया! यह टोकनाइज़र परीक्षण है।" + }, + { + "id": "devanagari-sanskrit", + "input_ids": [ + 0, + 14238, + 18537, + 14238, + 9264, + 14238, + 18537, + 22919, + 8384, + 14238, + 15722, + 22919, + 862, + 14238, + 10470, + 22214, + 12410, + 35636, + 14238, + 7487, + 14238, + 10470, + 22214, + 15722, + 22919, + 7471, + 22214, + 9357, + 14238, + 15722, + 22214, + 10278, + 22919, + 8384, + 14238, + 7487, + 35636, + 14238, + 15113, + 22919, + 7471, + 14238, + 11423, + 22214, + 12410, + 35636, + 14238, + 18400, + 35636, + 22214, + 9253, + 22919, + 23133, + 22919, + 10470, + 2 + ], + "tokens": [ + "", + "à¤", + "¸", + "à¤", + "Ĥ", + "à¤", + "¸", + "à¥", + "į", + "à¤", + "ķ", + "à¥", + "ĥ", + "à¤", + "¤", + "Ġà¤", + "Ń", + "ा", + "à¤", + "°", + "à¤", + "¤", + "Ġà¤", + "ķ", + "à¥", + "Ģ", + "Ġà¤", + "ı", + "à¤", + "ķ", + "Ġà¤", + "ª", + "à¥", + "į", + "à¤", + "°", + "ा", + "à¤", + "ļ", + "à¥", + "Ģ", + "à¤", + "¨", + "Ġà¤", + "Ń", + "ा", + "à¤", + "·", + "ा", + "Ġà¤", + "¹", + "à¥", + "Ī", + "à¥", + "¤", + "" + ], + "decoded_with_special": "संस्कृत भारत की एक प्राचीन भाषा है।", + "decoded_skip_special": "संस्कृत भारत की एक प्राचीन भाषा है।" + }, + { + "id": "thai-combining-marks-greeting", + "input_ids": [ + 0, + 24107, + 10278, + 24107, + 6248, + 24107, + 15389, + 24107, + 10278, + 24107, + 10674, + 24107, + 8906, + 24107, + 27969, + 24107, + 14292, + 24107, + 6248, + 42348, + 9264, + 24107, + 8210, + 24107, + 10172, + 328, + 1437, + 24107, + 27, + 24107, + 8906, + 42348, + 23133, + 24107, + 11936, + 24107, + 18400, + 24107, + 12410, + 24107, + 10172, + 24107, + 14292, + 24107, + 2469, + 24107, + 6800, + 24107, + 10674, + 24107, + 10278, + 24107, + 12410, + 24107, + 15113, + 42348, + 7471, + 24107, + 11936, + 24107, + 2469, + 24107, + 18400, + 42348, + 23133, + 24107, + 12410, + 24107, + 6382, + 24107, + 15722, + 24107, + 15389, + 24107, + 10674, + 24107, + 11936, + 24107, + 15264, + 2 + ], + "tokens": [ + "", + "à¸", + "ª", + "à¸", + "§", + "à¸", + "±", + "à¸", + "ª", + "à¸", + "Ķ", + "à¸", + "µ", + "à¸", + "Ĭ", + "à¸", + "²", + "à¸", + "§", + "à¹", + "Ĥ", + "à¸", + "¥", + "à¸", + "ģ", + "!", + "Ġ", + "à¸", + "Ļ", + "à¸", + "µ", + "à¹", + "Ī", + "à¸", + "Ħ", + "à¸", + "·", + "à¸", + "Ń", + "à¸", + "ģ", + "à¸", + "²", + "à¸", + "£", + "à¸", + "Ĺ", + "à¸", + "Ķ", + "à¸", + "ª", + "à¸", + "Ń", + "à¸", + "ļ", + "à¹", + "Ģ", + "à¸", + "Ħ", + "à¸", + "£", + "à¸", + "·", + "à¹", + "Ī", + "à¸", + "Ń", + "à¸", + "ĩ", + "à¸", + "ķ", + "à¸", + "±", + "à¸", + "Ķ", + "à¸", + "Ħ", + "à¸", + "³", + "" + ], + "decoded_with_special": "สวัสดีชาวโลก! นี่คือการทดสอบเครื่องตัดคำ", + "decoded_skip_special": "สวัสดีชาวโลก! นี่คือการทดสอบเครื่องตัดคำ" + }, + { + "id": "thai-combining-marks-prose", + "input_ids": [ + 0, + 24107, + 21402, + 24107, + 14292, + 24107, + 15375, + 24107, + 14292, + 42348, + 11936, + 24107, + 6800, + 24107, + 7258, + 42348, + 11936, + 24107, + 5543, + 42348, + 23133, + 42348, + 862, + 24107, + 27969, + 42348, + 23171, + 24107, + 27969, + 42348, + 23133, + 24107, + 12410, + 24107, + 6382, + 24107, + 6248, + 42348, + 23133, + 24107, + 14292, + 24107, + 6382, + 24107, + 2469, + 24107, + 7487, + 24107, + 4958, + 24107, + 6248, + 42348, + 23133, + 24107, + 14292, + 24107, + 6382, + 24107, + 11936, + 24107, + 15264, + 2 + ], + "tokens": [ + "", + "à¸", + "ł", + "à¸", + "²", + "à¸", + "©", + "à¸", + "²", + "à¹", + "Ħ", + "à¸", + "Ĺ", + "à¸", + "¢", + "à¹", + "Ħ", + "à¸", + "¡", + "à¹", + "Ī", + "à¹", + "ĥ", + "à¸", + "Ĭ", + "à¹", + "ī", + "à¸", + "Ĭ", + "à¹", + "Ī", + "à¸", + "Ń", + "à¸", + "ĩ", + "à¸", + "§", + "à¹", + "Ī", + "à¸", + "²", + "à¸", + "ĩ", + "à¸", + "£", + "à¸", + "°", + "à¸", + "«", + "à¸", + "§", + "à¹", + "Ī", + "à¸", + "²", + "à¸", + "ĩ", + "à¸", + "Ħ", + "à¸", + "³", + "" + ], + "decoded_with_special": "ภาษาไทยไม่ใช้ช่องว่างระหว่างคำ", + "decoded_skip_special": "ภาษาไทยไม่ใช้ช่องว่างระหว่างคำ" + }, + { + "id": "mixed-script-cjk-latin", + "input_ids": [ + 0, + 133, + 47111, + 14285, + 6800, + 46499, + 11582, + 1805, + 11, + 291, + 2481, + 93, + 2664, + 8025, + 29, + 741, + 2495, + 328, + 2 + ], + "tokens": [ + "", + "The", + "Ġå", + "Į", + "Ĺ", + "äº", + "¬", + "Ġtrip", + "Ġin", + "Ġ20", + "26", + "ĠâĢĶ", + "Ġtr", + "è", + "s", + "Ġb", + "ien", + "!", + "" + ], + "decoded_with_special": "The 北京 trip in 2026 — très bien!", + "decoded_skip_special": "The 北京 trip in 2026 — très bien!" + }, + { + "id": "mixed-script-katakana-code", + "input_ids": [ + 0, + 49195, + 48943, + 23171, + 49390, + 49622, + 49432, + 48438, + 35, + 13673, + 4460, + 11, + 6225, + 10, + 246, + 506, + 466, + 438, + 176, + 428, + 4, + 2 + ], + "tokens": [ + "", + "ãĤ³", + "ãĥ¼ãĥ", + "ī", + "ãĥ¬", + "ãĥĵ", + "ãĥ¥", + "ãĥ¼", + ":", + "Ġbug", + "Ġfixed", + "Ġin", + "Ġcommit", + "Ġa", + "3", + "f", + "9", + "c", + "2", + "b", + ".", + "" + ], + "decoded_with_special": "コードレビュー: bug fixed in commit a3f9c2b.", + "decoded_skip_special": "コードレビュー: bug fixed in commit a3f9c2b." + }, + { + "id": "math-symbols-summation", + "input_ids": [ + 0, + 48136, + 35, + 43662, + 2469, + 3023, + 1215, + 118, + 5457, + 3330, + 36, + 8569, + 3023, + 47444, + 23133, + 14333, + 11936, + 46, + 322, + 2 + ], + "tokens": [ + "", + "Result", + ":", + "ĠÎ", + "£", + "Ġx", + "_", + "i", + "Ġ=", + "Ġ42", + "Ġ(", + "where", + "Ġx", + "ĠâĪ", + "Ī", + "Ġâ", + "Ħ", + "Ŀ", + ").", + "" + ], + "decoded_with_special": "Result: Σ x_i = 42 (where x ∈ ℝ).", + "decoded_skip_special": "Result: Σ x_i = 42 (where x ∈ ℝ)." + }, + { + "id": "math-symbols-epsilon-delta", + "input_ids": [ + 0, + 133, + 43375, + 35, + 47444, + 7471, + 47016, + 15698, + 288, + 47444, + 862, + 41335, + 20024, + 15698, + 288, + 215, + 14, + 1721, + 1178, + 12, + 1178, + 24987, + 9264, + 7471, + 15483, + 41552, + 41335, + 20024, + 48850, + 10659, + 1721, + 506, + 1640, + 1178, + 19281, + 506, + 1640, + 1178, + 24987, + 9264, + 7471, + 49275, + 41552, + 47016, + 4, + 2 + ], + "tokens": [ + "", + "The", + "orem", + ":", + "ĠâĪ", + "Ģ", + "ε", + ">", + "0", + "ĠâĪ", + "ĥ", + "Î", + "´", + ">", + "0", + "Ġsuch", + "Ġthat", + "Ġ|", + "x", + "-", + "x", + "â", + "Ĥ", + "Ģ", + "|", + "<", + "Î", + "´", + "Ġâĩ", + "Ĵ", + "Ġ|", + "f", + "(", + "x", + ")-", + "f", + "(", + "x", + "â", + "Ĥ", + "Ģ", + ")|", + "<", + "ε", + ".", + "" + ], + "decoded_with_special": "Theorem: ∀ε>0 ∃δ>0 such that |x-x₀|<δ ⇒ |f(x)-f(x₀)|<ε.", + "decoded_skip_special": "Theorem: ∀ε>0 ∃δ>0 such that |x-x₀|<δ ⇒ |f(x)-f(x₀)|<ε." + }, + { + "id": "currency-symbols-multi", + "input_ids": [ + 0, + 34985, + 35, + 4480, + 1092, + 4, + 1096, + 42484, + 30844, + 134, + 6, + 34212, + 48981, + 23133, + 984, + 698, + 4, + 2545, + 36, + 3340, + 31726, + 6, + 2704, + 14333, + 9264, + 9470, + 288, + 4, + 151, + 306, + 322, + 2 + ], + "tokens": [ + "", + "Cost", + ":", + "ĠâĤ¬", + "12", + ".", + "50", + "ĠâĨĴ", + "ĠÂ¥", + "1", + ",", + "820", + "Ġâī", + "Ī", + "Ġ£", + "10", + ".", + "75", + "Ġ(", + "app", + "rox", + ",", + "Ġplus", + "Ġâ", + "Ĥ", + "¿", + "0", + ".", + "000", + "4", + ").", + "" + ], + "decoded_with_special": "Cost: €12.50 → ¥1,820 ≈ £10.75 (approx, plus ₿0.0004).", + "decoded_skip_special": "Cost: €12.50 → ¥1,820 ≈ £10.75 (approx, plus ₿0.0004)." + }, + { + "id": "box-drawing", + "input_ids": [ + 0, + 26621, + 35, + 46277, + 14285, + 49884, + 49546, + 49192, + 47534, + 16948, + 48925, + 20920, + 48925, + 46277, + 10674, + 49884, + 49546, + 49192, + 47534, + 711, + 93, + 626, + 4, + 2 + ], + "tokens": [ + "", + "Box", + ":", + "ĠâĶ", + "Į", + "âĶĢâĶĢâĶĢâĶĢ", + "âĶĢâĶĢ", + "âĶĢ", + "âĶ", + "IJ", + "ĠâĶĤ", + "ĠHello", + "ĠâĶĤ", + "ĠâĶ", + "Ķ", + "âĶĢâĶĢâĶĢâĶĢ", + "âĶĢâĶĢ", + "âĶĢ", + "âĶ", + "ĺ", + "ĠâĢĶ", + "Ġdone", + ".", + "" + ], + "decoded_with_special": "Box: ┌───────┐ │ Hello │ └───────┘ — done.", + "decoded_skip_special": "Box: ┌───────┐ │ Hello │ └───────┘ — done." + }, + { + "id": "dingbats-chess", + "input_ids": [ + 0, + 40941, + 4643, + 35, + 41270, + 42699, + 27819, + 29293, + 21402, + 29293, + 2469, + 29293, + 8210, + 29293, + 18164, + 29293, + 10674, + 29293, + 15722, + 29293, + 25448, + 29293, + 711, + 29293, + 27, + 93, + 25109, + 278, + 4, + 2 + ], + "tokens": [ + "", + "Pie", + "ces", + ":", + "Ġâĺħ", + "Ġâĺ", + "Ĩ", + "ĠâĻ", + "ł", + "ĠâĻ", + "£", + "ĠâĻ", + "¥", + "ĠâĻ", + "¦", + "ĠâĻ", + "Ķ", + "ĠâĻ", + "ķ", + "ĠâĻ", + "ĸ", + "ĠâĻ", + "ĺ", + "ĠâĻ", + "Ļ", + "ĠâĢĶ", + "Ġchess", + "Ġset", + ".", + "" + ], + "decoded_with_special": "Pieces: ★ ☆ ♠ ♣ ♥ ♦ ♔ ♕ ♖ ♘ ♙ — chess set.", + "decoded_skip_special": "Pieces: ★ ☆ ♠ ♣ ♥ ♦ ♔ ♕ ♖ ♘ ♙ — chess set." + }, + { + "id": "math-astral-bold-script", + "input_ids": [ + 0, + 44853, + 7457, + 35, + 1437, + 40756, + 16948, + 7471, + 40756, + 16948, + 10172, + 40756, + 16948, + 9264, + 40756, + 16948, + 862, + 1437, + 40756, + 4333, + 12736, + 40756, + 4333, + 9357, + 40756, + 4333, + 16948, + 40756, + 4333, + 3602, + 93, + 10638, + 8543, + 35, + 1437, + 40756, + 9085, + 16948, + 40756, + 9085, + 3602, + 40756, + 9085, + 10659, + 40756, + 9085, + 9085, + 4, + 2 + ], + "tokens": [ + "", + "Math", + "Ġbold", + ":", + "Ġ", + "ðĿ", + "IJ", + "Ģ", + "ðĿ", + "IJ", + "ģ", + "ðĿ", + "IJ", + "Ĥ", + "ðĿ", + "IJ", + "ĥ", + "Ġ", + "ðĿ", + "Ł", + "İ", + "ðĿ", + "Ł", + "ı", + "ðĿ", + "Ł", + "IJ", + "ðĿ", + "Ł", + "ij", + "ĠâĢĶ", + "Ġmath", + "Ġscript", + ":", + "Ġ", + "ðĿ", + "ĵ", + "IJ", + "ðĿ", + "ĵ", + "ij", + "ðĿ", + "ĵ", + "Ĵ", + "ðĿ", + "ĵ", + "ĵ", + ".", + "" + ], + "decoded_with_special": "Math bold: 𝐀𝐁𝐂𝐃 𝟎𝟏𝟐𝟑 — math script: 𝓐𝓑𝓒𝓓.", + "decoded_skip_special": "Math bold: 𝐀𝐁𝐂𝐃 𝟎𝟏𝟐𝟑 — math script: 𝓐𝓑𝓒𝓓." + }, + { + "id": "astral-egyptian-hieroglyphs", + "input_ids": [ + 0, + 37552, + 811, + 29588, + 39218, + 3792, + 29, + 35, + 1437, + 49585, + 9085, + 7471, + 7471, + 1437, + 49585, + 9085, + 9264, + 7471, + 1437, + 49585, + 9085, + 862, + 7471, + 1437, + 49585, + 9085, + 27819, + 12736, + 49585, + 9085, + 5782, + 9085, + 49585, + 9085, + 9357, + 9357, + 49585, + 9085, + 27969, + 25448, + 4, + 2 + ], + "tokens": [ + "", + "Egypt", + "ian", + "Ġhier", + "ogly", + "ph", + "s", + ":", + "Ġ", + "ð", + "ĵ", + "Ģ", + "Ģ", + "Ġ", + "ð", + "ĵ", + "Ĥ", + "Ģ", + "Ġ", + "ð", + "ĵ", + "ĥ", + "Ģ", + "Ġ", + "ð", + "ĵ", + "Ĩ", + "İ", + "ð", + "ĵ", + "ħ", + "ĵ", + "ð", + "ĵ", + "ı", + "ı", + "ð", + "ĵ", + "Ĭ", + "ĸ", + ".", + "" + ], + "decoded_with_special": "Egyptian hieroglyphs: 𓀀 𓂀 𓃀 𓆎𓅓𓏏𓊖.", + "decoded_skip_special": "Egyptian hieroglyphs: 𓀀 𓂀 𓃀 𓆎𓅓𓏏𓊖." + }, + { + "id": "astral-cuneiform", + "input_ids": [ + 0, + 347, + 4438, + 38263, + 35, + 1437, + 49585, + 10659, + 7471, + 12410, + 1437, + 49585, + 10659, + 23133, + 21402, + 1437, + 49585, + 10659, + 9264, + 6800, + 1437, + 49585, + 10659, + 27819, + 21402, + 93, + 1203, + 889, + 7931, + 4, + 2 + ], + "tokens": [ + "", + "C", + "une", + "iform", + ":", + "Ġ", + "ð", + "Ĵ", + "Ģ", + "Ń", + "Ġ", + "ð", + "Ĵ", + "Ī", + "ł", + "Ġ", + "ð", + "Ĵ", + "Ĥ", + "Ĺ", + "Ġ", + "ð", + "Ĵ", + "Ĩ", + "ł", + "ĠâĢĶ", + "Ġsign", + "Ġlist", + "Ġsamples", + ".", + "" + ], + "decoded_with_special": "Cuneiform: 𒀭 𒈠 𒂗 𒆠 — sign list samples.", + "decoded_skip_special": "Cuneiform: 𒀭 𒈠 𒂗 𒆠 — sign list samples." + }, + { + "id": "astral-mahjong", + "input_ids": [ + 0, + 28681, + 40435, + 26779, + 35, + 8103, + 7471, + 7471, + 8103, + 7471, + 10172, + 8103, + 7471, + 9264, + 8103, + 7471, + 862, + 8103, + 7471, + 11936, + 8103, + 7471, + 5782, + 8103, + 7471, + 27819, + 8103, + 7471, + 6382, + 4, + 2 + ], + "tokens": [ + "", + "Mah", + "jong", + "Ġtiles", + ":", + "ĠðŁ", + "Ģ", + "Ģ", + "ĠðŁ", + "Ģ", + "ģ", + "ĠðŁ", + "Ģ", + "Ĥ", + "ĠðŁ", + "Ģ", + "ĥ", + "ĠðŁ", + "Ģ", + "Ħ", + "ĠðŁ", + "Ģ", + "ħ", + "ĠðŁ", + "Ģ", + "Ĩ", + "ĠðŁ", + "Ģ", + "ĩ", + ".", + "" + ], + "decoded_with_special": "Mahjong tiles: 🀀 🀁 🀂 🀃 🀄 🀅 🀆 🀇.", + "decoded_skip_special": "Mahjong tiles: 🀀 🀁 🀂 🀃 🀄 🀅 🀆 🀇." + }, + { + "id": "astral-cards", + "input_ids": [ + 0, + 347, + 5954, + 35, + 8103, + 862, + 10172, + 8103, + 862, + 9264, + 8103, + 862, + 862, + 8103, + 9264, + 5543, + 8103, + 9264, + 7258, + 8103, + 9264, + 2469, + 4, + 2 + ], + "tokens": [ + "", + "C", + "ards", + ":", + "ĠðŁ", + "ĥ", + "ģ", + "ĠðŁ", + "ĥ", + "Ĥ", + "ĠðŁ", + "ĥ", + "ĥ", + "ĠðŁ", + "Ĥ", + "¡", + "ĠðŁ", + "Ĥ", + "¢", + "ĠðŁ", + "Ĥ", + "£", + ".", + "" + ], + "decoded_with_special": "Cards: 🃁 🃂 🃃 🂡 🂢 🂣.", + "decoded_skip_special": "Cards: 🃁 🃂 🃃 🂡 🂢 🂣." + }, + { + "id": "emoji-bmp-and-astral", + "input_ids": [ + 0, + 31193, + 13859, + 20920, + 328, + 8103, + 14285, + 8384, + 623, + 8103, + 15113, + 7471, + 1709, + 8103, + 12736, + 23171, + 537, + 8103, + 12736, + 9264, + 4, + 2 + ], + "tokens": [ + "", + "ðŁij", + "ĭ", + "ĠHello", + "!", + "ĠðŁ", + "Į", + "į", + "ĠWorld", + "ĠðŁ", + "ļ", + "Ģ", + "Ġlaunch", + "ĠðŁ", + "İ", + "ī", + "Ġparty", + "ĠðŁ", + "İ", + "Ĥ", + ".", + "" + ], + "decoded_with_special": "👋 Hello! 🌍 World 🚀 launch 🎉 party 🎂.", + "decoded_skip_special": "👋 Hello! 🌍 World 🚀 launch 🎉 party 🎂." + }, + { + "id": "emoji-zwj-family-pride-skin", + "input_ids": [ + 0, + 27818, + 35, + 26964, + 11423, + 17, + 8384, + 31193, + 15375, + 17, + 8384, + 31193, + 6248, + 17, + 8384, + 31193, + 18164, + 93, + 13170, + 3794, + 35, + 8103, + 9357, + 15264, + 12605, + 17, + 8384, + 6569, + 14285, + 23133, + 93, + 21814, + 12, + 32612, + 35, + 26964, + 8384, + 6569, + 9357, + 10809, + 36174, + 27969, + 6569, + 9357, + 9470, + 26964, + 19002, + 6569, + 9357, + 2023, + 4, + 2 + ], + "tokens": [ + "", + "Family", + ":", + "ĠðŁij", + "¨", + "âĢ", + "į", + "ðŁij", + "©", + "âĢ", + "į", + "ðŁij", + "§", + "âĢ", + "į", + "ðŁij", + "¦", + "ĠâĢĶ", + "ĠPride", + "Ġflag", + ":", + "ĠðŁ", + "ı", + "³", + "ï¸ı", + "âĢ", + "į", + "ðŁ", + "Į", + "Ī", + "ĠâĢĶ", + "ĠSkin", + "-", + "tone", + ":", + "ĠðŁij", + "į", + "ðŁ", + "ı", + "½", + "Ġâľ", + "Ĭ", + "ðŁ", + "ı", + "¿", + "ĠðŁij", + "¶", + "ðŁ", + "ı", + "»", + ".", + "" + ], + "decoded_with_special": "Family: 👨‍👩‍👧‍👦 — Pride flag: 🏳️‍🌈 — Skin-tone: 👍🏽 ✊🏿 👶🏻.", + "decoded_skip_special": "Family: 👨‍👩‍👧‍👦 — Pride flag: 🏳️‍🌈 — Skin-tone: 👍🏽 ✊🏿 👶🏻." + }, + { + "id": "emoji-zwj-with-text-prefix", + "input_ids": [ + 0, + 6569, + 9357, + 15264, + 12605, + 17, + 8384, + 6569, + 14285, + 23133, + 21077, + 1949, + 2744, + 31193, + 11423, + 17, + 8384, + 31193, + 15375, + 17, + 8384, + 31193, + 6248, + 17, + 8384, + 31193, + 18164, + 27818, + 2744, + 6569, + 6382, + 15375, + 6569, + 6382, + 10278, + 6569, + 6382, + 10965, + 6569, + 6382, + 8906, + 6569, + 6382, + 3070, + 6569, + 6382, + 18537, + 46760, + 2744, + 21959, + 2 + ], + "tokens": [ + "", + "ðŁ", + "ı", + "³", + "ï¸ı", + "âĢ", + "į", + "ðŁ", + "Į", + "Ī", + "Pr", + "ide", + "+", + "ðŁij", + "¨", + "âĢ", + "į", + "ðŁij", + "©", + "âĢ", + "į", + "ðŁij", + "§", + "âĢ", + "į", + "ðŁij", + "¦", + "Family", + "+", + "ðŁ", + "ĩ", + "©", + "ðŁ", + "ĩ", + "ª", + "ðŁ", + "ĩ", + "¯", + "ðŁ", + "ĩ", + "µ", + "ðŁ", + "ĩ", + "º", + "ðŁ", + "ĩ", + "¸", + "flags", + "+", + "test", + "" + ], + "decoded_with_special": "🏳️‍🌈Pride+👨‍👩‍👧‍👦Family+🇩🇪🇯🇵🇺🇸flags+test", + "decoded_skip_special": "🏳️‍🌈Pride+👨‍👩‍👧‍👦Family+🇩🇪🇯🇵🇺🇸flags+test" + }, + { + "id": "emoji-keycap-and-flags", + "input_ids": [ + 0, + 28152, + 25879, + 35, + 112, + 12605, + 24987, + 862, + 2469, + 132, + 12605, + 24987, + 862, + 2469, + 155, + 12605, + 24987, + 862, + 2469, + 93, + 3794, + 35, + 8103, + 6382, + 10965, + 6569, + 6382, + 8906, + 8103, + 6382, + 15375, + 6569, + 6382, + 10278, + 8103, + 6382, + 3070, + 6569, + 6382, + 18537, + 4, + 2 + ], + "tokens": [ + "", + "Key", + "caps", + ":", + "Ġ1", + "ï¸ı", + "â", + "ĥ", + "£", + "Ġ2", + "ï¸ı", + "â", + "ĥ", + "£", + "Ġ3", + "ï¸ı", + "â", + "ĥ", + "£", + "ĠâĢĶ", + "Ġflag", + ":", + "ĠðŁ", + "ĩ", + "¯", + "ðŁ", + "ĩ", + "µ", + "ĠðŁ", + "ĩ", + "©", + "ðŁ", + "ĩ", + "ª", + "ĠðŁ", + "ĩ", + "º", + "ðŁ", + "ĩ", + "¸", + ".", + "" + ], + "decoded_with_special": "Keycaps: 1️⃣ 2️⃣ 3️⃣ — flag: 🇯🇵 🇩🇪 🇺🇸.", + "decoded_skip_special": "Keycaps: 1️⃣ 2️⃣ 3️⃣ — flag: 🇯🇵 🇩🇪 🇺🇸." + }, + { + "id": "url-with-query-fragment", + "input_ids": [ + 0, + 6423, + 5214, + 13082, + 640, + 46781, + 4, + 175, + 73, + 22609, + 73, + 560, + 73, + 44814, + 116, + 1343, + 5214, + 48359, + 947, + 4901, + 5214, + 428, + 1222, + 947, + 282, + 5214, + 3714, + 10431, + 22580, + 12, + 246, + 2 + ], + "tokens": [ + "", + "url", + "=", + "https", + "://", + "example", + ".", + "com", + "/", + "path", + "/", + "to", + "/", + "resource", + "?", + "q", + "=", + "foo", + "&", + "bar", + "=", + "b", + "az", + "&", + "n", + "=", + "42", + "#", + "section", + "-", + "3", + "" + ], + "decoded_with_special": "url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3", + "decoded_skip_special": "url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3" + }, + { + "id": "hex-and-base64", + "input_ids": [ + 0, + 43726, + 5214, + 288, + 1178, + 10089, + 2606, + 387, + 9993, + 5268, + 8573, + 717, + 3813, + 8827, + 1542, + 4027, + 5214, + 975, + 771, + 863, + 267, + 1301, + 37838, + 119, + 1301, + 176, + 24548, + 424, + 1872, + 428, + 771, + 245, + 36940, + 250, + 43292, + 2 + ], + "tokens": [ + "", + "hex", + "=", + "0", + "x", + "DE", + "AD", + "B", + "EE", + "FC", + "AF", + "E", + "BA", + "BE", + "Ġbase", + "64", + "=", + "Y", + "W", + "J", + "j", + "Z", + "GV", + "m", + "Z", + "2", + "hp", + "am", + "ts", + "b", + "W", + "5", + "vc", + "A", + "==", + "" + ], + "decoded_with_special": "hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==", + "decoded_skip_special": "hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==" + }, + { + "id": "sha256-hash", + "input_ids": [ + 0, + 25410, + 24554, + 5214, + 242, + 246, + 428, + 288, + 438, + 3305, + 32646, + 25484, + 134, + 438, + 22928, + 2001, + 36920, + 306, + 438, + 398, + 39659, + 36028, + 466, + 1978, + 2518, + 4791, + 4006, + 242, + 306, + 33787, + 428, + 466, + 3079, + 3245, + 30973, + 2831, + 134, + 428, + 5479, + 4429, + 428, + 33450, + 2 + ], + "tokens": [ + "", + "sha", + "256", + "=", + "e", + "3", + "b", + "0", + "c", + "44", + "298", + "fc", + "1", + "c", + "149", + "af", + "bf", + "4", + "c", + "8", + "996", + "fb", + "9", + "24", + "27", + "ae", + "41", + "e", + "4", + "649", + "b", + "9", + "34", + "ca", + "495", + "99", + "1", + "b", + "78", + "52", + "b", + "855", + "" + ], + "decoded_with_special": "sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "decoded_skip_special": "sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "id": "code-javascript-functional", + "input_ids": [ + 0, + 6166, + 4, + 32557, + 1640, + 1178, + 46161, + 3023, + 1009, + 132, + 322, + 46617, + 1640, + 1178, + 46161, + 3023, + 8061, + 321, + 322, + 2050, + 15431, + 48461, + 102, + 6, + 741, + 43, + 46161, + 10, + 2055, + 741, + 4397, + 2 + ], + "tokens": [ + "", + "arr", + ".", + "map", + "(", + "x", + "Ġ=>", + "Ġx", + "Ġ*", + "Ġ2", + ").", + "filter", + "(", + "x", + "Ġ=>", + "Ġx", + "Ġ>", + "Ġ0", + ").", + "red", + "uce", + "((", + "a", + ",", + "Ġb", + ")", + "Ġ=>", + "Ġa", + "Ġ+", + "Ġb", + ");", + "" + ], + "decoded_with_special": "arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);", + "decoded_skip_special": "arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);" + }, + { + "id": "code-cplusplus-template", + "input_ids": [ + 0, + 48790, + 41552, + 565, + 6, + 791, + 15698, + 4485, + 856, + 1640, + 565, + 49145, + 90, + 6, + 121, + 49145, + 257, + 43, + 43839, + 16163, + 12528, + 1640, + 90, + 2744, + 257, + 43, + 25522, + 671, + 326, + 2055, + 1717, + 131, + 35524, + 2 + ], + "tokens": [ + "", + "template", + "<", + "T", + ",", + "U", + ">", + "Ġauto", + "Ġf", + "(", + "T", + "&&", + "t", + ",", + "ĠU", + "&&", + "u", + ")", + "Ġ->", + "Ġdecl", + "type", + "(", + "t", + "+", + "u", + ")", + "Ġ{", + "Ġreturn", + "Ġt", + "Ġ+", + "Ġu", + ";", + "Ġ}", + "" + ], + "decoded_with_special": "template auto f(T&&t, U&&u) -> decltype(t+u) { return t + u; }", + "decoded_skip_special": "template auto f(T&&t, U&&u) -> decltype(t+u) { return t + u; }" + }, + { + "id": "code-bitops", + "input_ids": [ + 0, + 43155, + 5457, + 36, + 102, + 8488, + 132, + 43, + 1721, + 36, + 428, + 48188, + 204, + 43, + 359, + 321, + 1178, + 7389, + 131, + 2 + ], + "tokens": [ + "", + "result", + "Ġ=", + "Ġ(", + "a", + "Ġ>>", + "Ġ2", + ")", + "Ġ|", + "Ġ(", + "b", + "Ġ<<", + "Ġ4", + ")", + "Ġ&", + "Ġ0", + "x", + "FF", + ";", + "" + ], + "decoded_with_special": "result = (a >> 2) | (b << 4) & 0xFF;", + "decoded_skip_special": "result = (a >> 2) | (b << 4) & 0xFF;" + }, + { + "id": "code-javascript-modern", + "input_ids": [ + 0, + 1594, + 36, + 1178, + 27785, + 43292, + 23796, + 48200, + 1423, + 49095, + 321, + 43, + 25522, + 671, + 3023, + 116, + 4, + 48359, + 42254, + 6814, + 33977, + 131, + 35524, + 2 + ], + "tokens": [ + "", + "if", + "Ġ(", + "x", + "Ġ!", + "==", + "Ġnull", + "Ġ&&", + "Ġy", + "Ġ>=", + "Ġ0", + ")", + "Ġ{", + "Ġreturn", + "Ġx", + "?", + ".", + "foo", + "Ġ??", + "Ġdefault", + "Value", + ";", + "Ġ}", + "" + ], + "decoded_with_special": "if (x !== null && y >= 0) { return x?.foo ?? defaultValue; }", + "decoded_skip_special": "if (x !== null && y >= 0) { return x?.foo ?? defaultValue; }" + }, + { + "id": "code-python-def", + "input_ids": [ + 0, + 9232, + 19961, + 261, + 29522, + 1640, + 282, + 3256, + 2 + ], + "tokens": [ + "", + "def", + "Ġfib", + "on", + "acci", + "(", + "n", + "):", + "" + ], + "decoded_with_special": "def fibonacci(n):", + "decoded_skip_special": "def fibonacci(n):" + }, + { + "id": "code-python-if", + "input_ids": [ + 0, + 1437, + 1437, + 1437, + 114, + 295, + 28696, + 132, + 35, + 2 + ], + "tokens": [ + "", + "Ġ", + "Ġ", + "Ġ", + "Ġif", + "Ġn", + "Ġ<", + "Ġ2", + ":", + "" + ], + "decoded_with_special": " if n < 2:", + "decoded_skip_special": " if n < 2:" + }, + { + "id": "code-python-return", + "input_ids": [ + 0, + 1437, + 1437, + 1437, + 1437, + 1437, + 1437, + 1437, + 671, + 295, + 2 + ], + "tokens": [ + "", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġreturn", + "Ġn", + "" + ], + "decoded_with_special": " return n", + "decoded_skip_special": " return n" + }, + { + "id": "code-python-recurse", + "input_ids": [ + 0, + 1437, + 1437, + 1437, + 671, + 19961, + 261, + 29522, + 1640, + 282, + 111, + 112, + 43, + 2055, + 19961, + 261, + 29522, + 1640, + 282, + 111, + 132, + 43, + 2 + ], + "tokens": [ + "", + "Ġ", + "Ġ", + "Ġ", + "Ġreturn", + "Ġfib", + "on", + "acci", + "(", + "n", + "Ġ-", + "Ġ1", + ")", + "Ġ+", + "Ġfib", + "on", + "acci", + "(", + "n", + "Ġ-", + "Ġ2", + ")", + "" + ], + "decoded_with_special": " return fibonacci(n - 1) + fibonacci(n - 2)", + "decoded_skip_special": " return fibonacci(n - 1) + fibonacci(n - 2)" + }, + { + "id": "code-go-func", + "input_ids": [ + 0, + 48901, + 17395, + 1640, + 13650, + 6755, + 43, + 25522, + 2 + ], + "tokens": [ + "", + "func", + "Ġgreet", + "(", + "name", + "Ġstring", + ")", + "Ġ{", + "" + ], + "decoded_with_special": "func greet(name string) {", + "decoded_skip_special": "func greet(name string) {" + }, + { + "id": "code-go-tab-if", + "input_ids": [ + 0, + 50117, + 1594, + 766, + 49333, + 41039, + 25522, + 2 + ], + "tokens": [ + "", + "ĉ", + "if", + "Ġname", + "Ġ!=", + "Ġ\"\"", + "Ġ{", + "" + ], + "decoded_with_special": "\tif name != \"\" {", + "decoded_skip_special": "\tif name != \"\" {" + }, + { + "id": "code-go-tab-printf", + "input_ids": [ + 0, + 50117, + 50117, + 506, + 16100, + 4, + 43945, + 506, + 46469, + 31414, + 6, + 7606, + 29, + 328, + 37457, + 282, + 1297, + 766, + 43, + 2 + ], + "tokens": [ + "", + "ĉ", + "ĉ", + "f", + "mt", + ".", + "Print", + "f", + "(\"", + "Hello", + ",", + "Ġ%", + "s", + "!", + "\\", + "n", + "\",", + "Ġname", + ")", + "" + ], + "decoded_with_special": "\t\tfmt.Printf(\"Hello, %s!\\n\", name)", + "decoded_skip_special": "\t\tfmt.Printf(\"Hello, %s!\\n\", name)" + }, + { + "id": "code-go-tab-close", + "input_ids": [ + 0, + 50117, + 24303, + 2 + ], + "tokens": [ + "", + "ĉ", + "}", + "" + ], + "decoded_with_special": "\t}", + "decoded_skip_special": "\t}" + }, + { + "id": "code-go-close", + "input_ids": [ + 0, + 24303, + 2 + ], + "tokens": [ + "", + "}", + "" + ], + "decoded_with_special": "}", + "decoded_skip_special": "}" + }, + { + "id": "programming-identifiers-mixed", + "input_ids": [ + 0, + 6460, + 1215, + 12105, + 1215, + 808, + 28066, + 1215, + 19159, + 45234, + 1215, + 49340, + 255, + 1215, + 19320, + 1215, + 46657, + 1215, + 808, + 15400, + 2492, + 10463, + 21308, + 49602, + 2 + ], + "tokens": [ + "", + "get", + "_", + "user", + "_", + "id", + "ĠMAX", + "_", + "BU", + "FFER", + "_", + "SIZE", + "ĠT", + "_", + "pad", + "_", + "token", + "_", + "id", + "ĠNS", + "UR", + "LS", + "ession", + "Configuration", + "" + ], + "decoded_with_special": "get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration", + "decoded_skip_special": "get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration" + }, + { + "id": "programming-identifiers-cases", + "input_ids": [ + 0, + 438, + 22044, + 38834, + 27717, + 27014, + 38834, + 16173, + 1215, + 11173, + 1215, + 10806, + 4998, + 28057, + 1862, + 1215, + 104, + 4444, + 6961, + 449, + 398, + 29, + 1215, + 40762, + 1215, + 11432, + 1215, + 705, + 176, + 2 + ], + "tokens": [ + "", + "c", + "amel", + "Case", + "Var", + "ĠPascal", + "Case", + "Ġsnake", + "_", + "case", + "_", + "var", + "ĠSC", + "REAM", + "ING", + "_", + "S", + "NA", + "KE", + "Ġk", + "8", + "s", + "_", + "pod", + "_", + "count", + "_", + "v", + "2", + "" + ], + "decoded_with_special": "camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2", + "decoded_skip_special": "camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2" + }, + { + "id": "code-string-multiscript", + "input_ids": [ + 0, + 571, + 14081, + 154, + 5457, + 22, + 25417, + 4333, + 24269, + 35328, + 42907, + 25482, + 35555, + 6, + 18697, + 4394, + 35328, + 24269, + 2901, + 2055, + 22, + 93, + 31, + 22, + 2055, + 22, + 48414, + 15389, + 46499, + 11582, + 113, + 2 + ], + "tokens": [ + "", + "g", + "reet", + "ing", + "Ġ=", + "Ġ\"", + "Ð", + "Ł", + "ÑĢ", + "и", + "в", + "е", + "ÑĤ", + ",", + "ĠÐ", + "¼", + "и", + "ÑĢ", + "!\"", + "Ġ+", + "Ġ\"", + "ĠâĢĶ", + "Ġfrom", + "Ġ\"", + "Ġ+", + "Ġ\"", + "æĿ", + "±", + "äº", + "¬", + "\"", + "" + ], + "decoded_with_special": "greeting = \"Привет, мир!\" + \" — from \" + \"東京\"", + "decoded_skip_special": "greeting = \"Привет, мир!\" + \" — from \" + \"東京\"" + }, + { + "id": "code-comment-diacritics", + "input_ids": [ + 0, + 42326, + 15367, + 18581, + 1140, + 93, + 9345, + 1129, + 19, + 2269, + 1043, + 3961, + 2857, + 2 + ], + "tokens": [ + "", + "//", + "Ġré", + "sum", + "é", + "ĠâĢĶ", + "ĠSwift", + "Ġcomment", + "Ġwith", + "Ġdi", + "ac", + "rit", + "ics", + "" + ], + "decoded_with_special": "// résumé — Swift comment with diacritics", + "decoded_skip_special": "// résumé — Swift comment with diacritics" + }, + { + "id": "code-comment-greek-math", + "input_ids": [ + 0, + 49051, + 47188, + 2055, + 47743, + 5457, + 43662, + 15264, + 93, + 5617, + 12, + 36419, + 10638, + 1129, + 48404, + 2 + ], + "tokens": [ + "", + "/*", + "Ġα", + "Ġ+", + "Ġβ", + "Ġ=", + "ĠÎ", + "³", + "ĠâĢĶ", + "ĠGreek", + "-", + "letter", + "Ġmath", + "Ġcomment", + "Ġ*/", + "" + ], + "decoded_with_special": "/* α + β = γ — Greek-letter math comment */", + "decoded_skip_special": "/* α + β = γ — Greek-letter math comment */" + }, + { + "id": "code-string-multiscript-emoji", + "input_ids": [ + 0, + 2716, + 1270, + 5457, + 22, + 6569, + 15113, + 7471, + 22217, + 93, + 18697, + 9470, + 42488, + 36709, + 41171, + 93, + 47897, + 8906, + 18400, + 48675, + 15722, + 113, + 2 + ], + "tokens": [ + "", + "let", + "Ġtitle", + "Ġ=", + "Ġ\"", + "ðŁ", + "ļ", + "Ģ", + "ĠLaunch", + "ĠâĢĶ", + "ĠÐ", + "¿", + "Ñĥ", + "Ñģ", + "к", + "ĠâĢĶ", + "Ġè", + "µ", + "·", + "åĭ", + "ķ", + "\"", + "" + ], + "decoded_with_special": "let title = \"🚀 Launch — пуск — 起動\"", + "decoded_skip_special": "let title = \"🚀 Launch — пуск — 起動\"" + }, + { + "id": "ipa-phonetic-transcription", + "input_ids": [ + 0, + 133, + 34472, + 37118, + 1589, + 506, + 35423, + 10278, + 38155, + 862, + 73, + 13, + 22, + 9106, + 113, + 93, + 19, + 3992, + 35, + 1437, + 49066, + 506, + 35423, + 10278, + 38155, + 862, + 4, + 2 + ], + "tokens": [ + "", + "The", + "ĠIPA", + "Ġtranscription", + "Ġ/", + "f", + "É", + "ª", + "Ê", + "ĥ", + "/", + "Ġfor", + "Ġ\"", + "fish", + "\"", + "ĠâĢĶ", + "Ġwith", + "Ġstress", + ":", + "Ġ", + "ËĪ", + "f", + "É", + "ª", + "Ê", + "ĥ", + ".", + "" + ], + "decoded_with_special": "The IPA transcription /fɪʃ/ for \"fish\" — with stress: ˈfɪʃ.", + "decoded_skip_special": "The IPA transcription /fɪʃ/ for \"fish\" — with stress: ˈfɪʃ." + }, + { + "id": "mandarin-tones", + "input_ids": [ + 0, + 565, + 6909, + 35, + 475, + 1526, + 475, + 47040, + 12736, + 475, + 5269, + 475, + 18195, + 36, + 37136, + 15394, + 23225, + 93, + 239, + 6, + 2227, + 6, + 26501, + 6, + 3064, + 322, + 2 + ], + "tokens": [ + "", + "T", + "ones", + ":", + "Ġm", + "á", + "Ġm", + "Ç", + "İ", + "Ġm", + "Ãł", + "Ġm", + "Äģ", + "Ġ(", + "Mand", + "arin", + "Ġtones", + "ĠâĢĶ", + "Ġhigh", + ",", + "Ġrising", + ",", + "Ġdipping", + ",", + "Ġfalling", + ").", + "" + ], + "decoded_with_special": "Tones: má mǎ mà mā (Mandarin tones — high, rising, dipping, falling).", + "decoded_skip_special": "Tones: má mǎ mà mā (Mandarin tones — high, rising, dipping, falling)." + }, + { + "id": "whitespace-runs", + "input_ids": [ + 0, + 42891, + 1437, + 1437, + 1437, + 232, + 1437, + 1437, + 1437, + 1437, + 1437, + 1437, + 1437, + 1437, + 479, + 1437, + 1437, + 1437, + 1437, + 1437, + 1437, + 253, + 2 + ], + "tokens": [ + "", + "hello", + "Ġ", + "Ġ", + "Ġ", + "Ġworld", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ.", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġ", + "Ġend", + "" + ], + "decoded_with_special": "hello world . end", + "decoded_skip_special": "hello world . end" + }, + { + "id": "whitespace-trailing-tabs", + "input_ids": [ + 0, + 9738, + 7022, + 17942, + 18851, + 137, + 92, + 1902, + 35, + 50117, + 1640, + 34108, + 43, + 1437, + 1437, + 36, + 4182, + 11667, + 43, + 2 + ], + "tokens": [ + "", + "tra", + "iling", + "Ġwhites", + "pace", + "Ġbefore", + "Ġnew", + "line", + ":", + "ĉ", + "(", + "tab", + ")", + "Ġ", + "Ġ", + "Ġ(", + "sp", + "aces", + ")", + "" + ], + "decoded_with_special": "trailing whitespace before newline:\t(tab) (spaces)", + "decoded_skip_special": "trailing whitespace before newline:\t(tab) (spaces)" + }, + { + "id": "punctuation-period", + "input_ids": [ + 0, + 4, + 2 + ], + "tokens": [ + "", + ".", + "" + ], + "decoded_with_special": ".", + "decoded_skip_special": "." + }, + { + "id": "punctuation-comma", + "input_ids": [ + 0, + 6, + 2 + ], + "tokens": [ + "", + ",", + "" + ], + "decoded_with_special": ",", + "decoded_skip_special": "," + }, + { + "id": "punctuation-exclamation", + "input_ids": [ + 0, + 328, + 2 + ], + "tokens": [ + "", + "!", + "" + ], + "decoded_with_special": "!", + "decoded_skip_special": "!" + }, + { + "id": "punctuation-nested-quotes", + "input_ids": [ + 0, + 113, + 2264, + 1917, + 79, + 553, + 93, + 22, + 20982, + 43775, + 93, + 4420, + 734, + 2 + ], + "tokens": [ + "", + "\"", + "What", + "?\"", + "Ġshe", + "Ġasked", + "ĠâĢĶ", + "Ġ\"", + "really", + "?!\"", + "ĠâĢĶ", + "Ġyes", + "...", + "" + ], + "decoded_with_special": "\"What?\" she asked — \"really?!\" — yes...", + "decoded_skip_special": "\"What?\" she asked — \"really?!\" — yes..." + }, + { + "id": "punctuation-dash-variants", + "input_ids": [ + 0, + 734, + 8, + 44, + 10659, + 67, + 126, + 8, + 93, + 8, + 15130, + 4, + 2 + ], + "tokens": [ + "", + "...", + "Ġand", + "ĠâĢ", + "Ĵ", + "Ġalso", + "ĠâĢĵ", + "Ġand", + "ĠâĢĶ", + "Ġand", + "ĠâĢķ", + ".", + "" + ], + "decoded_with_special": "... and ‒ also – and — and ―.", + "decoded_skip_special": "... and ‒ also – and — and ―." + }, + { + "id": "multiscript-greetings", + "input_ids": [ + 0, + 31414, + 1437, + 47856, + 21402, + 48975, + 10809, + 46747, + 15722, + 23133, + 45209, + 5782, + 15722, + 48897, + 9085, + 48913, + 48016, + 46311, + 5543, + 48549, + 22214, + 11423, + 14238, + 2840, + 14238, + 18537, + 22919, + 8384, + 14238, + 10470, + 22919, + 6382, + 42620, + 29434, + 21593, + 12410, + 41117, + 29438, + 42455, + 15375, + 47759, + 46174, + 42358, + 46, + 1437, + 24107, + 10278, + 24107, + 6248, + 24107, + 15389, + 24107, + 10278, + 24107, + 10674, + 24107, + 8906, + 18697, + 4333, + 24269, + 35328, + 42907, + 25482, + 35555, + 43662, + 15113, + 44721, + 48191, + 41335, + 18400, + 47049, + 41335, + 12410, + 47630, + 44721, + 4, + 2 + ], + "tokens": [ + "", + "Hello", + "Ġ", + "ä½", + "ł", + "å¥", + "½", + "Ġì", + "ķ", + "Ī", + "ë", + "ħ", + "ķ", + "Ġãģ", + "ĵ", + "ãĤĵ", + "ãģ«", + "ãģ", + "¡", + "ãģ¯", + "Ġà¤", + "¨", + "à¤", + "®", + "à¤", + "¸", + "à¥", + "į", + "à¤", + "¤", + "à¥", + "ĩ", + "ĠÙħ", + "ر", + "Ø", + "Ń", + "ب", + "ا", + "Ġ×", + "©", + "׾", + "×ķ", + "×", + "Ŀ", + "Ġ", + "à¸", + "ª", + "à¸", + "§", + "à¸", + "±", + "à¸", + "ª", + "à¸", + "Ķ", + "à¸", + "µ", + "ĠÐ", + "Ł", + "ÑĢ", + "и", + "в", + "е", + "ÑĤ", + "ĠÎ", + "ļ", + "α", + "λ", + "Î", + "·", + "μ", + "Î", + "Ń", + "Ïģ", + "α", + ".", + "" + ], + "decoded_with_special": "Hello 你好 안녕 こんにちは नमस्ते مرحبا שלום สวัสดี Привет Καλημέρα.", + "decoded_skip_special": "Hello 你好 안녕 こんにちは नमस्ते مرحبا שלום สวัสดี Привет Καλημέρα." + }, + { + "id": "multiscript-no-spaces", + "input_ids": [ + 0, + 44721, + 47856, + 21402, + 47328, + 43998, + 15722, + 23133, + 48381, + 14238, + 11423, + 41335, + 20024, + 14238, + 18537, + 6569, + 10659, + 4958, + 41688, + 31193, + 13859, + 48352, + 6569, + 15113, + 7471, + 38605, + 48104, + 7471, + 48104, + 862, + 48104, + 23133, + 24987, + 6382, + 10659, + 2 + ], + "tokens": [ + "", + "α", + "ä½", + "ł", + "β", + "ì", + "ķ", + "Ī", + "γ", + "à¤", + "¨", + "Î", + "´", + "à¤", + "¸", + "ðŁ", + "Ĵ", + "«", + "ع", + "ðŁij", + "ĭ", + "ש", + "ðŁ", + "ļ", + "Ģ", + "ت", + "âĪ", + "Ģ", + "âĪ", + "ĥ", + "âĪ", + "Ī", + "â", + "ĩ", + "Ĵ", + "" + ], + "decoded_with_special": "α你β안γनδस💫ع👋ש🚀ت∀∃∈⇒", + "decoded_skip_special": "α你β안γनδस💫ع👋ש🚀ت∀∃∈⇒" + }, + { + "id": "multiscript-zwj-mixed", + "input_ids": [ + 0, + 47643, + 47240, + 6382, + 48589, + 48, + 46873, + 18537, + 7471, + 47954, + 8210, + 44636, + 11582, + 48537, + 17772, + 37127, + 18400, + 18400, + 31193, + 13859, + 48589, + 15375, + 48444, + 23133, + 48444, + 10470, + 6569, + 12736, + 23171, + 29015, + 37127, + 18400, + 18400, + 47504, + 23133, + 2 + ], + "tokens": [ + "", + "ä¸Ń", + "æĸ", + "ĩ", + "íķ", + "ľ", + "ê", + "¸", + "Ģ", + "æĹ", + "¥", + "æľ", + "¬", + "èª", + "ŀ", + "æ", + "·", + "·", + "ðŁij", + "ĭ", + "íķ", + "©", + "ëĭ", + "Ī", + "ëĭ", + "¤", + "ðŁ", + "İ", + "ī", + "text", + "æ", + "·", + "·", + "åIJ", + "Ī", + "" + ], + "decoded_with_special": "中文한글日本語混👋합니다🎉text混合", + "decoded_skip_special": "中文한글日本語混👋합니다🎉text混合" + }, + { + "id": "multiscript-string-concat", + "input_ids": [ + 0, + 571, + 14081, + 154, + 40635, + 47856, + 21402, + 48975, + 10809, + 113, + 2744, + 6569, + 14285, + 8384, + 2744, + 113, + 43998, + 15722, + 23133, + 45209, + 5782, + 15722, + 113, + 2744, + 14238, + 11423, + 14238, + 2840, + 14238, + 18537, + 22919, + 8384, + 14238, + 10470, + 22919, + 6382, + 2744, + 31193, + 11423, + 17, + 8384, + 31193, + 15375, + 17, + 8384, + 31193, + 6248, + 2 + ], + "tokens": [ + "", + "g", + "reet", + "ing", + "=\"", + "ä½", + "ł", + "å¥", + "½", + "\"", + "+", + "ðŁ", + "Į", + "į", + "+", + "\"", + "ì", + "ķ", + "Ī", + "ë", + "ħ", + "ķ", + "\"", + "+", + "à¤", + "¨", + "à¤", + "®", + "à¤", + "¸", + "à¥", + "į", + "à¤", + "¤", + "à¥", + "ĩ", + "+", + "ðŁij", + "¨", + "âĢ", + "į", + "ðŁij", + "©", + "âĢ", + "į", + "ðŁij", + "§", + "" + ], + "decoded_with_special": "greeting=\"你好\"+🌍+\"안녕\"+नमस्ते+👨‍👩‍👧", + "decoded_skip_special": "greeting=\"你好\"+🌍+\"안녕\"+नमस्ते+👨‍👩‍👧" + }, + { + "id": "multiscript-emoji-arrows", + "input_ids": [ + 0, + 41335, + 2469, + 3023, + 1215, + 118, + 8103, + 6382, + 10965, + 6569, + 6382, + 8906, + 48981, + 21402, + 47111, + 14285, + 6800, + 46499, + 11582, + 42484, + 28168, + 282, + 8224, + 13339, + 255, + 38183, + 4122, + 38183, + 13339, + 1437, + 48827, + 48818, + 18400, + 36174, + 11423, + 253, + 4, + 2 + ], + "tokens": [ + "", + "Î", + "£", + "Ġx", + "_", + "i", + "ĠðŁ", + "ĩ", + "¯", + "ðŁ", + "ĩ", + "µ", + "Ġâī", + "ł", + "Ġå", + "Į", + "Ĺ", + "äº", + "¬", + "ĠâĨĴ", + "ĠMü", + "n", + "chen", + "Ġ·", + "ĠT", + "Åį", + "ky", + "Åį", + "Ġ·", + "Ġ", + "ä¸Ĭ", + "æµ", + "·", + "Ġâľ", + "¨", + "Ġend", + ".", + "" + ], + "decoded_with_special": "Σ x_i 🇯🇵 ≠ 北京 → München · Tōkyō · 上海 ✨ end.", + "decoded_skip_special": "Σ x_i 🇯🇵 ≠ 北京 → München · Tōkyō · 上海 ✨ end." + }, + { + "id": "multiscript-arrow-chain", + "input_ids": [ + 0, + 6569, + 15113, + 7471, + 14238, + 11423, + 14238, + 2840, + 14238, + 18537, + 22919, + 8384, + 14238, + 10470, + 22919, + 6382, + 48552, + 25417, + 4333, + 24269, + 35328, + 42907, + 25482, + 35555, + 48552, + 47856, + 21402, + 48975, + 10809, + 48552, + 43998, + 15722, + 23133, + 45209, + 5782, + 15722, + 48552, + 33070, + 29434, + 21593, + 12410, + 41117, + 29438, + 48552, + 30151, + 10470, + 12605, + 2 + ], + "tokens": [ + "", + "ðŁ", + "ļ", + "Ģ", + "à¤", + "¨", + "à¤", + "®", + "à¤", + "¸", + "à¥", + "į", + "à¤", + "¤", + "à¥", + "ĩ", + "âĨĴ", + "Ð", + "Ł", + "ÑĢ", + "и", + "в", + "е", + "ÑĤ", + "âĨĴ", + "ä½", + "ł", + "å¥", + "½", + "âĨĴ", + "ì", + "ķ", + "Ī", + "ë", + "ħ", + "ķ", + "âĨĴ", + "Ùħ", + "ر", + "Ø", + "Ń", + "ب", + "ا", + "âĨĴ", + "âĿ", + "¤", + "ï¸ı", + "" + ], + "decoded_with_special": "🚀नमस्ते→Привет→你好→안녕→مرحبا→❤️", + "decoded_skip_special": "🚀नमस्ते→Привет→你好→안녕→مرحبا→❤️" + }, + { + "id": "multiscript-function-call", + "input_ids": [ + 0, + 48359, + 1640, + 47643, + 47240, + 6382, + 6, + 48589, + 48, + 46873, + 18537, + 7471, + 6, + 47954, + 8210, + 44636, + 11582, + 48537, + 17772, + 6, + 14238, + 9253, + 14238, + 9470, + 14238, + 11423, + 22919, + 8384, + 14238, + 18164, + 22919, + 7471, + 6, + 24269, + 42488, + 36709, + 36709, + 41171, + 35328, + 25417, + 9253, + 43, + 48552, + 45152, + 48104, + 7471, + 35, + 48104, + 862, + 6, + 24987, + 27969, + 15722, + 35, + 24987, + 27969, + 6800, + 24303, + 2 + ], + "tokens": [ + "", + "foo", + "(", + "ä¸Ń", + "æĸ", + "ĩ", + ",", + "íķ", + "ľ", + "ê", + "¸", + "Ģ", + ",", + "æĹ", + "¥", + "æľ", + "¬", + "èª", + "ŀ", + ",", + "à¤", + "¹", + "à¤", + "¿", + "à¤", + "¨", + "à¥", + "į", + "à¤", + "¦", + "à¥", + "Ģ", + ",", + "ÑĢ", + "Ñĥ", + "Ñģ", + "Ñģ", + "к", + "и", + "Ð", + "¹", + ")", + "âĨĴ", + "{", + "âĪ", + "Ģ", + ":", + "âĪ", + "ĥ", + ",", + "â", + "Ĭ", + "ķ", + ":", + "â", + "Ĭ", + "Ĺ", + "}", + "" + ], + "decoded_with_special": "foo(中文,한글,日本語,हिन्दी,русский)→{∀:∃,⊕:⊗}", + "decoded_skip_special": "foo(中文,한글,日本語,हिन्दी,русский)→{∀:∃,⊕:⊗}" + }, + { + "id": "multiscript-legacy-symbols", + "input_ids": [ + 0, + 24987, + 11936, + 25448, + 3714, + 14333, + 41735, + 12605, + 3705, + 39110, + 46445, + 39207, + 4480, + 4056, + 8210, + 29254, + 24987, + 9264, + 9470, + 43662, + 2469, + 48104, + 7471, + 1437, + 47856, + 21402, + 48975, + 10809, + 46747, + 15722, + 23133, + 45209, + 5782, + 15722, + 32681, + 2 + ], + "tokens": [ + "", + "â", + "Ħ", + "ĸ", + "42", + "Ġâ", + "Ħ¢", + "ï¸ı", + "Ġ©", + "Ġ®", + "Ġ¶", + "Ġ§", + "ĠâĤ¬", + "Â", + "¥", + "£", + "â", + "Ĥ", + "¿", + "ĠÎ", + "£", + "âĪ", + "Ģ", + "Ġ", + "ä½", + "ł", + "å¥", + "½", + "Ġì", + "ķ", + "Ī", + "ë", + "ħ", + "ķ", + "ĠاÙĦ", + "" + ], + "decoded_with_special": "№42 ™️ © ® ¶ § €¥£₿ Σ∀ 你好 안녕 ال", + "decoded_skip_special": "№42 ™️ © ® ¶ § €¥£₿ Σ∀ 你好 안녕 ال" + }, + { + "id": "escape-sequences-with-zwj", + "input_ids": [ + 0, + 49728, + 37457, + 288, + 463, + 37457, + 5967, + 873, + 37457, + 282, + 4651, + 1902, + 37127, + 18400, + 18400, + 45895, + 8210, + 17, + 8384, + 1301, + 771, + 863, + 12605, + 38450, + 994, + 2 + ], + "tokens": [ + "", + "NULL", + "\\", + "0", + "and", + "\\", + "tt", + "ab", + "\\", + "n", + "new", + "line", + "æ", + "·", + "·", + "åħ", + "¥", + "âĢ", + "į", + "Z", + "W", + "J", + "ï¸ı", + "select", + "ors", + "" + ], + "decoded_with_special": "NULL\\0and\\ttab\\nnewline混入‍ZWJ️selectors", + "decoded_skip_special": "NULL\\0and\\ttab\\nnewline混入‍ZWJ️selectors" + }, + { + "id": "german-compound-short", + "input_ids": [ + 0, + 40676, + 5172, + 9182, + 4, + 926, + 20530, + 992, + 324, + 6083, + 4900, + 329, + 90, + 3069, + 16981, + 11474, + 405, + 895, + 424, + 2089, + 10, + 687, + 1935, + 14171, + 9244, + 967, + 605, + 13760, + 2158, + 16148, + 611, + 1792, + 462, + 1829, + 4, + 17841, + 11936, + 2 + ], + "tokens": [ + "", + "Nat", + "ür", + "lich", + ".", + "ĠDe", + "utsch", + "Ġz", + "ie", + "ht", + "Ġjet", + "z", + "t", + "Ġden", + "ĠKom", + "pos", + "it", + "ah", + "am", + "mer", + "Ġa", + "us", + "Ġder", + "ĠGram", + "mat", + "ik", + "w", + "erk", + "ze", + "ugs", + "ch", + "ub", + "l", + "ade", + ".", + "ĠðŁĺ", + "Ħ", + "" + ], + "decoded_with_special": "Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. 😄", + "decoded_skip_special": "Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. 😄" + }, + { + "id": "german-compound-long-1", + "input_ids": [ + 0, + 20470, + 32720, + 18504, + 8494, + 4462, + 3153, + 225, + 2802, + 1561, + 620, + 523, + 1545, + 4184, + 5618, + 1413, + 364, + 179, + 229, + 17459, + 594, + 2401, + 225, + 22103, + 39142, + 366, + 6673, + 324, + 1610, + 2413, + 1488, + 1334, + 3087, + 5720, + 364, + 5101, + 6304, + 1290, + 611, + 9856, + 2802, + 571, + 3361, + 9182, + 1071, + 2629, + 30650, + 2432, + 45, + 906, + 859, + 11, + 842, + 179, + 4969, + 3999, + 1073, + 119, + 1957, + 225, + 1949, + 40497, + 1097, + 3654, + 1210, + 428, + 4272, + 1597, + 3335, + 620, + 7381, + 5895, + 700, + 289, + 1561, + 2951, + 1023, + 1071, + 405, + 5689, + 15615, + 3121, + 5760, + 24981, + 1792, + 1478, + 19596, + 462, + 5398, + 13043, + 14726, + 1499, + 225, + 4, + 2 + ], + "tokens": [ + "", + "Im", + "ĠMorg", + "enne", + "bel", + "ged", + "ank", + "en", + "ver", + "ä", + "st", + "el", + "ung", + "sw", + "ald", + "Ġstand", + "Ġe", + "in", + "ĠK", + "affe", + "et", + "ass", + "en", + "rand", + "phil", + "os", + "oph", + "ie", + "be", + "ob", + "ach", + "ter", + "Ġne", + "ben", + "Ġe", + "iner", + "ĠReg", + "ens", + "ch", + "irm", + "ver", + "g", + "ess", + "lich", + "ke", + "its", + "station", + "Ġund", + "Ġnot", + "ier", + "te", + "Ġin", + "Ġse", + "in", + "ĠSon", + "nt", + "ag", + "m", + "org", + "en", + "ide", + "enf", + "ang", + "not", + "iz", + "b", + "uch", + "Ġdie", + "Ġer", + "st", + "aun", + "lic", + "he", + "ĠH", + "ä", + "uf", + "ig", + "ke", + "it", + "Ġvon", + "ĠFen", + "ster", + "bank", + "sta", + "ub", + "son", + "nen", + "l", + "icht", + "ref", + "lex", + "ion", + "en", + ".", + "" + ], + "decoded_with_special": "Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen.", + "decoded_skip_special": "Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen." + }, + { + "id": "german-compound-long-2", + "input_ids": [ + 0, + 487, + 3209, + 225, + 939, + 13492, + 32933, + 859, + 364, + 833, + 229, + 2768, + 298, + 6634, + 611, + 40081, + 5246, + 225, + 1610, + 459, + 4272, + 90, + 1545, + 12778, + 12247, + 1545, + 29, + 13738, + 611, + 833, + 6, + 885, + 1561, + 298, + 10082, + 364, + 179, + 1127, + 17170, + 625, + 3314, + 15086, + 11127, + 5576, + 3671, + 3145, + 282, + 1545, + 29, + 35692, + 906, + 1545, + 29, + 1610, + 1180, + 2543, + 18937, + 1334, + 6286, + 364, + 5101, + 6213, + 3807, + 225, + 19493, + 677, + 4193, + 967, + 2768, + 1943, + 5079, + 611, + 1545, + 14832, + 7776, + 1090, + 9988, + 611, + 1561, + 2543, + 1023, + 90, + 997, + 4, + 16675, + 1935, + 6838, + 858, + 295, + 1561, + 1843, + 859, + 579, + 1725, + 364, + 179, + 13847, + 6944, + 330, + 6528, + 6403, + 506, + 895, + 338, + 620, + 2957, + 462, + 13792, + 21609, + 7474, + 261, + 661, + 10, + 2951, + 364, + 10748, + 381, + 179, + 7822, + 2802, + 1071, + 298, + 4926, + 4950, + 523, + 17745, + 11629, + 1545, + 29, + 2403, + 1561, + 90, + 2432, + 910, + 14134, + 35, + 44, + 17772, + 250, + 8797, + 1545, + 6, + 385, + 2723, + 748, + 12266, + 6105, + 19377, + 364, + 833, + 272, + 196, + 3153, + 40497, + 20223, + 2802, + 330, + 3654, + 1545, + 29, + 1899, + 506, + 895, + 2558, + 23428, + 9762, + 328, + 17, + 48, + 2 + ], + "tokens": [ + "", + "N", + "eb", + "en", + "Ġi", + "hm", + "Ġsumm", + "te", + "Ġe", + "ine", + "ĠK", + "ü", + "h", + "ls", + "ch", + "rank", + "inn", + "en", + "be", + "le", + "uch", + "t", + "ung", + "ser", + "inner", + "ung", + "s", + "mas", + "ch", + "ine", + ",", + "Ġw", + "ä", + "h", + "rend", + "Ġe", + "in", + "ĠMar", + "mel", + "ad", + "eng", + "las", + "dec", + "kel", + "ö", + "ff", + "n", + "ung", + "s", + "optim", + "ier", + "ung", + "s", + "be", + "au", + "ft", + "rag", + "ter", + "Ġmit", + "Ġe", + "iner", + "ĠTre", + "pp", + "en", + "haus", + "ak", + "ust", + "ik", + "ü", + "ber", + "ras", + "ch", + "ung", + "san", + "aly", + "se", + "Ġbes", + "ch", + "ä", + "ft", + "ig", + "t", + "Ġwar", + ".", + "ĠAus", + "Ġder", + "ĠFer", + "ne", + "Ġn", + "ä", + "her", + "te", + "Ġs", + "ich", + "Ġe", + "in", + "ĠWol", + "ken", + "k", + "rat", + "zer", + "f", + "ah", + "r", + "st", + "uh", + "l", + "mus", + "ikk", + "omp", + "on", + "ist", + "Ġa", + "uf", + "Ġe", + "inem", + "ĠE", + "in", + "rad", + "ver", + "ke", + "h", + "rs", + "reg", + "el", + "miss", + "acht", + "ung", + "s", + "ger", + "ä", + "t", + "Ġund", + "Ġr", + "ief", + ":", + "ĠâĢ", + "ŀ", + "A", + "cht", + "ung", + ",", + "Ġd", + "ort", + "Ġv", + "orne", + "Ġlie", + "gt", + "Ġe", + "ine", + "ĠG", + "ed", + "ank", + "enf", + "aden", + "ver", + "k", + "not", + "ung", + "s", + "ge", + "f", + "ah", + "ren", + "sit", + "uation", + "!", + "âĢ", + "ľ", + "" + ], + "decoded_with_special": "Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“", + "decoded_skip_special": "Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“" + }, + { + "id": "german-compound-long-3", + "input_ids": [ + 0, + 16213, + 3671, + 12527, + 9182, + 952, + 19002, + 3145, + 4135, + 242, + 579, + 1725, + 364, + 179, + 22221, + 1545, + 29, + 1090, + 405, + 225, + 4179, + 462, + 1561, + 90, + 1334, + 9428, + 620, + 139, + 7804, + 3427, + 337, + 6, + 2432, + 37, + 763, + 687, + 9695, + 10101, + 3869, + 385, + 241, + 118, + 163, + 2768, + 1001, + 330, + 5112, + 2089, + 6852, + 2723, + 906, + 605, + 6607, + 1794, + 16729, + 7794, + 2543, + 6526, + 718, + 858, + 298, + 2089, + 5246, + 225, + 6, + 364, + 179, + 272, + 2013, + 16481, + 605, + 11249, + 119, + 2768, + 90, + 7889, + 15476, + 5720, + 1943, + 5109, + 2432, + 364, + 179, + 1368, + 4306, + 18252, + 1023, + 4342, + 605, + 853, + 338, + 1334, + 13256, + 225, + 462, + 3671, + 3145, + 523, + 13043, + 14726, + 2485, + 5646, + 8258, + 2459, + 17625, + 4, + 14435, + 242, + 1344, + 424, + 4435, + 2768, + 1187, + 594, + 225, + 579, + 324, + 3069, + 27117, + 2802, + 9484, + 856, + 5172, + 234, + 1488, + 22141, + 8299, + 9182, + 1872, + 611, + 21739, + 330, + 927, + 225, + 14600, + 338, + 11629, + 1545, + 2432, + 36403, + 906, + 859, + 15982, + 677, + 3006, + 20533, + 225, + 2802, + 859, + 808, + 1023, + 1545, + 29, + 6031, + 27586, + 2495, + 4, + 2 + ], + "tokens": [ + "", + "Pl", + "ö", + "tz", + "lich", + "ĠÃ", + "¶", + "ff", + "net", + "e", + "Ġs", + "ich", + "Ġe", + "in", + "ĠZeit", + "ung", + "s", + "se", + "it", + "en", + "umb", + "l", + "ä", + "t", + "ter", + "wind", + "st", + "o", + "ÃŁ", + "port", + "al", + ",", + "Ġund", + "Ġhe", + "ra", + "us", + "Ġpur", + "zel", + "ten", + "Ġd", + "re", + "i", + "ĠB", + "ü", + "ro", + "k", + "lam", + "mer", + "ns", + "ort", + "ier", + "w", + "elt", + "me", + "isters", + "cha", + "ft", + "ste", + "il", + "ne", + "h", + "mer", + "inn", + "en", + ",", + "Ġe", + "in", + "ĠG", + "art", + "enz", + "w", + "erg", + "m", + "ü", + "t", + "zen", + "far", + "ben", + "ber", + "ater", + "Ġund", + "Ġe", + "in", + "Ġh", + "och", + "grad", + "ig", + "Ġver", + "w", + "ir", + "r", + "ter", + "ĠSupp", + "en", + "l", + "ö", + "ff", + "el", + "ref", + "lex", + "ions", + "met", + "aph", + "ys", + "iker", + ".", + "ĠGem", + "e", + "ins", + "am", + "Ġgr", + "ü", + "nd", + "et", + "en", + "Ġs", + "ie", + "Ġden", + "ĠBundes", + "ver", + "band", + "Ġf", + "ür", + "ĠN", + "ach", + "mitt", + "ags", + "lich", + "ts", + "ch", + "atten", + "k", + "ant", + "en", + "bet", + "r", + "acht", + "ung", + "Ġund", + "Ġimprovis", + "ier", + "te", + "ĠSof", + "ak", + "iss", + "enburg", + "en", + "ver", + "te", + "id", + "ig", + "ung", + "s", + "str", + "ateg", + "ien", + ".", + "" + ], + "decoded_with_special": "Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien.", + "decoded_skip_special": "Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien." + }, + { + "id": "german-compound-long-4", + "input_ids": [ + 0, + 10127, + 2060, + 1397, + 19431, + 12614, + 523, + 3869, + 579, + 1725, + 22626, + 4356, + 21387, + 25383, + 9433, + 225, + 3121, + 9772, + 2262, + 24981, + 1792, + 7210, + 19903, + 329, + 15912, + 6, + 19958, + 1935, + 229, + 17459, + 594, + 2401, + 225, + 22103, + 39142, + 366, + 6673, + 324, + 1610, + 2413, + 1488, + 1334, + 364, + 833, + 10668, + 906, + 5895, + 700, + 21351, + 4781, + 2050, + 242, + 952, + 4394, + 1943, + 1597, + 305, + 5398, + 1023, + 1071, + 405, + 5689, + 1890, + 29, + 5246, + 29, + 26790, + 7282, + 2485, + 18178, + 6343, + 6, + 14933, + 1488, + 13792, + 1071, + 4779, + 14897, + 282, + 2768, + 428, + 1545, + 225, + 2432, + 1599, + 5247, + 3914, + 17884, + 611, + 1594, + 3145, + 895, + 338, + 1872, + 5641, + 17150, + 7794, + 2543, + 7771, + 1115, + 405, + 1561, + 282, + 9426, + 2768, + 90, + 7889, + 15204, + 1517, + 506, + 268, + 15586, + 859, + 718, + 16768, + 611, + 3707, + 1545, + 29, + 4892, + 5225, + 2262, + 20280, + 6607, + 4, + 2 + ], + "tokens": [ + "", + "Am", + "ĠAb", + "end", + "Ġvers", + "amm", + "el", + "ten", + "Ġs", + "ich", + "Ġalle", + "Ġim", + "ĠMond", + "sche", + "inf", + "en", + "ster", + "rah", + "men", + "sta", + "ub", + "gl", + "itzer", + "z", + "immer", + ",", + "Ġwo", + "Ġder", + "ĠK", + "affe", + "et", + "ass", + "en", + "rand", + "phil", + "os", + "oph", + "ie", + "be", + "ob", + "ach", + "ter", + "Ġe", + "ine", + "Ġfe", + "ier", + "lic", + "he", + "ĠSchl", + "uss", + "red", + "e", + "ĠÃ", + "¼", + "ber", + "Ġdie", + "ĠW", + "icht", + "ig", + "ke", + "it", + "Ġvon", + "ĠUn", + "s", + "inn", + "s", + "produ", + "kt", + "ions", + "fre", + "ude", + ",", + "ĠSpr", + "ach", + "mus", + "ke", + "ld", + "eh", + "n", + "ü", + "b", + "ung", + "en", + "Ġund", + "ĠDon", + "aud", + "amp", + "fs", + "ch", + "if", + "ff", + "ah", + "r", + "ts", + "ges", + "ells", + "cha", + "ft", + "sk", + "ap", + "it", + "ä", + "n", + "sm", + "ü", + "t", + "zen", + "kn", + "op", + "f", + "ers", + "atz", + "te", + "il", + "bes", + "ch", + "aff", + "ung", + "s", + "pro", + "ble", + "men", + "Ġhi", + "elt", + ".", + "" + ], + "decoded_with_special": "Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt.", + "decoded_skip_special": "Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt." + }, + { + "id": "german-compound-long-5", + "input_ids": [ + 0, + 25887, + 1488, + 69, + 4926, + 611, + 859, + 70, + 23922, + 242, + 833, + 525, + 2951, + 10382, + 225, + 24212, + 6, + 8477, + 7804, + 254, + 28, + 118, + 1935, + 229, + 2768, + 298, + 6634, + 611, + 40081, + 5246, + 225, + 1610, + 459, + 4272, + 90, + 1545, + 12778, + 12247, + 1545, + 29, + 13738, + 611, + 833, + 6, + 1597, + 579, + 1725, + 952, + 4394, + 1943, + 313, + 24612, + 1187, + 242, + 255, + 5172, + 3671, + 3145, + 282, + 1545, + 29, + 1180, + 506, + 2089, + 2258, + 424, + 1071, + 2629, + 605, + 2399, + 12265, + 1561, + 12527, + 1545, + 28, + 330, + 23546, + 859, + 4, + 2 + ], + "tokens": [ + "", + "Dan", + "ach", + "Ġher", + "rs", + "ch", + "te", + "Ġall", + "gem", + "e", + "ine", + "ĠZ", + "uf", + "ried", + "en", + "heit", + ",", + "Ġau", + "ÃŁ", + "er", + "Ġbe", + "i", + "Ġder", + "ĠK", + "ü", + "h", + "ls", + "ch", + "rank", + "inn", + "en", + "be", + "le", + "uch", + "t", + "ung", + "ser", + "inner", + "ung", + "s", + "mas", + "ch", + "ine", + ",", + "Ġdie", + "Ġs", + "ich", + "ĠÃ", + "¼", + "ber", + "Ġman", + "gel", + "nd", + "e", + "ĠT", + "ür", + "ö", + "ff", + "n", + "ung", + "s", + "au", + "f", + "mer", + "ks", + "am", + "ke", + "its", + "w", + "ert", + "sch", + "ä", + "tz", + "ung", + "Ġbe", + "k", + "lag", + "te", + ".", + "" + ], + "decoded_with_special": "Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte.", + "decoded_skip_special": "Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte." + } + ] +} diff --git a/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/t5_small_multilingual.json b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/t5_small_multilingual.json new file mode 100644 index 00000000..0c4ebb9c --- /dev/null +++ b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/t5_small_multilingual.json @@ -0,0 +1,4571 @@ +{ + "metadata": { + "model_id": "google-t5/t5-small", + "transformers_version": "4.57.1", + "generated_at": "2026-05-15T23:58:33+00:00", + "input_count": 83 + }, + "entries": [ + { + "id": "latin-diacritics-french", + "input_ids": [ + 11949, + 1417, + 4078, + 154, + 3, + 29, + 9, + 2, + 162, + 24369, + 3, + 318, + 20435, + 29925, + 721, + 47, + 1287, + 5, + 1 + ], + "tokens": [ + "▁café", + "▁ré", + "sum", + "é", + "▁", + "n", + "a", + "", + "ve", + "▁façade", + "▁", + "—", + "▁crème", + "▁brûl", + "ée", + "▁was", + "▁excellent", + ".", + "" + ], + "decoded_with_special": "café résumé nave façade — crème brûlée was excellent.", + "decoded_skip_special": "café résumé nave façade — crème brûlée was excellent." + }, + { + "id": "latin-diacritics-mixed", + "input_ids": [ + 454, + 1272, + 1803, + 510, + 19460, + 8849, + 5, + 1534, + 2, + 152, + 9, + 3, + 9845, + 3, + 9, + 50, + 158, + 2, + 9, + 975, + 26816, + 5, + 1 + ], + "tokens": [ + "▁H", + "ü", + "gel", + "▁über", + "▁Brü", + "cke", + ".", + "▁Ma", + "", + "an", + "a", + "▁", + "voy", + "▁", + "a", + "▁la", + "▁pe", + "", + "a", + "▁con", + "▁José", + ".", + "" + ], + "decoded_with_special": "Hügel über Brücke. Maana voy a la pea con José.", + "decoded_skip_special": "Hügel über Brücke. Maana voy a la pea con José." + }, + { + "id": "latin-diacritics-portuguese", + "input_ids": [ + 180, + 2, + 32, + 1838, + 32, + 3, + 154, + 561, + 9, + 3, + 75, + 15644, + 15, + 19130, + 117, + 18552, + 319, + 5, + 1 + ], + "tokens": [ + "▁S", + "", + "o", + "▁Paul", + "o", + "▁", + "é", + "▁um", + "a", + "▁", + "c", + "idad", + "e", + "▁interessante", + ";", + "▁Köln", + "▁auch", + ".", + "" + ], + "decoded_with_special": "So Paulo é uma cidade interessante; Köln auch.", + "decoded_skip_special": "So Paulo é uma cidade interessante; Köln auch." + }, + { + "id": "cyrillic-greeting", + "input_ids": [ + 3, + 2, + 14709, + 6609, + 15042, + 6, + 3, + 21325, + 8452, + 55, + 3, + 2, + 9592, + 3, + 14982, + 10458, + 3, + 9592, + 6652, + 21302, + 2, + 2533, + 9592, + 7948, + 5, + 1 + ], + "tokens": [ + "▁", + "", + "ри", + "в", + "ет", + ",", + "▁", + "ми", + "р", + "!", + "▁", + "", + "то", + "▁", + "те", + "ст", + "▁", + "то", + "к", + "ени", + "", + "а", + "то", + "ра", + ".", + "" + ], + "decoded_with_special": "ривет, мир! то тест токениатора.", + "decoded_skip_special": "ривет, мир! то тест токениатора." + }, + { + "id": "cyrillic-prose", + "input_ids": [ + 3, + 2, + 18737, + 6652, + 16624, + 3, + 318, + 12681, + 9592, + 14601, + 2, + 2533, + 3, + 2, + 18737, + 26798, + 2795, + 5, + 3, + 2, + 2533, + 22036, + 6588, + 21302, + 1757, + 3, + 2, + 17674, + 1757, + 1757, + 586, + 3, + 21325, + 6588, + 14601, + 2044, + 31545, + 5, + 1 + ], + "tokens": [ + "▁", + "", + "ос", + "к", + "ва", + "▁", + "—", + "▁с", + "то", + "ли", + "", + "а", + "▁", + "", + "ос", + "си", + "и", + ".", + "▁", + "", + "а", + "се", + "л", + "ени", + "е", + "▁", + "", + "ол", + "е", + "е", + "▁12", + "▁", + "ми", + "л", + "ли", + "о", + "нов", + ".", + "" + ], + "decoded_with_special": "осква — столиа оссии. аселение олее 12 миллионов.", + "decoded_skip_special": "осква — столиа оссии. аселение олее 12 миллионов." + }, + { + "id": "greek-greeting", + "input_ids": [ + 3, + 2, + 3, + 2, + 55, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁", + "", + "▁", + "", + "!", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": " ! .", + "decoded_skip_special": "! ." + }, + { + "id": "cjk-simplified-greeting", + "input_ids": [ + 3, + 2, + 6, + 2, + 55, + 2, + 1 + ], + "tokens": [ + "▁", + "", + ",", + "", + "!", + "", + "" + ], + "decoded_with_special": ",!", + "decoded_skip_special": ",!" + }, + { + "id": "cjk-simplified-prose", + "input_ids": [ + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "" + ], + "decoded_with_special": "", + "decoded_skip_special": "" + }, + { + "id": "cjk-traditional-compare", + "input_ids": [ + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "" + ], + "decoded_with_special": "", + "decoded_skip_special": "" + }, + { + "id": "cjk-traditional-hongkong", + "input_ids": [ + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "" + ], + "decoded_with_special": "", + "decoded_skip_special": "" + }, + { + "id": "japanese-voiced-kana-greeting", + "input_ids": [ + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "" + ], + "decoded_with_special": "", + "decoded_skip_special": "" + }, + { + "id": "japanese-voiced-kana-prose", + "input_ids": [ + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "" + ], + "decoded_with_special": "", + "decoded_skip_special": "" + }, + { + "id": "hangul-syllables-greeting", + "input_ids": [ + 3, + 2, + 6, + 3, + 2, + 55, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁", + "", + ",", + "▁", + "", + "!", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": ", ! .", + "decoded_skip_special": ",! ." + }, + { + "id": "hangul-syllables-prose", + "input_ids": [ + 3, + 2, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁", + "", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": " .", + "decoded_skip_special": " ." + }, + { + "id": "arabic-greeting", + "input_ids": [ + 3, + 2, + 3, + 2, + 55, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁", + "", + "▁", + "", + "!", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": " ! .", + "decoded_skip_special": "! ." + }, + { + "id": "arabic-prose", + "input_ids": [ + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": " .", + "decoded_skip_special": " ." + }, + { + "id": "hebrew-greeting", + "input_ids": [ + 3, + 2, + 3, + 2, + 55, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁", + "", + "▁", + "", + "!", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": " ! .", + "decoded_skip_special": "! ." + }, + { + "id": "devanagari-hindi", + "input_ids": [ + 3, + 2, + 3, + 2, + 55, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "▁", + "", + "!", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "" + ], + "decoded_with_special": " ! ", + "decoded_skip_special": "! " + }, + { + "id": "devanagari-sanskrit", + "input_ids": [ + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "" + ], + "decoded_with_special": " ", + "decoded_skip_special": " " + }, + { + "id": "thai-combining-marks-greeting", + "input_ids": [ + 3, + 2, + 55, + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "!", + "▁", + "", + "" + ], + "decoded_with_special": "! ", + "decoded_skip_special": "! " + }, + { + "id": "thai-combining-marks-prose", + "input_ids": [ + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "" + ], + "decoded_with_special": "", + "decoded_skip_special": "" + }, + { + "id": "mixed-script-cjk-latin", + "input_ids": [ + 37, + 3, + 2, + 1469, + 16, + 460, + 2688, + 3, + 318, + 1264, + 923, + 55, + 1 + ], + "tokens": [ + "▁The", + "▁", + "", + "▁trip", + "▁in", + "▁20", + "26", + "▁", + "—", + "▁très", + "▁bien", + "!", + "" + ], + "decoded_with_special": "The trip in 2026 — très bien!", + "decoded_skip_special": "The trip in 2026 — très bien!" + }, + { + "id": "mixed-script-katakana-code", + "input_ids": [ + 3, + 2, + 10, + 8143, + 3599, + 16, + 10042, + 3, + 9, + 519, + 89, + 1298, + 75, + 357, + 115, + 5, + 1 + ], + "tokens": [ + "▁", + "", + ":", + "▁bug", + "▁fixed", + "▁in", + "▁commit", + "▁", + "a", + "3", + "f", + "9", + "c", + "2", + "b", + ".", + "" + ], + "decoded_with_special": ": bug fixed in commit a3f9c2b.", + "decoded_skip_special": ": bug fixed in commit a3f9c2b." + }, + { + "id": "math-symbols-summation", + "input_ids": [ + 3, + 20119, + 10, + 3, + 2, + 3, + 226, + 834, + 23, + 3274, + 6426, + 41, + 8352, + 3, + 226, + 3, + 2, + 391, + 137, + 1 + ], + "tokens": [ + "▁", + "Result", + ":", + "▁", + "", + "▁", + "x", + "_", + "i", + "▁=", + "▁42", + "▁(", + "where", + "▁", + "x", + "▁", + "", + "▁R", + ").", + "" + ], + "decoded_with_special": "Result: x_i = 42 (where x R).", + "decoded_skip_special": "Result: x_i = 42 (where x R)." + }, + { + "id": "math-symbols-epsilon-delta", + "input_ids": [ + 37, + 127, + 15, + 51, + 10, + 3, + 2, + 3155, + 632, + 3, + 2, + 3155, + 632, + 224, + 24, + 1820, + 226, + 18, + 226, + 632, + 9175, + 2, + 3, + 2, + 1820, + 89, + 599, + 226, + 61, + 18, + 89, + 599, + 226, + 632, + 61, + 9175, + 2, + 5, + 1 + ], + "tokens": [ + "▁The", + "or", + "e", + "m", + ":", + "▁", + "", + ">", + "0", + "▁", + "", + ">", + "0", + "▁such", + "▁that", + "▁|", + "x", + "-", + "x", + "0", + "|", + "", + "▁", + "", + "▁|", + "f", + "(", + "x", + ")", + "-", + "f", + "(", + "x", + "0", + ")", + "|", + "", + ".", + "" + ], + "decoded_with_special": "Theorem: >0 >0 such that |x-x0| |f(x)-f(x0)|.", + "decoded_skip_special": "Theorem: >0 >0 such that |x-x0| |f(x)-f(x0)|." + }, + { + "id": "currency-symbols-multi", + "input_ids": [ + 6409, + 10, + 3416, + 9368, + 1752, + 3, + 2, + 3, + 2, + 4347, + 927, + 1755, + 3, + 2, + 26016, + 5, + 3072, + 41, + 12497, + 226, + 6, + 303, + 3, + 2, + 5311, + 7256, + 5, + 1 + ], + "tokens": [ + "▁Cost", + ":", + "▁€", + "12.", + "50", + "▁", + "", + "▁", + "", + "1,", + "8", + "20", + "▁", + "", + "▁£10", + ".", + "75", + "▁(", + "appro", + "x", + ",", + "▁plus", + "▁", + "", + "0.000", + "4)", + ".", + "" + ], + "decoded_with_special": "Cost: €12.50 1,820 £10.75 (approx, plus 0.0004).", + "decoded_skip_special": "Cost: €12.50 1,820 £10.75 (approx, plus 0.0004)." + }, + { + "id": "box-drawing", + "input_ids": [ + 5179, + 10, + 3, + 2, + 3, + 2, + 8774, + 3, + 2, + 3, + 2, + 3, + 318, + 612, + 5, + 1 + ], + "tokens": [ + "▁Box", + ":", + "▁", + "", + "▁", + "", + "▁Hello", + "▁", + "", + "▁", + "", + "▁", + "—", + "▁done", + ".", + "" + ], + "decoded_with_special": "Box: Hello — done.", + "decoded_skip_special": "Box: Hello — done." + }, + { + "id": "dingbats-chess", + "input_ids": [ + 21177, + 7, + 10, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 318, + 3, + 2951, + 7, + 356, + 5, + 1 + ], + "tokens": [ + "▁Piece", + "s", + ":", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "—", + "▁", + "ches", + "s", + "▁set", + ".", + "" + ], + "decoded_with_special": "Pieces: — chess set.", + "decoded_skip_special": "Pieces: — chess set." + }, + { + "id": "math-astral-bold-script", + "input_ids": [ + 9762, + 8197, + 10, + 14213, + 308, + 7088, + 2773, + 3, + 318, + 7270, + 4943, + 10, + 14213, + 308, + 5, + 1 + ], + "tokens": [ + "▁Math", + "▁bold", + ":", + "▁ABC", + "D", + "▁01", + "23", + "▁", + "—", + "▁math", + "▁script", + ":", + "▁ABC", + "D", + ".", + "" + ], + "decoded_with_special": "Math bold: ABCD 0123 — math script: ABCD.", + "decoded_skip_special": "Math bold: ABCD 0123 — math script: ABCD." + }, + { + "id": "astral-egyptian-hieroglyphs", + "input_ids": [ + 16341, + 1382, + 32, + 29929, + 7, + 10, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁Egyptian", + "▁hier", + "o", + "glyph", + "s", + ":", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": "Egyptian hieroglyphs: .", + "decoded_skip_special": "Egyptian hieroglyphs: ." + }, + { + "id": "astral-cuneiform", + "input_ids": [ + 205, + 444, + 23, + 2032, + 10, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 318, + 1320, + 570, + 5977, + 5, + 1 + ], + "tokens": [ + "▁C", + "une", + "i", + "form", + ":", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "—", + "▁sign", + "▁list", + "▁samples", + ".", + "" + ], + "decoded_with_special": "Cuneiform: — sign list samples.", + "decoded_skip_special": "Cuneiform: — sign list samples." + }, + { + "id": "astral-mahjong", + "input_ids": [ + 8555, + 354, + 2444, + 10426, + 10, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁Mah", + "j", + "ong", + "▁tiles", + ":", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": "Mahjong tiles: .", + "decoded_skip_special": "Mahjong tiles: ." + }, + { + "id": "astral-cards", + "input_ids": [ + 18085, + 10, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁Cards", + ":", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": "Cards: .", + "decoded_skip_special": "Cards: ." + }, + { + "id": "emoji-bmp-and-astral", + "input_ids": [ + 3, + 2, + 8774, + 55, + 3, + 2, + 1150, + 3, + 2, + 3289, + 3, + 2, + 1088, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁", + "", + "▁Hello", + "!", + "▁", + "", + "▁World", + "▁", + "", + "▁launch", + "▁", + "", + "▁party", + "▁", + "", + ".", + "" + ], + "decoded_with_special": " Hello! World launch party .", + "decoded_skip_special": " Hello! World launch party." + }, + { + "id": "emoji-zwj-family-pride-skin", + "input_ids": [ + 3712, + 10, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 318, + 24252, + 5692, + 10, + 3, + 2, + 3, + 2, + 3, + 318, + 13038, + 18, + 6948, + 10, + 3, + 2, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁Family", + ":", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "—", + "▁Pride", + "▁flag", + ":", + "▁", + "", + "▁", + "", + "▁", + "—", + "▁Skin", + "-", + "tone", + ":", + "▁", + "", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": "Family: — Pride flag: — Skin-tone: .", + "decoded_skip_special": "Family: — Pride flag: — Skin-tone: ." + }, + { + "id": "emoji-zwj-with-text-prefix", + "input_ids": [ + 3, + 2, + 3, + 2, + 7855, + 221, + 1220, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 371, + 3690, + 120, + 1220, + 2, + 89, + 5430, + 7, + 1220, + 4377, + 1 + ], + "tokens": [ + "▁", + "", + "▁", + "", + "Pri", + "de", + "+", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "F", + "ami", + "ly", + "+", + "", + "f", + "lag", + "s", + "+", + "test", + "" + ], + "decoded_with_special": " Pride+ Family+flags+test", + "decoded_skip_special": " Pride+ Family+flags+test" + }, + { + "id": "emoji-keycap-and-flags", + "input_ids": [ + 4420, + 4010, + 7, + 10, + 209, + 2, + 204, + 2, + 220, + 2, + 3, + 318, + 5692, + 10, + 3, + 2, + 3, + 2, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁Key", + "cap", + "s", + ":", + "▁1", + "", + "▁2", + "", + "▁3", + "", + "▁", + "—", + "▁flag", + ":", + "▁", + "", + "▁", + "", + "▁", + "", + ".", + "" + ], + "decoded_with_special": "Keycaps: 1 2 3 — flag: .", + "decoded_skip_special": "Keycaps: 1 2 3 — flag: ." + }, + { + "id": "url-with-query-fragment", + "input_ids": [ + 3, + 16137, + 2423, + 5948, + 7, + 1303, + 994, + 9, + 9208, + 5, + 287, + 87, + 8292, + 87, + 235, + 87, + 60, + 7928, + 58, + 1824, + 2423, + 89, + 32, + 32, + 184, + 1047, + 2423, + 115, + 9, + 172, + 184, + 29, + 2423, + 4165, + 4663, + 14309, + 3486, + 1 + ], + "tokens": [ + "▁", + "url", + "=", + "http", + "s", + "://", + "ex", + "a", + "mple", + ".", + "com", + "/", + "path", + "/", + "to", + "/", + "re", + "source", + "?", + "q", + "=", + "f", + "o", + "o", + "&", + "bar", + "=", + "b", + "a", + "z", + "&", + "n", + "=", + "42", + "#", + "section", + "-3", + "" + ], + "decoded_with_special": "url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3", + "decoded_skip_special": "url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3" + }, + { + "id": "hex-and-base64", + "input_ids": [ + 3, + 88, + 226, + 2423, + 632, + 226, + 25473, + 9213, + 5080, + 5390, + 188, + 6392, + 4882, + 10059, + 1247, + 4389, + 2423, + 476, + 518, + 683, + 354, + 956, + 24403, + 51, + 956, + 357, + 107, + 102, + 265, + 17, + 7, + 115, + 518, + 755, + 208, + 75, + 188, + 2423, + 2423, + 1 + ], + "tokens": [ + "▁", + "he", + "x", + "=", + "0", + "x", + "DEA", + "DB", + "EE", + "FC", + "A", + "FE", + "BA", + "BE", + "▁base", + "64", + "=", + "Y", + "W", + "J", + "j", + "Z", + "GV", + "m", + "Z", + "2", + "h", + "p", + "am", + "t", + "s", + "b", + "W", + "5", + "v", + "c", + "A", + "=", + "=", + "" + ], + "decoded_with_special": "hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==", + "decoded_skip_special": "hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==" + }, + { + "id": "sha256-hash", + "input_ids": [ + 6660, + 9, + 19337, + 2423, + 15, + 519, + 115, + 632, + 75, + 3628, + 357, + 3916, + 89, + 75, + 536, + 75, + 24816, + 9, + 89, + 115, + 89, + 591, + 75, + 3914, + 4314, + 89, + 115, + 4508, + 591, + 2555, + 9, + 15, + 4853, + 15, + 4448, + 3647, + 115, + 4271, + 591, + 658, + 3647, + 3390, + 4729, + 115, + 3940, + 5373, + 115, + 4433, + 755, + 1 + ], + "tokens": [ + "▁sh", + "a", + "256", + "=", + "e", + "3", + "b", + "0", + "c", + "44", + "2", + "98", + "f", + "c", + "1", + "c", + "149", + "a", + "f", + "b", + "f", + "4", + "c", + "89", + "96", + "f", + "b", + "92", + "4", + "27", + "a", + "e", + "41", + "e", + "46", + "49", + "b", + "93", + "4", + "ca", + "49", + "59", + "91", + "b", + "78", + "52", + "b", + "85", + "5", + "" + ], + "decoded_with_special": "sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "decoded_skip_special": "sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "id": "code-javascript-functional", + "input_ids": [ + 1584, + 52, + 5, + 11576, + 599, + 226, + 3, + 15425, + 3, + 226, + 1429, + 3, + 15070, + 22240, + 599, + 226, + 3, + 15425, + 3, + 226, + 2490, + 3, + 632, + 137, + 60, + 12160, + 599, + 599, + 9, + 6, + 3, + 115, + 61, + 3, + 15425, + 3, + 9, + 1768, + 3, + 115, + 3670, + 1 + ], + "tokens": [ + "▁ar", + "r", + ".", + "map", + "(", + "x", + "▁", + "=>", + "▁", + "x", + "▁*", + "▁", + "2).", + "filter", + "(", + "x", + "▁", + "=>", + "▁", + "x", + "▁>", + "▁", + "0", + ").", + "re", + "duce", + "(", + "(", + "a", + ",", + "▁", + "b", + ")", + "▁", + "=>", + "▁", + "a", + "▁+", + "▁", + "b", + ");", + "" + ], + "decoded_with_special": "arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);", + "decoded_skip_special": "arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);" + }, + { + "id": "code-cplusplus-template", + "input_ids": [ + 3847, + 2, + 382, + 6, + 1265, + 3155, + 1510, + 3, + 89, + 599, + 382, + 184, + 184, + 17, + 6, + 412, + 184, + 184, + 76, + 61, + 3, + 13114, + 20, + 75, + 40, + 6137, + 599, + 17, + 1220, + 76, + 61, + 3, + 2, + 1205, + 3, + 17, + 1768, + 3, + 76, + 117, + 3, + 2, + 1 + ], + "tokens": [ + "▁template", + "", + "T", + ",", + "U", + ">", + "▁auto", + "▁", + "f", + "(", + "T", + "&", + "&", + "t", + ",", + "▁U", + "&", + "&", + "u", + ")", + "▁", + "->", + "▁de", + "c", + "l", + "type", + "(", + "t", + "+", + "u", + ")", + "▁", + "", + "▁return", + "▁", + "t", + "▁+", + "▁", + "u", + ";", + "▁", + "", + "" + ], + "decoded_with_special": "templateT,U> auto f(T&&t, U&&u) -> decltype(t+u) return t + u; ", + "decoded_skip_special": "templateT,U> auto f(T&&t, U&&u) -> decltype(t+u) return t + u; " + }, + { + "id": "code-bitops", + "input_ids": [ + 741, + 3274, + 41, + 9, + 14052, + 9266, + 1820, + 41, + 115, + 3, + 2, + 3, + 7256, + 3, + 184, + 3, + 632, + 226, + 9089, + 117, + 1 + ], + "tokens": [ + "▁result", + "▁=", + "▁(", + "a", + "▁>>", + "▁2)", + "▁|", + "▁(", + "b", + "▁", + "", + "▁", + "4)", + "▁", + "&", + "▁", + "0", + "x", + "FF", + ";", + "" + ], + "decoded_with_special": "result = (a >> 2) | (b 4) & 0xFF;", + "decoded_skip_special": "result = (a >> 2) | (b 4) & 0xFF;" + }, + { + "id": "code-javascript-modern", + "input_ids": [ + 3, + 99, + 41, + 226, + 3, + 55, + 2423, + 2423, + 206, + 195, + 3, + 184, + 184, + 3, + 63, + 2490, + 2423, + 3, + 632, + 61, + 3, + 2, + 1205, + 3, + 226, + 58, + 5, + 89, + 32, + 32, + 3, + 8546, + 4647, + 18392, + 76, + 15, + 117, + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "if", + "▁(", + "x", + "▁", + "!", + "=", + "=", + "▁nu", + "ll", + "▁", + "&", + "&", + "▁", + "y", + "▁>", + "=", + "▁", + "0", + ")", + "▁", + "", + "▁return", + "▁", + "x", + "?", + ".", + "f", + "o", + "o", + "▁", + "??", + "▁default", + "Val", + "u", + "e", + ";", + "▁", + "", + "" + ], + "decoded_with_special": "if (x!== null && y >= 0) return x?.foo?? defaultValue; ", + "decoded_skip_special": "if (x!== null && y >= 0) return x?.foo?? defaultValue; " + }, + { + "id": "code-python-def", + "input_ids": [ + 20, + 89, + 361, + 5407, + 9, + 9920, + 599, + 29, + 61, + 10, + 1 + ], + "tokens": [ + "▁de", + "f", + "▁fi", + "bon", + "a", + "cci", + "(", + "n", + ")", + ":", + "" + ], + "decoded_with_special": "def fibonacci(n):", + "decoded_skip_special": "def fibonacci(n):" + }, + { + "id": "code-python-if", + "input_ids": [ + 3, + 99, + 3, + 29, + 3, + 2, + 204, + 10, + 1 + ], + "tokens": [ + "▁", + "if", + "▁", + "n", + "▁", + "", + "▁2", + ":", + "" + ], + "decoded_with_special": "if n 2:", + "decoded_skip_special": "if n 2:" + }, + { + "id": "code-python-return", + "input_ids": [ + 1205, + 3, + 29, + 1 + ], + "tokens": [ + "▁return", + "▁", + "n", + "" + ], + "decoded_with_special": "return n", + "decoded_skip_special": "return n" + }, + { + "id": "code-python-recurse", + "input_ids": [ + 1205, + 361, + 5407, + 9, + 9920, + 599, + 29, + 3, + 18, + 8925, + 1768, + 361, + 5407, + 9, + 9920, + 599, + 29, + 3, + 18, + 9266, + 1 + ], + "tokens": [ + "▁return", + "▁fi", + "bon", + "a", + "cci", + "(", + "n", + "▁", + "-", + "▁1)", + "▁+", + "▁fi", + "bon", + "a", + "cci", + "(", + "n", + "▁", + "-", + "▁2)", + "" + ], + "decoded_with_special": "return fibonacci(n - 1) + fibonacci(n - 2)", + "decoded_skip_special": "return fibonacci(n - 1) + fibonacci(n - 2)" + }, + { + "id": "code-go-func", + "input_ids": [ + 3, + 25322, + 22428, + 599, + 4350, + 6108, + 61, + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "func", + "▁greet", + "(", + "name", + "▁string", + ")", + "▁", + "", + "" + ], + "decoded_with_special": "func greet(name string) ", + "decoded_skip_special": "func greet(name string) " + }, + { + "id": "code-go-tab-if", + "input_ids": [ + 3, + 99, + 564, + 3, + 55, + 2423, + 96, + 121, + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "if", + "▁name", + "▁", + "!", + "=", + "▁\"", + "\"", + "▁", + "", + "" + ], + "decoded_with_special": "if name!= \"\" ", + "decoded_skip_special": "if name!= \"\" " + }, + { + "id": "code-go-tab-printf", + "input_ids": [ + 3, + 89, + 51, + 17, + 5, + 28592, + 89, + 599, + 121, + 566, + 7126, + 6, + 3, + 1454, + 7, + 55, + 2, + 29, + 1686, + 564, + 61, + 1 + ], + "tokens": [ + "▁", + "f", + "m", + "t", + ".", + "Print", + "f", + "(", + "\"", + "H", + "ello", + ",", + "▁", + "%", + "s", + "!", + "", + "n", + "\",", + "▁name", + ")", + "" + ], + "decoded_with_special": "fmt.Printf(\"Hello, %s!n\", name)", + "decoded_skip_special": "fmt.Printf(\"Hello, %s!n\", name)" + }, + { + "id": "code-go-tab-close", + "input_ids": [ + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "" + ], + "decoded_with_special": "", + "decoded_skip_special": "" + }, + { + "id": "code-go-close", + "input_ids": [ + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "" + ], + "decoded_with_special": "", + "decoded_skip_special": "" + }, + { + "id": "programming-identifiers-mixed", + "input_ids": [ + 129, + 834, + 10041, + 834, + 23, + 26, + 4800, + 4, + 834, + 8855, + 371, + 20805, + 834, + 134, + 196, + 13358, + 332, + 834, + 5612, + 834, + 235, + 2217, + 834, + 23, + 26, + 3, + 7369, + 21274, + 134, + 15, + 7, + 1938, + 4302, + 26703, + 1 + ], + "tokens": [ + "▁get", + "_", + "user", + "_", + "i", + "d", + "▁MA", + "X", + "_", + "BU", + "F", + "FER", + "_", + "S", + "I", + "ZE", + "▁T", + "_", + "pad", + "_", + "to", + "ken", + "_", + "i", + "d", + "▁", + "NS", + "URL", + "S", + "e", + "s", + "sion", + "Con", + "figuration", + "" + ], + "decoded_with_special": "get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration", + "decoded_skip_special": "get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration" + }, + { + "id": "programming-identifiers-cases", + "input_ids": [ + 764, + 40, + 254, + 9, + 7, + 15, + 553, + 291, + 25402, + 254, + 9, + 7, + 15, + 17599, + 834, + 6701, + 834, + 4331, + 180, + 4545, + 22684, + 2365, + 834, + 8544, + 17253, + 3, + 157, + 927, + 7, + 834, + 11410, + 834, + 13362, + 834, + 208, + 357, + 1 + ], + "tokens": [ + "▁came", + "l", + "C", + "a", + "s", + "e", + "V", + "ar", + "▁Pascal", + "C", + "a", + "s", + "e", + "▁snake", + "_", + "case", + "_", + "var", + "▁S", + "CR", + "EAM", + "ING", + "_", + "SN", + "AKE", + "▁", + "k", + "8", + "s", + "_", + "pod", + "_", + "count", + "_", + "v", + "2", + "" + ], + "decoded_with_special": "camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2", + "decoded_skip_special": "camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2" + }, + { + "id": "code-string-multiscript", + "input_ids": [ + 18660, + 3274, + 96, + 2, + 14709, + 6609, + 15042, + 6, + 3, + 21325, + 8452, + 4720, + 1768, + 96, + 3, + 318, + 45, + 96, + 1768, + 96, + 2, + 121, + 1 + ], + "tokens": [ + "▁greeting", + "▁=", + "▁\"", + "", + "ри", + "в", + "ет", + ",", + "▁", + "ми", + "р", + "!\"", + "▁+", + "▁\"", + "▁", + "—", + "▁from", + "▁\"", + "▁+", + "▁\"", + "", + "\"", + "" + ], + "decoded_with_special": "greeting = \"ривет, мир!\" + \" — from \" + \"\"", + "decoded_skip_special": "greeting = \"ривет, мир!\" + \" — from \" + \"\"" + }, + { + "id": "code-comment-diacritics", + "input_ids": [ + 13751, + 1417, + 4078, + 154, + 3, + 318, + 20477, + 1670, + 28, + 1227, + 9, + 2685, + 7636, + 1 + ], + "tokens": [ + "▁//", + "▁ré", + "sum", + "é", + "▁", + "—", + "▁Swift", + "▁comment", + "▁with", + "▁di", + "a", + "cri", + "tics", + "" + ], + "decoded_with_special": "// résumé — Swift comment with diacritics", + "decoded_skip_special": "// résumé — Swift comment with diacritics" + }, + { + "id": "code-comment-greek-math", + "input_ids": [ + 3, + 87, + 1935, + 3, + 2, + 1768, + 3, + 2, + 3274, + 3, + 2, + 3, + 318, + 6781, + 18, + 16024, + 7270, + 1670, + 1429, + 87, + 1 + ], + "tokens": [ + "▁", + "/", + "*", + "▁", + "", + "▁+", + "▁", + "", + "▁=", + "▁", + "", + "▁", + "—", + "▁Greek", + "-", + "letter", + "▁math", + "▁comment", + "▁*", + "/", + "" + ], + "decoded_with_special": "/* + = — Greek-letter math comment */", + "decoded_skip_special": "/* + = — Greek-letter math comment */" + }, + { + "id": "code-string-multiscript-emoji", + "input_ids": [ + 752, + 2233, + 3274, + 96, + 2, + 17113, + 3, + 318, + 3, + 2, + 3700, + 5345, + 6652, + 3, + 318, + 3, + 2, + 121, + 1 + ], + "tokens": [ + "▁let", + "▁title", + "▁=", + "▁\"", + "", + "▁Launch", + "▁", + "—", + "▁", + "", + "у", + "с", + "к", + "▁", + "—", + "▁", + "", + "\"", + "" + ], + "decoded_with_special": "let title = \" Launch — уск — \"", + "decoded_skip_special": "let title = \" Launch — уск — \"" + }, + { + "id": "ipa-phonetic-transcription", + "input_ids": [ + 37, + 3, + 25981, + 20267, + 3, + 87, + 89, + 2, + 87, + 21, + 96, + 6779, + 121, + 3, + 318, + 28, + 2189, + 10, + 3, + 2, + 89, + 2, + 5, + 1 + ], + "tokens": [ + "▁The", + "▁", + "IPA", + "▁transcription", + "▁", + "/", + "f", + "", + "/", + "▁for", + "▁\"", + "fish", + "\"", + "▁", + "—", + "▁with", + "▁stress", + ":", + "▁", + "", + "f", + "", + ".", + "" + ], + "decoded_with_special": "The IPA transcription /f/ for \"fish\" — with stress: f.", + "decoded_skip_special": "The IPA transcription /f/ for \"fish\" — with stress: f." + }, + { + "id": "mandarin-tones", + "input_ids": [ + 304, + 1496, + 10, + 3, + 51, + 2975, + 3, + 51, + 2, + 3, + 51, + 85, + 3, + 51, + 2, + 41, + 7296, + 3439, + 77, + 12, + 1496, + 3, + 318, + 306, + 6, + 6937, + 6, + 3, + 30127, + 6, + 7306, + 137, + 1 + ], + "tokens": [ + "▁To", + "nes", + ":", + "▁", + "m", + "á", + "▁", + "m", + "", + "▁", + "m", + "à", + "▁", + "m", + "", + "▁(", + "Man", + "dar", + "in", + "▁to", + "nes", + "▁", + "—", + "▁high", + ",", + "▁rising", + ",", + "▁", + "dipping", + ",", + "▁falling", + ").", + "" + ], + "decoded_with_special": "Tones: má m mà m (Mandarin tones — high, rising, dipping, falling).", + "decoded_skip_special": "Tones: má m mà m (Mandarin tones — high, rising, dipping, falling)." + }, + { + "id": "whitespace-runs", + "input_ids": [ + 21820, + 296, + 3, + 5, + 414, + 1 + ], + "tokens": [ + "▁hello", + "▁world", + "▁", + ".", + "▁end", + "" + ], + "decoded_with_special": "hello world. end", + "decoded_skip_special": "hello world. end" + }, + { + "id": "whitespace-trailing-tabs", + "input_ids": [ + 5032, + 53, + 872, + 6633, + 274, + 126, + 747, + 10, + 41, + 10309, + 61, + 41, + 6633, + 7, + 61, + 1 + ], + "tokens": [ + "▁trail", + "ing", + "▁white", + "space", + "▁before", + "▁new", + "line", + ":", + "▁(", + "tab", + ")", + "▁(", + "space", + "s", + ")", + "" + ], + "decoded_with_special": "trailing whitespace before newline: (tab) (spaces)", + "decoded_skip_special": "trailing whitespace before newline: (tab) (spaces)" + }, + { + "id": "punctuation-period", + "input_ids": [ + 3, + 5, + 1 + ], + "tokens": [ + "▁", + ".", + "" + ], + "decoded_with_special": ".", + "decoded_skip_special": "." + }, + { + "id": "punctuation-comma", + "input_ids": [ + 3, + 6, + 1 + ], + "tokens": [ + "▁", + ",", + "" + ], + "decoded_with_special": ",", + "decoded_skip_special": "," + }, + { + "id": "punctuation-exclamation", + "input_ids": [ + 3, + 55, + 1 + ], + "tokens": [ + "▁", + "!", + "" + ], + "decoded_with_special": "!", + "decoded_skip_special": "!" + }, + { + "id": "punctuation-nested-quotes", + "input_ids": [ + 96, + 5680, + 4609, + 255, + 1380, + 3, + 318, + 96, + 60, + 1427, + 58, + 4720, + 3, + 318, + 4273, + 233, + 1 + ], + "tokens": [ + "▁\"", + "What", + "?\"", + "▁she", + "▁asked", + "▁", + "—", + "▁\"", + "re", + "ally", + "?", + "!\"", + "▁", + "—", + "▁yes", + "...", + "" + ], + "decoded_with_special": "\"What?\" she asked — \"really?!\" — yes...", + "decoded_skip_special": "\"What?\" she asked — \"really?!\" — yes..." + }, + { + "id": "punctuation-dash-variants", + "input_ids": [ + 3, + 233, + 11, + 3, + 2, + 92, + 3, + 104, + 11, + 3, + 318, + 11, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁", + "...", + "▁and", + "▁", + "", + "▁also", + "▁", + "–", + "▁and", + "▁", + "—", + "▁and", + "▁", + "", + ".", + "" + ], + "decoded_with_special": "... and also – and — and .", + "decoded_skip_special": "... and also – and — and." + }, + { + "id": "multiscript-greetings", + "input_ids": [ + 8774, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 14709, + 6609, + 15042, + 3, + 2, + 5, + 1 + ], + "tokens": [ + "▁Hello", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "ри", + "в", + "ет", + "▁", + "", + ".", + "" + ], + "decoded_with_special": "Hello ривет .", + "decoded_skip_special": "Hello ривет." + }, + { + "id": "multiscript-no-spaces", + "input_ids": [ + 3, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "" + ], + "decoded_with_special": "", + "decoded_skip_special": "" + }, + { + "id": "multiscript-zwj-mixed", + "input_ids": [ + 3, + 2, + 6327, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "text", + "", + "" + ], + "decoded_with_special": "text", + "decoded_skip_special": "text" + }, + { + "id": "multiscript-string-concat", + "input_ids": [ + 18660, + 17592, + 2, + 121, + 1220, + 2, + 1220, + 121, + 2, + 121, + 1220, + 2, + 1220, + 2, + 3, + 2, + 3, + 2, + 1 + ], + "tokens": [ + "▁greeting", + "=\"", + "", + "\"", + "+", + "", + "+", + "\"", + "", + "\"", + "+", + "", + "+", + "", + "▁", + "", + "▁", + "", + "" + ], + "decoded_with_special": "greeting=\"\"++\"\"++ ", + "decoded_skip_special": "greeting=\"\"++\"\"++ " + }, + { + "id": "multiscript-emoji-arrows", + "input_ids": [ + 3, + 2, + 3, + 226, + 834, + 23, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 13405, + 3, + 2, + 332, + 2, + 3781, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 414, + 5, + 1 + ], + "tokens": [ + "▁", + "", + "▁", + "x", + "_", + "i", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁München", + "▁", + "", + "▁T", + "", + "ky", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁end", + ".", + "" + ], + "decoded_with_special": " x_i München Tky end.", + "decoded_skip_special": " x_i München Tky end." + }, + { + "id": "multiscript-arrow-chain", + "input_ids": [ + 3, + 2, + 14709, + 6609, + 15042, + 2, + 1 + ], + "tokens": [ + "▁", + "", + "ри", + "в", + "ет", + "", + "" + ], + "decoded_with_special": "ривет", + "decoded_skip_special": "ривет" + }, + { + "id": "multiscript-function-call", + "input_ids": [ + 5575, + 32, + 599, + 2, + 6, + 2, + 6, + 2, + 6, + 2, + 6, + 23912, + 5345, + 30610, + 2, + 61, + 2, + 10, + 2, + 6, + 2, + 10, + 2, + 1 + ], + "tokens": [ + "▁fo", + "o", + "(", + "", + ",", + "", + ",", + "", + ",", + "", + ",", + "ру", + "с", + "ски", + "", + ")", + "", + ":", + "", + ",", + "", + ":", + "", + "" + ], + "decoded_with_special": "foo(,,,,русски):,:", + "decoded_skip_special": "foo(,,,,русски):,:" + }, + { + "id": "multiscript-legacy-symbols", + "input_ids": [ + 465, + 4165, + 3, + 2305, + 2, + 3, + 2, + 3, + 1732, + 3, + 2, + 3, + 2, + 3416, + 2, + 19853, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 1 + ], + "tokens": [ + "▁No", + "42", + "▁", + "TM", + "", + "▁", + "", + "▁", + "®", + "▁", + "", + "▁", + "", + "▁€", + "", + "£", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "▁", + "", + "" + ], + "decoded_with_special": "No42 TM ® €£ ", + "decoded_skip_special": "No42 TM ® €£ " + }, + { + "id": "escape-sequences-with-zwj", + "input_ids": [ + 13046, + 10376, + 2, + 632, + 232, + 2, + 17, + 10309, + 2, + 29, + 5534, + 747, + 2, + 1027, + 518, + 683, + 2, + 7, + 400, + 5317, + 7, + 1 + ], + "tokens": [ + "▁NU", + "LL", + "", + "0", + "and", + "", + "t", + "tab", + "", + "n", + "new", + "line", + "", + "▁Z", + "W", + "J", + "", + "s", + "ele", + "ctor", + "s", + "" + ], + "decoded_with_special": "NULL0andttabnnewline ZWJselectors", + "decoded_skip_special": "NULL0andttabnnewline ZWJselectors" + }, + { + "id": "german-compound-short", + "input_ids": [ + 14649, + 5, + 3, + 18609, + 3, + 15925, + 3780, + 177, + 5837, + 19882, + 17, + 9, + 1483, + 935, + 403, + 74, + 20278, + 27373, + 3458, + 12584, + 7, + 8019, + 4605, + 221, + 5, + 3, + 2, + 1 + ], + "tokens": [ + "▁Natürlich", + ".", + "▁", + "Deutsch", + "▁", + "zieht", + "▁jetzt", + "▁den", + "▁Kom", + "posi", + "t", + "a", + "ham", + "mer", + "▁aus", + "▁der", + "▁Gram", + "matik", + "werk", + "zeug", + "s", + "chu", + "bla", + "de", + ".", + "▁", + "", + "" + ], + "decoded_with_special": "Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. ", + "decoded_skip_special": "Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. " + }, + { + "id": "german-compound-long-1", + "input_ids": [ + 1318, + 20147, + 29, + 15, + 2370, + 397, + 23020, + 624, + 18711, + 15, + 17454, + 11611, + 1518, + 236, + 14854, + 17, + 9, + 4932, + 52, + 232, + 17704, + 23, + 15, + 346, + 32, + 6425, + 449, + 7709, + 645, + 12412, + 27912, + 624, + 2897, + 7, + 9788, + 7, + 6682, + 64, + 59, + 3930, + 16, + 1110, + 14392, + 2528, + 729, + 1599, + 35, + 89, + 1468, + 2264, + 23, + 172, + 5671, + 67, + 5248, + 9, + 202, + 2626, + 13991, + 76, + 89, + 5335, + 193, + 16814, + 4739, + 2427, + 76, + 115, + 7, + 5993, + 29, + 15519, + 60, + 31898, + 35, + 5, + 1 + ], + "tokens": [ + "▁Im", + "▁Morgen", + "n", + "e", + "bel", + "ge", + "danken", + "ver", + "äst", + "e", + "lungs", + "wald", + "▁stand", + "▁ein", + "▁Kaffee", + "t", + "a", + "ssen", + "r", + "and", + "philosoph", + "i", + "e", + "be", + "o", + "bach", + "ter", + "▁neben", + "▁einer", + "▁Regen", + "schirm", + "ver", + "ges", + "s", + "lichkeit", + "s", + "station", + "▁und", + "▁not", + "ierte", + "▁in", + "▁sein", + "▁Sonntag", + "mor", + "gen", + "ide", + "en", + "f", + "ang", + "not", + "i", + "z", + "buch", + "▁die", + "▁erst", + "a", + "un", + "liche", + "▁Hä", + "u", + "f", + "igkeit", + "▁von", + "▁Fenster", + "bank", + "sta", + "u", + "b", + "s", + "onne", + "n", + "licht", + "re", + "flexion", + "en", + ".", + "" + ], + "decoded_with_special": "Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen.", + "decoded_skip_special": "Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen." + }, + { + "id": "german-compound-long-2", + "input_ids": [ + 7974, + 6889, + 4505, + 51, + 17, + 15, + 266, + 27322, + 21328, + 4015, + 2370, + 15, + 2295, + 13387, + 6655, + 29, + 6367, + 7, + 11938, + 6, + 3, + 5729, + 236, + 1571, + 526, + 9514, + 16985, + 9622, + 15, + 9081, + 27132, + 7, + 12331, + 51, + 3194, + 7, + 30082, + 52, + 181, + 645, + 4691, + 1572, + 35, + 2989, + 16296, + 7, + 4414, + 6638, + 52, + 9, + 7, + 23131, + 7, + 14616, + 23029, + 615, + 5, + 1392, + 74, + 12225, + 15, + 19875, + 17, + 15, + 289, + 236, + 3488, + 40, + 2217, + 9669, + 17, + 2558, + 16589, + 25775, + 22934, + 4524, + 5041, + 343, + 219, + 665, + 890, + 5672, + 17105, + 7, + 12835, + 11502, + 1836, + 13387, + 19274, + 64, + 3, + 1753, + 89, + 10, + 395, + 188, + 3997, + 425, + 6, + 5048, + 27429, + 5282, + 266, + 18952, + 89, + 9, + 537, + 624, + 24458, + 13387, + 26997, + 23717, + 55, + 735, + 1 + ], + "tokens": [ + "▁Neben", + "▁ihm", + "▁sum", + "m", + "t", + "e", + "▁eine", + "▁Kühl", + "schrank", + "innen", + "bel", + "e", + "uch", + "tungs", + "erin", + "n", + "erung", + "s", + "maschine", + ",", + "▁", + "während", + "▁ein", + "▁Mar", + "me", + "laden", + "glas", + "deck", + "e", + "lö", + "ffnung", + "s", + "opti", + "m", + "ierung", + "s", + "beauftragte", + "r", + "▁mit", + "▁einer", + "▁Tre", + "pp", + "en", + "haus", + "aku", + "s", + "tik", + "über", + "r", + "a", + "s", + "chung", + "s", + "analyse", + "▁beschäftigt", + "▁war", + ".", + "▁Aus", + "▁der", + "▁Fern", + "e", + "▁näher", + "t", + "e", + "▁sich", + "▁ein", + "▁Wo", + "l", + "ken", + "kra", + "t", + "zer", + "fahr", + "stuhl", + "musik", + "kom", + "pon", + "ist", + "▁auf", + "▁einem", + "▁Ein", + "rad", + "verkehr", + "s", + "regel", + "miss", + "ach", + "tungs", + "gerät", + "▁und", + "▁", + "rie", + "f", + ":", + "▁„", + "A", + "cht", + "ung", + ",", + "▁dort", + "▁vorne", + "▁liegt", + "▁eine", + "▁Gedanken", + "f", + "a", + "den", + "ver", + "kno", + "tungs", + "gefahren", + "situation", + "!", + "“", + "" + ], + "decoded_with_special": "Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“", + "decoded_skip_special": "Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“" + }, + { + "id": "german-compound-long-3", + "input_ids": [ + 276, + 9081, + 17, + 172, + 1411, + 3, + 29304, + 15, + 289, + 236, + 23542, + 7, + 23675, + 440, + 29040, + 5165, + 20497, + 1493, + 138, + 6, + 64, + 8964, + 3990, + 4650, + 324, + 4052, + 15833, + 8142, + 635, + 49, + 29, + 9309, + 972, + 9304, + 29724, + 7, + 2919, + 16477, + 4015, + 6, + 236, + 10469, + 172, + 3321, + 122, + 51, + 17022, + 35, + 22337, + 29217, + 64, + 236, + 6012, + 3484, + 5572, + 548, + 11802, + 52, + 449, + 1923, + 1572, + 35, + 9081, + 16387, + 60, + 31898, + 7, + 3493, + 9, + 6941, + 7, + 10109, + 5, + 25400, + 3, + 12880, + 324, + 680, + 177, + 6387, + 17575, + 218, + 26332, + 7, + 15519, + 7, + 16842, + 324, + 3304, + 324, + 346, + 17, + 21136, + 4437, + 64, + 3, + 16260, + 3466, + 3930, + 17547, + 2168, + 7, + 7, + 13576, + 35, + 3027, + 22033, + 425, + 7, + 21889, + 29, + 5, + 1 + ], + "tokens": [ + "▁P", + "lö", + "t", + "z", + "lich", + "▁", + "öffnet", + "e", + "▁sich", + "▁ein", + "▁Zeitung", + "s", + "seiten", + "um", + "blätter", + "wind", + "stoß", + "port", + "al", + ",", + "▁und", + "▁heraus", + "▁pur", + "zel", + "ten", + "▁drei", + "▁Büro", + "kla", + "mm", + "er", + "n", + "sort", + "ier", + "welt", + "meisterschaft", + "s", + "teil", + "nehmer", + "innen", + ",", + "▁ein", + "▁Garten", + "z", + "wer", + "g", + "m", + "ütz", + "en", + "farben", + "berater", + "▁und", + "▁ein", + "▁hoch", + "gra", + "dig", + "▁ver", + "wir", + "r", + "ter", + "▁Su", + "pp", + "en", + "lö", + "ffel", + "re", + "flexion", + "s", + "met", + "a", + "phy", + "s", + "iker", + ".", + "▁Gemeinsam", + "▁", + "gründe", + "ten", + "▁sie", + "▁den", + "▁Bundes", + "verband", + "▁für", + "▁Nachmittag", + "s", + "licht", + "s", + "chat", + "ten", + "kan", + "ten", + "be", + "t", + "rach", + "tung", + "▁und", + "▁", + "impro", + "vis", + "ierte", + "▁Sofa", + "ki", + "s", + "s", + "enburg", + "en", + "vert", + "eidig", + "ung", + "s", + "strategie", + "n", + ".", + "" + ], + "decoded_with_special": "Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien.", + "decoded_skip_special": "Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien." + }, + { + "id": "german-compound-long-4", + "input_ids": [ + 736, + 9436, + 548, + 21852, + 35, + 289, + 1854, + 256, + 2963, + 26, + 7837, + 25524, + 17475, + 35, + 2427, + 76, + 115, + 122, + 40, + 14777, + 11647, + 6, + 2275, + 74, + 14854, + 17, + 9, + 4932, + 52, + 232, + 17704, + 23, + 15, + 346, + 32, + 6425, + 449, + 266, + 3, + 14178, + 2626, + 20280, + 1271, + 15, + 510, + 67, + 549, + 29214, + 193, + 597, + 7, + 77, + 29, + 7, + 27839, + 7, + 31889, + 6, + 20684, + 3252, + 5768, + 221, + 107, + 29, + 1272, + 115, + 1818, + 64, + 1008, + 402, + 26, + 9, + 8078, + 26453, + 8255, + 7, + 16852, + 10717, + 5230, + 6125, + 7, + 51, + 17022, + 35, + 24458, + 102, + 1010, + 5606, + 2919, + 346, + 7, + 24635, + 7, + 19307, + 35, + 3, + 16219, + 17, + 5, + 1 + ], + "tokens": [ + "▁Am", + "▁Abend", + "▁ver", + "sammelt", + "en", + "▁sich", + "▁alle", + "▁im", + "▁Mon", + "d", + "schein", + "fenster", + "rahm", + "en", + "sta", + "u", + "b", + "g", + "l", + "itzer", + "zimmer", + ",", + "▁wo", + "▁der", + "▁Kaffee", + "t", + "a", + "ssen", + "r", + "and", + "philosoph", + "i", + "e", + "be", + "o", + "bach", + "ter", + "▁eine", + "▁", + "feier", + "liche", + "▁Schluss", + "red", + "e", + "▁über", + "▁die", + "▁W", + "ichtigkeit", + "▁von", + "▁Un", + "s", + "in", + "n", + "s", + "produktion", + "s", + "freude", + ",", + "▁Sprach", + "mus", + "kel", + "de", + "h", + "n", + "ü", + "b", + "ungen", + "▁und", + "▁Don", + "au", + "d", + "a", + "mpf", + "schiff", + "fahrt", + "s", + "gesellschaft", + "ska", + "pit", + "än", + "s", + "m", + "ütz", + "en", + "kno", + "p", + "fer", + "satz", + "teil", + "be", + "s", + "chaffung", + "s", + "problem", + "en", + "▁", + "hiel", + "t", + ".", + "" + ], + "decoded_with_special": "Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt.", + "decoded_skip_special": "Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt." + }, + { + "id": "german-compound-long-5", + "input_ids": [ + 2744, + 1836, + 3, + 29020, + 15, + 20712, + 1811, + 7704, + 35, + 4474, + 6, + 24402, + 468, + 74, + 27322, + 21328, + 4015, + 2370, + 15, + 2295, + 13387, + 6655, + 29, + 6367, + 7, + 11938, + 6, + 67, + 289, + 510, + 3, + 24347, + 29, + 221, + 12087, + 1872, + 27132, + 7, + 3019, + 12458, + 7, + 265, + 5754, + 7, + 6354, + 28910, + 3, + 115, + 27457, + 5, + 1 + ], + "tokens": [ + "▁Dan", + "ach", + "▁", + "herrscht", + "e", + "▁allgemeine", + "▁Zu", + "fried", + "en", + "heit", + ",", + "▁außer", + "▁bei", + "▁der", + "▁Kühl", + "schrank", + "innen", + "bel", + "e", + "uch", + "tungs", + "erin", + "n", + "erung", + "s", + "maschine", + ",", + "▁die", + "▁sich", + "▁über", + "▁", + "mangel", + "n", + "de", + "▁Tür", + "ö", + "ffnung", + "s", + "auf", + "merk", + "s", + "am", + "keit", + "s", + "wert", + "schätzung", + "▁", + "b", + "eklagte", + ".", + "" + ], + "decoded_with_special": "Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte.", + "decoded_skip_special": "Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte." + } + ] +} diff --git a/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/tinyllama_multilingual.json b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/tinyllama_multilingual.json new file mode 100644 index 00000000..5cdee31b --- /dev/null +++ b/Tests/TokenizersTests/Resources/MultilingualConformance/baselines/tinyllama_multilingual.json @@ -0,0 +1,5969 @@ +{ + "metadata": { + "model_id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "transformers_version": "4.57.1", + "generated_at": "2026-05-15T23:58:38+00:00", + "input_count": 83 + }, + "entries": [ + { + "id": "latin-diacritics-french", + "input_ids": [ + 1, + 274, + 28059, + 6896, + 398, + 29948, + 1055, + 30085, + 345, + 2258, + 30019, + 1943, + 813, + 2181, + 5000, + 1506, + 30095, + 29880, + 1318, + 471, + 15129, + 29889 + ], + "tokens": [ + "", + "▁c", + "afé", + "▁rés", + "um", + "é", + "▁na", + "ï", + "ve", + "▁fa", + "ç", + "ade", + "▁—", + "▁cr", + "ème", + "▁br", + "û", + "l", + "ée", + "▁was", + "▁excellent", + "." + ], + "decoded_with_special": " café résumé naïve façade — crème brûlée was excellent.", + "decoded_skip_special": "café résumé naïve façade — crème brûlée was excellent." + }, + { + "id": "latin-diacritics-mixed", + "input_ids": [ + 1, + 379, + 25146, + 2939, + 1771, + 17959, + 29889, + 3219, + 30046, + 1648, + 12278, + 263, + 425, + 1236, + 9658, + 378, + 7942, + 29889 + ], + "tokens": [ + "", + "▁H", + "ügel", + "▁über", + "▁Br", + "ücke", + ".", + "▁Ma", + "ñ", + "ana", + "▁voy", + "▁a", + "▁la", + "▁pe", + "ña", + "▁con", + "▁José", + "." + ], + "decoded_with_special": " Hügel über Brücke. Mañana voy a la peña con José.", + "decoded_skip_special": "Hügel über Brücke. Mañana voy a la peña con José." + }, + { + "id": "latin-diacritics-portuguese", + "input_ids": [ + 1, + 12281, + 16038, + 904, + 3672, + 17931, + 26636, + 1647, + 29936, + 20763, + 2907, + 29889 + ], + "tokens": [ + "", + "▁São", + "▁Paulo", + "▁é", + "▁uma", + "▁cidade", + "▁interess", + "ante", + ";", + "▁Köln", + "▁auch", + "." + ], + "decoded_with_special": " São Paulo é uma cidade interessante; Köln auch.", + "decoded_skip_special": "São Paulo é uma cidade interessante; Köln auch." + }, + { + "id": "cyrillic-greeting", + "input_ids": [ + 1, + 7203, + 7616, + 29892, + 4157, + 29927, + 29991, + 20782, + 2399, + 464, + 2721, + 2476, + 507, + 1902, + 9011, + 29889 + ], + "tokens": [ + "", + "▁При", + "вет", + ",", + "▁ми", + "р", + "!", + "▁Это", + "▁те", + "ст", + "▁то", + "ке", + "ни", + "за", + "тора", + "." + ], + "decoded_with_special": " Привет, мир! Это тест токенизатора.", + "decoded_skip_special": "Привет, мир! Это тест токенизатора." + }, + { + "id": "cyrillic-prose", + "input_ids": [ + 1, + 27156, + 813, + 7349, + 644, + 3432, + 11325, + 29889, + 21636, + 13388, + 29871, + 29896, + 29906, + 4157, + 15216, + 26119, + 29889 + ], + "tokens": [ + "", + "▁Москва", + "▁—", + "▁сто", + "ли", + "ца", + "▁России", + ".", + "▁Население", + "▁более", + "▁", + "1", + "2", + "▁ми", + "лли", + "онов", + "." + ], + "decoded_with_special": " Москва — столица России. Население более 12 миллионов.", + "decoded_skip_special": "Москва — столица России. Население более 12 миллионов." + }, + { + "id": "greek-greeting", + "input_ids": [ + 1, + 29871, + 30314, + 30110, + 30142, + 30183, + 30167, + 30273, + 30147, + 30110, + 29871, + 30173, + 30228, + 30164, + 30167, + 30151, + 29991, + 29871, + 30835, + 29871, + 30197, + 30142, + 30407, + 30164, + 30164, + 30110, + 29871, + 30151, + 30207, + 30133, + 30110, + 30136, + 29871, + 30228, + 30167, + 30123, + 30147, + 30237, + 30183, + 29889 + ], + "tokens": [ + "", + "▁", + "Κ", + "α", + "λ", + "η", + "μ", + "έ", + "ρ", + "α", + "▁", + "κ", + "ό", + "σ", + "μ", + "ε", + "!", + "▁", + "Η", + "▁", + "γ", + "λ", + "ώ", + "σ", + "σ", + "α", + "▁", + "ε", + "ί", + "ν", + "α", + "ι", + "▁", + "ό", + "μ", + "ο", + "ρ", + "φ", + "η", + "." + ], + "decoded_with_special": " Καλημέρα κόσμε! Η γλώσσα είναι όμορφη.", + "decoded_skip_special": "Καλημέρα κόσμε! Η γλώσσα είναι όμορφη." + }, + { + "id": "cjk-simplified-greeting", + "input_ids": [ + 1, + 29871, + 30919, + 31076, + 29892, + 30793, + 30967, + 29991, + 30810, + 30392, + 30748, + 235, + 178, + 144, + 30943, + 31851, + 31787, + 30267 + ], + "tokens": [ + "", + "▁", + "你", + "好", + ",", + "世", + "界", + "!", + "这", + "是", + "分", + "<0xE8>", + "<0xAF>", + "<0x8D>", + "器", + "测", + "试", + "。" + ], + "decoded_with_special": " 你好,世界!这是分词器测试。", + "decoded_skip_special": "你好,世界!这是分词器测试。" + }, + { + "id": "cjk-simplified-prose", + "input_ids": [ + 1, + 29871, + 30275, + 30333, + 30392, + 30793, + 30967, + 30429, + 30785, + 30406, + 30313, + 30856, + 30878, + 30923, + 30210, + 31505, + 31243, + 30577, + 30287, + 30267 + ], + "tokens": [ + "", + "▁", + "中", + "文", + "是", + "世", + "界", + "上", + "使", + "用", + "人", + "口", + "最", + "多", + "的", + "语", + "言", + "之", + "一", + "。" + ], + "decoded_with_special": " 中文是世界上使用人口最多的语言之一。", + "decoded_skip_special": "中文是世界上使用人口最多的语言之一。" + }, + { + "id": "cjk-traditional-compare", + "input_ids": [ + 1, + 29871, + 234, + 188, + 132, + 236, + 174, + 151, + 30275, + 30333, + 235, + 139, + 138, + 234, + 179, + 164, + 236, + 174, + 151, + 30275, + 30333, + 30417, + 30744, + 30413, + 30980, + 30267 + ], + "tokens": [ + "", + "▁", + "<0xE7>", + "<0xB9>", + "<0x81>", + "<0xE9>", + "<0xAB>", + "<0x94>", + "中", + "文", + "<0xE8>", + "<0x88>", + "<0x87>", + "<0xE7>", + "<0xB0>", + "<0xA1>", + "<0xE9>", + "<0xAB>", + "<0x94>", + "中", + "文", + "有", + "所", + "不", + "同", + "。" + ], + "decoded_with_special": " 繁體中文與簡體中文有所不同。", + "decoded_skip_special": "繁體中文與簡體中文有所不同。" + }, + { + "id": "cjk-traditional-hongkong", + "input_ids": [ + 1, + 29871, + 31113, + 31449, + 30210, + 234, + 188, + 132, + 236, + 174, + 151, + 30275, + 30333, + 30417, + 234, + 144, + 171, + 31141, + 30210, + 235, + 172, + 161, + 232, + 192, + 156, + 30267 + ], + "tokens": [ + "", + "▁", + "香", + "港", + "的", + "<0xE7>", + "<0xB9>", + "<0x81>", + "<0xE9>", + "<0xAB>", + "<0x94>", + "中", + "文", + "有", + "<0xE7>", + "<0x8D>", + "<0xA8>", + "特", + "的", + "<0xE8>", + "<0xA9>", + "<0x9E>", + "<0xE5>", + "<0xBD>", + "<0x99>", + "。" + ], + "decoded_with_special": " 香港的繁體中文有獨特的詞彙。", + "decoded_skip_special": "香港的繁體中文有獨特的詞彙。" + }, + { + "id": "japanese-voiced-kana-greeting", + "input_ids": [ + 1, + 29871, + 30589, + 30389, + 30353, + 30644, + 30449, + 30330, + 30793, + 30967, + 30267, + 30279, + 30185, + 30305, + 30576, + 30260, + 31123, + 30185, + 30199, + 30572, + 30255, + 30279, + 30499, + 30427, + 30267 + ], + "tokens": [ + "", + "▁", + "こ", + "ん", + "に", + "ち", + "は", + "、", + "世", + "界", + "。", + "ト", + "ー", + "ク", + "ナ", + "イ", + "ザ", + "ー", + "の", + "テ", + "ス", + "ト", + "で", + "す", + "。" + ], + "decoded_with_special": " こんにちは、世界。トークナイザーのテストです。", + "decoded_skip_special": "こんにちは、世界。トークナイザーのテストです。" + }, + { + "id": "japanese-voiced-kana-prose", + "input_ids": [ + 1, + 29871, + 30591, + 30675, + 30369, + 31028, + 30185, + 30449, + 30364, + 30466, + 30723, + 235, + 182, + 148, + 31111, + 30412, + 30371, + 235, + 169, + 182, + 30867, + 30533, + 30499, + 30427, + 30267 + ], + "tokens": [ + "", + "▁", + "東", + "京", + "タ", + "ワ", + "ー", + "は", + "と", + "て", + "も", + "<0xE8>", + "<0xB3>", + "<0x91>", + "や", + "か", + "な", + "<0xE8>", + "<0xA6>", + "<0xB3>", + "光", + "地", + "で", + "す", + "。" + ], + "decoded_with_special": " 東京タワーはとても賑やかな観光地です。", + "decoded_skip_special": "東京タワーはとても賑やかな観光地です。" + }, + { + "id": "hangul-syllables-greeting", + "input_ids": [ + 1, + 29871, + 31734, + 238, + 136, + 152, + 30944, + 31578, + 31527, + 29892, + 29871, + 31578, + 237, + 182, + 135, + 29991, + 29871, + 240, + 137, + 163, + 240, + 132, + 175, + 31207, + 30393, + 239, + 163, + 131, + 29871, + 240, + 136, + 143, + 30784, + 31177, + 239, + 161, + 136, + 31063, + 30709, + 29889 + ], + "tokens": [ + "", + "▁", + "안", + "<0xEB>", + "<0x85>", + "<0x95>", + "하", + "세", + "요", + ",", + "▁", + "세", + "<0xEA>", + "<0xB3>", + "<0x84>", + "!", + "▁", + "<0xED>", + "<0x86>", + "<0xA0>", + "<0xED>", + "<0x81>", + "<0xAC>", + "나", + "이", + "<0xEC>", + "<0xA0>", + "<0x80>", + "▁", + "<0xED>", + "<0x85>", + "<0x8C>", + "스", + "트", + "<0xEC>", + "<0x9E>", + "<0x85>", + "니", + "다", + "." + ], + "decoded_with_special": " 안녕하세요, 세계! 토크나이저 테스트입니다.", + "decoded_skip_special": "안녕하세요, 세계! 토크나이저 테스트입니다." + }, + { + "id": "hangul-syllables-prose", + "input_ids": [ + 1, + 29871, + 30877, + 31293, + 31129, + 31081, + 29871, + 30877, + 237, + 187, + 131, + 30906, + 29871, + 240, + 148, + 159, + 30827, + 31980, + 31063, + 30709, + 29889 + ], + "tokens": [ + "", + "▁", + "한", + "국", + "어", + "는", + "▁", + "한", + "<0xEA>", + "<0xB8>", + "<0x80>", + "로", + "▁", + "<0xED>", + "<0x91>", + "<0x9C>", + "기", + "합", + "니", + "다", + "." + ], + "decoded_with_special": " 한국어는 한글로 표기합니다.", + "decoded_skip_special": "한국어는 한글로 표기합니다." + }, + { + "id": "arabic-greeting", + "input_ids": [ + 1, + 29871, + 30159, + 30156, + 30240, + 30177, + 30112, + 29871, + 30177, + 19233, + 30218, + 19233, + 30159, + 29991, + 29871, + 30204, + 30851, + 30112, + 29871, + 30112, + 30321, + 30195, + 30177, + 30112, + 30156, + 29871, + 30138, + 30138, + 30159, + 30240, + 30138, + 30138, + 24508, + 30138, + 30611, + 30171, + 30163, + 29889 + ], + "tokens": [ + "", + "▁", + "م", + "ر", + "ح", + "ب", + "ا", + "▁", + "ب", + "ال", + "ع", + "ال", + "م", + "!", + "▁", + "ه", + "ذ", + "ا", + "▁", + "ا", + "خ", + "ت", + "ب", + "ا", + "ر", + "▁", + "ل", + "ل", + "م", + "ح", + "ل", + "ل", + "▁ال", + "ل", + "غ", + "و", + "ي", + "." + ], + "decoded_with_special": " مرحبا بالعالم! هذا اختبار للمحلل اللغوي.", + "decoded_skip_special": "مرحبا بالعالم! هذا اختبار للمحلل اللغوي." + }, + { + "id": "arabic-prose", + "input_ids": [ + 1, + 24508, + 30138, + 30611, + 30242, + 24508, + 30218, + 30156, + 30177, + 30163, + 30242, + 29871, + 30138, + 30611, + 30242, + 29871, + 30198, + 30112, + 30159, + 30163, + 30242, + 29871, + 30195, + 30283, + 30195, + 30177, + 29871, + 30159, + 30162, + 24508, + 30163, + 30159, + 30163, + 30162, + 29871, + 30754, + 30138, + 30480, + 24508, + 30163, + 30198, + 30112, + 30156, + 29889 + ], + "tokens": [ + "", + "▁ال", + "ل", + "غ", + "ة", + "▁ال", + "ع", + "ر", + "ب", + "ي", + "ة", + "▁", + "ل", + "غ", + "ة", + "▁", + "س", + "ا", + "م", + "ي", + "ة", + "▁", + "ت", + "ك", + "ت", + "ب", + "▁", + "م", + "ن", + "▁ال", + "ي", + "م", + "ي", + "ن", + "▁", + "إ", + "ل", + "ى", + "▁ال", + "ي", + "س", + "ا", + "ر", + "." + ], + "decoded_with_special": " اللغة العربية لغة سامية تكتب من اليمين إلى اليسار.", + "decoded_skip_special": "اللغة العربية لغة سامية تكتب من اليمين إلى اليسار." + }, + { + "id": "hebrew-greeting", + "input_ids": [ + 1, + 29871, + 30294, + 30249, + 30205, + 30404, + 29871, + 30324, + 30205, + 30249, + 30404, + 29991, + 29871, + 30776, + 30235, + 29871, + 30285, + 30276, + 30428, + 30447, + 29871, + 30294, + 30249, + 29871, + 30235, + 30639, + 30205, + 30433, + 30328, + 30196, + 30196, + 30776, + 30236, + 29889 + ], + "tokens": [ + "", + "▁", + "ש", + "ל", + "ו", + "ם", + "▁", + "ע", + "ו", + "ל", + "ם", + "!", + "▁", + "ז", + "ה", + "▁", + "מ", + "ב", + "ח", + "ן", + "▁", + "ש", + "ל", + "▁", + "ה", + "ט", + "ו", + "ק", + "נ", + "י", + "י", + "ז", + "ר", + "." + ], + "decoded_with_special": " שלום עולם! זה מבחן של הטוקנייזר.", + "decoded_skip_special": "שלום עולם! זה מבחן של הטוקנייזר." + }, + { + "id": "devanagari-hindi", + "input_ids": [ + 1, + 29871, + 30424, + 30485, + 30489, + 30296, + 30475, + 30569, + 29871, + 30694, + 30702, + 30424, + 30436, + 30640, + 30269, + 29991, + 29871, + 30640, + 30714, + 29871, + 31329, + 30799, + 30444, + 30424, + 30269, + 227, + 167, + 138, + 30871, + 31821, + 30316, + 29871, + 30621, + 30316, + 30580, + 30444, + 30296, + 31330, + 31377, + 29871, + 30714, + 31678, + 31776 + ], + "tokens": [ + "", + "▁", + "न", + "म", + "स", + "्", + "त", + "े", + "▁", + "द", + "ु", + "न", + "ि", + "य", + "ा", + "!", + "▁", + "य", + "ह", + "▁", + "ट", + "ो", + "क", + "न", + "ा", + "<0xE0>", + "<0xA4>", + "<0x87>", + "ज", + "़", + "र", + "▁", + "प", + "र", + "ी", + "क", + "्", + "ष", + "ण", + "▁", + "ह", + "ै", + "।" + ], + "decoded_with_special": " नमस्ते दुनिया! यह टोकनाइज़र परीक्षण है।", + "decoded_skip_special": "नमस्ते दुनिया! यह टोकनाइज़र परीक्षण है।" + }, + { + "id": "devanagari-sanskrit", + "input_ids": [ + 1, + 29871, + 30489, + 30757, + 30489, + 30296, + 30444, + 227, + 168, + 134, + 30475, + 29871, + 31380, + 30269, + 30316, + 30475, + 29871, + 30444, + 30580, + 29871, + 227, + 167, + 146, + 30444, + 29871, + 30621, + 30296, + 30316, + 30269, + 31132, + 30580, + 30424, + 29871, + 31380, + 30269, + 31330, + 30269, + 29871, + 30714, + 31678, + 31776 + ], + "tokens": [ + "", + "▁", + "स", + "ं", + "स", + "्", + "क", + "<0xE0>", + "<0xA5>", + "<0x83>", + "त", + "▁", + "भ", + "ा", + "र", + "त", + "▁", + "क", + "ी", + "▁", + "<0xE0>", + "<0xA4>", + "<0x8F>", + "क", + "▁", + "प", + "्", + "र", + "ा", + "च", + "ी", + "न", + "▁", + "भ", + "ा", + "ष", + "ा", + "▁", + "ह", + "ै", + "।" + ], + "decoded_with_special": " संस्कृत भारत की एक प्राचीन भाषा है।", + "decoded_skip_special": "संस्कृत भारत की एक प्राचीन भाषा है।" + }, + { + "id": "thai-combining-marks-greeting", + "input_ids": [ + 1, + 29871, + 30547, + 30492, + 30510, + 30547, + 30718, + 30691, + 30913, + 30289, + 30492, + 31364, + 30496, + 30425, + 29991, + 29871, + 30348, + 30691, + 30543, + 30759, + 31422, + 30351, + 30425, + 30289, + 30297, + 30595, + 30718, + 30547, + 30351, + 30526, + 30401, + 30759, + 30297, + 31422, + 30543, + 30351, + 30398, + 30618, + 30510, + 30718, + 30759, + 30747 + ], + "tokens": [ + "", + "▁", + "ส", + "ว", + "ั", + "ส", + "ด", + "ี", + "ช", + "า", + "ว", + "โ", + "ล", + "ก", + "!", + "▁", + "น", + "ี", + "่", + "ค", + "ื", + "อ", + "ก", + "า", + "ร", + "ท", + "ด", + "ส", + "อ", + "บ", + "เ", + "ค", + "ร", + "ื", + "่", + "อ", + "ง", + "ต", + "ั", + "ด", + "ค", + "ำ" + ], + "decoded_with_special": " สวัสดีชาวโลก! นี่คือการทดสอบเครื่องตัดคำ", + "decoded_skip_special": "สวัสดีชาวโลก! นี่คือการทดสอบเครื่องตัดคำ" + }, + { + "id": "thai-combining-marks-prose", + "input_ids": [ + 1, + 29871, + 31070, + 30289, + 31964, + 30289, + 31252, + 30595, + 30549, + 31252, + 30501, + 30543, + 227, + 188, + 134, + 30913, + 30652, + 30913, + 30543, + 30351, + 30398, + 30492, + 30543, + 30289, + 30398, + 30297, + 30823, + 30663, + 30492, + 30543, + 30289, + 30398, + 30759, + 30747 + ], + "tokens": [ + "", + "▁", + "ภ", + "า", + "ษ", + "า", + "ไ", + "ท", + "ย", + "ไ", + "ม", + "่", + "<0xE0>", + "<0xB9>", + "<0x83>", + "ช", + "้", + "ช", + "่", + "อ", + "ง", + "ว", + "่", + "า", + "ง", + "ร", + "ะ", + "ห", + "ว", + "่", + "า", + "ง", + "ค", + "ำ" + ], + "decoded_with_special": " ภาษาไทยไม่ใช้ช่องว่างระหว่างคำ", + "decoded_skip_special": "ภาษาไทยไม่ใช้ช่องว่างระหว่างคำ" + }, + { + "id": "mixed-script-cjk-latin", + "input_ids": [ + 1, + 450, + 29871, + 30662, + 30675, + 17487, + 297, + 29871, + 29906, + 29900, + 29906, + 29953, + 813, + 9577, + 6079, + 29991 + ], + "tokens": [ + "", + "▁The", + "▁", + "北", + "京", + "▁trip", + "▁in", + "▁", + "2", + "0", + "2", + "6", + "▁—", + "▁très", + "▁bien", + "!" + ], + "decoded_with_special": " The 北京 trip in 2026 — très bien!", + "decoded_skip_special": "The 北京 trip in 2026 — très bien!" + }, + { + "id": "mixed-script-katakana-code", + "input_ids": [ + 1, + 29871, + 30459, + 30185, + 30335, + 30420, + 30809, + 30645, + 30185, + 29901, + 6494, + 4343, + 297, + 9063, + 263, + 29941, + 29888, + 29929, + 29883, + 29906, + 29890, + 29889 + ], + "tokens": [ + "", + "▁", + "コ", + "ー", + "ド", + "レ", + "ビ", + "ュ", + "ー", + ":", + "▁bug", + "▁fixed", + "▁in", + "▁commit", + "▁a", + "3", + "f", + "9", + "c", + "2", + "b", + "." + ], + "decoded_with_special": " コードレビュー: bug fixed in commit a3f9c2b.", + "decoded_skip_special": "コードレビュー: bug fixed in commit a3f9c2b." + }, + { + "id": "math-symbols-summation", + "input_ids": [ + 1, + 7867, + 29901, + 29871, + 30291, + 921, + 29918, + 29875, + 353, + 29871, + 29946, + 29906, + 313, + 3062, + 921, + 29871, + 30264, + 29871, + 30463, + 467 + ], + "tokens": [ + "", + "▁Result", + ":", + "▁", + "Σ", + "▁x", + "_", + "i", + "▁=", + "▁", + "4", + "2", + "▁(", + "where", + "▁x", + "▁", + "∈", + "▁", + "ℝ", + ")." + ], + "decoded_with_special": " Result: Σ x_i = 42 (where x ∈ ℝ).", + "decoded_skip_special": "Result: Σ x_i = 42 (where x ∈ ℝ)." + }, + { + "id": "math-symbols-epsilon-delta", + "input_ids": [ + 1, + 10244, + 29901, + 29871, + 30315, + 30151, + 29958, + 29900, + 29871, + 30619, + 30201, + 29958, + 29900, + 1316, + 393, + 891, + 29916, + 29899, + 29916, + 30220, + 29989, + 29966, + 30201, + 29871, + 30381, + 891, + 29888, + 29898, + 29916, + 6817, + 29888, + 29898, + 29916, + 30220, + 10531, + 29966, + 30151, + 29889 + ], + "tokens": [ + "", + "▁Theorem", + ":", + "▁", + "∀", + "ε", + ">", + "0", + "▁", + "∃", + "δ", + ">", + "0", + "▁such", + "▁that", + "▁|", + "x", + "-", + "x", + "₀", + "|", + "<", + "δ", + "▁", + "⇒", + "▁|", + "f", + "(", + "x", + ")-", + "f", + "(", + "x", + "₀", + ")|", + "<", + "ε", + "." + ], + "decoded_with_special": " Theorem: ∀ε>0 ∃δ>0 such that |x-x₀|<δ ⇒ |f(x)-f(x₀)|<ε.", + "decoded_skip_special": "Theorem: ∀ε>0 ∃δ>0 such that |x-x₀|<δ ⇒ |f(x)-f(x₀)|<ε." + }, + { + "id": "currency-symbols-multi", + "input_ids": [ + 1, + 9839, + 29901, + 25540, + 29896, + 29906, + 29889, + 29945, + 29900, + 10309, + 29871, + 30563, + 29896, + 29892, + 29947, + 29906, + 29900, + 29871, + 30583, + 15151, + 29896, + 29900, + 29889, + 29955, + 29945, + 313, + 14850, + 29892, + 2298, + 29871, + 229, + 133, + 194, + 29900, + 29889, + 29900, + 29900, + 29900, + 29946, + 467 + ], + "tokens": [ + "", + "▁Cost", + ":", + "▁€", + "1", + "2", + ".", + "5", + "0", + "▁→", + "▁", + "¥", + "1", + ",", + "8", + "2", + "0", + "▁", + "≈", + "▁£", + "1", + "0", + ".", + "7", + "5", + "▁(", + "approx", + ",", + "▁plus", + "▁", + "<0xE2>", + "<0x82>", + "<0xBF>", + "0", + ".", + "0", + "0", + "0", + "4", + ")." + ], + "decoded_with_special": " Cost: €12.50 → ¥1,820 ≈ £10.75 (approx, plus ₿0.0004).", + "decoded_skip_special": "Cost: €12.50 → ¥1,820 ≈ £10.75 (approx, plus ₿0.0004)." + }, + { + "id": "box-drawing", + "input_ids": [ + 1, + 11773, + 29901, + 29871, + 31002, + 19820, + 8539, + 30091, + 31082, + 11879, + 15043, + 11879, + 29871, + 30227, + 19820, + 8539, + 30091, + 31119, + 813, + 2309, + 29889 + ], + "tokens": [ + "", + "▁Box", + ":", + "▁", + "┌", + "────", + "──", + "─", + "┐", + "▁│", + "▁Hello", + "▁│", + "▁", + "└", + "────", + "──", + "─", + "┘", + "▁—", + "▁done", + "." + ], + "decoded_with_special": " Box: ┌───────┐ │ Hello │ └───────┘ — done.", + "decoded_skip_special": "Box: ┌───────┐ │ Hello │ └───────┘ — done." + }, + { + "id": "dingbats-chess", + "input_ids": [ + 1, + 26005, + 778, + 29901, + 29871, + 30950, + 29871, + 31048, + 29871, + 31067, + 29871, + 31275, + 29871, + 30922, + 29871, + 30649, + 29871, + 229, + 156, + 151, + 29871, + 229, + 156, + 152, + 29871, + 229, + 156, + 153, + 29871, + 229, + 156, + 155, + 29871, + 229, + 156, + 156, + 813, + 521, + 404, + 731, + 29889 + ], + "tokens": [ + "", + "▁Pie", + "ces", + ":", + "▁", + "★", + "▁", + "☆", + "▁", + "♠", + "▁", + "♣", + "▁", + "♥", + "▁", + "♦", + "▁", + "<0xE2>", + "<0x99>", + "<0x94>", + "▁", + "<0xE2>", + "<0x99>", + "<0x95>", + "▁", + "<0xE2>", + "<0x99>", + "<0x96>", + "▁", + "<0xE2>", + "<0x99>", + "<0x98>", + "▁", + "<0xE2>", + "<0x99>", + "<0x99>", + "▁—", + "▁ch", + "ess", + "▁set", + "." + ], + "decoded_with_special": " Pieces: ★ ☆ ♠ ♣ ♥ ♦ ♔ ♕ ♖ ♘ ♙ — chess set.", + "decoded_skip_special": "Pieces: ★ ☆ ♠ ♣ ♥ ♦ ♔ ♕ ♖ ♘ ♙ — chess set." + }, + { + "id": "math-astral-bold-script", + "input_ids": [ + 1, + 5792, + 14288, + 29901, + 29871, + 243, + 160, + 147, + 131, + 243, + 160, + 147, + 132, + 243, + 160, + 147, + 133, + 243, + 160, + 147, + 134, + 29871, + 243, + 160, + 162, + 145, + 243, + 160, + 162, + 146, + 243, + 160, + 162, + 147, + 243, + 160, + 162, + 148, + 813, + 5844, + 2471, + 29901, + 29871, + 243, + 160, + 150, + 147, + 243, + 160, + 150, + 148, + 243, + 160, + 150, + 149, + 243, + 160, + 150, + 150, + 29889 + ], + "tokens": [ + "", + "▁Math", + "▁bold", + ":", + "▁", + "<0xF0>", + "<0x9D>", + "<0x90>", + "<0x80>", + "<0xF0>", + "<0x9D>", + "<0x90>", + "<0x81>", + "<0xF0>", + "<0x9D>", + "<0x90>", + "<0x82>", + "<0xF0>", + "<0x9D>", + "<0x90>", + "<0x83>", + "▁", + "<0xF0>", + "<0x9D>", + "<0x9F>", + "<0x8E>", + "<0xF0>", + "<0x9D>", + "<0x9F>", + "<0x8F>", + "<0xF0>", + "<0x9D>", + "<0x9F>", + "<0x90>", + "<0xF0>", + "<0x9D>", + "<0x9F>", + "<0x91>", + "▁—", + "▁math", + "▁script", + ":", + "▁", + "<0xF0>", + "<0x9D>", + "<0x93>", + "<0x90>", + "<0xF0>", + "<0x9D>", + "<0x93>", + "<0x91>", + "<0xF0>", + "<0x9D>", + "<0x93>", + "<0x92>", + "<0xF0>", + "<0x9D>", + "<0x93>", + "<0x93>", + "." + ], + "decoded_with_special": " Math bold: 𝐀𝐁𝐂𝐃 𝟎𝟏𝟐𝟑 — math script: 𝓐𝓑𝓒𝓓.", + "decoded_skip_special": "Math bold: 𝐀𝐁𝐂𝐃 𝟎𝟏𝟐𝟑 — math script: 𝓐𝓑𝓒𝓓." + }, + { + "id": "astral-egyptian-hieroglyphs", + "input_ids": [ + 1, + 12892, + 713, + 6128, + 468, + 27026, + 29879, + 29901, + 29871, + 243, + 150, + 131, + 131, + 29871, + 243, + 150, + 133, + 131, + 29871, + 243, + 150, + 134, + 131, + 29871, + 243, + 150, + 137, + 145, + 243, + 150, + 136, + 150, + 243, + 150, + 146, + 146, + 243, + 150, + 141, + 153, + 29889 + ], + "tokens": [ + "", + "▁Egypt", + "ian", + "▁hier", + "og", + "lyph", + "s", + ":", + "▁", + "<0xF0>", + "<0x93>", + "<0x80>", + "<0x80>", + "▁", + "<0xF0>", + "<0x93>", + "<0x82>", + "<0x80>", + "▁", + "<0xF0>", + "<0x93>", + "<0x83>", + "<0x80>", + "▁", + "<0xF0>", + "<0x93>", + "<0x86>", + "<0x8E>", + "<0xF0>", + "<0x93>", + "<0x85>", + "<0x93>", + "<0xF0>", + "<0x93>", + "<0x8F>", + "<0x8F>", + "<0xF0>", + "<0x93>", + "<0x8A>", + "<0x96>", + "." + ], + "decoded_with_special": " Egyptian hieroglyphs: 𓀀 𓂀 𓃀 𓆎𓅓𓏏𓊖.", + "decoded_skip_special": "Egyptian hieroglyphs: 𓀀 𓂀 𓃀 𓆎𓅓𓏏𓊖." + }, + { + "id": "astral-cuneiform", + "input_ids": [ + 1, + 315, + 1540, + 5560, + 29901, + 29871, + 243, + 149, + 131, + 176, + 29871, + 243, + 149, + 139, + 163, + 29871, + 243, + 149, + 133, + 154, + 29871, + 243, + 149, + 137, + 163, + 813, + 1804, + 1051, + 11916, + 29889 + ], + "tokens": [ + "", + "▁C", + "une", + "iform", + ":", + "▁", + "<0xF0>", + "<0x92>", + "<0x80>", + "<0xAD>", + "▁", + "<0xF0>", + "<0x92>", + "<0x88>", + "<0xA0>", + "▁", + "<0xF0>", + "<0x92>", + "<0x82>", + "<0x97>", + "▁", + "<0xF0>", + "<0x92>", + "<0x86>", + "<0xA0>", + "▁—", + "▁sign", + "▁list", + "▁samples", + "." + ], + "decoded_with_special": " Cuneiform: 𒀭 𒈠 𒂗 𒆠 — sign list samples.", + "decoded_skip_special": "Cuneiform: 𒀭 𒈠 𒂗 𒆠 — sign list samples." + }, + { + "id": "astral-mahjong", + "input_ids": [ + 1, + 10082, + 29926, + 549, + 260, + 5475, + 29901, + 29871, + 243, + 162, + 131, + 131, + 29871, + 243, + 162, + 131, + 132, + 29871, + 243, + 162, + 131, + 133, + 29871, + 243, + 162, + 131, + 134, + 29871, + 243, + 162, + 131, + 135, + 29871, + 243, + 162, + 131, + 136, + 29871, + 243, + 162, + 131, + 137, + 29871, + 243, + 162, + 131, + 138, + 29889 + ], + "tokens": [ + "", + "▁Mah", + "j", + "ong", + "▁t", + "iles", + ":", + "▁", + "<0xF0>", + "<0x9F>", + "<0x80>", + "<0x80>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x80>", + "<0x81>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x80>", + "<0x82>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x80>", + "<0x83>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x80>", + "<0x84>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x80>", + "<0x85>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x80>", + "<0x86>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x80>", + "<0x87>", + "." + ], + "decoded_with_special": " Mahjong tiles: 🀀 🀁 🀂 🀃 🀄 🀅 🀆 🀇.", + "decoded_skip_special": "Mahjong tiles: 🀀 🀁 🀂 🀃 🀄 🀅 🀆 🀇." + }, + { + "id": "astral-cards", + "input_ids": [ + 1, + 315, + 3163, + 29901, + 29871, + 243, + 162, + 134, + 132, + 29871, + 243, + 162, + 134, + 133, + 29871, + 243, + 162, + 134, + 134, + 29871, + 243, + 162, + 133, + 164, + 29871, + 243, + 162, + 133, + 165, + 29871, + 243, + 162, + 133, + 166, + 29889 + ], + "tokens": [ + "", + "▁C", + "ards", + ":", + "▁", + "<0xF0>", + "<0x9F>", + "<0x83>", + "<0x81>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x83>", + "<0x82>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x83>", + "<0x83>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x82>", + "<0xA1>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x82>", + "<0xA2>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x82>", + "<0xA3>", + "." + ], + "decoded_with_special": " Cards: 🃁 🃂 🃃 🂡 🂢 🂣.", + "decoded_skip_special": "Cards: 🃁 🃂 🃃 🂡 🂢 🂣." + }, + { + "id": "emoji-bmp-and-astral", + "input_ids": [ + 1, + 29871, + 243, + 162, + 148, + 142, + 15043, + 29991, + 29871, + 31494, + 2787, + 29871, + 243, + 162, + 157, + 131, + 6826, + 29871, + 243, + 162, + 145, + 140, + 6263, + 29871, + 243, + 162, + 145, + 133, + 29889 + ], + "tokens": [ + "", + "▁", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0x8B>", + "▁Hello", + "!", + "▁", + "🌍", + "▁World", + "▁", + "<0xF0>", + "<0x9F>", + "<0x9A>", + "<0x80>", + "▁launch", + "▁", + "<0xF0>", + "<0x9F>", + "<0x8E>", + "<0x89>", + "▁party", + "▁", + "<0xF0>", + "<0x9F>", + "<0x8E>", + "<0x82>", + "." + ], + "decoded_with_special": " 👋 Hello! 🌍 World 🚀 launch 🎉 party 🎂.", + "decoded_skip_special": "👋 Hello! 🌍 World 🚀 launch 🎉 party 🎂." + }, + { + "id": "emoji-zwj-family-pride-skin", + "input_ids": [ + 1, + 14662, + 29901, + 29871, + 243, + 162, + 148, + 171, + 30722, + 243, + 162, + 148, + 172, + 30722, + 243, + 162, + 148, + 170, + 30722, + 243, + 162, + 148, + 169, + 813, + 349, + 2426, + 7353, + 29901, + 29871, + 243, + 162, + 146, + 182, + 30598, + 30722, + 243, + 162, + 143, + 139, + 813, + 4971, + 262, + 29899, + 29873, + 650, + 29901, + 29871, + 243, + 162, + 148, + 144, + 243, + 162, + 146, + 192, + 29871, + 229, + 159, + 141, + 243, + 162, + 146, + 194, + 29871, + 243, + 162, + 148, + 185, + 243, + 162, + 146, + 190, + 29889 + ], + "tokens": [ + "", + "▁Family", + ":", + "▁", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA8>", + "‍", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA9>", + "‍", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA7>", + "‍", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA6>", + "▁—", + "▁P", + "ride", + "▁flag", + ":", + "▁", + "<0xF0>", + "<0x9F>", + "<0x8F>", + "<0xB3>", + "️", + "‍", + "<0xF0>", + "<0x9F>", + "<0x8C>", + "<0x88>", + "▁—", + "▁Sk", + "in", + "-", + "t", + "one", + ":", + "▁", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0x8D>", + "<0xF0>", + "<0x9F>", + "<0x8F>", + "<0xBD>", + "▁", + "<0xE2>", + "<0x9C>", + "<0x8A>", + "<0xF0>", + "<0x9F>", + "<0x8F>", + "<0xBF>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xB6>", + "<0xF0>", + "<0x9F>", + "<0x8F>", + "<0xBB>", + "." + ], + "decoded_with_special": " Family: 👨‍👩‍👧‍👦 — Pride flag: 🏳️‍🌈 — Skin-tone: 👍🏽 ✊🏿 👶🏻.", + "decoded_skip_special": "Family: 👨‍👩‍👧‍👦 — Pride flag: 🏳️‍🌈 — Skin-tone: 👍🏽 ✊🏿 👶🏻." + }, + { + "id": "emoji-zwj-with-text-prefix", + "input_ids": [ + 1, + 29871, + 243, + 162, + 146, + 182, + 30598, + 30722, + 243, + 162, + 143, + 139, + 29925, + 2426, + 29974, + 243, + 162, + 148, + 171, + 30722, + 243, + 162, + 148, + 172, + 30722, + 243, + 162, + 148, + 170, + 30722, + 243, + 162, + 148, + 169, + 27104, + 29974, + 243, + 162, + 138, + 172, + 243, + 162, + 138, + 173, + 243, + 162, + 138, + 178, + 243, + 162, + 138, + 184, + 243, + 162, + 138, + 189, + 243, + 162, + 138, + 187, + 15764, + 29974, + 1688 + ], + "tokens": [ + "", + "▁", + "<0xF0>", + "<0x9F>", + "<0x8F>", + "<0xB3>", + "️", + "‍", + "<0xF0>", + "<0x9F>", + "<0x8C>", + "<0x88>", + "P", + "ride", + "+", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA8>", + "‍", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA9>", + "‍", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA7>", + "‍", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA6>", + "Family", + "+", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xA9>", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xAA>", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xAF>", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xB5>", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xBA>", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xB8>", + "flags", + "+", + "test" + ], + "decoded_with_special": " 🏳️‍🌈Pride+👨‍👩‍👧‍👦Family+🇩🇪🇯🇵🇺🇸flags+test", + "decoded_skip_special": "🏳️‍🌈Pride+👨‍👩‍👧‍👦Family+🇩🇪🇯🇵🇺🇸flags+test" + }, + { + "id": "emoji-keycap-and-flags", + "input_ids": [ + 1, + 7670, + 29883, + 2547, + 29901, + 29871, + 29896, + 30598, + 229, + 134, + 166, + 29871, + 29906, + 30598, + 229, + 134, + 166, + 29871, + 29941, + 30598, + 229, + 134, + 166, + 813, + 7353, + 29901, + 29871, + 243, + 162, + 138, + 178, + 243, + 162, + 138, + 184, + 29871, + 243, + 162, + 138, + 172, + 243, + 162, + 138, + 173, + 29871, + 243, + 162, + 138, + 189, + 243, + 162, + 138, + 187, + 29889 + ], + "tokens": [ + "", + "▁Key", + "c", + "aps", + ":", + "▁", + "1", + "️", + "<0xE2>", + "<0x83>", + "<0xA3>", + "▁", + "2", + "️", + "<0xE2>", + "<0x83>", + "<0xA3>", + "▁", + "3", + "️", + "<0xE2>", + "<0x83>", + "<0xA3>", + "▁—", + "▁flag", + ":", + "▁", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xAF>", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xB5>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xA9>", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xAA>", + "▁", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xBA>", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xB8>", + "." + ], + "decoded_with_special": " Keycaps: 1️⃣ 2️⃣ 3️⃣ — flag: 🇯🇵 🇩🇪 🇺🇸.", + "decoded_skip_special": "Keycaps: 1️⃣ 2️⃣ 3️⃣ — flag: 🇯🇵 🇩🇪 🇺🇸." + }, + { + "id": "url-with-query-fragment", + "input_ids": [ + 1, + 3142, + 29922, + 991, + 597, + 4773, + 29889, + 510, + 29914, + 2084, + 29914, + 517, + 29914, + 10314, + 29973, + 29939, + 29922, + 5431, + 29987, + 1646, + 29922, + 27975, + 29987, + 29876, + 29922, + 29946, + 29906, + 29937, + 2042, + 29899, + 29941 + ], + "tokens": [ + "", + "▁url", + "=", + "https", + "://", + "example", + ".", + "com", + "/", + "path", + "/", + "to", + "/", + "resource", + "?", + "q", + "=", + "foo", + "&", + "bar", + "=", + "baz", + "&", + "n", + "=", + "4", + "2", + "#", + "section", + "-", + "3" + ], + "decoded_with_special": " url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3", + "decoded_skip_special": "url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3" + }, + { + "id": "hex-and-base64", + "input_ids": [ + 1, + 15090, + 29922, + 29900, + 29916, + 2287, + 3035, + 15349, + 29923, + 8610, + 5098, + 25752, + 2882, + 29923, + 2967, + 29953, + 29946, + 29922, + 29979, + 29956, + 29967, + 29926, + 29999, + 22928, + 29885, + 29999, + 29906, + 28887, + 314, + 1372, + 29890, + 29956, + 29945, + 7071, + 29909, + 1360 + ], + "tokens": [ + "", + "▁hex", + "=", + "0", + "x", + "DE", + "AD", + "BE", + "E", + "FC", + "AF", + "EB", + "AB", + "E", + "▁base", + "6", + "4", + "=", + "Y", + "W", + "J", + "j", + "Z", + "GV", + "m", + "Z", + "2", + "hp", + "am", + "ts", + "b", + "W", + "5", + "vc", + "A", + "==" + ], + "decoded_with_special": " hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==", + "decoded_skip_special": "hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==" + }, + { + "id": "sha256-hash", + "input_ids": [ + 1, + 528, + 29874, + 29906, + 29945, + 29953, + 29922, + 29872, + 29941, + 29890, + 29900, + 29883, + 29946, + 29946, + 29906, + 29929, + 29947, + 13801, + 29896, + 29883, + 29896, + 29946, + 29929, + 2142, + 1635, + 29946, + 29883, + 29947, + 29929, + 29929, + 29953, + 14943, + 29929, + 29906, + 29946, + 29906, + 29955, + 3660, + 29946, + 29896, + 29872, + 29946, + 29953, + 29946, + 29929, + 29890, + 29929, + 29941, + 29946, + 1113, + 29946, + 29929, + 29945, + 29929, + 29929, + 29896, + 29890, + 29955, + 29947, + 29945, + 29906, + 29890, + 29947, + 29945, + 29945 + ], + "tokens": [ + "", + "▁sh", + "a", + "2", + "5", + "6", + "=", + "e", + "3", + "b", + "0", + "c", + "4", + "4", + "2", + "9", + "8", + "fc", + "1", + "c", + "1", + "4", + "9", + "af", + "bf", + "4", + "c", + "8", + "9", + "9", + "6", + "fb", + "9", + "2", + "4", + "2", + "7", + "ae", + "4", + "1", + "e", + "4", + "6", + "4", + "9", + "b", + "9", + "3", + "4", + "ca", + "4", + "9", + "5", + "9", + "9", + "1", + "b", + "7", + "8", + "5", + "2", + "b", + "8", + "5", + "5" + ], + "decoded_with_special": " sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "decoded_skip_special": "sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "id": "code-javascript-functional", + "input_ids": [ + 1, + 3948, + 29889, + 1958, + 29898, + 29916, + 1149, + 921, + 334, + 29871, + 29906, + 467, + 4572, + 29898, + 29916, + 1149, + 921, + 1405, + 29871, + 29900, + 467, + 17469, + 3552, + 29874, + 29892, + 289, + 29897, + 1149, + 263, + 718, + 289, + 416 + ], + "tokens": [ + "", + "▁arr", + ".", + "map", + "(", + "x", + "▁=>", + "▁x", + "▁*", + "▁", + "2", + ").", + "filter", + "(", + "x", + "▁=>", + "▁x", + "▁>", + "▁", + "0", + ").", + "reduce", + "((", + "a", + ",", + "▁b", + ")", + "▁=>", + "▁a", + "▁+", + "▁b", + ");" + ], + "decoded_with_special": " arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);", + "decoded_skip_special": "arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);" + }, + { + "id": "code-cplusplus-template", + "input_ids": [ + 1, + 4472, + 29966, + 29911, + 29892, + 29965, + 29958, + 4469, + 285, + 29898, + 29911, + 12774, + 29873, + 29892, + 501, + 12774, + 29884, + 29897, + 1599, + 4845, + 1853, + 29898, + 29873, + 29974, + 29884, + 29897, + 426, + 736, + 260, + 718, + 318, + 29936, + 500 + ], + "tokens": [ + "", + "▁template", + "<", + "T", + ",", + "U", + ">", + "▁auto", + "▁f", + "(", + "T", + "&&", + "t", + ",", + "▁U", + "&&", + "u", + ")", + "▁->", + "▁decl", + "type", + "(", + "t", + "+", + "u", + ")", + "▁{", + "▁return", + "▁t", + "▁+", + "▁u", + ";", + "▁}" + ], + "decoded_with_special": " template auto f(T&&t, U&&u) -> decltype(t+u) { return t + u; }", + "decoded_skip_special": "template auto f(T&&t, U&&u) -> decltype(t+u) { return t + u; }" + }, + { + "id": "code-bitops", + "input_ids": [ + 1, + 1121, + 353, + 313, + 29874, + 5099, + 29871, + 29906, + 29897, + 891, + 313, + 29890, + 3532, + 29871, + 29946, + 29897, + 669, + 29871, + 29900, + 29916, + 4198, + 29936 + ], + "tokens": [ + "", + "▁result", + "▁=", + "▁(", + "a", + "▁>>", + "▁", + "2", + ")", + "▁|", + "▁(", + "b", + "▁<<", + "▁", + "4", + ")", + "▁&", + "▁", + "0", + "x", + "FF", + ";" + ], + "decoded_with_special": " result = (a >> 2) | (b << 4) & 0xFF;", + "decoded_skip_special": "result = (a >> 2) | (b << 4) & 0xFF;" + }, + { + "id": "code-javascript-modern", + "input_ids": [ + 1, + 565, + 313, + 29916, + 15639, + 1870, + 2607, + 343, + 6736, + 29871, + 29900, + 29897, + 426, + 736, + 921, + 9808, + 5431, + 13626, + 2322, + 1917, + 29936, + 500 + ], + "tokens": [ + "", + "▁if", + "▁(", + "x", + "▁!==", + "▁null", + "▁&&", + "▁y", + "▁>=", + "▁", + "0", + ")", + "▁{", + "▁return", + "▁x", + "?.", + "foo", + "▁??", + "▁default", + "Value", + ";", + "▁}" + ], + "decoded_with_special": " if (x !== null && y >= 0) { return x?.foo ?? defaultValue; }", + "decoded_skip_special": "if (x !== null && y >= 0) { return x?.foo ?? defaultValue; }" + }, + { + "id": "code-python-def", + "input_ids": [ + 1, + 822, + 18755, + 265, + 21566, + 29898, + 29876, + 1125 + ], + "tokens": [ + "", + "▁def", + "▁fib", + "on", + "acci", + "(", + "n", + "):" + ], + "decoded_with_special": " def fibonacci(n):", + "decoded_skip_special": "def fibonacci(n):" + }, + { + "id": "code-python-if", + "input_ids": [ + 1, + 268, + 565, + 302, + 529, + 29871, + 29906, + 29901 + ], + "tokens": [ + "", + "▁▁▁▁", + "▁if", + "▁n", + "▁<", + "▁", + "2", + ":" + ], + "decoded_with_special": " if n < 2:", + "decoded_skip_special": " if n < 2:" + }, + { + "id": "code-python-return", + "input_ids": [ + 1, + 308, + 736, + 302 + ], + "tokens": [ + "", + "▁▁▁▁▁▁▁▁", + "▁return", + "▁n" + ], + "decoded_with_special": " return n", + "decoded_skip_special": " return n" + }, + { + "id": "code-python-recurse", + "input_ids": [ + 1, + 268, + 736, + 18755, + 265, + 21566, + 29898, + 29876, + 448, + 29871, + 29896, + 29897, + 718, + 18755, + 265, + 21566, + 29898, + 29876, + 448, + 29871, + 29906, + 29897 + ], + "tokens": [ + "", + "▁▁▁▁", + "▁return", + "▁fib", + "on", + "acci", + "(", + "n", + "▁-", + "▁", + "1", + ")", + "▁+", + "▁fib", + "on", + "acci", + "(", + "n", + "▁-", + "▁", + "2", + ")" + ], + "decoded_with_special": " return fibonacci(n - 1) + fibonacci(n - 2)", + "decoded_skip_special": " return fibonacci(n - 1) + fibonacci(n - 2)" + }, + { + "id": "code-go-func", + "input_ids": [ + 1, + 3653, + 1395, + 300, + 29898, + 978, + 1347, + 29897, + 426 + ], + "tokens": [ + "", + "▁func", + "▁gre", + "et", + "(", + "name", + "▁string", + ")", + "▁{" + ], + "decoded_with_special": " func greet(name string) {", + "decoded_skip_special": "func greet(name string) {" + }, + { + "id": "code-go-tab-if", + "input_ids": [ + 1, + 29871, + 12, + 361, + 1024, + 2804, + 5124, + 426 + ], + "tokens": [ + "", + "▁", + "<0x09>", + "if", + "▁name", + "▁!=", + "▁\"\"", + "▁{" + ], + "decoded_with_special": " \tif name != \"\" {", + "decoded_skip_special": "\tif name != \"\" {" + }, + { + "id": "code-go-tab-printf", + "input_ids": [ + 1, + 29871, + 12, + 12, + 23479, + 29889, + 11816, + 29888, + 703, + 10994, + 29892, + 1273, + 29879, + 9903, + 29876, + 613, + 1024, + 29897 + ], + "tokens": [ + "", + "▁", + "<0x09>", + "<0x09>", + "fmt", + ".", + "Print", + "f", + "(\"", + "Hello", + ",", + "▁%", + "s", + "!\\", + "n", + "\",", + "▁name", + ")" + ], + "decoded_with_special": " \t\tfmt.Printf(\"Hello, %s!\\n\", name)", + "decoded_skip_special": "\t\tfmt.Printf(\"Hello, %s!\\n\", name)" + }, + { + "id": "code-go-tab-close", + "input_ids": [ + 1, + 29871, + 12, + 29913 + ], + "tokens": [ + "", + "▁", + "<0x09>", + "}" + ], + "decoded_with_special": " \t}", + "decoded_skip_special": "\t}" + }, + { + "id": "code-go-close", + "input_ids": [ + 1, + 500 + ], + "tokens": [ + "", + "▁}" + ], + "decoded_with_special": " }", + "decoded_skip_special": "}" + }, + { + "id": "programming-identifiers-mixed", + "input_ids": [ + 1, + 679, + 29918, + 1792, + 29918, + 333, + 18134, + 29918, + 7838, + 28483, + 29918, + 14226, + 323, + 29918, + 8305, + 29918, + 6979, + 29918, + 333, + 3865, + 4219, + 7317, + 8614 + ], + "tokens": [ + "", + "▁get", + "_", + "user", + "_", + "id", + "▁MAX", + "_", + "BU", + "FFER", + "_", + "SIZE", + "▁T", + "_", + "pad", + "_", + "token", + "_", + "id", + "▁NS", + "URL", + "Session", + "Configuration" + ], + "decoded_with_special": " get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration", + "decoded_skip_special": "get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration" + }, + { + "id": "programming-identifiers-cases", + "input_ids": [ + 1, + 3949, + 295, + 8259, + 9037, + 29671, + 8259, + 269, + 21040, + 29918, + 4878, + 29918, + 1707, + 12314, + 1525, + 5194, + 4214, + 29918, + 29903, + 3521, + 6059, + 413, + 29947, + 29879, + 29918, + 15334, + 29918, + 2798, + 29918, + 29894, + 29906 + ], + "tokens": [ + "", + "▁cam", + "el", + "Case", + "Var", + "▁Pascal", + "Case", + "▁s", + "nake", + "_", + "case", + "_", + "var", + "▁SC", + "RE", + "AM", + "ING", + "_", + "S", + "NA", + "KE", + "▁k", + "8", + "s", + "_", + "pod", + "_", + "count", + "_", + "v", + "2" + ], + "decoded_with_special": " camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2", + "decoded_skip_special": "camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2" + }, + { + "id": "code-string-multiscript", + "input_ids": [ + 1, + 1395, + 15133, + 353, + 376, + 30013, + 641, + 7616, + 29892, + 4157, + 29927, + 3850, + 718, + 376, + 813, + 515, + 376, + 718, + 376, + 30591, + 30675, + 29908 + ], + "tokens": [ + "", + "▁gre", + "eting", + "▁=", + "▁\"", + "П", + "ри", + "вет", + ",", + "▁ми", + "р", + "!\"", + "▁+", + "▁\"", + "▁—", + "▁from", + "▁\"", + "▁+", + "▁\"", + "東", + "京", + "\"" + ], + "decoded_with_special": " greeting = \"Привет, мир!\" + \" — from \" + \"東京\"", + "decoded_skip_special": "greeting = \"Привет, мир!\" + \" — from \" + \"東京\"" + }, + { + "id": "code-comment-diacritics", + "input_ids": [ + 1, + 849, + 6896, + 398, + 29948, + 813, + 14156, + 3440, + 411, + 9766, + 9695, + 1199 + ], + "tokens": [ + "", + "▁//", + "▁rés", + "um", + "é", + "▁—", + "▁Swift", + "▁comment", + "▁with", + "▁dia", + "crit", + "ics" + ], + "decoded_with_special": " // résumé — Swift comment with diacritics", + "decoded_skip_special": "// résumé — Swift comment with diacritics" + }, + { + "id": "code-comment-greek-math", + "input_ids": [ + 1, + 4949, + 19883, + 718, + 29871, + 30187, + 353, + 29871, + 30197, + 813, + 12311, + 29899, + 15670, + 5844, + 3440, + 3776 + ], + "tokens": [ + "", + "▁/*", + "▁α", + "▁+", + "▁", + "β", + "▁=", + "▁", + "γ", + "▁—", + "▁Greek", + "-", + "letter", + "▁math", + "▁comment", + "▁*/" + ], + "decoded_with_special": " /* α + β = γ — Greek-letter math comment */", + "decoded_skip_special": "/* α + β = γ — Greek-letter math comment */" + }, + { + "id": "code-string-multiscript-emoji", + "input_ids": [ + 1, + 1235, + 3611, + 353, + 376, + 243, + 162, + 157, + 131, + 997, + 3322, + 813, + 8430, + 6231, + 813, + 29871, + 31558, + 31124, + 29908 + ], + "tokens": [ + "", + "▁let", + "▁title", + "▁=", + "▁\"", + "<0xF0>", + "<0x9F>", + "<0x9A>", + "<0x80>", + "▁La", + "unch", + "▁—", + "▁пу", + "ск", + "▁—", + "▁", + "起", + "動", + "\"" + ], + "decoded_with_special": " let title = \"🚀 Launch — пуск — 起動\"", + "decoded_skip_special": "let title = \"🚀 Launch — пуск — 起動\"" + }, + { + "id": "ipa-phonetic-transcription", + "input_ids": [ + 1, + 450, + 5641, + 29909, + 1301, + 3395, + 847, + 29888, + 30312, + 30376, + 29914, + 363, + 376, + 15161, + 29908, + 813, + 411, + 22884, + 29901, + 29871, + 30176, + 29888, + 30312, + 30376, + 29889 + ], + "tokens": [ + "", + "▁The", + "▁IP", + "A", + "▁trans", + "cription", + "▁/", + "f", + "ɪ", + "ʃ", + "/", + "▁for", + "▁\"", + "fish", + "\"", + "▁—", + "▁with", + "▁stress", + ":", + "▁", + "ˈ", + "f", + "ɪ", + "ʃ", + "." + ], + "decoded_with_special": " The IPA transcription /fɪʃ/ for \"fish\" — with stress: ˈfɪʃ.", + "decoded_skip_special": "The IPA transcription /fɪʃ/ for \"fish\" — with stress: ˈfɪʃ." + }, + { + "id": "mandarin-tones", + "input_ids": [ + 1, + 323, + 2873, + 29901, + 10269, + 286, + 30359, + 286, + 30001, + 286, + 30107, + 313, + 29924, + 392, + 27139, + 260, + 2873, + 813, + 1880, + 29892, + 20493, + 29892, + 652, + 3262, + 29892, + 20327, + 467 + ], + "tokens": [ + "", + "▁T", + "ones", + ":", + "▁má", + "▁m", + "ǎ", + "▁m", + "à", + "▁m", + "ā", + "▁(", + "M", + "and", + "arin", + "▁t", + "ones", + "▁—", + "▁high", + ",", + "▁rising", + ",", + "▁di", + "pping", + ",", + "▁falling", + ")." + ], + "decoded_with_special": " Tones: má mǎ mà mā (Mandarin tones — high, rising, dipping, falling).", + "decoded_skip_special": "Tones: má mǎ mà mā (Mandarin tones — high, rising, dipping, falling)." + }, + { + "id": "whitespace-runs", + "input_ids": [ + 1, + 22172, + 1678, + 3186, + 308, + 869, + 539, + 1095 + ], + "tokens": [ + "", + "▁hello", + "▁▁▁", + "▁world", + "▁▁▁▁▁▁▁▁", + "▁.", + "▁▁▁▁▁▁", + "▁end" + ], + "decoded_with_special": " hello world . end", + "decoded_skip_special": "hello world . end" + }, + { + "id": "whitespace-trailing-tabs", + "input_ids": [ + 1, + 25053, + 24358, + 1434, + 25899, + 29901, + 12, + 29898, + 3891, + 29897, + 259, + 313, + 22854, + 29897 + ], + "tokens": [ + "", + "▁trailing", + "▁whitespace", + "▁before", + "▁newline", + ":", + "<0x09>", + "(", + "tab", + ")", + "▁▁", + "▁(", + "spaces", + ")" + ], + "decoded_with_special": " trailing whitespace before newline:\t(tab) (spaces)", + "decoded_skip_special": "trailing whitespace before newline:\t(tab) (spaces)" + }, + { + "id": "punctuation-period", + "input_ids": [ + 1, + 869 + ], + "tokens": [ + "", + "▁." + ], + "decoded_with_special": " .", + "decoded_skip_special": "." + }, + { + "id": "punctuation-comma", + "input_ids": [ + 1, + 1919 + ], + "tokens": [ + "", + "▁," + ], + "decoded_with_special": " ,", + "decoded_skip_special": "," + }, + { + "id": "punctuation-exclamation", + "input_ids": [ + 1, + 1738 + ], + "tokens": [ + "", + "▁!" + ], + "decoded_with_special": " !", + "decoded_skip_special": "!" + }, + { + "id": "punctuation-nested-quotes", + "input_ids": [ + 1, + 376, + 5618, + 3026, + 1183, + 4433, + 813, + 376, + 276, + 635, + 29973, + 3850, + 813, + 4874, + 856 + ], + "tokens": [ + "", + "▁\"", + "What", + "?\"", + "▁she", + "▁asked", + "▁—", + "▁\"", + "re", + "ally", + "?", + "!\"", + "▁—", + "▁yes", + "..." + ], + "decoded_with_special": " \"What?\" she asked — \"really?!\" — yes...", + "decoded_skip_special": "\"What?\" she asked — \"really?!\" — yes..." + }, + { + "id": "punctuation-dash-variants", + "input_ids": [ + 1, + 2023, + 322, + 29871, + 30921, + 884, + 785, + 322, + 813, + 322, + 29871, + 30217, + 29889 + ], + "tokens": [ + "", + "▁...", + "▁and", + "▁", + "‒", + "▁also", + "▁–", + "▁and", + "▁—", + "▁and", + "▁", + "―", + "." + ], + "decoded_with_special": " ... and ‒ also – and — and ―.", + "decoded_skip_special": "... and ‒ also – and — and ―." + }, + { + "id": "multiscript-greetings", + "input_ids": [ + 1, + 15043, + 29871, + 30919, + 31076, + 29871, + 31734, + 238, + 136, + 152, + 29871, + 30589, + 30389, + 30353, + 30644, + 30449, + 29871, + 30424, + 30485, + 30489, + 30296, + 30475, + 30569, + 29871, + 30159, + 30156, + 30240, + 30177, + 30112, + 29871, + 30294, + 30249, + 30205, + 30404, + 29871, + 30547, + 30492, + 30510, + 30547, + 30718, + 30691, + 7203, + 7616, + 29871, + 30314, + 30110, + 30142, + 30183, + 30167, + 30273, + 30147, + 30110, + 29889 + ], + "tokens": [ + "", + "▁Hello", + "▁", + "你", + "好", + "▁", + "안", + "<0xEB>", + "<0x85>", + "<0x95>", + "▁", + "こ", + "ん", + "に", + "ち", + "は", + "▁", + "न", + "म", + "स", + "्", + "त", + "े", + "▁", + "م", + "ر", + "ح", + "ب", + "ا", + "▁", + "ש", + "ל", + "ו", + "ם", + "▁", + "ส", + "ว", + "ั", + "ส", + "ด", + "ี", + "▁При", + "вет", + "▁", + "Κ", + "α", + "λ", + "η", + "μ", + "έ", + "ρ", + "α", + "." + ], + "decoded_with_special": " Hello 你好 안녕 こんにちは नमस्ते مرحبا שלום สวัสดี Привет Καλημέρα.", + "decoded_skip_special": "Hello 你好 안녕 こんにちは नमस्ते مرحبا שלום สวัสดี Привет Καλημέρα." + }, + { + "id": "multiscript-no-spaces", + "input_ids": [ + 1, + 19883, + 30919, + 30187, + 31734, + 30197, + 30424, + 30201, + 30489, + 243, + 162, + 149, + 174, + 30218, + 243, + 162, + 148, + 142, + 30294, + 243, + 162, + 157, + 131, + 30195, + 30315, + 30619, + 30264, + 30381 + ], + "tokens": [ + "", + "▁α", + "你", + "β", + "안", + "γ", + "न", + "δ", + "स", + "<0xF0>", + "<0x9F>", + "<0x92>", + "<0xAB>", + "ع", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0x8B>", + "ש", + "<0xF0>", + "<0x9F>", + "<0x9A>", + "<0x80>", + "ت", + "∀", + "∃", + "∈", + "⇒" + ], + "decoded_with_special": " α你β안γनδस💫ع👋ש🚀ت∀∃∈⇒", + "decoded_skip_special": "α你β안γनδस💫ع👋ש🚀ت∀∃∈⇒" + }, + { + "id": "multiscript-zwj-mixed", + "input_ids": [ + 1, + 29871, + 30275, + 30333, + 30877, + 237, + 187, + 131, + 30325, + 30346, + 30968, + 233, + 186, + 186, + 243, + 162, + 148, + 142, + 31980, + 31063, + 30709, + 243, + 162, + 145, + 140, + 726, + 233, + 186, + 186, + 30733 + ], + "tokens": [ + "", + "▁", + "中", + "文", + "한", + "<0xEA>", + "<0xB8>", + "<0x80>", + "日", + "本", + "語", + "<0xE6>", + "<0xB7>", + "<0xB7>", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0x8B>", + "합", + "니", + "다", + "<0xF0>", + "<0x9F>", + "<0x8E>", + "<0x89>", + "text", + "<0xE6>", + "<0xB7>", + "<0xB7>", + "合" + ], + "decoded_with_special": " 中文한글日本語混👋합니다🎉text混合", + "decoded_skip_special": "中文한글日本語混👋합니다🎉text混合" + }, + { + "id": "multiscript-string-concat", + "input_ids": [ + 1, + 1395, + 15133, + 543, + 30919, + 31076, + 17969, + 31494, + 13578, + 31734, + 238, + 136, + 152, + 17969, + 30424, + 30485, + 30489, + 30296, + 30475, + 30569, + 29974, + 243, + 162, + 148, + 171, + 30722, + 243, + 162, + 148, + 172, + 30722, + 243, + 162, + 148, + 170 + ], + "tokens": [ + "", + "▁gre", + "eting", + "=\"", + "你", + "好", + "\"+", + "🌍", + "+\"", + "안", + "<0xEB>", + "<0x85>", + "<0x95>", + "\"+", + "न", + "म", + "स", + "्", + "त", + "े", + "+", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA8>", + "‍", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA9>", + "‍", + "<0xF0>", + "<0x9F>", + "<0x91>", + "<0xA7>" + ], + "decoded_with_special": " greeting=\"你好\"+🌍+\"안녕\"+नमस्ते+👨‍👩‍👧", + "decoded_skip_special": "greeting=\"你好\"+🌍+\"안녕\"+नमस्ते+👨‍👩‍👧" + }, + { + "id": "multiscript-emoji-arrows", + "input_ids": [ + 1, + 29871, + 30291, + 921, + 29918, + 29875, + 29871, + 243, + 162, + 138, + 178, + 243, + 162, + 138, + 184, + 29871, + 30487, + 29871, + 30662, + 30675, + 10309, + 10864, + 2880, + 323, + 30099, + 3459, + 30099, + 2880, + 29871, + 30429, + 30581, + 29871, + 229, + 159, + 171, + 1095, + 29889 + ], + "tokens": [ + "", + "▁", + "Σ", + "▁x", + "_", + "i", + "▁", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xAF>", + "<0xF0>", + "<0x9F>", + "<0x87>", + "<0xB5>", + "▁", + "≠", + "▁", + "北", + "京", + "▁→", + "▁München", + "▁·", + "▁T", + "ō", + "ky", + "ō", + "▁·", + "▁", + "上", + "海", + "▁", + "<0xE2>", + "<0x9C>", + "<0xA8>", + "▁end", + "." + ], + "decoded_with_special": " Σ x_i 🇯🇵 ≠ 北京 → München · Tōkyō · 上海 ✨ end.", + "decoded_skip_special": "Σ x_i 🇯🇵 ≠ 北京 → München · Tōkyō · 上海 ✨ end." + }, + { + "id": "multiscript-arrow-chain", + "input_ids": [ + 1, + 29871, + 243, + 162, + 157, + 131, + 30424, + 30485, + 30489, + 30296, + 30475, + 30569, + 30121, + 30013, + 641, + 7616, + 30121, + 30919, + 31076, + 30121, + 31734, + 238, + 136, + 152, + 30121, + 30159, + 30156, + 30240, + 30177, + 30112, + 30121, + 229, + 160, + 167, + 30598 + ], + "tokens": [ + "", + "▁", + "<0xF0>", + "<0x9F>", + "<0x9A>", + "<0x80>", + "न", + "म", + "स", + "्", + "त", + "े", + "→", + "П", + "ри", + "вет", + "→", + "你", + "好", + "→", + "안", + "<0xEB>", + "<0x85>", + "<0x95>", + "→", + "م", + "ر", + "ح", + "ب", + "ا", + "→", + "<0xE2>", + "<0x9D>", + "<0xA4>", + "️" + ], + "decoded_with_special": " 🚀नमस्ते→Привет→你好→안녕→مرحبا→❤️", + "decoded_skip_special": "🚀नमस्ते→Привет→你好→안녕→مرحبا→❤️" + }, + { + "id": "multiscript-function-call", + "input_ids": [ + 1, + 7953, + 29898, + 30275, + 30333, + 29892, + 30877, + 237, + 187, + 131, + 29892, + 30325, + 30346, + 30968, + 29892, + 30714, + 30436, + 30424, + 30296, + 30694, + 30580, + 29892, + 16007, + 2542, + 29897, + 30121, + 29912, + 30315, + 29901, + 30619, + 29892, + 31200, + 29901, + 31486, + 29913 + ], + "tokens": [ + "", + "▁foo", + "(", + "中", + "文", + ",", + "한", + "<0xEA>", + "<0xB8>", + "<0x80>", + ",", + "日", + "本", + "語", + ",", + "ह", + "ि", + "न", + "्", + "द", + "ी", + ",", + "рус", + "ский", + ")", + "→", + "{", + "∀", + ":", + "∃", + ",", + "⊕", + ":", + "⊗", + "}" + ], + "decoded_with_special": " foo(中文,한글,日本語,हिन्दी,русский)→{∀:∃,⊕:⊗}", + "decoded_skip_special": "foo(中文,한글,日本語,हिन्दी,русский)→{∀:∃,⊕:⊗}" + }, + { + "id": "multiscript-legacy-symbols", + "input_ids": [ + 1, + 8981, + 29946, + 29906, + 29871, + 30536, + 30598, + 29871, + 30211, + 29871, + 30342, + 29871, + 30497, + 16683, + 25540, + 30563, + 30155, + 229, + 133, + 194, + 29871, + 30291, + 30315, + 29871, + 30919, + 31076, + 29871, + 31734, + 238, + 136, + 152, + 24508 + ], + "tokens": [ + "", + "▁№", + "4", + "2", + "▁", + "™", + "️", + "▁", + "©", + "▁", + "®", + "▁", + "¶", + "▁§", + "▁€", + "¥", + "£", + "<0xE2>", + "<0x82>", + "<0xBF>", + "▁", + "Σ", + "∀", + "▁", + "你", + "好", + "▁", + "안", + "<0xEB>", + "<0x85>", + "<0x95>", + "▁ال" + ], + "decoded_with_special": " №42 ™️ © ® ¶ § €¥£₿ Σ∀ 你好 안녕 ال", + "decoded_skip_special": "№42 ™️ © ® ¶ § €¥£₿ Σ∀ 你好 안녕 ال" + }, + { + "id": "escape-sequences-with-zwj", + "input_ids": [ + 1, + 4265, + 29905, + 29900, + 392, + 29905, + 698, + 370, + 29905, + 29876, + 1482, + 1220, + 233, + 186, + 186, + 30752, + 30722, + 29999, + 29956, + 29967, + 30598, + 2622, + 943 + ], + "tokens": [ + "", + "▁NULL", + "\\", + "0", + "and", + "\\", + "tt", + "ab", + "\\", + "n", + "new", + "line", + "<0xE6>", + "<0xB7>", + "<0xB7>", + "入", + "‍", + "Z", + "W", + "J", + "️", + "select", + "ors" + ], + "decoded_with_special": " NULL\\0and\\ttab\\nnewline混入‍ZWJ️selectors", + "decoded_skip_special": "NULL\\0and\\ttab\\nnewline混入‍ZWJ️selectors" + }, + { + "id": "german-compound-short", + "input_ids": [ + 1, + 19259, + 1276, + 1470, + 29889, + 4493, + 28395, + 400, + 432, + 7401, + 972, + 7107, + 1066, + 277, + 801, + 28527, + 1770, + 589, + 16878, + 2922, + 638, + 9888, + 13289, + 816, + 431, + 23373, + 29889, + 29871, + 243, + 162, + 155, + 135 + ], + "tokens": [ + "", + "▁Nat", + "ür", + "lich", + ".", + "▁Deutsch", + "▁zie", + "ht", + "▁j", + "etzt", + "▁den", + "▁Kom", + "pos", + "it", + "ah", + "ammer", + "▁aus", + "▁der", + "▁Gram", + "mat", + "ik", + "werk", + "zeug", + "sch", + "ub", + "lade", + ".", + "▁", + "<0xF0>", + "<0x9F>", + "<0x98>", + "<0x84>" + ], + "decoded_with_special": " Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. 😄", + "decoded_skip_special": "Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. 😄" + }, + { + "id": "german-compound-long-1", + "input_ids": [ + 1, + 1954, + 341, + 990, + 4584, + 6596, + 3192, + 804, + 264, + 369, + 16522, + 295, + 686, + 2774, + 2741, + 2317, + 1011, + 476, + 3470, + 29872, + 300, + 11257, + 9502, + 561, + 309, + 7708, + 347, + 915, + 711, + 496, + 357, + 20104, + 3422, + 2169, + 3586, + 3568, + 369, + 29887, + 404, + 1470, + 446, + 1169, + 19569, + 563, + 451, + 5423, + 297, + 2757, + 5791, + 593, + 351, + 29885, + 18996, + 680, + 264, + 11363, + 1333, + 466, + 8458, + 762, + 604, + 5173, + 348, + 4545, + 379, + 23240, + 6239, + 1005, + 20685, + 2475, + 9157, + 5173, + 431, + 1100, + 4566, + 10881, + 999, + 2506, + 17725, + 29889 + ], + "tokens": [ + "", + "▁Im", + "▁M", + "org", + "enne", + "bel", + "ged", + "ank", + "en", + "ver", + "äst", + "el", + "ung", + "sw", + "ald", + "▁stand", + "▁ein", + "▁K", + "aff", + "e", + "et", + "assen", + "rand", + "ph", + "il", + "osoph", + "ie", + "be", + "ob", + "ach", + "ter", + "▁neben", + "▁einer", + "▁Reg", + "ensch", + "irm", + "ver", + "g", + "ess", + "lich", + "ke", + "its", + "station", + "▁und", + "▁not", + "ierte", + "▁in", + "▁sein", + "▁Son", + "nt", + "ag", + "m", + "orgen", + "ide", + "en", + "fang", + "not", + "iz", + "buch", + "▁die", + "▁er", + "sta", + "un", + "liche", + "▁H", + "äufig", + "keit", + "▁von", + "▁Fen", + "ster", + "bank", + "sta", + "ub", + "son", + "nen", + "licht", + "ref", + "lex", + "ionen", + "." + ], + "decoded_with_special": " Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen.", + "decoded_skip_special": "Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen." + }, + { + "id": "german-compound-long-2", + "input_ids": [ + 1, + 19332, + 9771, + 22792, + 371, + 2128, + 19276, + 4415, + 816, + 10003, + 10190, + 915, + 280, + 10236, + 686, + 643, + 3993, + 3085, + 8247, + 305, + 457, + 29892, + 12829, + 1011, + 1085, + 12873, + 328, + 996, + 3333, + 16565, + 295, + 8850, + 9442, + 578, + 415, + 326, + 21212, + 915, + 585, + 25221, + 357, + 1380, + 3422, + 6479, + 7785, + 6208, + 557, + 504, + 638, + 12465, + 3417, + 305, + 3085, + 24209, + 344, + 8702, + 14133, + 5523, + 1370, + 29889, + 4028, + 589, + 7756, + 484, + 22248, + 2276, + 371, + 2160, + 1011, + 11902, + 1717, + 29895, + 3605, + 3298, + 18693, + 303, + 29884, + 4415, + 8366, + 638, + 7218, + 1112, + 391, + 1622, + 4093, + 2694, + 3665, + 27271, + 29879, + 1727, + 295, + 9894, + 5860, + 3085, + 914, + 3003, + 563, + 364, + 2575, + 29901, + 1768, + 29909, + 2570, + 686, + 29892, + 11529, + 3764, + 484, + 11478, + 2128, + 16836, + 804, + 23578, + 4858, + 24484, + 1333, + 3085, + 479, + 18693, + 575, + 1981, + 362, + 29991, + 30015 + ], + "tokens": [ + "", + "▁Neben", + "▁ihm", + "▁summ", + "te", + "▁eine", + "▁Kü", + "hl", + "sch", + "rank", + "innen", + "be", + "le", + "ucht", + "ung", + "ser", + "inner", + "ungs", + "mas", + "ch", + "ine", + ",", + "▁während", + "▁ein", + "▁Mar", + "mel", + "ad", + "eng", + "las", + "deck", + "el", + "öff", + "nung", + "so", + "pt", + "im", + "ierungs", + "be", + "au", + "ftrag", + "ter", + "▁mit", + "▁einer", + "▁Tre", + "ppen", + "haus", + "ak", + "ust", + "ik", + "über", + "ras", + "ch", + "ungs", + "analy", + "se", + "▁besch", + "äft", + "igt", + "▁war", + ".", + "▁Aus", + "▁der", + "▁Fer", + "ne", + "▁nä", + "her", + "te", + "▁sich", + "▁ein", + "▁Wol", + "ken", + "k", + "rat", + "zer", + "fahr", + "st", + "u", + "hl", + "mus", + "ik", + "kom", + "pon", + "ist", + "▁auf", + "▁einem", + "▁Ein", + "rad", + "verkehr", + "s", + "reg", + "el", + "miss", + "acht", + "ungs", + "ger", + "ät", + "▁und", + "▁r", + "ief", + ":", + "▁„", + "A", + "cht", + "ung", + ",", + "▁dort", + "▁vor", + "ne", + "▁liegt", + "▁eine", + "▁Ged", + "ank", + "enf", + "aden", + "verk", + "not", + "ungs", + "ge", + "fahr", + "ens", + "itu", + "ation", + "!", + "“" + ], + "decoded_with_special": " Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“", + "decoded_skip_special": "Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“" + }, + { + "id": "german-compound-long-3", + "input_ids": [ + 1, + 1858, + 9618, + 29920, + 1470, + 22113, + 18977, + 2160, + 1011, + 5347, + 3085, + 344, + 3537, + 398, + 2204, + 29118, + 14800, + 7864, + 30034, + 25089, + 29892, + 563, + 20964, + 3708, + 10533, + 841, + 9697, + 16144, + 307, + 6321, + 4850, + 824, + 6605, + 631, + 20581, + 28110, + 1655, + 309, + 23450, + 10190, + 29892, + 1011, + 402, + 442, + 4666, + 556, + 29887, + 29885, + 6621, + 2256, + 15641, + 1785, + 495, + 1008, + 563, + 1011, + 27568, + 5105, + 335, + 9540, + 381, + 29878, + 357, + 2166, + 7785, + 29880, + 8850, + 295, + 999, + 2506, + 1080, + 2527, + 12451, + 952, + 5603, + 29889, + 23832, + 314, + 27921, + 6302, + 2686, + 972, + 8457, + 25400, + 1865, + 3975, + 18344, + 810, + 10881, + 816, + 8606, + 29895, + 10204, + 6878, + 14493, + 686, + 563, + 4857, + 1730, + 5423, + 21541, + 557, + 13119, + 8399, + 1885, + 25996, + 333, + 335, + 3085, + 710, + 1845, + 819, + 29889 + ], + "tokens": [ + "", + "▁Pl", + "öt", + "z", + "lich", + "▁öff", + "nete", + "▁sich", + "▁ein", + "▁Zeit", + "ungs", + "se", + "iten", + "um", + "bl", + "ätter", + "wind", + "sto", + "ß", + "portal", + ",", + "▁und", + "▁heraus", + "▁pur", + "zel", + "ten", + "▁drei", + "▁Bü", + "ro", + "kl", + "amm", + "ern", + "sort", + "ier", + "welt", + "meisterschaft", + "ste", + "il", + "nehmer", + "innen", + ",", + "▁ein", + "▁G", + "art", + "enz", + "wer", + "g", + "m", + "üt", + "zen", + "far", + "ben", + "ber", + "ater", + "▁und", + "▁ein", + "▁hoch", + "grad", + "ig", + "▁verw", + "ir", + "r", + "ter", + "▁Su", + "ppen", + "l", + "öff", + "el", + "ref", + "lex", + "ions", + "met", + "aph", + "ys", + "iker", + ".", + "▁Gemeins", + "am", + "▁gründ", + "eten", + "▁sie", + "▁den", + "▁Bundes", + "verband", + "▁für", + "▁Nach", + "mitt", + "ags", + "licht", + "sch", + "atten", + "k", + "anten", + "bet", + "racht", + "ung", + "▁und", + "▁impro", + "vis", + "ierte", + "▁Sof", + "ak", + "issen", + "bur", + "gen", + "verte", + "id", + "ig", + "ungs", + "str", + "ateg", + "ien", + "." + ], + "decoded_with_special": " Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien.", + "decoded_skip_special": "Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien." + }, + { + "id": "german-compound-long-4", + "input_ids": [ + 1, + 1913, + 1976, + 355, + 1224, + 4850, + 295, + 841, + 2160, + 4788, + 527, + 12125, + 5955, + 7192, + 264, + 2475, + 19856, + 1527, + 5173, + 431, + 3820, + 13510, + 29920, + 19400, + 29892, + 8879, + 589, + 476, + 3470, + 29872, + 300, + 11257, + 9502, + 561, + 309, + 7708, + 347, + 915, + 711, + 496, + 357, + 2128, + 1238, + 631, + 4545, + 1102, + 13596, + 276, + 311, + 2939, + 762, + 399, + 1428, + 9918, + 1005, + 853, + 5223, + 1983, + 5498, + 1193, + 1080, + 10745, + 1151, + 29892, + 1706, + 10221, + 8366, + 10265, + 311, + 3123, + 20518, + 2469, + 563, + 360, + 2681, + 566, + 16817, + 816, + 361, + 600, + 5610, + 1372, + 26175, + 24892, + 17975, + 1983, + 29885, + 6621, + 2256, + 3959, + 15818, + 414, + 4101, + 7939, + 29890, + 1968, + 3470, + 3085, + 17199, + 264, + 298, + 9304, + 29889 + ], + "tokens": [ + "", + "▁Am", + "▁Ab", + "end", + "▁vers", + "amm", + "el", + "ten", + "▁sich", + "▁alle", + "▁im", + "▁Mond", + "sche", + "inf", + "en", + "ster", + "rah", + "men", + "sta", + "ub", + "gl", + "itzer", + "z", + "immer", + ",", + "▁wo", + "▁der", + "▁K", + "aff", + "e", + "et", + "assen", + "rand", + "ph", + "il", + "osoph", + "ie", + "be", + "ob", + "ach", + "ter", + "▁eine", + "▁fe", + "ier", + "liche", + "▁Sch", + "luss", + "re", + "de", + "▁über", + "▁die", + "▁W", + "icht", + "igkeit", + "▁von", + "▁Un", + "sin", + "ns", + "produ", + "kt", + "ions", + "fre", + "ude", + ",", + "▁Sp", + "rach", + "mus", + "kel", + "de", + "hn", + "üb", + "ungen", + "▁und", + "▁D", + "ona", + "ud", + "ampf", + "sch", + "if", + "ff", + "ahr", + "ts", + "gesellschaft", + "skap", + "itä", + "ns", + "m", + "üt", + "zen", + "kn", + "opf", + "ers", + "atz", + "teil", + "b", + "esch", + "aff", + "ungs", + "problem", + "en", + "▁h", + "ielt", + "." + ], + "decoded_with_special": " Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt.", + "decoded_skip_special": "Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt." + }, + { + "id": "german-compound-long-5", + "input_ids": [ + 1, + 3951, + 496, + 902, + 29878, + 816, + 371, + 599, + 19371, + 457, + 796, + 1137, + 1255, + 264, + 6884, + 29892, + 18842, + 2862, + 589, + 19276, + 4415, + 816, + 10003, + 10190, + 915, + 280, + 10236, + 686, + 643, + 3993, + 3085, + 8247, + 305, + 457, + 29892, + 762, + 2160, + 2939, + 286, + 9477, + 17720, + 25424, + 8850, + 29876, + 3085, + 4987, + 1050, + 29107, + 6239, + 581, + 29873, + 816, + 3003, + 16857, + 7505, + 3110, + 371, + 29889 + ], + "tokens": [ + "", + "▁Dan", + "ach", + "▁her", + "r", + "sch", + "te", + "▁all", + "geme", + "ine", + "▁Z", + "uf", + "ried", + "en", + "heit", + ",", + "▁außer", + "▁bei", + "▁der", + "▁Kü", + "hl", + "sch", + "rank", + "innen", + "be", + "le", + "ucht", + "ung", + "ser", + "inner", + "ungs", + "mas", + "ch", + "ine", + ",", + "▁die", + "▁sich", + "▁über", + "▁m", + "angel", + "nde", + "▁Tür", + "öff", + "n", + "ungs", + "auf", + "mer", + "ksam", + "keit", + "swer", + "t", + "sch", + "ät", + "zung", + "▁bek", + "lag", + "te", + "." + ], + "decoded_with_special": " Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte.", + "decoded_skip_special": "Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte." + } + ] +} diff --git a/Tests/TokenizersTests/Resources/MultilingualConformance/inputs.json b/Tests/TokenizersTests/Resources/MultilingualConformance/inputs.json new file mode 100644 index 00000000..8992e37e --- /dev/null +++ b/Tests/TokenizersTests/Resources/MultilingualConformance/inputs.json @@ -0,0 +1,417 @@ +[ + { + "id": "latin-diacritics-french", + "category": "latin-diacritics", + "text": "café résumé naïve façade — crème brûlée was excellent." + }, + { + "id": "latin-diacritics-mixed", + "category": "latin-diacritics", + "text": "Hügel über Brücke. Mañana voy a la peña con José." + }, + { + "id": "latin-diacritics-portuguese", + "category": "latin-diacritics", + "text": "São Paulo é uma cidade interessante; Köln auch." + }, + { + "id": "cyrillic-greeting", + "category": "cyrillic", + "text": "Привет, мир! Это тест токенизатора." + }, + { + "id": "cyrillic-prose", + "category": "cyrillic", + "text": "Москва — столица России. Население более 12 миллионов." + }, + { + "id": "greek-greeting", + "category": "greek", + "text": "Καλημέρα κόσμε! Η γλώσσα είναι όμορφη." + }, + { + "id": "cjk-simplified-greeting", + "category": "cjk-simplified", + "text": "你好,世界!这是分词器测试。" + }, + { + "id": "cjk-simplified-prose", + "category": "cjk-simplified", + "text": "中文是世界上使用人口最多的语言之一。" + }, + { + "id": "cjk-traditional-compare", + "category": "cjk-traditional", + "text": "繁體中文與簡體中文有所不同。" + }, + { + "id": "cjk-traditional-hongkong", + "category": "cjk-traditional", + "text": "香港的繁體中文有獨特的詞彙。" + }, + { + "id": "japanese-voiced-kana-greeting", + "category": "japanese-voiced-kana", + "text": "こんにちは、世界。トークナイザーのテストです。" + }, + { + "id": "japanese-voiced-kana-prose", + "category": "japanese-voiced-kana", + "text": "東京タワーはとても賑やかな観光地です。" + }, + { + "id": "hangul-syllables-greeting", + "category": "hangul-syllables", + "text": "안녕하세요, 세계! 토크나이저 테스트입니다." + }, + { + "id": "hangul-syllables-prose", + "category": "hangul-syllables", + "text": "한국어는 한글로 표기합니다." + }, + { + "id": "arabic-greeting", + "category": "arabic-rtl", + "text": "مرحبا بالعالم! هذا اختبار للمحلل اللغوي." + }, + { + "id": "arabic-prose", + "category": "arabic-rtl", + "text": "اللغة العربية لغة سامية تكتب من اليمين إلى اليسار." + }, + { + "id": "hebrew-greeting", + "category": "hebrew-rtl", + "text": "שלום עולם! זה מבחן של הטוקנייזר." + }, + { + "id": "devanagari-hindi", + "category": "devanagari", + "text": "नमस्ते दुनिया! यह टोकनाइज़र परीक्षण है।" + }, + { + "id": "devanagari-sanskrit", + "category": "devanagari", + "text": "संस्कृत भारत की एक प्राचीन भाषा है।" + }, + { + "id": "thai-combining-marks-greeting", + "category": "thai-combining-marks", + "text": "สวัสดีชาวโลก! นี่คือการทดสอบเครื่องตัดคำ" + }, + { + "id": "thai-combining-marks-prose", + "category": "thai-combining-marks", + "text": "ภาษาไทยไม่ใช้ช่องว่างระหว่างคำ" + }, + { + "id": "mixed-script-cjk-latin", + "category": "mixed-script", + "text": "The 北京 trip in 2026 — très bien!" + }, + { + "id": "mixed-script-katakana-code", + "category": "mixed-script", + "text": "コードレビュー: bug fixed in commit a3f9c2b." + }, + { + "id": "math-symbols-summation", + "category": "math-symbols", + "text": "Result: Σ x_i = 42 (where x ∈ ℝ)." + }, + { + "id": "math-symbols-epsilon-delta", + "category": "math-symbols", + "text": "Theorem: ∀ε>0 ∃δ>0 such that |x-x₀|<δ ⇒ |f(x)-f(x₀)|<ε." + }, + { + "id": "currency-symbols-multi", + "category": "currency-symbols", + "text": "Cost: €12.50 → ¥1,820 ≈ £10.75 (approx, plus ₿0.0004)." + }, + { + "id": "box-drawing", + "category": "box-drawing-dingbats", + "text": "Box: ┌───────┐ │ Hello │ └───────┘ — done." + }, + { + "id": "dingbats-chess", + "category": "box-drawing-dingbats", + "text": "Pieces: ★ ☆ ♠ ♣ ♥ ♦ ♔ ♕ ♖ ♘ ♙ — chess set." + }, + { + "id": "math-astral-bold-script", + "category": "math-astral-glyphs", + "text": "Math bold: 𝐀𝐁𝐂𝐃 𝟎𝟏𝟐𝟑 — math script: 𝓐𝓑𝓒𝓓." + }, + { + "id": "astral-egyptian-hieroglyphs", + "category": "astral-plane-historic", + "text": "Egyptian hieroglyphs: 𓀀 𓂀 𓃀 𓆎𓅓𓏏𓊖." + }, + { + "id": "astral-cuneiform", + "category": "astral-plane-historic", + "text": "Cuneiform: 𒀭 𒈠 𒂗 𒆠 — sign list samples." + }, + { + "id": "astral-mahjong", + "category": "astral-plane-game-symbols", + "text": "Mahjong tiles: 🀀 🀁 🀂 🀃 🀄 🀅 🀆 🀇." + }, + { + "id": "astral-cards", + "category": "astral-plane-game-symbols", + "text": "Cards: 🃁 🃂 🃃 🂡 🂢 🂣." + }, + { + "id": "emoji-bmp-and-astral", + "category": "emoji-basic", + "text": "👋 Hello! 🌍 World 🚀 launch 🎉 party 🎂." + }, + { + "id": "emoji-zwj-family-pride-skin", + "category": "emoji-zwj-sequence", + "text": "Family: 👨‍👩‍👧‍👦 — Pride flag: 🏳️‍🌈 — Skin-tone: 👍🏽 ✊🏿 👶🏻." + }, + { + "id": "emoji-zwj-with-text-prefix", + "category": "emoji-zwj-sequence", + "text": "🏳️‍🌈Pride+👨‍👩‍👧‍👦Family+🇩🇪🇯🇵🇺🇸flags+test" + }, + { + "id": "emoji-keycap-and-flags", + "category": "emoji-keycap", + "text": "Keycaps: 1️⃣ 2️⃣ 3️⃣ — flag: 🇯🇵 🇩🇪 🇺🇸." + }, + { + "id": "url-with-query-fragment", + "category": "url-hex-base64", + "text": "url=https://example.com/path/to/resource?q=foo&bar=baz&n=42#section-3" + }, + { + "id": "hex-and-base64", + "category": "url-hex-base64", + "text": "hex=0xDEADBEEFCAFEBABE base64=YWJjZGVmZ2hpamtsbW5vcA==" + }, + { + "id": "sha256-hash", + "category": "url-hex-base64", + "text": "sha256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "id": "code-javascript-functional", + "category": "programming-code", + "text": "arr.map(x => x * 2).filter(x => x > 0).reduce((a, b) => a + b);" + }, + { + "id": "code-cplusplus-template", + "category": "programming-code", + "text": "template auto f(T&&t, U&&u) -> decltype(t+u) { return t + u; }" + }, + { + "id": "code-bitops", + "category": "programming-code", + "text": "result = (a >> 2) | (b << 4) & 0xFF;" + }, + { + "id": "code-javascript-modern", + "category": "programming-code", + "text": "if (x !== null && y >= 0) { return x?.foo ?? defaultValue; }" + }, + { + "id": "code-python-def", + "category": "programming-code", + "text": "def fibonacci(n):" + }, + { + "id": "code-python-if", + "category": "programming-code", + "text": " if n < 2:" + }, + { + "id": "code-python-return", + "category": "programming-code", + "text": " return n" + }, + { + "id": "code-python-recurse", + "category": "programming-code", + "text": " return fibonacci(n - 1) + fibonacci(n - 2)" + }, + { + "id": "code-go-func", + "category": "programming-code", + "text": "func greet(name string) {" + }, + { + "id": "code-go-tab-if", + "category": "programming-code", + "text": "\tif name != \"\" {" + }, + { + "id": "code-go-tab-printf", + "category": "programming-code", + "text": "\t\tfmt.Printf(\"Hello, %s!\\n\", name)" + }, + { + "id": "code-go-tab-close", + "category": "programming-code", + "text": "\t}" + }, + { + "id": "code-go-close", + "category": "programming-code", + "text": "}" + }, + { + "id": "programming-identifiers-mixed", + "category": "programming-identifiers", + "text": "get_user_id MAX_BUFFER_SIZE T_pad_token_id NSURLSessionConfiguration" + }, + { + "id": "programming-identifiers-cases", + "category": "programming-identifiers", + "text": "camelCaseVar PascalCase snake_case_var SCREAMING_SNAKE k8s_pod_count_v2" + }, + { + "id": "code-string-multiscript", + "category": "programming-code", + "text": "greeting = \"Привет, мир!\" + \" — from \" + \"東京\"" + }, + { + "id": "code-comment-diacritics", + "category": "programming-code", + "text": "// résumé — Swift comment with diacritics" + }, + { + "id": "code-comment-greek-math", + "category": "programming-code", + "text": "/* α + β = γ — Greek-letter math comment */" + }, + { + "id": "code-string-multiscript-emoji", + "category": "programming-code", + "text": "let title = \"🚀 Launch — пуск — 起動\"" + }, + { + "id": "ipa-phonetic-transcription", + "category": "ipa-phonetic", + "text": "The IPA transcription /fɪʃ/ for \"fish\" — with stress: ˈfɪʃ." + }, + { + "id": "mandarin-tones", + "category": "ipa-phonetic", + "text": "Tones: má mǎ mà mā (Mandarin tones — high, rising, dipping, falling)." + }, + { + "id": "whitespace-runs", + "category": "whitespace-edge", + "text": "hello world . end" + }, + { + "id": "whitespace-trailing-tabs", + "category": "whitespace-edge", + "text": "trailing whitespace before newline:\t(tab) (spaces)" + }, + { + "id": "punctuation-period", + "category": "punctuation-edge", + "text": "." + }, + { + "id": "punctuation-comma", + "category": "punctuation-edge", + "text": "," + }, + { + "id": "punctuation-exclamation", + "category": "punctuation-edge", + "text": "!" + }, + { + "id": "punctuation-nested-quotes", + "category": "punctuation-edge", + "text": "\"What?\" she asked — \"really?!\" — yes..." + }, + { + "id": "punctuation-dash-variants", + "category": "punctuation-edge", + "text": "... and ‒ also – and — and ―." + }, + { + "id": "multiscript-greetings", + "category": "multiscript-stress", + "text": "Hello 你好 안녕 こんにちは नमस्ते مرحبا שלום สวัสดี Привет Καλημέρα." + }, + { + "id": "multiscript-no-spaces", + "category": "multiscript-stress", + "text": "α你β안γनδस💫ع👋ש🚀ت∀∃∈⇒" + }, + { + "id": "multiscript-zwj-mixed", + "category": "multiscript-stress", + "text": "中文한글日本語混👋합니다🎉text混合" + }, + { + "id": "multiscript-string-concat", + "category": "multiscript-stress", + "text": "greeting=\"你好\"+🌍+\"안녕\"+नमस्ते+👨‍👩‍👧" + }, + { + "id": "multiscript-emoji-arrows", + "category": "multiscript-stress", + "text": "Σ x_i 🇯🇵 ≠ 北京 → München · Tōkyō · 上海 ✨ end." + }, + { + "id": "multiscript-arrow-chain", + "category": "multiscript-stress", + "text": "🚀नमस्ते→Привет→你好→안녕→مرحبا→❤️" + }, + { + "id": "multiscript-function-call", + "category": "multiscript-stress", + "text": "foo(中文,한글,日本語,हिन्दी,русский)→{∀:∃,⊕:⊗}" + }, + { + "id": "multiscript-legacy-symbols", + "category": "multiscript-stress", + "text": "№42 ™️ © ® ¶ § €¥£₿ Σ∀ 你好 안녕 ال" + }, + { + "id": "escape-sequences-with-zwj", + "category": "escape-sequences", + "text": "NULL\\0and\\ttab\\nnewline混入‍ZWJ️selectors" + }, + { + "id": "german-compound-short", + "category": "german-compound", + "text": "Natürlich. Deutsch zieht jetzt den Kompositahammer aus der Grammatikwerkzeugschublade. 😄" + }, + { + "id": "german-compound-long-1", + "category": "german-compound", + "text": "Im Morgennebelgedankenverästelungswald stand ein Kaffeetassenrandphilosophiebeobachter neben einer Regenschirmvergesslichkeitsstation und notierte in sein Sonntagmorgenideenfangnotizbuch die erstaunliche Häufigkeit von Fensterbankstaubsonnenlichtreflexionen." + }, + { + "id": "german-compound-long-2", + "category": "german-compound", + "text": "Neben ihm summte eine Kühlschrankinnenbeleuchtungserinnerungsmaschine, während ein Marmeladenglasdeckelöffnungsoptimierungsbeauftragter mit einer Treppenhausakustiküberraschungsanalyse beschäftigt war. Aus der Ferne näherte sich ein Wolkenkratzerfahrstuhlmusikkomponist auf einem Einradverkehrsregelmissachtungsgerät und rief: „Achtung, dort vorne liegt eine Gedankenfadenverknotungsgefahrensituation!“" + }, + { + "id": "german-compound-long-3", + "category": "german-compound", + "text": "Plötzlich öffnete sich ein Zeitungsseitenumblätterwindstoßportal, und heraus purzelten drei Büroklammernsortierweltmeisterschaftsteilnehmerinnen, ein Gartenzwergmützenfarbenberater und ein hochgradig verwirrter Suppenlöffelreflexionsmetaphysiker. Gemeinsam gründeten sie den Bundesverband für Nachmittagslichtschattenkantenbetrachtung und improvisierte Sofakissenburgenverteidigungsstrategien." + }, + { + "id": "german-compound-long-4", + "category": "german-compound", + "text": "Am Abend versammelten sich alle im Mondscheinfensterrahmenstaubglitzerzimmer, wo der Kaffeetassenrandphilosophiebeobachter eine feierliche Schlussrede über die Wichtigkeit von Unsinnsproduktionsfreude, Sprachmuskeldehnübungen und Donaudampfschifffahrtsgesellschaftskapitänsmützenknopfersatzteilbeschaffungsproblemen hielt." + }, + { + "id": "german-compound-long-5", + "category": "german-compound", + "text": "Danach herrschte allgemeine Zufriedenheit, außer bei der Kühlschrankinnenbeleuchtungserinnerungsmaschine, die sich über mangelnde Türöffnungsaufmerksamkeitswertschätzung beklagte." + } +] diff --git a/Tools/generate_tokenizer_baselines.py b/Tools/generate_tokenizer_baselines.py new file mode 100644 index 00000000..4f88d749 --- /dev/null +++ b/Tools/generate_tokenizer_baselines.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Regenerate per-kernel multilingual baselines from +`transformers.AutoTokenizer` for the MultilingualConformanceTests target. + +The Python reference is the authoritative source of truth; the Swift port +must produce byte-identical output to it. Each baseline file is keyed by the +stable input id from `inputs.json` and holds: + - input_ids: [int] canonical token-id sequence (the parity target) + - tokens: [str] convert_ids_to_tokens(input_ids) — diagnostic + - decoded_with_special: str tokenizer.decode(input_ids) + - decoded_skip_special: str tokenizer.decode(input_ids, skip_special_tokens=True) + +The two decoded fields are forward-compatible material for a decoder-side +parity test; the current Swift target only consumes `input_ids` and `tokens`. + +A baseline file's top-level shape: + { + "metadata": { + "model_id": str, + "transformers_version": str, + "generated_at": str (ISO-8601 UTC), + "input_count": int + }, + "entries": [ + { + "id": str, # stable id from inputs.json + "input_ids": [int], + "tokens": [str], + "decoded_with_special": str, + "decoded_skip_special": str + }, + ... + ] + } + +Usage: + python -m venv .venv && source .venv/bin/activate + pip install -r Tools/requirements.txt + python Tools/generate_tokenizer_baselines.py # regenerate all kernels + python Tools/generate_tokenizer_baselines.py bge_small # regenerate one kernel + +A `--check` mode (regenerate and diff against the committed baselines) is +intentionally omitted; the byte-identity gate is the Swift test target. +""" + +import argparse +import datetime +import json +import sys +from pathlib import Path + +# (kernel_slug, hf_model_id) — kernel matrix. +# +# Picked to span every tokenizer kernel and post-processor surface used by the +# Swift port today: +# - bge_small WordPiece (Bert family) — most used Apple-Silicon embedder +# - t5_small Unigram (SentencePiece) — canonical Unigram + Metaspace +# - gpt2 Byte-level BPE — round-trip baseline +# - roberta_base Byte-level BPE + RobertaProcessing — different post-processor +# - qwen2_5 Byte-level BPE (modern vocab/merges) — catches kernel- vs vocab-shape bugs +# - tinyllama SentencePiece BPE + byte-fallback — Llama family without HF auth gate +FAMILIES = [ + ("bge_small", "BAAI/bge-small-en-v1.5"), + ("t5_small", "google-t5/t5-small"), + ("gpt2", "openai-community/gpt2"), + ("roberta_base", "FacebookAI/roberta-base"), + ("qwen2_5", "Qwen/Qwen2.5-0.5B"), + ("tinyllama", "TinyLlama/TinyLlama-1.1B-Chat-v1.0"), +] + +ROOT = Path(__file__).resolve().parent.parent +INPUTS = ROOT / "Tests" / "TokenizersTests" / "Resources" / "MultilingualConformance" / "inputs.json" +OUTDIR = ROOT / "Tests" / "TokenizersTests" / "Resources" / "MultilingualConformance" / "baselines" + + +def load_inputs(): + with open(INPUTS) as f: + return json.load(f) + + +def regenerate(slug: str, model_id: str) -> None: + from transformers import AutoTokenizer, __version__ as transformers_version + + print(f"[{slug}] loading {model_id} …", flush=True) + tok = AutoTokenizer.from_pretrained(model_id, use_fast=True) + + # The conformance contract is parity with the *fast* (Rust) tokenizer. + # A slow-only model can still produce a sensible reference but the + # comparison is no longer apples-to-apples. + if not getattr(tok, "is_fast", False): + raise SystemExit( + f"[{slug}] {model_id} is not a fast tokenizer — parity with the " + "Swift port would be undefined. Pick a model that ships tokenizer.json." + ) + + inputs = load_inputs() + entries = [] + for entry in inputs: + text = entry["text"] + ids = tok(text, add_special_tokens=True)["input_ids"] + # `convert_ids_to_tokens` is per-token introspection; spaces/specials + # come through with their canonical sentinel form (e.g. `▁`, ``). + tokens = tok.convert_ids_to_tokens(ids) + decoded_with = tok.decode(ids, skip_special_tokens=False) + decoded_without = tok.decode(ids, skip_special_tokens=True) + entries.append({ + "id": entry["id"], + "input_ids": ids, + "tokens": tokens, + "decoded_with_special": decoded_with, + "decoded_skip_special": decoded_without, + }) + + payload = { + "metadata": { + "model_id": model_id, + "transformers_version": transformers_version, + "generated_at": datetime.datetime.now(datetime.timezone.utc) + .replace(microsecond=0).isoformat(), + "input_count": len(entries), + }, + "entries": entries, + } + + OUTDIR.mkdir(parents=True, exist_ok=True) + out_path = OUTDIR / f"{slug}_multilingual.json" + with open(out_path, "w") as f: + json.dump(payload, f, ensure_ascii=False, indent=2) + f.write("\n") + print(f"[{slug}] wrote {out_path} ({len(entries)} entries)") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "kernels", + nargs="*", + help="Subset of kernel slugs to regenerate (default: all).", + ) + args = parser.parse_args() + + known = {slug: model_id for slug, model_id in FAMILIES} + targets = args.kernels or list(known.keys()) + for slug in targets: + if slug not in known: + print(f"unknown kernel slug: {slug} (known: {sorted(known)})", file=sys.stderr) + sys.exit(2) + for slug in targets: + regenerate(slug, known[slug]) + + +if __name__ == "__main__": + main() diff --git a/Tools/requirements.txt b/Tools/requirements.txt new file mode 100644 index 00000000..6b1f03ff --- /dev/null +++ b/Tools/requirements.txt @@ -0,0 +1,2 @@ +transformers==4.57.1 +# `tokenizers` is pulled transitively at the version transformers expects.