implement import

Author: tusharinqueue

setup.py

@@ -7,10 +7,7 @@
     description="A bilingual BPE tokenizer trained on English + Hindi transcripts",
     packages=find_packages(),
     package_data={
-        "tokenizer": [
-            "../data/vocab.json",
-            "../data/merges.txt",
-        ]
+        "tewtoken": ["vocab.json", "merges.txt"]  # ← updated path
     },
     python_requires=">=3.7",
 )

tewtoken/bpe.py

@@ -128,13 +128,13 @@ def merge_pair(pair, vocab):
 print(f"\nFinal vocab size: {len(token_to_id)}")
 
 #Save vocab ────────────────────────────────────────────
-with open(r"F:\RandomProjects\tewtoken\data\vocab.json", "w", encoding="utf-8") as f:
+with open(r"/tewtoken/vocab.json", "w", encoding="utf-8") as f:
     json.dump(token_to_id, f, ensure_ascii=False, indent=2)
 
 print("Saved vocab.json")
 
 #Save merges ───────────────────────────────────────────
-with open(r"F:\RandomProjects\tewtoken\data\merges.txt", "w", encoding="utf-8") as f:
+with open(r"/tewtoken/merges.txt", "w", encoding="utf-8") as f:
     for pair in merges:
         f.write(f"{pair[0]} {pair[1]}\n")
 

data/merges.txt tewtoken/merges.txtdata/merges.txt renamed to tewtoken/merges.txt

@@ -3,9 +3,9 @@
 import os
 
 # ── 1. Dynamic paths ─────────────────────────────────────────
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-VOCAB_PATH = os.path.join(BASE_DIR, "data", "vocab.json")
-MERGES_PATH = os.path.join(BASE_DIR, "data", "merges.txt")
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+VOCAB_PATH = os.path.join(BASE_DIR, "vocab.json")
+MERGES_PATH = os.path.join(BASE_DIR, "merges.txt")
 
 # ── 2. Load vocab and merges ─────────────────────────────────
 with open(VOCAB_PATH, "r", encoding="utf-8") as f: