Deploy nanochat

.gitignore

@@ -0,0 +1,20 @@
+# Binary tokenizer files (available at https://huggingface.co/BrianGuo/nanochat-d20-chat)
+tokenizer/token_bytes.pt
+tokenizer/tokenizer.pkl
+
+# Python cache
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo

app.py

@@ -28,9 +28,38 @@ def download_model():
     )
     print("Model downloaded successfully!")
 
+def download_tokenizer():
+    """Download tokenizer files from Hugging Face."""
+    tokenizer_dir = "/data/tokenizer"
+
+    if os.path.exists(tokenizer_dir) and os.listdir(tokenizer_dir):
+        print(f"Tokenizer found, skipping download")
+        return
+
+    print("Downloading tokenizer from BrianGuo/nanochat-d20-chat...")
+    from huggingface_hub import hf_hub_download
+
+    os.makedirs(tokenizer_dir, exist_ok=True)
+
+    # Download tokenizer files
+    hf_hub_download(
+        repo_id="BrianGuo/nanochat-d20-chat",
+        filename="tokenizer/token_bytes.pt",
+        local_dir="/data",
+        local_dir_use_symlinks=False
+    )
+    hf_hub_download(
+        repo_id="BrianGuo/nanochat-d20-chat",
+        filename="tokenizer/tokenizer.pkl",
+        local_dir="/data",
+        local_dir_use_symlinks=False
+    )
+    print("Tokenizer downloaded successfully!")
+
 if __name__ == "__main__":
-    # Download model before starting
+    # Download model and tokenizer before starting
     download_model()
+    download_tokenizer()
 
     # Override sys.argv to pass default arguments for HF Spaces
     sys.argv = [