Duplicate from herimor/voxtream

Co-authored-by: Nikita Torgashov <herimor@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,74 @@
+---
+language:
+- en
+license: cc-by-4.0
+pipeline_tag: text-to-speech
+tags:
+- voxtream
+- text-to-speech
+---
+
+# Model Card for VoXtream
+
+VoXtream, a fully autoregressive, zero-shot streaming text-to-speech system for real-time use that begins speaking from the first word.
+
+### Key features
+
+- **Streaming**: Support a full-stream scenario, where the full sentence is not known in advance. The model takes the text stream coming word-by-word as input and outputs an audio stream in 80ms chunks.
+- **Speed**: Works **5x** times faster than real-time and achieves **102 ms** first packet latency on GPU.
+- **Quality and efficiency**: With only 9k hours of training data, it matches or surpasses the quality and intelligibility of larger models or models trained on large datasets.
+
+### Model Sources 
+
+- **Repository:** [repo](https://github.com/herimor/voxtream) 
+- **Paper:** [paper](https://arxiv.org/pdf/2509.15969) 
+- **Demo:** [demo](https://herimor.github.io/voxtream) 
+
+## Get started
+
+### Installation
+
+```bash
+pip install voxtream
+```
+
+### Usage
+
+#### Output streaming
+```bash
+voxtream \
+    --prompt-audio assets/audio/male.wav \
+    --prompt-text "The liquor was first created as 'Brandy Milk', produced with milk, brandy and vanilla." \
+    --text "In general, however, some method is then needed to evaluate each approximation." \
+    --output "output_stream.wav"
+```
+* Note: Initial run may take some time to download model weights.
+
+#### Full streaming
+```bash
+voxtream \
+    --prompt-audio assets/audio/female.wav \
+    --prompt-text "Betty Cooper helps Archie with cleaning a store room, when Reggie attacks her." \
+    --text "Staff do not always do enough to prevent violence." \
+    --output "full_stream.wav" \
+    --full-stream
+```
+
+### Out-of-Scope Use
+
+Any organization or individual is prohibited from using any technology mentioned in this paper to generate someone's speech without his/her consent, including but not limited to government leaders, political figures, and celebrities. If you do not comply with this item, you could be in violation of copyright laws.
+
+## Training Data
+
+The model was trained on a 9k-hour subset from [Emilia](https://huggingface.co/datasets/amphion/Emilia-Dataset) and [HiFiTTS2](https://huggingface.co/datasets/nvidia/hifitts-2) datasets. You can download it [here](https://huggingface.co/datasets/herimor/voxtream-train-9k). For more details, please check our paper. 
+
+## Citation 
+
+```
+@article{torgashov2025voxtream,
+  author    = {Torgashov, Nikita and Henter, Gustav Eje and Skantze, Gabriel},
+  title     = {Vo{X}tream: Full-Stream Text-to-Speech with Extremely Low Latency},
+  journal   = {arXiv:2509.15969},
+  year      = {2025}
+}
+```

config.json

@@ -0,0 +1,15 @@
+{
+    "phone_former": "phone_former",
+    "temp_former": "temp_former",
+    "dep_former": "dep_former_csm",
+    "phone_vocab_size": 73,
+    "audio_vocab_size": 2049,
+    "embedding_dim": 1024,
+    "spk_embedding_dim": 192,
+    "num_codebooks": 12,
+    "num_phone_states": 4,
+    "amortization_divisor": 16,
+    "look_ahead": 2,
+    "audio_window_size": 250,
+    "phone_window_size": 350
+}

dep_former_csm.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6a8b06be6e4a5aee244b6218a5ce7bd28c8b288a2c5c994af021d2579e6a2fc
+size 637669544

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73b7039e40434ebe7a7f0faeb91406cd54fc358185e5f836fdd10e36aef377f9
+size 1767213632

phoneme_to_token.json

@@ -0,0 +1,73 @@
+{
+    "AA0": 0,
+    "AA1": 1,
+    "AA2": 2,
+    "AE0": 3,
+    "AE1": 4,
+    "AE2": 5,
+    "AH0": 6,
+    "AH1": 7,
+    "AH2": 8,
+    "AO0": 9,
+    "AO1": 10,
+    "AO2": 11,
+    "AW0": 12,
+    "AW1": 13,
+    "AW2": 14,
+    "AY0": 15,
+    "AY1": 16,
+    "AY2": 17,
+    "B": 18,
+    "CH": 19,
+    "D": 20,
+    "DH": 21,
+    "EH0": 22,
+    "EH1": 23,
+    "EH2": 24,
+    "ER0": 25,
+    "ER1": 26,
+    "ER2": 27,
+    "EY0": 28,
+    "EY1": 29,
+    "EY2": 30,
+    "F": 31,
+    "G": 32,
+    "HH": 33,
+    "IH0": 34,
+    "IH1": 35,
+    "IH2": 36,
+    "IY0": 37,
+    "IY1": 38,
+    "IY2": 39,
+    "JH": 40,
+    "K": 41,
+    "L": 42,
+    "M": 43,
+    "N": 44,
+    "NG": 45,
+    "OW0": 46,
+    "OW1": 47,
+    "OW2": 48,
+    "OY0": 49,
+    "OY1": 50,
+    "OY2": 51,
+    "P": 52,
+    "R": 53,
+    "S": 54,
+    "SH": 55,
+    "T": 56,
+    "TH": 57,
+    "UH0": 58,
+    "UH1": 59,
+    "UH2": 60,
+    "UW0": 61,
+    "UW1": 62,
+    "UW2": 63,
+    "V": 64,
+    "W": 65,
+    "Y": 66,
+    "Z": 67,
+    "ZH": 68,
+    "sil": 69,
+    "spn": 70
+}