init

README.md

@@ -0,0 +1,77 @@
+---
+language: en
+datasets:
+- librispeech_asr
+- common_voice
+tags:
+- speech
+license: apache-2.0
+---
+
+# M-CTC-T 
+​
+Massively multilingual speech recognizer from Meta AI. The model is a 1B-param transformer encoder, with a CTC head over 8065 character labels and a language identification head over 60 language ID labels. It is trained on Common Voice (version 6.1, December 2020 release) and VoxPopuli. After training on Common Voice and VoxPopuli, the model is trained on Common Voice only. The labels are unnormalized character-level transcripts (punctuation and capitalization are not removed). The model takes as input Mel filterbank features from a 16Khz audio signal.
+​
+![model image](https://raw.githubusercontent.com/cwkeam/scientific-images/main/MCTCT/mctct-arch.png) 
+​
+The original Flashlight code, model checkpoints, and Colab notebook can be found at https://github.com/flashlight/wav2letter/tree/main/recipes/mling_pl .
+​
+​
+## Citation
+​
+[Paper](https://arxiv.org/abs/2111.00161)
+​
+Authors: Loren Lugosch, Tatiana Likhomanenko, Gabriel Synnaeve, Ronan Collobert
+​
+```
+@article{lugosch2021pseudo,
+  title={Pseudo-Labeling for Massively Multilingual Speech Recognition},
+  author={Lugosch, Loren and Likhomanenko, Tatiana and Synnaeve, Gabriel and Collobert, Ronan},
+  journal={ICASSP},
+  year={2022}
+}
+```
+​
+Additional thanks to [Chan Woo Kim](https://huggingface.co/cwkeam) and [Patrick von Platen](https://huggingface.co/patrickvonplaten) for porting the model from Flashlight to PyTorch.
+​
+# Training method
+​
+![model image](https://raw.githubusercontent.com/cwkeam/scientific-images/main/MCTCT/mctct-slimipl.png) TO-DO: replace with the training diagram from paper
+​
+For more information on how the model was trained, please take a look at the [official paper](https://arxiv.org/abs/2111.00161).
+​
+# Usage
+​
+To transcribe audio files the model can be used as a standalone acoustic model as follows:
+​
+```python
+import torch
+import torchaudio
+from datasets import load_dataset
+from transformers import MCTCTForCTC, MCTCTProcessor
+
+model = MCTCTForCTC.from_pretrained("speechbrain/mctct-large")
+processor = MCTCTProcessor.from_pretrained("speechbrain/mctct-large")
+
+ # load dummy dataset and read soundfiles
+ds = load_dataset("patrickvonplaten/librispeech_asr_dummy", "clean", split="validation")
+ 
+# tokenize
+input_features = processor(ds[0]["audio"]["array"], return_tensors="pt").input_features 
+
+# retrieve logits
+logits = model(input_features).logits
+
+# take argmax and decode
+predicted_ids = torch.argmax(logits, dim=-1)
+transcription = processor.batch_decode(predicted_ids)
+```
+ 
+Results for Common Voice, averaged over all languages:
+​
+*Character error rate (CER)*:
+​
+| Valid | Test |
+|-------|------|
+| 21.4  | 23.3 |
+

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 8065, "</s>": 8066}

config.json

@@ -0,0 +1,162 @@
+{
+  "architectures": [
+    "MCTCTForAudioFrameClassification"
+  ],
+  "attention_head_dim": 384,
+  "attention_probs_dropout_prob": 0.3,
+  "bos_token_id": 0,
+  "conv_channels": null,
+  "conv_dropout": 0.3,
+  "conv_glu_dim": 1,
+  "conv_kernel": [
+    7
+  ],
+  "conv_stride": [
+    3
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "eos_token_id": 2,
+  "hidden_act": "relu",
+  "hidden_dropout_prob": 0.3,
+  "hidden_size": 1536,
+  "id2label": {
+    "0": "ab",
+    "1": "ar",
+    "10": "dv",
+    "11": "el",
+    "12": "en",
+    "13": "eo",
+    "14": "es",
+    "15": "et",
+    "16": "eu",
+    "17": "fa",
+    "18": "fi",
+    "19": "fr",
+    "2": "as",
+    "20": "fy-NL",
+    "21": "ga-IE",
+    "22": "hi",
+    "23": "hsb",
+    "24": "hu",
+    "25": "ia",
+    "26": "id",
+    "27": "it",
+    "28": "ja",
+    "29": "ka",
+    "3": "br",
+    "30": "kab",
+    "31": "ky",
+    "32": "lg",
+    "33": "lt",
+    "34": "lv",
+    "35": "mn",
+    "36": "mt",
+    "37": "nl",
+    "38": "or",
+    "39": "pa-IN",
+    "4": "ca",
+    "40": "pl",
+    "41": "pt",
+    "42": "rm-sursilv",
+    "43": "rm-vallader",
+    "44": "ro",
+    "45": "ru",
+    "46": "rw",
+    "47": "sah",
+    "48": "sl",
+    "49": "sv-SE",
+    "5": "cnh",
+    "50": "ta",
+    "51": "th",
+    "52": "tr",
+    "53": "tt",
+    "54": "uk",
+    "55": "vi",
+    "56": "vot",
+    "57": "zh-CN",
+    "58": "zh-HK",
+    "59": "zh-TW",
+    "6": "cs",
+    "7": "cv",
+    "8": "cy",
+    "9": "de"
+  },
+  "initializer_range": 0.02,
+  "input_channels": 1,
+  "input_feat_per_channel": 80,
+  "intermediate_size": 6144,
+  "label2id": {
+    "ab": "0",
+    "ar": "1",
+    "as": "2",
+    "br": "3",
+    "ca": "4",
+    "cnh": "5",
+    "cs": "6",
+    "cv": "7",
+    "cy": "8",
+    "de": "9",
+    "dv": "10",
+    "el": "11",
+    "en": "12",
+    "eo": "13",
+    "es": "14",
+    "et": "15",
+    "eu": "16",
+    "fa": "17",
+    "fi": "18",
+    "fr": "19",
+    "fy-NL": "20",
+    "ga-IE": "21",
+    "hi": "22",
+    "hsb": "23",
+    "hu": "24",
+    "ia": "25",
+    "id": "26",
+    "it": "27",
+    "ja": "28",
+    "ka": "29",
+    "kab": "30",
+    "ky": "31",
+    "lg": "32",
+    "lt": "33",
+    "lv": "34",
+    "mn": "35",
+    "mt": "36",
+    "nl": "37",
+    "or": "38",
+    "pa-IN": "39",
+    "pl": "40",
+    "pt": "41",
+    "rm-sursilv": "42",
+    "rm-vallader": "43",
+    "ro": "44",
+    "ru": "45",
+    "rw": "46",
+    "sah": "47",
+    "sl": "48",
+    "sv-SE": "49",
+    "ta": "50",
+    "th": "51",
+    "tr": "52",
+    "tt": "53",
+    "uk": "54",
+    "vi": "55",
+    "vot": "56",
+    "zh-CN": "57",
+    "zh-HK": "58",
+    "zh-TW": "59"
+  },
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.3,
+  "max_position_embeddings": 920,
+  "model_type": "mctct",
+  "num_attention_heads": 4,
+  "num_conv_layers": 1,
+  "num_hidden_layers": 36,
+  "pad_token_id": 1,
+  "torch_dtype": "float32",
+  "transformers_version": "4.20.0.dev0",
+  "vocab_size": 8065
+}

preprocessor_config.json

@@ -0,0 +1,21 @@
+{
+  "K": 257,
+  "do_normalize": true,
+  "feature_extractor_type": "MCTCFeatureExtractor",
+  "feature_size": 80,
+  "frame_signal_scale": 32768.0,
+  "hop_length": 10,
+  "mel_floor": 1.0,
+  "n_fft": 512,
+  "normalize_means": true,
+  "normalize_vars": true,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "preemphasis_coeff": 0.97,
+  "return_attention_mask": false,
+  "sample_size": 400,
+  "sample_stride": 160,
+  "sampling_rate": 16000,
+  "win_function": "hamming_window",
+  "win_length": 25
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ebe460f70052e0f0b8c1039e11b9bb0a9e2c135fb98b8e88562e5a5936d073f
+size 4186831517

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "return_attention_mask": false, "do_normalize": true, "special_tokens_map_file": "./mctc-large/special_tokens_map.json", "name_or_path": "./mctc-large", "tokenizer_class": "Wav2Vec2CTCTokenizer"}