add evaluation results with language model

.ipynb_checkpoints/add lm decoder-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.ipynb_checkpoints/preprocessor_config-checkpoint.json

@@ -0,0 +1,10 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0,
+  "processor_class": "Wav2Vec2ProcessorWithLM",
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

add lm decoder.ipynb

@@ -0,0 +1,399 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "db2971a9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "29f15da8fd9549188347df46955b078d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center>\\n<img src=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import notebook_login\n",
+    "\n",
+    "notebook_login()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2377a1e5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Cloning https://huggingface.co/Tahsin-Mayeesha/wav2vec2-bn-300m into local empty directory.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9eacbc3325314c0b9e70d738ea655554",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Download file pytorch_model.bin:   0%|          | 594/1.18G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "abce727d7bdc4291b5b8158ba46ca359",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Download file runs/Feb02_18-57-15_job-adbfa1a2-412e-4cc9-8438-18b8de11318f/events.out.tfevents.1643828376.job-…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "990958b3a4894a58bbc7b275683aadc5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Download file runs/Feb02_18-57-15_job-adbfa1a2-412e-4cc9-8438-18b8de11318f/1643828376.0908198/events.out.tfeve…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0e7b5cf37e13411f861ab6f97a7086f5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Download file training_args.bin: 100%|##########| 2.92k/2.92k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "083dfac65da1440bbb1b44d0ef9736b2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Clean file runs/Feb02_18-57-15_job-adbfa1a2-412e-4cc9-8438-18b8de11318f/1643828376.0908198/events.out.tfevents…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "de458e6cd4844842a9518ab9a6555742",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Clean file training_args.bin:  34%|###4      | 1.00k/2.92k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "83b6f9d8c39649e382025e1462d12ee0",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Clean file runs/Feb02_18-57-15_job-adbfa1a2-412e-4cc9-8438-18b8de11318f/events.out.tfevents.1643828376.job-adb…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "95142f5b1abc4218b5ed797b9edffbbc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Clean file pytorch_model.bin:   0%|          | 1.00k/1.18G [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import Repository\n",
+    "\n",
+    "repo = Repository(local_dir=\"wav2vec2-bn-300m\", clone_from=\"Tahsin-Mayeesha/wav2vec2-bn-300m\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "091991cd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ab4eb980966140b7a6bca34bc51c3fd2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/212 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "442c24c800b949238d33dd9ad2100c09",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/260 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "69504d7b2d994c6789cdc9b27c00bc5b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/1.99k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f8290e53eb4f402e99b6d71bd5836955",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/1.13k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "abaa5f4a82de4ec8b27a67178addfcb9",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/25.0 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9b6186e1a4df4999b71e8ea2e8f9d392",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/309 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from transformers import AutoProcessor\n",
+    "processor = AutoProcessor.from_pretrained(\"Tahsin-Mayeesha/wav2vec2-bn-300m\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "3507c167",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vocab_dict = processor.tokenizer.get_vocab()\n",
+    "sorted_vocab_dict = {k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1])}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "15ee83a8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Found entries of length > 1 in alphabet. This is unusual unless style is BPE, but the alphabet was not recognized as BPE type. Is this correct?\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pyctcdecode import build_ctcdecoder\n",
+    "\n",
+    "decoder = build_ctcdecoder(\n",
+    "    labels=list(sorted_vocab_dict.keys()),\n",
+    "    kenlm_model_path=\"5gram.arpa\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "46585ac6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import Wav2Vec2ProcessorWithLM\n",
+    "\n",
+    "processor_with_lm = Wav2Vec2ProcessorWithLM(\n",
+    "    feature_extractor=processor.feature_extractor,\n",
+    "    tokenizer=processor.tokenizer,\n",
+    "    decoder=decoder\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "c17befdc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "processor_with_lm.save_pretrained(\"wav2vec2-bn-300m\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "f3ec60c4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Adding files tracked by Git LFS: ['language_model/unigrams.txt']. This may take a bit of time if the files are large.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7aa6e28e8a9c49b79b09f5d2884383d7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Upload file language_model/unigrams.txt:   0%|          | 3.38k/22.3M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "To https://huggingface.co/Tahsin-Mayeesha/wav2vec2-bn-300m\n",
+      "   b6e6996..258816a  main -> main\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'https://huggingface.co/Tahsin-Mayeesha/wav2vec2-bn-300m/commit/258816acfe8e1e49f41b4edcf9f20f812b4bf00d'"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "repo.push_to_hub(commit_message=\"Upload lm-boosted decoder\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "add2d4ca",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

evaluations_with_lm/.ipynb_checkpoints/openslr_bn_test_eval_results-checkpoint.txt

@@ -0,0 +1,2 @@
+WER: 0.17776164652632478
+CER: 0.04394092712884769

evaluations_with_lm/openslr_bn_test_eval_results.txt

@@ -0,0 +1,2 @@
+WER: 0.17776164652632478
+CER: 0.04394092712884769