update

.gitignore

@@ -3,7 +3,7 @@
 .idea/
 
 #data/
-#pretrained_models/
+pretrained_models/
 temp/
 
 **/cache/

Dockerfile

@@ -1,7 +1,7 @@
 # read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
 # you will also find guides on how best to write your Dockerfile
 
-FROM python:3.8
+FROM python:3.6
 
 WORKDIR /code
 
@@ -27,4 +27,6 @@ WORKDIR $HOME/app
 # Copy the current directory contents into the container at $HOME/app setting the owner to the user
 COPY --chown=user . $HOME/app
 
+RUN bash -c 'bash install.sh --stage 1 --stop_stage 1 --system_version ubuntu'
+
 CMD ["python", "main.py"]

install.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+# bash install.sh --stage 1 --stop_stage 1 --system_version centos
+
+verbose=true;
+stage=-1
+stop_stage=2
+
+work_dir="$(pwd)"
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+
+$verbose && echo "system_version: ${system_version}"
+
+pretrained_models_dir="$(pwd)/pretrained_models"
+
+mkdir -p "${pretrained_models_dir}"
+
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  $verbose && echo "stage 1: download fasttext models"
+  cd "${pretrained_models_dir}" || exit 1;
+
+  wget https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin
+  wget https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz
+
+fi

language_identification.md

@@ -16,3 +16,16 @@ https://github.com/saffsd/langid.py/tree/master/langid/train
 4. 训练 NB (Naive Bayes) 概率模型, 即每个 item 对每个类型的概率贡献. 
 
 ```
+
+
+### fasttext
+
+识别 176 种语言。
+https://fasttext.cc/docs/en/language-identification.html
+
+
+### 参考
+
+```text
+https://zhuanlan.zhihu.com/p/600245782
+```

main.py

@@ -6,14 +6,12 @@ https://huggingface.co/spaces/sayakpaul/demo-docker-gradio
 import argparse
 import json
 import platform
-from typing import Tuple
 
+import fasttext
+from fasttext.FastText import load_model, _FastText
 import gradio as gr
-import langid
+from gradio import inputs, outputs
 from langid.langid import LanguageIdentifier, model
-import matplotlib.pyplot as plt
-import numpy as np
-from PIL import Image
 
 from project_settings import project_path, temp_directory
 
@@ -30,29 +28,40 @@ def get_args():
         default=(project_path / "lang_id_examples.json").as_posix(),
         type=str
     )
+    parser.add_argument(
+        "--fasttext_model",
+        default=(project_path / "pretrained_models/lid.176.bin").as_posix(),
+        type=str
+    )
     args = parser.parse_args()
     return args
 
 
-lang_id_identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)
+lang_id_identifier: LanguageIdentifier = None
+fasttext_model: _FastText = None
 
 
 def click_lang_id_button(text: str, ground_true: str, model_name: str):
     global lang_id_identifier
+    global fasttext_model
 
     if model_name == "langid":
         label, prob = lang_id_identifier.classify(text)
+    elif model_name == "fasttext":
+        label, prob = fasttext_model.predict(text, k=1)
+        label = label[0][9:]
+        prob = prob[0]
     else:
         label = "model_name not available."
-        prob = 0.0
-    return label, round(prob, 4)
+        prob = -1
+    return label, str(round(prob, 4))
 
 
 def main():
     args = get_args()
 
     brief_description = """
-    ### Language Identification
+    Language Identification
     """
 
     # description
@@ -63,56 +72,28 @@ def main():
     with open(args.lang_id_examples_file, "r", encoding="utf-8") as f:
         lang_id_examples = json.load(f)
 
-    # ui
-    with gr.Blocks() as blocks:
-        gr.Markdown(value=brief_description)
-
-        with gr.Row():
-            with gr.Column(scale=5):
-                with gr.Tabs():
-                    with gr.TabItem("lang_id"):
-                        gr.Markdown(value="")
-
-                        with gr.Row():
-                            with gr.Column(scale=1):
-                                lang_id_text = gr.Textbox(lines=2, max_lines=50, label="text")
-                                lang_id_ground_true = gr.Textbox(label="ground_true")
-
-                                lang_id_model_name = gr.Dropdown(choices=["langid"], value="langid", label="model_name")
-                                lang_id_button = gr.Button("run", variant="primary")
-
-                            with gr.Column(scale=1):
-                                lang_id_label = gr.Textbox(label="label")
-                                lang_id_prob = gr.Number(label="prob")
-
-                        gr.Examples(
-                            examples=lang_id_examples,
-                            inputs=[
-                                lang_id_text,
-                                lang_id_ground_true,
-                                lang_id_model_name,
-                            ],
-                            outputs=[lang_id_label, lang_id_prob],
-                            fn=click_lang_id_button
-                        )
-
-                        # click event
-                        lang_id_button.click(
-                            click_lang_id_button,
-                            inputs=[
-                                lang_id_text,
-                                lang_id_ground_true,
-                                lang_id_model_name,
-                            ],
-                            outputs=[lang_id_label, lang_id_prob],
-                        )
-
-                    gr.Markdown(value=description)
-
-    blocks.queue().launch(
+    global lang_id_identifier
+    global fasttext_model
+    lang_id_identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)
+    fasttext_model = fasttext.load_model(args.fasttext_model)
+
+    blocks = gr.Interface(
+        click_lang_id_button,
+        inputs=[
+            inputs.Textbox(lines=3, label="text"),
+            inputs.Textbox(label="ground_true"),
+            inputs.Dropdown(choices=["langid", "fasttext"], default="langid", label="model_name"),
+        ],
+        outputs=[
+            outputs.Textbox(label="label"),
+            outputs.Textbox(label="prob"),
+        ],
+        examples=lang_id_examples,
+        description=brief_description
+    )
+
+    blocks.launch(
         share=False if platform.system() == "Windows" else False,
-        server_name="127.0.0.1" if platform.system() == "Windows" else "0.0.0.0",
-        server_port=7860
     )
     return
 

requirements.txt

@@ -1,2 +1,3 @@
-gradio==4.28.3
-langid==1.1.6
+gradio==2.1.1
+langid==1.1.6
+fasttext==0.9.2