update

.gitignore

@@ -6,6 +6,7 @@
 flagged/
 pretrained_models/
 temp/
+trained_models/
 
 **/cache/
 **/__pycache__/

main.py

@@ -7,6 +7,8 @@ import argparse
 import json
 import platform
 
+from allennlp.models.archival import archive_model, load_archive
+from allennlp.predictors.text_classifier import TextClassifierPredictor
 import fasttext
 from fasttext.FastText import load_model, _FastText
 import gradio as gr
@@ -14,6 +16,7 @@ from gradio import inputs, outputs
 from langid.langid import LanguageIdentifier, model
 
 from project_settings import project_path, temp_directory
+from toolbox.os.command import Command
 
 
 def get_args():
@@ -39,11 +42,34 @@ def get_args():
 
 lang_id_identifier: LanguageIdentifier = None
 fasttext_model: _FastText = None
+qgyd_lang_id_predictor: TextClassifierPredictor = None
+
+
+trained_model_dir = project_path / "trained_models/huggingface"
+trained_model_dir.mkdir(parents=True, exist_ok=True)
+
+
+def init_qgyd_lang_id_predictor() -> TextClassifierPredictor:
+    model_name = "qgyd2021/language_identification"
+    model_path = trained_model_dir / model_name
+    if not model_path.exists():
+        model_path.parent.mkdir(exist_ok=True)
+        Command.cd(model_path.parent.as_posix())
+        Command.popen("git clone https://huggingface.co/{}".format(model_name))
+
+    archive = load_archive(archive_file=model_path.as_posix())
+
+    predictor = TextClassifierPredictor(
+        model=archive.model,
+        dataset_reader=archive.dataset_reader,
+    )
+    return predictor
 
 
 def click_lang_id_button(text: str, ground_true: str, model_name: str):
     global lang_id_identifier
     global fasttext_model
+    global qgyd_lang_id_predictor
 
     text = str(text).strip()
 
@@ -53,6 +79,16 @@ def click_lang_id_button(text: str, ground_true: str, model_name: str):
         label, prob = fasttext_model.predict(text, k=1)
         label = label[0][9:]
         prob = prob[0]
+    elif model_name == "qgyd_lang_id_1":
+        json_dict = {
+            "sentence": text
+        }
+        outputs = qgyd_lang_id_predictor.predict_json(
+            json_dict
+        )
+        label = outputs["label"]
+        probs = outputs["probs"]
+        prob = max(probs)
     else:
         label = "model_name not available."
         prob = -1
@@ -76,15 +112,17 @@ def main():
 
     global lang_id_identifier
     global fasttext_model
+    global qgyd_lang_id_predictor
     lang_id_identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)
     fasttext_model = fasttext.load_model(args.fasttext_model)
+    qgyd_lang_id_predictor = init_qgyd_lang_id_predictor()
 
     blocks = gr.Interface(
         click_lang_id_button,
         inputs=[
             inputs.Textbox(lines=3, label="text"),
             inputs.Textbox(label="ground_true"),
-            inputs.Dropdown(choices=["langid", "fasttext"], default="langid", label="model_name"),
+            inputs.Dropdown(choices=["langid", "fasttext", "qgyd_lang_id_1"], default="langid", label="model_name"),
         ],
         outputs=[
             outputs.Textbox(label="label"),

requirements.txt

@@ -1,3 +1,4 @@
-gradio==2.1.1
-langid==1.1.6
 fasttext==0.9.2
+langid==1.1.6
+gradio==2.3.0
+allennlp

toolbox/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+if __name__ == "__main__":
+    pass

toolbox/os/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/os/command.py

@@ -0,0 +1,53 @@
+import os
+
+
+class Command(object):
+    custom_command = [
+        'cd'
+    ]
+
+    @staticmethod
+    def _get_cmd(command):
+        command = str(command).strip()
+        if command == '':
+            return None
+        cmd_and_args = command.split(sep=' ')
+        cmd = cmd_and_args[0]
+        args = ' '.join(cmd_and_args[1:])
+        return cmd, args
+
+    @classmethod
+    def popen(cls, command):
+        cmd, args = cls._get_cmd(command)
+        if cmd in cls.custom_command:
+            method = getattr(cls, cmd)
+            return method(args)
+        else:
+            resp = os.popen(command)
+            result = resp.read()
+            resp.close()
+            return result
+
+    @classmethod
+    def cd(cls, args):
+        if args.startswith('/'):
+            os.chdir(args)
+        else:
+            pwd = os.getcwd()
+            path = os.path.join(pwd, args)
+            os.chdir(path)
+
+    @classmethod
+    def system(cls, command):
+        return os.system(command)
+
+    def __init__(self):
+        pass
+
+
+def ps_ef_grep(keyword: str):
+    cmd = 'ps -ef | grep {}'.format(keyword)
+    rows = Command.popen(cmd)
+    rows = str(rows).split('\n')
+    rows = [row for row in rows if row.__contains__(keyword) and not row.__contains__('grep')]
+    return rows

toolbox/os/environment.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import json
+import os
+
+from dotenv import load_dotenv
+from dotenv.main import DotEnv
+
+from smart.json.misc import traverse
+
+
+class EnvironmentManager(object):
+    def __init__(self, path, env, override=False):
+        filename = os.path.join(path, '{}.env'.format(env))
+        self.filename = filename
+
+        load_dotenv(
+            dotenv_path=filename,
+            override=override
+        )
+
+        self._environ = dict()
+
+    def open_dotenv(self, filename: str = None):
+        filename = filename or self.filename
+        dotenv = DotEnv(
+            dotenv_path=filename,
+            stream=None,
+            verbose=False,
+            interpolate=False,
+            override=False,
+            encoding="utf-8",
+        )
+        result = dotenv.dict()
+        return result
+
+    def get(self, key, default=None, dtype=str):
+        result = os.environ.get(key)
+        if result is None:
+            if default is None:
+                result = None
+            else:
+                result = default
+        else:
+            result = dtype(result)
+        self._environ[key] = result
+        return result
+
+
+_DEFAULT_DTYPE_MAP = {
+    'int': int,
+    'float': float,
+    'str': str,
+    'json.loads': json.loads
+}
+
+
+class JsonConfig(object):
+    """
+    将 json 中, 形如 `$float:threshold` 的值, 处理为:
+    从环境变量中查到 threshold, 再将其转换为 float 类型.
+    """
+    def __init__(self, dtype_map: dict = None, environment: EnvironmentManager = None):
+        self.dtype_map = dtype_map or _DEFAULT_DTYPE_MAP
+        self.environment = environment or os.environ
+
+    def sanitize_by_filename(self, filename: str):
+        with open(filename, 'r', encoding='utf-8') as f:
+            js = json.load(f)
+
+        return self.sanitize_by_json(js)
+
+    def sanitize_by_json(self, js):
+        js = traverse(
+            js,
+            callback=self.sanitize,
+            environment=self.environment
+        )
+        return js
+
+    def sanitize(self, string, environment):
+        """支持 $ 符开始的, 环境变量配置"""
+        if isinstance(string, str) and string.startswith('$'):
+            dtype, key = string[1:].split(':')
+            dtype = self.dtype_map[dtype]
+
+            value = environment.get(key)
+            if value is None:
+                raise AssertionError('environment not exist. key: {}'.format(key))
+
+            value = dtype(value)
+            result = value
+        else:
+            result = string
+        return result
+
+
+def demo1():
+    import json
+
+    from project_settings import project_path
+
+    environment = EnvironmentManager(
+        path=os.path.join(project_path, 'server/callbot_server/dotenv'),
+        env='dev',
+    )
+    init_scenes = environment.get(key='init_scenes', dtype=json.loads)
+    print(init_scenes)
+    print(environment._environ)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/other.py

@@ -0,0 +1,9 @@
+import os
+import inspect
+
+
+def pwd():
+    """你在哪个文件调用此函数, 它就会返回那个文件所在的 dir 目标"""
+    frame = inspect.stack()[1]
+    module = inspect.getmodule(frame[0])
+    return os.path.dirname(os.path.abspath(module.__file__))