update

main.py

@@ -6,11 +6,7 @@ https://huggingface.co/spaces/sayakpaul/demo-docker-gradio
 import argparse
 import json
 import platform
-from collections import OrderedDict
-# from typing import OrderedDict
 
-from allennlp.models.archival import archive_model, load_archive
-from allennlp.predictors.text_classifier import TextClassifierPredictor
 import fasttext
 from fasttext.FastText import load_model, _FastText
 import gradio as gr
@@ -18,7 +14,6 @@ from gradio import inputs, outputs
 from langid.langid import LanguageIdentifier, model
 
 from project_settings import project_path, temp_directory
-from toolbox.os.command import Command
 
 
 def get_args():
@@ -44,34 +39,11 @@ def get_args():
 
 lang_id_identifier: LanguageIdentifier = None
 fasttext_model: _FastText = None
-qgyd_lang_id_predictor: TextClassifierPredictor = None
-
-
-trained_model_dir = project_path / "trained_models/huggingface"
-trained_model_dir.mkdir(parents=True, exist_ok=True)
-
-
-def init_qgyd_lang_id_predictor() -> TextClassifierPredictor:
-    model_name = "qgyd2021/language_identification"
-    model_path = trained_model_dir / model_name
-    if not model_path.exists():
-        model_path.parent.mkdir(exist_ok=True)
-        Command.cd(model_path.parent.as_posix())
-        Command.popen("git clone https://huggingface.co/{}".format(model_name))
-
-    archive = load_archive(archive_file=model_path.as_posix())
-
-    predictor = TextClassifierPredictor(
-        model=archive.model,
-        dataset_reader=archive.dataset_reader,
-    )
-    return predictor
 
 
 def click_lang_id_button(text: str, ground_true: str, model_name: str):
     global lang_id_identifier
     global fasttext_model
-    global qgyd_lang_id_predictor
 
     text = str(text).strip()
 
@@ -81,16 +53,6 @@ def click_lang_id_button(text: str, ground_true: str, model_name: str):
         label, prob = fasttext_model.predict(text, k=1)
         label = label[0][9:]
         prob = prob[0]
-    elif model_name == "qgyd_lang_id_1":
-        json_dict = {
-            "sentence": text
-        }
-        outputs = qgyd_lang_id_predictor.predict_json(
-            json_dict
-        )
-        label = outputs["label"]
-        probs = outputs["probs"]
-        prob = max(probs)
     else:
         label = "model_name not available."
         prob = -1
@@ -114,10 +76,8 @@ def main():
 
     global lang_id_identifier
     global fasttext_model
-    global qgyd_lang_id_predictor
     lang_id_identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)
     fasttext_model = fasttext.load_model(args.fasttext_model)
-    qgyd_lang_id_predictor = init_qgyd_lang_id_predictor()
 
     blocks = gr.Interface(
         click_lang_id_button,

requirements.txt

@@ -1,5 +1,3 @@
 fasttext==0.9.2
 langid==1.1.6
 gradio==2.3.0
-allennlp==0.9.0
-overrides==1.9.0

toolbox/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
-
-if __name__ == "__main__":
-    pass

toolbox/os/__init__.py

@@ -1,6 +0,0 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
-
-
-if __name__ == '__main__':
-    pass

toolbox/os/command.py

@@ -1,53 +0,0 @@
-import os
-
-
-class Command(object):
-    custom_command = [
-        'cd'
-    ]
-
-    @staticmethod
-    def _get_cmd(command):
-        command = str(command).strip()
-        if command == '':
-            return None
-        cmd_and_args = command.split(sep=' ')
-        cmd = cmd_and_args[0]
-        args = ' '.join(cmd_and_args[1:])
-        return cmd, args
-
-    @classmethod
-    def popen(cls, command):
-        cmd, args = cls._get_cmd(command)
-        if cmd in cls.custom_command:
-            method = getattr(cls, cmd)
-            return method(args)
-        else:
-            resp = os.popen(command)
-            result = resp.read()
-            resp.close()
-            return result
-
-    @classmethod
-    def cd(cls, args):
-        if args.startswith('/'):
-            os.chdir(args)
-        else:
-            pwd = os.getcwd()
-            path = os.path.join(pwd, args)
-            os.chdir(path)
-
-    @classmethod
-    def system(cls, command):
-        return os.system(command)
-
-    def __init__(self):
-        pass
-
-
-def ps_ef_grep(keyword: str):
-    cmd = 'ps -ef | grep {}'.format(keyword)
-    rows = Command.popen(cmd)
-    rows = str(rows).split('\n')
-    rows = [row for row in rows if row.__contains__(keyword) and not row.__contains__('grep')]
-    return rows

toolbox/os/environment.py

@@ -1,114 +0,0 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
-import json
-import os
-
-from dotenv import load_dotenv
-from dotenv.main import DotEnv
-
-from smart.json.misc import traverse
-
-
-class EnvironmentManager(object):
-    def __init__(self, path, env, override=False):
-        filename = os.path.join(path, '{}.env'.format(env))
-        self.filename = filename
-
-        load_dotenv(
-            dotenv_path=filename,
-            override=override
-        )
-
-        self._environ = dict()
-
-    def open_dotenv(self, filename: str = None):
-        filename = filename or self.filename
-        dotenv = DotEnv(
-            dotenv_path=filename,
-            stream=None,
-            verbose=False,
-            interpolate=False,
-            override=False,
-            encoding="utf-8",
-        )
-        result = dotenv.dict()
-        return result
-
-    def get(self, key, default=None, dtype=str):
-        result = os.environ.get(key)
-        if result is None:
-            if default is None:
-                result = None
-            else:
-                result = default
-        else:
-            result = dtype(result)
-        self._environ[key] = result
-        return result
-
-
-_DEFAULT_DTYPE_MAP = {
-    'int': int,
-    'float': float,
-    'str': str,
-    'json.loads': json.loads
-}
-
-
-class JsonConfig(object):
-    """
-    将 json 中, 形如 `$float:threshold` 的值, 处理为:
-    从环境变量中查到 threshold, 再将其转换为 float 类型.
-    """
-    def __init__(self, dtype_map: dict = None, environment: EnvironmentManager = None):
-        self.dtype_map = dtype_map or _DEFAULT_DTYPE_MAP
-        self.environment = environment or os.environ
-
-    def sanitize_by_filename(self, filename: str):
-        with open(filename, 'r', encoding='utf-8') as f:
-            js = json.load(f)
-
-        return self.sanitize_by_json(js)
-
-    def sanitize_by_json(self, js):
-        js = traverse(
-            js,
-            callback=self.sanitize,
-            environment=self.environment
-        )
-        return js
-
-    def sanitize(self, string, environment):
-        """支持 $ 符开始的, 环境变量配置"""
-        if isinstance(string, str) and string.startswith('$'):
-            dtype, key = string[1:].split(':')
-            dtype = self.dtype_map[dtype]
-
-            value = environment.get(key)
-            if value is None:
-                raise AssertionError('environment not exist. key: {}'.format(key))
-
-            value = dtype(value)
-            result = value
-        else:
-            result = string
-        return result
-
-
-def demo1():
-    import json
-
-    from project_settings import project_path
-
-    environment = EnvironmentManager(
-        path=os.path.join(project_path, 'server/callbot_server/dotenv'),
-        env='dev',
-    )
-    init_scenes = environment.get(key='init_scenes', dtype=json.loads)
-    print(init_scenes)
-    print(environment._environ)
-    return
-
-
-if __name__ == '__main__':
-    demo1()

toolbox/os/other.py

@@ -1,9 +0,0 @@
-import os
-import inspect
-
-
-def pwd():
-    """你在哪个文件调用此函数, 它就会返回那个文件所在的 dir 目标"""
-    frame = inspect.stack()[1]
-    module = inspect.getmodule(frame[0])
-    return os.path.dirname(os.path.abspath(module.__file__))