fix: update mineru setup

requirements.txt

@@ -5,10 +5,10 @@ rapid-table>=1.0.3,<2.0.0
 rapidocr-paddle
 rapidocr-onnxruntime
 gradio-pdf>=0.0.21
-git+https://github.com/opendatalab/MinerU.git@dev
+git+https://github.com/opendatalab/MinerU.git@release-1.3.4
 git+https://github.com/VikParuchuri/marker
 docling
-PyMuPDF>=1.24.9,<1.24.14
+PyMuPDF
 pymupdf4llm
 unstructured[pdf]
 ultralytics>=8.3.48
@@ -22,3 +22,5 @@ unimernet==0.2.3
 transformers<5.0.0,>=4.45.2
 pypdf
 opencv-contrib-python
+numpy==1.26.4
+pdfminer.six

utils.py

@@ -62,12 +62,71 @@ def fix_problematic_imports():
     sys.modules[
         "magic_pdf.model.sub_modules.mfr.unimernet.Unimernet"
     ] = fake_unimernet_module
+    
+    
+def setup_mineru_config():
+    import json
+    import os
+    import requests
+    from huggingface_hub import snapshot_download
+
+
+    def download_json(url):
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.json()
+
+
+    def download_and_modify_json(url, local_filename, modifications):
+        if os.path.exists(local_filename):
+            data = json.load(open(local_filename))
+            config_version = data.get('config_version', '0.0.0')
+            if config_version < '1.2.0':
+                data = download_json(url)
+        else:
+            data = download_json(url)
+
+        for key, value in modifications.items():
+            data[key] = value
+
+        with open(local_filename, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=4)
+
+    mineru_patterns = [
+        "models/Layout/YOLO/*",
+        "models/MFD/YOLO/*",
+        "models/MFR/unimernet_hf_small_2503/*",
+        "models/OCR/paddleocr_torch/*",
+    ]
+    model_dir = snapshot_download('opendatalab/PDF-Extract-Kit-1.0', allow_patterns=mineru_patterns)
+
+    layoutreader_pattern = [
+        "*.json",
+        "*.safetensors",
+    ]
+    layoutreader_model_dir = snapshot_download('hantian/layoutreader', allow_patterns=layoutreader_pattern)
+
+    model_dir = model_dir + '/models'
+    print(f'model_dir is: {model_dir}')
+    print(f'layoutreader_model_dir is: {layoutreader_model_dir}')
+
+    json_url = 'https://raw.githubusercontent.com/opendatalab/MinerU/refs/heads/release-1.3.12/magic-pdf.template.json'
+    config_file_name = 'magic-pdf.json'
+    home_dir = os.path.expanduser('~')
+    config_file = os.path.join(home_dir, config_file_name)
+
+    json_mods = {
+        'models-dir': model_dir,
+        'layoutreader-model-dir': layoutreader_model_dir,
+    }
+
+    download_and_modify_json(json_url, config_file, json_mods)
+    print(f'The configuration file has been configured successfully, the path is: {config_file}')
 
 
 def prepare_env_mineru():
     import json
     import os
-
     import nltk
 
     # download nltk data
@@ -82,12 +141,7 @@ def prepare_env_mineru():
         return
 
     # download models
-    os.system(
-        "wget https://raw.githubusercontent.com/opendatalab/MinerU/"
-        "refs/heads/release-1.3.12/scripts/download_models_hf.py"
-        " -O download_models_hf.py"
-    )
-    os.system("python3 download_models_hf.py")
+    setup_mineru_config()
 
     with open(config_path, "r") as file:
         data = json.load(file)