loading torch doesn't play nice, just use config for now

config.yml

@@ -9,3 +9,6 @@ chat:
     - "</s>"
     - "<unk>"
     - "### User:"
+queue:
+  max_size: 16
+  concurrency_count: 2

requirements.txt

@@ -1,7 +1,6 @@
 --extra-index-url https://pypi.ngc.nvidia.com
 nvidia-cuda-runtime
 nvidia-cublas
-torch
 llama-cpp-python @ https://github.com/OpenAccess-AI-Collective/ggml-webui/releases/download/v0.1.50-rc3/llama_cpp_python-gpu-0.1.50-cp38-cp38-linux_x86_64.whl
 pyyaml
 torch

tabbed.py

@@ -1,7 +1,4 @@
-import math
-
 import gradio as gr
-import torch
 import yaml
 from huggingface_hub import hf_hub_download
 from huggingface_hub.utils import LocalEntryNotFoundError
@@ -147,14 +144,4 @@ with gr.Blocks() as demo:
         )
         stop.click(fn=None, inputs=None, outputs=None, cancels=[submit_click_event, message_submit_event], queue=False)
 
-
-# figure out how much VRAM is available to see if we can increase concurrency
-concurrency_count = 1
-model_vram_size_in_gb = 11
-if torch.cuda.is_available():
-    device = torch.cuda.current_device()
-    total_memory = torch.cuda.get_device_properties(device).total_memory
-    total_memory_in_gb = total_memory / 1024**3
-    concurrency_count = int(math.floor(total_memory_in_gb / model_vram_size_in_gb))
-
-demo.queue(max_size=16, concurrency_count=1).launch(debug=True, server_name="0.0.0.0", server_port=7860)
+demo.queue(**config["queue"]).launch(debug=True, server_name="0.0.0.0", server_port=7860)