models.py

from gradio.components import Component
import gradio as gr
from abc import ABC, abstractclassmethod
import inspect

class BaseTCOModel(ABC):
    # TO DO: Find way to specify which component should be used for computing cost
    def __setattr__(self, name, value):
        if isinstance(value, Component):
            self._components.append(value)
        self.__dict__[name] = value

    def __init__(self):
        super(BaseTCOModel, self).__setattr__("_components", [])

    def get_components(self) -> list[Component]:
        return self._components
    
    def get_components_for_cost_computing(self):
        return self.components_for_cost_computing
    
    def get_name(self):
        return self.name
    
    def register_components_for_cost_computing(self):
        args = inspect.getfullargspec(self.compute_cost_per_token)[0][1:]
        self.components_for_cost_computing = [self.__getattribute__(arg) for arg in args]
    
    @abstractclassmethod
    def compute_cost_per_token(self):
        pass
    
    @abstractclassmethod
    def render(self):
        pass
    
    def set_name(self, name):
        self.name = name
    
    def set_formula(self, formula):
        self.formula = formula
    
    def get_formula(self):
        return self.formula

class OpenAIModel(BaseTCOModel):

    def __init__(self):
        self.set_name("(SaaS) OpenAI")
        self.set_formula(r"""$CT = \frac{CT\_1K \times 1000}{L}$  <br>
                         with: <br>
                         CT = Cost per output Token <br>
                         CT_1K = Cost per 1000 Tokens (from OpenAI's pricing web page) <br>
                         L = Input Length 
                         """)
        super().__init__()

    def render(self):
        def on_model_change(model):
            
            if model == "GPT-4":
                return gr.Dropdown.update(choices=["8K", "32K"])
            else:
                return gr.Dropdown.update(choices=["4K", "16K"], value="4K")

        self.model = gr.Dropdown(["GPT-4", "GPT-3.5 Turbo"], value="GPT-4",
                                 label="OpenAI models",
                                 interactive=True, visible=False)
        self.context_length = gr.Dropdown(["8K", "32K"], value="8K", interactive=True,
                                          label="Context size",
                                          visible=False, info="Number of tokens the model considers when processing text")
        self.model.change(on_model_change, inputs=self.model, outputs=self.context_length)
        self.input_length = gr.Number(350, label="Average number of input tokens", 
                                      interactive=True, visible=False)

    def compute_cost_per_token(self, model, context_length, input_length):
        """Cost per token = """
        model = model[0]
        context_length = context_length[0]

        if model == "GPT-4" and context_length == "8K":
            cost_per_1k_input_tokens = 0.03
        elif model == "GPT-4" and context_length == "32K":
            cost_per_1k_input_tokens = 0.06
        elif model == "GPT-3.5" and context_length == "4K":
            cost_per_1k_input_tokens = 0.0015
        else:
            cost_per_1k_input_tokens = 0.003

        cost_per_output_token = cost_per_1k_input_tokens * input_length / 1000

        return cost_per_output_token

class OpenSourceLlama2Model(BaseTCOModel):
    
    def __init__(self):
        self.set_name("(Open source) Llama 2")
        self.set_formula(r"""$CT = \frac{VM\_CH}{TS \times 3600 \times MO \times U}$<br>
                         with: <br>
                         CT = Cost per Token <br>
                         VM_CH = VM Cost per Hour <br>
                         TS = Tokens per Second <br>
                         MO = Maxed Out <br>
                         U = Used
                         """)
        super().__init__()
    
    def render(self):
        vm_choices = ["1x Nvidia A100 (Azure NC24ads A100 v4)",
                      "2x Nvidia A100 (Azure NC48ads A100 v4)", 
                      "4x Nvidia A100 (Azure NC48ads A100 v4)"]
        
        def on_model_change(model):
            if model == "Llama 2 7B":
                return [gr.Dropdown.update(choices=vm_choices), 
                        gr.Markdown.update(value="To see the script used to benchmark the Llama2-7B model, [click here](https://example.com/script)"), 
                        gr.Number.update(value=3.6730),
                        gr.Number.update(value=694.38),
                        gr.Number.update(visible=True)
                        ]
            else:
                not_supported_vm = ["1x Nvidia A100 (Azure NC24ads A100 v4)", "2x Nvidia A100 (Azure NC48ads A100 v4)"]
                choices = [x for x in vm_choices if x not in not_supported_vm]
                return [gr.Dropdown.update(choices=choices, value="4x Nvidia A100 (Azure NC48ads A100 v4)"), 
                        gr.Markdown.update(value="To see the benchmark results used for the Llama2-70B model, [click here](https://www.cursor.so/blog/llama-inference#user-content-fn-llama-paper)"),
                        gr.Number.update(value=14.692),
                        gr.Number.update(value=18.6),
                        gr.Number.update(visible=False)
                        ]

        def on_vm_change(model, vm):
            # TO DO: load info from CSV
            if model == "Llama 2 7B" and vm == "1x Nvidia A100 (Azure NC24ads A100 v4)":
                return [gr.Number.update(value=3.6730), gr.Number.update(value=694.38)]
            elif model == "Llama 2 7B" and vm == "2x Nvidia A100 (Azure NC48ads A100 v4)":
                return [gr.Number.update(value=7.346), gr.Number.update(value=1388.76)]
            elif model == "Llama 2 7B" and vm == "4x Nvidia A100 (Azure NC48ads A100 v4)":
                return [gr.Number.update(value=14.692), gr.Number.update(value=2777.52)]
            elif model == "Llama 2 70B" and vm == "4x Nvidia A100 (Azure NC48ads A100 v4)":
                return [gr.Number.update(value=14.692), gr.Number.update(value=18.6)]
        
        self.model = gr.Dropdown(["Llama 2 7B", "Llama 2 70B"], value="Llama 2 7B", label="OpenSource models", visible=False)
        self.vm = gr.Dropdown(vm_choices,
                              value="1x Nvidia A100 (Azure NC24ads A100 v4)", 
                              visible=False,
                              label="Instance of VM with GPU",
                              info="Your options for this choice depend on the model you previously chose"
                              )
        self.vm_cost_per_hour = gr.Number(3.6730, label="VM instance cost per hour", 
                                      interactive=False, visible=False)
        self.tokens_per_second = gr.Number(694.38, visible=False,
                                           label="Number of tokens per second for this specific model and VM instance",
                                           interactive=False
                                           )
        self.input_length = gr.Number(233, label="Average number of input tokens", info="This is the number of input tokens used when the model was benchmarked to get the number of tokens/second it processes",
                                      interactive=False, visible=False)
        self.info = gr.Markdown("To see the script used to benchmark the Llama2-7B model, [click here](https://example.com/script)", interactive=False, visible=False)
        
        self.model.change(on_model_change, inputs=self.model, outputs=[self.vm, self.info, self.vm_cost_per_hour, self.tokens_per_second, self.input_length])
        self.vm.change(on_vm_change, inputs=[self.model, self.vm], outputs=[self.vm_cost_per_hour, self.tokens_per_second])
        self.maxed_out = gr.Slider(minimum=0.01, value=50., step=0.01, label="% maxed out", 
                                   info="How much the GPU is fully used",
                                   interactive=True,
                                   visible=False)
        self.used = gr.Slider(minimum=0.01, value=50., step=0.01, label="% used", 
                                   info="Percentage of time the GPU is used",
                                   interactive=True,
                                   visible=False)

    def compute_cost_per_token(self, vm_cost_per_hour, tokens_per_second, maxed_out, used):
        cost_per_token = vm_cost_per_hour * 10000 / (tokens_per_second * 3600 * maxed_out * used)
        return cost_per_token 

class OpenSourceDIY(BaseTCOModel):
    
    def __init__(self):
        self.set_name("(Open source) DIY")
        self.set_formula(r"""$CT = \frac{VM\_CH}{TS \times 3600 \times MO \times U}$<br>
                         with: <br>
                         CT = Cost per Token <br>
                         VM_CH = VM Cost per Hour <br>
                         TS = Tokens per Second <br>
                         MO = Maxed Out <br>
                         U = Used
                         """)
        super().__init__()
    
    def render(self):   
        self.info = gr.Markdown("Compute the cost/token based on our formula below, using your own parameters", visible=False)
        self.display_formula = gr.Markdown(r"""$CT = \frac{VM\_CH}{TS \times 3600 \times MO \times U}$<br>
                         with: <br>
                         CT = Cost per Token <br>
                         VM_CH = VM Cost per Hour <br>
                         TS = Tokens per Second <br>
                         MO = Maxed Out <br>
                         U = Used
                         """, visible=False)
        self.vm_cost_per_hour = gr.Number(3.5, label="VM instance cost per hour", 
                                      interactive=True, visible=False)
        self.tokens_per_second = gr.Number(700, visible=False,
                                           label="Number of tokens per second for this specific model and VM instance",
                                           interactive=True
                                           )
        self.maxed_out = gr.Slider(minimum=0.01, value=50., step=0.01, label="% maxed out", 
                                   info="How much the GPU is fully used",
                                   interactive=True,
                                   visible=False)
        self.used = gr.Slider(minimum=0.01, value=50., step=0.01, label="% used", 
                                   info="Percentage of time the GPU is used",
                                   interactive=True,
                                   visible=False)

    def compute_cost_per_token(self, vm_cost_per_hour, tokens_per_second, maxed_out, used):
        cost_per_token = vm_cost_per_hour * 10000 / (tokens_per_second * 3600 * maxed_out * used)
        return cost_per_token 

class CohereModel(BaseTCOModel):
    
    def __init__(self):
        self.set_name("(SaaS) Cohere")
        self.set_formula(r"""$CT = \frac{CT\_1K \times 1000}{L}$  <br>
                         with: <br>
                         CT = Cost per output Token <br>
                         CT_1M = Cost per one million Tokens (from Cohere's pricing web page) <br>
                         L = Input Length 
                         """)
        super().__init__()
    
    def render(self):
        def on_use_case_change(use_case):
            if use_case == "Summarize":
                return gr.Dropdown.update(choices=["Default"])
            else:
                return gr.Dropdown.update(choices=["Default", "Custom"])
            
        self.use_case = gr.Dropdown(["Generate", "Summarize"], value="Generate",
                                 label="API",
                                 interactive=True, visible=False)
        self.model = gr.Dropdown(["Default", "Custom"], value="Default",
                                 label="Model",
                                 interactive=True, visible=False)
        self.use_case.change(on_use_case_change, inputs=self.use_case, outputs=self.model)        
        self.input_length = gr.Number(350, label="Average number of input tokens", 
                                      interactive=True, visible=False)

    def compute_cost_per_token(self, use_case, model, input_length):
        """Cost per token = """
        use_case = use_case[0]        
        model = model[0]

        if use_case == "Generate":
            if model == "Default":
                cost_per_1M_input_tokens = 15
            else: 
                cost_per_1M_input_tokens = 30
        else:
            cost_per_1M_input_tokens = 15

        cost_per_output_token = cost_per_1M_input_tokens * input_length / 1000000

        return cost_per_output_token
   
class ModelPage:
    
    def __init__(self, Models: BaseTCOModel):
        self.models: list[BaseTCOModel] = []
        for Model in Models:
            model = Model()
            self.models.append(model)

    def render(self):
        for model in self.models:
            model.render()
            model.register_components_for_cost_computing() 

    def get_all_components(self) -> list[Component]:
        output = []
        for model in self.models:
            output += model.get_components()
        return output
    
    def get_all_components_for_cost_computing(self) -> list[Component]:
        output = []
        for model in self.models:
            output += model.get_components_for_cost_computing()
        return output

    def make_model_visible(self, name:str):
        # First decide which indexes
        output = []
        for model in self.models:
            if model.get_name() == name:
                output+= [gr.update(visible=True)] * len(model.get_components())
            else:
                output+= [gr.update(visible=False)] * len(model.get_components())
        return output
    
    def compute_cost_per_token(self, *args):
        begin=0
        current_model = args[-1]
        for model in self.models:
            model_n_args = len(model.get_components_for_cost_computing())
            if current_model == model.get_name():
                
                model_args = args[begin:begin+model_n_args]
                model_tco = model.compute_cost_per_token(*model_args)
                formula = model.get_formula()
                return f"Model {current_model} has a TCO of: ${model_tco}", model_tco, formula
            
            begin = begin+model_n_args