reafctor

packages.txt

@@ -0,0 +1,3 @@
+ffmpeg
+flash-attn
+triton

prometheus/eval/model_qa.py

@@ -6,8 +6,8 @@ import json
 from tqdm import tqdm
 import shortuuid
 
-from llava.conversation import default_conversation
-from llava.utils import disable_torch_init
+from prometheus.conversation import default_conversation
+from prometheus.utils import disable_torch_init
 
 
 # new stopping implementation

prometheus/eval/model_vqa.py

@@ -5,11 +5,11 @@ import json
 from tqdm import tqdm
 import shortuuid
 
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from llava.conversation import conv_templates, SeparatorStyle
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+from prometheus.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from prometheus.conversation import conv_templates, SeparatorStyle
+from prometheus.model.builder import load_pretrained_model
+from prometheus.utils import disable_torch_init
+from prometheus.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
 
 from PIL import Image
 import math

prometheus/eval/model_vqa_loader.py

@@ -5,11 +5,11 @@ import json
 from tqdm import tqdm
 import shortuuid
 
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from llava.conversation import conv_templates, SeparatorStyle
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, process_images, get_model_name_from_path
+from prometheus.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from prometheus.conversation import conv_templates, SeparatorStyle
+from prometheus.model.builder import load_pretrained_model
+from prometheus.utils import disable_torch_init
+from prometheus.mm_utils import tokenizer_image_token, process_images, get_model_name_from_path
 from torch.utils.data import Dataset, DataLoader
 
 from PIL import Image

prometheus/eval/model_vqa_mmbench.py

@@ -6,11 +6,11 @@ import pandas as pd
 from tqdm import tqdm
 import shortuuid
 
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from llava.conversation import conv_templates, SeparatorStyle
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, process_images, load_image_from_base64, get_model_name_from_path
+from prometheus.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from prometheus.conversation import conv_templates, SeparatorStyle
+from prometheus.model.builder import load_pretrained_model
+from prometheus.utils import disable_torch_init
+from prometheus.mm_utils import tokenizer_image_token, process_images, load_image_from_base64, get_model_name_from_path
 
 from PIL import Image
 import math

prometheus/eval/model_vqa_qbench.py

@@ -3,11 +3,11 @@ import torch
 from tqdm import tqdm
 import json
 
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from llava.conversation import conv_templates, SeparatorStyle
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+from prometheus.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from prometheus.conversation import conv_templates, SeparatorStyle
+from prometheus.model.builder import load_pretrained_model
+from prometheus.utils import disable_torch_init
+from prometheus.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
 
 from PIL import Image
 

prometheus/eval/model_vqa_science.py

@@ -5,11 +5,11 @@ import json
 from tqdm import tqdm
 import shortuuid
 
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from llava.conversation import conv_templates, SeparatorStyle
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+from prometheus.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from prometheus.conversation import conv_templates, SeparatorStyle
+from prometheus.model.builder import load_pretrained_model
+from prometheus.utils import disable_torch_init
+from prometheus.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
 
 from PIL import Image
 import math

prometheus/eval/run_llava.py

@@ -1,17 +1,17 @@
 import argparse
 import torch
 
-from llava.constants import (
+from prometheus.constants import (
     IMAGE_TOKEN_INDEX,
     DEFAULT_IMAGE_TOKEN,
     DEFAULT_IM_START_TOKEN,
     DEFAULT_IM_END_TOKEN,
     IMAGE_PLACEHOLDER,
 )
-from llava.conversation import conv_templates, SeparatorStyle
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import (
+from prometheus.conversation import conv_templates, SeparatorStyle
+from prometheus.model.builder import load_pretrained_model
+from prometheus.utils import disable_torch_init
+from prometheus.mm_utils import (
     process_images,
     tokenizer_image_token,
     get_model_name_from_path,

prometheus/mm_utils.py

@@ -4,7 +4,7 @@ import base64
 
 import torch
 from transformers import StoppingCriteria
-from vision.constants import IMAGE_TOKEN_INDEX
+from .constants import IMAGE_TOKEN_INDEX
 
 
 def load_image_from_base64(image):

prometheus/model/builder.py

@@ -19,8 +19,8 @@ import shutil
 
 from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BitsAndBytesConfig
 import torch
-from llava.model import *
-from llava.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from prometheus.model import *
+from prometheus.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
 
 
 def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto", device="cuda", **kwargs):

prometheus/model/consolidate.py

@@ -6,8 +6,8 @@ import argparse
 
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM
-from llava.model import *
-from llava.model.utils import auto_upgrade
+from prometheus.model import *
+from prometheus.model.utils import auto_upgrade
 
 
 def consolidate_ckpt(src_path, dst_path):

prometheus/model/language_model/llava_mpt.py

@@ -24,7 +24,7 @@ from transformers import AutoConfig, AutoModelForCausalLM
 from transformers.modeling_outputs import CausalLMOutputWithPast
 
 from .mpt.modeling_mpt import MPTConfig, MPTForCausalLM, MPTModel
-from llava.model.llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+from prometheus.model.llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
 
 
 class LlavaMPTConfig(MPTConfig):

prometheus/model/llava_arch.py

@@ -21,7 +21,7 @@ import torch.nn as nn
 from .multimodal_encoder.builder import build_vision_tower
 from .multimodal_projector.builder import build_vision_projector
 
-from llava.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from prometheus.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
 
 
 class LlavaMetaModel:

prometheus/model/make_delta.py

@@ -7,7 +7,7 @@ import argparse
 import torch
 from tqdm import tqdm
 from transformers import AutoTokenizer, AutoModelForCausalLM
-from llava.model.utils import auto_upgrade
+from prometheus.model.utils import auto_upgrade
 
 
 def make_delta(base_model_path, target_model_path, delta_path, hub_repo_id):

prometheus/serve/cli.py

@@ -1,11 +1,11 @@
 import argparse
 import torch
 
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from llava.conversation import conv_templates, SeparatorStyle
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import process_images, tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+from prometheus.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from prometheus.conversation import conv_templates, SeparatorStyle
+from prometheus.model.builder import load_pretrained_model
+from prometheus.utils import disable_torch_init
+from prometheus.mm_utils import process_images, tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
 
 from PIL import Image
 

prometheus/serve/controller.py

@@ -18,8 +18,8 @@ import numpy as np
 import requests
 import uvicorn
 
-from llava.constants import CONTROLLER_HEART_BEAT_EXPIRATION
-from llava.utils import build_logger, server_error_msg
+from prometheus.constants import CONTROLLER_HEART_BEAT_EXPIRATION
+from prometheus.utils import build_logger, server_error_msg
 
 
 logger = build_logger("controller", "controller.log")

prometheus/serve/gradio_web_server.py

@@ -7,10 +7,10 @@ import time
 import gradio as gr
 import requests
 
-from llava.conversation import (default_conversation, conv_templates,
+from prometheus.conversation import (default_conversation, conv_templates,
                                    SeparatorStyle)
-from llava.constants import LOGDIR
-from llava.utils import (build_logger, server_error_msg,
+from prometheus.constants import LOGDIR
+from prometheus.utils import (build_logger, server_error_msg,
     violates_moderation, moderation_msg)
 import hashlib
 

prometheus/serve/model_worker.py

@@ -15,12 +15,12 @@ import torch
 import uvicorn
 from functools import partial
 
-from llava.constants import WORKER_HEART_BEAT_INTERVAL
-from llava.utils import (build_logger, server_error_msg,
+from prometheus.constants import WORKER_HEART_BEAT_INTERVAL
+from prometheus.utils import (build_logger, server_error_msg,
     pretty_print_semaphore)
-from llava.model.builder import load_pretrained_model
-from llava.mm_utils import process_images, load_image_from_base64, tokenizer_image_token, KeywordsStoppingCriteria
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from prometheus.model.builder import load_pretrained_model
+from prometheus.mm_utils import process_images, load_image_from_base64, tokenizer_image_token, KeywordsStoppingCriteria
+from prometheus.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
 from transformers import TextIteratorStreamer
 from threading import Thread
 

prometheus/serve/test_message.py

@@ -3,7 +3,7 @@ import json
 
 import requests
 
-from llava.conversation import default_conversation
+from prometheus.conversation import default_conversation
 
 
 def main():

prometheus/train/train.py

@@ -26,13 +26,13 @@ import torch
 
 import transformers
 
-from llava.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from prometheus.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
 from torch.utils.data import Dataset
-from llava.train.llava_trainer import LLaVATrainer
+from llprometheusava.train.llava_trainer import LLaVATrainer
 
-from llava import conversation as conversation_lib
-from llava.model import *
-from llava.mm_utils import tokenizer_image_token
+from prometheus import conversation as conversation_lib
+from prometheus.model import *
+from prometheus.mm_utils import tokenizer_image_token
 
 from PIL import Image
 

prometheus/train/train_mem.py

@@ -3,11 +3,11 @@
 # Make it more memory efficient by monkey patching the LLaMA model with FlashAttn.
 
 # Need to call this before importing transformers.
-from llava.train.llama_flash_attn_monkey_patch import replace_llama_attn_with_flash_attn
+from prometheus.train.llama_flash_attn_monkey_patch import replace_llama_attn_with_flash_attn
 
 replace_llama_attn_with_flash_attn()
 
-from llava.train.train import train
+from prometheus.train.train import train
 
 if __name__ == "__main__":
     train()

prometheus/utils.py

@@ -6,7 +6,7 @@ import sys
 
 import requests
 
-from llava.constants import LOGDIR
+from prometheus.constants import LOGDIR
 
 server_error_msg = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
 moderation_msg = "YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES. PLEASE TRY AGAIN."