enable upload and model selection in dropdown
Browse files- Dockerfile +1 -2
- app.py +7 -4
- utils/config.py +41 -0
- utils/uploadAndExample.py +39 -0
- utils/vulnerability_classifier.py +19 -0
Dockerfile
CHANGED
|
@@ -15,8 +15,7 @@ COPY --chown=user ./requirements.txt requirements.txt
|
|
| 15 |
RUN pip3 install --no-cache-dir -r requirements.txt
|
| 16 |
|
| 17 |
# Copy the current directory contents into the container at /app setting the owner to the user
|
| 18 |
-
#
|
| 19 |
-
|
| 20 |
COPY --link --chown=1000 ./ /app
|
| 21 |
|
| 22 |
# app will need to expose the port for taking browser inputs
|
|
|
|
| 15 |
RUN pip3 install --no-cache-dir -r requirements.txt
|
| 16 |
|
| 17 |
# Copy the current directory contents into the container at /app setting the owner to the user
|
| 18 |
+
# also set the user as owner for root directory to enable update permissions
|
|
|
|
| 19 |
COPY --link --chown=1000 ./ /app
|
| 20 |
|
| 21 |
# app will need to expose the port for taking browser inputs
|
app.py
CHANGED
|
@@ -1,5 +1,8 @@
|
|
| 1 |
import streamlit as st
|
| 2 |
-
import
|
|
|
|
|
|
|
|
|
|
| 3 |
with st.sidebar:
|
| 4 |
# upload and example doc
|
| 5 |
choice = st.sidebar.radio(label = 'Select the Document',
|
|
@@ -7,16 +10,16 @@ with st.sidebar:
|
|
| 7 |
or else you can try a example document',
|
| 8 |
options = ('Upload Document', 'Try Example'),
|
| 9 |
horizontal = True)
|
| 10 |
-
|
| 11 |
|
| 12 |
# Create a list of options for the dropdown
|
| 13 |
model_options = ['Llama3.1-8B','Llama3.1-70B','Llama3.1-405B','Zephyr 7B β','Mistral-7B','Mixtral-8x7B']
|
| 14 |
|
| 15 |
# Dropdown selectbox: model
|
| 16 |
model_sel = st.selectbox('Select a model:', model_options)
|
| 17 |
-
|
| 18 |
|
| 19 |
-
|
| 20 |
|
| 21 |
with st.container():
|
| 22 |
st.markdown("<h2 style='text-align: center;'> Vulnerability Analysis 3.1 </h2>", unsafe_allow_html=True)
|
|
|
|
| 1 |
import streamlit as st
|
| 2 |
+
from utils.uploadAndExample import add_upload
|
| 3 |
+
from utils.config import model_dict
|
| 4 |
+
from utils.vulnerability_classifier import label_dict
|
| 5 |
+
|
| 6 |
with st.sidebar:
|
| 7 |
# upload and example doc
|
| 8 |
choice = st.sidebar.radio(label = 'Select the Document',
|
|
|
|
| 10 |
or else you can try a example document',
|
| 11 |
options = ('Upload Document', 'Try Example'),
|
| 12 |
horizontal = True)
|
| 13 |
+
add_upload(choice)
|
| 14 |
|
| 15 |
# Create a list of options for the dropdown
|
| 16 |
model_options = ['Llama3.1-8B','Llama3.1-70B','Llama3.1-405B','Zephyr 7B β','Mistral-7B','Mixtral-8x7B']
|
| 17 |
|
| 18 |
# Dropdown selectbox: model
|
| 19 |
model_sel = st.selectbox('Select a model:', model_options)
|
| 20 |
+
model_sel_name = model_dict[model_sel]
|
| 21 |
|
| 22 |
+
st.session_state['model_sel_name'] = model_sel_name
|
| 23 |
|
| 24 |
with st.container():
|
| 25 |
st.markdown("<h2 style='text-align: center;'> Vulnerability Analysis 3.1 </h2>", unsafe_allow_html=True)
|
utils/config.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import configparser
|
| 2 |
+
import logging
|
| 3 |
+
|
| 4 |
+
def getconfig(configfile_path:str):
|
| 5 |
+
"""
|
| 6 |
+
configfile_path: file path of .cfg file
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
config = configparser.ConfigParser()
|
| 10 |
+
|
| 11 |
+
try:
|
| 12 |
+
config.read_file(open(configfile_path))
|
| 13 |
+
return config
|
| 14 |
+
except:
|
| 15 |
+
logging.warning("config file not found")
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# Declare all the necessary variables
|
| 19 |
+
def get_classifier_params(model_name):
|
| 20 |
+
config = getconfig('paramconfig.cfg')
|
| 21 |
+
params = {}
|
| 22 |
+
params['model_name'] = config.get(model_name,'MODEL')
|
| 23 |
+
params['split_by'] = config.get(model_name,'SPLIT_BY')
|
| 24 |
+
params['split_length'] = int(config.get(model_name,'SPLIT_LENGTH'))
|
| 25 |
+
params['split_overlap'] = int(config.get(model_name,'SPLIT_OVERLAP'))
|
| 26 |
+
params['remove_punc'] = bool(int(config.get(model_name,'REMOVE_PUNC')))
|
| 27 |
+
params['split_respect_sentence_boundary'] = bool(int(config.get(model_name,'RESPECT_SENTENCE_BOUNDARY')))
|
| 28 |
+
params['threshold'] = float(config.get(model_name,'THRESHOLD'))
|
| 29 |
+
params['top_n'] = int(config.get(model_name,'TOP_KEY'))
|
| 30 |
+
|
| 31 |
+
return params
|
| 32 |
+
|
| 33 |
+
# Model select
|
| 34 |
+
model_dict = {
|
| 35 |
+
'Llama3.1-8B': 'meta-llama/Meta-Llama-3.1-8B-Instruct',
|
| 36 |
+
'Llama3.1-70B': 'meta-llama/Meta-Llama-3.1-70B-Instruct',
|
| 37 |
+
'Llama3.1-405B': 'meta-llama/Meta-Llama-3.1-405B-Instruct',
|
| 38 |
+
'Zephyr 7B β': 'HuggingFaceH4/zephyr-7b-beta',
|
| 39 |
+
'Mistral-7B': 'mistralai/Mistral-7B-Instruct-v0.2',
|
| 40 |
+
'Mixtral-8x7B': 'mistralai/Mixtral-8x7B-Instruct-v0.1',
|
| 41 |
+
}
|
utils/uploadAndExample.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import streamlit as st
|
| 2 |
+
import tempfile
|
| 3 |
+
import json
|
| 4 |
+
|
| 5 |
+
def add_upload(choice):
|
| 6 |
+
"""
|
| 7 |
+
Provdies the user with choice to either 'Upload Document' or 'Try Example'.
|
| 8 |
+
Based on user choice runs streamlit processes and save the path and name of
|
| 9 |
+
the 'file' to streamlit session_state which then can be fetched later.
|
| 10 |
+
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
if choice == 'Upload Document':
|
| 14 |
+
|
| 15 |
+
# if 'filename' in st.session_state:
|
| 16 |
+
# Delete all the items in Session state
|
| 17 |
+
# for key in st.session_state.keys():
|
| 18 |
+
# del st.session_state[key]
|
| 19 |
+
|
| 20 |
+
uploaded_file = st.sidebar.file_uploader('Upload the File',
|
| 21 |
+
type=['pdf', 'docx', 'txt'])
|
| 22 |
+
if uploaded_file is not None:
|
| 23 |
+
with tempfile.NamedTemporaryFile(mode="wb", delete = False) as temp:
|
| 24 |
+
bytes_data = uploaded_file.getvalue()
|
| 25 |
+
temp.write(bytes_data)
|
| 26 |
+
st.session_state['filename'] = uploaded_file.name
|
| 27 |
+
st.session_state['filepath'] = temp.name
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
else:
|
| 31 |
+
# listing the options
|
| 32 |
+
with open('docStore/sample/files.json','r') as json_file:
|
| 33 |
+
files = json.load(json_file)
|
| 34 |
+
|
| 35 |
+
option = st.sidebar.selectbox('Select the example document',
|
| 36 |
+
list(files.keys()))
|
| 37 |
+
file_name = file_path = files[option]
|
| 38 |
+
st.session_state['filename'] = file_name
|
| 39 |
+
st.session_state['filepath'] = file_path
|
utils/vulnerability_classifier.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# labels dictionary
|
| 2 |
+
label_dict= {0: 'Agricultural communities',
|
| 3 |
+
1: 'Children',
|
| 4 |
+
2: 'Coastal communities',
|
| 5 |
+
3: 'Ethnic, racial or other minorities',
|
| 6 |
+
4: 'Fishery communities',
|
| 7 |
+
5: 'Informal sector workers',
|
| 8 |
+
6: 'Members of indigenous and local communities',
|
| 9 |
+
7: 'Migrants and displaced persons',
|
| 10 |
+
8: 'Older persons',
|
| 11 |
+
9: 'Other',
|
| 12 |
+
10: 'Persons living in poverty',
|
| 13 |
+
11: 'Persons with disabilities',
|
| 14 |
+
12: 'Persons with pre-existing health conditions',
|
| 15 |
+
13: 'Residents of drought-prone regions',
|
| 16 |
+
14: 'Rural populations',
|
| 17 |
+
15: 'Sexual minorities (LGBTQI+)',
|
| 18 |
+
16: 'Urban populations',
|
| 19 |
+
17: 'Women and other genders'}
|