Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -15,6 +15,7 @@ db_name = os.environ.get("MONGO_DB", "grant_docs")
|
|
| 15 |
mongo_uri = f"mongodb+srv://{user}:{password}@{cluster}/{db_name}?retryWrites=true&w=majority&tls=true&tlsAllowInvalidCertificates=true"
|
| 16 |
client = MongoClient(mongo_uri, tls=True, tlsAllowInvalidCertificates=True, serverSelectionTimeoutMS=20000)
|
| 17 |
db = client[db_name]
|
|
|
|
| 18 |
def gate_ui():
|
| 19 |
APP_PASSWORD=st.secrets.get("APP_PASSWORD", os.getenv("APP_PASSWORD")).strip()
|
| 20 |
if "authed" not in st.session_state:
|
|
@@ -35,81 +36,82 @@ def gate_ui():
|
|
| 35 |
st.error("Incorrect password.")
|
| 36 |
return False
|
| 37 |
# === Streamlit UI ===
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
st.title("📄 Document Chunker & Uploader")
|
| 42 |
-
|
| 43 |
-
with st.sidebar:
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
uploaded_file = st.file_uploader("Upload a DOCX, TXT, or PDF file", type=["docx", "txt", "pdf"])
|
| 66 |
-
|
| 67 |
-
if uploaded_file:
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
else:
|
| 84 |
-
st.write("⏳ Processing with DocumentChunker...")
|
| 85 |
-
chunker = DocumentChunker()
|
| 86 |
-
chunks = chunker.process_document(str(temp_path))
|
| 87 |
-
|
| 88 |
-
if chunks:
|
| 89 |
-
for chunk in chunks:
|
| 90 |
-
chunk['collection_category']=selected_category
|
| 91 |
-
chunk['metadata'].update({
|
| 92 |
-
"title": uploaded_file.name,
|
| 93 |
-
"uploaded_at": modified_time,
|
| 94 |
-
"is_grant_app": is_grant_app,
|
| 95 |
-
})
|
| 96 |
-
collection.insert_one(chunk)
|
| 97 |
-
|
| 98 |
-
st.success(f"✅ {len(chunks)} chunks inserted into `final_chunks` (category: `{selected_category}`)")
|
| 99 |
-
|
| 100 |
-
# Show a few previews
|
| 101 |
-
for i, c in enumerate(chunks[:3]):
|
| 102 |
-
st.subheader(f"Chunk {i+1}: {c['metadata'].get('header') or 'No Header'}")
|
| 103 |
-
st.markdown(c['text'][:400] + "...")
|
| 104 |
-
st.caption(f"Topics: {', '.join(c['metadata']['topics'])} | Category: {c['metadata']['category']}")
|
| 105 |
-
st.progress(c['metadata']['confidence_score'])
|
| 106 |
-
|
| 107 |
-
if len(chunks) > 3:
|
| 108 |
-
st.info(f"... and {len(chunks)-3} more chunks processed.")
|
| 109 |
-
|
| 110 |
else:
|
| 111 |
-
st.
|
| 112 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
# try:
|
| 114 |
# os.remove(temp_path)
|
| 115 |
# except Exception as e:
|
|
|
|
| 15 |
mongo_uri = f"mongodb+srv://{user}:{password}@{cluster}/{db_name}?retryWrites=true&w=majority&tls=true&tlsAllowInvalidCertificates=true"
|
| 16 |
client = MongoClient(mongo_uri, tls=True, tlsAllowInvalidCertificates=True, serverSelectionTimeoutMS=20000)
|
| 17 |
db = client[db_name]
|
| 18 |
+
st.set_page_config(page_title="Doc Chunker", layout="wide")
|
| 19 |
def gate_ui():
|
| 20 |
APP_PASSWORD=st.secrets.get("APP_PASSWORD", os.getenv("APP_PASSWORD")).strip()
|
| 21 |
if "authed" not in st.session_state:
|
|
|
|
| 36 |
st.error("Incorrect password.")
|
| 37 |
return False
|
| 38 |
# === Streamlit UI ===
|
| 39 |
+
def main():
|
| 40 |
+
if not gate_ui():
|
| 41 |
+
return
|
| 42 |
+
st.title("📄 Document Chunker & Uploader")
|
| 43 |
+
|
| 44 |
+
with st.sidebar:
|
| 45 |
+
st.header("Settings")
|
| 46 |
+
|
| 47 |
+
# Fetch collection names for dropdown
|
| 48 |
+
try:
|
| 49 |
+
existing_categories = db["final_chunks"].distinct("collection_category") or []
|
| 50 |
+
except Exception:
|
| 51 |
+
existing_categories = []
|
| 52 |
+
existing_categories=sorted([c for c in existing_categories if c])+["Create New Category"]
|
| 53 |
+
selected_category = st.selectbox(
|
| 54 |
+
"Choose Category (collection_category)",
|
| 55 |
+
existing_categories,
|
| 56 |
+
index=existing_categories.index("Create New Category") if "Create New Category" in existing_categories else 0
|
| 57 |
+
)
|
| 58 |
+
if selected_category == "Create New Category":
|
| 59 |
+
selected_category = st.sidebar.text_input("Enter Category Name:")
|
| 60 |
+
if not selected_category:
|
| 61 |
+
st.warning("⚠️ Enter a category name to proceed.")
|
| 62 |
+
st.stop()
|
| 63 |
+
|
| 64 |
+
is_grant_app = st.toggle("Is this a Grant Application?", value=False)
|
| 65 |
+
|
| 66 |
+
uploaded_file = st.file_uploader("Upload a DOCX, TXT, or PDF file", type=["docx", "txt", "pdf"])
|
| 67 |
+
|
| 68 |
+
if uploaded_file:
|
| 69 |
+
temp_path = Path(tempfile.gettempdir()) / uploaded_file.name
|
| 70 |
+
with open(temp_path, "wb") as f:
|
| 71 |
+
f.write(uploaded_file.getbuffer())
|
| 72 |
+
|
| 73 |
+
st.success(f"Uploaded `{uploaded_file.name}`")
|
| 74 |
+
|
| 75 |
+
modified_time = datetime.now().isoformat()
|
| 76 |
+
collection = db['final_chunks']
|
| 77 |
+
already = collection.find_one({
|
| 78 |
+
"metadata.title": uploaded_file.name,
|
| 79 |
+
"collection_category": selected_category
|
| 80 |
+
})
|
| 81 |
+
|
| 82 |
+
if already:
|
| 83 |
+
st.warning(f"⚠️ `{uploaded_file.name}` already exists in category `{selected_category}`. Skipping…")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
else:
|
| 85 |
+
st.write("⏳ Processing with DocumentChunker...")
|
| 86 |
+
chunker = DocumentChunker()
|
| 87 |
+
chunks = chunker.process_document(str(temp_path))
|
| 88 |
+
|
| 89 |
+
if chunks:
|
| 90 |
+
for chunk in chunks:
|
| 91 |
+
chunk['collection_category']=selected_category
|
| 92 |
+
chunk['metadata'].update({
|
| 93 |
+
"title": uploaded_file.name,
|
| 94 |
+
"uploaded_at": modified_time,
|
| 95 |
+
"is_grant_app": is_grant_app,
|
| 96 |
+
})
|
| 97 |
+
collection.insert_one(chunk)
|
| 98 |
+
|
| 99 |
+
st.success(f"✅ {len(chunks)} chunks inserted into `final_chunks` (category: `{selected_category}`)")
|
| 100 |
+
|
| 101 |
+
# Show a few previews
|
| 102 |
+
for i, c in enumerate(chunks[:3]):
|
| 103 |
+
st.subheader(f"Chunk {i+1}: {c['metadata'].get('header') or 'No Header'}")
|
| 104 |
+
st.markdown(c['text'][:400] + "...")
|
| 105 |
+
st.caption(f"Topics: {', '.join(c['metadata']['topics'])} | Category: {c['metadata']['category']}")
|
| 106 |
+
st.progress(c['metadata']['confidence_score'])
|
| 107 |
+
|
| 108 |
+
if len(chunks) > 3:
|
| 109 |
+
st.info(f"... and {len(chunks)-3} more chunks processed.")
|
| 110 |
+
|
| 111 |
+
else:
|
| 112 |
+
st.warning("⚠️ No chunks were generated.")
|
| 113 |
+
if __name__ == "__main__":
|
| 114 |
+
main()
|
| 115 |
# try:
|
| 116 |
# os.remove(temp_path)
|
| 117 |
# except Exception as e:
|