diff --git a/App_Function_Libraries/Article_Extractor_Lib.py b/App_Function_Libraries/Article_Extractor_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..7b6d92db162bf32d06c19991afd188889f66a42b --- /dev/null +++ b/App_Function_Libraries/Article_Extractor_Lib.py @@ -0,0 +1,107 @@ +# Article_Extractor_Lib.py +######################################### +# Article Extraction Library +# This library is used to handle scraping and extraction of articles from web pages. +# Currently, uses a combination of beatifulsoup4 and trafilatura to extract article text. +# Firecrawl would be a better option for this, but it is not yet implemented. +#### +# +#################### +# Function List +# +# 1. get_page_title(url) +# 2. get_article_text(url) +# 3. get_article_title(article_url_arg) +# +#################### +# +# Import necessary libraries +import logging +# 3rd-Party Imports +import asyncio +from playwright.async_api import async_playwright +from bs4 import BeautifulSoup +import requests +import trafilatura +# Import Local +# +####################################################################################################################### +# Function Definitions +# + +def get_page_title(url: str) -> str: + try: + response = requests.get(url) + response.raise_for_status() + soup = BeautifulSoup(response.text, 'html.parser') + title_tag = soup.find('title') + return title_tag.string.strip() if title_tag else "Untitled" + except requests.RequestException as e: + logging.error(f"Error fetching page title: {e}") + return "Untitled" + + +def get_artice_title(article_url_arg: str) -> str: + # Use beautifulsoup to get the page title - Really should be using ytdlp for this.... + article_title = get_page_title(article_url_arg) + + +def scrape_article(url): + async def fetch_html(url: str) -> str: + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + context = await browser.new_context( + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3") + page = await context.new_page() + await page.goto(url) + await page.wait_for_load_state("networkidle") # Wait for the network to be idle + content = await page.content() + await browser.close() + return content + + def extract_article_data(html: str) -> dict: + downloaded = trafilatura.extract(html, include_comments=False, include_tables=False, include_images=False) + if downloaded: + metadata = trafilatura.extract_metadata(html) + if metadata: + return { + 'title': metadata.title if metadata.title else 'N/A', + 'author': metadata.author if metadata.author else 'N/A', + 'content': downloaded, + 'date': metadata.date if metadata.date else 'N/A', + } + else: + print("Metadata extraction failed.") + return None + else: + print("Content extraction failed.") + return None + + def convert_html_to_markdown(html: str) -> str: + soup = BeautifulSoup(html, 'html.parser') + # Convert each paragraph to markdown + for para in soup.find_all('p'): + para.append('\n') # Add a newline at the end of each paragraph for markdown separation + + # Use .get_text() with separator to keep paragraph separation + text = soup.get_text(separator='\n\n') + + return text + + async def fetch_and_extract_article(url: str): + html = await fetch_html(url) + print("HTML Content:", html[:500]) # Print first 500 characters of the HTML for inspection + article_data = extract_article_data(html) + if article_data: + article_data['content'] = convert_html_to_markdown(article_data['content']) + return article_data + else: + return None + + # Using asyncio.run to handle event loop creation and execution + article_data = asyncio.run(fetch_and_extract_article(url)) + return article_data + +# +# +####################################################################################################################### \ No newline at end of file diff --git a/App_Function_Libraries/Article_Summarization_Lib.py b/App_Function_Libraries/Article_Summarization_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..58a87c22e074d370e20d601985c1d32d506e768e --- /dev/null +++ b/App_Function_Libraries/Article_Summarization_Lib.py @@ -0,0 +1,284 @@ +# Article_Summarization_Lib.py +######################################### +# Article Summarization Library +# This library is used to handle summarization of articles. + +# +#### +# +#################### +# Function List +# +# 1. +# +#################### +# +# Import necessary libraries +import datetime +from datetime import datetime +import gradio as gr +import json +import os +import logging +import requests +# 3rd-Party Imports +from tqdm import tqdm + +from App_Function_Libraries.Utils import sanitize_filename +# Local Imports +from Article_Extractor_Lib import scrape_article +from Local_Summarization_Lib import summarize_with_llama, summarize_with_oobabooga, summarize_with_tabbyapi, \ + summarize_with_vllm, summarize_with_kobold, save_summary_to_file, summarize_with_local_llm +from Summarization_General_Lib import summarize_with_openai, summarize_with_anthropic, summarize_with_cohere, summarize_with_groq, summarize_with_openrouter, summarize_with_deepseek, summarize_with_huggingface +from SQLite_DB import Database, create_tables, add_media_with_keywords +# +####################################################################################################################### +# Function Definitions +# + +def ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date, custom_prompt): + try: + # Check if content is not empty or whitespace + if not content.strip(): + raise ValueError("Content is empty.") + + db = Database() + create_tables() + keyword_list = keywords.split(',') if keywords else ["default"] + keyword_str = ', '.join(keyword_list) + + # Set default values for missing fields + url = url or 'Unknown' + title = title or 'Unknown' + author = author or 'Unknown' + keywords = keywords or 'default' + summary = summary or 'No summary available' + ingestion_date = ingestion_date or datetime.datetime.now().strftime('%Y-%m-%d') + + # Log the values of all fields before calling add_media_with_keywords + logging.debug(f"URL: {url}") + logging.debug(f"Title: {title}") + logging.debug(f"Author: {author}") + logging.debug(f"Content: {content[:50]}... (length: {len(content)})") # Log first 50 characters of content + logging.debug(f"Keywords: {keywords}") + logging.debug(f"Summary: {summary}") + logging.debug(f"Ingestion Date: {ingestion_date}") + logging.debug(f"Custom Prompt: {custom_prompt}") + + # Check if any required field is empty and log the specific missing field + if not url: + logging.error("URL is missing.") + raise ValueError("URL is missing.") + if not title: + logging.error("Title is missing.") + raise ValueError("Title is missing.") + if not content: + logging.error("Content is missing.") + raise ValueError("Content is missing.") + if not keywords: + logging.error("Keywords are missing.") + raise ValueError("Keywords are missing.") + if not summary: + logging.error("Summary is missing.") + raise ValueError("Summary is missing.") + if not ingestion_date: + logging.error("Ingestion date is missing.") + raise ValueError("Ingestion date is missing.") + if not custom_prompt: + logging.error("Custom prompt is missing.") + raise ValueError("Custom prompt is missing.") + + # Add media with keywords to the database + result = add_media_with_keywords( + url=url, + title=title, + media_type='article', + content=content, + keywords=keyword_str or "article_default", + prompt=custom_prompt or None, + summary=summary or "No summary generated", + transcription_model=None, # or some default value if applicable + author=author or 'Unknown', + ingestion_date=ingestion_date + ) + return result + except Exception as e: + logging.error(f"Failed to ingest article to the database: {e}") + return str(e) + + +def scrape_and_summarize_multiple(urls, custom_prompt_arg, api_name, api_key, keywords, custom_article_titles): + urls = [url.strip() for url in urls.split('\n') if url.strip()] + custom_titles = custom_article_titles.split('\n') if custom_article_titles else [] + + results = [] + errors = [] + + # Create a progress bar + progress = gr.Progress() + + for i, url in tqdm(enumerate(urls), total=len(urls), desc="Processing URLs"): + custom_title = custom_titles[i] if i < len(custom_titles) else None + try: + result = scrape_and_summarize(url, custom_prompt_arg, api_name, api_key, keywords, custom_title) + results.append(f"Results for URL {i + 1}:\n{result}") + except Exception as e: + error_message = f"Error processing URL {i + 1} ({url}): {str(e)}" + errors.append(error_message) + results.append(f"Failed to process URL {i + 1}: {url}") + + # Update progress + progress((i + 1) / len(urls), desc=f"Processed {i + 1}/{len(urls)} URLs") + + # Combine results and errors + combined_output = "\n".join(results) + if errors: + combined_output += "\n\nErrors encountered:\n" + "\n".join(errors) + + return combined_output + + +def scrape_and_summarize(url, custom_prompt_arg, api_name, api_key, keywords, custom_article_title): + try: + # Step 1: Scrape the article + article_data = scrape_article(url) + print(f"Scraped Article Data: {article_data}") # Debugging statement + if not article_data: + return "Failed to scrape the article." + + # Use the custom title if provided, otherwise use the scraped title + title = custom_article_title.strip() if custom_article_title else article_data.get('title', 'Untitled') + author = article_data.get('author', 'Unknown') + content = article_data.get('content', '') + ingestion_date = datetime.now().strftime('%Y-%m-%d') + + print(f"Title: {title}, Author: {author}, Content Length: {len(content)}") # Debugging statement + + # Custom prompt for the article + article_custom_prompt = custom_prompt_arg or "Summarize this article." + + # Step 2: Summarize the article + summary = None + if api_name: + logging.debug(f"Article_Summarizer: Summarization being performed by {api_name}") + + # Sanitize filename for saving the JSON file + sanitized_title = sanitize_filename(title) + json_file_path = os.path.join("Results", f"{sanitized_title}_segments.json") + + with open(json_file_path, 'w') as json_file: + json.dump([{'text': content}], json_file, indent=2) + + try: + if api_name.lower() == 'openai': + # def summarize_with_openai(api_key, input_data, custom_prompt_arg) + summary = summarize_with_openai(api_key, json_file_path, article_custom_prompt) + + elif api_name.lower() == "anthropic": + # def summarize_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5): + summary = summarize_with_anthropic(api_key, json_file_path, article_custom_prompt) + elif api_name.lower() == "cohere": + # def summarize_with_cohere(api_key, input_data, model, custom_prompt_arg) + summary = summarize_with_cohere(api_key, json_file_path, article_custom_prompt) + + elif api_name.lower() == "groq": + logging.debug(f"MAIN: Trying to summarize with groq") + # def summarize_with_groq(api_key, input_data, model, custom_prompt_arg): + summary = summarize_with_groq(api_key, json_file_path, article_custom_prompt) + + elif api_name.lower() == "openrouter": + logging.debug(f"MAIN: Trying to summarize with OpenRouter") + # def summarize_with_openrouter(api_key, input_data, custom_prompt_arg): + summary = summarize_with_openrouter(api_key, json_file_path, article_custom_prompt) + + elif api_name.lower() == "deepseek": + logging.debug(f"MAIN: Trying to summarize with DeepSeek") + # def summarize_with_deepseek(api_key, input_data, custom_prompt_arg): + summary = summarize_with_deepseek(api_key, json_file_path, article_custom_prompt) + + elif api_name.lower() == "llama.cpp": + logging.debug(f"MAIN: Trying to summarize with Llama.cpp") + # def summarize_with_llama(api_url, file_path, token, custom_prompt) + summary = summarize_with_llama(json_file_path, article_custom_prompt) + + elif api_name.lower() == "kobold": + logging.debug(f"MAIN: Trying to summarize with Kobold.cpp") + # def summarize_with_kobold(input_data, kobold_api_token, custom_prompt_input, api_url): + summary = summarize_with_kobold(json_file_path, api_key, article_custom_prompt) + + elif api_name.lower() == "ooba": + # def summarize_with_oobabooga(input_data, api_key, custom_prompt, api_url): + summary = summarize_with_oobabooga(json_file_path, api_key, article_custom_prompt) + + elif api_name.lower() == "tabbyapi": + # def summarize_with_tabbyapi(input_data, tabby_model, custom_prompt_input, api_key=None, api_IP): + summary = summarize_with_tabbyapi(json_file_path, article_custom_prompt) + + elif api_name.lower() == "vllm": + logging.debug(f"MAIN: Trying to summarize with VLLM") + # def summarize_with_vllm(api_key, input_data, custom_prompt_input): + summary = summarize_with_vllm(json_file_path, article_custom_prompt) + + elif api_name.lower() == "local-llm": + logging.debug(f"MAIN: Trying to summarize with Local LLM") + summary = summarize_with_local_llm(json_file_path, article_custom_prompt) + + elif api_name.lower() == "huggingface": + logging.debug(f"MAIN: Trying to summarize with huggingface") + # def summarize_with_huggingface(api_key, input_data, custom_prompt_arg): + summarize_with_huggingface(api_key, json_file_path, article_custom_prompt) + # Add additional API handlers here... + except requests.exceptions.ConnectionError as e: + logging.error(f"Connection error while trying to summarize with {api_name}: {str(e)}") + + if summary: + logging.info(f"Article_Summarizer: Summary generated using {api_name} API") + save_summary_to_file(summary, json_file_path) + else: + summary = "Summary not available" + logging.warning(f"Failed to generate summary using {api_name} API") + + else: + summary = "Article Summarization: No API provided for summarization." + + print(f"Summary: {summary}") # Debugging statement + + # Step 3: Ingest the article into the database + ingestion_result = ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date, + article_custom_prompt) + + return f"Title: {title}\nAuthor: {author}\nIngestion Result: {ingestion_result}\n\nSummary: {summary}\n\nArticle Contents: {content}" + except Exception as e: + logging.error(f"Error processing URL {url}: {str(e)}") + return f"Failed to process URL {url}: {str(e)}" + + +def ingest_unstructured_text(text, custom_prompt, api_name, api_key, keywords, custom_article_title): + title = custom_article_title.strip() if custom_article_title else "Unstructured Text" + author = "Unknown" + ingestion_date = datetime.now().strftime('%Y-%m-%d') + + # Summarize the unstructured text + if api_name: + json_file_path = f"Results/{title.replace(' ', '_')}_segments.json" + with open(json_file_path, 'w') as json_file: + json.dump([{'text': text}], json_file, indent=2) + + if api_name.lower() == 'openai': + summary = summarize_with_openai(api_key, json_file_path, custom_prompt) + # Add other APIs as needed + else: + summary = "Unsupported API." + else: + summary = "No API provided for summarization." + + # Ingest the unstructured text into the database + ingestion_result = ingest_article_to_db('Unstructured Text', title, author, text, keywords, summary, ingestion_date, + custom_prompt) + return f"Title: {title}\nSummary: {summary}\nIngestion Result: {ingestion_result}" + + + +# +# +####################################################################################################################### \ No newline at end of file diff --git a/App_Function_Libraries/Audio_Files.py b/App_Function_Libraries/Audio_Files.py new file mode 100644 index 0000000000000000000000000000000000000000..bb51099c97edc62a8db2e5111f6c8bddbf44bae4 --- /dev/null +++ b/App_Function_Libraries/Audio_Files.py @@ -0,0 +1,629 @@ +# Audio_Files.py +######################################### +# Audio Processing Library +# This library is used to download or load audio files from a local directory. +# +#### +# +# Functions: +# +# download_audio_file(url, save_path) +# process_audio( +# process_audio_file(audio_url, audio_file, whisper_model="small.en", api_name=None, api_key=None) +# +# +######################################### +# Imports +import json +import logging +import subprocess +import sys +import tempfile +import uuid +from datetime import datetime + +import requests +import os +from gradio import gradio +import yt_dlp + +from App_Function_Libraries.Audio_Transcription_Lib import speech_to_text +from App_Function_Libraries.Chunk_Lib import improved_chunking_process +# +# Local Imports +from App_Function_Libraries.SQLite_DB import add_media_to_database, add_media_with_keywords +from App_Function_Libraries.Utils import create_download_directory, save_segments_to_json +from App_Function_Libraries.Summarization_General_Lib import save_transcription_and_summary, perform_transcription, \ + perform_summarization +from App_Function_Libraries.Video_DL_Ingestion_Lib import extract_metadata + +# +####################################################################################################################### +# Function Definitions +# + +MAX_FILE_SIZE = 500 * 1024 * 1024 + + +def download_audio_file(url, use_cookies=False, cookies=None): + try: + # Set up the request headers + headers = {} + if use_cookies and cookies: + try: + cookie_dict = json.loads(cookies) + headers['Cookie'] = '; '.join([f'{k}={v}' for k, v in cookie_dict.items()]) + except json.JSONDecodeError: + logging.warning("Invalid cookie format. Proceeding without cookies.") + + # Make the request + response = requests.get(url, headers=headers, stream=True) + response.raise_for_status() # Raise an exception for bad status codes + + # Get the file size + file_size = int(response.headers.get('content-length', 0)) + if file_size > 500 * 1024 * 1024: # 500 MB limit + raise ValueError("File size exceeds the 500MB limit.") + + # Generate a unique filename + file_name = f"audio_{uuid.uuid4().hex[:8]}.mp3" + save_path = os.path.join('downloads', file_name) + + # Ensure the downloads directory exists + os.makedirs('downloads', exist_ok=True) + + # Download the file + with open(save_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + + logging.info(f"Audio file downloaded successfully: {save_path}") + return save_path + + except requests.RequestException as e: + logging.error(f"Error downloading audio file: {str(e)}") + raise + except ValueError as e: + logging.error(str(e)) + raise + except Exception as e: + logging.error(f"Unexpected error downloading audio file: {str(e)}") + raise + + +def process_audio( + audio_file_path, + num_speakers=2, + whisper_model="small.en", + custom_prompt_input=None, + offset=0, + api_name=None, + api_key=None, + vad_filter=False, + rolling_summarization=False, + detail_level=0.01, + keywords="default,no_keyword_set", + chunk_text_by_words=False, + max_words=0, + chunk_text_by_sentences=False, + max_sentences=0, + chunk_text_by_paragraphs=False, + max_paragraphs=0, + chunk_text_by_tokens=False, + max_tokens=0 +): + try: + + # Perform transcription + audio_file_path, segments = perform_transcription(audio_file_path, offset, whisper_model, vad_filter) + + if audio_file_path is None or segments is None: + logging.error("Process_Audio: Transcription failed or segments not available.") + return "Process_Audio: Transcription failed.", None, None, None, None, None + + logging.debug(f"Process_Audio: Transcription audio_file: {audio_file_path}") + logging.debug(f"Process_Audio: Transcription segments: {segments}") + + transcription_text = {'audio_file': audio_file_path, 'transcription': segments} + logging.debug(f"Process_Audio: Transcription text: {transcription_text}") + + # Save segments to JSON + segments_json_path = save_segments_to_json(segments) + + # Perform summarization + summary_text = None + if api_name: + if rolling_summarization is not None: + pass + # FIXME rolling summarization + # summary_text = rolling_summarize_function( + # transcription_text, + # detail=detail_level, + # api_name=api_name, + # api_key=api_key, + # custom_prompt=custom_prompt_input, + # chunk_by_words=chunk_text_by_words, + # max_words=max_words, + # chunk_by_sentences=chunk_text_by_sentences, + # max_sentences=max_sentences, + # chunk_by_paragraphs=chunk_text_by_paragraphs, + # max_paragraphs=max_paragraphs, + # chunk_by_tokens=chunk_text_by_tokens, + # max_tokens=max_tokens + # ) + else: + summary_text = perform_summarization(api_name, segments_json_path, custom_prompt_input, api_key) + + if summary_text is None: + logging.error("Summary text is None. Check summarization function.") + summary_file_path = None + else: + summary_text = 'Summary not available' + summary_file_path = None + + # Save transcription and summary + download_path = create_download_directory("Audio_Processing") + json_file_path, summary_file_path = save_transcription_and_summary(transcription_text, summary_text, + download_path) + + # Update function call to add_media_to_database so that it properly applies the title, author and file type + # Add to database + add_media_to_database(None, {'title': 'Audio File', 'author': 'Unknown'}, segments, summary_text, keywords, + custom_prompt_input, whisper_model) + + return transcription_text, summary_text, json_file_path, summary_file_path, None, None + + except Exception as e: + logging.error(f"Error in process_audio: {str(e)}") + return str(e), None, None, None, None, None + + +def process_single_audio(audio_file_path, whisper_model, api_name, api_key, keep_original,custom_keywords, source, + custom_prompt_input, chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking, + use_multi_level_chunking, chunk_language): + progress = [] + transcription = "" + summary = "" + + def update_progress(message): + progress.append(message) + return "\n".join(progress) + + try: + # Check file size before processing + file_size = os.path.getsize(audio_file_path) + if file_size > MAX_FILE_SIZE: + update_progress(f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds the maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f} MB. Skipping this file.") + return "\n".join(progress), "", "" + + # Perform transcription + update_progress("Starting transcription...") + segments = speech_to_text(audio_file_path, whisper_model=whisper_model) + transcription = " ".join([segment['Text'] for segment in segments]) + update_progress("Audio transcribed successfully.") + + # Perform summarization if API is provided + if api_name and api_key: + update_progress("Starting summarization...") + summary = perform_summarization(api_name, transcription, "Summarize the following audio transcript", + api_key) + update_progress("Audio summarized successfully.") + else: + summary = "No summary available" + + # Prepare keywords + keywords = "audio,transcription" + if custom_keywords: + keywords += f",{custom_keywords}" + + # Add to database + add_media_with_keywords( + url=source, + title=os.path.basename(audio_file_path), + media_type='audio', + content=transcription, + keywords=keywords, + prompt="Summarize the following audio transcript", + summary=summary, + transcription_model=whisper_model, + author="Unknown", + ingestion_date=None # This will use the current date + ) + update_progress("Audio file added to database successfully.") + + if not keep_original and source != "Uploaded File": + os.remove(audio_file_path) + update_progress(f"Temporary file {audio_file_path} removed.") + elif keep_original and source != "Uploaded File": + update_progress(f"Original audio file kept at: {audio_file_path}") + + except Exception as e: + update_progress(f"Error processing {source}: {str(e)}") + transcription = f"Error: {str(e)}" + summary = "No summary due to error" + + return "\n".join(progress), transcription, summary + + +def process_audio_files(audio_urls, audio_file, whisper_model, api_name, api_key, use_cookies, cookies, keep_original, + custom_keywords, custom_prompt_input, chunk_method, max_chunk_size, chunk_overlap, + use_adaptive_chunking, use_multi_level_chunking, chunk_language, diarize): + progress = [] + temp_files = [] + all_transcriptions = [] + all_summaries = [] + + def update_progress(message): + progress.append(message) + return "\n".join(progress) + + def cleanup_files(): + for file in temp_files: + try: + if os.path.exists(file): + os.remove(file) + update_progress(f"Temporary file {file} removed.") + except Exception as e: + update_progress(f"Failed to remove temporary file {file}: {str(e)}") + + def reencode_mp3(mp3_file_path): + try: + reencoded_mp3_path = mp3_file_path.replace(".mp3", "_reencoded.mp3") + subprocess.run([ffmpeg_cmd, '-i', mp3_file_path, '-codec:a', 'libmp3lame', reencoded_mp3_path], check=True) + update_progress(f"Re-encoded {mp3_file_path} to {reencoded_mp3_path}.") + return reencoded_mp3_path + except subprocess.CalledProcessError as e: + update_progress(f"Error re-encoding {mp3_file_path}: {str(e)}") + raise + + def convert_mp3_to_wav(mp3_file_path): + try: + wav_file_path = mp3_file_path.replace(".mp3", ".wav") + subprocess.run([ffmpeg_cmd, '-i', mp3_file_path, wav_file_path], check=True) + update_progress(f"Converted {mp3_file_path} to {wav_file_path}.") + return wav_file_path + except subprocess.CalledProcessError as e: + update_progress(f"Error converting {mp3_file_path} to WAV: {str(e)}") + raise + + try: + # Check and set the ffmpeg command + global ffmpeg_cmd + if os.name == "nt": + logging.debug("Running on Windows") + ffmpeg_cmd = os.path.join(os.getcwd(), "Bin", "ffmpeg.exe") + else: + ffmpeg_cmd = 'ffmpeg' # Assume 'ffmpeg' is in PATH for non-Windows systems + + # Ensure ffmpeg is accessible + if not os.path.exists(ffmpeg_cmd) and os.name == "nt": + raise FileNotFoundError(f"ffmpeg executable not found at path: {ffmpeg_cmd}") + + # Define chunk options early to avoid undefined errors + chunk_options = { + 'method': chunk_method, + 'max_size': max_chunk_size, + 'overlap': chunk_overlap, + 'adaptive': use_adaptive_chunking, + 'multi_level': use_multi_level_chunking, + 'language': chunk_language + } + + # Process multiple URLs + urls = [url.strip() for url in audio_urls.split('\n') if url.strip()] + + for i, url in enumerate(urls): + update_progress(f"Processing URL {i + 1}/{len(urls)}: {url}") + + # Download and process audio file + audio_file_path = download_audio_file(url, use_cookies, cookies) + if not os.path.exists(audio_file_path): + update_progress(f"Downloaded file not found: {audio_file_path}") + continue + + temp_files.append(audio_file_path) + update_progress("Audio file downloaded successfully.") + + # Re-encode MP3 to fix potential issues + reencoded_mp3_path = reencode_mp3(audio_file_path) + if not os.path.exists(reencoded_mp3_path): + update_progress(f"Re-encoded file not found: {reencoded_mp3_path}") + continue + + temp_files.append(reencoded_mp3_path) + + # Convert re-encoded MP3 to WAV + wav_file_path = convert_mp3_to_wav(reencoded_mp3_path) + if not os.path.exists(wav_file_path): + update_progress(f"Converted WAV file not found: {wav_file_path}") + continue + + temp_files.append(wav_file_path) + + # Initialize transcription + transcription = "" + + # Transcribe audio + if diarize: + segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True) + else: + segments = speech_to_text(wav_file_path, whisper_model=whisper_model) + + # Handle segments nested under 'segments' key + if isinstance(segments, dict) and 'segments' in segments: + segments = segments['segments'] + + if isinstance(segments, list): + transcription = " ".join([segment.get('Text', '') for segment in segments]) + update_progress("Audio transcribed successfully.") + else: + update_progress("Unexpected segments format received from speech_to_text.") + logging.error(f"Unexpected segments format: {segments}") + continue + + if not transcription.strip(): + update_progress("Transcription is empty.") + else: + # Apply chunking + chunked_text = improved_chunking_process(transcription, chunk_options) + + # Summarize + if api_name: + try: + summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key) + update_progress("Audio summarized successfully.") + except Exception as e: + logging.error(f"Error during summarization: {str(e)}") + summary = "Summary generation failed" + else: + summary = "No summary available (API not provided)" + + all_transcriptions.append(transcription) + all_summaries.append(summary) + + # Add to database + add_media_with_keywords( + url=url, + title=os.path.basename(wav_file_path), + media_type='audio', + content=transcription, + keywords=custom_keywords, + prompt=custom_prompt_input, + summary=summary, + transcription_model=whisper_model, + author="Unknown", + ingestion_date=datetime.now().strftime('%Y-%m-%d') + ) + update_progress("Audio file processed and added to database.") + + # Process uploaded file if provided + if audio_file: + if os.path.getsize(audio_file.name) > MAX_FILE_SIZE: + update_progress( + f"Uploaded file size exceeds the maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f}MB. Skipping this file.") + else: + # Re-encode MP3 to fix potential issues + reencoded_mp3_path = reencode_mp3(audio_file.name) + if not os.path.exists(reencoded_mp3_path): + update_progress(f"Re-encoded file not found: {reencoded_mp3_path}") + return update_progress("Processing failed: Re-encoded file not found"), "", "" + + temp_files.append(reencoded_mp3_path) + + # Convert re-encoded MP3 to WAV + wav_file_path = convert_mp3_to_wav(reencoded_mp3_path) + if not os.path.exists(wav_file_path): + update_progress(f"Converted WAV file not found: {wav_file_path}") + return update_progress("Processing failed: Converted WAV file not found"), "", "" + + temp_files.append(wav_file_path) + + # Initialize transcription + transcription = "" + + if diarize: + segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True) + else: + segments = speech_to_text(wav_file_path, whisper_model=whisper_model) + + # Handle segments nested under 'segments' key + if isinstance(segments, dict) and 'segments' in segments: + segments = segments['segments'] + + if isinstance(segments, list): + transcription = " ".join([segment.get('Text', '') for segment in segments]) + else: + update_progress("Unexpected segments format received from speech_to_text.") + logging.error(f"Unexpected segments format: {segments}") + + chunked_text = improved_chunking_process(transcription, chunk_options) + + if api_name and api_key: + try: + summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key) + update_progress("Audio summarized successfully.") + except Exception as e: + logging.error(f"Error during summarization: {str(e)}") + summary = "Summary generation failed" + else: + summary = "No summary available (API not provided)" + + all_transcriptions.append(transcription) + all_summaries.append(summary) + + add_media_with_keywords( + url="Uploaded File", + title=os.path.basename(wav_file_path), + media_type='audio', + content=transcription, + keywords=custom_keywords, + prompt=custom_prompt_input, + summary=summary, + transcription_model=whisper_model, + author="Unknown", + ingestion_date=datetime.now().strftime('%Y-%m-%d') + ) + update_progress("Uploaded file processed and added to database.") + + # Final cleanup + if not keep_original: + cleanup_files() + + final_progress = update_progress("All processing complete.") + final_transcriptions = "\n\n".join(all_transcriptions) + final_summaries = "\n\n".join(all_summaries) + + return final_progress, final_transcriptions, final_summaries + + except Exception as e: + logging.error(f"Error processing audio files: {str(e)}") + cleanup_files() + return update_progress(f"Processing failed: {str(e)}"), "", "" + + +def download_youtube_audio(url: str) -> str: + ydl_opts = { + 'format': 'bestaudio/best', + 'postprocessors': [{ + 'key': 'FFmpegExtractAudio', + 'preferredcodec': 'wav', + 'preferredquality': '192', + }], + 'outtmpl': '%(title)s.%(ext)s' + } + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + info = ydl.extract_info(url, download=True) + filename = ydl.prepare_filename(info) + return filename.rsplit('.', 1)[0] + '.wav' + + +def process_podcast(url, title, author, keywords, custom_prompt, api_name, api_key, whisper_model, + keep_original=False, enable_diarization=False, use_cookies=False, cookies=None, + chunk_method=None, max_chunk_size=300, chunk_overlap=0, use_adaptive_chunking=False, + use_multi_level_chunking=False, chunk_language='english'): + progress = [] + error_message = "" + temp_files = [] + + def update_progress(message): + progress.append(message) + return "\n".join(progress) + + def cleanup_files(): + if not keep_original: + for file in temp_files: + try: + if os.path.exists(file): + os.remove(file) + update_progress(f"Temporary file {file} removed.") + except Exception as e: + update_progress(f"Failed to remove temporary file {file}: {str(e)}") + + try: + # Download podcast + audio_file = download_audio_file(url, use_cookies, cookies) + temp_files.append(audio_file) + update_progress("Podcast downloaded successfully.") + + # Extract metadata + metadata = extract_metadata(url) + title = title or metadata.get('title', 'Unknown Podcast') + author = author or metadata.get('uploader', 'Unknown Author') + + # Format metadata for storage + metadata_text = f""" +Metadata: +Title: {title} +Author: {author} +Series: {metadata.get('series', 'N/A')} +Episode: {metadata.get('episode', 'N/A')} +Season: {metadata.get('season', 'N/A')} +Upload Date: {metadata.get('upload_date', 'N/A')} +Duration: {metadata.get('duration', 'N/A')} seconds +Description: {metadata.get('description', 'N/A')} +""" + + # Update keywords + new_keywords = [] + if metadata.get('series'): + new_keywords.append(f"series:{metadata['series']}") + if metadata.get('episode'): + new_keywords.append(f"episode:{metadata['episode']}") + if metadata.get('season'): + new_keywords.append(f"season:{metadata['season']}") + + keywords = f"{keywords},{','.join(new_keywords)}" if keywords else ','.join(new_keywords) + + update_progress(f"Metadata extracted - Title: {title}, Author: {author}, Keywords: {keywords}") + + # Transcribe the podcast + try: + if enable_diarization: + segments = speech_to_text(audio_file, whisper_model=whisper_model, diarize=True) + else: + segments = speech_to_text(audio_file, whisper_model=whisper_model) + transcription = " ".join([segment['Text'] for segment in segments]) + update_progress("Podcast transcribed successfully.") + except Exception as e: + error_message = f"Transcription failed: {str(e)}" + raise + + # Apply chunking + chunk_options = { + 'method': chunk_method, + 'max_size': max_chunk_size, + 'overlap': chunk_overlap, + 'adaptive': use_adaptive_chunking, + 'multi_level': use_multi_level_chunking, + 'language': chunk_language + } + chunked_text = improved_chunking_process(transcription, chunk_options) + + # Combine metadata and transcription + full_content = metadata_text + "\n\nTranscription:\n" + transcription + + # Summarize if API is provided + summary = None + if api_name and api_key: + try: + summary = perform_summarization(api_name, chunked_text, custom_prompt, api_key) + update_progress("Podcast summarized successfully.") + except Exception as e: + error_message = f"Summarization failed: {str(e)}" + raise + + # Add to database + try: + add_media_with_keywords( + url=url, + title=title, + media_type='podcast', + content=full_content, + keywords=keywords, + prompt=custom_prompt, + summary=summary or "No summary available", + transcription_model=whisper_model, + author=author, + ingestion_date=datetime.now().strftime('%Y-%m-%d') + ) + update_progress("Podcast added to database successfully.") + except Exception as e: + error_message = f"Error adding podcast to database: {str(e)}" + raise + + # Cleanup + cleanup_files() + + return (update_progress("Processing complete."), full_content, summary or "No summary generated.", + title, author, keywords, error_message) + + except Exception as e: + logging.error(f"Error processing podcast: {str(e)}") + cleanup_files() + return update_progress(f"Processing failed: {str(e)}"), "", "", "", "", "", str(e) + + +# +# +####################################################################################################################### \ No newline at end of file diff --git a/App_Function_Libraries/Audio_Transcription_Lib.py b/App_Function_Libraries/Audio_Transcription_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..d4a703f18b272026db64683331b0563cb8b81ba4 --- /dev/null +++ b/App_Function_Libraries/Audio_Transcription_Lib.py @@ -0,0 +1,158 @@ +# Audio_Transcription_Lib.py +######################################### +# Transcription Library +# This library is used to perform transcription of audio files. +# Currently, uses faster_whisper for transcription. +# +#### +import configparser +#################### +# Function List +# +# 1. convert_to_wav(video_file_path, offset=0, overwrite=False) +# 2. speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='small.en', vad_filter=False) +# +#################### +# +# Import necessary libraries to run solo for testing +import json +import logging +import os +import sys +import subprocess +import time + +# Import Local +# +####################################################################################################################### +# Function Definitions +# + +# Convert video .m4a into .wav using ffmpeg +# ffmpeg -i "example.mp4" -ar 16000 -ac 1 -c:a pcm_s16le "output.wav" +# https://www.gyan.dev/ffmpeg/builds/ +# + + +# os.system(r'.\Bin\ffmpeg.exe -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"') +def convert_to_wav(video_file_path, offset=0, overwrite=False): + out_path = os.path.splitext(video_file_path)[0] + ".wav" + + if os.path.exists(out_path) and not overwrite: + print(f"File '{out_path}' already exists. Skipping conversion.") + logging.info(f"Skipping conversion as file already exists: {out_path}") + return out_path + print("Starting conversion process of .m4a to .WAV") + out_path = os.path.splitext(video_file_path)[0] + ".wav" + + try: + if os.name == "nt": + logging.debug("ffmpeg being ran on windows") + + if sys.platform.startswith('win'): + ffmpeg_cmd = ".\\Bin\\ffmpeg.exe" + logging.debug(f"ffmpeg_cmd: {ffmpeg_cmd}") + else: + ffmpeg_cmd = 'ffmpeg' # Assume 'ffmpeg' is in PATH for non-Windows systems + + command = [ + ffmpeg_cmd, # Assuming the working directory is correctly set where .\Bin exists + "-ss", "00:00:00", # Start at the beginning of the video + "-i", video_file_path, + "-ar", "16000", # Audio sample rate + "-ac", "1", # Number of audio channels + "-c:a", "pcm_s16le", # Audio codec + out_path + ] + try: + # Redirect stdin from null device to prevent ffmpeg from waiting for input + with open(os.devnull, 'rb') as null_file: + result = subprocess.run(command, stdin=null_file, text=True, capture_output=True) + if result.returncode == 0: + logging.info("FFmpeg executed successfully") + logging.debug("FFmpeg output: %s", result.stdout) + else: + logging.error("Error in running FFmpeg") + logging.error("FFmpeg stderr: %s", result.stderr) + raise RuntimeError(f"FFmpeg error: {result.stderr}") + except Exception as e: + logging.error("Error occurred - ffmpeg doesn't like windows") + raise RuntimeError("ffmpeg failed") + elif os.name == "posix": + os.system(f'ffmpeg -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"') + else: + raise RuntimeError("Unsupported operating system") + logging.info("Conversion to WAV completed: %s", out_path) + except subprocess.CalledProcessError as e: + logging.error("Error executing FFmpeg command: %s", str(e)) + raise RuntimeError("Error converting video file to WAV") + except Exception as e: + logging.error("speech-to-text: Error transcribing audio: %s", str(e)) + return {"error": str(e)} + return out_path + + +# Transcribe .wav into .segments.json +def speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='medium.en', vad_filter=False, diarize=False): + logging.info('speech-to-text: Loading faster_whisper model: %s', whisper_model) + from faster_whisper import WhisperModel + # Retrieve processing choice from the configuration file + config = configparser.ConfigParser() + config.read('config.txt') + processing_choice = config.get('Processing', 'processing_choice', fallback='cpu') + model = WhisperModel(whisper_model, device=f"{processing_choice}") + time_start = time.time() + if audio_file_path is None: + raise ValueError("speech-to-text: No audio file provided") + logging.info("speech-to-text: Audio file path: %s", audio_file_path) + + try: + _, file_ending = os.path.splitext(audio_file_path) + out_file = audio_file_path.replace(file_ending, ".segments.json") + prettified_out_file = audio_file_path.replace(file_ending, ".segments_pretty.json") + if os.path.exists(out_file): + logging.info("speech-to-text: Segments file already exists: %s", out_file) + with open(out_file) as f: + global segments + segments = json.load(f) + return segments + + logging.info('speech-to-text: Starting transcription...') + options = dict(language=selected_source_lang, beam_size=5, best_of=5, vad_filter=vad_filter) + transcribe_options = dict(task="transcribe", **options) + segments_raw, info = model.transcribe(audio_file_path, **transcribe_options) + + segments = [] + for segment_chunk in segments_raw: + chunk = { + "Time_Start": segment_chunk.start, + "Time_End": segment_chunk.end, + "Text": segment_chunk.text + } + logging.debug("Segment: %s", chunk) + segments.append(chunk) + if not segments: + raise RuntimeError("No transcription produced. The audio file may be invalid or empty.") + logging.info("speech-to-text: Transcription completed in %.2f seconds", time.time() - time_start) + + # Create a dictionary with the 'segments' key + output_data = {'segments': segments} + + # Save prettified JSON + logging.info("speech-to-text: Saving prettified JSON to %s", prettified_out_file) + with open(prettified_out_file, 'w') as f: + json.dump(output_data, f, indent=2) + + # Save non-prettified JSON + logging.info("speech-to-text: Saving JSON to %s", out_file) + with open(out_file, 'w') as f: + json.dump(output_data, f) + + except Exception as e: + logging.error("speech-to-text: Error transcribing audio: %s", str(e)) + raise RuntimeError("speech-to-text: Error transcribing audio") + return segments + +# +# +####################################################################################################################### \ No newline at end of file diff --git a/App_Function_Libraries/Book_Ingestion_Lib.py b/App_Function_Libraries/Book_Ingestion_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..b667b4682881bf7749ef2407bc1e89c96316f906 --- /dev/null +++ b/App_Function_Libraries/Book_Ingestion_Lib.py @@ -0,0 +1,95 @@ +# Book_Ingestion_Lib.py +######################################### +# Library to hold functions for ingesting book files.# +# +#################### +# Function List +# +# 1. ingest_text_file(file_path, title=None, author=None, keywords=None): +# 2. +# +# +#################### + + +# Import necessary libraries +import os +import re +from datetime import datetime +import logging + + +# Import Local +from SQLite_DB import add_media_with_keywords + +####################################################################################################################### +# Function Definitions +# + +# Ingest a text file into the database with Title/Author/Keywords + +def extract_epub_metadata(content): + title_match = re.search(r'Title:\s*(.*?)\n', content) + author_match = re.search(r'Author:\s*(.*?)\n', content) + + title = title_match.group(1) if title_match else None + author = author_match.group(1) if author_match else None + + return title, author + + +def ingest_text_file(file_path, title=None, author=None, keywords=None): + try: + with open(file_path, 'r', encoding='utf-8') as file: + content = file.read() + + # Check if it's a converted epub and extract metadata if so + if 'epub_converted' in (keywords or ''): + extracted_title, extracted_author = extract_epub_metadata(content) + title = title or extracted_title + author = author or extracted_author + + # If title is still not provided, use the filename without extension + if not title: + title = os.path.splitext(os.path.basename(file_path))[0] + + # If author is still not provided, set it to 'Unknown' + if not author: + author = 'Unknown' + + # If keywords are not provided, use a default keyword + if not keywords: + keywords = 'text_file,epub_converted' + else: + keywords = f'text_file,epub_converted,{keywords}' + + # Add the text file to the database + add_media_with_keywords( + url=file_path, + title=title, + media_type='document', + content=content, + keywords=keywords, + prompt='No prompt for text files', + summary='No summary for text files', + transcription_model='None', + author=author, + ingestion_date=datetime.now().strftime('%Y-%m-%d') + ) + + return f"Text file '{title}' by {author} ingested successfully." + except Exception as e: + logging.error(f"Error ingesting text file: {str(e)}") + return f"Error ingesting text file: {str(e)}" + + +def ingest_folder(folder_path, keywords=None): + results = [] + for filename in os.listdir(folder_path): + if filename.lower().endswith('.txt'): + file_path = os.path.join(folder_path, filename) + result = ingest_text_file(file_path, keywords=keywords) + results.append(result) + + + diff --git a/App_Function_Libraries/Chunk_Lib.py b/App_Function_Libraries/Chunk_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..39abc998b7a07d3892102a4f43f7d764ccd1d5b8 --- /dev/null +++ b/App_Function_Libraries/Chunk_Lib.py @@ -0,0 +1,467 @@ +# Chunk_Lib.py +######################################### +# Chunking Library +# This library is used to perform chunking of input files. +# Currently, uses naive approaches. Nothing fancy. +# +#### +# Import necessary libraries +import logging +import re + +from typing import List, Optional, Tuple, Dict, Any + +from openai import OpenAI +from tqdm import tqdm +# +# Import 3rd party +from transformers import GPT2Tokenizer +import nltk +from nltk.tokenize import sent_tokenize, word_tokenize +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.metrics.pairwise import cosine_similarity +# +# Import Local +from App_Function_Libraries.Tokenization_Methods_Lib import openai_tokenize +from App_Function_Libraries.Utils import load_comprehensive_config + + +# +####################################################################################################################### +# Function Definitions +# + +# FIXME - Make sure it only downloads if it already exists, and does a check first. +# Ensure NLTK data is downloaded +def ntlk_prep(): + nltk.download('punkt') + +# Load GPT2 tokenizer +tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + +# Load Config file for API keys +config = load_comprehensive_config() +openai_api_key = config.get('API', 'openai_api_key', fallback=None) + +def load_document(file_path): + with open(file_path, 'r') as file: + text = file.read() + return re.sub('\\s+', ' ', text).strip() + + +def improved_chunking_process(text: str, chunk_options: Dict[str, Any]) -> List[Dict[str, Any]]: + chunk_method = chunk_options.get('method', 'words') + max_chunk_size = chunk_options.get('max_size', 300) + overlap = chunk_options.get('overlap', 0) + language = chunk_options.get('language', 'english') + adaptive = chunk_options.get('adaptive', False) + multi_level = chunk_options.get('multi_level', False) + + if adaptive: + max_chunk_size = adaptive_chunk_size(text, max_chunk_size) + + if multi_level: + chunks = multi_level_chunking(text, chunk_method, max_chunk_size, overlap, language) + else: + if chunk_method == 'words': + chunks = chunk_text_by_words(text, max_chunk_size, overlap) + elif chunk_method == 'sentences': + chunks = chunk_text_by_sentences(text, max_chunk_size, overlap, language) + elif chunk_method == 'paragraphs': + chunks = chunk_text_by_paragraphs(text, max_chunk_size, overlap) + elif chunk_method == 'tokens': + chunks = chunk_text_by_tokens(text, max_chunk_size, overlap) + else: + chunks = [text] # No chunking applied + + return [{'text': chunk, 'metadata': get_chunk_metadata(chunk, text)} for chunk in chunks] + + +def adaptive_chunk_size(text: str, base_size: int) -> int: + # Simple adaptive logic: adjust chunk size based on text complexity + avg_word_length = sum(len(word) for word in text.split()) / len(text.split()) + if avg_word_length > 6: # Arbitrary threshold for "complex" text + return int(base_size * 0.8) # Reduce chunk size for complex text + return base_size + + +def multi_level_chunking(text: str, method: str, max_size: int, overlap: int, language: str) -> List[str]: + # First level: chunk by paragraphs + paragraphs = chunk_text_by_paragraphs(text, max_size * 2, overlap) + + # Second level: chunk each paragraph further + chunks = [] + for para in paragraphs: + if method == 'words': + chunks.extend(chunk_text_by_words(para, max_size, overlap)) + elif method == 'sentences': + chunks.extend(chunk_text_by_sentences(para, max_size, overlap, language)) + else: + chunks.append(para) + + return chunks + + +def chunk_text_by_words(text: str, max_words: int = 300, overlap: int = 0) -> List[str]: + words = text.split() + chunks = [] + for i in range(0, len(words), max_words - overlap): + chunk = ' '.join(words[i:i + max_words]) + chunks.append(chunk) + return post_process_chunks(chunks) + + +def chunk_text_by_sentences(text: str, max_sentences: int = 10, overlap: int = 0, language: str = 'english') -> List[ + str]: + nltk.download('punkt', quiet=True) + sentences = nltk.sent_tokenize(text, language=language) + chunks = [] + for i in range(0, len(sentences), max_sentences - overlap): + chunk = ' '.join(sentences[i:i + max_sentences]) + chunks.append(chunk) + return post_process_chunks(chunks) + + +def chunk_text_by_paragraphs(text: str, max_paragraphs: int = 5, overlap: int = 0) -> List[str]: + paragraphs = re.split(r'\n\s*\n', text) + chunks = [] + for i in range(0, len(paragraphs), max_paragraphs - overlap): + chunk = '\n\n'.join(paragraphs[i:i + max_paragraphs]) + chunks.append(chunk) + return post_process_chunks(chunks) + + +def chunk_text_by_tokens(text: str, max_tokens: int = 1000, overlap: int = 0) -> List[str]: + # This is a simplified token-based chunking. For more accurate tokenization, + # consider using a proper tokenizer like GPT-2 TokenizerFast + words = text.split() + chunks = [] + current_chunk = [] + current_token_count = 0 + + for word in words: + word_token_count = len(word) // 4 + 1 # Rough estimate of token count + if current_token_count + word_token_count > max_tokens and current_chunk: + chunks.append(' '.join(current_chunk)) + current_chunk = current_chunk[-overlap:] if overlap > 0 else [] + current_token_count = sum(len(w) // 4 + 1 for w in current_chunk) + + current_chunk.append(word) + current_token_count += word_token_count + + if current_chunk: + chunks.append(' '.join(current_chunk)) + + return post_process_chunks(chunks) + + +def post_process_chunks(chunks: List[str]) -> List[str]: + return [chunk.strip() for chunk in chunks if chunk.strip()] + + +def get_chunk_metadata(chunk: str, full_text: str) -> Dict[str, Any]: + start_index = full_text.index(chunk) + return { + 'start_index': start_index, + 'end_index': start_index + len(chunk), + 'word_count': len(chunk.split()), + 'char_count': len(chunk) + } + + +# Hybrid approach, chunk each sentence while ensuring total token size does not exceed a maximum number +def chunk_text_hybrid(text, max_tokens=1000): + sentences = nltk.tokenize.sent_tokenize(text) + chunks = [] + current_chunk = [] + current_length = 0 + + for sentence in sentences: + tokens = tokenizer.encode(sentence) + if current_length + len(tokens) <= max_tokens: + current_chunk.append(sentence) + current_length += len(tokens) + else: + chunks.append(' '.join(current_chunk)) + current_chunk = [sentence] + current_length = len(tokens) + + if current_chunk: + chunks.append(' '.join(current_chunk)) + + return chunks + +# Thanks openai +def chunk_on_delimiter(input_string: str, + max_tokens: int, + delimiter: str) -> List[str]: + chunks = input_string.split(delimiter) + combined_chunks, _, dropped_chunk_count = combine_chunks_with_no_minimum( + chunks, max_tokens, chunk_delimiter=delimiter, add_ellipsis_for_overflow=True) + if dropped_chunk_count > 0: + print(f"Warning: {dropped_chunk_count} chunks were dropped due to exceeding the token limit.") + combined_chunks = [f"{chunk}{delimiter}" for chunk in combined_chunks] + return combined_chunks + + +def recursive_summarize_chunks(chunks, summarize_func, custom_prompt): + summarized_chunks = [] + current_summary = "" + + for i, chunk in enumerate(chunks): + if i == 0: + current_summary = summarize_func(chunk, custom_prompt) + else: + combined_text = current_summary + "\n\n" + chunk + current_summary = summarize_func(combined_text, custom_prompt) + + summarized_chunks.append(current_summary) + + return summarized_chunks + + +# Sample text for testing +sample_text = """ +Natural language processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence +concerned with the interactions between computers and human language, in particular how to program computers +to process and analyze large amounts of natural language data. The result is a computer capable of "understanding" +the contents of documents, including the contextual nuances of the language within them. The technology can then +accurately extract information and insights contained in the documents as well as categorize and organize the documents themselves. + +Challenges in natural language processing frequently involve speech recognition, natural language understanding, +and natural language generation. + +Natural language processing has its roots in the 1950s. Already in 1950, Alan Turing published an article titled +"Computing Machinery and Intelligence" which proposed what is now called the Turing test as a criterion of intelligence. +""" + +# Example usage of different chunking methods +# print("Chunking by words:") +# print(chunk_text_by_words(sample_text, max_words=50)) +# +# print("\nChunking by sentences:") +# print(chunk_text_by_sentences(sample_text, max_sentences=2)) +# +# print("\nChunking by paragraphs:") +# print(chunk_text_by_paragraphs(sample_text, max_paragraphs=1)) +# +# print("\nChunking by tokens:") +# print(chunk_text_by_tokens(sample_text, max_tokens=50)) +# +# print("\nHybrid chunking:") +# print(chunk_text_hybrid(sample_text, max_tokens=50)) + + + +####################################################################################################################### +# +# Experimental Semantic Chunking +# + +# Chunk text into segments based on semantic similarity +def count_units(text, unit='tokens'): + if unit == 'words': + return len(text.split()) + elif unit == 'tokens': + return len(word_tokenize(text)) + elif unit == 'characters': + return len(text) + else: + raise ValueError("Invalid unit. Choose 'words', 'tokens', or 'characters'.") + + +def semantic_chunking(text, max_chunk_size=2000, unit='words'): + nltk.download('punkt', quiet=True) + sentences = sent_tokenize(text) + vectorizer = TfidfVectorizer() + sentence_vectors = vectorizer.fit_transform(sentences) + + chunks = [] + current_chunk = [] + current_size = 0 + + for i, sentence in enumerate(sentences): + sentence_size = count_units(sentence, unit) + if current_size + sentence_size > max_chunk_size and current_chunk: + chunks.append(' '.join(current_chunk)) + overlap_size = count_units(' '.join(current_chunk[-3:]), unit) # Use last 3 sentences for overlap + current_chunk = current_chunk[-3:] # Keep last 3 sentences for overlap + current_size = overlap_size + + current_chunk.append(sentence) + current_size += sentence_size + + if i + 1 < len(sentences): + current_vector = sentence_vectors[i] + next_vector = sentence_vectors[i + 1] + similarity = cosine_similarity(current_vector, next_vector)[0][0] + if similarity < 0.5 and current_size >= max_chunk_size // 2: + chunks.append(' '.join(current_chunk)) + overlap_size = count_units(' '.join(current_chunk[-3:]), unit) + current_chunk = current_chunk[-3:] + current_size = overlap_size + + if current_chunk: + chunks.append(' '.join(current_chunk)) + + return chunks + + +def semantic_chunk_long_file(file_path, max_chunk_size=1000, overlap=100): + try: + with open(file_path, 'r', encoding='utf-8') as file: + content = file.read() + + chunks = semantic_chunking(content, max_chunk_size, overlap) + return chunks + except Exception as e: + logging.error(f"Error chunking text file: {str(e)}") + return None +####################################################################################################################### + + + + + + +####################################################################################################################### +# +# OpenAI Rolling Summarization +# + +client = OpenAI(api_key=openai_api_key) +def get_chat_completion(messages, model='gpt-4-turbo'): + response = client.chat.completions.create( + model=model, + messages=messages, + temperature=0, + ) + return response.choices[0].message.content + + +# This function combines text chunks into larger blocks without exceeding a specified token count. +# It returns the combined chunks, their original indices, and the number of dropped chunks due to overflow. +def combine_chunks_with_no_minimum( + chunks: List[str], + max_tokens: int, + chunk_delimiter="\n\n", + header: Optional[str] = None, + add_ellipsis_for_overflow=False, +) -> Tuple[List[str], List[int]]: + dropped_chunk_count = 0 + output = [] # list to hold the final combined chunks + output_indices = [] # list to hold the indices of the final combined chunks + candidate = ( + [] if header is None else [header] + ) # list to hold the current combined chunk candidate + candidate_indices = [] + for chunk_i, chunk in enumerate(chunks): + chunk_with_header = [chunk] if header is None else [header, chunk] + # FIXME MAKE NOT OPENAI SPECIFIC + if len(openai_tokenize(chunk_delimiter.join(chunk_with_header))) > max_tokens: + print(f"warning: chunk overflow") + if ( + add_ellipsis_for_overflow + # FIXME MAKE NOT OPENAI SPECIFIC + and len(openai_tokenize(chunk_delimiter.join(candidate + ["..."]))) <= max_tokens + ): + candidate.append("...") + dropped_chunk_count += 1 + continue # this case would break downstream assumptions + # estimate token count with the current chunk added + # FIXME MAKE NOT OPENAI SPECIFIC + extended_candidate_token_count = len(openai_tokenize(chunk_delimiter.join(candidate + [chunk]))) + # If the token count exceeds max_tokens, add the current candidate to output and start a new candidate + if extended_candidate_token_count > max_tokens: + output.append(chunk_delimiter.join(candidate)) + output_indices.append(candidate_indices) + candidate = chunk_with_header # re-initialize candidate + candidate_indices = [chunk_i] + # otherwise keep extending the candidate + else: + candidate.append(chunk) + candidate_indices.append(chunk_i) + # add the remaining candidate to output if it's not empty + if (header is not None and len(candidate) > 1) or (header is None and len(candidate) > 0): + output.append(chunk_delimiter.join(candidate)) + output_indices.append(candidate_indices) + return output, output_indices, dropped_chunk_count + + +def rolling_summarize(text: str, + detail: float = 0, + model: str = 'gpt-4-turbo', + additional_instructions: Optional[str] = None, + minimum_chunk_size: Optional[int] = 500, + chunk_delimiter: str = ".", + summarize_recursively=False, + verbose=False): + """ + Summarizes a given text by splitting it into chunks, each of which is summarized individually. + The level of detail in the summary can be adjusted, and the process can optionally be made recursive. + + Parameters: + - text (str): The text to be summarized. + - detail (float, optional): A value between 0 and 1 + indicating the desired level of detail in the summary. 0 leads to a higher level summary, and 1 results in a more + detailed summary. Defaults to 0. + - additional_instructions (Optional[str], optional): Additional instructions to provide to the + model for customizing summaries. - minimum_chunk_size (Optional[int], optional): The minimum size for text + chunks. Defaults to 500. + - chunk_delimiter (str, optional): The delimiter used to split the text into chunks. Defaults to ".". + - summarize_recursively (bool, optional): If True, summaries are generated recursively, using previous summaries for context. + - verbose (bool, optional): If True, prints detailed information about the chunking process. + Returns: + - str: The final compiled summary of the text. + + The function first determines the number of chunks by interpolating between a minimum and a maximum chunk count + based on the `detail` parameter. It then splits the text into chunks and summarizes each chunk. If + `summarize_recursively` is True, each summary is based on the previous summaries, adding more context to the + summarization process. The function returns a compiled summary of all chunks. + """ + + # check detail is set correctly + assert 0 <= detail <= 1 + + # interpolate the number of chunks based to get specified level of detail + max_chunks = len(chunk_on_delimiter(text, minimum_chunk_size, chunk_delimiter)) + min_chunks = 1 + num_chunks = int(min_chunks + detail * (max_chunks - min_chunks)) + + # adjust chunk_size based on interpolated number of chunks + # FIXME MAKE NOT OPENAI SPECIFIC + document_length = len(openai_tokenize(text)) + chunk_size = max(minimum_chunk_size, document_length // num_chunks) + text_chunks = chunk_on_delimiter(text, chunk_size, chunk_delimiter) + if verbose: + print(f"Splitting the text into {len(text_chunks)} chunks to be summarized.") + # FIXME MAKE NOT OPENAI SPECIFIC + print(f"Chunk lengths are {[len(openai_tokenize(x)) for x in text_chunks]}") + + # set system message - FIXME + system_message_content = "Rewrite this text in summarized form." + if additional_instructions is not None: + system_message_content += f"\n\n{additional_instructions}" + + accumulated_summaries = [] + for i, chunk in enumerate(tqdm(text_chunks)): + if summarize_recursively and accumulated_summaries: + # Combine previous summary with current chunk for recursive summarization + combined_text = accumulated_summaries[-1] + "\n\n" + chunk + user_message_content = f"Previous summary and new content to summarize:\n\n{combined_text}" + else: + user_message_content = chunk + + messages = [ + {"role": "system", "content": system_message_content}, + {"role": "user", "content": user_message_content} + ] + + response = get_chat_completion(messages, model=model) + accumulated_summaries.append(response) + + final_summary = '\n\n'.join(accumulated_summaries) + return final_summary + + + diff --git a/App_Function_Libraries/Diarization_Lib.py b/App_Function_Libraries/Diarization_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..d4dc035c06123adf5dcc220bca09ff34da6284cd --- /dev/null +++ b/App_Function_Libraries/Diarization_Lib.py @@ -0,0 +1,177 @@ +# Diarization_Lib.py +######################################### +# Diarization Library +# This library is used to perform diarization of audio files. +# Currently, uses FIXME for transcription. +# +#################### +#################### +# Function List +# +# 1. speaker_diarize(video_file_path, segments, embedding_model = "pyannote/embedding", embedding_size=512, num_speakers=0) +# +#################### +# Import necessary libraries +import configparser +import json +import logging +import os +from pathlib import Path +import time +# Import Local +from App_Function_Libraries.Audio_Transcription_Lib import speech_to_text +# +# Import 3rd Party +from pyannote.audio import Model +from pyannote.audio.pipelines.speaker_diarization import SpeakerDiarization +import torch +import yaml +# +####################################################################################################################### +# Function Definitions +# + +def load_pipeline_from_pretrained(path_to_config: str | Path) -> SpeakerDiarization: + path_to_config = Path(path_to_config).resolve() + print(f"Loading pyannote pipeline from {path_to_config}...") + + if not path_to_config.exists(): + raise FileNotFoundError(f"Config file not found: {path_to_config}") + + # Load the YAML configuration + with open(path_to_config, 'r') as config_file: + config = yaml.safe_load(config_file) + + # Store current working directory + cwd = Path.cwd().resolve() + + # Change to the directory containing the config file + cd_to = path_to_config.parent.resolve() + print(f"Changing working directory to {cd_to}") + os.chdir(cd_to) + + try: + # Create a SpeakerDiarization pipeline + pipeline = SpeakerDiarization() + + # Load models explicitly from local paths + embedding_path = Path(config['pipeline']['params']['embedding']).resolve() + segmentation_path = Path(config['pipeline']['params']['segmentation']).resolve() + + if not embedding_path.exists(): + raise FileNotFoundError(f"Embedding model file not found: {embedding_path}") + if not segmentation_path.exists(): + raise FileNotFoundError(f"Segmentation model file not found: {segmentation_path}") + + # Load the models from local paths using pyannote's Model class + pipeline.embedding = Model.from_pretrained(str(embedding_path), map_location=torch.device('cpu')) + pipeline.segmentation = Model.from_pretrained(str(segmentation_path), map_location=torch.device('cpu')) + + # Set other parameters + pipeline.clustering = config['pipeline']['params']['clustering'] + pipeline.embedding_batch_size = config['pipeline']['params']['embedding_batch_size'] + pipeline.embedding_exclude_overlap = config['pipeline']['params']['embedding_exclude_overlap'] + pipeline.segmentation_batch_size = config['pipeline']['params']['segmentation_batch_size'] + + # Set additional parameters + pipeline.instantiate(config['params']) + + finally: + # Change back to the original working directory + print(f"Changing working directory back to {cwd}") + os.chdir(cwd) + + return pipeline + +def audio_diarization(audio_file_path): + logging.info('audio-diarization: Loading pyannote pipeline') + config = configparser.ConfigParser() + config.read('config.txt') + processing_choice = config.get('Processing', 'processing_choice', fallback='cpu') + + base_dir = Path(__file__).parent.resolve() + config_path = base_dir / 'models' / 'config.yaml' + pipeline = load_pipeline_from_pretrained(config_path) + + time_start = time.time() + if audio_file_path is None: + raise ValueError("audio-diarization: No audio file provided") + logging.info("audio-diarization: Audio file path: %s", audio_file_path) + + try: + _, file_ending = os.path.splitext(audio_file_path) + out_file = audio_file_path.replace(file_ending, ".diarization.json") + prettified_out_file = audio_file_path.replace(file_ending, ".diarization_pretty.json") + if os.path.exists(out_file): + logging.info("audio-diarization: Diarization file already exists: %s", out_file) + with open(out_file) as f: + global diarization_result + diarization_result = json.load(f) + return diarization_result + + logging.info('audio-diarization: Starting diarization...') + diarization_result = pipeline(audio_file_path) + + segments = [] + for turn, _, speaker in diarization_result.itertracks(yield_label=True): + chunk = { + "Time_Start": turn.start, + "Time_End": turn.end, + "Speaker": speaker + } + logging.debug("Segment: %s", chunk) + segments.append(chunk) + logging.info("audio-diarization: Diarization completed with pyannote") + + output_data = {'segments': segments} + + logging.info("audio-diarization: Saving prettified JSON to %s", prettified_out_file) + with open(prettified_out_file, 'w') as f: + json.dump(output_data, f, indent=2) + + logging.info("audio-diarization: Saving JSON to %s", out_file) + with open(out_file, 'w') as f: + json.dump(output_data, f) + + except Exception as e: + logging.error("audio-diarization: Error performing diarization: %s", str(e)) + raise RuntimeError("audio-diarization: Error performing diarization") + return segments + +def combine_transcription_and_diarization(audio_file_path): + logging.info('combine-transcription-and-diarization: Starting transcription and diarization...') + + transcription_result = speech_to_text(audio_file_path) + + diarization_result = audio_diarization(audio_file_path) + + combined_result = [] + for transcription_segment in transcription_result: + for diarization_segment in diarization_result: + if transcription_segment['Time_Start'] >= diarization_segment['Time_Start'] and transcription_segment[ + 'Time_End'] <= diarization_segment['Time_End']: + combined_segment = { + "Time_Start": transcription_segment['Time_Start'], + "Time_End": transcription_segment['Time_End'], + "Speaker": diarization_segment['Speaker'], + "Text": transcription_segment['Text'] + } + combined_result.append(combined_segment) + break + + _, file_ending = os.path.splitext(audio_file_path) + out_file = audio_file_path.replace(file_ending, ".combined.json") + prettified_out_file = audio_file_path.replace(file_ending, ".combined_pretty.json") + + logging.info("combine-transcription-and-diarization: Saving prettified JSON to %s", prettified_out_file) + with open(prettified_out_file, 'w') as f: + json.dump(combined_result, f, indent=2) + + logging.info("combine-transcription-and-diarization: Saving JSON to %s", out_file) + with open(out_file, 'w') as f: + json.dump(combined_result, f) + + return combined_result +# +# +####################################################################################################################### \ No newline at end of file diff --git a/App_Function_Libraries/Gradio_Related.py b/App_Function_Libraries/Gradio_Related.py new file mode 100644 index 0000000000000000000000000000000000000000..8a9cdc5b9c6b4d9c9c4ef121c62170afc74986db --- /dev/null +++ b/App_Function_Libraries/Gradio_Related.py @@ -0,0 +1,2226 @@ +# Gradio_Related.py +######################################### +# Gradio UI Functions Library +# This library is used to hold all UI-related functions for Gradio. +# I fucking hate Gradio. +# +##### +# Functions: +# +# download_audio_file(url, save_path) +# process_audio( +# process_audio_file(audio_url, audio_file, whisper_model="small.en", api_name=None, api_key=None) +# +# +######################################### +# +# Built-In Imports +from datetime import datetime +import json +import logging +import os.path +from pathlib import Path +import sqlite3 +from typing import Dict, List, Tuple +import traceback +from functools import wraps +# +# Import 3rd-Party Libraries +import yt_dlp +import gradio as gr +# +# Local Imports +from App_Function_Libraries.Article_Summarization_Lib import scrape_and_summarize_multiple +from App_Function_Libraries.Audio_Files import process_audio_files, process_podcast +from App_Function_Libraries.Chunk_Lib import improved_chunking_process, get_chat_completion +from App_Function_Libraries.PDF_Ingestion_Lib import process_and_cleanup_pdf +from App_Function_Libraries.Local_LLM_Inference_Engine_Lib import local_llm_gui_function +from App_Function_Libraries.Local_Summarization_Lib import summarize_with_llama, summarize_with_kobold, \ + summarize_with_oobabooga, summarize_with_tabbyapi, summarize_with_vllm, summarize_with_local_llm +from App_Function_Libraries.Summarization_General_Lib import summarize_with_openai, summarize_with_cohere, \ + summarize_with_anthropic, summarize_with_groq, summarize_with_openrouter, summarize_with_deepseek, \ + summarize_with_huggingface, perform_summarization, save_transcription_and_summary, \ + perform_transcription, summarize_chunk +from App_Function_Libraries.SQLite_DB import update_media_content, list_prompts, search_and_display, db, DatabaseError, \ + fetch_prompt_details, keywords_browser_interface, add_keyword, delete_keyword, \ + export_keywords_to_csv, export_to_file, add_media_to_database, insert_prompt_to_db +from App_Function_Libraries.Utils import sanitize_filename, extract_text_from_segments, create_download_directory, \ + convert_to_seconds, load_comprehensive_config +from App_Function_Libraries.Video_DL_Ingestion_Lib import parse_and_expand_urls, \ + generate_timestamped_url, extract_metadata, download_video + +# +####################################################################################################################### +# Function Definitions +# + +whisper_models = ["small", "medium", "small.en", "medium.en", "medium", "large", "large-v1", "large-v2", "large-v3", + "distil-large-v2", "distil-medium.en", "distil-small.en"] +custom_prompt_input = None +server_mode = False +share_public = False + + +def load_preset_prompts(): + return list_prompts() + + +def gradio_download_youtube_video(url): + """Download video using yt-dlp with specified options.""" + # Determine ffmpeg path based on the operating system. + ffmpeg_path = './Bin/ffmpeg.exe' if os.name == 'nt' else 'ffmpeg' + + # Extract information about the video + with yt_dlp.YoutubeDL({'quiet': True}) as ydl: + info_dict = ydl.extract_info(url, download=False) + sanitized_title = sanitize_filename(info_dict['title']) + original_ext = info_dict['ext'] + + # Setup the final directory and filename + download_dir = Path(f"results/{sanitized_title}") + download_dir.mkdir(parents=True, exist_ok=True) + output_file_path = download_dir / f"{sanitized_title}.{original_ext}" + + # Initialize yt-dlp with generic options and the output template + ydl_opts = { + 'format': 'bestvideo+bestaudio/best', + 'ffmpeg_location': ffmpeg_path, + 'outtmpl': str(output_file_path), + 'noplaylist': True, 'quiet': True + } + + # Execute yt-dlp to download the video + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + ydl.download([url]) + + # Final check to ensure file exists + if not output_file_path.exists(): + raise FileNotFoundError(f"Expected file was not found: {output_file_path}") + + return str(output_file_path) + + + + +def format_transcription(content): + # Add extra space after periods for better readability + content = content.replace('.', '. ').replace('. ', '. ') + # Split the content into lines for multiline display + lines = content.split('. ') + # Join lines with HTML line break for better presentation in Markdown + formatted_content = "
".join(lines) + return formatted_content + + +def format_file_path(file_path, fallback_path=None): + if file_path and os.path.exists(file_path): + logging.debug(f"File exists: {file_path}") + return file_path + elif fallback_path and os.path.exists(fallback_path): + logging.debug(f"File does not exist: {file_path}. Returning fallback path: {fallback_path}") + return fallback_path + else: + logging.debug(f"File does not exist: {file_path}. No fallback path available.") + return None + + +def search_media(query, fields, keyword, page): + try: + results = search_and_display(query, fields, keyword, page) + return results + except Exception as e: + logger = logging.getLogger() + logger.error(f"Error searching media: {e}") + return str(e) + + + + +# Sample data +prompts_category_1 = [ + "What are the key points discussed in the video?", + "Summarize the main arguments made by the speaker.", + "Describe the conclusions of the study presented." +] + +prompts_category_2 = [ + "How does the proposed solution address the problem?", + "What are the implications of the findings?", + "Can you explain the theory behind the observed phenomenon?" +] + +all_prompts = prompts_category_1 + prompts_category_2 + + + + + +# Handle prompt selection +def handle_prompt_selection(prompt): + return f"You selected: {prompt}" + +def display_details(media_id): + # Gradio Search Function-related stuff + if media_id: + details = display_item_details(media_id) + details_html = "" + for detail in details: + details_html += f"

Prompt:

{detail[0]}

" + details_html += f"

Summary:

{detail[1]}

" + details_html += f"

Transcription:

{detail[2]}
" + return details_html + return "No details available." + + +def fetch_items_by_title_or_url(search_query: str, search_type: str): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + if search_type == 'Title': + cursor.execute("SELECT id, title, url FROM Media WHERE title LIKE ?", (f'%{search_query}%',)) + elif search_type == 'URL': + cursor.execute("SELECT id, title, url FROM Media WHERE url LIKE ?", (f'%{search_query}%',)) + results = cursor.fetchall() + return results + except sqlite3.Error as e: + raise DatabaseError(f"Error fetching items by {search_type}: {e}") + + +def fetch_items_by_keyword(search_query: str): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT m.id, m.title, m.url + FROM Media m + JOIN MediaKeywords mk ON m.id = mk.media_id + JOIN Keywords k ON mk.keyword_id = k.id + WHERE k.keyword LIKE ? + """, (f'%{search_query}%',)) + results = cursor.fetchall() + return results + except sqlite3.Error as e: + raise DatabaseError(f"Error fetching items by keyword: {e}") + + +def fetch_items_by_content(search_query: str): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT id, title, url FROM Media WHERE content LIKE ?", (f'%{search_query}%',)) + results = cursor.fetchall() + return results + except sqlite3.Error as e: + raise DatabaseError(f"Error fetching items by content: {e}") + + +def fetch_item_details_single(media_id: int): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT prompt, summary + FROM MediaModifications + WHERE media_id = ? + ORDER BY modification_date DESC + LIMIT 1 + """, (media_id,)) + prompt_summary_result = cursor.fetchone() + cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,)) + content_result = cursor.fetchone() + + prompt = prompt_summary_result[0] if prompt_summary_result else "" + summary = prompt_summary_result[1] if prompt_summary_result else "" + content = content_result[0] if content_result else "" + + return prompt, summary, content + except sqlite3.Error as e: + raise Exception(f"Error fetching item details: {e}") + + +def fetch_item_details(media_id: int): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT prompt, summary + FROM MediaModifications + WHERE media_id = ? + ORDER BY modification_date DESC + LIMIT 1 + """, (media_id,)) + prompt_summary_result = cursor.fetchone() + cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,)) + content_result = cursor.fetchone() + + prompt = prompt_summary_result[0] if prompt_summary_result else "" + summary = prompt_summary_result[1] if prompt_summary_result else "" + content = content_result[0] if content_result else "" + + return content, prompt, summary + except sqlite3.Error as e: + logging.error(f"Error fetching item details: {e}") + return "", "", "" # Return empty strings if there's an error + + +def browse_items(search_query, search_type): + if search_type == 'Keyword': + results = fetch_items_by_keyword(search_query) + elif search_type == 'Content': + results = fetch_items_by_content(search_query) + else: + results = fetch_items_by_title_or_url(search_query, search_type) + return results + + +def display_item_details(media_id): + # Function to display item details + prompt_summary_results, content = fetch_item_details(media_id) + content_section = f"

Transcription:

{content}
" + prompt_summary_section = "" + for prompt, summary in prompt_summary_results: + prompt_summary_section += f"

Prompt:

{prompt}

" + prompt_summary_section += f"

Summary:

{summary}

" + return prompt_summary_section, content_section + + +def update_dropdown(search_query, search_type): + results = browse_items(search_query, search_type) + item_options = [f"{item[1]} ({item[2]})" for item in results] + new_item_mapping = {f"{item[1]} ({item[2]})": item[0] for item in results} + print(f"Debug - Update Dropdown - New Item Mapping: {new_item_mapping}") + return gr.update(choices=item_options), new_item_mapping + + + +def get_media_id(selected_item, item_mapping): + return item_mapping.get(selected_item) + + +def update_detailed_view(item, item_mapping): + # Function to update the detailed view based on selected item + if item: + item_id = item_mapping.get(item) + if item_id: + content, prompt, summary = fetch_item_details(item_id) + if content or prompt or summary: + details_html = "

Details:

" + if prompt: + details_html += f"

Prompt:

{prompt}

" + if summary: + details_html += f"

Summary:

{summary}

" + # Format the transcription content for better readability + content_html = f"

Transcription:

{format_transcription(content)}
" + return details_html, content_html + else: + return "No details available.", "No details available." + else: + return "No item selected", "No item selected" + else: + return "No item selected", "No item selected" + + +def format_content(content): + # Format content using markdown + formatted_content = f"```\n{content}\n```" + return formatted_content + + +def update_prompt_dropdown(): + prompt_names = list_prompts() + return gr.update(choices=prompt_names) + + +def display_prompt_details(selected_prompt): + if selected_prompt: + details = fetch_prompt_details(selected_prompt) + if details: + details_str = f"

Details:

{details[0]}

" + system_str = f"

System:

{details[1]}

" + user_str = f"

User:

{details[2]}

" if details[2] else "" + return details_str + system_str + user_str + return "No details available." + + +def display_search_results(query): + if not query.strip(): + return "Please enter a search query." + + results = search_prompts(query) + + # Debugging: Print the results to the console to see what is being returned + print(f"Processed search results for query '{query}': {results}") + + if results: + result_md = "## Search Results:\n" + for result in results: + # Debugging: Print each result to see its format + print(f"Result item: {result}") + + if len(result) == 2: + name, details = result + result_md += f"**Title:** {name}\n\n**Description:** {details}\n\n---\n" + else: + result_md += "Error: Unexpected result format.\n\n---\n" + return result_md + return "No results found." + + +def search_media_database(query: str) -> List[Tuple[int, str, str]]: + return browse_items(query, 'Title') + + +def load_media_content(media_id: int) -> dict: + try: + print(f"Debug - Load Media Content - Media ID: {media_id}") + item_details = fetch_item_details(media_id) + print(f"Debug - Load Media Content - Item Details: {item_details}") + + if isinstance(item_details, tuple) and len(item_details) == 3: + content, prompt, summary = item_details + else: + print(f"Debug - Load Media Content - Unexpected item_details format: {item_details}") + content, prompt, summary = "", "", "" + + return { + "content": content or "No content available", + "prompt": prompt or "No prompt available", + "summary": summary or "No summary available" + } + except Exception as e: + print(f"Debug - Load Media Content - Error: {str(e)}") + return {"content": "", "prompt": "", "summary": ""} + +def load_preset_prompts(): + return list_prompts() + +def chat(message, history, media_content, selected_parts, api_endpoint, api_key, prompt): + try: + print(f"Debug - Chat Function - Message: {message}") + print(f"Debug - Chat Function - Media Content: {media_content}") + print(f"Debug - Chat Function - Selected Parts: {selected_parts}") + print(f"Debug - Chat Function - API Endpoint: {api_endpoint}") + print(f"Debug - Chat Function - Prompt: {prompt}") + + # Ensure selected_parts is a list + if not isinstance(selected_parts, (list, tuple)): + selected_parts = [selected_parts] if selected_parts else [] + + print(f"Debug - Chat Function - Selected Parts (after check): {selected_parts}") + + # Combine the selected parts of the media content + combined_content = "\n\n".join([f"{part.capitalize()}: {media_content.get(part, '')}" for part in selected_parts if part in media_content]) + print(f"Debug - Chat Function - Combined Content: {combined_content[:500]}...") # Print first 500 chars + + # Prepare the input for the API + input_data = f"{combined_content}\n\nUser: {message}\nAI:" + print(f"Debug - Chat Function - Input Data: {input_data[:500]}...") # Print first 500 chars + + # Use the existing API request code based on the selected endpoint + if api_endpoint.lower() == 'openai': + response = summarize_with_openai(api_key, input_data, prompt) + elif api_endpoint.lower() == "anthropic": + response = summarize_with_anthropic(api_key, input_data, prompt) + elif api_endpoint.lower() == "cohere": + response = summarize_with_cohere(api_key, input_data, prompt) + elif api_endpoint.lower() == "groq": + response = summarize_with_groq(api_key, input_data, prompt) + elif api_endpoint.lower() == "openrouter": + response = summarize_with_openrouter(api_key, input_data, prompt) + elif api_endpoint.lower() == "deepseek": + response = summarize_with_deepseek(api_key, input_data, prompt) + elif api_endpoint.lower() == "llama.cpp": + response = summarize_with_llama(input_data, prompt) + elif api_endpoint.lower() == "kobold": + response = summarize_with_kobold(input_data, api_key, prompt) + elif api_endpoint.lower() == "ooba": + response = summarize_with_oobabooga(input_data, api_key, prompt) + elif api_endpoint.lower() == "tabbyapi": + response = summarize_with_tabbyapi(input_data, prompt) + elif api_endpoint.lower() == "vllm": + response = summarize_with_vllm(input_data, prompt) + elif api_endpoint.lower() == "local-llm": + response = summarize_with_local_llm(input_data, prompt) + elif api_endpoint.lower() == "huggingface": + response = summarize_with_huggingface(api_key, input_data, prompt) + else: + raise ValueError(f"Unsupported API endpoint: {api_endpoint}") + + return response + + except Exception as e: + logging.error(f"Error in chat function: {str(e)}") + return f"An error occurred: {str(e)}" + + +def save_chat_history(history: List[List[str]], media_content: Dict[str, str], selected_parts: List[str], + api_endpoint: str, prompt: str): + """ + Save the chat history along with context information to a JSON file. + """ + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"chat_history_{timestamp}.json" + + chat_data = { + "timestamp": timestamp, + "history": history, + "context": { + "selected_media": { + part: media_content.get(part, "") for part in selected_parts + }, + "api_endpoint": api_endpoint, + "prompt": prompt + } + } + + json_data = json.dumps(chat_data, indent=2) + + return filename, json_data + + +def error_handler(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + error_message = f"Error in {func.__name__}: {str(e)}" + logging.error(f"{error_message}\n{traceback.format_exc()}") + return {"error": error_message, "details": traceback.format_exc()} + return wrapper + + +def create_chunking_inputs(): + chunk_text_by_words_checkbox = gr.Checkbox(label="Chunk Text by Words", value=False, visible=True) + max_words_input = gr.Number(label="Max Words", value=300, precision=0, visible=True) + chunk_text_by_sentences_checkbox = gr.Checkbox(label="Chunk Text by Sentences", value=False, visible=True) + max_sentences_input = gr.Number(label="Max Sentences", value=10, precision=0, visible=True) + chunk_text_by_paragraphs_checkbox = gr.Checkbox(label="Chunk Text by Paragraphs", value=False, visible=True) + max_paragraphs_input = gr.Number(label="Max Paragraphs", value=5, precision=0, visible=True) + chunk_text_by_tokens_checkbox = gr.Checkbox(label="Chunk Text by Tokens", value=False, visible=True) + max_tokens_input = gr.Number(label="Max Tokens", value=1000, precision=0, visible=True) + gr_semantic_chunk_long_file = gr.Checkbox(label="Semantic Chunking by Sentence similarity", value=False, visible=True) + gr_semantic_chunk_long_file_size = gr.Number(label="Max Chunk Size", value=2000, visible=True) + gr_semantic_chunk_long_file_overlap = gr.Number(label="Max Chunk Overlap Size", value=100, visible=True) + return [chunk_text_by_words_checkbox, max_words_input, chunk_text_by_sentences_checkbox, max_sentences_input, + chunk_text_by_paragraphs_checkbox, max_paragraphs_input, chunk_text_by_tokens_checkbox, max_tokens_input] + + + +def create_video_transcription_tab(): + with gr.TabItem("Video Transcription + Summarization"): + gr.Markdown("# Transcribe & Summarize Videos from URLs") + with gr.Row(): + gr.Markdown("""Follow this project at [tldw - GitHub](https://github.com/rmusser01/tldw)""") + with gr.Row(): + with gr.Column(): + url_input = gr.Textbox(label="URL(s) (Mandatory)", + placeholder="Enter video URLs here, one per line. Supports YouTube, Vimeo, and playlists.", + lines=5) + diarize_input = gr.Checkbox(label="Enable Speaker Diarization", value=False) + whisper_model_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model") + custom_prompt_checkbox = gr.Checkbox(label="Use Custom Prompt", value=False, visible=True) + custom_prompt_input = gr.Textbox(label="Custom Prompt", placeholder="Enter custom prompt here", lines=3, visible=False) + custom_prompt_checkbox.change( + fn=lambda x: gr.update(visible=x), + inputs=[custom_prompt_checkbox], + outputs=[custom_prompt_input] + ) + api_name_input = gr.Dropdown( + choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "OpenRouter", + "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "HuggingFace"], + value=None, label="API Name (Mandatory)") + api_key_input = gr.Textbox(label="API Key (Mandatory)", placeholder="Enter your API key here") + keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords here (comma-separated)", + value="default,no_keyword_set") + batch_size_input = gr.Slider(minimum=1, maximum=10, value=1, step=1, + label="Batch Size (Number of videos to process simultaneously)") + timestamp_option = gr.Radio(choices=["Include Timestamps", "Exclude Timestamps"], + value="Include Timestamps", label="Timestamp Option") + keep_original_video = gr.Checkbox(label="Keep Original Video", value=False) + # First, create a checkbox to toggle the chunking options + chunking_options_checkbox = gr.Checkbox(label="Show Chunking Options", value=False) + summarize_recursively = gr.Checkbox(label="Enable Recursive Summarization", value=False) + use_cookies_input = gr.Checkbox(label="Use cookies for authenticated download", value=False) + use_time_input = gr.Checkbox(label="Use Start and End Time", value=False) + + with gr.Row(visible=False) as time_input_box: + gr.Markdown("### Start and End time") + with gr.Column(): + start_time_input = gr.Textbox(label="Start Time (Optional)", + placeholder="e.g., 1:30 or 90 (in seconds)") + end_time_input = gr.Textbox(label="End Time (Optional)", placeholder="e.g., 5:45 or 345 (in seconds)") + + use_time_input.change( + fn=lambda x: gr.update(visible=x), + inputs=[use_time_input], + outputs=[time_input_box] + ) + + cookies_input = gr.Textbox( + label="User Session Cookies", + placeholder="Paste your cookies here (JSON format)", + lines=3, + visible=False + ) + + use_cookies_input.change( + fn=lambda x: gr.update(visible=x), + inputs=[use_cookies_input], + outputs=[cookies_input] + ) + # Then, create a Box to group the chunking options + with gr.Row(visible=False) as chunking_options_box: + gr.Markdown("### Chunking Options") + with gr.Column(): + chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens'], + label="Chunking Method") + max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50, label="Max Chunk Size") + chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap") + use_adaptive_chunking = gr.Checkbox(label="Use Adaptive Chunking") + use_multi_level_chunking = gr.Checkbox(label="Use Multi-level Chunking") + chunk_language = gr.Dropdown(choices=['english', 'french', 'german', 'spanish'], + label="Chunking Language") + + # Add JavaScript to toggle the visibility of the chunking options box + chunking_options_checkbox.change( + fn=lambda x: gr.update(visible=x), + inputs=[chunking_options_checkbox], + outputs=[chunking_options_box] + ) + process_button = gr.Button("Process Videos") + + with gr.Column(): + progress_output = gr.Textbox(label="Progress") + error_output = gr.Textbox(label="Errors", visible=False) + results_output = gr.HTML(label="Results") + download_transcription = gr.File(label="Download All Transcriptions as JSON") + download_summary = gr.File(label="Download All Summaries as Text") + + @error_handler + def process_videos_with_error_handling(urls, start_time, end_time, diarize, whisper_model, + custom_prompt_checkbox, custom_prompt, chunking_options_checkbox, + chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking, + use_multi_level_chunking, chunk_language, api_name, + api_key, keywords, use_cookies, cookies, batch_size, + timestamp_option, keep_original_video, summarize_recursively, + progress: gr.Progress = gr.Progress()) -> tuple: + try: + logging.info("Entering process_videos_with_error_handling") + logging.info(f"Received URLs: {urls}") + + if not urls: + raise ValueError("No URLs provided") + + logging.debug("Input URL(s) is(are) valid") + + # Ensure batch_size is an integer + try: + batch_size = int(batch_size) + except (ValueError, TypeError): + batch_size = 1 # Default to processing one video at a time if invalid + + expanded_urls = parse_and_expand_urls(urls) + logging.info(f"Expanded URLs: {expanded_urls}") + + total_videos = len(expanded_urls) + logging.info(f"Total videos to process: {total_videos}") + results = [] + errors = [] + results_html = "" + all_transcriptions = {} + all_summaries = "" + + for i in range(0, total_videos, batch_size): + batch = expanded_urls[i:i + batch_size] + batch_results = [] + + for url in batch: + try: + start_seconds = convert_to_seconds(start_time) + end_seconds = convert_to_seconds(end_time) if end_time else None + + logging.info(f"Attempting to extract metadata for {url}") + video_metadata = extract_metadata(url, use_cookies, cookies) + if not video_metadata: + raise ValueError(f"Failed to extract metadata for {url}") + + chunk_options = { + 'method': chunk_method, + 'max_size': max_chunk_size, + 'overlap': chunk_overlap, + 'adaptive': use_adaptive_chunking, + 'multi_level': use_multi_level_chunking, + 'language': chunk_language + } if chunking_options_checkbox else None + + result = process_url_with_metadata( + url, 2, whisper_model, + custom_prompt if custom_prompt_checkbox else None, + start_seconds, api_name, api_key, + False, False, False, False, 0.01, None, keywords, None, diarize, + end_time=end_seconds, + include_timestamps=(timestamp_option == "Include Timestamps"), + metadata=video_metadata, + use_chunking=chunking_options_checkbox, + chunk_options=chunk_options, + keep_original_video=keep_original_video + ) + + if result[0] is None: # Check if the first return value is None + error_message = "Processing failed without specific error" + batch_results.append((url, error_message, "Error", video_metadata, None, None)) + errors.append(f"Error processing {url}: {error_message}") + else: + url, transcription, summary, json_file, summary_file, result_metadata = result + if transcription is None: + error_message = f"Processing failed for {url}: Transcription is None" + batch_results.append((url, error_message, "Error", result_metadata, None, None)) + errors.append(error_message) + else: + batch_results.append( + (url, transcription, "Success", result_metadata, json_file, summary_file)) + + except Exception as e: + error_message = f"Error processing {url}: {str(e)}" + logging.error(error_message, exc_info=True) + batch_results.append((url, error_message, "Error", {}, None, None)) + errors.append(error_message) + + results.extend(batch_results) + if isinstance(progress, gr.Progress): + progress((i + len(batch)) / total_videos, + f"Processed {i + len(batch)}/{total_videos} videos") + + # Generate HTML for results + for url, transcription, status, metadata, json_file, summary_file in results: + if status == "Success": + title = metadata.get('title', 'Unknown Title') + + # Check if transcription is a string (which it should be now) + if isinstance(transcription, str): + # Split the transcription into metadata and actual transcription + parts = transcription.split('\n\n', 1) + if len(parts) == 2: + metadata_text, transcription_text = parts + else: + metadata_text = "Metadata not found" + transcription_text = transcription + else: + metadata_text = "Metadata format error" + transcription_text = "Transcription format error" + + summary = open(summary_file, 'r').read() if summary_file else "No summary available" + + results_html += f""" +
+ + +

URL: {url}

+

Metadata:

+ 
{metadata_text}
+

Transcription:

+
{transcription_text}
+

Summary:

+
{summary}
+ + +
+ """ + logging.debug(f"Transcription for {url}: {transcription[:200]}...") + all_transcriptions[url] = transcription + all_summaries += f"Title: {title}\nURL: {url}\n\n{metadata_text}\n\nTranscription:\n{transcription_text}\n\nSummary:\n{summary}\n\n---\n\n" + else: + results_html += f""" +
+

Error processing {url}

+

{transcription}

+
+ """ + + # Save all transcriptions and summaries to files + with open('all_transcriptions.json', 'w') as f: + json.dump(all_transcriptions, f, indent=2) + + with open('all_summaries.txt', 'w') as f: + f.write(all_summaries) + + error_summary = "\n".join(errors) if errors else "No errors occurred." + + return ( + f"Processed {total_videos} videos. {len(errors)} errors occurred.", + error_summary, + results_html, + 'all_transcriptions.json', + 'all_summaries.txt' + ) + except Exception as e: + logging.error(f"Unexpected error in process_videos_with_error_handling: {str(e)}", exc_info=True) + return ( + f"An unexpected error occurred: {str(e)}", + str(e), + "

Unexpected Error

" + str(e) + "

", + None, + None + ) + + def process_videos_wrapper(urls, start_time, end_time, diarize, whisper_model, + custom_prompt_checkbox, custom_prompt, chunking_options_checkbox, + chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking, + use_multi_level_chunking, chunk_language, summarize_recursively, api_name, + api_key, keywords, use_cookies, cookies, batch_size, + timestamp_option, keep_original_video): + try: + logging.info("process_videos_wrapper called") + result = process_videos_with_error_handling( + urls, start_time, end_time, diarize, whisper_model, + custom_prompt_checkbox, custom_prompt, chunking_options_checkbox, + chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking, + use_multi_level_chunking, chunk_language, api_name, + api_key, keywords, use_cookies, cookies, batch_size, + timestamp_option, keep_original_video, summarize_recursively + ) + logging.info("process_videos_with_error_handling completed") + + # Ensure that result is a tuple with 5 elements + if not isinstance(result, tuple) or len(result) != 5: + raise ValueError( + f"Expected 5 outputs, but got {len(result) if isinstance(result, tuple) else 1}") + + return result + except Exception as e: + logging.error(f"Error in process_videos_wrapper: {str(e)}", exc_info=True) + # Return a tuple with 5 elements in case of any error + return ( + f"An error occurred: {str(e)}", # progress_output + str(e), # error_output + f"
Error: {str(e)}
", # results_output + None, # download_transcription + None # download_summary + ) + + # FIXME - remove dead args for process_url_with_metadata + @error_handler + def process_url_with_metadata(url, num_speakers, whisper_model, custom_prompt, offset, api_name, api_key, + vad_filter, download_video_flag, download_audio, rolling_summarization, + detail_level, question_box, keywords, local_file_path, diarize, end_time=None, + include_timestamps=True, metadata=None, use_chunking=False, + chunk_options=None, keep_original_video=False): + + try: + logging.info(f"Starting process_url_metadata for URL: {url}") + # Create download path + download_path = create_download_directory("Video_Downloads") + logging.info(f"Download path created at: {download_path}") + + # Initialize info_dict + info_dict = {} + + # Handle URL or local file + if local_file_path: + video_file_path = local_file_path + # Extract basic info from local file + info_dict = { + 'webpage_url': local_file_path, + 'title': os.path.basename(local_file_path), + 'description': "Local file", + 'channel_url': None, + 'duration': None, + 'channel': None, + 'uploader': None, + 'upload_date': None + } + else: + # Extract video information + with yt_dlp.YoutubeDL({'quiet': True}) as ydl: + try: + full_info = ydl.extract_info(url, download=False) + + # Create a safe subset of info to log + safe_info = { + 'title': full_info.get('title', 'No title'), + 'duration': full_info.get('duration', 'Unknown duration'), + 'upload_date': full_info.get('upload_date', 'Unknown upload date'), + 'uploader': full_info.get('uploader', 'Unknown uploader'), + 'view_count': full_info.get('view_count', 'Unknown view count') + } + + logging.debug(f"Full info extracted for {url}: {safe_info}") + except Exception as e: + logging.error(f"Error extracting video info: {str(e)}") + return None, None, None, None, None, None + + # Filter the required metadata + if full_info: + info_dict = { + 'webpage_url': full_info.get('webpage_url', url), + 'title': full_info.get('title'), + 'description': full_info.get('description'), + 'channel_url': full_info.get('channel_url'), + 'duration': full_info.get('duration'), + 'channel': full_info.get('channel'), + 'uploader': full_info.get('uploader'), + 'upload_date': full_info.get('upload_date') + } + logging.debug(f"Filtered info_dict: {info_dict}") + else: + logging.error("Failed to extract video information") + return None, None, None, None, None, None + + # Download video/audio + logging.info("Downloading video/audio...") + video_file_path = download_video(url, download_path, full_info, download_video_flag) + if not video_file_path: + logging.error(f"Failed to download video/audio from {url}") + return None, None, None, None, None, None + + logging.info(f"Processing file: {video_file_path}") + + # Perform transcription + logging.info("Starting transcription...") + audio_file_path, segments = perform_transcription(video_file_path, offset, whisper_model, + vad_filter) + + if audio_file_path is None or segments is None: + logging.error("Transcription failed or segments not available.") + return None, None, None, None, None, None + + logging.info(f"Transcription completed. Number of segments: {len(segments)}") + + # Add metadata to segments + segments_with_metadata = { + "metadata": info_dict, + "segments": segments + } + + # Save segments with metadata to JSON file + segments_json_path = os.path.splitext(audio_file_path)[0] + ".segments.json" + with open(segments_json_path, 'w') as f: + json.dump(segments_with_metadata, f, indent=2) + + # Delete the .wav file after successful transcription + files_to_delete = [audio_file_path] + for file_path in files_to_delete: + if file_path and os.path.exists(file_path): + try: + os.remove(file_path) + logging.info(f"Successfully deleted file: {file_path}") + except Exception as e: + logging.warning(f"Failed to delete file {file_path}: {str(e)}") + + # Delete the mp4 file after successful transcription if not keeping original audio + # Modify the file deletion logic to respect keep_original_video + if not keep_original_video: + files_to_delete = [audio_file_path, video_file_path] + for file_path in files_to_delete: + if file_path and os.path.exists(file_path): + try: + os.remove(file_path) + logging.info(f"Successfully deleted file: {file_path}") + except Exception as e: + logging.warning(f"Failed to delete file {file_path}: {str(e)}") + else: + logging.info(f"Keeping original video file: {video_file_path}") + logging.info(f"Keeping original audio file: {audio_file_path}") + + # Process segments based on the timestamp option + if not include_timestamps: + segments = [{'Text': segment['Text']} for segment in segments] + + logging.info(f"Segments processed for timestamp inclusion: {segments}") + + # Extract text from segments + transcription_text = extract_text_from_segments(segments) + + if transcription_text.startswith("Error:"): + logging.error(f"Failed to extract transcription: {transcription_text}") + return None, None, None, None, None, None + + # Use transcription_text instead of segments for further processing + full_text_with_metadata = f"{json.dumps(info_dict, indent=2)}\n\n{transcription_text}" + + logging.debug(f"Full text with metadata extracted: {full_text_with_metadata[:100]}...") + + # Perform summarization if API is provided + summary_text = None + if api_name: + # API key resolution handled at base of function if none provided + api_key = api_key if api_key else None + logging.info(f"Starting summarization with {api_name}...") + summary_text = perform_summarization(api_name, full_text_with_metadata, custom_prompt, api_key) + if summary_text is None: + logging.error("Summarization failed.") + return None, None, None, None, None, None + logging.debug(f"Summarization completed: {summary_text[:100]}...") + + # Save transcription and summary + logging.info("Saving transcription and summary...") + download_path = create_download_directory("Audio_Processing") + json_file_path, summary_file_path = save_transcription_and_summary(full_text_with_metadata, + summary_text, + download_path, info_dict) + logging.info( + f"Transcription and summary saved. JSON file: {json_file_path}, Summary file: {summary_file_path}") + + # Prepare keywords for database + if isinstance(keywords, str): + keywords_list = [kw.strip() for kw in keywords.split(',') if kw.strip()] + elif isinstance(keywords, (list, tuple)): + keywords_list = keywords + else: + keywords_list = [] + logging.info(f"Keywords prepared: {keywords_list}") + + # Add to database + logging.info("Adding to database...") + add_media_to_database(info_dict['webpage_url'], info_dict, full_text_with_metadata, summary_text, + keywords_list, custom_prompt, whisper_model) + logging.info(f"Media added to database: {info_dict['webpage_url']}") + + return info_dict[ + 'webpage_url'], full_text_with_metadata, summary_text, json_file_path, summary_file_path, info_dict + + except Exception as e: + logging.error(f"Error in process_url_with_metadata: {str(e)}", exc_info=True) + return None, None, None, None, None, None + + process_button.click( + fn=process_videos_wrapper, + inputs=[ + url_input, start_time_input, end_time_input, diarize_input, whisper_model_input, + custom_prompt_checkbox, custom_prompt_input, chunking_options_checkbox, + chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking, + use_multi_level_chunking, chunk_language, summarize_recursively, api_name_input, api_key_input, + keywords_input, use_cookies_input, cookies_input, batch_size_input, + timestamp_option, keep_original_video + ], + outputs=[progress_output, error_output, results_output, download_transcription, download_summary] + ) + + +def create_audio_processing_tab(): + with gr.TabItem("Audio File Transcription + Summarization"): + gr.Markdown("# Transcribe & Summarize Audio Files from URLs or Local Files!") + with gr.Row(): + with gr.Column(): + audio_url_input = gr.Textbox(label="Audio File URL(s)", placeholder="Enter the URL(s) of the audio file(s), one per line") + audio_file_input = gr.File(label="Upload Audio File", file_types=["audio/*"]) + + use_cookies_input = gr.Checkbox(label="Use cookies for authenticated download", value=False) + cookies_input = gr.Textbox( + label="Audio Download Cookies", + placeholder="Paste your cookies here (JSON format)", + lines=3, + visible=False + ) + + use_cookies_input.change( + fn=lambda x: gr.update(visible=x), + inputs=[use_cookies_input], + outputs=[cookies_input] + ) + + diarize_input = gr.Checkbox(label="Enable Speaker Diarization", value=False) + whisper_model_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model") + custom_prompt_checkbox = gr.Checkbox(label="Use Custom Prompt", value=False, visible=True) + custom_prompt_input = gr.Textbox(label="Custom Prompt", placeholder="Enter custom prompt here", lines=3, visible=False) + custom_prompt_checkbox.change( + fn=lambda x: gr.update(visible=x), + inputs=[custom_prompt_checkbox], + outputs=[custom_prompt_input] + ) + api_name_input = gr.Dropdown( + choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "OpenRouter", + "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "HuggingFace"], + value=None, + label="API for Summarization (Optional)" + ) + api_key_input = gr.Textbox(label="API Key (if required)", placeholder="Enter your API key here", type="password") + custom_keywords_input = gr.Textbox(label="Custom Keywords", placeholder="Enter custom keywords, comma-separated") + keep_original_input = gr.Checkbox(label="Keep original audio file", value=False) + + chunking_options_checkbox = gr.Checkbox(label="Show Chunking Options", value=False) + with gr.Row(visible=False) as chunking_options_box: + gr.Markdown("### Chunking Options") + with gr.Column(): + chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens'], label="Chunking Method") + max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50, label="Max Chunk Size") + chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap") + use_adaptive_chunking = gr.Checkbox(label="Use Adaptive Chunking") + use_multi_level_chunking = gr.Checkbox(label="Use Multi-level Chunking") + chunk_language = gr.Dropdown(choices=['english', 'french', 'german', 'spanish'], label="Chunking Language") + + chunking_options_checkbox.change( + fn=lambda x: gr.update(visible=x), + inputs=[chunking_options_checkbox], + outputs=[chunking_options_box] + ) + + process_audio_button = gr.Button("Process Audio File(s)") + + with gr.Column(): + audio_progress_output = gr.Textbox(label="Progress") + audio_transcription_output = gr.Textbox(label="Transcription") + audio_summary_output = gr.Textbox(label="Summary") + download_transcription = gr.File(label="Download All Transcriptions as JSON") + download_summary = gr.File(label="Download All Summaries as Text") + + process_audio_button.click( + fn=process_audio_files, + inputs=[audio_url_input, audio_file_input, whisper_model_input, api_name_input, api_key_input, + use_cookies_input, cookies_input, keep_original_input, custom_keywords_input, custom_prompt_input, + chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking, use_multi_level_chunking, + chunk_language, diarize_input], + outputs=[audio_progress_output, audio_transcription_output, audio_summary_output] + ) + + +def create_podcast_tab(): + with gr.TabItem("Podcast"): + gr.Markdown("# Podcast Transcription and Ingestion") + with gr.Row(): + with gr.Column(): + podcast_url_input = gr.Textbox(label="Podcast URL", placeholder="Enter the podcast URL here") + podcast_title_input = gr.Textbox(label="Podcast Title", placeholder="Will be auto-detected if possible") + podcast_author_input = gr.Textbox(label="Podcast Author", placeholder="Will be auto-detected if possible") + + podcast_keywords_input = gr.Textbox( + label="Keywords", + placeholder="Enter keywords here (comma-separated, include series name if applicable)", + value="podcast,audio", + elem_id="podcast-keywords-input" + ) + + custom_prompt_checkbox = gr.Checkbox(label="Use Custom Prompt", value=False, visible=True) + podcast_custom_prompt_input = gr.Textbox( + label="Custom Prompt", + placeholder="Enter custom prompt for summarization (optional)", + lines=3, + visible=False + ) + custom_prompt_checkbox.change( + fn=lambda x: gr.update(visible=x), + inputs=[custom_prompt_checkbox], + outputs=[podcast_custom_prompt_input] + ) + + podcast_api_name_input = gr.Dropdown( + choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "OpenRouter", "Llama.cpp", + "Kobold", "Ooba", "Tabbyapi", "VLLM", "HuggingFace"], + value=None, + label="API Name for Summarization (Optional)" + ) + podcast_api_key_input = gr.Textbox(label="API Key (if required)", type="password") + podcast_whisper_model_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model") + + keep_original_input = gr.Checkbox(label="Keep original audio file", value=False) + enable_diarization_input = gr.Checkbox(label="Enable speaker diarization", value=False) + + use_cookies_input = gr.Checkbox(label="Use cookies for yt-dlp", value=False) + cookies_input = gr.Textbox( + label="yt-dlp Cookies", + placeholder="Paste your cookies here (JSON format)", + lines=3, + visible=False + ) + + use_cookies_input.change( + fn=lambda x: gr.update(visible=x), + inputs=[use_cookies_input], + outputs=[cookies_input] + ) + + chunking_options_checkbox = gr.Checkbox(label="Show Chunking Options", value=False) + with gr.Row(visible=False) as chunking_options_box: + gr.Markdown("### Chunking Options") + with gr.Column(): + chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens'], label="Chunking Method") + max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50, label="Max Chunk Size") + chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap") + use_adaptive_chunking = gr.Checkbox(label="Use Adaptive Chunking") + use_multi_level_chunking = gr.Checkbox(label="Use Multi-level Chunking") + chunk_language = gr.Dropdown(choices=['english', 'french', 'german', 'spanish'], label="Chunking Language") + + chunking_options_checkbox.change( + fn=lambda x: gr.update(visible=x), + inputs=[chunking_options_checkbox], + outputs=[chunking_options_box] + ) + + podcast_process_button = gr.Button("Process Podcast") + + with gr.Column(): + podcast_progress_output = gr.Textbox(label="Progress") + podcast_error_output = gr.Textbox(label="Error Messages") + podcast_transcription_output = gr.Textbox(label="Transcription") + podcast_summary_output = gr.Textbox(label="Summary") + download_transcription = gr.File(label="Download Transcription as JSON") + download_summary = gr.File(label="Download Summary as Text") + + podcast_process_button.click( + fn=process_podcast, + inputs=[podcast_url_input, podcast_title_input, podcast_author_input, + podcast_keywords_input, podcast_custom_prompt_input, podcast_api_name_input, + podcast_api_key_input, podcast_whisper_model_input, keep_original_input, + enable_diarization_input, use_cookies_input, cookies_input, + chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking, + use_multi_level_chunking, chunk_language], + outputs=[podcast_progress_output, podcast_transcription_output, podcast_summary_output, + podcast_title_input, podcast_author_input, podcast_keywords_input, podcast_error_output, + download_transcription, download_summary] + ) + + +def create_website_scraping_tab(): + with gr.TabItem("Website Scraping"): + gr.Markdown("# Scrape Websites & Summarize Articles using a Headless Chrome Browser!") + with gr.Row(): + with gr.Column(): + url_input = gr.Textbox(label="Article URLs", placeholder="Enter article URLs here, one per line", lines=5) + custom_article_title_input = gr.Textbox(label="Custom Article Titles (Optional, one per line)", + placeholder="Enter custom titles for the articles, one per line", + lines=5) + custom_prompt_input = gr.Textbox(label="Custom Prompt (Optional)", + placeholder="Provide a custom prompt for summarization", lines=3) + api_name_input = gr.Dropdown( + choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "OpenRouter", + "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "HuggingFace"], value=None, label="API Name (Mandatory for Summarization)") + api_key_input = gr.Textbox(label="API Key (Mandatory if API Name is specified)", + placeholder="Enter your API key here; Ignore if using Local API or Built-in API") + keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords here (comma-separated)", + value="default,no_keyword_set", visible=True) + + scrape_button = gr.Button("Scrape and Summarize") + with gr.Column(): + result_output = gr.Textbox(label="Result", lines=20) + + scrape_button.click( + fn=scrape_and_summarize_multiple, + inputs=[url_input, custom_prompt_input, api_name_input, api_key_input, keywords_input, + custom_article_title_input], + outputs=result_output + ) + + +def create_pdf_ingestion_tab(): + with gr.TabItem("PDF Ingestion"): + # TODO - Add functionality to extract metadata from pdf as part of conversion process in marker + gr.Markdown("# Ingest PDF Files and Extract Metadata") + with gr.Row(): + with gr.Column(): + pdf_file_input = gr.File(label="Uploaded PDF File", file_types=[".pdf"], visible=False) + pdf_upload_button = gr.UploadButton("Click to Upload PDF", file_types=[".pdf"]) + pdf_title_input = gr.Textbox(label="Title (Optional)") + pdf_author_input = gr.Textbox(label="Author (Optional)") + pdf_keywords_input = gr.Textbox(label="Keywords (Optional, comma-separated)") + pdf_ingest_button = gr.Button("Ingest PDF") + + pdf_upload_button.upload(fn=lambda file: file, inputs=pdf_upload_button, outputs=pdf_file_input) + with gr.Column(): + pdf_result_output = gr.Textbox(label="Result") + + pdf_ingest_button.click( + fn=process_and_cleanup_pdf, + inputs=[pdf_file_input, pdf_title_input, pdf_author_input, pdf_keywords_input], + outputs=pdf_result_output + ) +# +# +################################################################################################################ +# Functions for Re-Summarization +# + + + +def create_resummary_tab(): + with gr.TabItem("Re-Summarize"): + gr.Markdown("# Re-Summarize Existing Content") + with gr.Row(): + search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...") + search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By") + search_button = gr.Button("Search") + + items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True) + item_mapping = gr.State({}) + + with gr.Row(): + api_name_input = gr.Dropdown( + choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "OpenRouter", + "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "HuggingFace"], + value="Local-LLM", label="API Name") + api_key_input = gr.Textbox(label="API Key", placeholder="Enter your API key here") + + chunking_options_checkbox = gr.Checkbox(label="Use Chunking", value=False) + with gr.Row(visible=False) as chunking_options_box: + chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens'], + label="Chunking Method", value='words') + max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50, label="Max Chunk Size") + chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap") + + custom_prompt_checkbox = gr.Checkbox(label="Use Custom Prompt", value=False) + custom_prompt_input = gr.Textbox(label="Custom Prompt", placeholder="Enter custom prompt here", lines=3, visible=False) + + resummary_button = gr.Button("Re-Summarize") + + result_output = gr.Textbox(label="Result") + + # Connect the UI elements + search_button.click( + fn=update_resummary_dropdown, + inputs=[search_query_input, search_type_input], + outputs=[items_output, item_mapping] + ) + + chunking_options_checkbox.change( + fn=lambda x: gr.update(visible=x), + inputs=[chunking_options_checkbox], + outputs=[chunking_options_box] + ) + + custom_prompt_checkbox.change( + fn=lambda x: gr.update(visible=x), + inputs=[custom_prompt_checkbox], + outputs=[custom_prompt_input] + ) + + resummary_button.click( + fn=resummary_content_wrapper, + inputs=[items_output, item_mapping, api_name_input, api_key_input, chunking_options_checkbox, chunk_method, + max_chunk_size, chunk_overlap, custom_prompt_checkbox, custom_prompt_input], + outputs=result_output + ) + + return search_query_input, search_type_input, search_button, items_output, item_mapping, api_name_input, api_key_input, chunking_options_checkbox, chunking_options_box, chunk_method, max_chunk_size, chunk_overlap, custom_prompt_checkbox, custom_prompt_input, resummary_button, result_output + + +def update_resummary_dropdown(search_query, search_type): + if search_type in ['Title', 'URL']: + results = fetch_items_by_title_or_url(search_query, search_type) + elif search_type == 'Keyword': + results = fetch_items_by_keyword(search_query) + else: # Content + results = fetch_items_by_content(search_query) + + item_options = [f"{item[1]} ({item[2]})" for item in results] + item_mapping = {f"{item[1]} ({item[2]})": item[0] for item in results} + return gr.update(choices=item_options), item_mapping + + +def resummary_content_wrapper(selected_item, item_mapping, api_name, api_key, chunking_options_checkbox, chunk_method, + max_chunk_size, chunk_overlap, custom_prompt_checkbox, custom_prompt): + if not selected_item or not api_name or not api_key: + return "Please select an item and provide API details." + + media_id = item_mapping.get(selected_item) + if not media_id: + return "Invalid selection." + + content, old_prompt, old_summary = fetch_item_details(media_id) + + if not content: + return "No content available for re-summarization." + + # Prepare chunking options + chunk_options = { + 'method': chunk_method, + 'max_size': int(max_chunk_size), + 'overlap': int(chunk_overlap), + 'language': 'english', + 'adaptive': True, + 'multi_level': False, + } if chunking_options_checkbox else None + + # Prepare summarization prompt + summarization_prompt = custom_prompt if custom_prompt_checkbox and custom_prompt else None + + # Call the resummary_content function + result = resummary_content(media_id, content, api_name, api_key, chunk_options, summarization_prompt) + + return result + + +def resummary_content(selected_item, item_mapping, api_name, api_key, chunking_options_checkbox, chunk_method, max_chunk_size, chunk_overlap, custom_prompt_checkbox, custom_prompt): + if not selected_item or not api_name or not api_key: + return "Please select an item and provide API details." + + media_id = item_mapping.get(selected_item) + if not media_id: + return "Invalid selection." + + content, old_prompt, old_summary = fetch_item_details(media_id) + + if not content: + return "No content available for re-summarization." + + # Load configuration + config = load_comprehensive_config() + + # Prepare chunking options + chunk_options = { + 'method': chunk_method, + 'max_size': int(max_chunk_size), + 'overlap': int(chunk_overlap), + 'language': 'english', + 'adaptive': True, + 'multi_level': False, + } + + # Chunking logic + if chunking_options_checkbox: + chunks = improved_chunking_process(content, chunk_options) + else: + chunks = [{'text': content, 'metadata': {}}] + + # Prepare summarization prompt + if custom_prompt_checkbox and custom_prompt: + summarization_prompt = custom_prompt + else: + summarization_prompt = config.get('Prompts', 'default_summary_prompt', fallback="Summarize the following text:") + + # Summarization logic + summaries = [] + for chunk in chunks: + chunk_text = chunk['text'] + try: + chunk_summary = summarize_chunk(api_name, chunk_text, summarization_prompt, api_key) + if chunk_summary: + summaries.append(chunk_summary) + else: + logging.warning(f"Summarization failed for chunk: {chunk_text[:100]}...") + except Exception as e: + logging.error(f"Error during summarization: {str(e)}") + return f"Error during summarization: {str(e)}" + + if not summaries: + return "Summarization failed for all chunks." + + new_summary = " ".join(summaries) + + # Update the database with the new summary + try: + update_result = update_media_content(selected_item, item_mapping, content, summarization_prompt, new_summary) + if "successfully" in update_result.lower(): + return f"Re-summarization complete. New summary: {new_summary[:500]}..." + else: + return f"Error during database update: {update_result}" + except Exception as e: + logging.error(f"Error updating database: {str(e)}") + return f"Error updating database: {str(e)}" + +# End of Re-Summarization Functions +# +############################################################################################################## +# +# Search Tab + +def add_or_update_prompt(title, description, system_prompt, user_prompt): + if not title: + return "Error: Title is required." + + existing_prompt = fetch_prompt_details(title) + if existing_prompt: + # Update existing prompt + result = update_prompt_in_db(title, description, system_prompt, user_prompt) + else: + # Insert new prompt + result = insert_prompt_to_db(title, description, system_prompt, user_prompt) + + # Refresh the prompt dropdown + update_prompt_dropdown() + return result + + +def load_prompt_details(selected_prompt): + if selected_prompt: + details = fetch_prompt_details(selected_prompt) + if details: + return details[0], details[1], details[2], details[3] + return "", "", "", "" + + +def update_prompt_in_db(title, description, system_prompt, user_prompt): + try: + conn = sqlite3.connect('prompts.db') + cursor = conn.cursor() + cursor.execute( + "UPDATE Prompts SET details = ?, system = ?, user = ? WHERE name = ?", + (description, system_prompt, user_prompt, title) + ) + conn.commit() + conn.close() + return "Prompt updated successfully!" + except sqlite3.Error as e: + return f"Error updating prompt: {e}" + + +def search_prompts(query): + try: + conn = sqlite3.connect('prompts.db') + cursor = conn.cursor() + cursor.execute("SELECT name, details, system, user FROM Prompts WHERE name LIKE ? OR details LIKE ?", + (f"%{query}%", f"%{query}%")) + results = cursor.fetchall() + conn.close() + return results + except sqlite3.Error as e: + print(f"Error searching prompts: {e}") + return [] + + +def create_search_tab(): + with gr.TabItem("Search / Detailed View"): + with gr.Row(): + with gr.Column(): + gr.Markdown("# Search across all ingested items in the Database") + gr.Markdown(" by Title / URL / Keyword / or Content via SQLite Full-Text-Search") + search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...") + search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By") + search_button = gr.Button("Search") + items_output = gr.Dropdown(label="Select Item", choices=[]) + item_mapping = gr.State({}) + prompt_summary_output = gr.HTML(label="Prompt & Summary", visible=True) + content_output = gr.Markdown(label="Content", visible=True) + + search_button.click( + fn=update_dropdown, + inputs=[search_query_input, search_type_input], + outputs=[items_output, item_mapping] + ) + with gr.Column(): + items_output.change( + fn=update_detailed_view, + inputs=[items_output, item_mapping], + outputs=[prompt_summary_output, content_output] + ) +def create_prompt_view_tab(): + def display_search_results(query): + if not query.strip(): + return "Please enter a search query." + + results = search_prompts(query) + + print(f"Processed search results for query '{query}': {results}") + + if results: + result_md = "## Search Results:\n" + for result in results: + print(f"Result item: {result}") + + if len(result) == 4: + name, details, system, user = result + result_md += f"**Title:** {name}\n\n" + result_md += f"**Description:** {details}\n\n" + result_md += f"**System Prompt:** {system}\n\n" + result_md += f"**User Prompt:** {user}\n\n" + result_md += "---\n" + else: + result_md += "Error: Unexpected result format.\n\n---\n" + return result_md + return "No results found." + with gr.TabItem("Search Prompts"): + with gr.Row(): + with gr.Column(): + gr.Markdown("# Search and View Prompt Details") + gr.Markdown("Currently has all of the https://github.com/danielmiessler/fabric prompts already available") + search_query_input = gr.Textbox(label="Search Prompts", placeholder="Enter your search query...") + search_button = gr.Button("Search Prompts") + with gr.Column(): + search_results_output = gr.Markdown() + prompt_details_output = gr.HTML() + search_button.click( + fn=display_search_results, + inputs=[search_query_input], + outputs=[search_results_output] + ) + + + +def create_prompt_edit_tab(): + with gr.TabItem("Edit Prompts"): + with gr.Row(): + with gr.Column(): + prompt_dropdown = gr.Dropdown( + label="Select Prompt", + choices=[], + interactive=True + ) + prompt_list_button = gr.Button("List Prompts") + + with gr.Column(): + title_input = gr.Textbox(label="Title", placeholder="Enter the prompt title") + description_input = gr.Textbox(label="Description", placeholder="Enter the prompt description", lines=3) + system_prompt_input = gr.Textbox(label="System Prompt", placeholder="Enter the system prompt", lines=3) + user_prompt_input = gr.Textbox(label="User Prompt", placeholder="Enter the user prompt", lines=3) + add_prompt_button = gr.Button("Add/Update Prompt") + add_prompt_output = gr.HTML() + + # Event handlers + prompt_list_button.click( + fn=update_prompt_dropdown, + outputs=prompt_dropdown + ) + + add_prompt_button.click( + fn=add_or_update_prompt, + inputs=[title_input, description_input, system_prompt_input, user_prompt_input], + outputs=add_prompt_output + ) + + # Load prompt details when selected + prompt_dropdown.change( + fn=load_prompt_details, + inputs=[prompt_dropdown], + outputs=[title_input, description_input, system_prompt_input, user_prompt_input] + ) + + +# End of Search Tab Functions +# +################################################################################################################ +# +# Llamafile Tab + + +def start_llamafile(*args): + # Unpack arguments + (am_noob, verbose_checked, threads_checked, threads_value, http_threads_checked, http_threads_value, + model_checked, model_value, hf_repo_checked, hf_repo_value, hf_file_checked, hf_file_value, + ctx_size_checked, ctx_size_value, ngl_checked, ngl_value, host_checked, host_value, port_checked, + port_value) = args + + # Construct command based on checked values + command = [] + if am_noob: + am_noob = True + if verbose_checked is not None and verbose_checked: + command.append('-v') + if threads_checked and threads_value is not None: + command.extend(['-t', str(threads_value)]) + if http_threads_checked and http_threads_value is not None: + command.extend(['--threads', str(http_threads_value)]) + if model_checked and model_value is not None: + model_path = model_value.name + command.extend(['-m', model_path]) + if hf_repo_checked and hf_repo_value is not None: + command.extend(['-hfr', hf_repo_value]) + if hf_file_checked and hf_file_value is not None: + command.extend(['-hff', hf_file_value]) + if ctx_size_checked and ctx_size_value is not None: + command.extend(['-c', str(ctx_size_value)]) + if ngl_checked and ngl_value is not None: + command.extend(['-ngl', str(ngl_value)]) + if host_checked and host_value is not None: + command.extend(['--host', host_value]) + if port_checked and port_value is not None: + command.extend(['--port', str(port_value)]) + + # Code to start llamafile with the provided configuration + local_llm_gui_function(am_noob, verbose_checked, threads_checked, threads_value, + http_threads_checked, http_threads_value, model_checked, + model_value, hf_repo_checked, hf_repo_value, hf_file_checked, + hf_file_value, ctx_size_checked, ctx_size_value, ngl_checked, + ngl_value, host_checked, host_value, port_checked, port_value, ) + + # Example command output to verify + return f"Command built and ran: {' '.join(command)} \n\nLlamafile started successfully." + +def stop_llamafile(): + # Code to stop llamafile + # ... + return "Llamafile stopped" + + +def create_llamafile_settings_tab(): + with gr.TabItem("Local LLM with Llamafile"): + gr.Markdown("# Settings for Llamafile") + am_noob = gr.Checkbox(label="Check this to enable sane defaults", value=False, visible=True) + advanced_mode_toggle = gr.Checkbox(label="Advanced Mode - Enable to show all settings", value=False) + + model_checked = gr.Checkbox(label="Enable Setting Local LLM Model Path", value=False, visible=True) + model_value = gr.Textbox(label="Select Local Model File", value="", visible=True) + ngl_checked = gr.Checkbox(label="Enable Setting GPU Layers", value=False, visible=True) + ngl_value = gr.Number(label="Number of GPU Layers", value=None, precision=0, visible=True) + + advanced_inputs = create_llamafile_advanced_inputs() + + start_button = gr.Button("Start Llamafile") + stop_button = gr.Button("Stop Llamafile") + output_display = gr.Markdown() + + start_button.click( + fn=start_llamafile, + inputs=[am_noob, model_checked, model_value, ngl_checked, ngl_value] + advanced_inputs, + outputs=output_display + ) + + +def create_llamafile_advanced_inputs(): + verbose_checked = gr.Checkbox(label="Enable Verbose Output", value=False, visible=False) + threads_checked = gr.Checkbox(label="Set CPU Threads", value=False, visible=False) + threads_value = gr.Number(label="Number of CPU Threads", value=None, precision=0, visible=False) + http_threads_checked = gr.Checkbox(label="Set HTTP Server Threads", value=False, visible=False) + http_threads_value = gr.Number(label="Number of HTTP Server Threads", value=None, precision=0, visible=False) + hf_repo_checked = gr.Checkbox(label="Use Huggingface Repo Model", value=False, visible=False) + hf_repo_value = gr.Textbox(label="Huggingface Repo Name", value="", visible=False) + hf_file_checked = gr.Checkbox(label="Set Huggingface Model File", value=False, visible=False) + hf_file_value = gr.Textbox(label="Huggingface Model File", value="", visible=False) + ctx_size_checked = gr.Checkbox(label="Set Prompt Context Size", value=False, visible=False) + ctx_size_value = gr.Number(label="Prompt Context Size", value=8124, precision=0, visible=False) + host_checked = gr.Checkbox(label="Set IP to Listen On", value=False, visible=False) + host_value = gr.Textbox(label="Host IP Address", value="", visible=False) + port_checked = gr.Checkbox(label="Set Server Port", value=False, visible=False) + port_value = gr.Number(label="Port Number", value=None, precision=0, visible=False) + + return [verbose_checked, threads_checked, threads_value, http_threads_checked, http_threads_value, + hf_repo_checked, hf_repo_value, hf_file_checked, hf_file_value, ctx_size_checked, ctx_size_value, + host_checked, host_value, port_checked, port_value] + +# +# End of Llamafile Tab Functions +################################################################################################################ +# +# Chat Interface Tab Functions + + +def create_chat_interface(): + with gr.TabItem("Remote LLM Chat"): + gr.Markdown("# Chat with a designated LLM Endpoint, using your selected item as starting context") + + with gr.Row(): + with gr.Column(scale=1): + search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...") + search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By") + search_button = gr.Button("Search") + + with gr.Column(scale=2): + items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True) + item_mapping = gr.State({}) + + with gr.Row(): + use_content = gr.Checkbox(label="Use Content") + use_summary = gr.Checkbox(label="Use Summary") + use_prompt = gr.Checkbox(label="Use Prompt") + + api_endpoint = gr.Dropdown(label="Select API Endpoint", choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "HuggingFace"]) + api_key = gr.Textbox(label="API Key (if required)", type="password") + preset_prompt = gr.Dropdown(label="Select Preset Prompt", choices=load_preset_prompts()) + user_prompt = gr.Textbox(label="Modify Prompt (Need to delete this after the first message, otherwise it'll " + "be used as the next message instead)", lines=3) + + chatbot = gr.Chatbot(height=500) + msg = gr.Textbox(label="Enter your message") + submit = gr.Button("Submit") + + chat_history = gr.State([]) + media_content = gr.State({}) + selected_parts = gr.State([]) + + save_button = gr.Button("Save Chat History") + download_file = gr.File(label="Download Chat History") + + def chat_wrapper(message, history, media_content, selected_parts, api_endpoint, api_key, user_prompt): + print(f"Debug - Chat Wrapper - Message: {message}") + print(f"Debug - Chat Wrapper - Media Content: {media_content}") + print(f"Debug - Chat Wrapper - Selected Parts: {selected_parts}") + print(f"Debug - Chat Wrapper - API Endpoint: {api_endpoint}") + print(f"Debug - Chat Wrapper - User Prompt: {user_prompt}") + + selected_content = "\n\n".join( + [f"{part.capitalize()}: {media_content.get(part, '')}" for part in selected_parts if + part in media_content]) + print(f"Debug - Chat Wrapper - Selected Content: {selected_content[:500]}...") # Print first 500 chars + + context = f"Selected content:\n{selected_content}\n\nUser message: {message}" + print(f"Debug - Chat Wrapper - Context: {context[:500]}...") # Print first 500 chars + + # Use a default API endpoint if none is selected + if not api_endpoint: + api_endpoint = "OpenAI" # You can change this to any default endpoint you prefer + print(f"Debug - Chat Wrapper - Using default API Endpoint: {api_endpoint}") + + bot_message = chat(context, history, media_content, selected_parts, api_endpoint, api_key, user_prompt) + print(f"Debug - Chat Wrapper - Bot Message: {bot_message[:500]}...") # Print first 500 chars + + history.append((message, bot_message)) + return "", history + + submit.click( + chat_wrapper, + inputs=[msg, chat_history, media_content, selected_parts, api_endpoint, api_key, user_prompt], + outputs=[msg, chatbot] + ) + + def save_chat_history(history): + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"chat_history_{timestamp}.json" + with open(filename, "w") as f: + json.dump(history, f) + return filename + + save_button.click(save_chat_history, inputs=[chat_history], outputs=[download_file]) + + search_button.click( + fn=update_dropdown, + inputs=[search_query_input, search_type_input], + outputs=[items_output, item_mapping] + ) + + def update_user_prompt(preset_name): + details = fetch_prompt_details(preset_name) + if details: + return details[1] # Return the system prompt + return "" + + preset_prompt.change(update_user_prompt, inputs=preset_prompt, outputs=user_prompt) + + def update_chat_content(selected_item, use_content, use_summary, use_prompt, item_mapping): + print(f"Debug - Update Chat Content - Selected Item: {selected_item}") + print(f"Debug - Update Chat Content - Use Content: {use_content}") + print(f"Debug - Update Chat Content - Use Summary: {use_summary}") + print(f"Debug - Update Chat Content - Use Prompt: {use_prompt}") + print(f"Debug - Update Chat Content - Item Mapping: {item_mapping}") + + if selected_item and selected_item in item_mapping: + media_id = item_mapping[selected_item] + content = load_media_content(media_id) + selected_parts = [] + if use_content and "content" in content: + selected_parts.append("content") + if use_summary and "summary" in content: + selected_parts.append("summary") + if use_prompt and "prompt" in content: + selected_parts.append("prompt") + print(f"Debug - Update Chat Content - Content: {content}") + print(f"Debug - Update Chat Content - Selected Parts: {selected_parts}") + return content, selected_parts + else: + print(f"Debug - Update Chat Content - No item selected or item not in mapping") + return {}, [] + + items_output.change( + update_chat_content, + inputs=[items_output, use_content, use_summary, use_prompt, item_mapping], + outputs=[media_content, selected_parts] + ) + + def update_selected_parts(use_content, use_summary, use_prompt): + selected_parts = [] + if use_content: + selected_parts.append("content") + if use_summary: + selected_parts.append("summary") + if use_prompt: + selected_parts.append("prompt") + print(f"Debug - Update Selected Parts: {selected_parts}") + return selected_parts + + use_content.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt], + outputs=[selected_parts]) + use_summary.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt], + outputs=[selected_parts]) + use_prompt.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt], + outputs=[selected_parts]) + + def update_selected_parts(use_content, use_summary, use_prompt): + selected_parts = [] + if use_content: + selected_parts.append("content") + if use_summary: + selected_parts.append("summary") + if use_prompt: + selected_parts.append("prompt") + print(f"Debug - Update Selected Parts: {selected_parts}") + return selected_parts + + use_content.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt], + outputs=[selected_parts]) + use_summary.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt], + outputs=[selected_parts]) + use_prompt.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt], + outputs=[selected_parts]) + + # Add debug output + def debug_output(media_content, selected_parts): + print(f"Debug - Media Content: {media_content}") + print(f"Debug - Selected Parts: {selected_parts}") + return "" + + items_output.change(debug_output, inputs=[media_content, selected_parts], outputs=[]) + +# +# End of Chat Interface Tab Functions +################################################################################################################ +# +# Media Edit Tab Functions + +def create_media_edit_tab(): + with gr.TabItem("Edit Existing Items"): + gr.Markdown("# Search and Edit Media Items") + + with gr.Row(): + search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...") + search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By") + search_button = gr.Button("Search") + + with gr.Row(): + items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True) + item_mapping = gr.State({}) + + content_input = gr.Textbox(label="Edit Content", lines=10) + prompt_input = gr.Textbox(label="Edit Prompt", lines=3) + summary_input = gr.Textbox(label="Edit Summary", lines=5) + + update_button = gr.Button("Update Media Content") + status_message = gr.Textbox(label="Status", interactive=False) + + search_button.click( + fn=update_dropdown, + inputs=[search_query_input, search_type_input], + outputs=[items_output, item_mapping] + ) + + def load_selected_media_content(selected_item, item_mapping): + if selected_item and item_mapping and selected_item in item_mapping: + media_id = item_mapping[selected_item] + content, prompt, summary = fetch_item_details(media_id) + return content, prompt, summary + return "No item selected or invalid selection", "", "" + + items_output.change( + fn=load_selected_media_content, + inputs=[items_output, item_mapping], + outputs=[content_input, prompt_input, summary_input] + ) + + update_button.click( + fn=update_media_content, + inputs=[items_output, item_mapping, content_input, prompt_input, summary_input], + outputs=status_message + ) +# +# +################################################################################################################ +# +# Import Items Tab Functions + + +def import_data(file, title, author, keywords, custom_prompt, summary, auto_summarize, api_name, api_key): + if file is None: + return "No file uploaded. Please upload a file." + + try: + logging.debug(f"File object type: {type(file)}") + logging.debug(f"File object attributes: {dir(file)}") + + if hasattr(file, 'name'): + file_name = file.name + else: + file_name = 'unknown_file' + + if isinstance(file, str): + # If file is a string, it's likely a file path + file_path = file + with open(file_path, 'r', encoding='utf-8') as f: + file_content = f.read() + elif hasattr(file, 'read'): + # If file has a 'read' method, it's likely a file-like object + file_content = file.read() + if isinstance(file_content, bytes): + file_content = file_content.decode('utf-8') + else: + # If it's neither a string nor a file-like object, try converting it to a string + file_content = str(file) + + logging.debug(f"File name: {file_name}") + logging.debug(f"File content (first 100 chars): {file_content[:100]}") + + # Create info_dict + info_dict = { + 'title': title or 'Untitled', + 'uploader': author or 'Unknown', + } + + # Create segments (assuming one segment for the entire content) + segments = [{'Text': file_content}] + + # Process keywords + keyword_list = [kw.strip() for kw in keywords.split(',') if kw.strip()] + + # Handle summarization + if auto_summarize and api_name and api_key: + summary = perform_summarization(api_name, file_content, custom_prompt, api_key) + elif not summary: + summary = "No summary provided" + + # Add to database + add_media_to_database( + url=file_name, # Using filename as URL + info_dict=info_dict, + segments=segments, + summary=summary, + keywords=keyword_list, + custom_prompt_input=custom_prompt, + whisper_model="Imported", # Indicating this was an imported file, + media_type = "document" + ) + + return f"File '{file_name}' successfully imported with title '{title}' and author '{author}'." + except Exception as e: + logging.error(f"Error importing file: {str(e)}") + return f"Error importing file: {str(e)}" + + +def create_import_item_tab(): + with gr.TabItem("Import Items"): + gr.Markdown("# Import a markdown file or text file into the database") + gr.Markdown("...and have it tagged + summarized") + with gr.Row(): + import_file = gr.File(label="Upload file for import", file_types=["txt", "md"]) + with gr.Row(): + title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content") + author_input = gr.Textbox(label="Author", placeholder="Enter the author's name") + with gr.Row(): + keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords, comma-separated") + custom_prompt_input = gr.Textbox(label="Custom Prompt", + placeholder="Enter a custom prompt for summarization (optional)") + with gr.Row(): + summary_input = gr.Textbox(label="Summary", + placeholder="Enter a summary or leave blank for auto-summarization", lines=3) + with gr.Row(): + auto_summarize_checkbox = gr.Checkbox(label="Auto-summarize", value=False) + api_name_input = gr.Dropdown( + choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "OpenRouter", + "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "HuggingFace"], + label="API for Auto-summarization" + ) + api_key_input = gr.Textbox(label="API Key", type="password") + with gr.Row(): + import_button = gr.Button("Import Data") + with gr.Row(): + import_output = gr.Textbox(label="Import Status") + + import_button.click( + fn=import_data, + inputs=[import_file, title_input, author_input, keywords_input, custom_prompt_input, + summary_input, auto_summarize_checkbox, api_name_input, api_key_input], + outputs=import_output + ) + +# +# End of Import Items Tab Functions +################################################################################################################ +# +# Export Items Tab Functions + + +def create_export_tab(): + with gr.Tab("Export"): + with gr.Tab("Export Search Results"): + search_query = gr.Textbox(label="Search Query", placeholder="Enter your search query here...") + search_fields = gr.CheckboxGroup(label="Search Fields", choices=["Title", "Content"], value=["Title"]) + keyword_input = gr.Textbox( + label="Keyword (Match ALL, can use multiple keywords, separated by ',' (comma) )", + placeholder="Enter keywords here...") + page_input = gr.Number(label="Page", value=1, precision=0) + results_per_file_input = gr.Number(label="Results per File", value=1000, precision=0) + export_format = gr.Radio(label="Export Format", choices=["csv", "markdown"], value="csv") + export_search_button = gr.Button("Export Search Results") + export_search_output = gr.File(label="Download Exported Keywords") + export_search_status = gr.Textbox(label="Export Status") + + export_search_button.click( + fn=export_to_file, + inputs=[search_query, search_fields, keyword_input, page_input, results_per_file_input, export_format], + outputs=[export_search_status, export_search_output] + ) + +# +# End of Export Items Tab Functions +################################################################################################################ +# +# Keyword Management Tab Functions + +def create_export_keywords_tab(): + with gr.Group(): + with gr.Tab("Export Keywords"): + export_keywords_button = gr.Button("Export Keywords") + export_keywords_output = gr.File(label="Download Exported Keywords") + export_keywords_status = gr.Textbox(label="Export Status") + + export_keywords_button.click( + fn=export_keywords_to_csv, + outputs=[export_keywords_status, export_keywords_output] + ) + +def create_view_keywords_tab(): + with gr.TabItem("View Keywords"): + gr.Markdown("# Browse Keywords") + browse_output = gr.Markdown() + browse_button = gr.Button("View Existing Keywords") + browse_button.click(fn=keywords_browser_interface, outputs=browse_output) + + +def create_add_keyword_tab(): + with gr.TabItem("Add Keywords"): + with gr.Row(): + gr.Markdown("# Add Keywords to the Database") + add_input = gr.Textbox(label="Add Keywords (comma-separated)", placeholder="Enter keywords here...") + add_button = gr.Button("Add Keywords") + with gr.Row(): + add_output = gr.Textbox(label="Result") + add_button.click(fn=add_keyword, inputs=add_input, outputs=add_output) + + +def create_delete_keyword_tab(): + with gr.Tab("Delete Keywords"): + with gr.Row(): + gr.Markdown("# Delete Keywords from the Database") + delete_input = gr.Textbox(label="Delete Keyword", placeholder="Enter keyword to delete here...") + delete_button = gr.Button("Delete Keyword") + with gr.Row(): + delete_output = gr.Textbox(label="Result") + delete_button.click(fn=delete_keyword, inputs=delete_input, outputs=delete_output) + +# +# End of Keyword Management Tab Functions +################################################################################################################ +# +# Utilities Tab Functions + + +def create_utilities_tab(): + with gr.Group(): + with gr.Tab("YouTube Video Downloader"): + gr.Markdown( + "

Youtube Video Downloader

This Input takes a Youtube URL as input and creates a webm file for you to download.
If you want a full-featured one: https://github.com/StefanLobbenmeier/youtube-dl-gui or https://github.com/yt-dlg/yt-dlg

") + youtube_url_input = gr.Textbox(label="YouTube URL", placeholder="Enter YouTube video URL here") + download_button = gr.Button("Download Video") + output_file = gr.File(label="Download Video") + + download_button.click( + fn=gradio_download_youtube_video, + inputs=youtube_url_input, + outputs=output_file + ) + + with gr.Tab("YouTube Audio Downloader"): + gr.Markdown( + "

Youtube Audio Downloader

This Input takes a Youtube URL as input and creates an audio file for you to download.
If you want a full-featured one: https://github.com/StefanLobbenmeier/youtube-dl-gui or https://github.com/yt-dlg/yt-dlg

") + youtube_url_input_audio = gr.Textbox(label="YouTube URL", placeholder="Enter YouTube video URL here") + download_button_audio = gr.Button("Download Audio") + output_file_audio = gr.File(label="Download Audio") + + # Implement the audio download functionality here + + with gr.Tab("Grammar Checker"): + gr.Markdown("# Grammar Check Utility to be added...") + + with gr.Tab("YouTube Timestamp URL Generator"): + gr.Markdown("## Generate YouTube URL with Timestamp") + with gr.Row(): + url_input = gr.Textbox(label="YouTube URL") + hours_input = gr.Number(label="Hours", value=0, minimum=0, precision=0) + minutes_input = gr.Number(label="Minutes", value=0, minimum=0, maximum=59, precision=0) + seconds_input = gr.Number(label="Seconds", value=0, minimum=0, maximum=59, precision=0) + + generate_button = gr.Button("Generate URL") + output_url = gr.Textbox(label="Timestamped URL") + + generate_button.click( + fn=generate_timestamped_url, + inputs=[url_input, hours_input, minutes_input, seconds_input], + outputs=output_url + ) + +# +# End of Utilities Tab Functions +################################################################################################################ + +# FIXME - Prompt sample box +# +# # Sample data +# prompts_category_1 = [ +# "What are the key points discussed in the video?", +# "Summarize the main arguments made by the speaker.", +# "Describe the conclusions of the study presented." +# ] +# +# prompts_category_2 = [ +# "How does the proposed solution address the problem?", +# "What are the implications of the findings?", +# "Can you explain the theory behind the observed phenomenon?" +# ] +# +# all_prompts2 = prompts_category_1 + prompts_category_2 + + +def launch_ui(share_public=None, server_mode=False): + share=share_public + css = """ + .result-box { + margin-bottom: 20px; + border: 1px solid #ddd; + padding: 10px; + } + .result-box.error { + border-color: #ff0000; + background-color: #ffeeee; + } + .transcription, .summary { + max-height: 300px; + overflow-y: auto; + border: 1px solid #eee; + padding: 10px; + margin-top: 10px; + } + """ + + with gr.Blocks(css=css) as iface: + gr.Markdown("# TL/DW: Too Long, Didn't Watch - Your Personal Research Multi-Tool") + with gr.Tabs(): + with gr.TabItem("Transcription / Summarization / Ingestion"): + with gr.Tabs(): + create_video_transcription_tab() + create_audio_processing_tab() + create_podcast_tab() + create_website_scraping_tab() + create_pdf_ingestion_tab() + create_resummary_tab() + + with gr.TabItem("Search / Detailed View"): + create_search_tab() + create_prompt_view_tab() + create_prompt_edit_tab() + + with gr.TabItem("Local LLM with Llamafile"): + create_llamafile_settings_tab() + + with gr.TabItem("Remote LLM Chat"): + create_chat_interface() + + with gr.TabItem("Edit Existing Items"): + create_media_edit_tab() + + with gr.TabItem("Keywords"): + with gr.Tabs(): + create_view_keywords_tab() + create_add_keyword_tab() + create_delete_keyword_tab() + create_export_keywords_tab() + + with gr.TabItem("Import/Export"): + create_import_item_tab() + create_export_tab() + + with gr.TabItem("Utilities"): + create_utilities_tab() + + # Launch the interface + server_port_variable = 7860 + if share==True: + iface.launch(share=True) + elif server_mode and not share_public: + iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable) + else: + iface.launch(share=False) + diff --git a/App_Function_Libraries/LLM_API_Calls.py b/App_Function_Libraries/LLM_API_Calls.py new file mode 100644 index 0000000000000000000000000000000000000000..9ca640e38887c06aa13502b1c9309f74ab890122 --- /dev/null +++ b/App_Function_Libraries/LLM_API_Calls.py @@ -0,0 +1,633 @@ +# Summarization_General_Lib.py +######################################### +# General Summarization Library +# This library is used to perform summarization. +# +#### +#################### +# Function List +# +# 1. extract_text_from_segments(segments: List[Dict]) -> str +# 2. chat_with_openai(api_key, file_path, custom_prompt_arg) +# 3. chat_with_anthropic(api_key, file_path, model, custom_prompt_arg, max_retries=3, retry_delay=5) +# 4. chat_with_cohere(api_key, file_path, model, custom_prompt_arg) +# 5. chat_with_groq(api_key, input_data, custom_prompt_arg, system_prompt=None): +# 6. chat_with_openrouter(api_key, input_data, custom_prompt_arg, system_prompt=None) +# 7. chat_with_huggingface(api_key, input_data, custom_prompt_arg, system_prompt=None) +# 8. chat_with_deepseek(api_key, input_data, custom_prompt_arg, system_prompt=None) +# 9. chat_with_vllm(input_data, custom_prompt_input, api_key=None, vllm_api_url="http://127.0.0.1:8000/v1/chat/completions", system_prompt=None) +# +# +#################### +import json +# Import necessary libraries +import os +import logging +import time +import requests +import configparser +# Import 3rd-Party Libraries +from openai import OpenAI +from requests import RequestException +# Import Local libraries +from App_Function_Libraries.Local_Summarization_Lib import openai_api_key, client +from App_Function_Libraries.Utils import load_and_log_configs +# +####################################################################################################################### +# Function Definitions +# + +def extract_text_from_segments(segments): + logging.debug(f"Segments received: {segments}") + logging.debug(f"Type of segments: {type(segments)}") + + text = "" + + if isinstance(segments, list): + for segment in segments: + logging.debug(f"Current segment: {segment}") + logging.debug(f"Type of segment: {type(segment)}") + if 'Text' in segment: + text += segment['Text'] + " " + else: + logging.warning(f"Skipping segment due to missing 'Text' key: {segment}") + else: + logging.warning(f"Unexpected type of 'segments': {type(segments)}") + + return text.strip() + + + + + +def chat_with_openai(api_key, input_data, custom_prompt_arg, system_prompt=None): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None or api_key.strip() == "": + logging.info("OpenAI: API key not provided as parameter") + logging.info("OpenAI: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['openai'] + + if api_key is None or api_key.strip() == "": + logging.error("OpenAI: API key not found or is empty") + return "OpenAI: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"OpenAI: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + logging.debug("OpenAI: Using provided string data for chat input") + data = input_data + + logging.debug(f"OpenAI: Loaded data: {data}") + logging.debug(f"OpenAI: Type of data: {type(data)}") + + if system_prompt is not None: + logging.debug(f"OpenAI: Using provided system prompt:\n\n {system_prompt}") + pass + else: + system_prompt = "You are a helpful assistant" + logging.debug(f"OpenAI: Using default system prompt:\n\n {system_prompt}") + + openai_model = loaded_config_data['models']['openai'] or "gpt-4o" + + headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json' + } + + logging.debug( + f"OpenAI API Key: {openai_api_key[:5]}...{openai_api_key[-5:] if openai_api_key else None}") + logging.debug("openai: Preparing data + prompt for submittal") + openai_prompt = f"{data} \n\n\n\n{custom_prompt_arg}" + data = { + "model": openai_model, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": openai_prompt} + ], + "max_tokens": 4096, + "temperature": 0.1 + } + + logging.debug("openai: Posting request") + response = requests.post('https://api.openai.com/v1/chat/completions', headers=headers, json=data) + + if response.status_code == 200: + response_data = response.json() + if 'choices' in response_data and len(response_data['choices']) > 0: + chat_response = response_data['choices'][0]['message']['content'].strip() + logging.debug("openai: Chat Sent successfully") + return chat_response + else: + logging.warning("openai: Chat response not found in the response data") + return "openai: Chat not available" + else: + logging.error(f"openai: Chat request failed with status code {response.status_code}") + logging.error(f"openai: Error response: {response.text}") + return f"openai: Failed to process chat request. Status code: {response.status_code}" + except Exception as e: + logging.error(f"openai: Error in processing: {str(e)}", exc_info=True) + return f"openai: Error occurred while processing chat request: {str(e)}" + + +def chat_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5, system_prompt=None): + try: + loaded_config_data = load_and_log_configs() + global anthropic_api_key + # API key validation + if api_key is None: + logging.info("Anthropic: API key not provided as parameter") + logging.info("Anthropic: Attempting to use API key from config file") + anthropic_api_key = loaded_config_data['api_keys']['anthropic'] + + if api_key is None or api_key.strip() == "": + logging.error("Anthropic: API key not found or is empty") + return "Anthropic: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"Anthropic: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + if system_prompt is not None: + logging.debug("Anthropic: Using provided system prompt") + pass + else: + system_prompt = "You are a helpful assistant" + + logging.debug(f"AnthropicAI: Loaded data: {input_data}") + logging.debug(f"AnthropicAI: Type of data: {type(input_data)}") + + anthropic_model = loaded_config_data['models']['anthropic'] + + headers = { + 'x-api-key': anthropic_api_key, + 'anthropic-version': '2023-06-01', + 'Content-Type': 'application/json' + } + + anthropic_user_prompt = custom_prompt_arg + logging.debug(f"Anthropic: User Prompt is {anthropic_user_prompt}") + user_message = { + "role": "user", + "content": f"{input_data} \n\n\n\n{anthropic_user_prompt}" + } + + data = { + "model": model, + "max_tokens": 4096, # max _possible_ tokens to return + "messages": [user_message], + "stop_sequences": ["\n\nHuman:"], + "temperature": 0.1, + "top_k": 0, + "top_p": 1.0, + "metadata": { + "user_id": "example_user_id", + }, + "stream": False, + "system": f"{system_prompt}" + } + + for attempt in range(max_retries): + try: + logging.debug("anthropic: Posting request to API") + response = requests.post('https://api.anthropic.com/v1/messages', headers=headers, json=data) + + # Check if the status code indicates success + if response.status_code == 200: + logging.debug("anthropic: Post submittal successful") + response_data = response.json() + try: + chat_response = response_data['content'][0]['text'].strip() + logging.debug("anthropic: Chat request successful") + print("Chat request processed successfully.") + return chat_response + except (IndexError, KeyError) as e: + logging.debug("anthropic: Unexpected data in response") + print("Unexpected response format from Anthropic API:", response.text) + return None + elif response.status_code == 500: # Handle internal server error specifically + logging.debug("anthropic: Internal server error") + print("Internal server error from API. Retrying may be necessary.") + time.sleep(retry_delay) + else: + logging.debug( + f"anthropic: Failed to process chat request, status code {response.status_code}: {response.text}") + print(f"Failed to process chat request, status code {response.status_code}: {response.text}") + return None + + except RequestException as e: + logging.error(f"anthropic: Network error during attempt {attempt + 1}/{max_retries}: {str(e)}") + if attempt < max_retries - 1: + time.sleep(retry_delay) + else: + return f"anthropic: Network error: {str(e)}" + except Exception as e: + logging.error(f"anthropic: Error in processing: {str(e)}") + return f"anthropic: Error occurred while processing summary with Anthropic: {str(e)}" + + +# Summarize with Cohere +def chat_with_cohere(api_key, input_data, model, custom_prompt_arg, system_prompt=None): + global cohere_api_key + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None: + logging.info("cohere: API key not provided as parameter") + logging.info("cohere: Attempting to use API key from config file") + cohere_api_key = loaded_config_data['api_keys']['cohere'] + + if api_key is None or api_key.strip() == "": + logging.error("cohere: API key not found or is empty") + return "cohere: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"cohere: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + logging.debug(f"Cohere: Loaded data: {input_data}") + logging.debug(f"Cohere: Type of data: {type(input_data)}") + + cohere_model = loaded_config_data['models']['cohere'] + + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + 'Authorization': f'Bearer {cohere_api_key}' + } + + if system_prompt is not None: + logging.debug("Anthropic: Using provided system prompt") + pass + else: + system_prompt = "You are a helpful assistant" + + cohere_prompt = f"{input_data} \n\n\n\n{custom_prompt_arg}" + logging.debug(f"cohere: User Prompt being sent is {cohere_prompt}") + + logging.debug(f"cohere: System Prompt being sent is {system_prompt}") + + data = { + "chat_history": [ + {"role": "SYSTEM", "message": f"system_prompt"}, + ], + "message": f"{cohere_prompt}", + "model": model, + "connectors": [{"id": "web-search"}] + } + + logging.debug("cohere: Submitting request to API endpoint") + print("cohere: Submitting request to API endpoint") + response = requests.post('https://api.cohere.ai/v1/chat', headers=headers, json=data) + response_data = response.json() + logging.debug("API Response Data: %s", response_data) + + if response.status_code == 200: + if 'text' in response_data: + chat_response = response_data['text'].strip() + logging.debug("cohere: Chat request successful") + print("Chat request processed successfully.") + return chat_response + else: + logging.error("Expected data not found in API response.") + return "Expected data not found in API response." + else: + logging.error(f"cohere: API request failed with status code {response.status_code}: {response.text}") + print(f"Failed to process summary, status code {response.status_code}: {response.text}") + return f"cohere: API request failed: {response.text}" + + except Exception as e: + logging.error("cohere: Error in processing: %s", str(e)) + return f"cohere: Error occurred while processing summary with Cohere: {str(e)}" + + +# https://console.groq.com/docs/quickstart +def chat_with_groq(api_key, input_data, custom_prompt_arg, system_prompt=None): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None: + logging.info("groq: API key not provided as parameter") + logging.info("groq: Attempting to use API key from config file") + groq_api_key = loaded_config_data['api_keys']['groq'] + + if api_key is None or api_key.strip() == "": + logging.error("groq: API key not found or is empty") + return "groq: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"groq: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + logging.debug(f"Groq: Loaded data: {input_data}") + logging.debug(f"Groq: Type of data: {type(input_data)}") + + # Set the model to be used + groq_model = loaded_config_data['models']['groq'] + + headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json' + } + + if system_prompt is not None: + logging.debug("Groq: Using provided system prompt") + pass + else: + system_prompt = "You are a helpful assistant" + + groq_prompt = f"{input_data} \n\n\n\n{custom_prompt_arg}" + logging.debug("groq: User Prompt being sent is {groq_prompt}") + + logging.debug("groq: System Prompt being sent is {system_prompt}") + + data = { + "messages": [ + { + "role": "system", + "content": f"{system_prompt}" + }, + { + "role": "user", + "content": groq_prompt + } + ], + "model": groq_model + } + + logging.debug("groq: Submitting request to API endpoint") + print("groq: Submitting request to API endpoint") + response = requests.post('https://api.groq.com/openai/v1/chat/completions', headers=headers, json=data) + + response_data = response.json() + logging.debug("API Response Data: %s", response_data) + + if response.status_code == 200: + if 'choices' in response_data and len(response_data['choices']) > 0: + summary = response_data['choices'][0]['message']['content'].strip() + logging.debug("groq: Summarization successful") + print("Summarization successful.") + return summary + else: + logging.error("Expected data not found in API response.") + return "Expected data not found in API response." + else: + logging.error(f"groq: API request failed with status code {response.status_code}: {response.text}") + return f"groq: API request failed: {response.text}" + + except Exception as e: + logging.error("groq: Error in processing: %s", str(e)) + return f"groq: Error occurred while processing summary with groq: {str(e)}" + + +def chat_with_openrouter(api_key, input_data, custom_prompt_arg, system_prompt=None): + loaded_config_data = load_and_log_configs() + import requests + import json + global openrouter_model, openrouter_api_key + # API key validation + if api_key is None: + logging.info("openrouter: API key not provided as parameter") + logging.info("openrouter: Attempting to use API key from config file") + openrouter_api_key = loaded_config_data['api_keys']['openrouter'] + + if api_key is None or api_key.strip() == "": + logging.error("openrouter: API key not found or is empty") + return "openrouter: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"openai: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + logging.debug(f"openrouter: Loaded data: {input_data}") + logging.debug(f"openrouter: Type of data: {type(input_data)}") + + config = configparser.ConfigParser() + file_path = 'config.txt' + + # Check if the file exists in the specified path + if os.path.exists(file_path): + config.read(file_path) + elif os.path.exists('config.txt'): # Check in the current directory + config.read('../config.txt') + else: + print("config.txt not found in the specified path or current directory.") + openrouter_model = loaded_config_data['models']['openrouter'] + + if system_prompt is not None: + logging.debug("OpenRouter: Using provided system prompt") + pass + else: + system_prompt = "You are a helpful assistant" + + openrouter_prompt = f"{input_data} \n\n\n\n{custom_prompt_arg}" + logging.debug(f"openrouter: User Prompt being sent is {openrouter_prompt}") + + logging.debug(f"openrouter: System Prompt being sent is {system_prompt}") + + try: + logging.debug("openrouter: Submitting request to API endpoint") + print("openrouter: Submitting request to API endpoint") + response = requests.post( + url="https://openrouter.ai/api/v1/chat/completions", + headers={ + "Authorization": f"Bearer {openrouter_api_key}", + }, + data=json.dumps({ + "model": f"{openrouter_model}", + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": openrouter_prompt} + ] + }) + ) + + response_data = response.json() + logging.debug("API Response Data: %s", response_data) + + if response.status_code == 200: + if 'choices' in response_data and len(response_data['choices']) > 0: + summary = response_data['choices'][0]['message']['content'].strip() + logging.debug("openrouter: Chat request successful") + print("openrouter: Chat request successful.") + return summary + else: + logging.error("openrouter: Expected data not found in API response.") + return "openrouter: Expected data not found in API response." + else: + logging.error(f"openrouter: API request failed with status code {response.status_code}: {response.text}") + return f"openrouter: API request failed: {response.text}" + except Exception as e: + logging.error("openrouter: Error in processing: %s", str(e)) + return f"openrouter: Error occurred while processing chat request with openrouter: {str(e)}" + +# FIXME: This function is not yet implemented properly +def chat_with_huggingface(api_key, input_data, custom_prompt_arg, system_prompt=None): + loaded_config_data = load_and_log_configs() + global huggingface_api_key + logging.debug(f"huggingface: Summarization process starting...") + try: + # API key validation + if api_key is None: + logging.info("HuggingFace: API key not provided as parameter") + logging.info("HuggingFace: Attempting to use API key from config file") + huggingface_api_key = loaded_config_data['api_keys']['openai'] + if api_key is None or api_key.strip() == "": + logging.error("HuggingFace: API key not found or is empty") + return "HuggingFace: API Key Not Provided/Found in Config file or is empty" + logging.debug(f"HuggingFace: Using API Key: {api_key[:5]}...{api_key[-5:]}") + headers = { + "Authorization": f"Bearer {api_key}" + } + + # Setup model + huggingface_model = loaded_config_data['models']['huggingface'] + + API_URL = f"https://api-inference.huggingface.co/models/{huggingface_model}" + if system_prompt is not None: + logging.debug("HuggingFace: Using provided system prompt") + pass + else: + system_prompt = "You are a helpful assistant" + + huggingface_prompt = f"{input_data}\n\n\n\n{custom_prompt_arg}" + logging.debug("huggingface: Prompt being sent is {huggingface_prompt}") + data = { + "inputs": f"{input_data}", + "parameters": {"max_length": 8192, "min_length": 100} # You can adjust max_length and min_length as needed + } + logging.debug("huggingface: Submitting request...") + + response = requests.post(API_URL, headers=headers, json=data) + + if response.status_code == 200: + summary = response.json()[0]['summary_text'] + logging.debug("huggingface: Chat request successful") + print("Chat request successful.") + return summary + else: + logging.error(f"huggingface: Chat request failed with status code {response.status_code}: {response.text}") + return f"Failed to process chat request, status code {response.status_code}: {response.text}" + except Exception as e: + logging.error("huggingface: Error in processing: %s", str(e)) + print(f"Error occurred while processing chat request with huggingface: {str(e)}") + return None + + +def chat_with_deepseek(api_key, input_data, custom_prompt_arg, system_prompt=None): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None or api_key.strip() == "": + logging.info("DeepSeek: API key not provided as parameter") + logging.info("DeepSeek: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['deepseek'] + + if api_key is None or api_key.strip() == "": + logging.error("DeepSeek: API key not found or is empty") + return "DeepSeek: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"DeepSeek: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + deepseek_model = loaded_config_data['models']['deepseek'] or "deepseek-chat" + + headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json' + } + + if system_prompt is not None: + logging.debug(f"Deepseek: Using provided system prompt: {system_prompt}") + pass + else: + system_prompt = "You are a helpful assistant" + + logging.debug( + f"Deepseek API Key: {api_key[:5]}...{api_key[-5:] if api_key else None}") + logging.debug("openai: Preparing data + prompt for submittal") + deepseek_prompt = f"{input_data} \n\n\n\n{custom_prompt_arg}" + data = { + "model": deepseek_model, + "messages": [ + {"role": "system", "content": f"{system_prompt}"}, + {"role": "user", "content": deepseek_prompt} + ], + "stream": False, + "temperature": 0.8 + } + + logging.debug("DeepSeek: Posting request") + response = requests.post('https://api.deepseek.com/chat/completions', headers=headers, json=data) + + if response.status_code == 200: + response_data = response.json() + if 'choices' in response_data and len(response_data['choices']) > 0: + summary = response_data['choices'][0]['message']['content'].strip() + logging.debug("DeepSeek: Chat request successful") + return summary + else: + logging.warning("DeepSeek: Chat response not found in the response data") + return "DeepSeek: Chat response not available" + else: + logging.error(f"DeepSeek: Chat request failed with status code {response.status_code}") + logging.error(f"DeepSeek: Error response: {response.text}") + return f"DeepSeek: Failed to chat request summary. Status code: {response.status_code}" + except Exception as e: + logging.error(f"DeepSeek: Error in processing: {str(e)}", exc_info=True) + return f"DeepSeek: Error occurred while processing chat request: {str(e)}" + + + +# Stashed in here since OpenAI usage.... #FIXME +# FIXME - https://docs.vllm.ai/en/latest/getting_started/quickstart.html .... Great docs. +def chat_with_vllm(input_data, custom_prompt_input, api_key=None, vllm_api_url="http://127.0.0.1:8000/v1/chat/completions", system_prompt=None): + loaded_config_data = load_and_log_configs() + llm_model = loaded_config_data['models']['vllm'] + # API key validation + if api_key is None: + logging.info("vLLM: API key not provided as parameter") + logging.info("vLLM: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['llama'] + + if api_key is None or api_key.strip() == "": + logging.info("vLLM: API key not found or is empty") + vllm_client = OpenAI( + base_url=vllm_api_url, + api_key=custom_prompt_input + ) + + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("vLLM: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("vLLM: Using provided string data for summarization") + data = input_data + + logging.debug(f"vLLM: Loaded data: {data}") + logging.debug(f"vLLM: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("vLLM: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Invalid input data format") + + + custom_prompt = custom_prompt_input + + completion = client.chat.completions.create( + model=llm_model, + messages=[ + {"role": "system", "content": f"{system_prompt}"}, + {"role": "user", "content": f"{text} \n\n\n\n{custom_prompt}"} + ] + ) + vllm_summary = completion.choices[0].message.content + return vllm_summary + + + +# +# +####################################################################################################################### \ No newline at end of file diff --git a/App_Function_Libraries/LLM_API_Calls_Local.py b/App_Function_Libraries/LLM_API_Calls_Local.py new file mode 100644 index 0000000000000000000000000000000000000000..a1cac7bae41d8184e4751052d501b12e64a69d03 --- /dev/null +++ b/App_Function_Libraries/LLM_API_Calls_Local.py @@ -0,0 +1,348 @@ +# Local_Summarization_Lib.py +######################################### +# Local Summarization Library +# This library is used to perform summarization with a 'local' inference engine. +# +#### + +#################### +# Function List +# FIXME - UPDATE Function Arguments +# 1. chat_with_local_llm(text, custom_prompt_arg) +# 2. chat_with_llama(api_url, text, token, custom_prompt) +# 3. chat_with_kobold(api_url, text, kobold_api_token, custom_prompt) +# 4. chat_with_oobabooga(api_url, text, ooba_api_token, custom_prompt) +# 5. chat_with_vllm(vllm_api_url, vllm_api_key_function_arg, llm_model, text, vllm_custom_prompt_function_arg) +# 6. chat_with_tabbyapi(tabby_api_key, tabby_api_IP, text, tabby_model, custom_prompt) +# 7. save_summary_to_file(summary, file_path) +# +# +#################### +# Import necessary libraries +import json +# Import Local +from Utils import * +# +####################################################################################################################### +# Function Definitions +# + + +def chat_with_local_llm(input_data, user_prompt, system_prompt=None): + try: + if system_prompt is None: + system_prompt_arg = "You are a helpful assistant." + + headers = { + 'Content-Type': 'application/json' + } + + logging.debug("Local LLM: Preparing data + prompt for submittal") + local_llm_prompt = f"{user_prompt}\n\n\n\n{input_data} " + data = { + "messages": [ + { + "role": "system", + "content": f"{system_prompt}" + }, + { + "role": "user", + "content": f"{local_llm_prompt}" + } + ], + "max_tokens": 28000, # Adjust tokens as needed + } + logging.debug("Local LLM: System Prompt to be used: %s", system_prompt) + logging.debug("Local LLM: User Prompt to be used: %s", user_prompt) + logging.debug("Local LLM: Posting request") + response = requests.post('http://127.0.0.1:8080/v1/chat/completions', headers=headers, json=data) + + if response.status_code == 200: + response_data = response.json() + if 'choices' in response_data and len(response_data['choices']) > 0: + summary = response_data['choices'][0]['message']['content'].strip() + logging.debug("Local LLM: Chat request successful") + print("Local LLM: Chat request successful.") + return summary + else: + logging.warning("Local LLM: Chat response not found in the response data") + return "Local LLM: Chat response not available" + else: + logging.debug("Local LLM: Chat request failed") + print("Local LLM: Failed to process Chat response:", response.text) + return "Local LLM: Failed to process Chat response" + except Exception as e: + logging.debug("Local LLM: Error in processing: %s", str(e)) + print("Error occurred while processing Chat request with Local LLM:", str(e)) + return "Local LLM: Error occurred while processing Chat response" + +def chat_with_llama(input_data, custom_prompt, api_url="http://127.0.0.1:8080/completion", api_key=None, system_prompt=None): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None: + logging.info("llama.cpp: API key not provided as parameter") + logging.info("llama.cpp: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['llama'] + + if api_key is None or api_key.strip() == "": + logging.info("llama.cpp: API key not found or is empty") + + logging.debug(f"llama.cpp: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + } + if len(api_key) > 5: + headers['Authorization'] = f'Bearer {api_key}' + + if system_prompt is None: + system_prompt = "You are a helpful AI assistant that provides accurate and concise information." + + logging.debug("Llama.cpp: System prompt being used is: %s", system_prompt) + logging.debug("Llama.cpp: User prompt being used is: %s", custom_prompt) + + + llama_prompt = f"{custom_prompt} \n\n\n\n{input_data}" + logging.debug(f"llama: Prompt being sent is {llama_prompt}") + + data = { + "prompt": f"{llama_prompt}", + "system_prompt": f"{system_prompt}" + } + + logging.debug("llama: Submitting request to API endpoint") + print("llama: Submitting request to API endpoint") + response = requests.post(api_url, headers=headers, json=data) + response_data = response.json() + logging.debug("API Response Data: %s", response_data) + + if response.status_code == 200: + # if 'X' in response_data: + logging.debug(response_data) + summary = response_data['content'].strip() + logging.debug("llama: Summarization successful") + print("Summarization successful.") + return summary + else: + logging.error(f"Llama: API request failed with status code {response.status_code}: {response.text}") + return f"Llama: API request failed: {response.text}" + + except Exception as e: + logging.error("Llama: Error in processing: %s", str(e)) + return f"Llama: Error occurred while processing summary with llama: {str(e)}" + + +# System prompts not supported through API requests. +# https://lite.koboldai.net/koboldcpp_api#/api%2Fv1/post_api_v1_generate +def chat_with_kobold(input_data, api_key, custom_prompt_input, kobold_api_IP="http://127.0.0.1:5001/api/v1/generate"): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None: + logging.info("Kobold.cpp: API key not provided as parameter") + logging.info("Kobold.cpp: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['kobold'] + + if api_key is None or api_key.strip() == "": + logging.info("Kobold.cpp: API key not found or is empty") + + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + } + + kobold_prompt = f"{custom_prompt_input} \n\n\n\n{input_data}" + logging.debug("kobold: Prompt being sent is {kobold_prompt}") + + # FIXME + # Values literally c/p from the api docs.... + data = { + "max_context_length": 8096, + "max_length": 4096, + "prompt": f"{custom_prompt_input}\n\n\n\n{input_data}" + } + + logging.debug("kobold: Submitting request to API endpoint") + print("kobold: Submitting request to API endpoint") + response = requests.post(kobold_api_IP, headers=headers, json=data) + response_data = response.json() + logging.debug("kobold: API Response Data: %s", response_data) + + if response.status_code == 200: + if 'results' in response_data and len(response_data['results']) > 0: + summary = response_data['results'][0]['text'].strip() + logging.debug("kobold: Chat request successful!") + print("Chat request successful!") + return summary + else: + logging.error("Expected data not found in API response.") + return "Expected data not found in API response." + else: + logging.error(f"kobold: API request failed with status code {response.status_code}: {response.text}") + return f"kobold: API request failed: {response.text}" + + except Exception as e: + logging.error("kobold: Error in processing: %s", str(e)) + return f"kobold: Error occurred while processing chat response with kobold: {str(e)}" + +# System prompt doesn't work. FIXME +# https://github.com/oobabooga/text-generation-webui/wiki/12-%E2%80%90-OpenAI-API +def chat_with_oobabooga(input_data, api_key, custom_prompt, api_url="http://127.0.0.1:5000/v1/chat/completions", system_prompt=None): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None: + logging.info("ooba: API key not provided as parameter") + logging.info("ooba: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['ooba'] + + if api_key is None or api_key.strip() == "": + logging.info("ooba: API key not found or is empty") + + if system_prompt is None: + system_prompt = "You are a helpful AI assistant that provides accurate and concise information." + + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + } + + # prompt_text = "I like to eat cake and bake cakes. I am a baker. I work in a French bakery baking cakes. It + # is a fun job. I have been baking cakes for ten years. I also bake lots of other baked goods, but cakes are + # my favorite." prompt_text += f"\n\n{text}" # Uncomment this line if you want to include the text variable + ooba_prompt = f"{input_data}" + f"\n\n\n\n{custom_prompt}" + logging.debug("ooba: Prompt being sent is {ooba_prompt}") + + data = { + "mode": "chat", + "character": "Example", + "messages": [{"role": "user", "content": ooba_prompt}] + } + + logging.debug("ooba: Submitting request to API endpoint") + print("ooba: Submitting request to API endpoint") + response = requests.post(api_url, headers=headers, json=data, verify=False) + logging.debug("ooba: API Response Data: %s", response) + + if response.status_code == 200: + response_data = response.json() + summary = response.json()['choices'][0]['message']['content'] + logging.debug("ooba: Summarization successful") + print("Summarization successful.") + return summary + else: + logging.error(f"oobabooga: API request failed with status code {response.status_code}: {response.text}") + return f"ooba: API request failed with status code {response.status_code}: {response.text}" + + except Exception as e: + logging.error("ooba: Error in processing: %s", str(e)) + return f"ooba: Error occurred while processing summary with oobabooga: {str(e)}" + + +# FIXME - Install is more trouble than care to deal with right now. +def chat_with_tabbyapi(input_data, custom_prompt_input, api_key=None, api_IP="http://127.0.0.1:5000/v1/chat/completions"): + loaded_config_data = load_and_log_configs() + model = loaded_config_data['models']['tabby'] + # API key validation + if api_key is None: + logging.info("tabby: API key not provided as parameter") + logging.info("tabby: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['tabby'] + + if api_key is None or api_key.strip() == "": + logging.info("tabby: API key not found or is empty") + + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("tabby: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("tabby: Using provided string data for summarization") + data = input_data + + logging.debug(f"tabby: Loaded data: {data}") + logging.debug(f"tabby: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("tabby: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Invalid input data format") + + headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json' + } + data2 = { + 'text': text, + 'model': 'tabby' # Specify the model if needed + } + tabby_api_ip = loaded_config_data['local_apis']['tabby']['ip'] + try: + response = requests.post(tabby_api_ip, headers=headers, json=data2) + response.raise_for_status() + summary = response.json().get('summary', '') + return summary + except requests.exceptions.RequestException as e: + logging.error(f"Error summarizing with TabbyAPI: {e}") + return "Error summarizing with TabbyAPI." + + +# FIXME aphrodite engine - code was literally tab complete in one go from copilot... :/ +def chat_with_aphrodite(input_data, custom_prompt_input, api_key=None, api_IP="http://" + load_and_log_configs()['local_apis']['aphrodite']['ip']): + loaded_config_data = load_and_log_configs() + model = loaded_config_data['models']['aphrodite'] + # API key validation + if api_key is None: + logging.info("aphrodite: API key not provided as parameter") + logging.info("aphrodite: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['aphrodite'] + + if api_key is None or api_key.strip() == "": + logging.info("aphrodite: API key not found or is empty") + + headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json' + } + data2 = { + 'text': input_data, + } + try: + response = requests.post(api_IP, headers=headers, json=data2) + response.raise_for_status() + summary = response.json().get('summary', '') + return summary + except requests.exceptions.RequestException as e: + logging.error(f"Error summarizing with Aphrodite: {e}") + return "Error summarizing with Aphrodite." + + + + +def save_summary_to_file(summary, file_path): + logging.debug("Now saving summary to file...") + base_name = os.path.splitext(os.path.basename(file_path))[0] + summary_file_path = os.path.join(os.path.dirname(file_path), base_name + '_summary.txt') + os.makedirs(os.path.dirname(summary_file_path), exist_ok=True) + logging.debug("Opening summary file for writing, *segments.json with *_summary.txt") + with open(summary_file_path, 'w') as file: + file.write(summary) + logging.info(f"Summary saved to file: {summary_file_path}") + +# +# +####################################################################################################################### + + + diff --git a/App_Function_Libraries/Local_File_Processing_Lib.py b/App_Function_Libraries/Local_File_Processing_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..649c0b19677711d6d4288e851982c8d98653ac69 --- /dev/null +++ b/App_Function_Libraries/Local_File_Processing_Lib.py @@ -0,0 +1,90 @@ +# Local_File_Processing_Lib.py +######################################### +# Local File Processing and File Path Handling Library +# This library is used to handle processing local filepaths and URLs. +# It checks for the OS, the availability of the GPU, and the availability of the ffmpeg executable. +# If the GPU is available, it asks the user if they would like to use it for processing. +# If ffmpeg is not found, it asks the user if they would like to download it. +# The script will exit if the user chooses not to download ffmpeg. +#### + +#################### +# Function List +# +# 1. read_paths_from_file(file_path) +# 2. process_path(path) +# 3. process_local_file(file_path) +# 4. read_paths_from_file(file_path: str) -> List[str] +# +#################### + +# Import necessary libraries +# Import Local +from App_Function_Libraries.Audio_Transcription_Lib import convert_to_wav +from App_Function_Libraries.Video_DL_Ingestion_Lib import * +from App_Function_Libraries.Video_DL_Ingestion_Lib import get_youtube +from App_Function_Libraries.Utils import normalize_title, create_download_directory + +####################################################################################################################### +# Function Definitions +# + +def read_paths_from_file(file_path): + """ Reads a file containing URLs or local file paths and returns them as a list. """ + paths = [] # Initialize paths as an empty list + with open(file_path, 'r') as file: + paths = file.readlines() + return [path.strip() for path in paths] + + +def process_path(path): + """ Decides whether the path is a URL or a local file and processes accordingly. """ + if path.startswith('http'): + logging.debug("file is a URL") + # For YouTube URLs, modify to download and extract info + return get_youtube(path) + elif os.path.exists(path): + logging.debug("File is a path") + # For local files, define a function to handle them + return process_local_file(path) + else: + logging.error(f"Path does not exist: {path}") + return None + + +# FIXME - ingest_text is not used, need to confirm. +def process_local_file(file_path, ingest_text=False): + logging.info(f"Processing local file: {file_path}") + file_extension = os.path.splitext(file_path)[1].lower() + + if os.path.isfile(file_path): + if file_path.lower().endswith('.txt'): + if ingest_text: + # Treat as content to be ingested + return os.path.dirname(file_path), {'title': os.path.basename(file_path)}, file_path + else: + # Treat as potential list of URLs + with open(file_path, 'r') as file: + urls = file.read().splitlines() + return None, None, urls + elif file_path.lower().endswith(('.mp4', '.avi', '.mov', '.wav', '.mp3', '.m4a')): + # Handle video and audio files (existing code) + title = normalize_title(os.path.splitext(os.path.basename(file_path))[0]) + info_dict = {'title': title} + logging.debug(f"Creating {title} directory...") + download_path = create_download_directory(title) + logging.debug(f"Converting '{title}' to an audio file (wav).") + audio_file = convert_to_wav(file_path) + logging.debug(f"'{title}' successfully converted to an audio file (wav).") + return download_path, info_dict, audio_file + else: + logging.error(f"File not found: {file_path}") + return None, None, None + + + + + +# +# +####################################################################################################################### \ No newline at end of file diff --git a/App_Function_Libraries/Local_LLM_Inference_Engine_Lib.py b/App_Function_Libraries/Local_LLM_Inference_Engine_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..62272adc4585dbc78137288ce2a77f20e1e86a9c --- /dev/null +++ b/App_Function_Libraries/Local_LLM_Inference_Engine_Lib.py @@ -0,0 +1,590 @@ +# Local_LLM_Inference_Engine_Lib.py +######################################### +# Local LLM Inference Engine Library +# This library is used to handle downloading, configuring, and launching the Local LLM Inference Engine +# via (llama.cpp via llamafile) +# +# +#### +#################### +# Function List +# +# 1. download_latest_llamafile(repo, asset_name_prefix, output_filename) +# 2. download_file(url, dest_path, expected_checksum=None, max_retries=3, delay=5) +# 3. verify_checksum(file_path, expected_checksum) +# 4. cleanup_process() +# 5. signal_handler(sig, frame) +# 6. local_llm_function() +# 7. launch_in_new_terminal_windows(executable, args) +# 8. launch_in_new_terminal_linux(executable, args) +# 9. launch_in_new_terminal_mac(executable, args) +# +#################### +# Import necessary libraries +from asyncio import subprocess +import atexit +import re +import sys +import time +# Import 3rd-pary Libraries +# +# Import Local +from Article_Summarization_Lib import * +from App_Function_Libraries.Utils import download_file +# +# +####################################################################################################################### +# Function Definitions +# + +# Download latest llamafile from Github + # Example usage + #repo = "Mozilla-Ocho/llamafile" + #asset_name_prefix = "llamafile-" + #output_filename = "llamafile" + #download_latest_llamafile(repo, asset_name_prefix, output_filename) + +# THIS SHOULD ONLY BE CALLED IF THE USER IS USING THE GUI TO SETUP LLAMAFILE +# Function is used to download only llamafile +def download_latest_llamafile_no_model(output_filename): + # Check if the file already exists + print("Checking for and downloading Llamafile it it doesn't already exist...") + if os.path.exists(output_filename): + print("Llamafile already exists. Skipping download.") + logging.debug(f"{output_filename} already exists. Skipping download.") + llamafile_exists = True + else: + llamafile_exists = False + + if llamafile_exists == True: + pass + else: + # Establish variables for Llamafile download + repo = "Mozilla-Ocho/llamafile" + asset_name_prefix = "llamafile-" + # Get the latest release information + latest_release_url = f"https://api.github.com/repos/{repo}/releases/latest" + response = requests.get(latest_release_url) + if response.status_code != 200: + raise Exception(f"Failed to fetch latest release info: {response.status_code}") + + latest_release_data = response.json() + tag_name = latest_release_data['tag_name'] + + # Get the release details using the tag name + release_details_url = f"https://api.github.com/repos/{repo}/releases/tags/{tag_name}" + response = requests.get(release_details_url) + if response.status_code != 200: + raise Exception(f"Failed to fetch release details for tag {tag_name}: {response.status_code}") + + release_data = response.json() + assets = release_data.get('assets', []) + + # Find the asset with the specified prefix + asset_url = None + for asset in assets: + if re.match(f"{asset_name_prefix}.*", asset['name']): + asset_url = asset['browser_download_url'] + break + + if not asset_url: + raise Exception(f"No asset found with prefix {asset_name_prefix}") + + # Download the asset + response = requests.get(asset_url) + if response.status_code != 200: + raise Exception(f"Failed to download asset: {response.status_code}") + + print("Llamafile downloaded successfully.") + logging.debug("Main: Llamafile downloaded successfully.") + + # Save the file + with open(output_filename, 'wb') as file: + file.write(response.content) + + logging.debug(f"Downloaded {output_filename} from {asset_url}") + print(f"Downloaded {output_filename} from {asset_url}") + return output_filename + + +# FIXME - Add option in GUI for selecting the other models for download +# Should only be called from 'local_llm_gui_function' - if its called from anywhere else, shits broken. +# Function is used to download llamafile + A model from Huggingface +def download_latest_llamafile_through_gui(repo, asset_name_prefix, output_filename): + # Check if the file already exists + print("Checking for and downloading Llamafile it it doesn't already exist...") + if os.path.exists(output_filename): + print("Llamafile already exists. Skipping download.") + logging.debug(f"{output_filename} already exists. Skipping download.") + llamafile_exists = True + else: + llamafile_exists = False + + if llamafile_exists == True: + pass + else: + # Get the latest release information + latest_release_url = f"https://api.github.com/repos/{repo}/releases/latest" + response = requests.get(latest_release_url) + if response.status_code != 200: + raise Exception(f"Failed to fetch latest release info: {response.status_code}") + + latest_release_data = response.json() + tag_name = latest_release_data['tag_name'] + + # Get the release details using the tag name + release_details_url = f"https://api.github.com/repos/{repo}/releases/tags/{tag_name}" + response = requests.get(release_details_url) + if response.status_code != 200: + raise Exception(f"Failed to fetch release details for tag {tag_name}: {response.status_code}") + + release_data = response.json() + assets = release_data.get('assets', []) + + # Find the asset with the specified prefix + asset_url = None + for asset in assets: + if re.match(f"{asset_name_prefix}.*", asset['name']): + asset_url = asset['browser_download_url'] + break + + if not asset_url: + raise Exception(f"No asset found with prefix {asset_name_prefix}") + + # Download the asset + response = requests.get(asset_url) + if response.status_code != 200: + raise Exception(f"Failed to download asset: {response.status_code}") + + print("Llamafile downloaded successfully.") + logging.debug("Main: Llamafile downloaded successfully.") + + # Save the file + with open(output_filename, 'wb') as file: + file.write(response.content) + + logging.debug(f"Downloaded {output_filename} from {asset_url}") + print(f"Downloaded {output_filename} from {asset_url}") + + # Check to see if the LLM already exists, and if not, download the LLM + print("Checking for and downloading LLM from Huggingface if needed...") + logging.debug("Main: Checking and downloading LLM from Huggingface if needed...") + mistral_7b_instruct_v0_2_q8_0_llamafile = "mistral-7b-instruct-v0.2.Q8_0.llamafile" + Samantha_Mistral_Instruct_7B_Bulleted_Notes_Q8 = "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" + Phi_3_mini_128k_instruct_Q8_0_gguf = "Phi-3-mini-128k-instruct-Q8_0.gguf" + if os.path.exists(mistral_7b_instruct_v0_2_q8_0_llamafile): + llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" + print("Model is already downloaded. Skipping download.") + pass + elif os.path.exists(Samantha_Mistral_Instruct_7B_Bulleted_Notes_Q8): + llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" + print("Model is already downloaded. Skipping download.") + pass + elif os.path.exists(mistral_7b_instruct_v0_2_q8_0_llamafile): + llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" + print("Model is already downloaded. Skipping download.") + pass + else: + logging.debug("Main: Checking and downloading LLM from Huggingface if needed...") + print("Downloading LLM from Huggingface...") + time.sleep(1) + print("Gonna be a bit...") + time.sleep(1) + print("Like seriously, an 8GB file...") + time.sleep(2) + # Not needed for GUI + # dl_check = input("Final chance to back out, hit 'N'/'n' to cancel, or 'Y'/'y' to continue: ") + #if dl_check == "N" or dl_check == "n": + # exit() + x = 2 + if x != 1: + print("Uhhhh how'd you get here...?") + exit() + else: + print("Downloading LLM from Huggingface...") + # Establish hash values for LLM models + mistral_7b_instruct_v0_2_q8_gguf_sha256 = "f326f5f4f137f3ad30f8c9cc21d4d39e54476583e8306ee2931d5a022cb85b06" + samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 = "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" + mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 = "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" + global llm_choice + + # FIXME - llm_choice + llm_choice = 2 + llm_choice = input("Which LLM model would you like to download? 1. Mistral-7B-Instruct-v0.2-GGUF or 2. Samantha-Mistral-Instruct-7B-Bulleted-Notes) (plain or 'custom') or MS Flavor: Phi-3-mini-128k-instruct-Q8_0.gguf \n\n\tPress '1' or '2' or '3' to specify: ") + while llm_choice != "1" and llm_choice != "2" and llm_choice != "3": + print("Invalid choice. Please try again.") + if llm_choice == "1": + llm_download_model = "Mistral-7B-Instruct-v0.2-Q8.llamafile" + mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 = "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" + llm_download_model_hash = mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 + llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" + llamafile_llm_output_filename = "mistral-7b-instruct-v0.2.Q8_0.llamafile" + download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) + elif llm_choice == "2": + llm_download_model = "Samantha-Mistral-Instruct-7B-Bulleted-Notes-Q8.gguf" + samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 = "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" + llm_download_model_hash = samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 + llamafile_llm_output_filename = "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" + llamafile_llm_url = "https://huggingface.co/cognitivetech/samantha-mistral-instruct-7b-bulleted-notes-GGUF/resolve/main/samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf?download=true" + download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) + elif llm_choice == "3": + llm_download_model = "Phi-3-mini-128k-instruct-Q8_0.gguf" + Phi_3_mini_128k_instruct_Q8_0_gguf_sha256 = "6817b66d1c3c59ab06822e9732f0e594eea44e64cae2110906eac9d17f75d193" + llm_download_model_hash = Phi_3_mini_128k_instruct_Q8_0_gguf_sha256 + llamafile_llm_output_filename = "Phi-3-mini-128k-instruct-Q8_0.gguf" + llamafile_llm_url = "https://huggingface.co/gaianet/Phi-3-mini-128k-instruct-GGUF/resolve/main/Phi-3-mini-128k-instruct-Q8_0.gguf?download=true" + download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) + elif llm_choice == "4": # FIXME - and meta_Llama_3_8B_Instruct_Q8_0_llamafile_exists == False: + meta_Llama_3_8B_Instruct_Q8_0_llamafile_sha256 = "406868a97f02f57183716c7e4441d427f223fdbc7fa42964ef10c4d60dd8ed37" + llm_download_model_hash = meta_Llama_3_8B_Instruct_Q8_0_llamafile_sha256 + llamafile_llm_output_filename = " Meta-Llama-3-8B-Instruct.Q8_0.llamafile" + llamafile_llm_url = "https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/resolve/main/Meta-Llama-3-8B-Instruct.Q8_0.llamafile?download=true" + else: + print("Invalid choice. Please try again.") + return output_filename + + +# Maybe replace/ dead code? FIXME +# Function is used to download llamafile + A model from Huggingface +def download_latest_llamafile(repo, asset_name_prefix, output_filename): + # Check if the file already exists + print("Checking for and downloading Llamafile it it doesn't already exist...") + if os.path.exists(output_filename): + print("Llamafile already exists. Skipping download.") + logging.debug(f"{output_filename} already exists. Skipping download.") + llamafile_exists = True + else: + llamafile_exists = False + + if llamafile_exists == True: + pass + else: + # Get the latest release information + latest_release_url = f"https://api.github.com/repos/{repo}/releases/latest" + response = requests.get(latest_release_url) + if response.status_code != 200: + raise Exception(f"Failed to fetch latest release info: {response.status_code}") + + latest_release_data = response.json() + tag_name = latest_release_data['tag_name'] + + # Get the release details using the tag name + release_details_url = f"https://api.github.com/repos/{repo}/releases/tags/{tag_name}" + response = requests.get(release_details_url) + if response.status_code != 200: + raise Exception(f"Failed to fetch release details for tag {tag_name}: {response.status_code}") + + release_data = response.json() + assets = release_data.get('assets', []) + + # Find the asset with the specified prefix + asset_url = None + for asset in assets: + if re.match(f"{asset_name_prefix}.*", asset['name']): + asset_url = asset['browser_download_url'] + break + + if not asset_url: + raise Exception(f"No asset found with prefix {asset_name_prefix}") + + # Download the asset + response = requests.get(asset_url) + if response.status_code != 200: + raise Exception(f"Failed to download asset: {response.status_code}") + + print("Llamafile downloaded successfully.") + logging.debug("Main: Llamafile downloaded successfully.") + + # Save the file + with open(output_filename, 'wb') as file: + file.write(response.content) + + logging.debug(f"Downloaded {output_filename} from {asset_url}") + print(f"Downloaded {output_filename} from {asset_url}") + + # Check to see if the LLM already exists, and if not, download the LLM + print("Checking for and downloading LLM from Huggingface if needed...") + logging.debug("Main: Checking and downloading LLM from Huggingface if needed...") + mistral_7b_instruct_v0_2_q8_0_llamafile = "mistral-7b-instruct-v0.2.Q8_0.llamafile" + Samantha_Mistral_Instruct_7B_Bulleted_Notes_Q8 = "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" + Phi_3_mini_128k_instruct_Q8_0_gguf = "Phi-3-mini-128k-instruct-Q8_0.gguf" + if os.path.exists(mistral_7b_instruct_v0_2_q8_0_llamafile): + llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" + print("Model is already downloaded. Skipping download.") + pass + elif os.path.exists(Samantha_Mistral_Instruct_7B_Bulleted_Notes_Q8): + llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" + print("Model is already downloaded. Skipping download.") + pass + elif os.path.exists(mistral_7b_instruct_v0_2_q8_0_llamafile): + llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" + print("Model is already downloaded. Skipping download.") + pass + else: + logging.debug("Main: Checking and downloading LLM from Huggingface if needed...") + print("Downloading LLM from Huggingface...") + time.sleep(1) + print("Gonna be a bit...") + time.sleep(1) + print("Like seriously, an 8GB file...") + time.sleep(2) + dl_check = input("Final chance to back out, hit 'N'/'n' to cancel, or 'Y'/'y' to continue: ") + if dl_check == "N" or dl_check == "n": + exit() + else: + print("Downloading LLM from Huggingface...") + # Establish hash values for LLM models + mistral_7b_instruct_v0_2_q8_gguf_sha256 = "f326f5f4f137f3ad30f8c9cc21d4d39e54476583e8306ee2931d5a022cb85b06" + samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 = "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" + mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 = "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" + + # FIXME - llm_choice + llm_choice = 2 + llm_choice = input("Which LLM model would you like to download? 1. Mistral-7B-Instruct-v0.2-GGUF or 2. Samantha-Mistral-Instruct-7B-Bulleted-Notes) (plain or 'custom') or MS Flavor: Phi-3-mini-128k-instruct-Q8_0.gguf \n\n\tPress '1' or '2' or '3' to specify: ") + while llm_choice != "1" and llm_choice != "2" and llm_choice != "3": + print("Invalid choice. Please try again.") + if llm_choice == "1": + llm_download_model = "Mistral-7B-Instruct-v0.2-Q8.llamafile" + mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 = "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" + llm_download_model_hash = mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 + llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" + llamafile_llm_output_filename = "mistral-7b-instruct-v0.2.Q8_0.llamafile" + download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) + elif llm_choice == "2": + llm_download_model = "Samantha-Mistral-Instruct-7B-Bulleted-Notes-Q8.gguf" + samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 = "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" + llm_download_model_hash = samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 + llamafile_llm_output_filename = "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" + llamafile_llm_url = "https://huggingface.co/cognitivetech/samantha-mistral-instruct-7b_bulleted-notes_GGUF/resolve/main/samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf?download=true" + download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) + elif llm_choice == "3": + llm_download_model = "Phi-3-mini-128k-instruct-Q8_0.gguf" + Phi_3_mini_128k_instruct_Q8_0_gguf_sha256 = "6817b66d1c3c59ab06822e9732f0e594eea44e64cae2110906eac9d17f75d193" + llm_download_model_hash = Phi_3_mini_128k_instruct_Q8_0_gguf_sha256 + llamafile_llm_output_filename = "Phi-3-mini-128k-instruct-Q8_0.gguf" + llamafile_llm_url = "https://huggingface.co/gaianet/Phi-3-mini-128k-instruct-GGUF/resolve/main/Phi-3-mini-128k-instruct-Q8_0.gguf?download=true" + download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) + elif llm_choice == "4": # FIXME - and meta_Llama_3_8B_Instruct_Q8_0_llamafile_exists == False: + meta_Llama_3_8B_Instruct_Q8_0_llamafile_sha256 = "406868a97f02f57183716c7e4441d427f223fdbc7fa42964ef10c4d60dd8ed37" + llm_download_model_hash = meta_Llama_3_8B_Instruct_Q8_0_llamafile_sha256 + llamafile_llm_output_filename = " Meta-Llama-3-8B-Instruct.Q8_0.llamafile" + llamafile_llm_url = "https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/resolve/main/Meta-Llama-3-8B-Instruct.Q8_0.llamafile?download=true" + else: + print("Invalid choice. Please try again.") + return output_filename + + + + +# FIXME / IMPLEMENT FULLY +# File download verification +#mistral_7b_llamafile_instruct_v02_q8_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" +#global mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 +#mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 = "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" + +#mistral_7b_v02_instruct_model_q8_gguf_url = "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q8_0.gguf?download=true" +#global mistral_7b_instruct_v0_2_q8_gguf_sha256 +#mistral_7b_instruct_v0_2_q8_gguf_sha256 = "f326f5f4f137f3ad30f8c9cc21d4d39e54476583e8306ee2931d5a022cb85b06" + +#samantha_instruct_model_q8_gguf_url = "https://huggingface.co/cognitivetech/samantha-mistral-instruct-7b_bulleted-notes_GGUF/resolve/main/samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf?download=true" +#global samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 +#samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 = "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" + + +process = None +# Function to close out llamafile process on script exit. +def cleanup_process(): + global process + if process is not None: + process.kill() + logging.debug("Main: Terminated the external process") + + +def signal_handler(sig, frame): + logging.info('Signal handler called with signal: %s', sig) + cleanup_process() + sys.exit(0) + + +# FIXME - Add callout to gradio UI +def local_llm_function(): + global process + repo = "Mozilla-Ocho/llamafile" + asset_name_prefix = "llamafile-" + useros = os.name + if useros == "nt": + output_filename = "llamafile.exe" + else: + output_filename = "llamafile" + print( + "WARNING - Checking for existence of llamafile and HuggingFace model, downloading if needed...This could be a while") + print("WARNING - and I mean a while. We're talking an 8 Gigabyte model here...") + print("WARNING - Hope you're comfy. Or it's already downloaded.") + time.sleep(6) + logging.debug("Main: Checking and downloading Llamafile from Github if needed...") + llamafile_path = download_latest_llamafile(repo, asset_name_prefix, output_filename) + logging.debug("Main: Llamafile downloaded successfully.") + + # FIXME - llm_choice + global llm_choice + llm_choice = 1 + # Launch the llamafile in an external process with the specified argument + if llm_choice == 1: + arguments = ["--ctx-size", "8192 ", " -m", "mistral-7b-instruct-v0.2.Q8_0.llamafile"] + elif llm_choice == 2: + arguments = ["--ctx-size", "8192 ", " -m", "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf"] + elif llm_choice == 3: + arguments = ["--ctx-size", "8192 ", " -m", "Phi-3-mini-128k-instruct-Q8_0.gguf"] + elif llm_choice == 4: + arguments = ["--ctx-size", "8192 ", " -m", "llama-3"] # FIXME + + try: + logging.info("Main: Launching the LLM (llamafile) in an external terminal window...") + if useros == "nt": + launch_in_new_terminal_windows(llamafile_path, arguments) + elif useros == "posix": + launch_in_new_terminal_linux(llamafile_path, arguments) + else: + launch_in_new_terminal_mac(llamafile_path, arguments) + # FIXME - pid doesn't exist in this context + #logging.info(f"Main: Launched the {llamafile_path} with PID {process.pid}") + atexit.register(cleanup_process, process) + except Exception as e: + logging.error(f"Failed to launch the process: {e}") + print(f"Failed to launch the process: {e}") + + +# This function is used to dl a llamafile binary + the Samantha Mistral Finetune model. +# It should only be called when the user is using the GUI to set up and interact with Llamafile. +def local_llm_gui_function(am_noob, verbose_checked, threads_checked, threads_value, http_threads_checked, http_threads_value, + model_checked, model_value, hf_repo_checked, hf_repo_value, hf_file_checked, hf_file_value, + ctx_size_checked, ctx_size_value, ngl_checked, ngl_value, host_checked, host_value, port_checked, + port_value): + # Identify running OS + useros = os.name + if useros == "nt": + output_filename = "llamafile.exe" + else: + output_filename = "llamafile" + + # Build up the commands for llamafile + built_up_args = [] + + # Identify if the user wants us to do everything for them + if am_noob == True: + print("You're a noob. (lol j/k; they're good settings)") + + # Setup variables for Model download from HF + repo = "Mozilla-Ocho/llamafile" + asset_name_prefix = "llamafile-" + print( + "WARNING - Checking for existence of llamafile or HuggingFace model (GGUF type), downloading if needed...This could be a while") + print("WARNING - and I mean a while. We're talking an 8 Gigabyte model here...") + print("WARNING - Hope you're comfy. Or it's already downloaded.") + time.sleep(6) + logging.debug("Main: Checking for Llamafile and downloading from Github if needed...\n\tAlso checking for a " + "local LLM model...\n\tDownloading if needed...\n\tThis could take a while...\n\tWill be the " + "'samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf' model...") + llamafile_path = download_latest_llamafile_through_gui(repo, asset_name_prefix, output_filename) + logging.debug("Main: Llamafile downloaded successfully.") + + arguments = [] + # FIXME - llm_choice + # This is the gui, we can add this as options later + llm_choice = 2 + # Launch the llamafile in an external process with the specified argument + if llm_choice == 1: + arguments = ["--ctx-size", "8192 ", " -m", "mistral-7b-instruct-v0.2.Q8_0.llamafile"] + elif llm_choice == 2: + arguments = """--ctx-size 8192 -m samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf""" + elif llm_choice == 3: + arguments = ["--ctx-size", "8192 ", " -m", "Phi-3-mini-128k-instruct-Q8_0.gguf"] + elif llm_choice == 4: + arguments = ["--ctx-size", "8192 ", " -m", "llama-3"] + + try: + logging.info("Main(Local-LLM-GUI-noob): Launching the LLM (llamafile) in an external terminal window...") + + if useros == "nt": + command = 'start cmd /k "llamafile.exe --ctx-size 8192 -m samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf"' + subprocess.Popen(command, shell=True) + elif useros == "posix": + command = "llamafile --ctx-size 8192 -m samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" + subprocess.Popen(command, shell=True) + else: + command = "llamafile.exe --ctx-size 8192 -m samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" + subprocess.Popen(command, shell=True) + # FIXME - pid doesn't exist in this context + # logging.info(f"Main: Launched the {llamafile_path} with PID {process.pid}") + atexit.register(cleanup_process, process) + except Exception as e: + logging.error(f"Failed to launch the process: {e}") + print(f"Failed to launch the process: {e}") + + else: + print("You're not a noob.") + llamafile_path = download_latest_llamafile_no_model(output_filename) + if verbose_checked == True: + print("Verbose mode enabled.") + built_up_args.append("--verbose") + if threads_checked == True: + print(f"Threads enabled with value: {threads_value}") + built_up_args.append(f"--threads {threads_value}") + if http_threads_checked == True: + print(f"HTTP Threads enabled with value: {http_threads_value}") + built_up_args.append(f"--http-threads {http_threads_value}") + if model_checked == True: + print(f"Model enabled with value: {model_value}") + built_up_args.append(f"--model {model_value}") + if hf_repo_checked == True: + print(f"Huggingface repo enabled with value: {hf_repo_value}") + built_up_args.append(f"--hf-repo {hf_repo_value}") + if hf_file_checked == True: + print(f"Huggingface file enabled with value: {hf_file_value}") + built_up_args.append(f"--hf-file {hf_file_value}") + if ctx_size_checked == True: + print(f"Context size enabled with value: {ctx_size_value}") + built_up_args.append(f"--ctx-size {ctx_size_value}") + if ngl_checked == True: + print(f"NGL enabled with value: {ngl_value}") + built_up_args.append(f"--ngl {ngl_value}") + if host_checked == True: + print(f"Host enabled with value: {host_value}") + built_up_args.append(f"--host {host_value}") + if port_checked == True: + print(f"Port enabled with value: {port_value}") + built_up_args.append(f"--port {port_value}") + + # Lets go ahead and finally launch the bastard... + try: + logging.info("Main(Local-LLM-GUI-Main): Launching the LLM (llamafile) in an external terminal window...") + if useros == "nt": + launch_in_new_terminal_windows(llamafile_path, built_up_args) + elif useros == "posix": + launch_in_new_terminal_linux(llamafile_path, built_up_args) + else: + launch_in_new_terminal_mac(llamafile_path, built_up_args) + # FIXME - pid doesn't exist in this context + #logging.info(f"Main: Launched the {llamafile_path} with PID {process.pid}") + atexit.register(cleanup_process, process) + except Exception as e: + logging.error(f"Failed to launch the process: {e}") + print(f"Failed to launch the process: {e}") + + +# Launch the executable in a new terminal window # FIXME - really should figure out a cleaner way of doing this... +def launch_in_new_terminal_windows(executable, args): + command = f'start cmd /k "{executable} {" ".join(args)}"' + subprocess.Popen(command, shell=True) + + +# FIXME +def launch_in_new_terminal_linux(executable, args): + command = f'gnome-terminal -- {executable} {" ".join(args)}' + subprocess.Popen(command, shell=True) + + +# FIXME +def launch_in_new_terminal_mac(executable, args): + command = f'open -a Terminal.app {executable} {" ".join(args)}' + subprocess.Popen(command, shell=True) diff --git a/App_Function_Libraries/Local_Summarization_Lib.py b/App_Function_Libraries/Local_Summarization_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..f0abf8dea571bcf5d31aa355443e7e6fd5959283 --- /dev/null +++ b/App_Function_Libraries/Local_Summarization_Lib.py @@ -0,0 +1,467 @@ +# Local_Summarization_Lib.py +######################################### +# Local Summarization Library +# This library is used to perform summarization with a 'local' inference engine. +# +#### +# +#################### +# Function List +# FIXME - UPDATE Function Arguments +# 1. summarize_with_local_llm(text, custom_prompt_arg) +# 2. summarize_with_llama(api_url, text, token, custom_prompt) +# 3. summarize_with_kobold(api_url, text, kobold_api_token, custom_prompt) +# 4. summarize_with_oobabooga(api_url, text, ooba_api_token, custom_prompt) +# 5. summarize_with_vllm(vllm_api_url, vllm_api_key_function_arg, llm_model, text, vllm_custom_prompt_function_arg) +# 6. summarize_with_tabbyapi(tabby_api_key, tabby_api_IP, text, tabby_model, custom_prompt) +# 7. save_summary_to_file(summary, file_path) +# +############################### +# Import necessary libraries +import json +import logging +import os +import requests +# Import 3rd-party Libraries +from openai import OpenAI +# Import Local +from App_Function_Libraries.Utils import load_and_log_configs +from App_Function_Libraries.Utils import extract_text_from_segments +# +####################################################################################################################### +# Function Definitions +# + +logger = logging.getLogger() + +# Dirty hack for vLLM +openai_api_key = "Fake_key" +client = OpenAI(api_key=openai_api_key) + +def summarize_with_local_llm(input_data, custom_prompt_arg): + try: + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("Local LLM: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("openai: Using provided string data for summarization") + data = input_data + + logging.debug(f"Local LLM: Loaded data: {data}") + logging.debug(f"Local LLM: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("Local LLM: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Invalid input data format") + + headers = { + 'Content-Type': 'application/json' + } + + logging.debug("Local LLM: Preparing data + prompt for submittal") + local_llm_prompt = f"{text} \n\n\n\n{custom_prompt_arg}" + data = { + "messages": [ + { + "role": "system", + "content": "You are a professional summarizer." + }, + { + "role": "user", + "content": local_llm_prompt + } + ], + "max_tokens": 28000, # Adjust tokens as needed + } + logging.debug("Local LLM: Posting request") + response = requests.post('http://127.0.0.1:8080/v1/chat/completions', headers=headers, json=data) + + if response.status_code == 200: + response_data = response.json() + if 'choices' in response_data and len(response_data['choices']) > 0: + summary = response_data['choices'][0]['message']['content'].strip() + logging.debug("Local LLM: Summarization successful") + print("Local LLM: Summarization successful.") + return summary + else: + logging.warning("Local LLM: Summary not found in the response data") + return "Local LLM: Summary not available" + else: + logging.debug("Local LLM: Summarization failed") + print("Local LLM: Failed to process summary:", response.text) + return "Local LLM: Failed to process summary" + except Exception as e: + logging.debug("Local LLM: Error in processing: %s", str(e)) + print("Error occurred while processing summary with Local LLM:", str(e)) + return "Local LLM: Error occurred while processing summary" + +def summarize_with_llama(input_data, custom_prompt, api_url="http://127.0.0.1:8080/completion", api_key=None): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None: + logging.info("llama.cpp: API key not provided as parameter") + logging.info("llama.cpp: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['llama'] + + if api_key is None or api_key.strip() == "": + logging.info("llama.cpp: API key not found or is empty") + + logging.debug(f"llama.cpp: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + # Load transcript + logging.debug("llama.cpp: Loading JSON data") + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("Llama.cpp: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("Llama.cpp: Using provided string data for summarization") + data = input_data + + logging.debug(f"Llama.cpp: Loaded data: {data}") + logging.debug(f"Llama.cpp: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("Llama.cpp: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Llama.cpp: Invalid input data format") + + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + } + if len(api_key) > 5: + headers['Authorization'] = f'Bearer {api_key}' + + llama_prompt = f"{text} \n\n\n\n{custom_prompt}" + logging.debug("llama: Prompt being sent is {llama_prompt}") + + data = { + "prompt": llama_prompt + } + + logging.debug("llama: Submitting request to API endpoint") + print("llama: Submitting request to API endpoint") + response = requests.post(api_url, headers=headers, json=data) + response_data = response.json() + logging.debug("API Response Data: %s", response_data) + + if response.status_code == 200: + # if 'X' in response_data: + logging.debug(response_data) + summary = response_data['content'].strip() + logging.debug("llama: Summarization successful") + print("Summarization successful.") + return summary + else: + logging.error(f"Llama: API request failed with status code {response.status_code}: {response.text}") + return f"Llama: API request failed: {response.text}" + + except Exception as e: + logging.error("Llama: Error in processing: %s", str(e)) + return f"Llama: Error occurred while processing summary with llama: {str(e)}" + + +# https://lite.koboldai.net/koboldcpp_api#/api%2Fv1/post_api_v1_generate +def summarize_with_kobold(input_data, api_key, custom_prompt_input, kobold_api_IP="http://127.0.0.1:5001/api/v1/generate"): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None: + logging.info("Kobold.cpp: API key not provided as parameter") + logging.info("Kobold.cpp: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['kobold'] + + if api_key is None or api_key.strip() == "": + logging.info("Kobold.cpp: API key not found or is empty") + + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("Kobold.cpp: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("Kobold.cpp: Using provided string data for summarization") + data = input_data + + logging.debug(f"Kobold.cpp: Loaded data: {data}") + logging.debug(f"Kobold.cpp: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("Kobold.cpp: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Kobold.cpp: Invalid input data format") + + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + } + + kobold_prompt = f"{text} \n\n\n\n{custom_prompt_input}" + logging.debug("kobold: Prompt being sent is {kobold_prompt}") + + # FIXME + # Values literally c/p from the api docs.... + data = { + "max_context_length": 8096, + "max_length": 4096, + "prompt": f"{text}\n\n\n\n{custom_prompt_input}" + } + + logging.debug("kobold: Submitting request to API endpoint") + print("kobold: Submitting request to API endpoint") + response = requests.post(kobold_api_IP, headers=headers, json=data) + response_data = response.json() + logging.debug("kobold: API Response Data: %s", response_data) + + if response.status_code == 200: + if 'results' in response_data and len(response_data['results']) > 0: + summary = response_data['results'][0]['text'].strip() + logging.debug("kobold: Summarization successful") + print("Summarization successful.") + return summary + else: + logging.error("Expected data not found in API response.") + return "Expected data not found in API response." + else: + logging.error(f"kobold: API request failed with status code {response.status_code}: {response.text}") + return f"kobold: API request failed: {response.text}" + + except Exception as e: + logging.error("kobold: Error in processing: %s", str(e)) + return f"kobold: Error occurred while processing summary with kobold: {str(e)}" + + +# https://github.com/oobabooga/text-generation-webui/wiki/12-%E2%80%90-OpenAI-API +def summarize_with_oobabooga(input_data, api_key, custom_prompt, api_url="http://127.0.0.1:5000/v1/chat/completions"): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None: + logging.info("ooba: API key not provided as parameter") + logging.info("ooba: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['ooba'] + + if api_key is None or api_key.strip() == "": + logging.info("ooba: API key not found or is empty") + + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("Oobabooga: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("Oobabooga: Using provided string data for summarization") + data = input_data + + logging.debug(f"Oobabooga: Loaded data: {data}") + logging.debug(f"Oobabooga: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("Oobabooga: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Invalid input data format") + + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + } + + # prompt_text = "I like to eat cake and bake cakes. I am a baker. I work in a French bakery baking cakes. It + # is a fun job. I have been baking cakes for ten years. I also bake lots of other baked goods, but cakes are + # my favorite." prompt_text += f"\n\n{text}" # Uncomment this line if you want to include the text variable + ooba_prompt = f"{text}" + f"\n\n\n\n{custom_prompt}" + logging.debug("ooba: Prompt being sent is {ooba_prompt}") + + data = { + "mode": "chat", + "character": "Example", + "messages": [{"role": "user", "content": ooba_prompt}] + } + + logging.debug("ooba: Submitting request to API endpoint") + print("ooba: Submitting request to API endpoint") + response = requests.post(api_url, headers=headers, json=data, verify=False) + logging.debug("ooba: API Response Data: %s", response) + + if response.status_code == 200: + response_data = response.json() + summary = response.json()['choices'][0]['message']['content'] + logging.debug("ooba: Summarization successful") + print("Summarization successful.") + return summary + else: + logging.error(f"oobabooga: API request failed with status code {response.status_code}: {response.text}") + return f"ooba: API request failed with status code {response.status_code}: {response.text}" + + except Exception as e: + logging.error("ooba: Error in processing: %s", str(e)) + return f"ooba: Error occurred while processing summary with oobabooga: {str(e)}" + + +# FIXME - Install is more trouble than care to deal with right now. +def summarize_with_tabbyapi(input_data, custom_prompt_input, api_key=None, api_IP="http://127.0.0.1:5000/v1/chat/completions"): + loaded_config_data = load_and_log_configs() + model = loaded_config_data['models']['tabby'] + # API key validation + if api_key is None: + logging.info("tabby: API key not provided as parameter") + logging.info("tabby: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['tabby'] + + if api_key is None or api_key.strip() == "": + logging.info("tabby: API key not found or is empty") + + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("tabby: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("tabby: Using provided string data for summarization") + data = input_data + + logging.debug(f"tabby: Loaded data: {data}") + logging.debug(f"tabby: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("tabby: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Invalid input data format") + + headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json' + } + data2 = { + 'text': text, + 'model': 'tabby' # Specify the model if needed + } + tabby_api_ip = loaded_config_data['local_apis']['tabby']['ip'] + try: + response = requests.post(tabby_api_ip, headers=headers, json=data2) + response.raise_for_status() + summary = response.json().get('summary', '') + return summary + except requests.exceptions.RequestException as e: + logger.error(f"Error summarizing with TabbyAPI: {e}") + return "Error summarizing with TabbyAPI." + + +# FIXME - https://docs.vllm.ai/en/latest/getting_started/quickstart.html .... Great docs. +def summarize_with_vllm(input_data, custom_prompt_input, api_key=None, vllm_api_url="http://127.0.0.1:8000/v1/chat/completions"): + loaded_config_data = load_and_log_configs() + llm_model = loaded_config_data['models']['vllm'] + # API key validation + if api_key is None: + logging.info("vLLM: API key not provided as parameter") + logging.info("vLLM: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['llama'] + + if api_key is None or api_key.strip() == "": + logging.info("vLLM: API key not found or is empty") + vllm_client = OpenAI( + base_url=vllm_api_url, + api_key=custom_prompt_input + ) + + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("vLLM: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("vLLM: Using provided string data for summarization") + data = input_data + + logging.debug(f"vLLM: Loaded data: {data}") + logging.debug(f"vLLM: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("vLLM: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Invalid input data format") + + + custom_prompt = custom_prompt_input + + completion = client.chat.completions.create( + model=llm_model, + messages=[ + {"role": "system", "content": "You are a professional summarizer."}, + {"role": "user", "content": f"{text} \n\n\n\n{custom_prompt}"} + ] + ) + vllm_summary = completion.choices[0].message.content + return vllm_summary + + +def save_summary_to_file(summary, file_path): + logging.debug("Now saving summary to file...") + base_name = os.path.splitext(os.path.basename(file_path))[0] + summary_file_path = os.path.join(os.path.dirname(file_path), base_name + '_summary.txt') + os.makedirs(os.path.dirname(summary_file_path), exist_ok=True) + logging.debug("Opening summary file for writing, *segments.json with *_summary.txt") + with open(summary_file_path, 'w') as file: + file.write(summary) + logging.info(f"Summary saved to file: {summary_file_path}") + +# +# +####################################################################################################################### + + + diff --git a/App_Function_Libraries/Markdown_Export-improvement.py b/App_Function_Libraries/Markdown_Export-improvement.py new file mode 100644 index 0000000000000000000000000000000000000000..764ce405fcdbceb3dc66ea0a3c84d670e092fb67 --- /dev/null +++ b/App_Function_Libraries/Markdown_Export-improvement.py @@ -0,0 +1,234 @@ +import gradio as gr +import logging +import sqlite3 +from typing import List, Dict +import os +import zipfile +import tempfile +import shutil + + +# Set up logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# Database connection (you'll need to set this up) +db = None # Replace with your actual database connection + + +class DatabaseError(Exception): + pass + + +# Database functions +def fetch_items_by_keyword(search_query: str) -> List[Dict]: + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT m.id, m.title, m.url + FROM Media m + JOIN MediaKeywords mk ON m.id = mk.media_id + JOIN Keywords k ON mk.keyword_id = k.id + WHERE k.keyword LIKE ? + """, (f'%{search_query}%',)) + results = cursor.fetchall() + return [{"id": r[0], "title": r[1], "url": r[2]} for r in results] + except sqlite3.Error as e: + logger.error(f"Error fetching items by keyword: {e}") + raise DatabaseError(f"Error fetching items by keyword: {e}") + + +def fetch_item_details(media_id: int) -> tuple: + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT prompt, summary + FROM MediaModifications + WHERE media_id = ? + ORDER BY modification_date DESC + LIMIT 1 + """, (media_id,)) + prompt_summary_result = cursor.fetchone() + cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,)) + content_result = cursor.fetchone() + + prompt = prompt_summary_result[0] if prompt_summary_result else "" + summary = prompt_summary_result[1] if prompt_summary_result else "" + content = content_result[0] if content_result else "" + + return content, prompt, summary + except sqlite3.Error as e: + logger.error(f"Error fetching item details: {e}") + return "", "", "" + + +def browse_items(search_query: str, search_type: str) -> List[Dict]: + try: + with db.get_connection() as conn: + cursor = conn.cursor() + if search_type == 'Title': + cursor.execute("SELECT id, title, url FROM Media WHERE title LIKE ?", (f'%{search_query}%',)) + elif search_type == 'URL': + cursor.execute("SELECT id, title, url FROM Media WHERE url LIKE ?", (f'%{search_query}%',)) + elif search_type == 'Keyword': + return fetch_items_by_keyword(search_query) + elif search_type == 'Content': + cursor.execute("SELECT id, title, url FROM Media WHERE content LIKE ?", (f'%{search_query}%',)) + else: + raise ValueError(f"Invalid search type: {search_type}") + + results = cursor.fetchall() + return [{"id": r[0], "title": r[1], "url": r[2]} for r in results] + except sqlite3.Error as e: + logger.error(f"Error fetching items by {search_type}: {e}") + raise DatabaseError(f"Error fetching items by {search_type}: {e}") + + +# Export functions +def export_item_as_markdown(media_id: int) -> str: + try: + content, prompt, summary = fetch_item_details(media_id) + title = f"Item {media_id}" # You might want to fetch the actual title + markdown_content = f"# {title}\n\n## Prompt\n{prompt}\n\n## Summary\n{summary}\n\n## Content\n{content}" + + filename = f"export_item_{media_id}.md" + with open(filename, "w", encoding='utf-8') as f: + f.write(markdown_content) + + logger.info(f"Successfully exported item {media_id} to {filename}") + return filename + except Exception as e: + logger.error(f"Error exporting item {media_id}: {str(e)}") + return None + + +def export_items_by_keyword(keyword: str) -> str: + try: + items = fetch_items_by_keyword(keyword) + if not items: + logger.warning(f"No items found for keyword: {keyword}") + return None + + # Create a temporary directory to store individual markdown files + with tempfile.TemporaryDirectory() as temp_dir: + folder_name = f"export_keyword_{keyword}" + export_folder = os.path.join(temp_dir, folder_name) + os.makedirs(export_folder) + + for item in items: + content, prompt, summary = fetch_item_details(item['id']) + markdown_content = f"# {item['title']}\n\n## Prompt\n{prompt}\n\n## Summary\n{summary}\n\n## Content\n{content}" + + # Create individual markdown file for each item + file_name = f"{item['id']}_{item['title'][:50]}.md" # Limit filename length + file_path = os.path.join(export_folder, file_name) + with open(file_path, "w", encoding='utf-8') as f: + f.write(markdown_content) + + # Create a zip file containing all markdown files + zip_filename = f"{folder_name}.zip" + shutil.make_archive(os.path.join(temp_dir, folder_name), 'zip', export_folder) + + # Move the zip file to a location accessible by Gradio + final_zip_path = os.path.join(os.getcwd(), zip_filename) + shutil.move(os.path.join(temp_dir, zip_filename), final_zip_path) + + logger.info(f"Successfully exported {len(items)} items for keyword '{keyword}' to {zip_filename}") + return final_zip_path + except Exception as e: + logger.error(f"Error exporting items for keyword '{keyword}': {str(e)}") + return None + + +def export_selected_items(selected_items: List[Dict]) -> str: + try: + if not selected_items: + logger.warning("No items selected for export") + return None + + markdown_content = "# Selected Items\n\n" + for item in selected_items: + content, prompt, summary = fetch_item_details(item['id']) + markdown_content += f"## {item['title']}\n\n### Prompt\n{prompt}\n\n### Summary\n{summary}\n\n### Content\n{content}\n\n---\n\n" + + filename = "export_selected_items.md" + with open(filename, "w", encoding='utf-8') as f: + f.write(markdown_content) + + logger.info(f"Successfully exported {len(selected_items)} selected items to {filename}") + return filename + except Exception as e: + logger.error(f"Error exporting selected items: {str(e)}") + return None + + +# Gradio interface functions +def display_search_results(search_query: str, search_type: str) -> List[Dict]: + try: + results = browse_items(search_query, search_type) + return [{"name": f"{item['title']} ({item['url']})", "value": item} for item in results] + except DatabaseError as e: + logger.error(f"Error in display_search_results: {str(e)}") + return [] + + +# Gradio interface +with gr.Blocks() as demo: + gr.Markdown("# Content Export Interface") + + with gr.Tab("Search and Export"): + search_query = gr.Textbox(label="Search Query") + search_type = gr.Radio(["Title", "URL", "Keyword", "Content"], label="Search By") + search_button = gr.Button("Search") + + search_results = gr.CheckboxGroup(label="Search Results") + export_selected_button = gr.Button("Export Selected Items") + + keyword_input = gr.Textbox(label="Enter keyword for export") + export_by_keyword_button = gr.Button("Export items by keyword") + + export_output = gr.File(label="Download Exported File") + + error_output = gr.Textbox(label="Status/Error Messages", interactive=False) + + search_button.click( + fn=display_search_results, + inputs=[search_query, search_type], + outputs=[search_results, error_output] + ) + + export_selected_button.click( + fn=lambda selected: (export_selected_items(selected), "Exported selected items") if selected else ( + None, "No items selected"), + inputs=[search_results], + outputs=[export_output, error_output] + ) + + export_by_keyword_button.click( + fn=lambda keyword: ( + export_items_by_keyword(keyword), f"Exported items for keyword: {keyword}") if keyword else ( + None, "No keyword provided"), + inputs=[keyword_input], + outputs=[export_output, error_output] + ) + + # Add functionality to export individual items + search_results.select( + fn=lambda item: (export_item_as_markdown(item['id']), f"Exported item: {item['title']}") if item else ( + None, "No item selected"), + inputs=[gr.State(lambda: search_results.value)], + outputs=[export_output, error_output] + ) + +demo.launch() + + +# This modified version of export_items_by_keyword does the following: +# +# Creates a temporary directory to store individual markdown files. +# For each item associated with the keyword, it creates a separate markdown file. +# Places all markdown files in a folder named export_keyword_{keyword}. +# Creates a zip file containing the folder with all markdown files. +# Moves the zip file to a location accessible by Gradio for download. \ No newline at end of file diff --git a/App_Function_Libraries/Obsidian-Importer.py b/App_Function_Libraries/Obsidian-Importer.py new file mode 100644 index 0000000000000000000000000000000000000000..02399c3e5d4946383ad7b66c39b50f8a60fd19aa --- /dev/null +++ b/App_Function_Libraries/Obsidian-Importer.py @@ -0,0 +1,210 @@ +import os +import re +import yaml +import sqlite3 +import traceback +import time +import zipfile +import tempfile +import shutil +import gradio as gr +import logging + +# Set up logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# Assume db connection is set up elsewhere +db = None # Replace with your actual database connection + + +class DatabaseError(Exception): + pass + + +def scan_obsidian_vault(vault_path): + markdown_files = [] + for root, dirs, files in os.walk(vault_path): + for file in files: + if file.endswith('.md'): + markdown_files.append(os.path.join(root, file)) + return markdown_files + + +def parse_obsidian_note(file_path): + with open(file_path, 'r', encoding='utf-8') as file: + content = file.read() + + frontmatter = {} + frontmatter_match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL) + if frontmatter_match: + frontmatter_text = frontmatter_match.group(1) + frontmatter = yaml.safe_load(frontmatter_text) + content = content[frontmatter_match.end():] + + tags = re.findall(r'#(\w+)', content) + links = re.findall(r'\[\[(.*?)\]\]', content) + + return { + 'title': os.path.basename(file_path).replace('.md', ''), + 'content': content, + 'frontmatter': frontmatter, + 'tags': tags, + 'links': links, + 'file_path': file_path # Add this line + } + + +def import_obsidian_note_to_db(note_data): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + + cursor.execute("SELECT id FROM Media WHERE title = ? AND type = 'obsidian_note'", (note_data['title'],)) + existing_note = cursor.fetchone() + + if existing_note: + media_id = existing_note[0] + cursor.execute(""" + UPDATE Media + SET content = ?, author = ?, ingestion_date = CURRENT_TIMESTAMP + WHERE id = ? + """, (note_data['content'], note_data['frontmatter'].get('author', 'Unknown'), media_id)) + + cursor.execute("DELETE FROM MediaKeywords WHERE media_id = ?", (media_id,)) + else: + cursor.execute(""" + INSERT INTO Media (title, content, type, author, ingestion_date, url) + VALUES (?, ?, 'obsidian_note', ?, CURRENT_TIMESTAMP, ?) + """, (note_data['title'], note_data['content'], note_data['frontmatter'].get('author', 'Unknown'), + note_data['file_path'])) + + media_id = cursor.lastrowid + + for tag in note_data['tags']: + cursor.execute("INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)", (tag,)) + cursor.execute("SELECT id FROM Keywords WHERE keyword = ?", (tag,)) + keyword_id = cursor.fetchone()[0] + cursor.execute("INSERT OR IGNORE INTO MediaKeywords (media_id, keyword_id) VALUES (?, ?)", + (media_id, keyword_id)) + + frontmatter_str = yaml.dump(note_data['frontmatter']) + cursor.execute(""" + INSERT INTO MediaModifications (media_id, prompt, summary, modification_date) + VALUES (?, 'Obsidian Frontmatter', ?, CURRENT_TIMESTAMP) + """, (media_id, frontmatter_str)) + + # Update full-text search index + cursor.execute('INSERT OR REPLACE INTO media_fts (rowid, title, content) VALUES (?, ?, ?)', + (media_id, note_data['title'], note_data['content'])) + + action = "Updated" if existing_note else "Imported" + logger.info(f"{action} Obsidian note: {note_data['title']}") + return True, None + except sqlite3.Error as e: + error_msg = f"Database error {'updating' if existing_note else 'importing'} note {note_data['title']}: {str(e)}" + logger.error(error_msg) + return False, error_msg + except Exception as e: + error_msg = f"Unexpected error {'updating' if existing_note else 'importing'} note {note_data['title']}: {str(e)}\n{traceback.format_exc()}" + logger.error(error_msg) + return False, error_msg + + +def import_obsidian_vault(vault_path, progress=gr.Progress()): + try: + markdown_files = scan_obsidian_vault(vault_path) + total_files = len(markdown_files) + imported_files = 0 + errors = [] + + for i, file_path in enumerate(markdown_files): + try: + note_data = parse_obsidian_note(file_path) + success, error_msg = import_obsidian_note_to_db(note_data) + if success: + imported_files += 1 + else: + errors.append(error_msg) + except Exception as e: + error_msg = f"Error processing {file_path}: {str(e)}" + logger.error(error_msg) + errors.append(error_msg) + + progress((i + 1) / total_files, f"Imported {imported_files} of {total_files} files") + time.sleep(0.1) # Small delay to prevent UI freezing + + return imported_files, total_files, errors + except Exception as e: + error_msg = f"Error scanning vault: {str(e)}\n{traceback.format_exc()}" + logger.error(error_msg) + return 0, 0, [error_msg] + + +def process_obsidian_zip(zip_file): + with tempfile.TemporaryDirectory() as temp_dir: + try: + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + zip_ref.extractall(temp_dir) + + imported_files, total_files, errors = import_obsidian_vault(temp_dir) + + return imported_files, total_files, errors + except zipfile.BadZipFile: + error_msg = "The uploaded file is not a valid zip file." + logger.error(error_msg) + return 0, 0, [error_msg] + except Exception as e: + error_msg = f"Error processing zip file: {str(e)}\n{traceback.format_exc()}" + logger.error(error_msg) + return 0, 0, [error_msg] + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + +# Gradio interface +with gr.Blocks() as demo: + gr.Markdown("# Content Export and Import Interface") + + # ... (your existing tabs and components) + + with gr.Tab("Import Obsidian Vault"): + gr.Markdown("## Import Obsidian Vault") + with gr.Row(): + vault_path_input = gr.Textbox(label="Obsidian Vault Path (Local)") + vault_zip_input = gr.File(label="Upload Obsidian Vault (Zip)") + import_vault_button = gr.Button("Import Obsidian Vault") + import_status = gr.Textbox(label="Import Status", interactive=False) + + + def import_vault(vault_path, vault_zip): + if vault_zip: + imported, total, errors = process_obsidian_zip(vault_zip.name) + elif vault_path: + imported, total, errors = import_obsidian_vault(vault_path) + else: + return "Please provide either a local vault path or upload a zip file." + + status = f"Imported {imported} out of {total} files.\n" + if errors: + status += f"Encountered {len(errors)} errors:\n" + "\n".join(errors) + return status + + + import_vault_button.click( + fn=import_vault, + inputs=[vault_path_input, vault_zip_input], + outputs=[import_status], + show_progress=True + ) + + # ... (rest of your existing code) + +demo.launch() + +# This comprehensive solution includes: +# +# Enhanced error handling throughout the import process. +# Progress updates for large vaults using Gradio's progress bar. +# The ability to update existing notes if they're reimported. +# Support for importing Obsidian vaults from both local directories and uploaded zip files. \ No newline at end of file diff --git a/App_Function_Libraries/Old_Chunking_Lib.py b/App_Function_Libraries/Old_Chunking_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..bf09df05860bf1a947cfde03d9faf77d0fabaa18 --- /dev/null +++ b/App_Function_Libraries/Old_Chunking_Lib.py @@ -0,0 +1,159 @@ +# Old_Chunking_Lib.py +######################################### +# Old Chunking Library +# This library is used to handle chunking of text for summarization. +# +#### +import logging +#################### +# Function List +# +# 1. chunk_transcript(transcript: str, chunk_duration: int, words_per_second) -> List[str] +# 2. summarize_chunks(api_name: str, api_key: str, transcript: List[dict], chunk_duration: int, words_per_second: int) -> str +# 3. get_chat_completion(messages, model='gpt-4-turbo') +# 4. chunk_on_delimiter(input_string: str, max_tokens: int, delimiter: str) -> List[str] +# 5. combine_chunks_with_no_minimum(chunks: List[str], max_tokens: int, chunk_delimiter="\n\n", header: Optional[str] = None, add_ellipsis_for_overflow=False) -> Tuple[List[str], List[int]] +# 6. rolling_summarize(text: str, detail: float = 0, model: str = 'gpt-4-turbo', additional_instructions: Optional[str] = None, minimum_chunk_size: Optional[int] = 500, chunk_delimiter: str = ".", summarize_recursively=False, verbose=False) +# 7. chunk_transcript(transcript: str, chunk_duration: int, words_per_second) -> List[str] +# 8. summarize_chunks(api_name: str, api_key: str, transcript: List[dict], chunk_duration: int, words_per_second: int) -> str +# +#################### + +# Import necessary libraries +import os +from typing import Optional, List, Tuple +# +# Import 3rd party +from openai import OpenAI +from App_Function_Libraries.Tokenization_Methods_Lib import openai_tokenize +# +# Import Local +# +####################################################################################################################### +# Function Definitions +# + +######### Words-per-second Chunking ######### +def chunk_transcript(transcript: str, chunk_duration: int, words_per_second) -> List[str]: + words = transcript.split() + words_per_chunk = chunk_duration * words_per_second + chunks = [' '.join(words[i:i + words_per_chunk]) for i in range(0, len(words), words_per_chunk)] + return chunks + + +# def summarize_chunks(api_name: str, api_key: str, transcript: List[dict], chunk_duration: int, +# words_per_second: int) -> str: +# if api_name not in summarizers: # See 'summarizers' dict in the main script +# return f"Unsupported API: {api_name}" +# +# summarizer = summarizers[api_name] +# text = extract_text_from_segments(transcript) +# chunks = chunk_transcript(text, chunk_duration, words_per_second) +# +# summaries = [] +# for chunk in chunks: +# if api_name == 'openai': +# # Ensure the correct model and prompt are passed +# summaries.append(summarizer(api_key, chunk, custom_prompt)) +# else: +# summaries.append(summarizer(api_key, chunk)) +# +# return "\n\n".join(summaries) + + +################## #################### + + +######### Token-size Chunking ######### FIXME - OpenAI only currently +# This is dirty and shameful and terrible. It should be replaced with a proper implementation. +# anyways lets get to it.... +openai_api_key = "Fake_key" # FIXME +client = OpenAI(api_key=openai_api_key) + + + + + +# This function chunks a text into smaller pieces based on a maximum token count and a delimiter +def chunk_on_delimiter(input_string: str, + max_tokens: int, + delimiter: str) -> List[str]: + chunks = input_string.split(delimiter) + combined_chunks, _, dropped_chunk_count = combine_chunks_with_no_minimum( + chunks, max_tokens, chunk_delimiter=delimiter, add_ellipsis_for_overflow=True) + if dropped_chunk_count > 0: + print(f"Warning: {dropped_chunk_count} chunks were dropped due to exceeding the token limit.") + combined_chunks = [f"{chunk}{delimiter}" for chunk in combined_chunks] + return combined_chunks + + + + + +####################################### + + +######### Words-per-second Chunking ######### +# FIXME - WHole section needs to be re-written +def chunk_transcript(transcript: str, chunk_duration: int, words_per_second) -> List[str]: + words = transcript.split() + words_per_chunk = chunk_duration * words_per_second + chunks = [' '.join(words[i:i + words_per_chunk]) for i in range(0, len(words), words_per_chunk)] + return chunks + + +# def summarize_chunks(api_name: str, api_key: str, transcript: List[dict], chunk_duration: int, +# words_per_second: int) -> str: + # if api_name not in summarizers: # See 'summarizers' dict in the main script + # return f"Unsupported API: {api_name}" + # + # if not transcript: + # logging.error("Empty or None transcript provided to summarize_chunks") + # return "Error: Empty or None transcript provided" + # + # text = extract_text_from_segments(transcript) + # chunks = chunk_transcript(text, chunk_duration, words_per_second) + # + # #FIXME + # custom_prompt = args.custom_prompt + # + # summaries = [] + # for chunk in chunks: + # if api_name == 'openai': + # # Ensure the correct model and prompt are passed + # summaries.append(summarize_with_openai(api_key, chunk, custom_prompt)) + # elif api_name == 'anthropic': + # summaries.append(summarize_with_cohere(api_key, chunk, anthropic_model, custom_prompt)) + # elif api_name == 'cohere': + # summaries.append(summarize_with_anthropic(api_key, chunk, cohere_model, custom_prompt)) + # elif api_name == 'groq': + # summaries.append(summarize_with_groq(api_key, chunk, groq_model, custom_prompt)) + # elif api_name == 'llama': + # summaries.append(summarize_with_llama(llama_api_IP, chunk, api_key, custom_prompt)) + # elif api_name == 'kobold': + # summaries.append(summarize_with_kobold(kobold_api_IP, chunk, api_key, custom_prompt)) + # elif api_name == 'ooba': + # summaries.append(summarize_with_oobabooga(ooba_api_IP, chunk, api_key, custom_prompt)) + # elif api_name == 'tabbyapi': + # summaries.append(summarize_with_vllm(api_key, tabby_api_IP, chunk, summarize.llm_model, custom_prompt)) + # elif api_name == 'local-llm': + # summaries.append(summarize_with_local_llm(chunk, custom_prompt)) + # else: + # return f"Unsupported API: {api_name}" + # + # return "\n\n".join(summaries) + +# FIXME - WHole section needs to be re-written +def summarize_with_detail_openai(text, detail, verbose=False): + summary_with_detail_variable = rolling_summarize(text, detail=detail, verbose=True) + print(len(openai_tokenize(summary_with_detail_variable))) + return summary_with_detail_variable + + +def summarize_with_detail_recursive_openai(text, detail, verbose=False): + summary_with_recursive_summarization = rolling_summarize(text, detail=detail, summarize_recursively=True) + print(summary_with_recursive_summarization) + +# +# +################################################################################# diff --git a/App_Function_Libraries/PDF_Ingestion_Lib.py b/App_Function_Libraries/PDF_Ingestion_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..341bd57e6549e88b455e93edbc39c69501cacaf7 --- /dev/null +++ b/App_Function_Libraries/PDF_Ingestion_Lib.py @@ -0,0 +1,166 @@ +# PDF_Ingestion_Lib.py +######################################### +# Library to hold functions for ingesting PDF files.# +# +#################### +# Function List +# +# 1. convert_pdf_to_markdown(pdf_path) +# 2. ingest_pdf_file(file_path, title=None, author=None, keywords=None): +# 3. +# +# +#################### + + +# Import necessary libraries +from datetime import datetime +import logging +import subprocess +import os +import shutil +import tempfile + + +# Import Local +from App_Function_Libraries.SQLite_DB import add_media_with_keywords + +####################################################################################################################### +# Function Definitions +# + +# Ingest a text file into the database with Title/Author/Keywords + + +# Constants +MAX_FILE_SIZE_MB = 50 +CONVERSION_TIMEOUT_SECONDS = 300 + + +def convert_pdf_to_markdown(pdf_path): + """ + Convert a PDF file to Markdown by calling a script in another virtual environment. + """ + + logging.debug(f"Marker: Converting PDF file to Markdown: {pdf_path}") + # Check if the file size exceeds the maximum allowed size + file_size_mb = os.path.getsize(pdf_path) / (1024 * 1024) + if file_size_mb > MAX_FILE_SIZE_MB: + raise ValueError(f"File size ({file_size_mb:.2f} MB) exceeds the maximum allowed size of {MAX_FILE_SIZE_MB} MB") + + logging.debug("Marker: Converting PDF file to Markdown using Marker virtual environment") + # Path to the Python interpreter in the other virtual environment + other_venv_python = "Helper_Scripts/marker_venv/bin/python" + + # Path to the conversion script + converter_script = "Helper_Scripts/PDF_Converter.py" + + logging.debug("Marker: Attempting to convert PDF file to Markdown...") + try: + result = subprocess.run( + [other_venv_python, converter_script, pdf_path], + capture_output=True, + text=True, + timeout=CONVERSION_TIMEOUT_SECONDS + ) + if result.returncode != 0: + raise Exception(f"Conversion failed: {result.stderr}") + return result.stdout + except subprocess.TimeoutExpired: + raise Exception(f"PDF conversion timed out after {CONVERSION_TIMEOUT_SECONDS} seconds") + + +def process_and_ingest_pdf(file, title, author, keywords): + if file is None: + return "Please select a PDF file to upload." + + try: + # Create a temporary directory + with tempfile.TemporaryDirectory() as temp_dir: + # Create a path for the temporary PDF file + temp_path = os.path.join(temp_dir, "temp.pdf") + + # Copy the contents of the uploaded file to the temporary file + shutil.copy(file.name, temp_path) + + # Call the ingest_pdf_file function with the temporary file path + result = ingest_pdf_file(temp_path, title, author, keywords) + + return result + except Exception as e: + return f"Error processing PDF: {str(e)}" + + +def ingest_pdf_file(file_path, title=None, author=None, keywords=None): + try: + # Convert PDF to Markdown + markdown_content = convert_pdf_to_markdown(file_path) + + # If title is not provided, use the filename without extension + if not title: + title = os.path.splitext(os.path.basename(file_path))[0] + + # If author is not provided, set it to 'Unknown' + if not author: + author = 'Unknown' + + # If keywords are not provided, use a default keyword + if not keywords: + keywords = 'pdf_file,markdown_converted' + else: + keywords = f'pdf_file,markdown_converted,{keywords}' + + # Add the markdown content to the database + add_media_with_keywords( + url=file_path, + title=title, + media_type='document', + content=markdown_content, + keywords=keywords, + prompt='No prompt for PDF files', + summary='No summary for PDF files', + transcription_model='None', + author=author, + ingestion_date=datetime.now().strftime('%Y-%m-%d') + ) + + return f"PDF file '{title}' converted to Markdown and ingested successfully.", file_path + except ValueError as e: + logging.error(f"File size error: {str(e)}") + return f"Error: {str(e)}", file_path + except Exception as e: + logging.error(f"Error ingesting PDF file: {str(e)}") + return f"Error ingesting PDF file: {str(e)}", file_path + + +def process_and_cleanup_pdf(file, title, author, keywords): + if file is None: + return "No file uploaded. Please upload a PDF file." + + temp_dir = tempfile.mkdtemp() + temp_file_path = os.path.join(temp_dir, "temp.pdf") + + try: + # Copy the uploaded file to a temporary location + shutil.copy2(file.name, temp_file_path) + + # Process the file + result, _ = ingest_pdf_file(temp_file_path, title, author, keywords) + + return result + except Exception as e: + logging.error(f"Error in processing and cleanup: {str(e)}") + return f"Error: {str(e)}" + finally: + # Clean up the temporary directory and its contents + try: + shutil.rmtree(temp_dir) + logging.info(f"Removed temporary directory: {temp_dir}") + except Exception as cleanup_error: + logging.error(f"Error during cleanup: {str(cleanup_error)}") + result += f"\nWarning: Could not remove temporary files: {str(cleanup_error)}" + + +# +# +####################################################################################################################### \ No newline at end of file diff --git a/App_Function_Libraries/RAG_Library.py b/App_Function_Libraries/RAG_Library.py new file mode 100644 index 0000000000000000000000000000000000000000..4384d1868f2c132b3b1ed80da97c4ec65dba7093 --- /dev/null +++ b/App_Function_Libraries/RAG_Library.py @@ -0,0 +1,812 @@ +# RAG_Library.py +######################################### +# RAG Search & Related Functions Library +# This library is used to hold any/all RAG-related operations. +# Currently, all of this code was generated from Sonnet 3.5. 0_0 +# +#### + +import os +from typing import List, Tuple, Callable, Optional +from contextlib import contextmanager +import sqlite3 +import numpy as np +from sentence_transformers import SentenceTransformer +from sklearn.metrics.pairwise import cosine_similarity +import logging +from dotenv import load_dotenv + +load_dotenv() + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + + +class RAGException(Exception): + """Custom exception class for RAG-related errors""" + pass + + +class BaseRAGSystem: + def __init__(self, db_path: str, model_name: Optional[str] = None): + """ + Initialize the RAG system. + + :param db_path: Path to the SQLite database + :param model_name: Name of the SentenceTransformer model to use + """ + self.db_path = db_path + self.model_name = model_name or os.getenv('DEFAULT_MODEL_NAME', 'all-MiniLM-L6-v2') + try: + self.model = SentenceTransformer(self.model_name) + logger.info(f"Initialized SentenceTransformer with model: {self.model_name}") + except Exception as e: + logger.error(f"Failed to initialize SentenceTransformer: {e}") + raise RAGException(f"Model initialization failed: {e}") + + self.init_db() + + @contextmanager + def get_db_connection(self): + conn = sqlite3.connect(self.db_path) + try: + yield conn + finally: + conn.close() + + def init_db(self): + try: + with self.get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute(''' + CREATE TABLE IF NOT EXISTS documents ( + id INTEGER PRIMARY KEY, + title TEXT, + content TEXT, + embedding BLOB + ) + ''') + conn.commit() + logger.info("Initialized database schema") + except sqlite3.Error as e: + logger.error(f"Failed to initialize database schema: {e}") + raise RAGException(f"Database schema initialization failed: {e}") + + def add_documents(self, documents: List[Tuple[str, str]]): + try: + embeddings = self.model.encode([content for _, content in documents]) + with self.get_db_connection() as conn: + cursor = conn.cursor() + cursor.executemany( + 'INSERT INTO documents (title, content, embedding) VALUES (?, ?, ?)', + [(title, content, embedding.tobytes()) for (title, content), embedding in zip(documents, embeddings)] + ) + conn.commit() + logger.info(f"Added {len(documents)} documents in batch") + except Exception as e: + logger.error(f"Failed to add documents in batch: {e}") + raise RAGException(f"Batch document addition failed: {e}") + + def get_documents(self) -> List[Tuple[int, str, str, np.ndarray]]: + try: + with self.get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute('SELECT id, title, content, embedding FROM documents') + documents = [(id, title, content, np.frombuffer(embedding, dtype=np.float32)) + for id, title, content, embedding in cursor.fetchall()] + logger.info(f"Retrieved {len(documents)} documents") + return documents + except sqlite3.Error as e: + logger.error(f"Failed to retrieve documents: {e}") + raise RAGException(f"Document retrieval failed: {e}") + + def close(self): + try: + self.conn.close() + logger.info("Closed database connection") + except sqlite3.Error as e: + logger.error(f"Error closing database connection: {e}") + + +class StandardRAGSystem(BaseRAGSystem): + def get_relevant_documents(self, query: str, top_k: int = 3) -> List[Tuple[int, str, str, float]]: + try: + query_embedding = self.model.encode([query])[0] + documents = self.get_documents() + similarities = [ + (id, title, content, cosine_similarity([query_embedding], [doc_embedding])[0][0]) + for id, title, content, doc_embedding in documents + ] + similarities.sort(key=lambda x: x[3], reverse=True) + logger.info(f"Retrieved top {top_k} relevant documents for query") + return similarities[:top_k] + except Exception as e: + logger.error(f"Error in getting relevant documents: {e}") + raise RAGException(f"Retrieval of relevant documents failed: {e}") + + def rag_query(self, query: str, llm_function: Callable[[str], str], top_k: int = 3) -> str: + try: + relevant_docs = self.get_relevant_documents(query, top_k) + context = "\n\n".join([f"Title: {title}\nContent: {content}" for _, title, content, _ in relevant_docs]) + + llm_prompt = f"Based on the following context, please answer the query:\n\nContext:\n{context}\n\nQuery: {query}" + + response = llm_function(llm_prompt) + logger.info("Generated response for query") + return response + except Exception as e: + logger.error(f"Error in RAG query: {e}") + raise RAGException(f"RAG query failed: {e}") + + +class HyDERAGSystem(BaseRAGSystem): + def generate_hypothetical_document(self, query: str, llm_function: Callable[[str], str]) -> str: + try: + prompt = f"Given the question '{query}', write a short paragraph that would answer this question. Do not include the question itself in your response." + hypothetical_doc = llm_function(prompt) + logger.info("Generated hypothetical document") + return hypothetical_doc + except Exception as e: + logger.error(f"Error generating hypothetical document: {e}") + raise RAGException(f"Hypothetical document generation failed: {e}") + + def get_relevant_documents(self, query: str, llm_function: Callable[[str], str], top_k: int = 3) -> List[ + Tuple[int, str, str, float]]: + try: + hypothetical_doc = self.generate_hypothetical_document(query, llm_function) + hyde_embedding = self.model.encode([hypothetical_doc])[0] + + documents = self.get_documents() + similarities = [ + (id, title, content, cosine_similarity([hyde_embedding], [doc_embedding])[0][0]) + for id, title, content, doc_embedding in documents + ] + similarities.sort(key=lambda x: x[3], reverse=True) + logger.info(f"Retrieved top {top_k} relevant documents using HyDE") + return similarities[:top_k] + except Exception as e: + logger.error(f"Error in getting relevant documents with HyDE: {e}") + raise RAGException(f"HyDE retrieval of relevant documents failed: {e}") + + def rag_query(self, query: str, llm_function: Callable[[str], str], top_k: int = 3) -> str: + try: + relevant_docs = self.get_relevant_documents(query, llm_function, top_k) + context = "\n\n".join([f"Title: {title}\nContent: {content}" for _, title, content, _ in relevant_docs]) + + llm_prompt = f"Based on the following context, please answer the query:\n\nContext:\n{context}\n\nQuery: {query}" + + response = llm_function(llm_prompt) + logger.info("Generated response for query using HyDE") + return response + except Exception as e: + logger.error(f"Error in HyDE RAG query: {e}") + raise RAGException(f"HyDE RAG query failed: {e}") + + +# Example usage with error handling +def mock_llm(prompt: str) -> str: + if "write a short paragraph" in prompt: + return "Paris, the capital of France, is renowned for its iconic Eiffel Tower and rich cultural heritage." + else: + return f"This is a mock LLM response for the prompt: {prompt}" + + +def main(): + use_hyde = False # Set this to True when you want to enable HyDE + + try: + if use_hyde: + rag_system = HyDERAGSystem('rag_database.db') + logger.info("Using HyDE RAG System") + else: + rag_system = StandardRAGSystem('rag_database.db') + logger.info("Using Standard RAG System") + + # Add sample documents in batch + sample_docs = [ + ("Paris", "Paris is the capital of France and is known for the Eiffel Tower."), + ("London", "London is the capital of the United Kingdom and home to Big Ben."), + ("Tokyo", "Tokyo is the capital of Japan and is famous for its bustling city life.") + ] + + for title, content in sample_docs: + rag_system.add_document(title, content) + + query = "What is the capital of France?" + result = rag_system.rag_query(query, mock_llm) + print(f"Query: {query}") + print(f"Result: {result}") + + except RAGException as e: + logger.error(f"RAG system error: {e}") + print(f"An error occurred: {e}") + except Exception as e: + logger.error(f"Unexpected error: {e}") + print(f"An unexpected error occurred: {e}") + finally: + if 'rag_system' in locals(): + rag_system.close() + + +if __name__ == "__main__": + main() + + + +#################################################################################### +# async: + +# import os +# import asyncio +# from typing import List, Tuple, Callable, Optional +# import aiosqlite +# import numpy as np +# from sentence_transformers import SentenceTransformer +# from sklearn.metrics.pairwise import cosine_similarity +# import logging +# from dotenv import load_dotenv +# +# load_dotenv() +# +# logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +# logger = logging.getLogger(__name__) +# +# +# class RAGException(Exception): +# """Custom exception class for RAG-related errors""" +# pass +# +# +# class BaseRAGSystem: +# def __init__(self, db_path: str, model_name: Optional[str] = None): +# """ +# Initialize the RAG system. +# +# :param db_path: Path to the SQLite database +# :param model_name: Name of the SentenceTransformer model to use +# """ +# self.db_path = db_path +# self.model_name = model_name or os.getenv('DEFAULT_MODEL_NAME', 'all-MiniLM-L6-v2') +# try: +# self.model = SentenceTransformer(self.model_name) +# logger.info(f"Initialized SentenceTransformer with model: {self.model_name}") +# except Exception as e: +# logger.error(f"Failed to initialize SentenceTransformer: {e}") +# raise RAGException(f"Model initialization failed: {e}") +# +# async def init_db(self): +# try: +# async with aiosqlite.connect(self.db_path) as db: +# await db.execute(''' +# CREATE TABLE IF NOT EXISTS documents ( +# id INTEGER PRIMARY KEY, +# title TEXT, +# content TEXT, +# embedding BLOB +# ) +# ''') +# await db.commit() +# logger.info("Initialized database schema") +# except aiosqlite.Error as e: +# logger.error(f"Failed to initialize database schema: {e}") +# raise RAGException(f"Database schema initialization failed: {e}") +# +# async def add_documents(self, documents: List[Tuple[str, str]]): +# try: +# embeddings = self.model.encode([content for _, content in documents]) +# async with aiosqlite.connect(self.db_path) as db: +# await db.executemany( +# 'INSERT INTO documents (title, content, embedding) VALUES (?, ?, ?)', +# [(title, content, embedding.tobytes()) for (title, content), embedding in +# zip(documents, embeddings)] +# ) +# await db.commit() +# logger.info(f"Added {len(documents)} documents in batch") +# except Exception as e: +# logger.error(f"Failed to add documents in batch: {e}") +# raise RAGException(f"Batch document addition failed: {e}") +# +# async def get_documents(self) -> List[Tuple[int, str, str, np.ndarray, str]]: +# try: +# async with aiosqlite.connect(self.db_path) as db: +# async with db.execute('SELECT id, title, content, embedding, source FROM documents') as cursor: +# documents = [ +# (id, title, content, np.frombuffer(embedding, dtype=np.float32), source) +# async for id, title, content, embedding, source in cursor +# ] +# logger.info(f"Retrieved {len(documents)} documents") +# return documents +# except aiosqlite.Error as e: +# logger.error(f"Failed to retrieve documents: {e}") +# raise RAGException(f"Document retrieval failed: {e}") +# +# +# class AsyncStandardRAGSystem(BaseRAGSystem): +# async def get_relevant_documents(self, query: str, top_k: int = 3) -> List[Tuple[int, str, str, float]]: +# try: +# query_embedding = self.model.encode([query])[0] +# documents = await self.get_documents() +# similarities = [ +# (id, title, content, cosine_similarity([query_embedding], [doc_embedding])[0][0]) +# for id, title, content, doc_embedding in documents +# ] +# similarities.sort(key=lambda x: x[3], reverse=True) +# logger.info(f"Retrieved top {top_k} relevant documents for query") +# return similarities[:top_k] +# except Exception as e: +# logger.error(f"Error in getting relevant documents: {e}") +# raise RAGException(f"Retrieval of relevant documents failed: {e}") +# +# async def rag_query(self, query: str, llm_function: Callable[[str], str], top_k: int = 3) -> str: +# try: +# relevant_docs = await self.get_relevant_documents(query, top_k) +# context = "\n\n".join([f"Title: {title}\nContent: {content}\nSource: {source}" for _, title, content, _, source in relevant_docs]) +# +# llm_prompt = f"Based on the following context, please answer the query. Include citations in your response using [Source] format:\n\nContext:\n{context}\n\nQuery: {query}" +# +# response = llm_function(llm_prompt) +# logger.info("Generated response for query") +# return response +# except Exception as e: +# logger.error(f"Error in RAG query: {e}") +# raise RAGException(f"RAG query failed: {e}") +# +# +# class AsyncHyDERAGSystem(BaseRAGSystem): +# async def generate_hypothetical_document(self, query: str, llm_function: Callable[[str], str]) -> str: +# try: +# prompt = f"Given the question '{query}', write a short paragraph that would answer this question. Do not include the question itself in your response." +# hypothetical_doc = llm_function(prompt) +# logger.info("Generated hypothetical document") +# return hypothetical_doc +# except Exception as e: +# logger.error(f"Error generating hypothetical document: {e}") +# raise RAGException(f"Hypothetical document generation failed: {e}") +# +# async def get_relevant_documents(self, query: str, llm_function: Callable[[str], str], top_k: int = 3) -> List[ +# Tuple[int, str, str, float]]: +# try: +# hypothetical_doc = await self.generate_hypothetical_document(query, llm_function) +# hyde_embedding = self.model.encode([hypothetical_doc])[0] +# +# documents = await self.get_documents() +# similarities = [ +# (id, title, content, cosine_similarity([hyde_embedding], [doc_embedding])[0][0]) +# for id, title, content, doc_embedding in documents +# ] +# similarities.sort(key=lambda x: x[3], reverse=True) +# logger.info(f"Retrieved top {top_k} relevant documents using HyDE") +# return similarities[:top_k] +# except Exception as e: +# logger.error(f"Error in getting relevant documents with HyDE: {e}") +# raise RAGException(f"HyDE retrieval of relevant documents failed: {e}") +# +# async def rag_query(self, query: str, llm_function: Callable[[str], str], top_k: int = 3) -> str: +# try: +# relevant_docs = await self.get_relevant_documents(query, llm_function, top_k) +# context = "\n\n".join([f"Title: {title}\nContent: {content}" for _, title, content, _ in relevant_docs]) +# +# llm_prompt = f"Based on the following context, please answer the query:\n\nContext:\n{context}\n\nQuery: {query}" +# +# response = llm_function(llm_prompt) +# logger.info("Generated response for query using HyDE") +# return response +# except Exception as e: +# logger.error(f"Error in HyDE RAG query: {e}") +# raise RAGException(f"HyDE RAG query failed: {e}") +# +# +# # Example usage with error handling +# def mock_llm(prompt: str) -> str: +# if "write a short paragraph" in prompt: +# return "Paris, the capital of France, is renowned for its iconic Eiffel Tower and rich cultural heritage." +# else: +# return f"This is a mock LLM response for the prompt: {prompt}" +# +# +# async def main(): +# use_hyde = False # Set this to True when you want to enable HyDE +# +# try: +# if use_hyde: +# rag_system = AsyncHyDERAGSystem('rag_database.db') +# logger.info("Using Async HyDE RAG System") +# else: +# rag_system = AsyncStandardRAGSystem('rag_database.db') +# logger.info("Using Async Standard RAG System") +# +# await rag_system.init_db() +# +# # Add sample documents +# sample_docs = [ +# ("Paris", "Paris is the capital of France and is known for the Eiffel Tower."), +# ("London", "London is the capital of the United Kingdom and home to Big Ben."), +# ("Tokyo", "Tokyo is the capital of Japan and is famous for its bustling city life.") +# ] +# +# await rag_system.add_documents(sample_docs) +# +# query = "What is the capital of France?" +# result = await rag_system.rag_query(query, mock_llm) +# print(f"Query: {query}") +# print(f"Result: {result}") +# +# except RAGException as e: +# logger.error(f"RAG system error: {e}") +# print(f"An error occurred: {e}") +# except Exception as e: +# logger.error(f"Unexpected error: {e}") +# print(f"An unexpected error occurred: {e}") +# +# +# if __name__ == "__main__": +# asyncio.run(main()) + + + +# +# from fastapi import FastAPI, HTTPException +# +# app = FastAPI() +# rag_system = AsyncStandardRAGSystem('rag_database.db') +# +# @app.on_event("startup") +# async def startup_event(): +# await rag_system.init_db() +# +# @app.get("/query") +# async def query(q: str): +# try: +# result = await rag_system.rag_query(q, mock_llm) +# return {"query": q, "result": result} +# except RAGException as e: +# raise HTTPException(status_code=500, detail=str(e)) +# + + +############################################################################################ +# Using FAISS +# +# +# +# Update DB +# async def init_db(self): +# try: +# async with aiosqlite.connect(self.db_path) as db: +# await db.execute(''' +# CREATE TABLE IF NOT EXISTS documents ( +# id INTEGER PRIMARY KEY, +# title TEXT, +# content TEXT, +# embedding BLOB, +# source TEXT +# ) +# ''') +# await db.commit() +# logger.info("Initialized database schema") +# except aiosqlite.Error as e: +# logger.error(f"Failed to initialize database schema: {e}") +# raise RAGException(f"Database schema initialization failed: {e}") +# +# + +# import os +# import asyncio +# from typing import List, Tuple, Callable, Optional +# import aiosqlite +# import numpy as np +# from sentence_transformers import SentenceTransformer +# import faiss +# import logging +# from dotenv import load_dotenv +# +# load_dotenv() +# +# logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +# logger = logging.getLogger(__name__) +# +# +# class RAGException(Exception): +# """Custom exception class for RAG-related errors""" +# pass +# +# +# class AsyncFAISSRAGSystem: +# def __init__(self, db_path: str, model_name: Optional[str] = None): +# self.db_path = db_path +# self.model_name = model_name or os.getenv('DEFAULT_MODEL_NAME', 'all-MiniLM-L6-v2') +# try: +# self.model = SentenceTransformer(self.model_name) +# logger.info(f"Initialized SentenceTransformer with model: {self.model_name}") +# except Exception as e: +# logger.error(f"Failed to initialize SentenceTransformer: {e}") +# raise RAGException(f"Model initialization failed: {e}") +# +# self.index = None +# self.document_lookup = {} +# +# async def init_db(self): +# try: +# async with aiosqlite.connect(self.db_path) as db: +# await db.execute(''' +# CREATE TABLE IF NOT EXISTS documents ( +# id INTEGER PRIMARY KEY, +# title TEXT, +# content TEXT +# ) +# ''') +# await db.commit() +# logger.info("Initialized database schema") +# except aiosqlite.Error as e: +# logger.error(f"Failed to initialize database schema: {e}") +# raise RAGException(f"Database schema initialization failed: {e}") +# +# async def add_documents(self, documents: List[Tuple[str, str, str]]): +# try: +# embeddings = self.model.encode([content for _, content, _ in documents]) +# async with aiosqlite.connect(self.db_path) as db: +# await db.executemany( +# 'INSERT INTO documents (title, content, embedding, source) VALUES (?, ?, ?, ?)', +# [(title, content, embedding.tobytes(), source) for (title, content, source), embedding in +# zip(documents, embeddings)] +# ) +# await db.commit() +# logger.info(f"Added {len(documents)} documents in batch") +# except Exception as e: +# logger.error(f"Failed to add documents in batch: {e}") +# raise RAGException(f"Batch document addition failed: {e}") +# +# async def get_relevant_documents(self, query: str, top_k: int = 3) -> List[Tuple[int, str, str, float, str]]: +# try: +# query_embedding = self.model.encode([query])[0] +# documents = await self.get_documents() +# similarities = [ +# (id, title, content, cosine_similarity([query_embedding], [doc_embedding])[0][0], source) +# for id, title, content, doc_embedding, source in documents +# ] +# similarities.sort(key=lambda x: x[3], reverse=True) +# logger.info(f"Retrieved top {top_k} relevant documents for query") +# return similarities[:top_k] +# except Exception as e: +# logger.error(f"Error in getting relevant documents: {e}") +# raise RAGException(f"Retrieval of relevant documents failed: {e}") +# +# async def rag_query(self, query: str, llm_function: Callable[[str], str], top_k: int = 3) -> str: +# try: +# relevant_docs = await self.get_relevant_documents(query, top_k) +# context = "\n\n".join([f"Title: {title}\nContent: {content}" for _, title, content, _ in relevant_docs]) +# +# llm_prompt = f"Based on the following context, please answer the query:\n\nContext:\n{context}\n\nQuery: {query}" +# +# response = llm_function(llm_prompt) +# logger.info("Generated response for query") +# return response +# except Exception as e: +# logger.error(f"Error in RAG query: {e}") +# raise RAGException(f"RAG query failed: {e}") +# +# +# class AsyncFAISSHyDERAGSystem(AsyncFAISSRAGSystem): +# async def generate_hypothetical_document(self, query: str, llm_function: Callable[[str], str]) -> str: +# try: +# prompt = f"Given the question '{query}', write a short paragraph that would answer this question. Do not include the question itself in your response." +# hypothetical_doc = llm_function(prompt) +# logger.info("Generated hypothetical document") +# return hypothetical_doc +# except Exception as e: +# logger.error(f"Error generating hypothetical document: {e}") +# raise RAGException(f"Hypothetical document generation failed: {e}") +# +# async def get_relevant_documents(self, query: str, llm_function: Callable[[str], str], top_k: int = 3) -> List[ +# Tuple[int, str, str, float]]: +# try: +# hypothetical_doc = await self.generate_hypothetical_document(query, llm_function) +# hyde_embedding = self.model.encode([hypothetical_doc])[0] +# +# distances, indices = self.index.search(np.array([hyde_embedding]), top_k) +# +# results = [] +# for i, idx in enumerate(indices[0]): +# doc_id = list(self.document_lookup.keys())[idx] +# title, content = self.document_lookup[doc_id] +# results.append((doc_id, title, content, distances[0][i])) +# +# logger.info(f"Retrieved top {top_k} relevant documents using HyDE") +# return results +# except Exception as e: +# logger.error(f"Error in getting relevant documents with HyDE: {e}") +# raise RAGException(f"HyDE retrieval of relevant documents failed: {e}") +# +# +# # Example usage +# def mock_llm(prompt: str) -> str: +# if "write a short paragraph" in prompt: +# return "Paris, the capital of France, is renowned for its iconic Eiffel Tower and rich cultural heritage." +# else: +# return f"This is a mock LLM response for the prompt: {prompt}" +# +# +# async def main(): +# use_hyde = False # Set this to True when you want to enable HyDE +# +# try: +# if use_hyde: +# rag_system = AsyncFAISSHyDERAGSystem('rag_database.db') +# logger.info("Using Async FAISS HyDE RAG System") +# else: +# rag_system = AsyncFAISSRAGSystem('rag_database.db') +# logger.info("Using Async FAISS RAG System") +# +# await rag_system.init_db() +# +# # Add sample documents +# sample_docs = [ +# ("Paris", "Paris is the capital of France and is known for the Eiffel Tower."), +# ("London", "London is the capital of the United Kingdom and home to Big Ben."), +# ("Tokyo", "Tokyo is the capital of Japan and is famous for its bustling city life.") +# ] +# +# await rag_system.add_documents(sample_docs) +# +# query = "What is the capital of France?" +# result = await rag_system.rag_query(query, mock_llm) +# print(f"Query: {query}") +# print(f"Result: {result}") +# +# except RAGException as e: +# logger.error(f"RAG system error: {e}") +# print(f"An error occurred: {e}") +# except Exception as e: +# logger.error(f"Unexpected error: {e}") +# print(f"An unexpected error occurred: {e}") +# +# +# if __name__ == "__main__": +# asyncio.run(main()) + + +""" +Key changes in this FAISS-integrated version: + +We've replaced the cosine similarity search with FAISS indexing and search. +The add_documents method now adds embeddings to the FAISS index as well as storing documents in the SQLite database. +We maintain a document_lookup dictionary to quickly retrieve document content based on FAISS search results. +The get_relevant_documents method now uses FAISS for similarity search instead of computing cosine similarities manually. +We've kept the asynchronous structure for database operations, while FAISS operations remain synchronous (as FAISS doesn't have built-in async support). + +Benefits of using FAISS: + +Scalability: FAISS can handle millions of vectors efficiently, making it suitable for large document collections. +Speed: FAISS is optimized for fast similarity search, which can significantly improve query times as your dataset grows. +Memory Efficiency: FAISS provides various index types that can trade off between search accuracy and memory usage, allowing you to optimize for your specific use case. + +Considerations: + +This implementation uses a simple IndexFlatL2 FAISS index, which performs exact search. For larger datasets, you might want to consider approximate search methods like IndexIVFFlat for better scalability. +The current implementation keeps all document content in memory (in the document_lookup dictionary). For very large datasets, you might want to modify this to fetch document content from the database as needed. +If you're dealing with a very large number of documents, you might want to implement batch processing for adding documents to the FAISS index. + +This FAISS-integrated version should provide better performance for similarity search, especially as your document collection grows larger +""" + + +############################################################################################################### +# Web Search +# Output from Sonnet 3.5 regarding how to add web searches to the RAG system +# Integrating web search into your RAG system can significantly enhance its capabilities by providing up-to-date information. Here's how you can modify your RAG system to include web search: +# +# First, you'll need to choose a web search API. Some popular options include: +# +# Google Custom Search API +# Bing Web Search API +# DuckDuckGo API +# SerpAPI (which can interface with multiple search engines) +# +# +# +# For this example, let's use the DuckDuckGo API, as it's free and doesn't require authentication. +# +# Install the required library: +# `pip install duckduckgo-search` +# +# Add a new method to your RAG system for web search: +# ``` +# from duckduckgo_search import ddg +# +# class AsyncRAGSystem: +# # ... (existing code) ... +# +# async def web_search(self, query: str, num_results: int = 3) -> List[Dict[str, str]]: +# try: +# results = ddg(query, max_results=num_results) +# return [{'title': r['title'], 'content': r['body'], 'source': r['href']} for r in results] +# except Exception as e: +# logger.error(f"Error in web search: {e}") +# raise RAGException(f"Web search failed: {e}") +# +# async def add_web_results_to_db(self, results: List[Dict[str, str]]): +# try: +# documents = [(r['title'], r['content'], r['source']) for r in results] +# await self.add_documents(documents) +# logger.info(f"Added {len(documents)} web search results to the database") +# except Exception as e: +# logger.error(f"Error adding web search results to database: {e}") +# raise RAGException(f"Adding web search results failed: {e}") +# +# async def rag_query_with_web_search(self, query: str, llm_function: Callable[[str], str], top_k: int = 3, +# use_web_search: bool = True, num_web_results: int = 3) -> str: +# try: +# if use_web_search: +# web_results = await self.web_search(query, num_web_results) +# await self.add_web_results_to_db(web_results) +# +# relevant_docs = await self.get_relevant_documents(query, top_k) +# context = "\n\n".join([f"Title: {title}\nContent: {content}\nSource: {source}" +# for _, title, content, _, source in relevant_docs]) +# +# llm_prompt = f"Based on the following context, please answer the query. Include citations in your response using [Source] format:\n\nContext:\n{context}\n\nQuery: {query}" +# +# response = llm_function(llm_prompt) +# logger.info("Generated response for query with web search") +# return response +# except Exception as e: +# logger.error(f"Error in RAG query with web search: {e}") +# raise RAGException(f"RAG query with web search failed: {e}") +# ``` +# +# Update your main function to use the new web search capability: +# ``` +# async def main(): +# use_hyde = False # Set this to True when you want to enable HyDE +# use_web_search = True # Set this to False if you don't want to use web search +# +# try: +# if use_hyde: +# rag_system = AsyncHyDERAGSystem('rag_database.db') +# logger.info("Using Async HyDE RAG System") +# else: +# rag_system = AsyncStandardRAGSystem('rag_database.db') +# logger.info("Using Async Standard RAG System") +# +# await rag_system.init_db() +# +# # Add sample documents +# sample_docs = [ +# ("Paris", "Paris is the capital of France and is known for the Eiffel Tower.", "Local Database"), +# ("London", "London is the capital of the United Kingdom and home to Big Ben.", "Local Database"), +# ("Tokyo", "Tokyo is the capital of Japan and is famous for its bustling city life.", "Local Database") +# ] +# +# await rag_system.add_documents(sample_docs) +# +# query = "What is the capital of France?" +# result = await rag_system.rag_query_with_web_search(query, mock_llm, use_web_search=use_web_search) +# print(f"Query: {query}") +# print(f"Result: {result}") +# +# except RAGException as e: +# logger.error(f"RAG system error: {e}") +# print(f"An error occurred: {e}") +# except Exception as e: +# logger.error(f"Unexpected error: {e}") +# print(f"An unexpected error occurred: {e}") +# ``` +# +# +# This implementation does the following: +# +# It adds a web_search method that uses the DuckDuckGo API to perform web searches. +# It adds an add_web_results_to_db method that adds the web search results to your existing database. +# It modifies the rag_query method (now called rag_query_with_web_search) to optionally perform a web search before retrieving relevant documents. +# +# When use_web_search is set to True, the system will: +# +# Perform a web search for the given query. +# Add the web search results to the database. +# Retrieve relevant documents (which now may include the newly added web search results). +# Use these documents to generate a response. +# +# This approach allows your RAG system to combine information from your existing database with fresh information from the web, potentially providing more up-to-date and comprehensive answers. +# Remember to handle rate limiting and respect the terms of service of the web search API you choose to use. Also, be aware that adding web search results to your database will increase its size over time, so you may need to implement a strategy to manage this growth (e.g., removing old web search results periodically). + + diff --git a/App_Function_Libraries/SQLite_DB.py b/App_Function_Libraries/SQLite_DB.py new file mode 100644 index 0000000000000000000000000000000000000000..f932587f06a2cbd698881d5f48a6a21311722b67 --- /dev/null +++ b/App_Function_Libraries/SQLite_DB.py @@ -0,0 +1,973 @@ +# SQLite_DB.py +######################################### +# SQLite_DB Library +# This library is used to perform any/all DB operations related to SQLite. +# +#### + +#################### +# Function List +# FIXME - UPDATE Function Arguments +# 1. get_connection(self) +# 2. execute_query(self, query: str, params: Tuple = ()) +# 3. create_tables() +# 4. add_keyword(keyword: str) +# 5. delete_keyword(keyword: str) +# 6. add_media_with_keywords(url, title, media_type, content, keywords, prompt, summary, transcription_model, author, ingestion_date) +# 7. fetch_all_keywords() +# 8. keywords_browser_interface() +# 9. display_keywords() +# 10. export_keywords_to_csv() +# 11. browse_items(search_query, search_type) +# 12. fetch_item_details(media_id: int) +# 13. add_media_version(media_id: int, prompt: str, summary: str) +# 14. search_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 10) +# 15. search_and_display(search_query, search_fields, keywords, page) +# 16. display_details(index, results) +# 17. get_details(index, dataframe) +# 18. format_results(results) +# 19. export_to_csv(search_query: str, search_fields: List[str], keyword: str, page: int = 1, results_per_file: int = 1000) +# 20. is_valid_url(url: str) -> bool +# 21. is_valid_date(date_string: str) -> bool +# 22. add_media_to_database(url, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model) +# 23. create_prompts_db() +# 24. add_prompt(name, details, system, user=None) +# 25. fetch_prompt_details(name) +# 26. list_prompts() +# 27. insert_prompt_to_db(title, description, system_prompt, user_prompt) +# 28. update_media_content(media_id: int, content: str, prompt: str, summary: str) +# 29. search_media_database(query: str) -> List[Tuple[int, str, str]] +# 30. load_media_content(media_id: int) +# 31. +# 32. +# +# +##################### +# +# Import necessary libraries +import csv +import logging +import os +import re +import sqlite3 +import time +from contextlib import contextmanager +from datetime import datetime +from typing import List, Tuple +# Third-Party Libraries +import gradio as gr +import pandas as pd +# Import Local Libraries +# +####################################################################################################################### +# Function Definitions +# + +# Set up logging +#logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +#logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + + +# Custom exceptions +class DatabaseError(Exception): + pass + + +class InputError(Exception): + pass + + +# Database connection function with connection pooling +class Database: + def __init__(self, db_name=None): + self.db_name = db_name or os.getenv('DB_NAME', 'media_summary.db') + self.pool = [] + self.pool_size = 10 + + @contextmanager + def get_connection(self): + retry_count = 5 + retry_delay = 1 + conn = None + while retry_count > 0: + try: + conn = self.pool.pop() if self.pool else sqlite3.connect(self.db_name, check_same_thread=False) + yield conn + self.pool.append(conn) + return + except sqlite3.OperationalError as e: + if 'database is locked' in str(e): + logging.warning(f"Database is locked, retrying in {retry_delay} seconds...") + retry_count -= 1 + time.sleep(retry_delay) + else: + raise DatabaseError(f"Database error: {e}") + except Exception as e: + raise DatabaseError(f"Unexpected error: {e}") + finally: + # Ensure the connection is returned to the pool even on failure + if conn: + self.pool.append(conn) + raise DatabaseError("Database is locked and retries have been exhausted") + + def execute_query(self, query: str, params: Tuple = ()) -> None: + with self.get_connection() as conn: + try: + cursor = conn.cursor() + cursor.execute(query, params) + conn.commit() + except sqlite3.Error as e: + raise DatabaseError(f"Database error: {e}, Query: {query}") + +db = Database() + + +# Function to create tables with the new media schema +def create_tables() -> None: + table_queries = [ + ''' + CREATE TABLE IF NOT EXISTS Media ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT, + title TEXT NOT NULL, + type TEXT NOT NULL, + content TEXT, + author TEXT, + ingestion_date TEXT, + prompt TEXT, + summary TEXT, + transcription_model TEXT + ) + ''', + ''' + CREATE TABLE IF NOT EXISTS Keywords ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + keyword TEXT NOT NULL UNIQUE + ) + ''', + ''' + CREATE TABLE IF NOT EXISTS MediaKeywords ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + media_id INTEGER NOT NULL, + keyword_id INTEGER NOT NULL, + FOREIGN KEY (media_id) REFERENCES Media(id), + FOREIGN KEY (keyword_id) REFERENCES Keywords(id) + ) + ''', + ''' + CREATE TABLE IF NOT EXISTS MediaVersion ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + media_id INTEGER NOT NULL, + version INTEGER NOT NULL, + prompt TEXT, + summary TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY (media_id) REFERENCES Media(id) + ) + ''', + ''' + CREATE TABLE IF NOT EXISTS MediaModifications ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + media_id INTEGER NOT NULL, + prompt TEXT, + summary TEXT, + modification_date TEXT, + FOREIGN KEY (media_id) REFERENCES Media(id) + ) + ''', + ''' + CREATE VIRTUAL TABLE IF NOT EXISTS media_fts USING fts5(title, content); + ''', + ''' + CREATE VIRTUAL TABLE IF NOT EXISTS keyword_fts USING fts5(keyword); + ''', + ''' + CREATE INDEX IF NOT EXISTS idx_media_title ON Media(title); + ''', + ''' + CREATE INDEX IF NOT EXISTS idx_media_type ON Media(type); + ''', + ''' + CREATE INDEX IF NOT EXISTS idx_media_author ON Media(author); + ''', + ''' + CREATE INDEX IF NOT EXISTS idx_media_ingestion_date ON Media(ingestion_date); + ''', + ''' + CREATE INDEX IF NOT EXISTS idx_keywords_keyword ON Keywords(keyword); + ''', + ''' + CREATE INDEX IF NOT EXISTS idx_mediakeywords_media_id ON MediaKeywords(media_id); + ''', + ''' + CREATE INDEX IF NOT EXISTS idx_mediakeywords_keyword_id ON MediaKeywords(keyword_id); + ''', + ''' + CREATE INDEX IF NOT EXISTS idx_media_version_media_id ON MediaVersion(media_id); + ''', + ''' + CREATE INDEX IF NOT EXISTS idx_mediamodifications_media_id ON MediaModifications(media_id); + ''', + ''' + CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_media_url ON Media(url); + ''', + ''' + CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_media_keyword ON MediaKeywords(media_id, keyword_id); + ''' + ] + for query in table_queries: + db.execute_query(query) + + logging.info("All tables and indexes created successfully.") + +create_tables() + + +####################################################################################################################### +# Keyword-related Functions +# + +# Function to add a keyword +def add_keyword(keyword: str) -> int: + keyword = keyword.strip().lower() + with db.get_connection() as conn: + cursor = conn.cursor() + try: + cursor.execute('INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)', (keyword,)) + cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,)) + keyword_id = cursor.fetchone()[0] + cursor.execute('INSERT OR IGNORE INTO keyword_fts (rowid, keyword) VALUES (?, ?)', (keyword_id, keyword)) + logging.info(f"Keyword '{keyword}' added to keyword_fts with ID: {keyword_id}") + conn.commit() + return keyword_id + except sqlite3.IntegrityError as e: + logging.error(f"Integrity error adding keyword: {e}") + raise DatabaseError(f"Integrity error adding keyword: {e}") + except sqlite3.Error as e: + logging.error(f"Error adding keyword: {e}") + raise DatabaseError(f"Error adding keyword: {e}") + + +# Function to delete a keyword +def delete_keyword(keyword: str) -> str: + keyword = keyword.strip().lower() + with db.get_connection() as conn: + cursor = conn.cursor() + try: + cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,)) + keyword_id = cursor.fetchone() + if keyword_id: + cursor.execute('DELETE FROM Keywords WHERE keyword = ?', (keyword,)) + cursor.execute('DELETE FROM keyword_fts WHERE rowid = ?', (keyword_id[0],)) + conn.commit() + return f"Keyword '{keyword}' deleted successfully." + else: + return f"Keyword '{keyword}' not found." + except sqlite3.Error as e: + raise DatabaseError(f"Error deleting keyword: {e}") + + + +# Function to add media with keywords +def add_media_with_keywords(url, title, media_type, content, keywords, prompt, summary, transcription_model, author, + ingestion_date): + # Set default values for missing fields + url = url or 'Unknown' + title = title or 'Untitled' + media_type = media_type or 'Unknown' + content = content or 'No content available' + keywords = keywords or 'default' + prompt = prompt or 'No prompt available' + summary = summary or 'No summary available' + transcription_model = transcription_model or 'Unknown' + author = author or 'Unknown' + ingestion_date = ingestion_date or datetime.now().strftime('%Y-%m-%d') + + # Ensure URL is valid + if not is_valid_url(url): + url = 'localhost' + + if media_type not in ['article', 'audio', 'document', 'obsidian_note', 'podcast', 'text', 'video', 'unknown']: + raise InputError("Invalid media type. Allowed types: article, audio file, document, obsidian_note podcast, text, video, unknown.") + + if ingestion_date and not is_valid_date(ingestion_date): + raise InputError("Invalid ingestion date format. Use YYYY-MM-DD.") + + # Handle keywords as either string or list + if isinstance(keywords, str): + keyword_list = [keyword.strip().lower() for keyword in keywords.split(',')] + elif isinstance(keywords, list): + keyword_list = [keyword.strip().lower() for keyword in keywords] + else: + keyword_list = ['default'] + + logging.info(f"Adding/updating media: URL={url}, Title={title}, Type={media_type}") + logging.debug(f"Content (first 500 chars): {content[:500]}...") + logging.debug(f"Keywords: {keyword_list}") + logging.info(f"Prompt: {prompt}") + logging.info(f"Summary: {summary}") + logging.info(f"Author: {author}") + logging.info(f"Ingestion Date: {ingestion_date}") + logging.info(f"Transcription Model: {transcription_model}") + + try: + with db.get_connection() as conn: + conn.execute("BEGIN TRANSACTION") + cursor = conn.cursor() + + # Check if media already exists + cursor.execute('SELECT id FROM Media WHERE url = ?', (url,)) + existing_media = cursor.fetchone() + + if existing_media: + media_id = existing_media[0] + logging.info(f"Updating existing media with ID: {media_id}") + + cursor.execute(''' + UPDATE Media + SET content = ?, transcription_model = ?, title = ?, type = ?, author = ?, ingestion_date = ? + WHERE id = ? + ''', (content, transcription_model, title, media_type, author, ingestion_date, media_id)) + else: + logging.info("Creating new media entry") + + cursor.execute(''' + INSERT INTO Media (url, title, type, content, author, ingestion_date, transcription_model) + VALUES (?, ?, ?, ?, ?, ?, ?) + ''', (url, title, media_type, content, author, ingestion_date, transcription_model)) + media_id = cursor.lastrowid + + logging.info(f"Adding new modification to MediaModifications for media ID: {media_id}") + cursor.execute(''' + INSERT INTO MediaModifications (media_id, prompt, summary, modification_date) + VALUES (?, ?, ?, ?) + ''', (media_id, prompt, summary, ingestion_date)) + logger.info("New modification added to MediaModifications") + + # Insert keywords and associate with media item + logging.info("Processing keywords") + for keyword in keyword_list: + keyword = keyword.strip().lower() + cursor.execute('INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)', (keyword,)) + cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,)) + keyword_id = cursor.fetchone()[0] + cursor.execute('INSERT OR IGNORE INTO MediaKeywords (media_id, keyword_id) VALUES (?, ?)', + (media_id, keyword_id)) + + # Update full-text search index + logging.info("Updating full-text search index") + cursor.execute('INSERT OR REPLACE INTO media_fts (rowid, title, content) VALUES (?, ?, ?)', + (media_id, title, content)) + + logging.info("Adding new media version") + add_media_version(media_id, prompt, summary) + + conn.commit() + logging.info(f"Media '{title}' successfully added/updated with ID: {media_id}") + + return f"Media '{title}' added/updated successfully with keywords: {', '.join(keyword_list)}" + + except sqlite3.Error as e: + conn.rollback() + logging.error(f"SQL Error: {e}") + raise DatabaseError(f"Error adding media with keywords: {e}") + except Exception as e: + conn.rollback() + logging.error(f"Unexpected Error: {e}") + raise DatabaseError(f"Unexpected error: {e}") + + +def fetch_all_keywords() -> List[str]: + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute('SELECT keyword FROM Keywords') + keywords = [row[0] for row in cursor.fetchall()] + return keywords + except sqlite3.Error as e: + raise DatabaseError(f"Error fetching keywords: {e}") + +def keywords_browser_interface(): + keywords = fetch_all_keywords() + return gr.Markdown("\n".join(f"- {keyword}" for keyword in keywords)) + +def display_keywords(): + try: + keywords = fetch_all_keywords() + return "\n".join(keywords) if keywords else "No keywords found." + except DatabaseError as e: + return str(e) + + +def export_keywords_to_csv(): + try: + keywords = fetch_all_keywords() + if not keywords: + return None, "No keywords found in the database." + + filename = "keywords.csv" + with open(filename, 'w', newline='', encoding='utf-8') as file: + writer = csv.writer(file) + writer.writerow(["Keyword"]) + for keyword in keywords: + writer.writerow([keyword]) + + return filename, f"Keywords exported to {filename}" + except Exception as e: + logger.error(f"Error exporting keywords to CSV: {e}") + return None, f"Error exporting keywords: {e}" + + +# Function to fetch items based on search query and type +def browse_items(search_query, search_type): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + if search_type == 'Title': + cursor.execute("SELECT id, title, url FROM Media WHERE title LIKE ?", (f'%{search_query}%',)) + elif search_type == 'URL': + cursor.execute("SELECT id, title, url FROM Media WHERE url LIKE ?", (f'%{search_query}%',)) + results = cursor.fetchall() + return results + except sqlite3.Error as e: + raise Exception(f"Error fetching items by {search_type}: {e}") + + +# Function to fetch item details +def fetch_item_details(media_id: int): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT prompt, summary + FROM MediaModifications + WHERE media_id = ? + ORDER BY modification_date DESC + LIMIT 1 + """, (media_id,)) + prompt_summary_result = cursor.fetchone() + cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,)) + content_result = cursor.fetchone() + + prompt = prompt_summary_result[0] if prompt_summary_result else "" + summary = prompt_summary_result[1] if prompt_summary_result else "" + content = content_result[0] if content_result else "" + + return content, prompt, summary + except sqlite3.Error as e: + logging.error(f"Error fetching item details: {e}") + return "", "", "" # Return empty strings if there's an error + +# +# +####################################################################################################################### + + + + +# Function to add a version of a prompt and summary +def add_media_version(media_id: int, prompt: str, summary: str) -> None: + try: + with db.get_connection() as conn: + cursor = conn.cursor() + + # Get the current version number + cursor.execute('SELECT MAX(version) FROM MediaVersion WHERE media_id = ?', (media_id,)) + current_version = cursor.fetchone()[0] or 0 + + # Insert the new version + cursor.execute(''' + INSERT INTO MediaVersion (media_id, version, prompt, summary, created_at) + VALUES (?, ?, ?, ?, ?) + ''', (media_id, current_version + 1, prompt, summary, datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) + conn.commit() + except sqlite3.Error as e: + raise DatabaseError(f"Error adding media version: {e}") + + +# Function to search the database with advanced options, including keyword search and full-text search +def search_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 10): + if page < 1: + raise ValueError("Page number must be 1 or greater.") + + # Prepare keywords by splitting and trimming + keywords = [keyword.strip().lower() for keyword in keywords.split(',') if keyword.strip()] + + with db.get_connection() as conn: + cursor = conn.cursor() + offset = (page - 1) * results_per_page + + # Prepare the search conditions for general fields + search_conditions = [] + params = [] + + for field in search_fields: + if search_query: # Ensure there's a search query before adding this condition + search_conditions.append(f"Media.{field} LIKE ?") + params.append(f'%{search_query}%') + + # Prepare the conditions for keywords filtering + keyword_conditions = [] + for keyword in keywords: + keyword_conditions.append( + f"EXISTS (SELECT 1 FROM MediaKeywords mk JOIN Keywords k ON mk.keyword_id = k.id WHERE mk.media_id = Media.id AND k.keyword LIKE ?)") + params.append(f'%{keyword}%') + + # Combine all conditions + where_clause = " AND ".join( + search_conditions + keyword_conditions) if search_conditions or keyword_conditions else "1=1" + + # Complete the query + query = f''' + SELECT DISTINCT Media.id, Media.url, Media.title, Media.type, Media.content, Media.author, Media.ingestion_date, + MediaModifications.prompt, MediaModifications.summary + FROM Media + LEFT JOIN MediaModifications ON Media.id = MediaModifications.media_id + WHERE {where_clause} + ORDER BY Media.ingestion_date DESC + LIMIT ? OFFSET ? + ''' + params.extend([results_per_page, offset]) + + cursor.execute(query, params) + results = cursor.fetchall() + + return results + + +# Gradio function to handle user input and display results with pagination, with better feedback +def search_and_display(search_query, search_fields, keywords, page): + results = search_db(search_query, search_fields, keywords, page) + + if isinstance(results, pd.DataFrame): + # Convert DataFrame to a list of tuples or lists + processed_results = results.values.tolist() # This converts DataFrame rows to lists + elif isinstance(results, list): + # Ensure that each element in the list is itself a list or tuple (not a dictionary) + processed_results = [list(item.values()) if isinstance(item, dict) else item for item in results] + else: + raise TypeError("Unsupported data type for results") + + return processed_results + + +def display_details(index, results): + if index is None or results is None: + return "Please select a result to view details." + + try: + # Ensure the index is an integer and access the row properly + index = int(index) + if isinstance(results, pd.DataFrame): + if index >= len(results): + return "Index out of range. Please select a valid index." + selected_row = results.iloc[index] + else: + # If results is not a DataFrame, but a list (assuming list of dicts) + selected_row = results[index] + except ValueError: + return "Index must be an integer." + except IndexError: + return "Index out of range. Please select a valid index." + + # Build HTML output safely + details_html = f""" +

{selected_row.get('Title', 'No Title')}

+

URL: {selected_row.get('URL', 'No URL')}

+

Type: {selected_row.get('Type', 'No Type')}

+

Author: {selected_row.get('Author', 'No Author')}

+

Ingestion Date: {selected_row.get('Ingestion Date', 'No Date')}

+

Prompt: {selected_row.get('Prompt', 'No Prompt')}

+

Summary: {selected_row.get('Summary', 'No Summary')}

+

Content: {selected_row.get('Content', 'No Content')}

+ """ + return details_html + + +def get_details(index, dataframe): + if index is None or dataframe is None or index >= len(dataframe): + return "Please select a result to view details." + row = dataframe.iloc[index] + details = f""" +

{row['Title']}

+

URL: {row['URL']}

+

Type: {row['Type']}

+

Author: {row['Author']}

+

Ingestion Date: {row['Ingestion Date']}

+

Prompt: {row['Prompt']}

+

Summary: {row['Summary']}

+

Content:

+ 
{row['Content']}
+ """ + return details + + +def format_results(results): + if not results: + return pd.DataFrame(columns=['URL', 'Title', 'Type', 'Content', 'Author', 'Ingestion Date', 'Prompt', 'Summary']) + + df = pd.DataFrame(results, columns=['URL', 'Title', 'Type', 'Content', 'Author', 'Ingestion Date', 'Prompt', 'Summary']) + logging.debug(f"Formatted DataFrame: {df}") + + return df + + +# Function to export search results to CSV or markdown with pagination +def export_to_file(search_query: str, search_fields: List[str], keyword: str, page: int = 1, results_per_file: int = 1000, export_format: str = 'csv'): + try: + results = search_db(search_query, search_fields, keyword, page, results_per_file) + if not results: + return "No results found to export." + + # Create an 'exports' directory if it doesn't exist + if not os.path.exists('exports'): + os.makedirs('exports') + + if export_format == 'csv': + filename = f'exports/search_results_page_{page}.csv' + with open(filename, 'w', newline='', encoding='utf-8') as file: + writer = csv.writer(file) + writer.writerow(['URL', 'Title', 'Type', 'Content', 'Author', 'Ingestion Date', 'Prompt', 'Summary']) + for row in results: + writer.writerow(row) + elif export_format == 'markdown': + filename = f'exports/search_results_page_{page}.md' + with open(filename, 'w', encoding='utf-8') as file: + for item in results: + markdown_content = convert_to_markdown({ + 'title': item[1], + 'url': item[0], + 'type': item[2], + 'content': item[3], + 'author': item[4], + 'ingestion_date': item[5], + 'summary': item[7], + 'keywords': item[8].split(',') if item[8] else [] + }) + file.write(markdown_content) + file.write("\n---\n\n") # Separator between items + else: + return f"Unsupported export format: {export_format}" + + return f"Results exported to {filename}" + except (DatabaseError, InputError) as e: + return str(e) + + +# Helper function to validate URL format +def is_valid_url(url: str) -> bool: + regex = re.compile( + r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4 + r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6 + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + return re.match(regex, url) is not None + + +# Helper function to validate date format +def is_valid_date(date_string: str) -> bool: + try: + datetime.strptime(date_string, '%Y-%m-%d') + return True + except ValueError: + return False + + +# Add ingested media to DB +def add_media_to_database(url, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model, media_type='video'): + try: + # Extract content from segments + if isinstance(segments, list): + content = ' '.join([segment.get('Text', '') for segment in segments if 'Text' in segment]) + elif isinstance(segments, dict): + content = segments.get('text', '') or segments.get('content', '') + else: + content = str(segments) + + logging.debug(f"Extracted content (first 500 chars): {content[:500]}") + + # Set default custom prompt if not provided + if custom_prompt_input is None: + custom_prompt_input = """ + You are a bulleted notes specialist. ```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes. + **Bulleted Note Creation Guidelines** + + **Headings**: + - Based on referenced topics, not categories like quotes or terms + - Surrounded by **bold** formatting + - Not listed as bullet points + - No space between headings and list items underneath + + **Emphasis**: + - **Important terms** set in bold font + - **Text ending in a colon**: also bolded + + **Review**: + - Ensure adherence to specified format + - Do not reference these instructions in your response.[INST] {{ .Prompt }} [/INST]""" + + logging.info(f"Adding media to database: URL={url}, Title={info_dict.get('title', 'Untitled')}, Type={media_type}") + + result = add_media_with_keywords( + url=url, + title=info_dict.get('title', 'Untitled'), + media_type=media_type, + content=content, + keywords=','.join(keywords) if isinstance(keywords, list) else keywords, + prompt=custom_prompt_input or 'No prompt provided', + summary=summary or 'No summary provided', + transcription_model=whisper_model, + author=info_dict.get('uploader', 'Unknown'), + ingestion_date=datetime.now().strftime('%Y-%m-%d') + ) + + logging.info(f"Media added successfully: {result}") + return result + + except Exception as e: + logging.error(f"Error in add_media_to_database: {str(e)}") + raise + + +# +# +####################################################################################################################### + + + + +####################################################################################################################### +# Functions to manage prompts DB +# + +def create_prompts_db(): + conn = sqlite3.connect('prompts.db') + cursor = conn.cursor() + cursor.execute(''' + CREATE TABLE IF NOT EXISTS Prompts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + details TEXT, + system TEXT, + user TEXT + ) + ''') + conn.commit() + conn.close() + +create_prompts_db() + + +def add_prompt(name, details, system, user=None): + try: + conn = sqlite3.connect('prompts.db') + cursor = conn.cursor() + cursor.execute(''' + INSERT INTO Prompts (name, details, system, user) + VALUES (?, ?, ?, ?) + ''', (name, details, system, user)) + conn.commit() + conn.close() + return "Prompt added successfully." + except sqlite3.IntegrityError: + return "Prompt with this name already exists." + except sqlite3.Error as e: + return f"Database error: {e}" + +def fetch_prompt_details(name): + conn = sqlite3.connect('prompts.db') + cursor = conn.cursor() + cursor.execute(''' + SELECT name, details, system, user + FROM Prompts + WHERE name = ? + ''', (name,)) + result = cursor.fetchone() + conn.close() + return result + +def list_prompts(): + conn = sqlite3.connect('prompts.db') + cursor = conn.cursor() + cursor.execute(''' + SELECT name + FROM Prompts + ''') + results = cursor.fetchall() + conn.close() + return [row[0] for row in results] + +def insert_prompt_to_db(title, description, system_prompt, user_prompt): + result = add_prompt(title, description, system_prompt, user_prompt) + return result + + + + +# +# +####################################################################################################################### + + +def update_media_content(selected_item, item_mapping, content_input, prompt_input, summary_input): + try: + if selected_item and item_mapping and selected_item in item_mapping: + media_id = item_mapping[selected_item] + + with db.get_connection() as conn: + cursor = conn.cursor() + + # Update the main content in the Media table + cursor.execute("UPDATE Media SET content = ? WHERE id = ?", (content_input, media_id)) + + # Check if a row already exists in MediaModifications for this media_id + cursor.execute("SELECT COUNT(*) FROM MediaModifications WHERE media_id = ?", (media_id,)) + exists = cursor.fetchone()[0] > 0 + + if exists: + # Update existing row + cursor.execute(""" + UPDATE MediaModifications + SET prompt = ?, summary = ?, modification_date = CURRENT_TIMESTAMP + WHERE media_id = ? + """, (prompt_input, summary_input, media_id)) + else: + # Insert new row + cursor.execute(""" + INSERT INTO MediaModifications (media_id, prompt, summary, modification_date) + VALUES (?, ?, ?, CURRENT_TIMESTAMP) + """, (media_id, prompt_input, summary_input)) + + conn.commit() + + return f"Content updated successfully for media ID: {media_id}" + else: + return "No item selected or invalid selection" + except Exception as e: + logging.error(f"Error updating media content: {e}") + return f"Error updating content: {str(e)}" + +def search_media_database(query: str) -> List[Tuple[int, str, str]]: + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT id, title, url FROM Media WHERE title LIKE ?", (f'%{query}%',)) + results = cursor.fetchall() + return results + except sqlite3.Error as e: + raise Exception(f"Error searching media database: {e}") + +def load_media_content(media_id: int) -> dict: + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT content, prompt, summary FROM Media WHERE id = ?", (media_id,)) + result = cursor.fetchone() + if result: + return { + "content": result[0], + "prompt": result[1], + "summary": result[2] + } + return {"content": "", "prompt": "", "summary": ""} + except sqlite3.Error as e: + raise Exception(f"Error loading media content: {e}") + +def insert_prompt_to_db(title, description, system_prompt, user_prompt): + try: + conn = sqlite3.connect('prompts.db') + cursor = conn.cursor() + cursor.execute( + "INSERT INTO Prompts (name, details, system, user) VALUES (?, ?, ?, ?)", + (title, description, system_prompt, user_prompt) + ) + conn.commit() + conn.close() + return "Prompt added successfully!" + except sqlite3.Error as e: + return f"Error adding prompt: {e}" + + +def fetch_items_by_title_or_url(search_query: str, search_type: str): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + if search_type == 'Title': + cursor.execute("SELECT id, title, url FROM Media WHERE title LIKE ?", (f'%{search_query}%',)) + elif search_type == 'URL': + cursor.execute("SELECT id, title, url FROM Media WHERE url LIKE ?", (f'%{search_query}%',)) + results = cursor.fetchall() + return results + except sqlite3.Error as e: + raise DatabaseError(f"Error fetching items by {search_type}: {e}") + + +def fetch_items_by_keyword(search_query: str): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT m.id, m.title, m.url + FROM Media m + JOIN MediaKeywords mk ON m.id = mk.media_id + JOIN Keywords k ON mk.keyword_id = k.id + WHERE k.keyword LIKE ? + """, (f'%{search_query}%',)) + results = cursor.fetchall() + return results + except sqlite3.Error as e: + raise DatabaseError(f"Error fetching items by keyword: {e}") + + +def fetch_items_by_content(search_query: str): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT id, title, url FROM Media WHERE content LIKE ?", (f'%{search_query}%',)) + results = cursor.fetchall() + return results + except sqlite3.Error as e: + raise DatabaseError(f"Error fetching items by content: {e}") + + +def fetch_item_details_single(media_id: int): + try: + with db.get_connection() as conn: + cursor = conn.cursor() + cursor.execute(""" + SELECT prompt, summary + FROM MediaModifications + WHERE media_id = ? + ORDER BY modification_date DESC + LIMIT 1 + """, (media_id,)) + prompt_summary_result = cursor.fetchone() + cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,)) + content_result = cursor.fetchone() + + prompt = prompt_summary_result[0] if prompt_summary_result else "" + summary = prompt_summary_result[1] if prompt_summary_result else "" + content = content_result[0] if content_result else "" + + return prompt, summary, content + except sqlite3.Error as e: + raise Exception(f"Error fetching item details: {e}") + + + +def convert_to_markdown(item): + markdown = f"# {item['title']}\n\n" + markdown += f"**URL:** {item['url']}\n\n" + markdown += f"**Author:** {item['author']}\n\n" + markdown += f"**Ingestion Date:** {item['ingestion_date']}\n\n" + markdown += f"**Type:** {item['type']}\n\n" + markdown += f"**Keywords:** {', '.join(item['keywords'])}\n\n" + markdown += "## Summary\n\n" + markdown += f"{item['summary']}\n\n" + markdown += "## Content\n\n" + markdown += f"{item['content']}\n\n" + return markdown \ No newline at end of file diff --git a/App_Function_Libraries/Summarization_General_Lib.py b/App_Function_Libraries/Summarization_General_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..37010013975035032da639fe777dc686cdedd99d --- /dev/null +++ b/App_Function_Libraries/Summarization_General_Lib.py @@ -0,0 +1,1388 @@ +# Summarization_General_Lib.py +######################################### +# General Summarization Library +# This library is used to perform summarization. +# +#### +import configparser +#################### +# Function List +# +# 1. extract_text_from_segments(segments: List[Dict]) -> str +# 2. summarize_with_openai(api_key, file_path, custom_prompt_arg) +# 3. summarize_with_anthropic(api_key, file_path, model, custom_prompt_arg, max_retries=3, retry_delay=5) +# 4. summarize_with_cohere(api_key, file_path, model, custom_prompt_arg) +# 5. summarize_with_groq(api_key, file_path, model, custom_prompt_arg) +# +# +#################### +# Import necessary libraries +import os +import logging +import time +import requests +import json +from requests import RequestException + +from App_Function_Libraries.Audio_Transcription_Lib import convert_to_wav, speech_to_text +from App_Function_Libraries.Chunk_Lib import semantic_chunking, rolling_summarize, recursive_summarize_chunks, \ + improved_chunking_process +from App_Function_Libraries.Diarization_Lib import combine_transcription_and_diarization +from App_Function_Libraries.Local_Summarization_Lib import summarize_with_llama, summarize_with_kobold, \ + summarize_with_oobabooga, summarize_with_tabbyapi, summarize_with_vllm, summarize_with_local_llm +from App_Function_Libraries.SQLite_DB import is_valid_url, add_media_to_database +# Import Local +from App_Function_Libraries.Utils import load_and_log_configs, load_comprehensive_config, sanitize_filename, \ + clean_youtube_url, extract_video_info, create_download_directory +from App_Function_Libraries.Video_DL_Ingestion_Lib import download_video + +# +####################################################################################################################### +# Function Definitions +# +config = load_comprehensive_config() +openai_api_key = config.get('API', 'openai_api_key', fallback=None) + +def extract_text_from_segments(segments): + logging.debug(f"Segments received: {segments}") + logging.debug(f"Type of segments: {type(segments)}") + + text = "" + + if isinstance(segments, list): + for segment in segments: + logging.debug(f"Current segment: {segment}") + logging.debug(f"Type of segment: {type(segment)}") + if 'Text' in segment: + text += segment['Text'] + " " + else: + logging.warning(f"Skipping segment due to missing 'Text' key: {segment}") + else: + logging.warning(f"Unexpected type of 'segments': {type(segments)}") + + return text.strip() + + +def summarize_with_openai(api_key, input_data, custom_prompt_arg): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None or api_key.strip() == "": + logging.info("OpenAI: API key not provided as parameter") + logging.info("OpenAI: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['openai'] + + if api_key is None or api_key.strip() == "": + logging.error("OpenAI: API key not found or is empty") + return "OpenAI: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"OpenAI: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + # Input data handling + logging.debug(f"OpenAI: Raw input data type: {type(input_data)}") + logging.debug(f"OpenAI: Raw input data (first 500 chars): {str(input_data)[:500]}...") + + if isinstance(input_data, str): + if input_data.strip().startswith('{'): + # It's likely a JSON string + logging.debug("OpenAI: Parsing provided JSON string data for summarization") + try: + data = json.loads(input_data) + except json.JSONDecodeError as e: + logging.error(f"OpenAI: Error parsing JSON string: {str(e)}") + return f"OpenAI: Error parsing JSON input: {str(e)}" + elif os.path.isfile(input_data): + logging.debug("OpenAI: Loading JSON data from file for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("OpenAI: Using provided string data for summarization") + data = input_data + else: + data = input_data + + logging.debug(f"OpenAI: Processed data type: {type(data)}") + logging.debug(f"OpenAI: Processed data (first 500 chars): {str(data)[:500]}...") + + # Text extraction + if isinstance(data, dict): + if 'summary' in data: + logging.debug("OpenAI: Summary already exists in the loaded data") + return data['summary'] + elif 'segments' in data: + text = extract_text_from_segments(data['segments']) + else: + text = json.dumps(data) # Convert dict to string if no specific format + elif isinstance(data, list): + text = extract_text_from_segments(data) + elif isinstance(data, str): + text = data + else: + raise ValueError(f"OpenAI: Invalid input data format: {type(data)}") + + openai_model = loaded_config_data['models']['openai'] or "gpt-4o" + logging.debug(f"OpenAI: Extracted text (first 500 chars): {text[:500]}...") + logging.debug(f"OpenAI: Custom prompt: {custom_prompt_arg}") + + openai_model = loaded_config_data['models']['openai'] or "gpt-4o" + logging.debug(f"OpenAI: Using model: {openai_model}") + + headers = { + 'Authorization': f'Bearer {openai_api_key}', + 'Content-Type': 'application/json' + } + + logging.debug( + f"OpenAI API Key: {openai_api_key[:5]}...{openai_api_key[-5:] if openai_api_key else None}") + logging.debug("openai: Preparing data + prompt for submittal") + openai_prompt = f"{text} \n\n\n\n{custom_prompt_arg}" + data = { + "model": openai_model, + "messages": [ + {"role": "system", "content": "You are a professional summarizer."}, + {"role": "user", "content": openai_prompt} + ], + "max_tokens": 4096, + "temperature": 0.1 + } + + logging.debug("OpenAI: Posting request") + response = requests.post('https://api.openai.com/v1/chat/completions', headers=headers, json=data) + + if response.status_code == 200: + response_data = response.json() + if 'choices' in response_data and len(response_data['choices']) > 0: + summary = response_data['choices'][0]['message']['content'].strip() + logging.debug("OpenAI: Summarization successful") + logging.debug(f"OpenAI: Summary (first 500 chars): {summary[:500]}...") + return summary + else: + logging.warning("OpenAI: Summary not found in the response data") + return "OpenAI: Summary not available" + else: + logging.error(f"OpenAI: Summarization failed with status code {response.status_code}") + logging.error(f"OpenAI: Error response: {response.text}") + return f"OpenAI: Failed to process summary. Status code: {response.status_code}" + except json.JSONDecodeError as e: + logging.error(f"OpenAI: Error decoding JSON: {str(e)}", exc_info=True) + return f"OpenAI: Error decoding JSON input: {str(e)}" + except requests.RequestException as e: + logging.error(f"OpenAI: Error making API request: {str(e)}", exc_info=True) + return f"OpenAI: Error making API request: {str(e)}" + except Exception as e: + logging.error(f"OpenAI: Unexpected error: {str(e)}", exc_info=True) + return f"OpenAI: Unexpected error occurred: {str(e)}" + + +def summarize_with_anthropic(api_key, input_data, custom_prompt_arg, max_retries=3, retry_delay=5): + try: + loaded_config_data = load_and_log_configs() + # API key validation + if api_key is None or api_key.strip() == "": + logging.info("Anthropic: API key not provided as parameter") + logging.info("Anthropic: Attempting to use API key from config file") + anthropic_api_key = loaded_config_data['api_keys']['anthropic'] + + # Sanity check to ensure API key is not empty in the config file + if api_key is None or api_key.strip() == "": + logging.error("Anthropic: API key not found or is empty") + return "Anthropic: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"Anthropic: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("AnthropicAI: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("AnthropicAI: Using provided string data for summarization") + data = input_data + + logging.debug(f"AnthropicAI: Loaded data: {data}") + logging.debug(f"AnthropicAI: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("Anthropic: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Anthropic: Invalid input data format") + + anthropic_model = loaded_config_data['models']['anthropic'] + + headers = { + 'x-api-key': anthropic_api_key, + 'anthropic-version': '2023-06-01', + 'Content-Type': 'application/json' + } + + anthropic_prompt = custom_prompt_arg + logging.debug(f"Anthropic: Prompt is {anthropic_prompt}") + user_message = { + "role": "user", + "content": f"{text} \n\n\n\n{anthropic_prompt}" + } + + model = loaded_config_data['models']['anthropic'] + + data = { + "model": model, + "max_tokens": 4096, # max _possible_ tokens to return + "messages": [user_message], + "stop_sequences": ["\n\nHuman:"], + "temperature": 0.1, + "top_k": 0, + "top_p": 1.0, + "metadata": { + "user_id": "example_user_id", + }, + "stream": False, + "system": "You are a professional summarizer." + } + + for attempt in range(max_retries): + try: + logging.debug("anthropic: Posting request to API") + response = requests.post('https://api.anthropic.com/v1/messages', headers=headers, json=data) + + # Check if the status code indicates success + if response.status_code == 200: + logging.debug("anthropic: Post submittal successful") + response_data = response.json() + try: + summary = response_data['content'][0]['text'].strip() + logging.debug("anthropic: Summarization successful") + print("Summary processed successfully.") + return summary + except (IndexError, KeyError) as e: + logging.debug("anthropic: Unexpected data in response") + print("Unexpected response format from Anthropic API:", response.text) + return None + elif response.status_code == 500: # Handle internal server error specifically + logging.debug("anthropic: Internal server error") + print("Internal server error from API. Retrying may be necessary.") + time.sleep(retry_delay) + else: + logging.debug( + f"anthropic: Failed to summarize, status code {response.status_code}: {response.text}") + print(f"Failed to process summary, status code {response.status_code}: {response.text}") + return None + + except RequestException as e: + logging.error(f"anthropic: Network error during attempt {attempt + 1}/{max_retries}: {str(e)}") + if attempt < max_retries - 1: + time.sleep(retry_delay) + else: + return f"anthropic: Network error: {str(e)}" + except FileNotFoundError as e: + logging.error(f"anthropic: File not found: {input_data}") + return f"anthropic: File not found: {input_data}" + except json.JSONDecodeError as e: + logging.error(f"anthropic: Invalid JSON format in file: {input_data}") + return f"anthropic: Invalid JSON format in file: {input_data}" + except Exception as e: + logging.error(f"anthropic: Error in processing: {str(e)}") + return f"anthropic: Error occurred while processing summary with Anthropic: {str(e)}" + + +# Summarize with Cohere +def summarize_with_cohere(api_key, input_data, custom_prompt_arg): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None or api_key.strip() == "": + logging.info("Cohere: API key not provided as parameter") + logging.info("Cohere: Attempting to use API key from config file") + cohere_api_key = loaded_config_data['api_keys']['cohere'] + + if api_key is None or api_key.strip() == "": + logging.error("Cohere: API key not found or is empty") + return "Cohere: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"Cohere: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("Cohere: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("Cohere: Using provided string data for summarization") + data = input_data + + logging.debug(f"Cohere: Loaded data: {data}") + logging.debug(f"Cohere: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("Cohere: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Invalid input data format") + + cohere_model = loaded_config_data['models']['cohere'] + + headers = { + 'accept': 'application/json', + 'content-type': 'application/json', + 'Authorization': f'Bearer {cohere_api_key}' + } + + cohere_prompt = f"{text} \n\n\n\n{custom_prompt_arg}" + logging.debug("cohere: Prompt being sent is {cohere_prompt}") + + model = loaded_config_data['models']['anthropic'] + + data = { + "chat_history": [ + {"role": "USER", "message": cohere_prompt} + ], + "message": "Please provide a summary.", + "model": model, + "connectors": [{"id": "web-search"}] + } + + logging.debug("cohere: Submitting request to API endpoint") + print("cohere: Submitting request to API endpoint") + response = requests.post('https://api.cohere.ai/v1/chat', headers=headers, json=data) + response_data = response.json() + logging.debug("API Response Data: %s", response_data) + + if response.status_code == 200: + if 'text' in response_data: + summary = response_data['text'].strip() + logging.debug("cohere: Summarization successful") + print("Summary processed successfully.") + return summary + else: + logging.error("Expected data not found in API response.") + return "Expected data not found in API response." + else: + logging.error(f"cohere: API request failed with status code {response.status_code}: {response.text}") + print(f"Failed to process summary, status code {response.status_code}: {response.text}") + return f"cohere: API request failed: {response.text}" + + except Exception as e: + logging.error("cohere: Error in processing: %s", str(e)) + return f"cohere: Error occurred while processing summary with Cohere: {str(e)}" + + +# https://console.groq.com/docs/quickstart +def summarize_with_groq(api_key, input_data, custom_prompt_arg): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None or api_key.strip() == "": + logging.info("Groq: API key not provided as parameter") + logging.info("Groq: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['groq'] + + if api_key is None or api_key.strip() == "": + logging.error("Groq: API key not found or is empty") + return "Groq: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"Groq: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + # Transcript data handling & Validation + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("Groq: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("Groq: Using provided string data for summarization") + data = input_data + + logging.debug(f"Groq: Loaded data: {data}") + logging.debug(f"Groq: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("Groq: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Groq: Invalid input data format") + + # Set the model to be used + groq_model = loaded_config_data['models']['groq'] + + headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json' + } + + groq_prompt = f"{text} \n\n\n\n{custom_prompt_arg}" + logging.debug("groq: Prompt being sent is {groq_prompt}") + + data = { + "messages": [ + { + "role": "user", + "content": groq_prompt + } + ], + "model": groq_model + } + + logging.debug("groq: Submitting request to API endpoint") + print("groq: Submitting request to API endpoint") + response = requests.post('https://api.groq.com/openai/v1/chat/completions', headers=headers, json=data) + + response_data = response.json() + logging.debug("API Response Data: %s", response_data) + + if response.status_code == 200: + if 'choices' in response_data and len(response_data['choices']) > 0: + summary = response_data['choices'][0]['message']['content'].strip() + logging.debug("groq: Summarization successful") + print("Summarization successful.") + return summary + else: + logging.error("Expected data not found in API response.") + return "Expected data not found in API response." + else: + logging.error(f"groq: API request failed with status code {response.status_code}: {response.text}") + return f"groq: API request failed: {response.text}" + + except Exception as e: + logging.error("groq: Error in processing: %s", str(e)) + return f"groq: Error occurred while processing summary with groq: {str(e)}" + + +def summarize_with_openrouter(api_key, input_data, custom_prompt_arg): + loaded_config_data = load_and_log_configs() + import requests + import json + global openrouter_model, openrouter_api_key + # API key validation + if api_key is None or api_key.strip() == "": + logging.info("OpenRouter: API key not provided as parameter") + logging.info("OpenRouter: Attempting to use API key from config file") + openrouter_api_key = loaded_config_data['api_keys']['openrouter'] + + if api_key is None or api_key.strip() == "": + logging.error("OpenRouter: API key not found or is empty") + return "OpenRouter: API Key Not Provided/Found in Config file or is empty" + + # Model Selection validation + if openrouter_model is None or openrouter_model.strip() == "": + logging.info("OpenRouter: model not provided as parameter") + logging.info("OpenRouter: Attempting to use model from config file") + openrouter_model = loaded_config_data['api_keys']['openrouter_model'] + + if api_key is None or api_key.strip() == "": + logging.error("OpenAI: API key not found or is empty") + return "OpenAI: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"OpenAI: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + logging.debug(f"openai: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("openrouter: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("openrouter: Using provided string data for summarization") + data = input_data + + logging.debug(f"openrouter: Loaded data: {data}") + logging.debug(f"openrouter: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("openrouter: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("Invalid input data format") + + config = configparser.ConfigParser() + file_path = 'config.txt' + + # Check if the file exists in the specified path + if os.path.exists(file_path): + config.read(file_path) + elif os.path.exists('config.txt'): # Check in the current directory + config.read('../config.txt') + else: + print("config.txt not found in the specified path or current directory.") + + openrouter_prompt = f"{input_data} \n\n\n\n{custom_prompt_arg}" + + try: + logging.debug("openrouter: Submitting request to API endpoint") + print("openrouter: Submitting request to API endpoint") + response = requests.post( + url="https://openrouter.ai/api/v1/chat/completions", + headers={ + "Authorization": f"Bearer {openrouter_api_key}", + }, + data=json.dumps({ + "model": f"{openrouter_model}", + "messages": [ + {"role": "user", "content": openrouter_prompt} + ] + }) + ) + + response_data = response.json() + logging.debug("API Response Data: %s", response_data) + + if response.status_code == 200: + if 'choices' in response_data and len(response_data['choices']) > 0: + summary = response_data['choices'][0]['message']['content'].strip() + logging.debug("openrouter: Summarization successful") + print("openrouter: Summarization successful.") + return summary + else: + logging.error("openrouter: Expected data not found in API response.") + return "openrouter: Expected data not found in API response." + else: + logging.error(f"openrouter: API request failed with status code {response.status_code}: {response.text}") + return f"openrouter: API request failed: {response.text}" + except Exception as e: + logging.error("openrouter: Error in processing: %s", str(e)) + return f"openrouter: Error occurred while processing summary with openrouter: {str(e)}" + +def summarize_with_huggingface(api_key, input_data, custom_prompt_arg): + loaded_config_data = load_and_log_configs() + global huggingface_api_key + logging.debug(f"huggingface: Summarization process starting...") + try: + # API key validation + if api_key is None or api_key.strip() == "": + logging.info("HuggingFace: API key not provided as parameter") + logging.info("HuggingFace: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['huggingface'] + + if api_key is None or api_key.strip() == "": + logging.error("HuggingFace: API key not found or is empty") + return "HuggingFace: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"HuggingFace: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("HuggingFace: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("HuggingFace: Using provided string data for summarization") + data = input_data + + logging.debug(f"HuggingFace: Loaded data: {data}") + logging.debug(f"HuggingFace: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("HuggingFace: Summary already exists in the loaded data") + return data['summary'] + + # If the loaded data is a list of segment dictionaries or a string, proceed with summarization + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("HuggingFace: Invalid input data format") + + print(f"HuggingFace: lets make sure the HF api key exists...\n\t {api_key}") + headers = { + "Authorization": f"Bearer {api_key}" + } + + huggingface_model = loaded_config_data['models']['huggingface'] + API_URL = f"https://api-inference.huggingface.co/models/{huggingface_model}" + + huggingface_prompt = f"{text}\n\n\n\n{custom_prompt_arg}" + logging.debug("huggingface: Prompt being sent is {huggingface_prompt}") + data = { + "inputs": text, + "parameters": {"max_length": 512, "min_length": 100} # You can adjust max_length and min_length as needed + } + + print(f"huggingface: lets make sure the HF api key is the same..\n\t {huggingface_api_key}") + + logging.debug("huggingface: Submitting request...") + + response = requests.post(API_URL, headers=headers, json=data) + + if response.status_code == 200: + summary = response.json()[0]['summary_text'] + logging.debug("huggingface: Summarization successful") + print("Summarization successful.") + return summary + else: + logging.error(f"huggingface: Summarization failed with status code {response.status_code}: {response.text}") + return f"Failed to process summary, status code {response.status_code}: {response.text}" + except Exception as e: + logging.error("huggingface: Error in processing: %s", str(e)) + print(f"Error occurred while processing summary with huggingface: {str(e)}") + return None + + +def summarize_with_deepseek(api_key, input_data, custom_prompt_arg): + loaded_config_data = load_and_log_configs() + try: + # API key validation + if api_key is None or api_key.strip() == "": + logging.info("DeepSeek: API key not provided as parameter") + logging.info("DeepSeek: Attempting to use API key from config file") + api_key = loaded_config_data['api_keys']['deepseek'] + + if api_key is None or api_key.strip() == "": + logging.error("DeepSeek: API key not found or is empty") + return "DeepSeek: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"DeepSeek: Using API Key: {api_key[:5]}...{api_key[-5:]}") + + # Input data handling + if isinstance(input_data, str) and os.path.isfile(input_data): + logging.debug("DeepSeek: Loading json data for summarization") + with open(input_data, 'r') as file: + data = json.load(file) + else: + logging.debug("DeepSeek: Using provided string data for summarization") + data = input_data + + logging.debug(f"DeepSeek: Loaded data: {data}") + logging.debug(f"DeepSeek: Type of data: {type(data)}") + + if isinstance(data, dict) and 'summary' in data: + # If the loaded data is a dictionary and already contains a summary, return it + logging.debug("DeepSeek: Summary already exists in the loaded data") + return data['summary'] + + # Text extraction + if isinstance(data, list): + segments = data + text = extract_text_from_segments(segments) + elif isinstance(data, str): + text = data + else: + raise ValueError("DeepSeek: Invalid input data format") + + deepseek_model = loaded_config_data['models']['deepseek'] or "deepseek-chat" + + headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json' + } + + logging.debug( + f"Deepseek API Key: {api_key[:5]}...{api_key[-5:] if api_key else None}") + logging.debug("openai: Preparing data + prompt for submittal") + deepseek_prompt = f"{text} \n\n\n\n{custom_prompt_arg}" + data = { + "model": deepseek_model, + "messages": [ + {"role": "system", "content": "You are a professional summarizer."}, + {"role": "user", "content": deepseek_prompt} + ], + "stream": False, + "temperature": 0.8 + } + + logging.debug("DeepSeek: Posting request") + response = requests.post('https://api.deepseek.com/chat/completions', headers=headers, json=data) + + if response.status_code == 200: + response_data = response.json() + if 'choices' in response_data and len(response_data['choices']) > 0: + summary = response_data['choices'][0]['message']['content'].strip() + logging.debug("DeepSeek: Summarization successful") + return summary + else: + logging.warning("DeepSeek: Summary not found in the response data") + return "DeepSeek: Summary not available" + else: + logging.error(f"DeepSeek: Summarization failed with status code {response.status_code}") + logging.error(f"DeepSeek: Error response: {response.text}") + return f"DeepSeek: Failed to process summary. Status code: {response.status_code}" + except Exception as e: + logging.error(f"DeepSeek: Error in processing: {str(e)}", exc_info=True) + return f"DeepSeek: Error occurred while processing summary: {str(e)}" + + +# +# +####################################################################################################################### +# +# +# Gradio File Processing + + +# Handle multiple videos as input +def process_video_urls(url_list, num_speakers, whisper_model, custom_prompt_input, offset, api_name, api_key, vad_filter, + download_video_flag, download_audio, rolling_summarization, detail_level, question_box, + keywords, chunk_text_by_words, max_words, chunk_text_by_sentences, max_sentences, + chunk_text_by_paragraphs, max_paragraphs, chunk_text_by_tokens, max_tokens, chunk_by_semantic, + semantic_chunk_size, semantic_chunk_overlap, recursive_summarization): + global current_progress + progress = [] # This must always be a list + status = [] # This must always be a list + + if custom_prompt_input is None: + custom_prompt_input = """ + You are a bulleted notes specialist. ```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes. + **Bulleted Note Creation Guidelines** + + **Headings**: + - Based on referenced topics, not categories like quotes or terms + - Surrounded by **bold** formatting + - Not listed as bullet points + - No space between headings and list items underneath + + **Emphasis**: + - **Important terms** set in bold font + - **Text ending in a colon**: also bolded + + **Review**: + - Ensure adherence to specified format + - Do not reference these instructions in your response.[INST] {{ .Prompt }} [/INST]""" + + def update_progress(index, url, message): + progress.append(f"Processing {index + 1}/{len(url_list)}: {url}") # Append to list + status.append(message) # Append to list + return "\n".join(progress), "\n".join(status) # Return strings for display + + + for index, url in enumerate(url_list): + try: + transcription, summary, json_file_path, summary_file_path, _, _ = process_url( + url=url, + num_speakers=num_speakers, + whisper_model=whisper_model, + custom_prompt_input=custom_prompt_input, + offset=offset, + api_name=api_name, + api_key=api_key, + vad_filter=vad_filter, + download_video_flag=download_video_flag, + download_audio=download_audio, + rolling_summarization=rolling_summarization, + detail_level=detail_level, + question_box=question_box, + keywords=keywords, + chunk_text_by_words=chunk_text_by_words, + max_words=max_words, + chunk_text_by_sentences=chunk_text_by_sentences, + max_sentences=max_sentences, + chunk_text_by_paragraphs=chunk_text_by_paragraphs, + max_paragraphs=max_paragraphs, + chunk_text_by_tokens=chunk_text_by_tokens, + max_tokens=max_tokens, + chunk_by_semantic=chunk_by_semantic, + semantic_chunk_size=semantic_chunk_size, + semantic_chunk_overlap=semantic_chunk_overlap, + recursive_summarization=recursive_summarization + ) + # Update progress and transcription properly + current_progress, current_status = update_progress(index, url, "Video processed and ingested into the database.") + except Exception as e: + current_progress, current_status = update_progress(index, url, f"Error: {str(e)}") + + success_message = "All videos have been transcribed, summarized, and ingested into the database successfully." + return current_progress, success_message, None, None, None, None + + +# stuff +def perform_transcription(video_path, offset, whisper_model, vad_filter, diarize=False): + global segments_json_path + audio_file_path = convert_to_wav(video_path, offset) + segments_json_path = audio_file_path.replace('.wav', '.segments.json') + + if diarize: + diarized_json_path = audio_file_path.replace('.wav', '.diarized.json') + + # Check if diarized JSON already exists + if os.path.exists(diarized_json_path): + logging.info(f"Diarized file already exists: {diarized_json_path}") + try: + with open(diarized_json_path, 'r') as file: + diarized_segments = json.load(file) + if not diarized_segments: + logging.warning(f"Diarized JSON file is empty, re-generating: {diarized_json_path}") + raise ValueError("Empty diarized JSON file") + logging.debug(f"Loaded diarized segments from {diarized_json_path}") + return audio_file_path, diarized_segments + except (json.JSONDecodeError, ValueError) as e: + logging.error(f"Failed to read or parse the diarized JSON file: {e}") + os.remove(diarized_json_path) + + # If diarized file doesn't exist or was corrupted, generate new diarized transcription + logging.info(f"Generating diarized transcription for {audio_file_path}") + diarized_segments = combine_transcription_and_diarization(audio_file_path) + + # Save diarized segments + with open(diarized_json_path, 'w') as file: + json.dump(diarized_segments, file, indent=2) + + return audio_file_path, diarized_segments + + # Non-diarized transcription (existing functionality) + if os.path.exists(segments_json_path): + logging.info(f"Segments file already exists: {segments_json_path}") + try: + with open(segments_json_path, 'r') as file: + segments = json.load(file) + if not segments: + logging.warning(f"Segments JSON file is empty, re-generating: {segments_json_path}") + raise ValueError("Empty segments JSON file") + logging.debug(f"Loaded segments from {segments_json_path}") + except (json.JSONDecodeError, ValueError) as e: + logging.error(f"Failed to read or parse the segments JSON file: {e}") + os.remove(segments_json_path) + logging.info(f"Re-generating transcription for {audio_file_path}") + audio_file, segments = re_generate_transcription(audio_file_path, whisper_model, vad_filter) + if segments is None: + return None, None + else: + audio_file, segments = re_generate_transcription(audio_file_path, whisper_model, vad_filter) + + return audio_file_path, segments + + +def re_generate_transcription(audio_file_path, whisper_model, vad_filter): + try: + segments = speech_to_text(audio_file_path, whisper_model=whisper_model, vad_filter=vad_filter) + # Save segments to JSON + with open(segments_json_path, 'w') as file: + json.dump(segments, file, indent=2) + logging.debug(f"Transcription segments saved to {segments_json_path}") + return audio_file_path, segments + except Exception as e: + logging.error(f"Error in re-generating transcription: {str(e)}") + return None, None + + +def save_transcription_and_summary(transcription_text, summary_text, download_path, info_dict): + try: + video_title = sanitize_filename(info_dict.get('title', 'Untitled')) + + # Save transcription + transcription_file_path = os.path.join(download_path, f"{video_title}_transcription.txt") + with open(transcription_file_path, 'w', encoding='utf-8') as f: + f.write(transcription_text) + + # Save summary if available + summary_file_path = None + if summary_text: + summary_file_path = os.path.join(download_path, f"{video_title}_summary.txt") + with open(summary_file_path, 'w', encoding='utf-8') as f: + f.write(summary_text) + + return transcription_file_path, summary_file_path + except Exception as e: + logging.error(f"Error in save_transcription_and_summary: {str(e)}", exc_info=True) + return None, None + + +def summarize_chunk(api_name, text, custom_prompt_input, api_key): + try: + if api_name.lower() == 'openai': + return summarize_with_openai(api_key, text, custom_prompt_input) + elif api_name.lower() == "anthropic": + return summarize_with_anthropic(api_key, text, custom_prompt_input) + elif api_name.lower() == "cohere": + return summarize_with_cohere(api_key, text, custom_prompt_input) + elif api_name.lower() == "groq": + return summarize_with_groq(api_key, text, custom_prompt_input) + elif api_name.lower() == "openrouter": + return summarize_with_openrouter(api_key, text, custom_prompt_input) + elif api_name.lower() == "deepseek": + return summarize_with_deepseek(api_key, text, custom_prompt_input) + elif api_name.lower() == "llama.cpp": + return summarize_with_llama(text, custom_prompt_input) + elif api_name.lower() == "kobold": + return summarize_with_kobold(text, api_key, custom_prompt_input) + elif api_name.lower() == "ooba": + return summarize_with_oobabooga(text, api_key, custom_prompt_input) + elif api_name.lower() == "tabbyapi": + return summarize_with_tabbyapi(text, custom_prompt_input) + elif api_name.lower() == "vllm": + return summarize_with_vllm(text, custom_prompt_input) + elif api_name.lower() == "local-llm": + return summarize_with_local_llm(text, custom_prompt_input) + elif api_name.lower() == "huggingface": + return summarize_with_huggingface(api_key, text, custom_prompt_input) + else: + logging.warning(f"Unsupported API: {api_name}") + return None + except Exception as e: + logging.error(f"Error in summarize_chunk with {api_name}: {str(e)}") + return None + + +def extract_metadata_and_content(input_data): + metadata = {} + content = "" + + if isinstance(input_data, str): + if os.path.exists(input_data): + with open(input_data, 'r', encoding='utf-8') as file: + data = json.load(file) + else: + try: + data = json.loads(input_data) + except json.JSONDecodeError: + return {}, input_data + elif isinstance(input_data, dict): + data = input_data + else: + return {}, str(input_data) + + # Extract metadata + metadata['title'] = data.get('title', 'No title available') + metadata['author'] = data.get('author', 'Unknown author') + + # Extract content + if 'transcription' in data: + content = extract_text_from_segments(data['transcription']) + elif 'segments' in data: + content = extract_text_from_segments(data['segments']) + elif 'content' in data: + content = data['content'] + else: + content = json.dumps(data) + + return metadata, content + +def extract_text_from_segments(segments): + if isinstance(segments, list): + return ' '.join([seg.get('Text', '') for seg in segments if 'Text' in seg]) + return str(segments) + +def format_input_with_metadata(metadata, content): + formatted_input = f"Title: {metadata.get('title', 'No title available')}\n" + formatted_input += f"Author: {metadata.get('author', 'Unknown author')}\n\n" + formatted_input += content + return formatted_input + +def perform_summarization(api_name, input_data, custom_prompt_input, api_key, recursive_summarization=False): + loaded_config_data = load_and_log_configs() + + if custom_prompt_input is None: + custom_prompt_input = """ + You are a bulleted notes specialist. ```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes. +**Bulleted Note Creation Guidelines** + +**Headings**: +- Based on referenced topics, not categories like quotes or terms +- Surrounded by **bold** formatting +- Not listed as bullet points +- No space between headings and list items underneath + +**Emphasis**: +- **Important terms** set in bold font +- **Text ending in a colon**: also bolded + +**Review**: +- Ensure adherence to specified format +- Do not reference these instructions in your response.[INST] {{ .Prompt }} [/INST]""" + + try: + logging.debug(f"Input data type: {type(input_data)}") + logging.debug(f"Input data (first 500 chars): {str(input_data)[:500]}...") + + # Extract metadata and content + metadata, content = extract_metadata_and_content(input_data) + + logging.debug(f"Extracted metadata: {metadata}") + logging.debug(f"Extracted content (first 500 chars): {content[:500]}...") + + # Prepare a structured input for summarization + structured_input = format_input_with_metadata(metadata, content) + + # Perform summarization on the structured input + if recursive_summarization: + chunk_options = { + 'method': 'words', # or 'sentences', 'paragraphs', 'tokens' based on your preference + 'max_size': 1000, # adjust as needed + 'overlap': 100, # adjust as needed + 'adaptive': False, + 'multi_level': False, + 'language': 'english' + } + chunks = improved_chunking_process(structured_input, chunk_options) + summary = recursive_summarize_chunks([chunk['text'] for chunk in chunks], + lambda x: summarize_chunk(api_name, x, custom_prompt_input, api_key), + custom_prompt_input) + else: + summary = summarize_chunk(api_name, structured_input, custom_prompt_input, api_key) + + if summary: + logging.info(f"Summary generated using {api_name} API") + if isinstance(input_data, str) and os.path.exists(input_data): + summary_file_path = input_data.replace('.json', '_summary.txt') + with open(summary_file_path, 'w', encoding='utf-8') as file: + file.write(summary) + else: + logging.warning(f"Failed to generate summary using {api_name} API") + + return summary + + except requests.exceptions.ConnectionError: + logging.error("Connection error while summarizing") + except Exception as e: + logging.error(f"Error summarizing with {api_name}: {str(e)}", exc_info=True) + return f"An error occurred during summarization: {str(e)}" + return None + +def extract_text_from_input(input_data): + if isinstance(input_data, str): + try: + # Try to parse as JSON + data = json.loads(input_data) + except json.JSONDecodeError: + # If not valid JSON, treat as plain text + return input_data + elif isinstance(input_data, dict): + data = input_data + else: + return str(input_data) + + # Extract relevant fields from the JSON object + text_parts = [] + if 'title' in data: + text_parts.append(f"Title: {data['title']}") + if 'description' in data: + text_parts.append(f"Description: {data['description']}") + if 'transcription' in data: + if isinstance(data['transcription'], list): + transcription_text = ' '.join([segment.get('Text', '') for segment in data['transcription']]) + elif isinstance(data['transcription'], str): + transcription_text = data['transcription'] + else: + transcription_text = str(data['transcription']) + text_parts.append(f"Transcription: {transcription_text}") + elif 'segments' in data: + segments_text = extract_text_from_segments(data['segments']) + text_parts.append(f"Segments: {segments_text}") + + return '\n\n'.join(text_parts) + + + +def process_url( + url, + num_speakers, + whisper_model, + custom_prompt_input, + offset, + api_name, + api_key, + vad_filter, + download_video_flag, + download_audio, + rolling_summarization, + detail_level, + # It's for the asking a question about a returned prompt - needs to be removed #FIXME + question_box, + keywords, + chunk_text_by_words, + max_words, + chunk_text_by_sentences, + max_sentences, + chunk_text_by_paragraphs, + max_paragraphs, + chunk_text_by_tokens, + max_tokens, + chunk_by_semantic, + semantic_chunk_size, + semantic_chunk_overlap, + local_file_path=None, + diarize=False, + recursive_summarization=False +): + # Handle the chunk summarization options + set_chunk_txt_by_words = chunk_text_by_words + set_max_txt_chunk_words = max_words + set_chunk_txt_by_sentences = chunk_text_by_sentences + set_max_txt_chunk_sentences = max_sentences + set_chunk_txt_by_paragraphs = chunk_text_by_paragraphs + set_max_txt_chunk_paragraphs = max_paragraphs + set_chunk_txt_by_tokens = chunk_text_by_tokens + set_max_txt_chunk_tokens = max_tokens + set_chunk_txt_by_semantic = chunk_by_semantic + set_semantic_chunk_size = semantic_chunk_size + set_semantic_chunk_overlap = semantic_chunk_overlap + + progress = [] + success_message = "All videos processed successfully. Transcriptions and summaries have been ingested into the database." + + if custom_prompt_input is None: + custom_prompt_input = """ + You are a bulleted notes specialist. ```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes. + **Bulleted Note Creation Guidelines** + + **Headings**: + - Based on referenced topics, not categories like quotes or terms + - Surrounded by **bold** formatting + - Not listed as bullet points + - No space between headings and list items underneath + + **Emphasis**: + - **Important terms** set in bold font + - **Text ending in a colon**: also bolded + + **Review**: + - Ensure adherence to specified format + - Do not reference these instructions in your response.[INST] {{ .Prompt }} [/INST]""" + + # Validate input + if not url and not local_file_path: + return "Process_URL: No URL provided.", "No URL provided.", None, None, None, None, None, None + + # FIXME - Chatgpt again? + if isinstance(url, str): + urls = url.strip().split('\n') + if len(urls) > 1: + return process_video_urls(urls, num_speakers, whisper_model, custom_prompt_input, offset, api_name, api_key, vad_filter, + download_video_flag, download_audio, rolling_summarization, detail_level, question_box, + keywords, chunk_text_by_words, max_words, chunk_text_by_sentences, max_sentences, + chunk_text_by_paragraphs, max_paragraphs, chunk_text_by_tokens, max_tokens, chunk_by_semantic, semantic_chunk_size, semantic_chunk_overlap) + else: + urls = [url] + + if url and not is_valid_url(url): + return "Process_URL: Invalid URL format.", "Invalid URL format.", None, None, None, None, None, None + + if url: + # Clean the URL to remove playlist parameters if any + url = clean_youtube_url(url) + logging.info(f"Process_URL: Processing URL: {url}") + + if api_name: + print("Process_URL: API Name received:", api_name) # Debugging line + + video_file_path = None + global info_dict + + # FIXME - need to handle local audio file processing + # If Local audio file is provided + if local_file_path: + try: + pass + # # insert code to process local audio file + # # Need to be able to add a title/author/etc for ingestion into the database + # # Also want to be able to optionally _just_ ingest it, and not ingest. + # # FIXME + # #download_path = create_download_directory(title) + # #audio_path = download_video(url, download_path, info_dict, download_video_flag) + # + # audio_file_path = local_file_path + # global segments + # audio_file_path, segments = perform_transcription(audio_file_path, offset, whisper_model, vad_filter) + # + # if audio_file_path is None or segments is None: + # logging.error("Process_URL: Transcription failed or segments not available.") + # return "Process_URL: Transcription failed.", "Transcription failed.", None, None, None, None + # + # logging.debug(f"Process_URL: Transcription audio_file: {audio_file_path}") + # logging.debug(f"Process_URL: Transcription segments: {segments}") + # + # transcription_text = {'audio_file': audio_file_path, 'transcription': segments} + # logging.debug(f"Process_URL: Transcription text: {transcription_text}") + + # Rolling Summarization Processing + # if rolling_summarization: + # text = extract_text_from_segments(segments) + # summary_text = rolling_summarize_function( + # transcription_text, + # detail=detail_level, + # api_name=api_name, + # api_key=api_key, + # custom_prompt=custom_prompt, + # chunk_by_words=chunk_text_by_words, + # max_words=max_words, + # chunk_by_sentences=chunk_text_by_sentences, + # max_sentences=max_sentences, + # chunk_by_paragraphs=chunk_text_by_paragraphs, + # max_paragraphs=max_paragraphs, + # chunk_by_tokens=chunk_text_by_tokens, + # max_tokens=max_tokens + # ) + # if api_name: + # summary_text = perform_summarization(api_name, segments_json_path, custom_prompt, api_key, config) + # if summary_text is None: + # logging.error("Summary text is None. Check summarization function.") + # summary_file_path = None # Set summary_file_path to None if summary is not generated + # else: + # summary_text = 'Summary not available' + # summary_file_path = None # Set summary_file_path to None if summary is not generated + # + # json_file_path, summary_file_path = save_transcription_and_summary(transcription_text, summary_text, download_path) + # + # add_media_to_database(url, info_dict, segments, summary_text, keywords, custom_prompt, whisper_model) + # + # return transcription_text, summary_text, json_file_path, summary_file_path, None, None + + except Exception as e: + logging.error(f": {e}") + return str(e), 'process_url: Error processing the request.', None, None, None, None + + + # If URL/Local video file is provided + try: + info_dict, title = extract_video_info(url) + download_path = create_download_directory(title) + video_path = download_video(url, download_path, info_dict, download_video_flag) + global segments + audio_file_path, segments = perform_transcription(video_path, offset, whisper_model, vad_filter) + + if diarize: + transcription_text = combine_transcription_and_diarization(audio_file_path) + else: + audio_file, segments = perform_transcription(video_path, offset, whisper_model, vad_filter) + transcription_text = {'audio_file': audio_file, 'transcription': segments} + + + if audio_file_path is None or segments is None: + logging.error("Process_URL: Transcription failed or segments not available.") + return "Process_URL: Transcription failed.", "Transcription failed.", None, None, None, None + + logging.debug(f"Process_URL: Transcription audio_file: {audio_file_path}") + logging.debug(f"Process_URL: Transcription segments: {segments}") + + logging.debug(f"Process_URL: Transcription text: {transcription_text}") + + # FIXME - Implement chunking calls here + # Implement chunking calls here + chunked_transcriptions = [] + if chunk_text_by_words: + chunked_transcriptions = chunk_text_by_words(transcription_text['transcription'], max_words) + elif chunk_text_by_sentences: + chunked_transcriptions = chunk_text_by_sentences(transcription_text['transcription'], max_sentences) + elif chunk_text_by_paragraphs: + chunked_transcriptions = chunk_text_by_paragraphs(transcription_text['transcription'], max_paragraphs) + elif chunk_text_by_tokens: + chunked_transcriptions = chunk_text_by_tokens(transcription_text['transcription'], max_tokens) + elif chunk_by_semantic: + chunked_transcriptions = semantic_chunking(transcription_text['transcription'], semantic_chunk_size, 'tokens') + + # If we did chunking, we now have the chunked transcripts in 'chunked_transcriptions' + elif rolling_summarization: + # FIXME - rolling summarization + # text = extract_text_from_segments(segments) + # summary_text = rolling_summarize_function( + # transcription_text, + # detail=detail_level, + # api_name=api_name, + # api_key=api_key, + # custom_prompt_input=custom_prompt_input, + # chunk_by_words=chunk_text_by_words, + # max_words=max_words, + # chunk_by_sentences=chunk_text_by_sentences, + # max_sentences=max_sentences, + # chunk_by_paragraphs=chunk_text_by_paragraphs, + # max_paragraphs=max_paragraphs, + # chunk_by_tokens=chunk_text_by_tokens, + # max_tokens=max_tokens + # ) + pass + else: + pass + + summarized_chunk_transcriptions = [] + + if chunk_text_by_words or chunk_text_by_sentences or chunk_text_by_paragraphs or chunk_text_by_tokens or chunk_by_semantic and api_name: + # Perform summarization based on chunks + for chunk in chunked_transcriptions: + summarized_chunks = [] + if api_name == "anthropic": + summary = summarize_with_anthropic(api_key, chunk, custom_prompt_input) + elif api_name == "cohere": + summary = summarize_with_cohere(api_key, chunk, custom_prompt_input) + elif api_name == "openai": + summary = summarize_with_openai(api_key, chunk, custom_prompt_input) + elif api_name == "Groq": + summary = summarize_with_groq(api_key, chunk, custom_prompt_input) + elif api_name == "DeepSeek": + summary = summarize_with_deepseek(api_key, chunk, custom_prompt_input) + elif api_name == "OpenRouter": + summary = summarize_with_openrouter(api_key, chunk, custom_prompt_input) + elif api_name == "Llama.cpp": + summary = summarize_with_llama(chunk, custom_prompt_input) + elif api_name == "Kobold": + summary = summarize_with_kobold(chunk, custom_prompt_input) + elif api_name == "Ooba": + summary = summarize_with_oobabooga(chunk, custom_prompt_input) + elif api_name == "Tabbyapi": + summary = summarize_with_tabbyapi(chunk, custom_prompt_input) + elif api_name == "VLLM": + summary = summarize_with_vllm(chunk, custom_prompt_input) + summarized_chunk_transcriptions.append(summary) + + # Combine chunked transcriptions into a single file + combined_transcription_text = '\n\n'.join(chunked_transcriptions) + combined_transcription_file_path = os.path.join(download_path, 'combined_transcription.txt') + with open(combined_transcription_file_path, 'w') as f: + f.write(combined_transcription_text) + + # Combine summarized chunk transcriptions into a single file + combined_summary_text = '\n\n'.join(summarized_chunk_transcriptions) + combined_summary_file_path = os.path.join(download_path, 'combined_summary.txt') + with open(combined_summary_file_path, 'w') as f: + f.write(combined_summary_text) + + # Handle rolling summarization + if rolling_summarization: + summary_text = rolling_summarize( + text=extract_text_from_segments(segments), + detail=detail_level, + model='gpt-4-turbo', + additional_instructions=custom_prompt_input, + summarize_recursively=recursive_summarization + ) + elif api_name: + summary_text = perform_summarization(api_name, segments_json_path, custom_prompt_input, api_key, + recursive_summarization) + else: + summary_text = 'Summary not available' + + # Check to see if chunking was performed, and if so, return that instead + if chunk_text_by_words or chunk_text_by_sentences or chunk_text_by_paragraphs or chunk_text_by_tokens or chunk_by_semantic: + # Combine chunked transcriptions into a single file + # FIXME - validate this works.... + json_file_path, summary_file_path = save_transcription_and_summary(combined_transcription_file_path, combined_summary_file_path, download_path) + add_media_to_database(url, info_dict, segments, summary_text, keywords, custom_prompt_input, whisper_model) + return transcription_text, summary_text, json_file_path, summary_file_path, None, None + else: + json_file_path, summary_file_path = save_transcription_and_summary(transcription_text, summary_text, download_path) + add_media_to_database(url, info_dict, segments, summary_text, keywords, custom_prompt_input, whisper_model) + return transcription_text, summary_text, json_file_path, summary_file_path, None, None + + except Exception as e: + logging.error(f": {e}") + return str(e), 'process_url: Error processing the request.', None, None, None, None + +# +# +############################################################################################################################################ diff --git a/App_Function_Libraries/System_Checks_Lib.py b/App_Function_Libraries/System_Checks_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..8f4e4aef799723ca9b742ab392a8f11e43dd4a33 --- /dev/null +++ b/App_Function_Libraries/System_Checks_Lib.py @@ -0,0 +1,184 @@ +# System_Checks_Lib.py +######################################### +# System Checks Library +# This library is used to check the system for the necessary dependencies to run the script. +# It checks for the OS, the availability of the GPU, and the availability of the ffmpeg executable. +# If the GPU is available, it asks the user if they would like to use it for processing. +# If ffmpeg is not found, it asks the user if they would like to download it. +# The script will exit if the user chooses not to download ffmpeg. +#### + +#################### +# Function List +# +# 1. platform_check() +# 2. cuda_check() +# 3. decide_cpugpu() +# 4. check_ffmpeg() +# 5. download_ffmpeg() +# +#################### + + + + +# Import necessary libraries +import logging +import os +import platform +import requests +import shutil +import subprocess +import zipfile +# Import Local Libraries +#from App_Function_Libraries import +# +####################################################################################################################### +# Function Definitions +# + +def platform_check(): + global userOS + if platform.system() == "Linux": + print("Linux OS detected \n Running Linux appropriate commands") + userOS = "Linux" + elif platform.system() == "Windows": + print("Windows OS detected \n Running Windows appropriate commands") + userOS = "Windows" + else: + print("Other OS detected \n Maybe try running things manually?") + exit() + + +# Check for NVIDIA GPU and CUDA availability +def cuda_check(): + global processing_choice + try: + # Run nvidia-smi to capture its output + nvidia_smi_output = subprocess.check_output("nvidia-smi", shell=True).decode() + + # Look for CUDA version in the output + if "CUDA Version" in nvidia_smi_output: + cuda_version = next( + (line.split(":")[-1].strip() for line in nvidia_smi_output.splitlines() if "CUDA Version" in line), + "Not found") + print(f"NVIDIA GPU with CUDA Version {cuda_version} is available.") + processing_choice = "cuda" + else: + print("CUDA is not installed or configured correctly.") + processing_choice = "cpu" + + except subprocess.CalledProcessError as e: + print(f"Failed to run 'nvidia-smi': {str(e)}") + processing_choice = "cpu" + except Exception as e: + print(f"An error occurred: {str(e)}") + processing_choice = "cpu" + + # Optionally, check for the CUDA_VISIBLE_DEVICES env variable as an additional check + if "CUDA_VISIBLE_DEVICES" in os.environ: + print("CUDA_VISIBLE_DEVICES is set:", os.environ["CUDA_VISIBLE_DEVICES"]) + else: + print("CUDA_VISIBLE_DEVICES not set.") + + +# Ask user if they would like to use either their GPU or their CPU for transcription +def decide_cpugpu(): + global processing_choice + processing_input = input("Would you like to use your GPU or CPU for transcription? (1/cuda)GPU/(2/cpu)CPU): ") + if processing_choice == "cuda" and (processing_input.lower() == "cuda" or processing_input == "1"): + print("You've chosen to use the GPU.") + logging.debug("GPU is being used for processing") + processing_choice = "cuda" + elif processing_input.lower() == "cpu" or processing_input == "2": + print("You've chosen to use the CPU.") + logging.debug("CPU is being used for processing") + processing_choice = "cpu" + else: + print("Invalid choice. Please select either GPU or CPU.") + + +# check for existence of ffmpeg +def check_ffmpeg(): + if shutil.which("ffmpeg") or (os.path.exists("Bin") and os.path.isfile(".\\Bin\\ffmpeg.exe")): + logging.debug("ffmpeg found installed on the local system, in the local PATH, or in the './Bin' folder") + pass + else: + logging.debug("ffmpeg not installed on the local system/in local PATH") + print( + "ffmpeg is not installed.\n\n You can either install it manually, or through your package manager of " + "choice.\n Windows users, builds are here: https://www.gyan.dev/ffmpeg/builds/") + if userOS == "Windows": + download_ffmpeg() + elif userOS == "Linux": + print( + "You should install ffmpeg using your platform's appropriate package manager, 'apt install ffmpeg'," + "'dnf install ffmpeg' or 'pacman', etc.") + else: + logging.debug("running an unsupported OS") + print("You're running an unspported/Un-tested OS") + exit_script = input("Let's exit the script, unless you're feeling lucky? (y/n)") + if exit_script == "y" or "yes" or "1": + exit() + + +# Download ffmpeg +def download_ffmpeg(): + user_choice = input("Do you want to download ffmpeg? (y)Yes/(n)No: ") + if user_choice.lower() in ['yes', 'y', '1']: + print("Downloading ffmpeg") + url = "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.zip" + response = requests.get(url) + + if response.status_code == 200: + print("Saving ffmpeg zip file") + logging.debug("Saving ffmpeg zip file") + zip_path = "ffmpeg-release-essentials.zip" + with open(zip_path, 'wb') as file: + file.write(response.content) + + logging.debug("Extracting the 'ffmpeg.exe' file from the zip") + print("Extracting ffmpeg.exe from zip file to '/Bin' folder") + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + # Find the ffmpeg.exe file within the zip + ffmpeg_path = None + for file_info in zip_ref.infolist(): + if file_info.filename.endswith("ffmpeg.exe"): + ffmpeg_path = file_info.filename + break + + if ffmpeg_path is None: + logging.error("ffmpeg.exe not found in the zip file.") + print("ffmpeg.exe not found in the zip file.") + return + + logging.debug("checking if the './Bin' folder exists, creating if not") + bin_folder = "Bin" + if not os.path.exists(bin_folder): + logging.debug("Creating a folder for './Bin', it didn't previously exist") + os.makedirs(bin_folder) + + logging.debug("Extracting 'ffmpeg.exe' to the './Bin' folder") + zip_ref.extract(ffmpeg_path, path=bin_folder) + + logging.debug("Moving 'ffmpeg.exe' to the './Bin' folder") + src_path = os.path.join(bin_folder, ffmpeg_path) + dst_path = os.path.join(bin_folder, "ffmpeg.exe") + shutil.move(src_path, dst_path) + + logging.debug("Removing ffmpeg zip file") + print("Deleting zip file (we've already extracted ffmpeg.exe, no worries)") + os.remove(zip_path) + + logging.debug("ffmpeg.exe has been downloaded and extracted to the './Bin' folder.") + print("ffmpeg.exe has been successfully downloaded and extracted to the './Bin' folder.") + else: + logging.error("Failed to download the zip file.") + print("Failed to download the zip file.") + else: + logging.debug("User chose to not download ffmpeg") + print("ffmpeg will not be downloaded.") + +# +# +####################################################################################################################### diff --git a/App_Function_Libraries/Tokenization_Methods_Lib.py b/App_Function_Libraries/Tokenization_Methods_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..3694f88940dfdb07a41d06557f79e26f376c7220 --- /dev/null +++ b/App_Function_Libraries/Tokenization_Methods_Lib.py @@ -0,0 +1,30 @@ +# Tokenization_Methods_Lib.py +######################################### +# Tokenization Methods Library +# This library is used to handle tokenization of text for summarization. +# +#### +import tiktoken + +# Import Local +from typing import List + +#################### +# Function List +# +# 1. openai_tokenize(text: str) -> List[str] +# +#################### + + +####################################################################################################################### +# Function Definitions +# + +def openai_tokenize(text: str) -> List[str]: + encoding = tiktoken.encoding_for_model('gpt-4-turbo') + return encoding.encode(text) + +# +# +####################################################################################################################### diff --git a/App_Function_Libraries/Tone-Changer.py b/App_Function_Libraries/Tone-Changer.py new file mode 100644 index 0000000000000000000000000000000000000000..5e0f8ced667a1dc6440d62d6b5ac8ff919e3b101 --- /dev/null +++ b/App_Function_Libraries/Tone-Changer.py @@ -0,0 +1,46 @@ +import gradio as gr +import json +from transformers import pipeline + +# Initialize the text generation pipeline +generator = pipeline('text-generation', model='gpt2') + + +def adjust_tone(text, concise, casual): + tones = [ + {"tone": "concise", "weight": concise}, + {"tone": "casual", "weight": casual}, + {"tone": "professional", "weight": 1 - casual}, + {"tone": "expanded", "weight": 1 - concise} + ] + tones = sorted(tones, key=lambda x: x['weight'], reverse=True)[:2] + + tone_prompt = " and ".join([f"{t['tone']} (weight: {t['weight']:.2f})" for t in tones]) + + prompt = f"Rewrite the following text to match these tones: {tone_prompt}. Text: {text}" + + result = generator(prompt, max_length=100, num_return_sequences=1) + return result[0]['generated_text'] + + +# Gradio Interface +with gr.Blocks() as demo: + gr.Markdown("# Tone Adjuster") + + input_text = gr.Textbox(label="Input Text") + + with gr.Row(): + concise_slider = gr.Slider(minimum=0, maximum=1, value=0.5, label="Concise vs Expanded") + casual_slider = gr.Slider(minimum=0, maximum=1, value=0.5, label="Casual vs Professional") + + output_text = gr.Textbox(label="Adjusted Text") + + adjust_btn = gr.Button("Adjust Tone") + + adjust_btn.click( + adjust_tone, + inputs=[input_text, concise_slider, casual_slider], + outputs=output_text + ) + +demo.launch() \ No newline at end of file diff --git a/App_Function_Libraries/Utils.py b/App_Function_Libraries/Utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d8cfc94ea218b798134e7293578e0dcef7e8b9bb --- /dev/null +++ b/App_Function_Libraries/Utils.py @@ -0,0 +1,440 @@ +# Utils.py +######################################### +# General Utilities Library +# This library is used to hold random utilities used by various other libraries. +# +#### +#################### +# Function List +# +# 1. extract_text_from_segments(segments: List[Dict]) -> str +# 2. download_file(url, dest_path, expected_checksum=None, max_retries=3, delay=5) +# 3. verify_checksum(file_path, expected_checksum) +# 4. create_download_directory(title) +# 5. sanitize_filename(filename) +# 6. normalize_title(title) +# 7. +# +# +# +#################### +# Import necessary libraries +import configparser +import hashlib +import json +import logging +from datetime import timedelta +from urllib.parse import urlparse, parse_qs, urlencode, urlunparse + +import requests +import time +from tqdm import tqdm +import os +import re +import unicodedata + +from App_Function_Libraries.Video_DL_Ingestion_Lib import get_youtube + + +####################################################################################################################### +# Function Definitions +# + +def extract_text_from_segments(segments): + logging.debug(f"Segments received: {segments}") + logging.debug(f"Type of segments: {type(segments)}") + + def extract_text_recursive(data): + if isinstance(data, dict): + for key, value in data.items(): + if key == 'Text': + return value + elif isinstance(value, (dict, list)): + result = extract_text_recursive(value) + if result: + return result + elif isinstance(data, list): + return ' '.join(filter(None, [extract_text_recursive(item) for item in data])) + return None + + text = extract_text_recursive(segments) + + if text: + return text.strip() + else: + logging.error(f"Unable to extract text from segments: {segments}") + return "Error: Unable to extract transcription" + + +def download_file(url, dest_path, expected_checksum=None, max_retries=3, delay=5): + temp_path = dest_path + '.tmp' + + for attempt in range(max_retries): + try: + # Check if a partial download exists and get its size + resume_header = {} + if os.path.exists(temp_path): + resume_header = {'Range': f'bytes={os.path.getsize(temp_path)}-'} + + response = requests.get(url, stream=True, headers=resume_header) + response.raise_for_status() + + # Get the total file size from headers + total_size = int(response.headers.get('content-length', 0)) + initial_pos = os.path.getsize(temp_path) if os.path.exists(temp_path) else 0 + + mode = 'ab' if 'Range' in response.headers else 'wb' + with open(temp_path, mode) as temp_file, tqdm( + total=total_size, unit='B', unit_scale=True, desc=dest_path, initial=initial_pos, ascii=True + ) as pbar: + for chunk in response.iter_content(chunk_size=8192): + if chunk: # filter out keep-alive new chunks + temp_file.write(chunk) + pbar.update(len(chunk)) + + # Verify the checksum if provided + if expected_checksum: + if not verify_checksum(temp_path, expected_checksum): + os.remove(temp_path) + raise ValueError("Downloaded file's checksum does not match the expected checksum") + + # Move the file to the final destination + os.rename(temp_path, dest_path) + print("Download complete and verified!") + return dest_path + + except Exception as e: + print(f"Attempt {attempt + 1} failed: {e}") + if attempt < max_retries - 1: + print(f"Retrying in {delay} seconds...") + time.sleep(delay) + else: + print("Max retries reached. Download failed.") + raise + + +def verify_checksum(file_path, expected_checksum): + sha256_hash = hashlib.sha256() + with open(file_path, 'rb') as f: + for byte_block in iter(lambda: f.read(4096), b''): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() == expected_checksum + + +def create_download_directory(title): + base_dir = "Results" + # Remove characters that are illegal in Windows filenames and normalize + safe_title = normalize_title(title) + logging.debug(f"{title} successfully normalized") + session_path = os.path.join(base_dir, safe_title) + if not os.path.exists(session_path): + os.makedirs(session_path, exist_ok=True) + logging.debug(f"Created directory for downloaded video: {session_path}") + else: + logging.debug(f"Directory already exists for downloaded video: {session_path}") + return session_path + + +def sanitize_filename(filename): + # Remove invalid characters and replace spaces with underscores + sanitized = re.sub(r'[<>:"/\\|?*]', '', filename) + sanitized = re.sub(r'\s+', ' ', sanitized).strip() + return sanitized + + +def normalize_title(title): + # Normalize the string to 'NFKD' form and encode to 'ascii' ignoring non-ascii characters + title = unicodedata.normalize('NFKD', title).encode('ascii', 'ignore').decode('ascii') + title = title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('"', '').replace('*', '').replace('?', + '').replace( + '<', '').replace('>', '').replace('|', '') + return title + + + + +def clean_youtube_url(url): + parsed_url = urlparse(url) + query_params = parse_qs(parsed_url.query) + if 'list' in query_params: + query_params.pop('list') + cleaned_query = urlencode(query_params, doseq=True) + cleaned_url = urlunparse(parsed_url._replace(query=cleaned_query)) + return cleaned_url + + +def extract_video_info(url): + info_dict = get_youtube(url) + title = info_dict.get('title', 'Untitled') + return info_dict, title + + +def clean_youtube_url(url): + parsed_url = urlparse(url) + query_params = parse_qs(parsed_url.query) + if 'list' in query_params: + query_params.pop('list') + cleaned_query = urlencode(query_params, doseq=True) + cleaned_url = urlunparse(parsed_url._replace(query=cleaned_query)) + return cleaned_url + +def extract_video_info(url): + info_dict = get_youtube(url) + title = info_dict.get('title', 'Untitled') + return info_dict, title + +def import_data(file): + # Implement this function to import data from a file + pass + + + + +####################### +# Config loading +# + +def load_comprehensive_config(): + # Get the directory of the current script + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Go up one level to the project root directory + project_root = os.path.dirname(current_dir) + # Construct the path to the config file in the project root directory + config_path = os.path.join(project_root, 'config.txt') + # Create a ConfigParser object + config = configparser.ConfigParser() + # Read the configuration file + files_read = config.read(config_path) + if not files_read: + raise FileNotFoundError(f"Config file not found at {config_path}") + return config + + +def load_and_log_configs(): + try: + config = load_comprehensive_config() + if config is None: + logging.error("Config is None, cannot proceed") + return None + # API Keys + anthropic_api_key = config.get('API', 'anthropic_api_key', fallback=None) + logging.debug( + f"Loaded Anthropic API Key: {anthropic_api_key[:5]}...{anthropic_api_key[-5:] if anthropic_api_key else None}") + + cohere_api_key = config.get('API', 'cohere_api_key', fallback=None) + logging.debug( + f"Loaded Cohere API Key: {cohere_api_key[:5]}...{cohere_api_key[-5:] if cohere_api_key else None}") + + groq_api_key = config.get('API', 'groq_api_key', fallback=None) + logging.debug(f"Loaded Groq API Key: {groq_api_key[:5]}...{groq_api_key[-5:] if groq_api_key else None}") + + openai_api_key = config.get('API', 'openai_api_key', fallback=None) + logging.debug( + f"Loaded OpenAI API Key: {openai_api_key[:5]}...{openai_api_key[-5:] if openai_api_key else None}") + + huggingface_api_key = config.get('API', 'huggingface_api_key', fallback=None) + logging.debug( + f"Loaded HuggingFace API Key: {huggingface_api_key[:5]}...{huggingface_api_key[-5:] if huggingface_api_key else None}") + + openrouter_api_key = config.get('API', 'openrouter_api_key', fallback=None) + logging.debug( + f"Loaded OpenRouter API Key: {openrouter_api_key[:5]}...{openrouter_api_key[-5:] if openrouter_api_key else None}") + + deepseek_api_key = config.get('API', 'deepseek_api_key', fallback=None) + logging.debug( + f"Loaded DeepSeek API Key: {deepseek_api_key[:5]}...{deepseek_api_key[-5:] if deepseek_api_key else None}") + + # Models + anthropic_model = config.get('API', 'anthropic_model', fallback='claude-3-sonnet-20240229') + cohere_model = config.get('API', 'cohere_model', fallback='command-r-plus') + groq_model = config.get('API', 'groq_model', fallback='llama3-70b-8192') + openai_model = config.get('API', 'openai_model', fallback='gpt-4-turbo') + huggingface_model = config.get('API', 'huggingface_model', fallback='CohereForAI/c4ai-command-r-plus') + openrouter_model = config.get('API', 'openrouter_model', fallback='microsoft/wizardlm-2-8x22b') + deepseek_model = config.get('API', 'deepseek_model', fallback='deepseek-chat') + + logging.debug(f"Loaded Anthropic Model: {anthropic_model}") + logging.debug(f"Loaded Cohere Model: {cohere_model}") + logging.debug(f"Loaded Groq Model: {groq_model}") + logging.debug(f"Loaded OpenAI Model: {openai_model}") + logging.debug(f"Loaded HuggingFace Model: {huggingface_model}") + logging.debug(f"Loaded OpenRouter Model: {openrouter_model}") + + # Local-Models + kobold_api_IP = config.get('Local-API', 'kobold_api_IP', fallback='http://127.0.0.1:5000/api/v1/generate') + kobold_api_key = config.get('Local-API', 'kobold_api_key', fallback='') + + llama_api_IP = config.get('Local-API', 'llama_api_IP', fallback='http://127.0.0.1:8080/v1/chat/completions') + llama_api_key = config.get('Local-API', 'llama_api_key', fallback='') + + ooba_api_IP = config.get('Local-API', 'ooba_api_IP', fallback='http://127.0.0.1:5000/v1/chat/completions') + ooba_api_key = config.get('Local-API', 'ooba_api_key', fallback='') + + tabby_api_IP = config.get('Local-API', 'tabby_api_IP', fallback='http://127.0.0.1:5000/api/v1/generate') + tabby_api_key = config.get('Local-API', 'tabby_api_key', fallback=None) + + vllm_api_url = config.get('Local-API', 'vllm_api_IP', fallback='http://127.0.0.1:500/api/v1/chat/completions') + vllm_api_key = config.get('Local-API', 'vllm_api_key', fallback=None) + + logging.debug(f"Loaded Kobold API IP: {kobold_api_IP}") + logging.debug(f"Loaded Llama API IP: {llama_api_IP}") + logging.debug(f"Loaded Ooba API IP: {ooba_api_IP}") + logging.debug(f"Loaded Tabby API IP: {tabby_api_IP}") + logging.debug(f"Loaded VLLM API URL: {vllm_api_url}") + + # Retrieve output paths from the configuration file + output_path = config.get('Paths', 'output_path', fallback='results') + logging.debug(f"Output path set to: {output_path}") + + # Retrieve processing choice from the configuration file + processing_choice = config.get('Processing', 'processing_choice', fallback='cpu') + logging.debug(f"Processing choice set to: {processing_choice}") + + # Prompts - FIXME + prompt_path = config.get('Prompts', 'prompt_path', fallback='prompts.db') + + return { + 'api_keys': { + 'anthropic': anthropic_api_key, + 'cohere': cohere_api_key, + 'groq': groq_api_key, + 'openai': openai_api_key, + 'huggingface': huggingface_api_key, + 'openrouter': openrouter_api_key, + 'deepseek': deepseek_api_key + }, + 'models': { + 'anthropic': anthropic_model, + 'cohere': cohere_model, + 'groq': groq_model, + 'openai': openai_model, + 'huggingface': huggingface_model, + 'openrouter': openrouter_model, + 'deepseek': deepseek_model + }, + 'local_apis': { + 'kobold': {'ip': kobold_api_IP, 'key': kobold_api_key}, + 'llama': {'ip': llama_api_IP, 'key': llama_api_key}, + 'ooba': {'ip': ooba_api_IP, 'key': ooba_api_key}, + 'tabby': {'ip': tabby_api_IP, 'key': tabby_api_key}, + 'vllm': {'ip': vllm_api_url, 'key': vllm_api_key} + }, + 'output_path': output_path, + 'processing_choice': processing_choice + } + + except Exception as e: + logging.error(f"Error loading config: {str(e)}") + return None + + + +# Log file +# logging.basicConfig(filename='debug-runtime.log', encoding='utf-8', level=logging.DEBUG) + + + + + + + +def format_metadata_as_text(metadata): + if not metadata: + return "No metadata available" + + formatted_text = "Video Metadata:\n" + for key, value in metadata.items(): + if value is not None: + if isinstance(value, list): + # Join list items with commas + formatted_value = ", ".join(str(item) for item in value) + elif key == 'upload_date' and len(str(value)) == 8: + # Format date as YYYY-MM-DD + formatted_value = f"{value[:4]}-{value[4:6]}-{value[6:]}" + elif key in ['view_count', 'like_count']: + # Format large numbers with commas + formatted_value = f"{value:,}" + elif key == 'duration': + # Convert seconds to HH:MM:SS format + hours, remainder = divmod(value, 3600) + minutes, seconds = divmod(remainder, 60) + formatted_value = f"{hours:02d}:{minutes:02d}:{seconds:02d}" + else: + formatted_value = str(value) + + formatted_text += f"{key.capitalize()}: {formatted_value}\n" + return formatted_text.strip() + +# # Example usage: +# example_metadata = { +# 'title': 'Sample Video Title', +# 'uploader': 'Channel Name', +# 'upload_date': '20230615', +# 'view_count': 1000000, +# 'like_count': 50000, +# 'duration': 3725, # 1 hour, 2 minutes, 5 seconds +# 'tags': ['tag1', 'tag2', 'tag3'], +# 'description': 'This is a sample video description.' +# } +# +# print(format_metadata_as_text(example_metadata)) + + + +def convert_to_seconds(time_str): + if not time_str: + return 0 + + # If it's already a number, assume it's in seconds + if time_str.isdigit(): + return int(time_str) + + # Parse time string in format HH:MM:SS, MM:SS, or SS + time_parts = time_str.split(':') + if len(time_parts) == 3: + return int(timedelta(hours=int(time_parts[0]), + minutes=int(time_parts[1]), + seconds=int(time_parts[2])).total_seconds()) + elif len(time_parts) == 2: + return int(timedelta(minutes=int(time_parts[0]), + seconds=int(time_parts[1])).total_seconds()) + elif len(time_parts) == 1: + return int(time_parts[0]) + else: + raise ValueError(f"Invalid time format: {time_str}") + + +def save_to_file(video_urls, filename): + with open(filename, 'w') as file: + file.write('\n'.join(video_urls)) + print(f"Video URLs saved to {filename}") + + +def save_segments_to_json(segments, file_name="transcription_segments.json"): + """ + Save transcription segments to a JSON file. + + Parameters: + segments (list): List of transcription segments + file_name (str): Name of the JSON file to save (default: "transcription_segments.json") + + Returns: + str: Path to the saved JSON file + """ + # Ensure the Results directory exists + os.makedirs("Results", exist_ok=True) + + # Full path for the JSON file + json_file_path = os.path.join("Results", file_name) + + # Save segments to JSON file + with open(json_file_path, 'w', encoding='utf-8') as json_file: + json.dump(segments, json_file, ensure_ascii=False, indent=4) + + return json_file_path + + + + + + diff --git a/App_Function_Libraries/Video_DL_Ingestion_Lib.py b/App_Function_Libraries/Video_DL_Ingestion_Lib.py new file mode 100644 index 0000000000000000000000000000000000000000..feecbbaea1dfb4079d69bbb3fe855414482c2e32 --- /dev/null +++ b/App_Function_Libraries/Video_DL_Ingestion_Lib.py @@ -0,0 +1,315 @@ +# Video_DL_Ingestion_Lib.py +######################################### +# Video Downloader and Ingestion Library +# This library is used to handle downloading videos from YouTube and other platforms. +# It also handles the ingestion of the videos into the database. +# It uses yt-dlp to extract video information and download the videos. +#### +import json +#################### +# Function List +# +# 1. get_video_info(url) +# 2. create_download_directory(title) +# 3. sanitize_filename(title) +# 4. normalize_title(title) +# 5. get_youtube(video_url) +# 6. get_playlist_videos(playlist_url) +# 7. download_video(video_url, download_path, info_dict, download_video_flag) +# 8. save_to_file(video_urls, filename) +# 9. save_summary_to_file(summary, file_path) +# 10. process_url(url, num_speakers, whisper_model, custom_prompt, offset, api_name, api_key, vad_filter, download_video, download_audio, rolling_summarization, detail_level, question_box, keywords, chunk_summarization, chunk_duration_input, words_per_second_input) +# +# +#################### +# Import necessary libraries to run solo for testing +import logging +import os +import re +import sys +from urllib.parse import urlparse, parse_qs + +import unicodedata +# 3rd-Party Imports +import yt_dlp +# Import Local +# +####################################################################################################################### +# Function Definitions +# + +def normalize_title(title): + # Normalize the string to 'NFKD' form and encode to 'ascii' ignoring non-ascii characters + title = unicodedata.normalize('NFKD', title).encode('ascii', 'ignore').decode('ascii') + title = title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('"', '').replace('*', '').replace('?', + '').replace( + '<', '').replace('>', '').replace('|', '') + return title + +def get_video_info(url: str) -> dict: + ydl_opts = { + 'quiet': True, + 'no_warnings': True, + 'skip_download': True, + } + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + try: + info_dict = ydl.extract_info(url, download=False) + return info_dict + except Exception as e: + logging.error(f"Error extracting video info: {e}") + return None + + +def get_youtube(video_url): + ydl_opts = { + 'format': 'bestaudio[ext=m4a]', + 'noplaylist': False, + 'quiet': True, + 'extract_flat': True + } + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + logging.debug("About to extract youtube info") + info_dict = ydl.extract_info(video_url, download=False) + logging.debug("Youtube info successfully extracted") + return info_dict + + +def get_playlist_videos(playlist_url): + ydl_opts = { + 'extract_flat': True, + 'skip_download': True, + 'quiet': True + } + + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + info = ydl.extract_info(playlist_url, download=False) + + if 'entries' in info: + video_urls = [entry['url'] for entry in info['entries']] + playlist_title = info['title'] + return video_urls, playlist_title + else: + print("No videos found in the playlist.") + return [], None + + +def download_video(video_url, download_path, info_dict, download_video_flag): + global video_file_path, ffmpeg_path + global audio_file_path + + # Normalize Video Title name + logging.debug("About to normalize downloaded video title") + if 'title' not in info_dict or 'ext' not in info_dict: + logging.error("info_dict is missing 'title' or 'ext'") + return None + + normalized_video_title = normalize_title(info_dict['title']) + video_file_path = os.path.join(download_path, f"{normalized_video_title}.{info_dict['ext']}") + + # Check for existence of video file + if os.path.exists(video_file_path): + logging.info(f"Video file already exists: {video_file_path}") + return video_file_path + + # Setup path handling for ffmpeg on different OSs + if sys.platform.startswith('win'): + ffmpeg_path = os.path.join(os.getcwd(), 'Bin', 'ffmpeg.exe') + elif sys.platform.startswith('linux'): + ffmpeg_path = 'ffmpeg' + elif sys.platform.startswith('darwin'): + ffmpeg_path = 'ffmpeg' + + if download_video_flag: + video_file_path = os.path.join(download_path, f"{normalized_video_title}.mp4") + ydl_opts_video = { + 'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', + 'outtmpl': video_file_path, + 'ffmpeg_location': ffmpeg_path + } + + try: + with yt_dlp.YoutubeDL(ydl_opts_video) as ydl: + logging.debug("yt_dlp: About to download video with youtube-dl") + ydl.download([video_url]) + logging.debug("yt_dlp: Video successfully downloaded with youtube-dl") + if os.path.exists(video_file_path): + return video_file_path + else: + logging.error("yt_dlp: Video file not found after download") + return None + except Exception as e: + logging.error(f"yt_dlp: Error downloading video: {e}") + return None + elif not download_video_flag: + video_file_path = os.path.join(download_path, f"{normalized_video_title}.mp4") + # Set options for video and audio + ydl_opts = { + 'format': 'bestaudio[ext=m4a]', + 'quiet': True, + 'outtmpl': video_file_path + } + + try: + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + logging.debug("yt_dlp: About to download video with youtube-dl") + ydl.download([video_url]) + logging.debug("yt_dlp: Video successfully downloaded with youtube-dl") + if os.path.exists(video_file_path): + return video_file_path + else: + logging.error("yt_dlp: Video file not found after download") + return None + except Exception as e: + logging.error(f"yt_dlp: Error downloading video: {e}") + return None + + else: + logging.debug("download_video: Download video flag is set to False and video file path is not found") + return None + + +def extract_video_info(url): + try: + with yt_dlp.YoutubeDL({'quiet': True}) as ydl: + info = ydl.extract_info(url, download=False) + + # Log only a subset of the info to avoid overwhelming the logs + log_info = { + 'title': info.get('title'), + 'duration': info.get('duration'), + 'upload_date': info.get('upload_date') + } + logging.debug(f"Extracted info for {url}: {log_info}") + + return info + except Exception as e: + logging.error(f"Error extracting video info for {url}: {str(e)}", exc_info=True) + return None + + +def get_youtube_playlist_urls(playlist_id): + ydl_opts = { + 'extract_flat': True, + 'quiet': True, + } + + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + result = ydl.extract_info(f'https://www.youtube.com/playlist?list={playlist_id}', download=False) + return [entry['url'] for entry in result['entries'] if entry.get('url')] + + +def parse_and_expand_urls(url_input): + logging.info(f"Starting parse_and_expand_urls with input: {url_input}") + urls = [url.strip() for url in url_input.split('\n') if url.strip()] + logging.info(f"Parsed URLs: {urls}") + expanded_urls = [] + + for url in urls: + try: + logging.info(f"Processing URL: {url}") + parsed_url = urlparse(url) + logging.debug(f"Parsed URL components: {parsed_url}") + + # YouTube playlist handling + if 'youtube.com' in parsed_url.netloc and 'list' in parsed_url.query: + playlist_id = parse_qs(parsed_url.query)['list'][0] + logging.info(f"Detected YouTube playlist with ID: {playlist_id}") + playlist_urls = get_youtube_playlist_urls(playlist_id) + logging.info(f"Expanded playlist URLs: {playlist_urls}") + expanded_urls.extend(playlist_urls) + + # YouTube short URL handling + elif 'youtu.be' in parsed_url.netloc: + video_id = parsed_url.path.lstrip('/') + full_url = f'https://www.youtube.com/watch?v={video_id}' + logging.info(f"Expanded YouTube short URL to: {full_url}") + expanded_urls.append(full_url) + + # Vimeo handling + elif 'vimeo.com' in parsed_url.netloc: + video_id = parsed_url.path.lstrip('/') + full_url = f'https://vimeo.com/{video_id}' + logging.info(f"Processed Vimeo URL: {full_url}") + expanded_urls.append(full_url) + + # Add more platform-specific handling here + + else: + logging.info(f"URL not recognized as special case, adding as-is: {url}") + expanded_urls.append(url) + + except Exception as e: + logging.error(f"Error processing URL {url}: {str(e)}", exc_info=True) + # Optionally, you might want to add the problematic URL to expanded_urls + # expanded_urls.append(url) + + logging.info(f"Final expanded URLs: {expanded_urls}") + return expanded_urls + + +def extract_metadata(url, use_cookies=False, cookies=None): + ydl_opts = { + 'quiet': True, + 'no_warnings': True, + 'extract_flat': True, + 'skip_download': True, + } + + if use_cookies and cookies: + try: + cookie_dict = json.loads(cookies) + ydl_opts['cookiefile'] = cookie_dict + except json.JSONDecodeError: + logging.warning("Invalid cookie format. Proceeding without cookies.") + + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + try: + info = ydl.extract_info(url, download=False) + metadata = { + 'title': info.get('title'), + 'uploader': info.get('uploader'), + 'upload_date': info.get('upload_date'), + 'view_count': info.get('view_count'), + 'like_count': info.get('like_count'), + 'duration': info.get('duration'), + 'tags': info.get('tags'), + 'description': info.get('description') + } + + # Create a safe subset of metadata to log + safe_metadata = { + 'title': metadata.get('title', 'No title'), + 'duration': metadata.get('duration', 'Unknown duration'), + 'upload_date': metadata.get('upload_date', 'Unknown upload date'), + 'uploader': metadata.get('uploader', 'Unknown uploader') + } + + logging.info(f"Successfully extracted metadata for {url}: {safe_metadata}") + return metadata + except Exception as e: + logging.error(f"Error extracting metadata for {url}: {str(e)}", exc_info=True) + return None + + +def generate_timestamped_url(url, hours, minutes, seconds): + # Extract video ID from the URL + video_id_match = re.search(r'(?:v=|\/)([0-9A-Za-z_-]{11}).*', url) + if not video_id_match: + return "Invalid YouTube URL" + + video_id = video_id_match.group(1) + + # Calculate total seconds + total_seconds = int(hours) * 3600 + int(minutes) * 60 + int(seconds) + + # Generate the new URL + new_url = f"https://www.youtube.com/watch?v={video_id}&t={total_seconds}s" + + return new_url + + + +# +# +####################################################################################################################### diff --git a/App_Function_Libraries/__Init__.py b/App_Function_Libraries/__Init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/App_Function_Libraries/__pycache__/Article_Extractor_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Article_Extractor_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5bde582ad7771c59fd816498d5e388d66522538f Binary files /dev/null and b/App_Function_Libraries/__pycache__/Article_Extractor_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Article_Summarization_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Article_Summarization_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c48aba1268b8655a57e23736668adf9b24ce61ce Binary files /dev/null and b/App_Function_Libraries/__pycache__/Article_Summarization_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Audio_Files.cpython-312.pyc b/App_Function_Libraries/__pycache__/Audio_Files.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..135f0a203c2727d0b18271e01b543df1d0ddf593 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Audio_Files.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Audio_Transcription_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Audio_Transcription_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..41fd7458b0aea7e1f9a7f1939bf4ca38de4ba5b5 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Audio_Transcription_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Book_Ingestion_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Book_Ingestion_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd4115aa5375a530e72919ff0c888da77bf6a758 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Book_Ingestion_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Chunk_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Chunk_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6e5e42a71597d61f6ee77bd904050c8af9bdcbe3 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Chunk_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Diarization_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Diarization_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8df0c43876266424200e67e4c06135a7f10b6fd8 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Diarization_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Gradio_Related.cpython-312.pyc b/App_Function_Libraries/__pycache__/Gradio_Related.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e1660c684f77b5c20605a66d0927cb5206b4a108 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Gradio_Related.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/LLM_API_Calls.cpython-312.pyc b/App_Function_Libraries/__pycache__/LLM_API_Calls.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc1f31541a5c6b94ddca3e2c1040fb57a24ce3c0 Binary files /dev/null and b/App_Function_Libraries/__pycache__/LLM_API_Calls.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Local_File_Processing_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Local_File_Processing_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..46294649bb3f376de879a98b0c7c6f3212b6fe20 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Local_File_Processing_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Local_LLM_Inference_Engine_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Local_LLM_Inference_Engine_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..994ab20153273af293d25430d7a3a2a1c1afdb61 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Local_LLM_Inference_Engine_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Local_Summarization_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Local_Summarization_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e1f1c6c00768710ab30474335054486d683ec88b Binary files /dev/null and b/App_Function_Libraries/__pycache__/Local_Summarization_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Old_Chunking_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Old_Chunking_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2b2504faa9eb3f5d0ea3830483c9366e0ee6cd1 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Old_Chunking_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/PDF_Ingestion_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/PDF_Ingestion_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e677bb258f77443c029f41058ed09da19bc9bdbe Binary files /dev/null and b/App_Function_Libraries/__pycache__/PDF_Ingestion_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/SQLite_DB.cpython-312.pyc b/App_Function_Libraries/__pycache__/SQLite_DB.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbb4c6642ee8e5866d6ea5d036d5bb35d489af77 Binary files /dev/null and b/App_Function_Libraries/__pycache__/SQLite_DB.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Summarization_General_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Summarization_General_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd3e29a5e13cde1345ae9d67b499273a324c9c44 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Summarization_General_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/System_Checks_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/System_Checks_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..982bdcbb30b1a6f1f4ab42f9bd9a08c87335e227 Binary files /dev/null and b/App_Function_Libraries/__pycache__/System_Checks_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Tokenization_Methods_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Tokenization_Methods_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..650f3db5426c39e36e1fbfc0da5676697fa00037 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Tokenization_Methods_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Utils.cpython-312.pyc b/App_Function_Libraries/__pycache__/Utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25426ca180e306b2c7bdddf00953469e5e17590e Binary files /dev/null and b/App_Function_Libraries/__pycache__/Utils.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/Video_DL_Ingestion_Lib.cpython-312.pyc b/App_Function_Libraries/__pycache__/Video_DL_Ingestion_Lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7e1672373204281af1817ba5cc11a5a59188d8e4 Binary files /dev/null and b/App_Function_Libraries/__pycache__/Video_DL_Ingestion_Lib.cpython-312.pyc differ diff --git a/App_Function_Libraries/__pycache__/__init__.cpython-312.pyc b/App_Function_Libraries/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d9e956224da92e5cbd4c9981785a8408709a811 Binary files /dev/null and b/App_Function_Libraries/__pycache__/__init__.cpython-312.pyc differ diff --git a/App_Function_Libraries/models/config.yaml b/App_Function_Libraries/models/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dae87c82ce56188f49be4ae8b3ebee8fbae36d9c --- /dev/null +++ b/App_Function_Libraries/models/config.yaml @@ -0,0 +1,21 @@ +version: 3.1.0 + +pipeline: + name: pyannote.audio.pipelines.SpeakerDiarization + params: + clustering: AgglomerativeClustering + # embedding: pyannote/wespeaker-voxceleb-resnet34-LM # If you want to use the HF model + embedding: pyannote_model_wespeaker-voxceleb-resnet34-LM.bin # If you want to use the local model + embedding_batch_size: 32 + embedding_exclude_overlap: true + # segmentation: pyannote/segmentation-3.0 # If you want to use the HF model + segmentation: pyannote_model_segmentation-3.0.bin # If you want to use the local model + segmentation_batch_size: 32 + +params: + clustering: + method: centroid + min_cluster_size: 12 + threshold: 0.7045654963945799 + segmentation: + min_duration_off: 0.0 diff --git a/app.py b/app.py index 106e0d229542122b49212112058565300ff9ffea..db6ba9bbe60c7181160c77ecb517b742546d1cda 100644 --- a/app.py +++ b/app.py @@ -1,910 +1,384 @@ -# Huggingface app.py file -# I just dumped the code from everything into this file -# sue me. - -# Article_Extractor_Lib.py -######################################### -# Article Extraction Library -# This library is used to handle scraping and extraction of articles from web pages. -# Currently, uses a combination of beatifulsoup4 and trafilatura to extract article text. -# Firecrawl would be a better option for this, but it is not yet implemented. -#### - -#################### -# Function List +#!/usr/bin/env python3 +# Std Lib Imports +import argparse +import atexit +import json +import logging +import os +import signal +import sys +import time +import webbrowser # -# 1. get_page_title(url) -# 2. get_article_text(url) -# 3. get_article_title(article_url_arg) +# Local Library Imports +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'App_Function_Libraries'))) +from App_Function_Libraries.Book_Ingestion_Lib import ingest_folder, ingest_text_file +from App_Function_Libraries.Chunk_Lib import semantic_chunk_long_file#, rolling_summarize_function, +from App_Function_Libraries.Gradio_Related import launch_ui +from App_Function_Libraries.Local_LLM_Inference_Engine_Lib import cleanup_process, local_llm_function +from App_Function_Libraries.Local_Summarization_Lib import summarize_with_llama, summarize_with_kobold, \ + summarize_with_oobabooga, summarize_with_tabbyapi, summarize_with_vllm, summarize_with_local_llm +from App_Function_Libraries.Summarization_General_Lib import summarize_with_openai, summarize_with_anthropic, \ + summarize_with_cohere, summarize_with_groq, summarize_with_openrouter, summarize_with_deepseek, \ + summarize_with_huggingface, perform_transcription, perform_summarization +from App_Function_Libraries.Audio_Transcription_Lib import convert_to_wav, speech_to_text +from App_Function_Libraries.Local_File_Processing_Lib import read_paths_from_file, process_local_file +from App_Function_Libraries.SQLite_DB import add_media_to_database, is_valid_url +from App_Function_Libraries.System_Checks_Lib import cuda_check, platform_check, check_ffmpeg +from App_Function_Libraries.Utils import load_and_log_configs, sanitize_filename, create_download_directory, extract_text_from_segments +from App_Function_Libraries.Video_DL_Ingestion_Lib import download_video, extract_video_info # -#################### - - - -# Import necessary libraries -import os -import logging -import huggingface_hub -import tokenizers -import torchvision -import transformers -# 3rd-Party Imports -import asyncio -import playwright -from playwright.async_api import async_playwright -from bs4 import BeautifulSoup +# 3rd-Party Module Imports import requests -import trafilatura -from typing import Callable, Dict, List, Optional, Tuple - - -####################################################################################################################### -# Function Definitions +# OpenAI Tokenizer support # - -def get_page_title(url: str) -> str: - try: - response = requests.get(url) - response.raise_for_status() - soup = BeautifulSoup(response.text, 'html.parser') - title_tag = soup.find('title') - return title_tag.string.strip() if title_tag else "Untitled" - except requests.RequestException as e: - logging.error(f"Error fetching page title: {e}") - return "Untitled" - - -def get_artice_title(article_url_arg: str) -> str: - # Use beautifulsoup to get the page title - Really should be using ytdlp for this.... - article_title = get_page_title(article_url_arg) - - -def scrape_article(url): - async def fetch_html(url: str) -> str: - async with async_playwright() as p: - browser = await p.chromium.launch(headless=True) - context = await browser.new_context( - user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3") - page = await context.new_page() - await page.goto(url) - await page.wait_for_load_state("networkidle") # Wait for the network to be idle - content = await page.content() - await browser.close() - return content - - def extract_article_data(html: str) -> dict: - downloaded = trafilatura.extract(html, include_comments=False, include_tables=False, include_images=False) - if downloaded: - metadata = trafilatura.extract_metadata(html) - if metadata: - return { - 'title': metadata.title if metadata.title else 'N/A', - 'author': metadata.author if metadata.author else 'N/A', - 'content': downloaded, - 'date': metadata.date if metadata.date else 'N/A', - } - else: - print("Metadata extraction failed.") - return None - else: - print("Content extraction failed.") - return None - - def convert_html_to_markdown(html: str) -> str: - soup = BeautifulSoup(html, 'html.parser') - # Convert each paragraph to markdown - for para in soup.find_all('p'): - para.append('\n') # Add a newline at the end of each paragraph for markdown separation - - # Use .get_text() with separator to keep paragraph separation - text = soup.get_text(separator='\n\n') - - return text - - async def fetch_and_extract_article(url: str): - html = await fetch_html(url) - print("HTML Content:", html[:500]) # Print first 500 characters of the HTML for inspection - article_data = extract_article_data(html) - if article_data: - article_data['content'] = convert_html_to_markdown(article_data['content']) - return article_data - else: - return None - - # Using asyncio.run to handle event loop creation and execution - article_data = asyncio.run(fetch_and_extract_article(url)) - return article_data - +# Other Tokenizers # +####################### +# Logging Setup # -####################################################################################################################### - - -# Article_Summarization_Lib.py -######################################### -# Article Summarization Library -# This library is used to handle summarization of articles. - +log_level = "DEBUG" +logging.basicConfig(level=getattr(logging, log_level), format='%(asctime)s - %(levelname)s - %(message)s') +os.environ["GRADIO_ANALYTICS_ENABLED"] = "False" # -#### - -#################### -# Function List +############# +# Global variables setup +custom_prompt_input = ("Above is the transcript of a video. Please read through the transcript carefully. Identify the " +"main topics that are discussed over the course of the transcript. Then, summarize the key points about each main " +"topic in bullet points. The bullet points should cover the key information conveyed about each topic in the video, " +"but should be much shorter than the full transcript. Please output your bullet point summary inside " +"tags.") # -# 1. +# Global variables +whisper_models = ["small", "medium", "small.en", "medium.en", "medium", "large", "large-v1", "large-v2", "large-v3", + "distil-large-v2", "distil-medium.en", "distil-small.en"] +server_mode = False +share_public = False # -#################### - - - -# Import necessary libraries -import datetime -from datetime import datetime -import json -import os -import logging -# 3rd-Party Imports -import bs4 -import huggingface_hub -import tokenizers -import torchvision -import transformers - - -####################################################################################################################### -# Function Definitions # +####################### -def ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date, custom_prompt): - try: - # Check if content is not empty or whitespace - if not content.strip(): - raise ValueError("Content is empty.") - - db = Database() - create_tables() - keyword_list = keywords.split(',') if keywords else ["default"] - keyword_str = ', '.join(keyword_list) - - # Set default values for missing fields - url = url or 'Unknown' - title = title or 'Unknown' - author = author or 'Unknown' - keywords = keywords or 'default' - summary = summary or 'No summary available' - ingestion_date = ingestion_date or datetime.datetime.now().strftime('%Y-%m-%d') - - # Log the values of all fields before calling add_media_with_keywords - logging.debug(f"URL: {url}") - logging.debug(f"Title: {title}") - logging.debug(f"Author: {author}") - logging.debug(f"Content: {content[:50]}... (length: {len(content)})") # Log first 50 characters of content - logging.debug(f"Keywords: {keywords}") - logging.debug(f"Summary: {summary}") - logging.debug(f"Ingestion Date: {ingestion_date}") - logging.debug(f"Custom Prompt: {custom_prompt}") - - # Check if any required field is empty and log the specific missing field - if not url: - logging.error("URL is missing.") - raise ValueError("URL is missing.") - if not title: - logging.error("Title is missing.") - raise ValueError("Title is missing.") - if not content: - logging.error("Content is missing.") - raise ValueError("Content is missing.") - if not keywords: - logging.error("Keywords are missing.") - raise ValueError("Keywords are missing.") - if not summary: - logging.error("Summary is missing.") - raise ValueError("Summary is missing.") - if not ingestion_date: - logging.error("Ingestion date is missing.") - raise ValueError("Ingestion date is missing.") - if not custom_prompt: - logging.error("Custom prompt is missing.") - raise ValueError("Custom prompt is missing.") - - # Add media with keywords to the database - result = add_media_with_keywords( - url=url, - title=title, - media_type='article', - content=content, - keywords=keyword_str or "article_default", - prompt=custom_prompt or None, - summary=summary or "No summary generated", - transcription_model=None, # or some default value if applicable - author=author or 'Unknown', - ingestion_date=ingestion_date - ) - return result - except Exception as e: - logging.error(f"Failed to ingest article to the database: {e}") - return str(e) - - -def scrape_and_summarize(url, custom_prompt_arg, api_name, api_key, keywords, custom_article_title): - # Step 1: Scrape the article - article_data = scrape_article(url) - print(f"Scraped Article Data: {article_data}") # Debugging statement - if not article_data: - return "Failed to scrape the article." - - # Use the custom title if provided, otherwise use the scraped title - title = custom_article_title.strip() if custom_article_title else article_data.get('title', 'Untitled') - author = article_data.get('author', 'Unknown') - content = article_data.get('content', '') - ingestion_date = datetime.now().strftime('%Y-%m-%d') - - print(f"Title: {title}, Author: {author}, Content Length: {len(content)}") # Debugging statement - - # Custom prompt for the article - article_custom_prompt = custom_prompt_arg or "Summarize this article." - - # Step 2: Summarize the article - summary = None - if api_name: - logging.debug(f"Article_Summarizer: Summarization being performed by {api_name}") - - # Sanitize filename for saving the JSON file - sanitized_title = sanitize_filename(title) - json_file_path = os.path.join("Results", f"{sanitized_title}_segments.json") - - with open(json_file_path, 'w') as json_file: - json.dump([{'text': content}], json_file, indent=2) - - try: - if api_name.lower() == 'openai': - # def summarize_with_openai(api_key, input_data, custom_prompt_arg) - summary = summarize_with_openai(api_key, json_file_path, article_custom_prompt) - - elif api_name.lower() == "anthropic": - # def summarize_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5): - summary = summarize_with_anthropic(api_key, json_file_path, article_custom_prompt) - elif api_name.lower() == "cohere": - # def summarize_with_cohere(api_key, input_data, model, custom_prompt_arg) - summary = summarize_with_cohere(api_key, json_file_path, article_custom_prompt) - - elif api_name.lower() == "groq": - logging.debug(f"MAIN: Trying to summarize with groq") - # def summarize_with_groq(api_key, input_data, model, custom_prompt_arg): - summary = summarize_with_groq(api_key, json_file_path, article_custom_prompt) +####################### +# Function Sections +# +abc_xyz = """ + Database Setup + Config Loading + System Checks + DataBase Functions + Processing Paths and local file handling + Video Download/Handling + Audio Transcription + Diarization + Chunking-related Techniques & Functions + Tokenization-related Techniques & Functions + Summarizers + Gradio UI + Main +""" +# +# +####################### +####################### +# +# TL/DW: Too Long Didn't Watch +# +# Project originally created by https://github.com/the-crypt-keeper +# Modifications made by https://github.com/rmusser01 +# All credit to the original authors, I've just glued shit together. +# +# +# Usage: +# +# Download Audio only from URL -> Transcribe audio: +# python summarize.py https://www.youtube.com/watch?v=4nd1CDZP21s` +# +# Download Audio+Video from URL -> Transcribe audio from Video:** +# python summarize.py -v https://www.youtube.com/watch?v=4nd1CDZP21s` +# +# Download Audio only from URL -> Transcribe audio -> Summarize using (`anthropic`/`cohere`/`openai`/`llama` (llama.cpp)/`ooba` (oobabooga/text-gen-webui)/`kobold` (kobold.cpp)/`tabby` (Tabbyapi)) API:** +# python summarize.py -v https://www.youtube.com/watch?v=4nd1CDZP21s -api ` - Make sure to put your API key into `config.txt` under the appropriate API variable +# +# Download Audio+Video from a list of videos in a text file (can be file paths or URLs) and have them all summarized:** +# python summarize.py ./local/file_on_your/system --api_name ` +# +# Run it as a WebApp** +# python summarize.py -gui` - This requires you to either stuff your API keys into the `config.txt` file, or pass them into the app every time you want to use it. +# Can be helpful for setting up a shared instance, but not wanting people to perform inference on your server. +# +####################### - elif api_name.lower() == "openrouter": - logging.debug(f"MAIN: Trying to summarize with OpenRouter") - # def summarize_with_openrouter(api_key, input_data, custom_prompt_arg): - summary = summarize_with_openrouter(api_key, json_file_path, article_custom_prompt) - elif api_name.lower() == "deepseek": - logging.debug(f"MAIN: Trying to summarize with DeepSeek") - # def summarize_with_deepseek(api_key, input_data, custom_prompt_arg): - summary = summarize_with_deepseek(api_key, json_file_path, article_custom_prompt) +####################### +# Random issues I've encountered and how I solved them: +# 1. Something about cuda nn library missing, even though cuda is installed... +# https://github.com/tensorflow/tensorflow/issues/54784 - Basically, installing zlib made it go away. idk. +# Or https://github.com/SYSTRAN/faster-whisper/issues/85 +# +# 2. ERROR: Could not install packages due to an OSError: [WinError 2] The system cannot find the file specified: 'C:\\Python312\\Scripts\\dateparser-download.exe' -> 'C:\\Python312\\Scripts\\dateparser-download.exe.deleteme' +# Resolved through adding --user to the pip install command +# +# 3. Windows: Could not locate cudnn_ops_infer64_8.dll. Please make sure it is in your library path! +# +# 4. +# +# 5. +# +# +# +####################### - elif api_name.lower() == "llama.cpp": - logging.debug(f"MAIN: Trying to summarize with Llama.cpp") - # def summarize_with_llama(api_url, file_path, token, custom_prompt) - summary = summarize_with_llama(json_file_path, article_custom_prompt) - elif api_name.lower() == "kobold": - logging.debug(f"MAIN: Trying to summarize with Kobold.cpp") - # def summarize_with_kobold(input_data, kobold_api_token, custom_prompt_input, api_url): - summary = summarize_with_kobold(json_file_path, api_key, article_custom_prompt) +####################### +# DB Setup - elif api_name.lower() == "ooba": - # def summarize_with_oobabooga(input_data, api_key, custom_prompt, api_url): - summary = summarize_with_oobabooga(json_file_path, api_key, article_custom_prompt) +# Handled by SQLite_DB.py - elif api_name.lower() == "tabbyapi": - # def summarize_with_tabbyapi(input_data, tabby_model, custom_prompt_input, api_key=None, api_IP): - summary = summarize_with_tabbyapi(json_file_path, article_custom_prompt) +####################### - elif api_name.lower() == "vllm": - logging.debug(f"MAIN: Trying to summarize with VLLM") - # def summarize_with_vllm(api_key, input_data, custom_prompt_input): - summary = summarize_with_vllm(json_file_path, article_custom_prompt) - elif api_name.lower() == "local-llm": - logging.debug(f"MAIN: Trying to summarize with Local LLM") - summary = summarize_with_local_llm(json_file_path, article_custom_prompt) +####################### +# Config loading +# +# 1. +# 2. +# +# +####################### - elif api_name.lower() == "huggingface": - logging.debug(f"MAIN: Trying to summarize with huggingface") - # def summarize_with_huggingface(api_key, input_data, custom_prompt_arg): - summarize_with_huggingface(api_key, json_file_path, article_custom_prompt) - # Add additional API handlers here... - except requests.exceptions.ConnectionError as e: - logging.error(f"Connection error while trying to summarize with {api_name}: {str(e)}") - if summary: - logging.info(f"Article_Summarizer: Summary generated using {api_name} API") - save_summary_to_file(summary, json_file_path) - else: - summary = "Summary not available" - logging.warning(f"Failed to generate summary using {api_name} API") +####################### +# System Startup Notice +# - else: - summary = "Article Summarization: No API provided for summarization." +# Dirty hack - sue me. - FIXME - fix this... +os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' - print(f"Summary: {summary}") # Debugging statement +whisper_models = ["small", "medium", "small.en", "medium.en", "medium", "large", "large-v1", "large-v2", "large-v3", + "distil-large-v2", "distil-medium.en", "distil-small.en"] +source_languages = { + "en": "English", + "zh": "Chinese", + "de": "German", + "es": "Spanish", + "ru": "Russian", + "ko": "Korean", + "fr": "French" +} +source_language_list = [key[0] for key in source_languages.items()] - # Step 3: Ingest the article into the database - ingestion_result = ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date, - article_custom_prompt) - return f"Title: {title}\nAuthor: {author}\nIngestion Result: {ingestion_result}\n\nSummary: {summary}\n\nArticle Contents: {content}" +def print_hello(): + print(r"""_____ _ ________ _ _ +|_ _|| | / /| _ \| | | | _ + | | | | / / | | | || | | |(_) + | | | | / / | | | || |/\| | + | | | |____ / / | |/ / \ /\ / _ + \_/ \_____//_/ |___/ \/ \/ (_) -def ingest_unstructured_text(text, custom_prompt, api_name, api_key, keywords, custom_article_title): - title = custom_article_title.strip() if custom_article_title else "Unstructured Text" - author = "Unknown" - ingestion_date = datetime.now().strftime('%Y-%m-%d') + _ _ +| | | | +| |_ ___ ___ | | ___ _ __ __ _ +| __| / _ \ / _ \ | | / _ \ | '_ \ / _` | +| |_ | (_) || (_) | | || (_) || | | || (_| | _ + \__| \___/ \___/ |_| \___/ |_| |_| \__, |( ) + __/ ||/ + |___/ + _ _ _ _ _ _ _ + | |(_) | | ( )| | | | | | + __| | _ __| | _ __ |/ | |_ __ __ __ _ | |_ ___ | |__ + / _` || | / _` || '_ \ | __| \ \ /\ / / / _` || __| / __|| '_ \ +| (_| || || (_| || | | | | |_ \ V V / | (_| || |_ | (__ | | | | + \__,_||_| \__,_||_| |_| \__| \_/\_/ \__,_| \__| \___||_| |_| +""") + time.sleep(1) + return - # Summarize the unstructured text - if api_name: - json_file_path = f"Results/{title.replace(' ', '_')}_segments.json" - with open(json_file_path, 'w') as json_file: - json.dump([{'text': text}], json_file, indent=2) - if api_name.lower() == 'openai': - summary = summarize_with_openai(api_key, json_file_path, custom_prompt) - # Add other APIs as needed - else: - summary = "Unsupported API." - else: - summary = "No API provided for summarization." +# +# +####################### - # Ingest the unstructured text into the database - ingestion_result = ingest_article_to_db('Unstructured Text', title, author, text, keywords, summary, ingestion_date, - custom_prompt) - return f"Title: {title}\nSummary: {summary}\nIngestion Result: {ingestion_result}" +####################### +# System Check Functions +# +# 1. platform_check() +# 2. cuda_check() +# 3. decide_cpugpu() +# 4. check_ffmpeg() +# 5. download_ffmpeg() +# +####################### +####################### +# DB Functions # +# create_tables() +# add_keyword() +# delete_keyword() +# add_keyword() +# add_media_with_keywords() +# search_db() +# format_results() +# search_and_display() +# export_to_csv() +# is_valid_url() +# is_valid_date() # -####################################################################################################################### +######################################################################################################################## -# Audio_Transcription_Lib.py -######################################### -# Transcription Library -# This library is used to perform transcription of audio files. -# Currently, uses faster_whisper for transcription. + +######################################################################################################################## +# Processing Paths and local file handling # -#### -import configparser -#################### # Function List +# 1. read_paths_from_file(file_path) +# 2. process_path(path) +# 3. process_local_file(file_path) +# 4. read_paths_from_file(file_path: str) -> List[str] # -# 1. convert_to_wav(video_file_path, offset=0, overwrite=False) -# 2. speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='small.en', vad_filter=False) # -#################### - - -# Import necessary libraries to run solo for testing -import json -import logging -import os -import sys -import subprocess -import time +######################################################################################################################## ####################################################################################################################### -# Function Definitions +# Online Article Extraction / Handling # - -# Convert video .m4a into .wav using ffmpeg -# ffmpeg -i "example.mp4" -ar 16000 -ac 1 -c:a pcm_s16le "output.wav" -# https://www.gyan.dev/ffmpeg/builds/ -# - - -# os.system(r'.\Bin\ffmpeg.exe -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"') -def convert_to_wav(video_file_path, offset=0, overwrite=False): - out_path = os.path.splitext(video_file_path)[0] + ".wav" - - if os.path.exists(out_path) and not overwrite: - print(f"File '{out_path}' already exists. Skipping conversion.") - logging.info(f"Skipping conversion as file already exists: {out_path}") - return out_path - print("Starting conversion process of .m4a to .WAV") - out_path = os.path.splitext(video_file_path)[0] + ".wav" - - try: - if os.name == "nt": - logging.debug("ffmpeg being ran on windows") - - if sys.platform.startswith('win'): - ffmpeg_cmd = ".\\Bin\\ffmpeg.exe" - logging.debug(f"ffmpeg_cmd: {ffmpeg_cmd}") - else: - ffmpeg_cmd = 'ffmpeg' # Assume 'ffmpeg' is in PATH for non-Windows systems - - command = [ - ffmpeg_cmd, # Assuming the working directory is correctly set where .\Bin exists - "-ss", "00:00:00", # Start at the beginning of the video - "-i", video_file_path, - "-ar", "16000", # Audio sample rate - "-ac", "1", # Number of audio channels - "-c:a", "pcm_s16le", # Audio codec - out_path - ] - try: - # Redirect stdin from null device to prevent ffmpeg from waiting for input - with open(os.devnull, 'rb') as null_file: - result = subprocess.run(command, stdin=null_file, text=True, capture_output=True) - if result.returncode == 0: - logging.info("FFmpeg executed successfully") - logging.debug("FFmpeg output: %s", result.stdout) - else: - logging.error("Error in running FFmpeg") - logging.error("FFmpeg stderr: %s", result.stderr) - raise RuntimeError(f"FFmpeg error: {result.stderr}") - except Exception as e: - logging.error("Error occurred - ffmpeg doesn't like windows") - raise RuntimeError("ffmpeg failed") - elif os.name == "posix": - os.system(f'ffmpeg -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"') - else: - raise RuntimeError("Unsupported operating system") - logging.info("Conversion to WAV completed: %s", out_path) - except subprocess.CalledProcessError as e: - logging.error("Error executing FFmpeg command: %s", str(e)) - raise RuntimeError("Error converting video file to WAV") - except Exception as e: - logging.error("speech-to-text: Error transcribing audio: %s", str(e)) - return {"error": str(e)} - return out_path - - -# Transcribe .wav into .segments.json -def speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='medium.en', vad_filter=False): - logging.info('speech-to-text: Loading faster_whisper model: %s', whisper_model) - from faster_whisper import WhisperModel - # Retrieve processing choice from the configuration file - config = configparser.ConfigParser() - config.read('config.txt') - processing_choice = config.get('Processing', 'processing_choice', fallback='cpu') - model = WhisperModel(whisper_model, device=f"{processing_choice}") - time_start = time.time() - if audio_file_path is None: - raise ValueError("speech-to-text: No audio file provided") - logging.info("speech-to-text: Audio file path: %s", audio_file_path) - - try: - _, file_ending = os.path.splitext(audio_file_path) - out_file = audio_file_path.replace(file_ending, ".segments.json") - prettified_out_file = audio_file_path.replace(file_ending, ".segments_pretty.json") - if os.path.exists(out_file): - logging.info("speech-to-text: Segments file already exists: %s", out_file) - with open(out_file) as f: - global segments - segments = json.load(f) - return segments - - logging.info('speech-to-text: Starting transcription...') - options = dict(language=selected_source_lang, beam_size=5, best_of=5, vad_filter=vad_filter) - transcribe_options = dict(task="transcribe", **options) - segments_raw, info = model.transcribe(audio_file_path, **transcribe_options) - - segments = [] - for segment_chunk in segments_raw: - chunk = { - "Time_Start": segment_chunk.start, - "Time_End": segment_chunk.end, - "Text": segment_chunk.text - } - logging.debug("Segment: %s", chunk) - segments.append(chunk) - logging.info("speech-to-text: Transcription completed with faster_whisper") - - # Create a dictionary with the 'segments' key - output_data = {'segments': segments} - - # Save prettified JSON - logging.info("speech-to-text: Saving prettified JSON to %s", prettified_out_file) - with open(prettified_out_file, 'w') as f: - json.dump(output_data, f, indent=2) - - # Save non-prettified JSON - logging.info("speech-to-text: Saving JSON to %s", out_file) - with open(out_file, 'w') as f: - json.dump(output_data, f) - - except Exception as e: - logging.error("speech-to-text: Error transcribing audio: %s", str(e)) - raise RuntimeError("speech-to-text: Error transcribing audio") - return segments - - - +# Function List +# 1. get_page_title(url) +# 2. get_article_text(url) +# 3. get_article_title(article_url_arg) # # ####################################################################################################################### -from transformers import GPT2Tokenizer -import nltk -import re - - -# FIXME - Make sure it only downloads if it already exists, and does a check first. -# Ensure NLTK data is downloaded -def ntlk_prep(): - nltk.download('punkt') - -# Load GPT2 tokenizer -tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - - -def load_document(file_path): - with open(file_path, 'r') as file: - text = file.read() - return re.sub('\s+', ' ', text).strip() - - -# Chunk based on maximum number of words, using ' ' (space) as a delimiter -def chunk_text_by_words(text, max_words=300): - words = text.split() - chunks = [' '.join(words[i:i + max_words]) for i in range(0, len(words), max_words)] - return chunks - - -# Chunk based on sentences, not exceeding a max amount, using nltk -def chunk_text_by_sentences(text, max_sentences=10): - sentences = nltk.tokenize.sent_tokenize(text) - chunks = [' '.join(sentences[i:i + max_sentences]) for i in range(0, len(sentences), max_sentences)] - return chunks - - -# Chunk text by paragraph, marking paragraphs by (delimiter) '\n\n' -def chunk_text_by_paragraphs(text, max_paragraphs=5): - paragraphs = text.split('\n\n') - chunks = ['\n\n'.join(paragraphs[i:i + max_paragraphs]) for i in range(0, len(paragraphs), max_paragraphs)] - return chunks - - -# Naive chunking based on token count -def chunk_text_by_tokens(text, max_tokens=1000): - tokens = tokenizer.encode(text) - chunks = [tokenizer.decode(tokens[i:i + max_tokens]) for i in range(0, len(tokens), max_tokens)] - return chunks - - -# Hybrid approach, chunk each sentence while ensuring total token size does not exceed a maximum number -def chunk_text_hybrid(text, max_tokens=1000): - sentences = nltk.tokenize.sent_tokenize(text) - chunks = [] - current_chunk = [] - current_length = 0 - - for sentence in sentences: - tokens = tokenizer.encode(sentence) - if current_length + len(tokens) <= max_tokens: - current_chunk.append(sentence) - current_length += len(tokens) - else: - chunks.append(' '.join(current_chunk)) - current_chunk = [sentence] - current_length = len(tokens) - - if current_chunk: - chunks.append(' '.join(current_chunk)) - - return chunks - -# Thanks openai -def chunk_on_delimiter(input_string: str, - max_tokens: int, - delimiter: str) -> list[str]: - chunks = input_string.split(delimiter) - combined_chunks, _, dropped_chunk_count = combine_chunks_with_no_minimum( - chunks, max_tokens, chunk_delimiter=delimiter, add_ellipsis_for_overflow=True) - if dropped_chunk_count > 0: - print(f"Warning: {dropped_chunk_count} chunks were dropped due to exceeding the token limit.") - combined_chunks = [f"{chunk}{delimiter}" for chunk in combined_chunks] - return combined_chunks - - -def rolling_summarize_function(text: str, - detail: float = 0, - api_name: str = None, - api_key: str = None, - model: str = None, - custom_prompt: str = None, - chunk_by_words: bool = False, - max_words: int = 300, - chunk_by_sentences: bool = False, - max_sentences: int = 10, - chunk_by_paragraphs: bool = False, - max_paragraphs: int = 5, - chunk_by_tokens: bool = False, - max_tokens: int = 1000, - summarize_recursively=False, - verbose=False): - """ - Summarizes a given text by splitting it into chunks, each of which is summarized individually. - Allows selecting the method for chunking (words, sentences, paragraphs, tokens). - - Parameters: - - text (str): The text to be summarized. - - detail (float, optional): A value between 0 and 1 indicating the desired level of detail in the summary. - - api_name (str, optional): Name of the API to use for summarization. - - api_key (str, optional): API key for the specified API. - - model (str, optional): Model identifier for the summarization engine. - - custom_prompt (str, optional): Custom prompt for the summarization. - - chunk_by_words (bool, optional): If True, chunks the text by words. - - max_words (int, optional): Maximum number of words per chunk. - - chunk_by_sentences (bool, optional): If True, chunks the text by sentences. - - max_sentences (int, optional): Maximum number of sentences per chunk. - - chunk_by_paragraphs (bool, optional): If True, chunks the text by paragraphs. - - max_paragraphs (int, optional): Maximum number of paragraphs per chunk. - - chunk_by_tokens (bool, optional): If True, chunks the text by tokens. - - max_tokens (int, optional): Maximum number of tokens per chunk. - - summarize_recursively (bool, optional): If True, summaries are generated recursively. - - verbose (bool, optional): If verbose, prints additional output. - - Returns: - - str: The final compiled summary of the text. - """ - - def extract_text_from_segments(segments): - text = ' '.join([segment['Text'] for segment in segments if 'Text' in segment]) - return text - # Validate input - if not text: - raise ValueError("Input text cannot be empty.") - if any([max_words <= 0, max_sentences <= 0, max_paragraphs <= 0, max_tokens <= 0]): - raise ValueError("All maximum chunk size parameters must be positive integers.") - global segments - - if isinstance(text, dict) and 'transcription' in text: - text = extract_text_from_segments(text['transcription']) - elif isinstance(text, list): - text = extract_text_from_segments(text) - - # Select the chunking function based on the method specified - if chunk_by_words: - chunks = chunk_text_by_words(text, max_words) - elif chunk_by_sentences: - chunks = chunk_text_by_sentences(text, max_sentences) - elif chunk_by_paragraphs: - chunks = chunk_text_by_paragraphs(text, max_paragraphs) - elif chunk_by_tokens: - chunks = chunk_text_by_tokens(text, max_tokens) - else: - chunks = [text] - - # Process each chunk for summarization - accumulated_summaries = [] - for chunk in chunks: - if summarize_recursively and accumulated_summaries: - # Creating a structured prompt for recursive summarization - previous_summaries = '\n\n'.join(accumulated_summaries) - user_message_content = f"Previous summaries:\n\n{previous_summaries}\n\nText to summarize next:\n\n{chunk}" - else: - # Directly passing the chunk for summarization without recursive context - user_message_content = chunk - - # Extracting the completion from the response - try: - if api_name.lower() == 'openai': - # def summarize_with_openai(api_key, input_data, custom_prompt_arg) - summary = summarize_with_openai(user_message_content, text, custom_prompt) - - elif api_name.lower() == "anthropic": - # def summarize_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5): - summary = summarize_with_anthropic(user_message_content, text, custom_prompt) - elif api_name.lower() == "cohere": - # def summarize_with_cohere(api_key, input_data, model, custom_prompt_arg) - summary = summarize_with_cohere(user_message_content, text, custom_prompt) - - elif api_name.lower() == "groq": - logging.debug(f"MAIN: Trying to summarize with groq") - # def summarize_with_groq(api_key, input_data, model, custom_prompt_arg): - summary = summarize_with_groq(user_message_content, text, custom_prompt) - - elif api_name.lower() == "openrouter": - logging.debug(f"MAIN: Trying to summarize with OpenRouter") - # def summarize_with_openrouter(api_key, input_data, custom_prompt_arg): - summary = summarize_with_openrouter(user_message_content, text, custom_prompt) - - elif api_name.lower() == "deepseek": - logging.debug(f"MAIN: Trying to summarize with DeepSeek") - # def summarize_with_deepseek(api_key, input_data, custom_prompt_arg): - summary = summarize_with_deepseek(api_key, user_message_content,custom_prompt) - - elif api_name.lower() == "llama.cpp": - logging.debug(f"MAIN: Trying to summarize with Llama.cpp") - # def summarize_with_llama(api_url, file_path, token, custom_prompt) - summary = summarize_with_llama(user_message_content, custom_prompt) - elif api_name.lower() == "kobold": - logging.debug(f"MAIN: Trying to summarize with Kobold.cpp") - # def summarize_with_kobold(input_data, kobold_api_token, custom_prompt_input, api_url): - summary = summarize_with_kobold(user_message_content, api_key, custom_prompt) - - elif api_name.lower() == "ooba": - # def summarize_with_oobabooga(input_data, api_key, custom_prompt, api_url): - summary = summarize_with_oobabooga(user_message_content, api_key, custom_prompt) - - elif api_name.lower() == "tabbyapi": - # def summarize_with_tabbyapi(input_data, tabby_model, custom_prompt_input, api_key=None, api_IP): - summary = summarize_with_tabbyapi(user_message_content, custom_prompt) - - elif api_name.lower() == "vllm": - logging.debug(f"MAIN: Trying to summarize with VLLM") - # def summarize_with_vllm(api_key, input_data, custom_prompt_input): - summary = summarize_with_vllm(user_message_content, custom_prompt) - - elif api_name.lower() == "local-llm": - logging.debug(f"MAIN: Trying to summarize with Local LLM") - summary = summarize_with_local_llm(user_message_content, custom_prompt) - - elif api_name.lower() == "huggingface": - logging.debug(f"MAIN: Trying to summarize with huggingface") - # def summarize_with_huggingface(api_key, input_data, custom_prompt_arg): - summarize_with_huggingface(api_key, user_message_content, custom_prompt) - # Add additional API handlers here... - else: - logging.warning(f"Unsupported API: {api_name}") - summary = None - except requests.exceptions.ConnectionError: - logging.error("Connection error while summarizing") - summary = None - except Exception as e: - logging.error(f"Error summarizing with {api_name}: {str(e)}") - summary = None - - if summary: - logging.info(f"Summary generated using {api_name} API") - accumulated_summaries.append(summary) - else: - logging.warning(f"Failed to generate summary using {api_name} API") - - # Compile final summary from partial summaries - final_summary = '\n\n'.join(accumulated_summaries) - return final_summary - - - -# Sample text for testing -sample_text = """ -Natural language processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence -concerned with the interactions between computers and human language, in particular how to program computers -to process and analyze large amounts of natural language data. The result is a computer capable of "understanding" -the contents of documents, including the contextual nuances of the language within them. The technology can then -accurately extract information and insights contained in the documents as well as categorize and organize the documents themselves. - -Challenges in natural language processing frequently involve speech recognition, natural language understanding, -and natural language generation. - -Natural language processing has its roots in the 1950s. Already in 1950, Alan Turing published an article titled -"Computing Machinery and Intelligence" which proposed what is now called the Turing test as a criterion of intelligence. -""" - -# Example usage of different chunking methods -# print("Chunking by words:") -# print(chunk_text_by_words(sample_text, max_words=50)) -# -# print("\nChunking by sentences:") -# print(chunk_text_by_sentences(sample_text, max_sentences=2)) +####################################################################################################################### +# Video Download/Handling +# Video-DL-Ingestion-Lib # -# print("\nChunking by paragraphs:") -# print(chunk_text_by_paragraphs(sample_text, max_paragraphs=1)) +# Function List +# 1. get_video_info(url) +# 2. create_download_directory(title) +# 3. sanitize_filename(title) +# 4. normalize_title(title) +# 5. get_youtube(video_url) +# 6. get_playlist_videos(playlist_url) +# 7. download_video(video_url, download_path, info_dict, download_video_flag) +# 8. save_to_file(video_urls, filename) +# 9. save_summary_to_file(summary, file_path) +# 10. process_url(url, num_speakers, whisper_model, custom_prompt, offset, api_name, api_key, vad_filter, download_video, download_audio, rolling_summarization, detail_level, question_box, keywords, ) # FIXME - UPDATE # -# print("\nChunking by tokens:") -# print(chunk_text_by_tokens(sample_text, max_tokens=50)) # -# print("\nHybrid chunking:") -# print(chunk_text_hybrid(sample_text, max_tokens=50)) - - - +####################################################################################################################### -# Local_File_Processing_Lib.py -######################################### -# Local File Processing and File Path Handling Library -# This library is used to handle processing local filepaths and URLs. -# It checks for the OS, the availability of the GPU, and the availability of the ffmpeg executable. -# If the GPU is available, it asks the user if they would like to use it for processing. -# If ffmpeg is not found, it asks the user if they would like to download it. -# The script will exit if the user chooses not to download ffmpeg. -#### -#################### +####################################################################################################################### +# Audio Transcription +# # Function List +# 1. convert_to_wav(video_file_path, offset=0, overwrite=False) +# 2. speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='small.en', vad_filter=False) # -# 1. read_paths_from_file(file_path) -# 2. process_path(path) -# 3. process_local_file(file_path) -# 4. read_paths_from_file(file_path: str) -> List[str] # -#################### - -# Import necessary libraries -import os -import logging +####################################################################################################################### ####################################################################################################################### -# Function Definitions +# Diarization +# +# Function List 1. speaker_diarize(video_file_path, segments, embedding_model = "pyannote/embedding", +# embedding_size=512, num_speakers=0) +# # +####################################################################################################################### -def read_paths_from_file(file_path): - """ Reads a file containing URLs or local file paths and returns them as a list. """ - paths = [] # Initialize paths as an empty list - with open(file_path, 'r') as file: - paths = file.readlines() - return [path.strip() for path in paths] +####################################################################################################################### +# Chunking-related Techniques & Functions +# +# +# FIXME +# +# +####################################################################################################################### -def process_path(path): - """ Decides whether the path is a URL or a local file and processes accordingly. """ - if path.startswith('http'): - logging.debug("file is a URL") - # For YouTube URLs, modify to download and extract info - return get_youtube(path) - elif os.path.exists(path): - logging.debug("File is a path") - # For local files, define a function to handle them - return process_local_file(path) - else: - logging.error(f"Path does not exist: {path}") - return None +####################################################################################################################### +# Tokenization-related Functions +# +# # FIXME -def process_local_file(file_path): - logging.info(f"Processing local file: {file_path}") - file_extension = os.path.splitext(file_path)[1].lower() - - if file_extension == '.txt': - # Handle text file containing URLs - with open(file_path, 'r') as file: - urls = file.read().splitlines() - return None, None, urls - else: - # Handle video file - title = normalize_title(os.path.splitext(os.path.basename(file_path))[0]) - info_dict = {'title': title} - logging.debug(f"Creating {title} directory...") - download_path = create_download_directory(title) - logging.debug(f"Converting '{title}' to an audio file (wav).") - audio_file = convert_to_wav(file_path) - logging.debug(f"'{title}' successfully converted to an audio file (wav).") - return download_path, info_dict, audio_file +# +# +####################################################################################################################### +####################################################################################################################### +# Website-related Techniques & Functions +# +# +# +# +####################################################################################################################### +####################################################################################################################### +# Summarizers +# +# Function List +# 1. extract_text_from_segments(segments: List[Dict]) -> str +# 2. summarize_with_openai(api_key, file_path, custom_prompt_arg) +# 3. summarize_with_anthropic(api_key, file_path, model, custom_prompt_arg, max_retries=3, retry_delay=5) +# 4. summarize_with_cohere(api_key, file_path, model, custom_prompt_arg) +# 5. summarize_with_groq(api_key, file_path, model, custom_prompt_arg) +# +################################# +# Local Summarization # +# Function List +# +# 1. summarize_with_local_llm(file_path, custom_prompt_arg) +# 2. summarize_with_llama(api_url, file_path, token, custom_prompt) +# 3. summarize_with_kobold(api_url, file_path, kobold_api_token, custom_prompt) +# 4. summarize_with_oobabooga(api_url, file_path, ooba_api_token, custom_prompt) +# 5. summarize_with_vllm(vllm_api_url, vllm_api_key_function_arg, llm_model, text, vllm_custom_prompt_function_arg) +# 6. summarize_with_tabbyapi(tabby_api_key, tabby_api_IP, text, tabby_model, custom_prompt) +# 7. save_summary_to_file(summary, file_path) # ####################################################################################################################### +####################################################################################################################### +# Summarization with Detail +# +# FIXME - see 'Old_Chunking_Lib.py' +# +# +####################################################################################################################### -# Local_LLM_Inference_Engine_Lib.py -######################################### -# Local LLM Inference Engine Library -# This library is used to handle downloading, configuring, and launching the Local LLM Inference Engine -# via (llama.cpp via llamafile) +####################################################################################################################### +# Gradio UI # # -#### -import atexit -import hashlib -#################### -# Function List # +# +# +################################################################################################################# +# +####################################################################################################################### +# Local LLM Setup / Running +# +# Function List # 1. download_latest_llamafile(repo, asset_name_prefix, output_filename) # 2. download_file(url, dest_path, expected_checksum=None, max_retries=3, delay=5) # 3. verify_checksum(file_path, expected_checksum) @@ -915,5591 +389,546 @@ import hashlib # 8. launch_in_new_terminal_linux(executable, args) # 9. launch_in_new_terminal_mac(executable, args) # -#################### - -# Import necessary libraries -import json -import logging -from multiprocessing import Process as MpProcess -import requests -import sys -import os -# Import 3rd-pary Libraries -import gradio as gr -from tqdm import tqdm - +# +####################################################################################################################### ####################################################################################################################### -# Function Definitions +# Helper Functions for Main() & process_url() +# # +# +####################################################################################################################### -# Download latest llamafile from Github - # Example usage - #repo = "Mozilla-Ocho/llamafile" - #asset_name_prefix = "llamafile-" - #output_filename = "llamafile" - #download_latest_llamafile(repo, asset_name_prefix, output_filename) -# THIS SHOULD ONLY BE CALLED IF THE USER IS USING THE GUI TO SETUP LLAMAFILE -# Function is used to download only llamafile -def download_latest_llamafile_no_model(output_filename): - # Check if the file already exists - print("Checking for and downloading Llamafile it it doesn't already exist...") - if os.path.exists(output_filename): - print("Llamafile already exists. Skipping download.") - logging.debug(f"{output_filename} already exists. Skipping download.") - llamafile_exists = True - else: - llamafile_exists = False +###################################################################################################################### +# Main() +# - if llamafile_exists == True: - pass - else: - # Establish variables for Llamafile download - repo = "Mozilla-Ocho/llamafile" - asset_name_prefix = "llamafile-" - # Get the latest release information - latest_release_url = f"https://api.github.com/repos/{repo}/releases/latest" - response = requests.get(latest_release_url) - if response.status_code != 200: - raise Exception(f"Failed to fetch latest release info: {response.status_code}") +def main(input_path, api_name=None, api_key=None, + num_speakers=2, + whisper_model="small.en", + offset=0, + vad_filter=False, + download_video_flag=False, + custom_prompt=None, + overwrite=False, + rolling_summarization=False, + detail=0.01, + keywords=None, + llm_model=None, + time_based=False, + set_chunk_txt_by_words=False, + set_max_txt_chunk_words=0, + set_chunk_txt_by_sentences=False, + set_max_txt_chunk_sentences=0, + set_chunk_txt_by_paragraphs=False, + set_max_txt_chunk_paragraphs=0, + set_chunk_txt_by_tokens=False, + set_max_txt_chunk_tokens=0, + ingest_text_file=False, + chunk=False, + max_chunk_size=2000, + chunk_overlap=100, + chunk_unit='tokens', + summarize_chunks=None, + diarize=False + ): + global detail_level_number, summary, audio_file, transcription_text, info_dict - latest_release_data = response.json() - tag_name = latest_release_data['tag_name'] + detail_level = detail - # Get the release details using the tag name - release_details_url = f"https://api.github.com/repos/{repo}/releases/tags/{tag_name}" - response = requests.get(release_details_url) - if response.status_code != 200: - raise Exception(f"Failed to fetch release details for tag {tag_name}: {response.status_code}") + print(f"Keywords: {keywords}") - release_data = response.json() - assets = release_data.get('assets', []) + if not input_path: + return [] - # Find the asset with the specified prefix - asset_url = None - for asset in assets: - if re.match(f"{asset_name_prefix}.*", asset['name']): - asset_url = asset['browser_download_url'] - break + start_time = time.monotonic() + paths = [input_path] if not os.path.isfile(input_path) else read_paths_from_file(input_path) + results = [] - if not asset_url: - raise Exception(f"No asset found with prefix {asset_name_prefix}") + for path in paths: + try: + if path.startswith('http'): + info_dict, title = extract_video_info(path) + download_path = create_download_directory(title) + video_path = download_video(path, download_path, info_dict, download_video_flag) - # Download the asset - response = requests.get(asset_url) - if response.status_code != 200: - raise Exception(f"Failed to download asset: {response.status_code}") + if video_path: + if diarize: + audio_file, segments = perform_transcription(video_path, offset, whisper_model, vad_filter, diarize=True) + transcription_text = {'audio_file': audio_file, 'transcription': segments} + else: + audio_file, segments = perform_transcription(video_path, offset, whisper_model, vad_filter) + transcription_text = {'audio_file': audio_file, 'transcription': segments} - print("Llamafile downloaded successfully.") - logging.debug("Main: Llamafile downloaded successfully.") + # FIXME rolling summarization + if rolling_summarization == True: + pass + # text = extract_text_from_segments(segments) + # detail = detail_level + # additional_instructions = custom_prompt_input + # chunk_text_by_words = set_chunk_txt_by_words + # max_words = set_max_txt_chunk_words + # chunk_text_by_sentences = set_chunk_txt_by_sentences + # max_sentences = set_max_txt_chunk_sentences + # chunk_text_by_paragraphs = set_chunk_txt_by_paragraphs + # max_paragraphs = set_max_txt_chunk_paragraphs + # chunk_text_by_tokens = set_chunk_txt_by_tokens + # max_tokens = set_max_txt_chunk_tokens + # # FIXME + # summarize_recursively = rolling_summarization + # verbose = False + # model = None + # summary = rolling_summarize_function(text, detail, api_name, api_key, model, custom_prompt_input, + # chunk_text_by_words, + # max_words, chunk_text_by_sentences, + # max_sentences, chunk_text_by_paragraphs, + # max_paragraphs, chunk_text_by_tokens, + # max_tokens, summarize_recursively, verbose + # ) - # Save the file - with open(output_filename, 'wb') as file: - file.write(response.content) - logging.debug(f"Downloaded {output_filename} from {asset_url}") - print(f"Downloaded {output_filename} from {asset_url}") - return output_filename + elif api_name: + summary = perform_summarization(api_name, transcription_text, custom_prompt_input, api_key) + else: + summary = None + if summary: + # Save the summary file in the download_path directory + summary_file_path = os.path.join(download_path, f"{transcription_text}_summary.txt") + with open(summary_file_path, 'w') as file: + file.write(summary) -# FIXME - Add option in GUI for selecting the other models for download -# Should only be called from 'local_llm_gui_function' - if its called from anywhere else, shits broken. -# Function is used to download llamafile + A model from Huggingface -def download_latest_llamafile_through_gui(repo, asset_name_prefix, output_filename): - # Check if the file already exists - print("Checking for and downloading Llamafile it it doesn't already exist...") - if os.path.exists(output_filename): - print("Llamafile already exists. Skipping download.") - logging.debug(f"{output_filename} already exists. Skipping download.") - llamafile_exists = True - else: - llamafile_exists = False + add_media_to_database(path, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model) + else: + logging.error(f"Failed to download video: {path}") - if llamafile_exists == True: - pass - else: - # Get the latest release information - latest_release_url = f"https://api.github.com/repos/{repo}/releases/latest" - response = requests.get(latest_release_url) - if response.status_code != 200: - raise Exception(f"Failed to fetch latest release info: {response.status_code}") + # FIXME - make sure this doesn't break ingesting multiple videos vs multiple text files + # FIXME - Need to update so that chunking is fully handled. + elif chunk and path.lower().endswith('.txt'): + chunks = semantic_chunk_long_file(path, max_chunk_size, chunk_overlap) + if chunks: + chunks_data = { + "file_path": path, + "chunk_unit": chunk_unit, + "max_chunk_size": max_chunk_size, + "chunk_overlap": chunk_overlap, + "chunks": [] + } + summaries_data = { + "file_path": path, + "summarization_method": summarize_chunks, + "summaries": [] + } - latest_release_data = response.json() - tag_name = latest_release_data['tag_name'] - - # Get the release details using the tag name - release_details_url = f"https://api.github.com/repos/{repo}/releases/tags/{tag_name}" - response = requests.get(release_details_url) - if response.status_code != 200: - raise Exception(f"Failed to fetch release details for tag {tag_name}: {response.status_code}") - - release_data = response.json() - assets = release_data.get('assets', []) - - # Find the asset with the specified prefix - asset_url = None - for asset in assets: - if re.match(f"{asset_name_prefix}.*", asset['name']): - asset_url = asset['browser_download_url'] - break - - if not asset_url: - raise Exception(f"No asset found with prefix {asset_name_prefix}") - - # Download the asset - response = requests.get(asset_url) - if response.status_code != 200: - raise Exception(f"Failed to download asset: {response.status_code}") - - print("Llamafile downloaded successfully.") - logging.debug("Main: Llamafile downloaded successfully.") - - # Save the file - with open(output_filename, 'wb') as file: - file.write(response.content) - - logging.debug(f"Downloaded {output_filename} from {asset_url}") - print(f"Downloaded {output_filename} from {asset_url}") + for i, chunk_text in enumerate(chunks): + chunk_info = { + "chunk_id": i + 1, + "text": chunk_text + } + chunks_data["chunks"].append(chunk_info) + + if summarize_chunks: + summary = None + if summarize_chunks == 'openai': + summary = summarize_with_openai(api_key, chunk_text, custom_prompt) + elif summarize_chunks == 'anthropic': + summary = summarize_with_anthropic(api_key, chunk_text, custom_prompt) + elif summarize_chunks == 'cohere': + summary = summarize_with_cohere(api_key, chunk_text, custom_prompt) + elif summarize_chunks == 'groq': + summary = summarize_with_groq(api_key, chunk_text, custom_prompt) + elif summarize_chunks == 'local-llm': + summary = summarize_with_local_llm(chunk_text, custom_prompt) + # FIXME - Add more summarization methods as needed - # Check to see if the LLM already exists, and if not, download the LLM - print("Checking for and downloading LLM from Huggingface if needed...") - logging.debug("Main: Checking and downloading LLM from Huggingface if needed...") - mistral_7b_instruct_v0_2_q8_0_llamafile = "mistral-7b-instruct-v0.2.Q8_0.llamafile" - Samantha_Mistral_Instruct_7B_Bulleted_Notes_Q8 = "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" - Phi_3_mini_128k_instruct_Q8_0_gguf = "Phi-3-mini-128k-instruct-Q8_0.gguf" - if os.path.exists(mistral_7b_instruct_v0_2_q8_0_llamafile): - llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" - print("Model is already downloaded. Skipping download.") - pass - elif os.path.exists(Samantha_Mistral_Instruct_7B_Bulleted_Notes_Q8): - llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" - print("Model is already downloaded. Skipping download.") - pass - elif os.path.exists(mistral_7b_instruct_v0_2_q8_0_llamafile): - llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" - print("Model is already downloaded. Skipping download.") - pass - else: - logging.debug("Main: Checking and downloading LLM from Huggingface if needed...") - print("Downloading LLM from Huggingface...") - time.sleep(1) - print("Gonna be a bit...") - time.sleep(1) - print("Like seriously, an 8GB file...") - time.sleep(2) - # Not needed for GUI - # dl_check = input("Final chance to back out, hit 'N'/'n' to cancel, or 'Y'/'y' to continue: ") - #if dl_check == "N" or dl_check == "n": - # exit() - x = 2 - if x != 1: - print("Uhhhh how'd you get here...?") - exit() - else: - print("Downloading LLM from Huggingface...") - # Establish hash values for LLM models - mistral_7b_instruct_v0_2_q8_gguf_sha256 = "f326f5f4f137f3ad30f8c9cc21d4d39e54476583e8306ee2931d5a022cb85b06" - samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 = "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" - mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 = "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" - global llm_choice + if summary: + summary_info = { + "chunk_id": i + 1, + "summary": summary + } + summaries_data["summaries"].append(summary_info) + else: + logging.warning(f"Failed to generate summary for chunk {i + 1}") + + # Save chunks to a single JSON file + chunks_file_path = f"{path}_chunks.json" + with open(chunks_file_path, 'w', encoding='utf-8') as f: + json.dump(chunks_data, f, ensure_ascii=False, indent=2) + logging.info(f"All chunks saved to {chunks_file_path}") + + # Save summaries to a single JSON file (if summarization was performed) + if summarize_chunks: + summaries_file_path = f"{path}_summaries.json" + with open(summaries_file_path, 'w', encoding='utf-8') as f: + json.dump(summaries_data, f, ensure_ascii=False, indent=2) + logging.info(f"All summaries saved to {summaries_file_path}") + + logging.info(f"File {path} chunked into {len(chunks)} parts using {chunk_unit} as the unit.") + else: + logging.error(f"Failed to chunk file {path}") - # FIXME - llm_choice - llm_choice = 2 - llm_choice = input("Which LLM model would you like to download? 1. Mistral-7B-Instruct-v0.2-GGUF or 2. Samantha-Mistral-Instruct-7B-Bulleted-Notes) (plain or 'custom') or MS Flavor: Phi-3-mini-128k-instruct-Q8_0.gguf \n\n\tPress '1' or '2' or '3' to specify: ") - while llm_choice != "1" and llm_choice != "2" and llm_choice != "3": - print("Invalid choice. Please try again.") - if llm_choice == "1": - llm_download_model = "Mistral-7B-Instruct-v0.2-Q8.llamafile" - mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 = "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" - llm_download_model_hash = mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 - llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" - llamafile_llm_output_filename = "mistral-7b-instruct-v0.2.Q8_0.llamafile" - download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) - elif llm_choice == "2": - llm_download_model = "Samantha-Mistral-Instruct-7B-Bulleted-Notes-Q8.gguf" - samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 = "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" - llm_download_model_hash = samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 - llamafile_llm_output_filename = "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" - llamafile_llm_url = "https://huggingface.co/cognitivetech/samantha-mistral-instruct-7b-bulleted-notes-GGUF/resolve/main/samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf?download=true" - download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) - elif llm_choice == "3": - llm_download_model = "Phi-3-mini-128k-instruct-Q8_0.gguf" - Phi_3_mini_128k_instruct_Q8_0_gguf_sha256 = "6817b66d1c3c59ab06822e9732f0e594eea44e64cae2110906eac9d17f75d193" - llm_download_model_hash = Phi_3_mini_128k_instruct_Q8_0_gguf_sha256 - llamafile_llm_output_filename = "Phi-3-mini-128k-instruct-Q8_0.gguf" - llamafile_llm_url = "https://huggingface.co/gaianet/Phi-3-mini-128k-instruct-GGUF/resolve/main/Phi-3-mini-128k-instruct-Q8_0.gguf?download=true" - download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) - elif llm_choice == "4": # FIXME - and meta_Llama_3_8B_Instruct_Q8_0_llamafile_exists == False: - meta_Llama_3_8B_Instruct_Q8_0_llamafile_sha256 = "406868a97f02f57183716c7e4441d427f223fdbc7fa42964ef10c4d60dd8ed37" - llm_download_model_hash = meta_Llama_3_8B_Instruct_Q8_0_llamafile_sha256 - llamafile_llm_output_filename = " Meta-Llama-3-8B-Instruct.Q8_0.llamafile" - llamafile_llm_url = "https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/resolve/main/Meta-Llama-3-8B-Instruct.Q8_0.llamafile?download=true" + # Handle downloading of URLs from a text file or processing local video/audio files else: - print("Invalid choice. Please try again.") - return output_filename - - -# Maybe replace/ dead code? FIXME -# Function is used to download llamafile + A model from Huggingface -def download_latest_llamafile(repo, asset_name_prefix, output_filename): - # Check if the file already exists - print("Checking for and downloading Llamafile it it doesn't already exist...") - if os.path.exists(output_filename): - print("Llamafile already exists. Skipping download.") - logging.debug(f"{output_filename} already exists. Skipping download.") - llamafile_exists = True - else: - llamafile_exists = False - - if llamafile_exists == True: - pass - else: - # Get the latest release information - latest_release_url = f"https://api.github.com/repos/{repo}/releases/latest" - response = requests.get(latest_release_url) - if response.status_code != 200: - raise Exception(f"Failed to fetch latest release info: {response.status_code}") + download_path, info_dict, urls_or_media_file = process_local_file(path) + if isinstance(urls_or_media_file, list): + # Text file containing URLs + for url in urls_or_media_file: + for item in urls_or_media_file: + if item.startswith(('http://', 'https://')): + info_dict, title = extract_video_info(url) + download_path = create_download_directory(title) + video_path = download_video(url, download_path, info_dict, download_video_flag) + + if video_path: + if diarize: + audio_file, segments = perform_transcription(video_path, offset, whisper_model, vad_filter, diarize=True) + else: + audio_file, segments = perform_transcription(video_path, offset, whisper_model, vad_filter) + + transcription_text = {'audio_file': audio_file, 'transcription': segments} + if rolling_summarization: + text = extract_text_from_segments(segments) + # FIXME + #summary = summarize_with_detail_openai(text, detail=detail) + elif api_name: + summary = perform_summarization(api_name, transcription_text, custom_prompt_input, api_key) + else: + summary = None + + if summary: + # Save the summary file in the download_path directory + summary_file_path = os.path.join(download_path, f"{transcription_text}_summary.txt") + with open(summary_file_path, 'w') as file: + file.write(summary) + + add_media_to_database(url, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model) + else: + logging.error(f"Failed to download video: {url}") - latest_release_data = response.json() - tag_name = latest_release_data['tag_name'] + else: + # Video or audio or txt file + media_path = urls_or_media_file - # Get the release details using the tag name - release_details_url = f"https://api.github.com/repos/{repo}/releases/tags/{tag_name}" - response = requests.get(release_details_url) - if response.status_code != 200: - raise Exception(f"Failed to fetch release details for tag {tag_name}: {response.status_code}") + if media_path.lower().endswith(('.txt', '.md')): + if media_path.lower().endswith('.txt'): + # Handle text file ingestion + result = ingest_text_file(media_path) + logging.info(result) + elif media_path.lower().endswith(('.mp4', '.avi', '.mov')): + if diarize: + audio_file, segments = perform_transcription(media_path, offset, whisper_model, vad_filter, diarize=True) + else: + audio_file, segments = perform_transcription(media_path, offset, whisper_model, vad_filter) + elif media_path.lower().endswith(('.wav', '.mp3', '.m4a')): + if diarize: + segments = speech_to_text(media_path, whisper_model=whisper_model, vad_filter=vad_filter, diarize=True) + else: + segments = speech_to_text(media_path, whisper_model=whisper_model, vad_filter=vad_filter) + else: + logging.error(f"Unsupported media file format: {media_path}") + continue - release_data = response.json() - assets = release_data.get('assets', []) + transcription_text = {'media_path': path, 'audio_file': media_path, 'transcription': segments} - # Find the asset with the specified prefix - asset_url = None - for asset in assets: - if re.match(f"{asset_name_prefix}.*", asset['name']): - asset_url = asset['browser_download_url'] - break + # FIXME + if rolling_summarization: + # text = extract_text_from_segments(segments) + # summary = summarize_with_detail_openai(text, detail=detail) + pass + elif api_name: + summary = perform_summarization(api_name, transcription_text, custom_prompt_input, api_key) + else: + summary = None - if not asset_url: - raise Exception(f"No asset found with prefix {asset_name_prefix}") + if summary: + # Save the summary file in the download_path directory + summary_file_path = os.path.join(download_path, f"{transcription_text}_summary.txt") + with open(summary_file_path, 'w') as file: + file.write(summary) - # Download the asset - response = requests.get(asset_url) - if response.status_code != 200: - raise Exception(f"Failed to download asset: {response.status_code}") + add_media_to_database(path, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model) - print("Llamafile downloaded successfully.") - logging.debug("Main: Llamafile downloaded successfully.") + except Exception as e: + logging.error(f"Error processing {path}: {str(e)}") + continue - # Save the file - with open(output_filename, 'wb') as file: - file.write(response.content) + return transcription_text - logging.debug(f"Downloaded {output_filename} from {asset_url}") - print(f"Downloaded {output_filename} from {asset_url}") - # Check to see if the LLM already exists, and if not, download the LLM - print("Checking for and downloading LLM from Huggingface if needed...") - logging.debug("Main: Checking and downloading LLM from Huggingface if needed...") - mistral_7b_instruct_v0_2_q8_0_llamafile = "mistral-7b-instruct-v0.2.Q8_0.llamafile" - Samantha_Mistral_Instruct_7B_Bulleted_Notes_Q8 = "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" - Phi_3_mini_128k_instruct_Q8_0_gguf = "Phi-3-mini-128k-instruct-Q8_0.gguf" - if os.path.exists(mistral_7b_instruct_v0_2_q8_0_llamafile): - llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" - print("Model is already downloaded. Skipping download.") - pass - elif os.path.exists(Samantha_Mistral_Instruct_7B_Bulleted_Notes_Q8): - llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" - print("Model is already downloaded. Skipping download.") - pass - elif os.path.exists(mistral_7b_instruct_v0_2_q8_0_llamafile): - llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" - print("Model is already downloaded. Skipping download.") - pass - else: - logging.debug("Main: Checking and downloading LLM from Huggingface if needed...") - print("Downloading LLM from Huggingface...") - time.sleep(1) - print("Gonna be a bit...") - time.sleep(1) - print("Like seriously, an 8GB file...") - time.sleep(2) - dl_check = input("Final chance to back out, hit 'N'/'n' to cancel, or 'Y'/'y' to continue: ") - if dl_check == "N" or dl_check == "n": - exit() - else: - print("Downloading LLM from Huggingface...") - # Establish hash values for LLM models - mistral_7b_instruct_v0_2_q8_gguf_sha256 = "f326f5f4f137f3ad30f8c9cc21d4d39e54476583e8306ee2931d5a022cb85b06" - samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 = "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" - mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 = "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" +def signal_handler(sig, frame): + logging.info('Signal handler called with signal: %s', sig) + cleanup_process() + sys.exit(0) - # FIXME - llm_choice - llm_choice = 2 - llm_choice = input("Which LLM model would you like to download? 1. Mistral-7B-Instruct-v0.2-GGUF or 2. Samantha-Mistral-Instruct-7B-Bulleted-Notes) (plain or 'custom') or MS Flavor: Phi-3-mini-128k-instruct-Q8_0.gguf \n\n\tPress '1' or '2' or '3' to specify: ") - while llm_choice != "1" and llm_choice != "2" and llm_choice != "3": - print("Invalid choice. Please try again.") - if llm_choice == "1": - llm_download_model = "Mistral-7B-Instruct-v0.2-Q8.llamafile" - mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 = "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" - llm_download_model_hash = mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 - llamafile_llm_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" - llamafile_llm_output_filename = "mistral-7b-instruct-v0.2.Q8_0.llamafile" - download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) - elif llm_choice == "2": - llm_download_model = "Samantha-Mistral-Instruct-7B-Bulleted-Notes-Q8.gguf" - samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 = "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" - llm_download_model_hash = samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 - llamafile_llm_output_filename = "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" - llamafile_llm_url = "https://huggingface.co/cognitivetech/samantha-mistral-instruct-7b_bulleted-notes_GGUF/resolve/main/samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf?download=true" - download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) - elif llm_choice == "3": - llm_download_model = "Phi-3-mini-128k-instruct-Q8_0.gguf" - Phi_3_mini_128k_instruct_Q8_0_gguf_sha256 = "6817b66d1c3c59ab06822e9732f0e594eea44e64cae2110906eac9d17f75d193" - llm_download_model_hash = Phi_3_mini_128k_instruct_Q8_0_gguf_sha256 - llamafile_llm_output_filename = "Phi-3-mini-128k-instruct-Q8_0.gguf" - llamafile_llm_url = "https://huggingface.co/gaianet/Phi-3-mini-128k-instruct-GGUF/resolve/main/Phi-3-mini-128k-instruct-Q8_0.gguf?download=true" - download_file(llamafile_llm_url, llamafile_llm_output_filename, llm_download_model_hash) - elif llm_choice == "4": # FIXME - and meta_Llama_3_8B_Instruct_Q8_0_llamafile_exists == False: - meta_Llama_3_8B_Instruct_Q8_0_llamafile_sha256 = "406868a97f02f57183716c7e4441d427f223fdbc7fa42964ef10c4d60dd8ed37" - llm_download_model_hash = meta_Llama_3_8B_Instruct_Q8_0_llamafile_sha256 - llamafile_llm_output_filename = " Meta-Llama-3-8B-Instruct.Q8_0.llamafile" - llamafile_llm_url = "https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/resolve/main/Meta-Llama-3-8B-Instruct.Q8_0.llamafile?download=true" - else: - print("Invalid choice. Please try again.") - return output_filename +############################## MAIN ############################## +# +# -def download_file(url, dest_path, expected_checksum=None, max_retries=3, delay=5): - temp_path = dest_path + '.tmp' +if __name__ == "__main__": + # Register signal handlers + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) - for attempt in range(max_retries): - try: - # Check if a partial download exists and get its size - resume_header = {} - if os.path.exists(temp_path): - resume_header = {'Range': f'bytes={os.path.getsize(temp_path)}-'} + # Logging setup + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') - response = requests.get(url, stream=True, headers=resume_header) - response.raise_for_status() + # Load Config + loaded_config_data = load_and_log_configs() - # Get the total file size from headers - total_size = int(response.headers.get('content-length', 0)) - initial_pos = os.path.getsize(temp_path) if os.path.exists(temp_path) else 0 + if loaded_config_data: + logging.info("Main: Configuration loaded successfully") + # You can access the configuration data like this: + # print(f"OpenAI API Key: {config_data['api_keys']['openai']}") + # print(f"Anthropic Model: {config_data['models']['anthropic']}") + # print(f"Kobold API IP: {config_data['local_apis']['kobold']['ip']}") + # print(f"Output Path: {config_data['output_path']}") + # print(f"Processing Choice: {config_data['processing_choice']}") + else: + print("Failed to load configuration") - mode = 'ab' if 'Range' in response.headers else 'wb' - with open(temp_path, mode) as temp_file, tqdm( - total=total_size, unit='B', unit_scale=True, desc=dest_path, initial=initial_pos, ascii=True - ) as pbar: - for chunk in response.iter_content(chunk_size=8192): - if chunk: # filter out keep-alive new chunks - temp_file.write(chunk) - pbar.update(len(chunk)) + # Print ascii_art + print_hello() - # Verify the checksum if provided - if expected_checksum: - if not verify_checksum(temp_path, expected_checksum): - os.remove(temp_path) - raise ValueError("Downloaded file's checksum does not match the expected checksum") + transcription_text = None - # Move the file to the final destination - os.rename(temp_path, dest_path) - print("Download complete and verified!") - return dest_path + parser = argparse.ArgumentParser( + description='Transcribe and summarize videos.', + epilog=''' +Sample commands: + 1. Simple Sample command structure: + summarize.py -api openai -k tag_one tag_two tag_three - except Exception as e: - print(f"Attempt {attempt + 1} failed: {e}") - if attempt < max_retries - 1: - print(f"Retrying in {delay} seconds...") - time.sleep(delay) - else: - print("Max retries reached. Download failed.") - raise + 2. Rolling Summary Sample command structure: + summarize.py -api openai -prompt "custom_prompt_goes_here-is-appended-after-transcription" -roll -detail 0.01 -k tag_one tag_two tag_three -# FIXME / IMPLEMENT FULLY -# File download verification -#mistral_7b_llamafile_instruct_v02_q8_url = "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true" -#global mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 -#mistral_7b_instruct_v0_2_q8_0_llamafile_sha256 = "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" + 3. FULL Sample command structure: + summarize.py -api openai -ns 2 -wm small.en -off 0 -vad -log INFO -prompt "custom_prompt" -overwrite -roll -detail 0.01 -k tag_one tag_two tag_three -#mistral_7b_v02_instruct_model_q8_gguf_url = "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q8_0.gguf?download=true" -#global mistral_7b_instruct_v0_2_q8_gguf_sha256 -#mistral_7b_instruct_v0_2_q8_gguf_sha256 = "f326f5f4f137f3ad30f8c9cc21d4d39e54476583e8306ee2931d5a022cb85b06" + 4. Sample command structure for UI: + summarize.py -gui -log DEBUG + ''', + formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument('input_path', type=str, help='Path or URL of the video', nargs='?') + parser.add_argument('-v', '--video', action='store_true', help='Download the video instead of just the audio') + parser.add_argument('-api', '--api_name', type=str, help='API name for summarization (optional)') + parser.add_argument('-key', '--api_key', type=str, help='API key for summarization (optional)') + parser.add_argument('-ns', '--num_speakers', type=int, default=2, help='Number of speakers (default: 2)') + parser.add_argument('-wm', '--whisper_model', type=str, default='small', + help='Whisper model (default: small)| Options: tiny.en, tiny, base.en, base, small.en, small, medium.en, ' + 'medium, large-v1, large-v2, large-v3, large, distil-large-v2, distil-medium.en, ' + 'distil-small.en') + parser.add_argument('-off', '--offset', type=int, default=0, help='Offset in seconds (default: 0)') + parser.add_argument('-vad', '--vad_filter', action='store_true', help='Enable VAD filter') + parser.add_argument('-log', '--log_level', type=str, default='INFO', + choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Log level (default: INFO)') + parser.add_argument('-gui', '--user_interface', action='store_true', help="Launch the Gradio user interface") + parser.add_argument('-demo', '--demo_mode', action='store_true', help='Enable demo mode') + parser.add_argument('-prompt', '--custom_prompt', type=str, + help='Pass in a custom prompt to be used in place of the existing one.\n (Probably should just ' + 'modify the script itself...)') + parser.add_argument('-overwrite', '--overwrite', action='store_true', help='Overwrite existing files') + parser.add_argument('-roll', '--rolling_summarization', action='store_true', help='Enable rolling summarization') + parser.add_argument('-detail', '--detail_level', type=float, help='Mandatory if rolling summarization is enabled, ' + 'defines the chunk size.\n Default is 0.01(lots ' + 'of chunks) -> 1.00 (few chunks)\n Currently ' + 'only OpenAI works. ', + default=0.01, ) + parser.add_argument('-model', '--llm_model', type=str, default='', + help='Model to use for LLM summarization (only used for vLLM/TabbyAPI)') + parser.add_argument('-k', '--keywords', nargs='+', default=['cli_ingest_no_tag'], + help='Keywords for tagging the media, can use multiple separated by spaces (default: cli_ingest_no_tag)') + parser.add_argument('--log_file', type=str, help='Where to save logfile (non-default)') + parser.add_argument('--local_llm', action='store_true', + help="Use a local LLM from the script(Downloads llamafile from github and 'mistral-7b-instruct-v0.2.Q8' - 8GB model from Huggingface)") + parser.add_argument('--server_mode', action='store_true', + help='Run in server mode (This exposes the GUI/Server to the network)') + parser.add_argument('-share', '--share_public', action='store_true', help="This will use Gradio's built-in ngrok tunneling to share the server publicly on the internet."), + parser.add_argument('--port', type=int, default=7860, help='Port to run the server on') + parser.add_argument('--ingest_text_file', action='store_true', + help='Ingest .txt files as content instead of treating them as URL lists') + parser.add_argument('--text_title', type=str, help='Title for the text file being ingested') + parser.add_argument('--text_author', type=str, help='Author of the text file being ingested') + parser.add_argument('--diarize', action='store_true', help='Enable speaker diarization') + # parser.add_argument('--offload', type=int, default=20, help='Numbers of layers to offload to GPU for Llamafile usage') + # parser.add_argument('-o', '--output_path', type=str, help='Path to save the output file') -#samantha_instruct_model_q8_gguf_url = "https://huggingface.co/cognitivetech/samantha-mistral-instruct-7b_bulleted-notes_GGUF/resolve/main/samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf?download=true" -#global samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 -#samantha_mistral_instruct_7b_bulleted_notes_q8_0_gguf_sha256 = "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" + args = parser.parse_args() + # Set Chunking values/variables + set_chunk_txt_by_words = False + set_max_txt_chunk_words = 0 + set_chunk_txt_by_sentences = False + set_max_txt_chunk_sentences = 0 + set_chunk_txt_by_paragraphs = False + set_max_txt_chunk_paragraphs = 0 + set_chunk_txt_by_tokens = False + set_max_txt_chunk_tokens = 0 + if args.share_public or args.share: + share_public = args.share_public + else: + share_public = None + if args.server_mode: + server_mode = args.server_mode + else: + server_mode = None + if args.server_mode is True: + server_mode = True + if args.port: + server_port = args.port + else: + server_port = None -def verify_checksum(file_path, expected_checksum): - sha256_hash = hashlib.sha256() - with open(file_path, 'rb') as f: - for byte_block in iter(lambda: f.read(4096), b''): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() == expected_checksum + ########## Logging setup + logger = logging.getLogger() + logger.setLevel(getattr(logging, args.log_level)) -process = None -# Function to close out llamafile process on script exit. -def cleanup_process(): - global process - if process is not None: - process.kill() - logging.debug("Main: Terminated the external process") + # Create console handler + console_handler = logging.StreamHandler() + console_handler.setLevel(getattr(logging, args.log_level)) + console_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + console_handler.setFormatter(console_formatter) + if args.log_file: + # Create file handler + file_handler = logging.FileHandler(args.log_file) + file_handler.setLevel(getattr(logging, args.log_level)) + file_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + file_handler.setFormatter(file_formatter) + logger.addHandler(file_handler) + logger.info(f"Log file created at: {args.log_file}") -def signal_handler(sig, frame): - logging.info('Signal handler called with signal: %s', sig) - cleanup_process() - sys.exit(0) + ########## Custom Prompt setup + custom_prompt_input = args.custom_prompt + if not args.custom_prompt: + logging.debug("No custom prompt defined, will use default") + args.custom_prompt_input = ( + "\n\nabove is the transcript of a video. " + "Please read through the transcript carefully. Identify the main topics that are " + "discussed over the course of the transcript. Then, summarize the key points about each " + "main topic in a concise bullet point. The bullet points should cover the key " + "information conveyed about each topic in the video, but should be much shorter than " + "the full transcript. Please output your bullet point summary inside " + "tags." + ) + print("No custom prompt defined, will use default") -# FIXME - Add callout to gradio UI -def local_llm_function(): - global process - repo = "Mozilla-Ocho/llamafile" - asset_name_prefix = "llamafile-" - useros = os.name - if useros == "nt": - output_filename = "llamafile.exe" + custom_prompt_input = args.custom_prompt else: - output_filename = "llamafile" - print( - "WARNING - Checking for existence of llamafile and HuggingFace model, downloading if needed...This could be a while") - print("WARNING - and I mean a while. We're talking an 8 Gigabyte model here...") - print("WARNING - Hope you're comfy. Or it's already downloaded.") - time.sleep(6) - logging.debug("Main: Checking and downloading Llamafile from Github if needed...") - llamafile_path = download_latest_llamafile(repo, asset_name_prefix, output_filename) - logging.debug("Main: Llamafile downloaded successfully.") + logging.debug(f"Custom prompt defined, will use \n\nf{custom_prompt_input} \n\nas the prompt") + print(f"Custom Prompt has been defined. Custom prompt: \n\n {args.custom_prompt}") - # FIXME - llm_choice - global llm_choice - llm_choice = 1 - # Launch the llamafile in an external process with the specified argument - if llm_choice == 1: - arguments = ["--ctx-size", "8192 ", " -m", "mistral-7b-instruct-v0.2.Q8_0.llamafile"] - elif llm_choice == 2: - arguments = ["--ctx-size", "8192 ", " -m", "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf"] - elif llm_choice == 3: - arguments = ["--ctx-size", "8192 ", " -m", "Phi-3-mini-128k-instruct-Q8_0.gguf"] - elif llm_choice == 4: - arguments = ["--ctx-size", "8192 ", " -m", "llama-3"] # FIXME + # Check if the user wants to use the local LLM from the script + local_llm = args.local_llm + logging.info(f'Local LLM flag: {local_llm}') - try: - logging.info("Main: Launching the LLM (llamafile) in an external terminal window...") - if useros == "nt": - launch_in_new_terminal_windows(llamafile_path, arguments) - elif useros == "posix": - launch_in_new_terminal_linux(llamafile_path, arguments) + # Check if the user wants to ingest a text file (singular or multiple from a folder) + if args.input_path is not None: + if os.path.isdir(args.input_path) and args.ingest_text_file: + results = ingest_folder(args.input_path, keywords=args.keywords) + for result in results: + print(result) + elif args.input_path.lower().endswith('.txt') and args.ingest_text_file: + result = ingest_text_file(args.input_path, title=args.text_title, author=args.text_author, + keywords=args.keywords) + print(result) + sys.exit(0) + + # Launch the GUI + if args.user_interface: + if local_llm: + local_llm_function() + time.sleep(2) + webbrowser.open_new_tab('http://127.0.0.1:7860') + launch_ui(share_public=False) + elif share_public is not None: + if local_llm: + local_llm_function() + time.sleep(2) + webbrowser.open_new_tab('http://127.0.0.1:7860') else: - launch_in_new_terminal_mac(llamafile_path, arguments) - # FIXME - pid doesn't exist in this context - #logging.info(f"Main: Launched the {llamafile_path} with PID {process.pid}") - atexit.register(cleanup_process, process) - except Exception as e: - logging.error(f"Failed to launch the process: {e}") - print(f"Failed to launch the process: {e}") - + launch_ui(share_public=True) + elif not args.input_path: + parser.print_help() + sys.exit(1) -# This function is used to dl a llamafile binary + the Samantha Mistral Finetune model. -# It should only be called when the user is using the GUI to set up and interact with Llamafile. -def local_llm_gui_function(am_noob, verbose_checked, threads_checked, threads_value, http_threads_checked, http_threads_value, - model_checked, model_value, hf_repo_checked, hf_repo_value, hf_file_checked, hf_file_value, - ctx_size_checked, ctx_size_value, ngl_checked, ngl_value, host_checked, host_value, port_checked, - port_value): - # Identify running OS - useros = os.name - if useros == "nt": - output_filename = "llamafile.exe" else: - output_filename = "llamafile" + logging.info('Starting the transcription and summarization process.') + logging.info(f'Input path: {args.input_path}') + logging.info(f'API Name: {args.api_name}') + logging.info(f'Number of speakers: {args.num_speakers}') + logging.info(f'Whisper model: {args.whisper_model}') + logging.info(f'Offset: {args.offset}') + logging.info(f'VAD filter: {args.vad_filter}') + logging.info(f'Log Level: {args.log_level}') + logging.info(f'Demo Mode: {args.demo_mode}') + logging.info(f'Custom Prompt: {args.custom_prompt}') + logging.info(f'Overwrite: {args.overwrite}') + logging.info(f'Rolling Summarization: {args.rolling_summarization}') + logging.info(f'User Interface: {args.user_interface}') + logging.info(f'Video Download: {args.video}') + # logging.info(f'Save File location: {args.output_path}') + # logging.info(f'Log File location: {args.log_file}') - # Build up the commands for llamafile - built_up_args = [] + global api_name + api_name = args.api_name - # Identify if the user wants us to do everything for them - if am_noob == True: - print("You're a noob. (lol j/k; they're good settings)") + summary = None # Initialize to ensure it's always defined + if args.detail_level == None: + args.detail_level = 0.01 - # Setup variables for Model download from HF - repo = "Mozilla-Ocho/llamafile" - asset_name_prefix = "llamafile-" - print( - "WARNING - Checking for existence of llamafile or HuggingFace model (GGUF type), downloading if needed...This could be a while") - print("WARNING - and I mean a while. We're talking an 8 Gigabyte model here...") - print("WARNING - Hope you're comfy. Or it's already downloaded.") - time.sleep(6) - logging.debug("Main: Checking for Llamafile and downloading from Github if needed...\n\tAlso checking for a " - "local LLM model...\n\tDownloading if needed...\n\tThis could take a while...\n\tWill be the " - "'samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf' model...") - llamafile_path = download_latest_llamafile_through_gui(repo, asset_name_prefix, output_filename) - logging.debug("Main: Llamafile downloaded successfully.") + # FIXME + # if args.api_name and args.rolling_summarization and any( + # key.startswith(args.api_name) and value is not None for key, value in api_keys.items()): + # logging.info(f'MAIN: API used: {args.api_name}') + # logging.info('MAIN: Rolling Summarization will be performed.') - arguments = [] - # FIXME - llm_choice - # This is the gui, we can add this as options later - llm_choice = 2 - # Launch the llamafile in an external process with the specified argument - if llm_choice == 1: - arguments = ["--ctx-size", "8192 ", " -m", "mistral-7b-instruct-v0.2.Q8_0.llamafile"] - elif llm_choice == 2: - arguments = """--ctx-size 8192 -m samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf""" - elif llm_choice == 3: - arguments = ["--ctx-size", "8192 ", " -m", "Phi-3-mini-128k-instruct-Q8_0.gguf"] - elif llm_choice == 4: - arguments = ["--ctx-size", "8192 ", " -m", "llama-3"] + elif args.api_name: + logging.info(f'MAIN: API used: {args.api_name}') + logging.info('MAIN: Summarization (not rolling) will be performed.') - try: - logging.info("Main(Local-LLM-GUI-noob): Launching the LLM (llamafile) in an external terminal window...") + else: + logging.info('No API specified. Summarization will not be performed.') - if useros == "nt": - command = 'start cmd /k "llamafile.exe --ctx-size 8192 -m samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf"' - subprocess.Popen(command, shell=True) - elif useros == "posix": - command = "llamafile --ctx-size 8192 -m samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" - subprocess.Popen(command, shell=True) - else: - command = "llamafile.exe --ctx-size 8192 -m samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf" - subprocess.Popen(command, shell=True) - # FIXME - pid doesn't exist in this context - # logging.info(f"Main: Launched the {llamafile_path} with PID {process.pid}") - atexit.register(cleanup_process, process) - except Exception as e: - logging.error(f"Failed to launch the process: {e}") - print(f"Failed to launch the process: {e}") + logging.debug("Platform check being performed...") + platform_check() + logging.debug("CUDA check being performed...") + cuda_check() + processing_choice = "cpu" + logging.debug("ffmpeg check being performed...") + check_ffmpeg() + # download_ffmpeg() - else: - print("You're not a noob.") - llamafile_path = download_latest_llamafile_no_model(output_filename) - if verbose_checked == True: - print("Verbose mode enabled.") - built_up_args.append("--verbose") - if threads_checked == True: - print(f"Threads enabled with value: {threads_value}") - built_up_args.append(f"--threads {threads_value}") - if http_threads_checked == True: - print(f"HTTP Threads enabled with value: {http_threads_value}") - built_up_args.append(f"--http-threads {http_threads_value}") - if model_checked == True: - print(f"Model enabled with value: {model_value}") - built_up_args.append(f"--model {model_value}") - if hf_repo_checked == True: - print(f"Huggingface repo enabled with value: {hf_repo_value}") - built_up_args.append(f"--hf-repo {hf_repo_value}") - if hf_file_checked == True: - print(f"Huggingface file enabled with value: {hf_file_value}") - built_up_args.append(f"--hf-file {hf_file_value}") - if ctx_size_checked == True: - print(f"Context size enabled with value: {ctx_size_value}") - built_up_args.append(f"--ctx-size {ctx_size_value}") - if ngl_checked == True: - print(f"NGL enabled with value: {ngl_value}") - built_up_args.append(f"--ngl {ngl_value}") - if host_checked == True: - print(f"Host enabled with value: {host_value}") - built_up_args.append(f"--host {host_value}") - if port_checked == True: - print(f"Port enabled with value: {port_value}") - built_up_args.append(f"--port {port_value}") + llm_model = args.llm_model or None + # FIXME - dirty hack + args.time_based = False - # Lets go ahead and finally launch the bastard... try: - logging.info("Main(Local-LLM-GUI-Main): Launching the LLM (llamafile) in an external terminal window...") - if useros == "nt": - launch_in_new_terminal_windows(llamafile_path, built_up_args) - elif useros == "posix": - launch_in_new_terminal_linux(llamafile_path, built_up_args) - else: - launch_in_new_terminal_mac(llamafile_path, built_up_args) - # FIXME - pid doesn't exist in this context - #logging.info(f"Main: Launched the {llamafile_path} with PID {process.pid}") - atexit.register(cleanup_process, process) - except Exception as e: - logging.error(f"Failed to launch the process: {e}") - print(f"Failed to launch the process: {e}") - - -# Launch the executable in a new terminal window # FIXME - really should figure out a cleaner way of doing this... -def launch_in_new_terminal_windows(executable, args): - command = f'start cmd /k "{executable} {" ".join(args)}"' - subprocess.Popen(command, shell=True) - - -# FIXME -def launch_in_new_terminal_linux(executable, args): - command = f'gnome-terminal -- {executable} {" ".join(args)}' - subprocess.Popen(command, shell=True) - - -# FIXME -def launch_in_new_terminal_mac(executable, args): - command = f'open -a Terminal.app {executable} {" ".join(args)}' - subprocess.Popen(command, shell=True) - - -# Local_Summarization_Lib.py -######################################### -# Local Summarization Library -# This library is used to perform summarization with a 'local' inference engine. -# -#### - -#################### -# Function List -# FIXME - UPDATE Function Arguments -# 1. summarize_with_local_llm(text, custom_prompt_arg) -# 2. summarize_with_llama(api_url, text, token, custom_prompt) -# 3. summarize_with_kobold(api_url, text, kobold_api_token, custom_prompt) -# 4. summarize_with_oobabooga(api_url, text, ooba_api_token, custom_prompt) -# 5. summarize_with_vllm(vllm_api_url, vllm_api_key_function_arg, llm_model, text, vllm_custom_prompt_function_arg) -# 6. summarize_with_tabbyapi(tabby_api_key, tabby_api_IP, text, tabby_model, custom_prompt) -# 7. save_summary_to_file(summary, file_path) -# -# -#################### - - -# Import necessary libraries -import os -import logging -from typing import Callable - -from openai import OpenAI - - - -####################################################################################################################### -# Function Definitions -# - -def summarize_with_local_llm(input_data, custom_prompt_arg): - try: - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("Local LLM: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("openai: Using provided string data for summarization") - data = input_data - - logging.debug(f"Local LLM: Loaded data: {data}") - logging.debug(f"Local LLM: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("Local LLM: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("Invalid input data format") - - headers = { - 'Content-Type': 'application/json' - } - - logging.debug("Local LLM: Preparing data + prompt for submittal") - local_llm_prompt = f"{text} \n\n\n\n{custom_prompt_arg}" - data = { - "messages": [ - { - "role": "system", - "content": "You are a professional summarizer." - }, - { - "role": "user", - "content": local_llm_prompt - } - ], - "max_tokens": 28000, # Adjust tokens as needed - } - logging.debug("Local LLM: Posting request") - response = requests.post('http://127.0.0.1:8080/v1/chat/completions', headers=headers, json=data) - - if response.status_code == 200: - response_data = response.json() - if 'choices' in response_data and len(response_data['choices']) > 0: - summary = response_data['choices'][0]['message']['content'].strip() - logging.debug("Local LLM: Summarization successful") - print("Local LLM: Summarization successful.") - return summary - else: - logging.warning("Local LLM: Summary not found in the response data") - return "Local LLM: Summary not available" - else: - logging.debug("Local LLM: Summarization failed") - print("Local LLM: Failed to process summary:", response.text) - return "Local LLM: Failed to process summary" - except Exception as e: - logging.debug("Local LLM: Error in processing: %s", str(e)) - print("Error occurred while processing summary with Local LLM:", str(e)) - return "Local LLM: Error occurred while processing summary" - -def summarize_with_llama(input_data, custom_prompt, api_url="http://127.0.0.1:8080/completion", api_key=None): - loaded_config_data = load_and_log_configs() - try: - # API key validation - if api_key is None: - logging.info("llama.cpp: API key not provided as parameter") - logging.info("llama.cpp: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['llama'] - - if api_key is None or api_key.strip() == "": - logging.info("llama.cpp: API key not found or is empty") - - logging.debug(f"llama.cpp: Using API Key: {api_key[:5]}...{api_key[-5:]}") - - # Load transcript - logging.debug("llama.cpp: Loading JSON data") - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("Llama.cpp: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("Llama.cpp: Using provided string data for summarization") - data = input_data - - logging.debug(f"Llama.cpp: Loaded data: {data}") - logging.debug(f"Llama.cpp: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("Llama.cpp: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("Llama.cpp: Invalid input data format") - - headers = { - 'accept': 'application/json', - 'content-type': 'application/json', - } - if len(api_key) > 5: - headers['Authorization'] = f'Bearer {api_key}' - - llama_prompt = f"{text} \n\n\n\n{custom_prompt}" - logging.debug("llama: Prompt being sent is {llama_prompt}") - - data = { - "prompt": llama_prompt - } - - logging.debug("llama: Submitting request to API endpoint") - print("llama: Submitting request to API endpoint") - response = requests.post(api_url, headers=headers, json=data) - response_data = response.json() - logging.debug("API Response Data: %s", response_data) - - if response.status_code == 200: - # if 'X' in response_data: - logging.debug(response_data) - summary = response_data['content'].strip() - logging.debug("llama: Summarization successful") - print("Summarization successful.") - return summary - else: - logging.error(f"Llama: API request failed with status code {response.status_code}: {response.text}") - return f"Llama: API request failed: {response.text}" - - except Exception as e: - logging.error("Llama: Error in processing: %s", str(e)) - return f"Llama: Error occurred while processing summary with llama: {str(e)}" - - -# https://lite.koboldai.net/koboldcpp_api#/api%2Fv1/post_api_v1_generate -def summarize_with_kobold(input_data, api_key, custom_prompt_input, kobold_api_IP="http://127.0.0.1:5001/api/v1/generate"): - loaded_config_data = load_and_log_configs() - try: - # API key validation - if api_key is None: - logging.info("Kobold.cpp: API key not provided as parameter") - logging.info("Kobold.cpp: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['kobold'] - - if api_key is None or api_key.strip() == "": - logging.info("Kobold.cpp: API key not found or is empty") - - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("Kobold.cpp: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("Kobold.cpp: Using provided string data for summarization") - data = input_data - - logging.debug(f"Kobold.cpp: Loaded data: {data}") - logging.debug(f"Kobold.cpp: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("Kobold.cpp: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("Kobold.cpp: Invalid input data format") - - headers = { - 'accept': 'application/json', - 'content-type': 'application/json', - } - - kobold_prompt = f"{text} \n\n\n\n{custom_prompt_input}" - logging.debug("kobold: Prompt being sent is {kobold_prompt}") - - # FIXME - # Values literally c/p from the api docs.... - data = { - "max_context_length": 8096, - "max_length": 4096, - "prompt": f"{text}\n\n\n\n{custom_prompt_input}" - } - - logging.debug("kobold: Submitting request to API endpoint") - print("kobold: Submitting request to API endpoint") - response = requests.post(kobold_api_IP, headers=headers, json=data) - response_data = response.json() - logging.debug("kobold: API Response Data: %s", response_data) - - if response.status_code == 200: - if 'results' in response_data and len(response_data['results']) > 0: - summary = response_data['results'][0]['text'].strip() - logging.debug("kobold: Summarization successful") - print("Summarization successful.") - return summary - else: - logging.error("Expected data not found in API response.") - return "Expected data not found in API response." - else: - logging.error(f"kobold: API request failed with status code {response.status_code}: {response.text}") - return f"kobold: API request failed: {response.text}" - - except Exception as e: - logging.error("kobold: Error in processing: %s", str(e)) - return f"kobold: Error occurred while processing summary with kobold: {str(e)}" - - -# https://github.com/oobabooga/text-generation-webui/wiki/12-%E2%80%90-OpenAI-API -def summarize_with_oobabooga(input_data, api_key, custom_prompt, api_url="http://127.0.0.1:5000/v1/chat/completions"): - loaded_config_data = load_and_log_configs() - try: - # API key validation - if api_key is None: - logging.info("ooba: API key not provided as parameter") - logging.info("ooba: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['ooba'] - - if api_key is None or api_key.strip() == "": - logging.info("ooba: API key not found or is empty") - - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("Oobabooga: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("Oobabooga: Using provided string data for summarization") - data = input_data - - logging.debug(f"Oobabooga: Loaded data: {data}") - logging.debug(f"Oobabooga: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("Oobabooga: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("Invalid input data format") - - headers = { - 'accept': 'application/json', - 'content-type': 'application/json', - } - - # prompt_text = "I like to eat cake and bake cakes. I am a baker. I work in a French bakery baking cakes. It - # is a fun job. I have been baking cakes for ten years. I also bake lots of other baked goods, but cakes are - # my favorite." prompt_text += f"\n\n{text}" # Uncomment this line if you want to include the text variable - ooba_prompt = f"{text}" + f"\n\n\n\n{custom_prompt}" - logging.debug("ooba: Prompt being sent is {ooba_prompt}") - - data = { - "mode": "chat", - "character": "Example", - "messages": [{"role": "user", "content": ooba_prompt}] - } - - logging.debug("ooba: Submitting request to API endpoint") - print("ooba: Submitting request to API endpoint") - response = requests.post(api_url, headers=headers, json=data, verify=False) - logging.debug("ooba: API Response Data: %s", response) - - if response.status_code == 200: - response_data = response.json() - summary = response.json()['choices'][0]['message']['content'] - logging.debug("ooba: Summarization successful") - print("Summarization successful.") - return summary - else: - logging.error(f"oobabooga: API request failed with status code {response.status_code}: {response.text}") - return f"ooba: API request failed with status code {response.status_code}: {response.text}" - - except Exception as e: - logging.error("ooba: Error in processing: %s", str(e)) - return f"ooba: Error occurred while processing summary with oobabooga: {str(e)}" - - -# FIXME - Install is more trouble than care to deal with right now. -def summarize_with_tabbyapi(input_data, custom_prompt_input, api_key=None, api_IP="http://127.0.0.1:5000/v1/chat/completions"): - loaded_config_data = load_and_log_configs() - model = loaded_config_data['models']['tabby'] - # API key validation - if api_key is None: - logging.info("tabby: API key not provided as parameter") - logging.info("tabby: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['tabby'] - - if api_key is None or api_key.strip() == "": - logging.info("tabby: API key not found or is empty") - - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("tabby: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("tabby: Using provided string data for summarization") - data = input_data - - logging.debug(f"tabby: Loaded data: {data}") - logging.debug(f"tabby: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("tabby: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("Invalid input data format") - - headers = { - 'Authorization': f'Bearer {api_key}', - 'Content-Type': 'application/json' - } - data2 = { - 'text': text, - 'model': 'tabby' # Specify the model if needed - } - tabby_api_ip = loaded_config_data['local_apis']['tabby']['ip'] - try: - response = requests.post(tabby_api_ip, headers=headers, json=data2) - response.raise_for_status() - summary = response.json().get('summary', '') - return summary - except requests.exceptions.RequestException as e: - logger.error(f"Error summarizing with TabbyAPI: {e}") - return "Error summarizing with TabbyAPI." - - -# FIXME - https://docs.vllm.ai/en/latest/getting_started/quickstart.html .... Great docs. -def summarize_with_vllm(input_data, custom_prompt_input, api_key=None, vllm_api_url="http://127.0.0.1:8000/v1/chat/completions"): - loaded_config_data = load_and_log_configs() - llm_model = loaded_config_data['models']['vllm'] - # API key validation - if api_key is None: - logging.info("vLLM: API key not provided as parameter") - logging.info("vLLM: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['llama'] - - if api_key is None or api_key.strip() == "": - logging.info("vLLM: API key not found or is empty") - vllm_client = OpenAI( - base_url=vllm_api_url, - api_key=custom_prompt_input - ) - - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("vLLM: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("vLLM: Using provided string data for summarization") - data = input_data - - logging.debug(f"vLLM: Loaded data: {data}") - logging.debug(f"vLLM: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("vLLM: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("Invalid input data format") - - - custom_prompt = custom_prompt_input - - completion = client.chat.completions.create( - model=llm_model, - messages=[ - {"role": "system", "content": "You are a professional summarizer."}, - {"role": "user", "content": f"{text} \n\n\n\n{custom_prompt}"} - ] - ) - vllm_summary = completion.choices[0].message.content - return vllm_summary - - -def save_summary_to_file(summary, file_path): - logging.debug("Now saving summary to file...") - base_name = os.path.splitext(os.path.basename(file_path))[0] - summary_file_path = os.path.join(os.path.dirname(file_path), base_name + '_summary.txt') - os.makedirs(os.path.dirname(summary_file_path), exist_ok=True) - logging.debug("Opening summary file for writing, *segments.json with *_summary.txt") - with open(summary_file_path, 'w') as file: - file.write(summary) - logging.info(f"Summary saved to file: {summary_file_path}") - -# -# -####################################################################################################################### - - - -# Old_Chunking_Lib.py -######################################### -# Old Chunking Library -# This library is used to handle chunking of text for summarization. -# -#### - - - -#################### -# Function List -# -# 1. chunk_transcript(transcript: str, chunk_duration: int, words_per_second) -> List[str] -# 2. summarize_chunks(api_name: str, api_key: str, transcript: List[dict], chunk_duration: int, words_per_second: int) -> str -# 3. get_chat_completion(messages, model='gpt-4-turbo') -# 4. chunk_on_delimiter(input_string: str, max_tokens: int, delimiter: str) -> List[str] -# 5. combine_chunks_with_no_minimum(chunks: List[str], max_tokens: int, chunk_delimiter="\n\n", header: Optional[str] = None, add_ellipsis_for_overflow=False) -> Tuple[List[str], List[int]] -# 6. rolling_summarize(text: str, detail: float = 0, model: str = 'gpt-4-turbo', additional_instructions: Optional[str] = None, minimum_chunk_size: Optional[int] = 500, chunk_delimiter: str = ".", summarize_recursively=False, verbose=False) -# 7. chunk_transcript(transcript: str, chunk_duration: int, words_per_second) -> List[str] -# 8. summarize_chunks(api_name: str, api_key: str, transcript: List[dict], chunk_duration: int, words_per_second: int) -> str -# -#################### - -# Import necessary libraries -import os -from typing import Optional - -# Import 3rd party -import openai -from openai import OpenAI - - - - - -####################################################################################################################### -# Function Definitions -# - -# ######### Words-per-second Chunking ######### -# def chunk_transcript(transcript: str, chunk_duration: int, words_per_second) -> List[str]: -# words = transcript.split() -# words_per_chunk = chunk_duration * words_per_second -# chunks = [' '.join(words[i:i + words_per_chunk]) for i in range(0, len(words), words_per_chunk)] -# return chunks -# -# -# def summarize_chunks(api_name: str, api_key: str, transcript: List[dict], chunk_duration: int, -# words_per_second: int) -> str: -# # if api_name not in summarizers: # See 'summarizers' dict in the main script -# # return f"Unsupported API: {api_name}" -# -# summarizer = summarizers[api_name] -# text = extract_text_from_segments(transcript) -# chunks = chunk_transcript(text, chunk_duration, words_per_second) -# -# summaries = [] -# for chunk in chunks: -# if api_name == 'openai': -# # Ensure the correct model and prompt are passed -# summaries.append(summarizer(api_key, chunk, custom_prompt)) -# else: -# summaries.append(summarizer(api_key, chunk)) -# -# return "\n\n".join(summaries) - - -################## #################### - - -######### Token-size Chunking ######### FIXME - OpenAI only currently -# This is dirty and shameful and terrible. It should be replaced with a proper implementation. -# anyways lets get to it.... -openai_api_key = "Fake_key" # FIXME -client = OpenAI(api_key=openai_api_key) - - -def get_chat_completion(messages, model='gpt-4-turbo'): - response = client.chat.completions.create( - model=model, - messages=messages, - temperature=0, - ) - return response.choices[0].message.content - - -# This function chunks a text into smaller pieces based on a maximum token count and a delimiter -def chunk_on_delimiter(input_string: str, - max_tokens: int, - delimiter: str) -> list[str]: - chunks = input_string.split(delimiter) - combined_chunks, _, dropped_chunk_count = combine_chunks_with_no_minimum( - chunks, max_tokens, chunk_delimiter=delimiter, add_ellipsis_for_overflow=True) - if dropped_chunk_count > 0: - print(f"Warning: {dropped_chunk_count} chunks were dropped due to exceeding the token limit.") - combined_chunks = [f"{chunk}{delimiter}" for chunk in combined_chunks] - return combined_chunks - - -# This function combines text chunks into larger blocks without exceeding a specified token count. -# It returns the combined chunks, their original indices, and the number of dropped chunks due to overflow. -def combine_chunks_with_no_minimum( - chunks: list[str], - max_tokens: int, - chunk_delimiter="\n\n", - header: Optional[str] = None, - add_ellipsis_for_overflow=False, -) -> Tuple[list[str], list[int]]: - dropped_chunk_count = 0 - output = [] # list to hold the final combined chunks - output_indices = [] # list to hold the indices of the final combined chunks - candidate = ( - [] if header is None else [header] - ) # list to hold the current combined chunk candidate - candidate_indices = [] - for chunk_i, chunk in enumerate(chunks): - chunk_with_header = [chunk] if header is None else [header, chunk] - # FIXME MAKE NOT OPENAI SPECIFIC - if len(openai_tokenize(chunk_delimiter.join(chunk_with_header))) > max_tokens: - print(f"warning: chunk overflow") - if ( - add_ellipsis_for_overflow - # FIXME MAKE NOT OPENAI SPECIFIC - and len(openai_tokenize(chunk_delimiter.join(candidate + ["..."]))) <= max_tokens - ): - candidate.append("...") - dropped_chunk_count += 1 - continue # this case would break downstream assumptions - # estimate token count with the current chunk added - # FIXME MAKE NOT OPENAI SPECIFIC - extended_candidate_token_count = len(openai_tokenize(chunk_delimiter.join(candidate + [chunk]))) - # If the token count exceeds max_tokens, add the current candidate to output and start a new candidate - if extended_candidate_token_count > max_tokens: - output.append(chunk_delimiter.join(candidate)) - output_indices.append(candidate_indices) - candidate = chunk_with_header # re-initialize candidate - candidate_indices = [chunk_i] - # otherwise keep extending the candidate - else: - candidate.append(chunk) - candidate_indices.append(chunk_i) - # add the remaining candidate to output if it's not empty - if (header is not None and len(candidate) > 1) or (header is None and len(candidate) > 0): - output.append(chunk_delimiter.join(candidate)) - output_indices.append(candidate_indices) - return output, output_indices, dropped_chunk_count - - -def rolling_summarize(text: str, - detail: float = 0, - model: str = 'gpt-4-turbo', - additional_instructions: Optional[str] = None, - minimum_chunk_size: Optional[int] = 500, - chunk_delimiter: str = ".", - summarize_recursively=False, - verbose=False): - """ - Summarizes a given text by splitting it into chunks, each of which is summarized individually. - The level of detail in the summary can be adjusted, and the process can optionally be made recursive. - - Parameters: - - text (str): The text to be summarized. - - detail (float, optional): A value between 0 and 1 - indicating the desired level of detail in the summary. 0 leads to a higher level summary, and 1 results in a more - detailed summary. Defaults to 0. - - additional_instructions (Optional[str], optional): Additional instructions to provide to the - model for customizing summaries. - minimum_chunk_size (Optional[int], optional): The minimum size for text - chunks. Defaults to 500. - - chunk_delimiter (str, optional): The delimiter used to split the text into chunks. Defaults to ".". - - summarize_recursively (bool, optional): If True, summaries are generated recursively, using previous summaries for context. - - verbose (bool, optional): If True, prints detailed information about the chunking process. - Returns: - - str: The final compiled summary of the text. - - The function first determines the number of chunks by interpolating between a minimum and a maximum chunk count - based on the `detail` parameter. It then splits the text into chunks and summarizes each chunk. If - `summarize_recursively` is True, each summary is based on the previous summaries, adding more context to the - summarization process. The function returns a compiled summary of all chunks. - """ - - # check detail is set correctly - assert 0 <= detail <= 1 - - # interpolate the number of chunks based to get specified level of detail - max_chunks = len(chunk_on_delimiter(text, minimum_chunk_size, chunk_delimiter)) - min_chunks = 1 - num_chunks = int(min_chunks + detail * (max_chunks - min_chunks)) - - # adjust chunk_size based on interpolated number of chunks - # FIXME MAKE NOT OPENAI SPECIFIC - document_length = len(openai_tokenize(text)) - chunk_size = max(minimum_chunk_size, document_length // num_chunks) - text_chunks = chunk_on_delimiter(text, chunk_size, chunk_delimiter) - if verbose: - print(f"Splitting the text into {len(text_chunks)} chunks to be summarized.") - # FIXME MAKE NOT OPENAI SPECIFIC - print(f"Chunk lengths are {[len(openai_tokenize(x)) for x in text_chunks]}") - - # set system message - system_message_content = "Rewrite this text in summarized form." - if additional_instructions is not None: - system_message_content += f"\n\n{additional_instructions}" - - accumulated_summaries = [] - for chunk in tqdm(text_chunks): - if summarize_recursively and accumulated_summaries: - # Creating a structured prompt for recursive summarization - accumulated_summaries_string = '\n\n'.join(accumulated_summaries) - user_message_content = f"Previous summaries:\n\n{accumulated_summaries_string}\n\nText to summarize next:\n\n{chunk}" - else: - # Directly passing the chunk for summarization without recursive context - user_message_content = chunk - - # Constructing messages based on whether recursive summarization is applied - messages = [ - {"role": "system", "content": system_message_content}, - {"role": "user", "content": user_message_content} - ] - - # Assuming this function gets the completion and works as expected - response = get_chat_completion(messages, model=model) - accumulated_summaries.append(response) - - # Compile final summary from partial summaries - global final_summary - final_summary = '\n\n'.join(accumulated_summaries) - - return final_summary - - -####################################### - -#!/usr/bin/env python3 -# Std Lib Imports -import argparse -import asyncio -import atexit -import configparser -from datetime import datetime -import hashlib -import json -import logging -import os -from pathlib import Path -import platform -import re -import shutil -import signal -import sqlite3 -import subprocess -import sys -import time -import unicodedata -from multiprocessing import process -from typing import Callable, Dict, List, Optional, Tuple -from urllib.parse import urlparse, parse_qs, urlencode, urlunparse -import webbrowser -import zipfile - -# 3rd-Party Module Imports -from bs4 import BeautifulSoup -import gradio as gr -import nltk -from playwright.async_api import async_playwright -import requests -from requests.exceptions import RequestException -import trafilatura -import yt_dlp - -# OpenAI Tokenizer support -from openai import OpenAI -from tqdm import tqdm -import tiktoken - -# Other Tokenizers -from transformers import GPT2Tokenizer - -####################### -# Logging Setup -# - -log_level = "DEBUG" -logging.basicConfig(level=getattr(logging, log_level), format='%(asctime)s - %(levelname)s - %(message)s') -os.environ["GRADIO_ANALYTICS_ENABLED"] = "False" - -############# -# Global variables setup - -custom_prompt_input = ("Above is the transcript of a video. Please read through the transcript carefully. Identify the " -"main topics that are discussed over the course of the transcript. Then, summarize the key points about each main " -"topic in bullet points. The bullet points should cover the key information conveyed about each topic in the video, " -"but should be much shorter than the full transcript. Please output your bullet point summary inside " -"tags.") - -# -# -####################### - -####################### -# Function Sections -# - - -abc_xyz = """ - Database Setup - Config Loading - System Checks - DataBase Functions - Processing Paths and local file handling - Video Download/Handling - Audio Transcription - Diarization - Chunking-related Techniques & Functions - Tokenization-related Techniques & Functions - Summarizers - Gradio UI - Main -""" - -# -# -####################### - - -####################### -# -# TL/DW: Too Long Didn't Watch -# -# Project originally created by https://github.com/the-crypt-keeper -# Modifications made by https://github.com/rmusser01 -# All credit to the original authors, I've just glued shit together. -# -# -# Usage: -# -# Download Audio only from URL -> Transcribe audio: -# python summarize.py https://www.youtube.com/watch?v=4nd1CDZP21s` -# -# Download Audio+Video from URL -> Transcribe audio from Video:** -# python summarize.py -v https://www.youtube.com/watch?v=4nd1CDZP21s` -# -# Download Audio only from URL -> Transcribe audio -> Summarize using (`anthropic`/`cohere`/`openai`/`llama` (llama.cpp)/`ooba` (oobabooga/text-gen-webui)/`kobold` (kobold.cpp)/`tabby` (Tabbyapi)) API:** -# python summarize.py -v https://www.youtube.com/watch?v=4nd1CDZP21s -api ` - Make sure to put your API key into `config.txt` under the appropriate API variable -# -# Download Audio+Video from a list of videos in a text file (can be file paths or URLs) and have them all summarized:** -# python summarize.py ./local/file_on_your/system --api_name ` -# -# Run it as a WebApp** -# python summarize.py -gui` - This requires you to either stuff your API keys into the `config.txt` file, or pass them into the app every time you want to use it. -# Can be helpful for setting up a shared instance, but not wanting people to perform inference on your server. -# -####################### - - -####################### -# Random issues I've encountered and how I solved them: -# 1. Something about cuda nn library missing, even though cuda is installed... -# https://github.com/tensorflow/tensorflow/issues/54784 - Basically, installing zlib made it go away. idk. -# Or https://github.com/SYSTRAN/faster-whisper/issues/85 -# -# 2. ERROR: Could not install packages due to an OSError: [WinError 2] The system cannot find the file specified: 'C:\\Python312\\Scripts\\dateparser-download.exe' -> 'C:\\Python312\\Scripts\\dateparser-download.exe.deleteme' -# Resolved through adding --user to the pip install command -# -# 3. ? -# -####################### - - -####################### -# DB Setup - -# Handled by SQLite_DB.py - -####################### - - -####################### -# Config loading -# - -def load_comprehensive_config(): - # Get the directory of the current script - current_dir = os.path.dirname(os.path.abspath(__file__)) - - # Construct the path to the config file in the current directory - config_path = os.path.join(current_dir, 'config.txt') - - # Create a ConfigParser object - config = configparser.ConfigParser() - - # Read the configuration file - files_read = config.read(config_path) - - if not files_read: - raise FileNotFoundError(f"Config file not found at {config_path}") - - return config - - -def load_and_log_configs(): - try: - config = load_comprehensive_config() - if config is None: - logging.error("Config is None, cannot proceed") - return None - # API Keys - anthropic_api_key = config.get('API', 'anthropic_api_key', fallback=None) - logging.debug( - f"Loaded Anthropic API Key: {anthropic_api_key[:5]}...{anthropic_api_key[-5:] if anthropic_api_key else None}") - - cohere_api_key = config.get('API', 'cohere_api_key', fallback=None) - logging.debug( - f"Loaded Cohere API Key: {cohere_api_key[:5]}...{cohere_api_key[-5:] if cohere_api_key else None}") - - groq_api_key = config.get('API', 'groq_api_key', fallback=None) - logging.debug(f"Loaded Groq API Key: {groq_api_key[:5]}...{groq_api_key[-5:] if groq_api_key else None}") - - openai_api_key = config.get('API', 'openai_api_key', fallback=None) - logging.debug( - f"Loaded OpenAI API Key: {openai_api_key[:5]}...{openai_api_key[-5:] if openai_api_key else None}") - - huggingface_api_key = config.get('API', 'huggingface_api_key', fallback=None) - logging.debug( - f"Loaded HuggingFace API Key: {huggingface_api_key[:5]}...{huggingface_api_key[-5:] if huggingface_api_key else None}") - - openrouter_api_key = config.get('API', 'openrouter_api_key', fallback=None) - logging.debug( - f"Loaded OpenRouter API Key: {openrouter_api_key[:5]}...{openrouter_api_key[-5:] if openrouter_api_key else None}") - - deepseek_api_key = config.get('API', 'deepseek_api_key', fallback=None) - logging.debug( - f"Loaded DeepSeek API Key: {deepseek_api_key[:5]}...{deepseek_api_key[-5:] if deepseek_api_key else None}") - - # Models - anthropic_model = config.get('API', 'anthropic_model', fallback='claude-3-sonnet-20240229') - cohere_model = config.get('API', 'cohere_model', fallback='command-r-plus') - groq_model = config.get('API', 'groq_model', fallback='llama3-70b-8192') - openai_model = config.get('API', 'openai_model', fallback='gpt-4-turbo') - huggingface_model = config.get('API', 'huggingface_model', fallback='CohereForAI/c4ai-command-r-plus') - openrouter_model = config.get('API', 'openrouter_model', fallback='microsoft/wizardlm-2-8x22b') - deepseek_model = config.get('API', 'deepseek_model', fallback='deepseek-chat') - - logging.debug(f"Loaded Anthropic Model: {anthropic_model}") - logging.debug(f"Loaded Cohere Model: {cohere_model}") - logging.debug(f"Loaded Groq Model: {groq_model}") - logging.debug(f"Loaded OpenAI Model: {openai_model}") - logging.debug(f"Loaded HuggingFace Model: {huggingface_model}") - logging.debug(f"Loaded OpenRouter Model: {openrouter_model}") - - # Local-Models - kobold_api_IP = config.get('Local-API', 'kobold_api_IP', fallback='http://127.0.0.1:5000/api/v1/generate') - kobold_api_key = config.get('Local-API', 'kobold_api_key', fallback='') - - llama_api_IP = config.get('Local-API', 'llama_api_IP', fallback='http://127.0.0.1:8080/v1/chat/completions') - llama_api_key = config.get('Local-API', 'llama_api_key', fallback='') - - ooba_api_IP = config.get('Local-API', 'ooba_api_IP', fallback='http://127.0.0.1:5000/v1/chat/completions') - ooba_api_key = config.get('Local-API', 'ooba_api_key', fallback='') - - tabby_api_IP = config.get('Local-API', 'tabby_api_IP', fallback='http://127.0.0.1:5000/api/v1/generate') - tabby_api_key = config.get('Local-API', 'tabby_api_key', fallback=None) - - vllm_api_url = config.get('Local-API', 'vllm_api_IP', fallback='http://127.0.0.1:500/api/v1/chat/completions') - vllm_api_key = config.get('Local-API', 'vllm_api_key', fallback=None) - - logging.debug(f"Loaded Kobold API IP: {kobold_api_IP}") - logging.debug(f"Loaded Llama API IP: {llama_api_IP}") - logging.debug(f"Loaded Ooba API IP: {ooba_api_IP}") - logging.debug(f"Loaded Tabby API IP: {tabby_api_IP}") - logging.debug(f"Loaded VLLM API URL: {vllm_api_url}") - - # Retrieve output paths from the configuration file - output_path = config.get('Paths', 'output_path', fallback='results') - logging.debug(f"Output path set to: {output_path}") - - # Retrieve processing choice from the configuration file - processing_choice = config.get('Processing', 'processing_choice', fallback='cpu') - logging.debug(f"Processing choice set to: {processing_choice}") - - # Prompts - FIXME - prompt_path = config.get('Prompts', 'prompt_path', fallback='prompts.db') - - return { - 'api_keys': { - 'anthropic': anthropic_api_key, - 'cohere': cohere_api_key, - 'groq': groq_api_key, - 'openai': openai_api_key, - 'huggingface': huggingface_api_key, - 'openrouter': openrouter_api_key, - 'deepseek': deepseek_api_key - }, - 'models': { - 'anthropic': anthropic_model, - 'cohere': cohere_model, - 'groq': groq_model, - 'openai': openai_model, - 'huggingface': huggingface_model, - 'openrouter': openrouter_model, - 'deepseek': deepseek_model - }, - 'local_apis': { - 'kobold': {'ip': kobold_api_IP, 'key': kobold_api_key}, - 'llama': {'ip': llama_api_IP, 'key': llama_api_key}, - 'ooba': {'ip': ooba_api_IP, 'key': ooba_api_key}, - 'tabby': {'ip': tabby_api_IP, 'key': tabby_api_key}, - 'vllm': {'ip': vllm_api_url, 'key': vllm_api_key} - }, - 'output_path': output_path, - 'processing_choice': processing_choice - } - - except Exception as e: - logging.error(f"Error loading config: {str(e)}") - return None - - - -# Log file -# logging.basicConfig(filename='debug-runtime.log', encoding='utf-8', level=logging.DEBUG) - -# -# -####################### - - -####################### -# System Startup Notice -# - -# Dirty hack - sue me. - FIXME - fix this... -os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' - -whisper_models = ["small", "medium", "small.en", "medium.en", "medium", "large", "large-v1", "large-v2", "large-v3", - "distil-large-v2", "distil-medium.en", "distil-small.en", "distil-large-v3"] -source_languages = { - "en": "English", - "zh": "Chinese", - "de": "German", - "es": "Spanish", - "ru": "Russian", - "ko": "Korean", - "fr": "French" -} -source_language_list = [key[0] for key in source_languages.items()] - - -def print_hello(): - print(r"""_____ _ ________ _ _ -|_ _|| | / /| _ \| | | | _ - | | | | / / | | | || | | |(_) - | | | | / / | | | || |/\| | - | | | |____ / / | |/ / \ /\ / _ - \_/ \_____//_/ |___/ \/ \/ (_) - - - _ _ -| | | | -| |_ ___ ___ | | ___ _ __ __ _ -| __| / _ \ / _ \ | | / _ \ | '_ \ / _` | -| |_ | (_) || (_) | | || (_) || | | || (_| | _ - \__| \___/ \___/ |_| \___/ |_| |_| \__, |( ) - __/ ||/ - |___/ - _ _ _ _ _ _ _ - | |(_) | | ( )| | | | | | - __| | _ __| | _ __ |/ | |_ __ __ __ _ | |_ ___ | |__ - / _` || | / _` || '_ \ | __| \ \ /\ / / / _` || __| / __|| '_ \ -| (_| || || (_| || | | | | |_ \ V V / | (_| || |_ | (__ | | | | - \__,_||_| \__,_||_| |_| \__| \_/\_/ \__,_| \__| \___||_| |_| -""") - time.sleep(1) - return - - -# -# -####################### - - -####################### -# System Check Functions -# -# 1. platform_check() -# 2. cuda_check() -# 3. decide_cpugpu() -# 4. check_ffmpeg() -# 5. download_ffmpeg() -# -####################### - - -####################### -# DB Functions -# -# create_tables() -# add_keyword() -# delete_keyword() -# add_keyword() -# add_media_with_keywords() -# search_db() -# format_results() -# search_and_display() -# export_to_csv() -# is_valid_url() -# is_valid_date() -# -######################################################################################################################## - - -######################################################################################################################## -# Processing Paths and local file handling -# -# Function List -# 1. read_paths_from_file(file_path) -# 2. process_path(path) -# 3. process_local_file(file_path) -# 4. read_paths_from_file(file_path: str) -> List[str] -# -# -######################################################################################################################## - - -####################################################################################################################### -# Online Article Extraction / Handling -# -# Function List -# 1. get_page_title(url) -# 2. get_article_text(url) -# 3. get_article_title(article_url_arg) -# -# -####################################################################################################################### - - -####################################################################################################################### -# Video Download/Handling -# Video-DL-Ingestion-Lib -# -# Function List -# 1. get_video_info(url) -# 2. create_download_directory(title) -# 3. sanitize_filename(title) -# 4. normalize_title(title) -# 5. get_youtube(video_url) -# 6. get_playlist_videos(playlist_url) -# 7. download_video(video_url, download_path, info_dict, download_video_flag) -# 8. save_to_file(video_urls, filename) -# 9. save_summary_to_file(summary, file_path) -# 10. process_url(url, num_speakers, whisper_model, custom_prompt, offset, api_name, api_key, vad_filter, download_video, download_audio, rolling_summarization, detail_level, question_box, keywords, ) # FIXME - UPDATE -# -# -####################################################################################################################### - - -####################################################################################################################### -# Audio Transcription -# -# Function List -# 1. convert_to_wav(video_file_path, offset=0, overwrite=False) -# 2. speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='small.en', vad_filter=False) -# -# -####################################################################################################################### - - -####################################################################################################################### -# Diarization -# -# Function List 1. speaker_diarize(video_file_path, segments, embedding_model = "pyannote/embedding", -# embedding_size=512, num_speakers=0) -# -# -####################################################################################################################### - - -####################################################################################################################### -# Chunking-related Techniques & Functions -# -# -# FIXME -# -# -####################################################################################################################### - - -####################################################################################################################### -# Tokenization-related Functions -# -# - -# FIXME - -# -# -####################################################################################################################### - - -####################################################################################################################### -# Website-related Techniques & Functions -# -# - -# -# -####################################################################################################################### - - -####################################################################################################################### -# Summarizers -# -# Function List -# 1. extract_text_from_segments(segments: List[Dict]) -> str -# 2. summarize_with_openai(api_key, file_path, custom_prompt_arg) -# 3. summarize_with_anthropic(api_key, file_path, model, custom_prompt_arg, max_retries=3, retry_delay=5) -# 4. summarize_with_cohere(api_key, file_path, model, custom_prompt_arg) -# 5. summarize_with_groq(api_key, file_path, model, custom_prompt_arg) -# -################################# -# Local Summarization -# -# Function List -# -# 1. summarize_with_local_llm(file_path, custom_prompt_arg) -# 2. summarize_with_llama(api_url, file_path, token, custom_prompt) -# 3. summarize_with_kobold(api_url, file_path, kobold_api_token, custom_prompt) -# 4. summarize_with_oobabooga(api_url, file_path, ooba_api_token, custom_prompt) -# 5. summarize_with_vllm(vllm_api_url, vllm_api_key_function_arg, llm_model, text, vllm_custom_prompt_function_arg) -# 6. summarize_with_tabbyapi(tabby_api_key, tabby_api_IP, text, tabby_model, custom_prompt) -# 7. save_summary_to_file(summary, file_path) -# -####################################################################################################################### - - -####################################################################################################################### -# Summarization with Detail -# - -# FIXME - see 'Old_Chunking_Lib.py' - -# -# -####################################################################################################################### - - -####################################################################################################################### -# Gradio UI -# -####################################################################################################################### -# Function Definitions -# - -# Only to be used when configured with Gradio for HF Space - -# New -def format_transcription(content): - # Add extra space after periods for better readability - content = content.replace('.', '. ').replace('. ', '. ') - # Split the content into lines for multiline display; assuming simple logic here - lines = content.split('. ') - # Join lines with HTML line break for better presentation in HTML - formatted_content = "
".join(lines) - return formatted_content - -# Old -# def format_transcription(transcription_text_arg): -# if transcription_text_arg: -# json_data = transcription_text_arg['transcription'] -# return json.dumps(json_data, indent=2) -# else: -# return "" - - -def format_file_path(file_path, fallback_path=None): - if file_path and os.path.exists(file_path): - logging.debug(f"File exists: {file_path}") - return file_path - elif fallback_path and os.path.exists(fallback_path): - logging.debug(f"File does not exist: {file_path}. Returning fallback path: {fallback_path}") - return fallback_path - else: - logging.debug(f"File does not exist: {file_path}. No fallback path available.") - return None - - -def search_media(query, fields, keyword, page): - try: - results = search_and_display(query, fields, keyword, page) - return results - except Exception as e: - logger.error(f"Error searching media: {e}") - return str(e) - - -####################################################### - -def display_details(media_id): - # Gradio Search Function-related stuff - if media_id: - details = display_item_details(media_id) - details_html = "" - for detail in details: - details_html += f"

Prompt:

{detail[0]}

" - details_html += f"

Summary:

{detail[1]}

" - details_html += f"

Transcription:

{detail[2]}
" - return details_html - return "No details available." - - -def fetch_items_by_title_or_url(search_query: str, search_type: str): - try: - with db.get_connection() as conn: - cursor = conn.cursor() - if search_type == 'Title': - cursor.execute("SELECT id, title, url FROM Media WHERE title LIKE ?", (f'%{search_query}%',)) - elif search_type == 'URL': - cursor.execute("SELECT id, title, url FROM Media WHERE url LIKE ?", (f'%{search_query}%',)) - results = cursor.fetchall() - return results - except sqlite3.Error as e: - raise DatabaseError(f"Error fetching items by {search_type}: {e}") - - -def fetch_items_by_keyword(search_query: str): - try: - with db.get_connection() as conn: - cursor = conn.cursor() - cursor.execute(""" - SELECT m.id, m.title, m.url - FROM Media m - JOIN MediaKeywords mk ON m.id = mk.media_id - JOIN Keywords k ON mk.keyword_id = k.id - WHERE k.keyword LIKE ? - """, (f'%{search_query}%',)) - results = cursor.fetchall() - return results - except sqlite3.Error as e: - raise DatabaseError(f"Error fetching items by keyword: {e}") - - -def fetch_items_by_content(search_query: str): - try: - with db.get_connection() as conn: - cursor = conn.cursor() - cursor.execute("SELECT id, title, url FROM Media WHERE content LIKE ?", (f'%{search_query}%',)) - results = cursor.fetchall() - return results - except sqlite3.Error as e: - raise DatabaseError(f"Error fetching items by content: {e}") - - -def fetch_item_details(media_id: int): - try: - with db.get_connection() as conn: - cursor = conn.cursor() - cursor.execute("SELECT prompt, summary FROM MediaModifications WHERE media_id = ?", (media_id,)) - prompt_summary_results = cursor.fetchall() - - cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,)) - content_result = cursor.fetchone() - content = content_result[0] if content_result else "" - - return prompt_summary_results, content - except sqlite3.Error as e: - raise DatabaseError(f"Error fetching item details: {e}") - - -def browse_items(search_query, search_type): - if search_type == 'Keyword': - results = fetch_items_by_keyword(search_query) - elif search_type == 'Content': - results = fetch_items_by_content(search_query) - else: - results = fetch_items_by_title_or_url(search_query, search_type) - return results - - -def display_item_details(media_id): - # Function to display item details - prompt_summary_results, content = fetch_item_details(media_id) - content_section = f"

Transcription:

{content}
" - prompt_summary_section = "" - for prompt, summary in prompt_summary_results: - prompt_summary_section += f"

Prompt:

{prompt}

" - prompt_summary_section += f"

Summary:

{summary}

" - return prompt_summary_section, content_section - - -def update_dropdown(search_query, search_type): - # Function to update the dropdown choices - results = browse_items(search_query, search_type) - item_options = [f"{item[1]} ({item[2]})" for item in results] - item_mapping = {f"{item[1]} ({item[2]})": item[0] for item in results} # Map item display to media ID - return gr.update(choices=item_options), item_mapping - - - -def get_media_id(selected_item, item_mapping): - return item_mapping.get(selected_item) - - -def update_detailed_view(item, item_mapping): - # Function to update the detailed view based on selected item - if item: - item_id = item_mapping.get(item) - if item_id: - prompt_summary_results, content = fetch_item_details(item_id) - if prompt_summary_results: - details_html = "

Details:

" - for prompt, summary in prompt_summary_results: - details_html += f"

Prompt:

{prompt}

" - details_html += f"

Summary:

{summary}

" - # Format the transcription content for better readability - content_html = f"

Transcription:

{format_transcription(content)}
" - return details_html, content_html - else: - return "No details available.", "No details available." - else: - return "No item selected", "No item selected" - else: - return "No item selected", "No item selected" - - -def format_content(content): - # Format content using markdown - formatted_content = f"```\n{content}\n```" - return formatted_content - - -def update_prompt_dropdown(): - prompt_names = list_prompts() - return gr.update(choices=prompt_names) - - -def display_prompt_details(selected_prompt): - if selected_prompt: - details = fetch_prompt_details(selected_prompt) - if details: - details_str = f"

Details:

{details[0]}

" - system_str = f"

System:

{details[1]}

" - user_str = f"

User:

{details[2]}

" if details[2] else "" - return details_str + system_str + user_str - return "No details available." - - -def insert_prompt_to_db(title, description, system_prompt, user_prompt): - try: - conn = sqlite3.connect('prompts.db') - cursor = conn.cursor() - cursor.execute( - "INSERT INTO Prompts (name, details, system, user) VALUES (?, ?, ?, ?)", - (title, description, system_prompt, user_prompt) - ) - conn.commit() - conn.close() - return "Prompt added successfully!" - except sqlite3.Error as e: - return f"Error adding prompt: {e}" - - -def display_search_results(query): - if not query.strip(): - return "Please enter a search query." - - results = search_prompts(query) - - # Debugging: Print the results to the console to see what is being returned - print(f"Processed search results for query '{query}': {results}") - - if results: - result_md = "## Search Results:\n" - for result in results: - # Debugging: Print each result to see its format - print(f"Result item: {result}") - - if len(result) == 2: - name, details = result - result_md += f"**Title:** {name}\n\n**Description:** {details}\n\n---\n" - else: - result_md += "Error: Unexpected result format.\n\n---\n" - return result_md - return "No results found." - - -# -# End of Gradio Search Function-related stuff -############################################################ - - -# def gradio UI -def launch_ui(demo_mode=False): - whisper_models = ["small", "medium", "small.en", "medium.en", "medium", "large", "large-v1", "large-v2", "large-v3", - "distil-large-v2", "distil-medium.en", "distil-small.en", "distil-large-v3"] - # Set theme value with https://www.gradio.app/guides/theming-guide - 'theme=' - my_theme = gr.Theme.from_hub("gradio/seafoam") - global custom_prompt_input - with gr.Blocks(theme=my_theme) as iface: - # Tab 1: Video Transcription + Summarization - with gr.Tab("Video Transcription + Summarization"): - - with gr.Row(): - # Light/Dark mode toggle switch - theme_toggle = gr.Radio(choices=["Light", "Dark"], value="Light", - label="Light/Dark Mode Toggle (Toggle to change UI color scheme)") - - # UI Mode toggle switch - ui_frontpage_mode_toggle = gr.Radio(choices=["Simple List", "Advanced List"], value="Simple List", - label="UI Mode Options Toggle(Toggle to show a few/all options)") - - # Add the new toggle switch - chunk_summarization_toggle = gr.Radio(choices=["Non-Chunked", "Chunked-Summarization"], - value="Non-Chunked", - label="Summarization Mode") - - # URL input is always visible - url_input = gr.Textbox(label="URL (Mandatory) --> Playlist URLs will be stripped and only the linked video" - " will be downloaded)", placeholder="Enter the video URL here. Multiple at once supported, one per line") - - # Inputs to be shown or hidden - num_speakers_input = gr.Number(value=2, label="Number of Speakers(Optional - Currently has no effect)", - visible=False) - whisper_model_input = gr.Dropdown(choices=whisper_models, value="small.en", - label="Whisper Model(This is the ML model used for transcription.)", - visible=False) - custom_prompt_input = gr.Textbox( - label="Custom Prompt (Customize your summarization, or ask a question about the video and have it " - "answered)\n Does not work against the summary currently.", - placeholder="Above is the transcript of a video. Please read " - "through the transcript carefully. Identify the main topics that are discussed over the " - "course of the transcript. Then, summarize the key points about each main topic in" - " bullet points. The bullet points should cover the key information conveyed about " - "each topic in the video, but should be much shorter than the full transcript. Please " - "output your bullet point summary inside tags.", - lines=3, visible=True) - offset_input = gr.Number(value=0, label="Offset (Seconds into the video to start transcribing at)", - visible=False) - api_name_input = gr.Dropdown( - choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "OpenRouter", "Llama.cpp", - "Kobold", "Ooba", "Tabbyapi", "VLLM", "HuggingFace",], - value=None, - label="API Name (Mandatory) --> Unless you just want a Transcription", visible=True) - api_key_input = gr.Textbox( - label="API Key (Mandatory) --> Unless you're running a local model/server OR have no API selected", - placeholder="Enter your API key here; Ignore if using Local API or Built-in API('Local-LLM')", - visible=True) - vad_filter_input = gr.Checkbox(label="VAD Filter (WIP)", value=False, - visible=False) - rolling_summarization_input = gr.Checkbox(label="Enable Rolling Summarization", value=False, - visible=False) - download_video_input = gr.components.Checkbox(label="Download Video(Select to allow for file download of " - "selected video)", value=False, visible=False) - download_audio_input = gr.components.Checkbox(label="Download Audio(Select to allow for file download of " - "selected Video's Audio)", value=False, visible=False) - detail_level_input = gr.Slider(minimum=0.01, maximum=1.0, value=0.01, step=0.01, interactive=True, - label="Summary Detail Level (Slide me) (Only OpenAI currently supported)", - visible=False) - keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords here (comma-separated Example: " - "tag_one,tag_two,tag_three)", - value="default,no_keyword_set", - visible=True) - question_box_input = gr.Textbox(label="Question", - placeholder="Enter a question to ask about the transcription", - visible=False) - # Add the additional input components - chunk_text_by_words_checkbox = gr.Checkbox(label="Chunk Text by Words", value=False, visible=False) - max_words_input = gr.Number(label="Max Words", value=300, precision=0, visible=False) - - chunk_text_by_sentences_checkbox = gr.Checkbox(label="Chunk Text by Sentences", value=False, - visible=False) - max_sentences_input = gr.Number(label="Max Sentences", value=10, precision=0, visible=False) - - chunk_text_by_paragraphs_checkbox = gr.Checkbox(label="Chunk Text by Paragraphs", value=False, - visible=False) - max_paragraphs_input = gr.Number(label="Max Paragraphs", value=5, precision=0, visible=False) - - chunk_text_by_tokens_checkbox = gr.Checkbox(label="Chunk Text by Tokens", value=False, visible=False) - max_tokens_input = gr.Number(label="Max Tokens", value=1000, precision=0, visible=False) - - inputs = [ - num_speakers_input, whisper_model_input, custom_prompt_input, offset_input, api_name_input, - api_key_input, vad_filter_input, download_video_input, download_audio_input, - rolling_summarization_input, detail_level_input, question_box_input, keywords_input, - chunk_text_by_words_checkbox, max_words_input, chunk_text_by_sentences_checkbox, - max_sentences_input, chunk_text_by_paragraphs_checkbox, max_paragraphs_input, - chunk_text_by_tokens_checkbox, max_tokens_input - ] - - - # FIgure out how to check for url vs list of urls - - all_inputs = [url_input] + inputs - - outputs = [ - gr.Textbox(label="Transcription (Resulting Transcription from your input URL)"), - gr.Textbox(label="Summary or Status Message (Current status of Summary or Summary itself)"), - gr.File(label="Download Transcription as JSON (Download the Transcription as a file)"), - gr.File(label="Download Summary as Text (Download the Summary as a file)"), - gr.File(label="Download Video (Download the Video as a file)", visible=False), - gr.File(label="Download Audio (Download the Audio as a file)", visible=False), - ] - - # Function to toggle visibility of advanced inputs - def toggle_frontpage_ui(mode): - visible_simple = mode == "Simple List" - visible_advanced = mode == "Advanced List" - - return [ - gr.update(visible=True), # URL input should always be visible - gr.update(visible=visible_advanced), # num_speakers_input - gr.update(visible=visible_advanced), # whisper_model_input - gr.update(visible=True), # custom_prompt_input - gr.update(visible=visible_advanced), # offset_input - gr.update(visible=True), # api_name_input - gr.update(visible=True), # api_key_input - gr.update(visible=visible_advanced), # vad_filter_input - gr.update(visible=visible_advanced), # download_video_input - gr.update(visible=visible_advanced), # download_audio_input - gr.update(visible=visible_advanced), # rolling_summarization_input - gr.update(visible_advanced), # detail_level_input - gr.update(visible_advanced), # question_box_input - gr.update(visible=True), # keywords_input - gr.update(visible_advanced), # chunk_text_by_words_checkbox - gr.update(visible_advanced), # max_words_input - gr.update(visible_advanced), # chunk_text_by_sentences_checkbox - gr.update(visible_advanced), # max_sentences_input - gr.update(visible_advanced), # chunk_text_by_paragraphs_checkbox - gr.update(visible_advanced), # max_paragraphs_input - gr.update(visible_advanced), # chunk_text_by_tokens_checkbox - gr.update(visible_advanced), # max_tokens_input - ] - - def toggle_chunk_summarization(mode): - visible = (mode == "Chunked-Summarization") - return [ - gr.update(visible=visible), # chunk_text_by_words_checkbox - gr.update(visible=visible), # max_words_input - gr.update(visible=visible), # chunk_text_by_sentences_checkbox - gr.update(visible=visible), # max_sentences_input - gr.update(visible=visible), # chunk_text_by_paragraphs_checkbox - gr.update(visible=visible), # max_paragraphs_input - gr.update(visible=visible), # chunk_text_by_tokens_checkbox - gr.update(visible=visible) # max_tokens_input - ] - - chunk_summarization_toggle.change(fn=toggle_chunk_summarization, inputs=chunk_summarization_toggle, - outputs=[ - chunk_text_by_words_checkbox, max_words_input, - chunk_text_by_sentences_checkbox, max_sentences_input, - chunk_text_by_paragraphs_checkbox, max_paragraphs_input, - chunk_text_by_tokens_checkbox, max_tokens_input - ]) - - def start_llamafile(*args): - # Unpack arguments - (am_noob, verbose_checked, threads_checked, threads_value, http_threads_checked, http_threads_value, - model_checked, model_value, hf_repo_checked, hf_repo_value, hf_file_checked, hf_file_value, - ctx_size_checked, ctx_size_value, ngl_checked, ngl_value, host_checked, host_value, port_checked, - port_value) = args - - # Construct command based on checked values - command = [] - if am_noob: - am_noob = True - if verbose_checked is not None and verbose_checked: - command.append('-v') - if threads_checked and threads_value is not None: - command.extend(['-t', str(threads_value)]) - if http_threads_checked and http_threads_value is not None: - command.extend(['--threads', str(http_threads_value)]) - if model_checked is not None and model_value is not None: - command.extend(['-m', model_value]) - if hf_repo_checked and hf_repo_value is not None: - command.extend(['-hfr', hf_repo_value]) - if hf_file_checked and hf_file_value is not None: - command.extend(['-hff', hf_file_value]) - if ctx_size_checked and ctx_size_value is not None: - command.extend(['-c', str(ctx_size_value)]) - if ngl_checked and ngl_value is not None: - command.extend(['-ngl', str(ngl_value)]) - if host_checked and host_value is not None: - command.extend(['--host', host_value]) - if port_checked and port_value is not None: - command.extend(['--port', str(port_value)]) - - # Code to start llamafile with the provided configuration - local_llm_gui_function(am_noob, verbose_checked, threads_checked, threads_value, - http_threads_checked, http_threads_value, model_checked, - model_value, hf_repo_checked, hf_repo_value, hf_file_checked, - hf_file_value, ctx_size_checked, ctx_size_value, ngl_checked, - ngl_value, host_checked, host_value, port_checked, port_value, ) - - # Example command output to verify - return f"Command built and ran: {' '.join(command)} \n\nLlamafile started successfully." - - def stop_llamafile(): - # Code to stop llamafile - # ... - return "Llamafile stopped" - - def toggle_light(mode): - if mode == "Dark": - return """ - - """ - else: - return """ - - """ - - # Set the event listener for the Light/Dark mode toggle switch - theme_toggle.change(fn=toggle_light, inputs=theme_toggle, outputs=gr.HTML()) - - ui_frontpage_mode_toggle.change(fn=toggle_frontpage_ui, inputs=ui_frontpage_mode_toggle, outputs=inputs) - - # Combine URL input and inputs lists - all_inputs = [url_input] + inputs - - # lets try embedding the theme here - FIXME? - # Adding a check in process_url to identify if passed multiple URLs or just one - gr.Interface( - fn=process_url, - inputs=all_inputs, - outputs=outputs, - title="Video Transcription and Summarization", - description="Submit a video URL for transcription and summarization. Ensure you input all necessary " - "information including API keys.", - theme='freddyaboulton/dracula_revamped', - allow_flagging="never" - ) - - - # Tab 2: Transcribe & Summarize Audio file - with gr.Tab("Audio File Processing"): - audio_url_input = gr.Textbox( - label="Audio File URL", - placeholder="Enter the URL of the audio file" - ) - audio_file_input = gr.File(label="Upload Audio File", file_types=["audio/*"]) - process_audio_button = gr.Button("Process Audio File") - audio_progress_output = gr.Textbox(label="Progress") - audio_transcriptions_output = gr.Textbox(label="Transcriptions") - - process_audio_button.click( - fn=process_audio_file, - inputs=[audio_url_input, audio_file_input], - outputs=[audio_progress_output, audio_transcriptions_output] - ) - - # Tab 3: Scrape & Summarize Articles/Websites - with gr.Tab("Scrape & Summarize Articles/Websites"): - url_input = gr.Textbox(label="Article URL", placeholder="Enter the article URL here") - custom_article_title_input = gr.Textbox(label="Custom Article Title (Optional)", - placeholder="Enter a custom title for the article") - custom_prompt_input = gr.Textbox( - label="Custom Prompt (Optional)", - placeholder="Provide a custom prompt for summarization", - lines=3 - ) - api_name_input = gr.Dropdown( - choices=[None, "huggingface", "deepseek", "openrouter", "openai", "anthropic", "cohere", "groq", "llama", "kobold", - "ooba"], - value=None, - label="API Name (Mandatory for Summarization)" - ) - api_key_input = gr.Textbox(label="API Key (Mandatory if API Name is specified)", - placeholder="Enter your API key here; Ignore if using Local API or Built-in API") - keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords here (comma-separated)", - value="default,no_keyword_set", visible=True) - - scrape_button = gr.Button("Scrape and Summarize") - result_output = gr.Textbox(label="Result") - - scrape_button.click(scrape_and_summarize, inputs=[url_input, custom_prompt_input, api_name_input, - api_key_input, keywords_input, - custom_article_title_input], outputs=result_output) - - gr.Markdown("### Or Paste Unstructured Text Below (Will use settings from above)") - text_input = gr.Textbox(label="Unstructured Text", placeholder="Paste unstructured text here", lines=10) - text_ingest_button = gr.Button("Ingest Unstructured Text") - text_ingest_result = gr.Textbox(label="Result") - - text_ingest_button.click(ingest_unstructured_text, - inputs=[text_input, custom_prompt_input, api_name_input, api_key_input, - keywords_input, custom_article_title_input], outputs=text_ingest_result) - - # Tab 4: Ingest & Summarize Documents - with gr.Tab("Ingest & Summarize Documents"): - gr.Markdown("Plan to put ingestion form for documents here") - gr.Markdown("Will ingest documents and store into SQLite DB") - gr.Markdown("RAG here we come....:/") - - # Function to update the visibility of the UI elements for Llamafile Settings - # def toggle_advanced_llamafile_mode(is_advanced): - # if is_advanced: - # return [gr.update(visible=True)] * 14 - # else: - # return [gr.update(visible=False)] * 11 + [gr.update(visible=True)] * 3 - # FIXME - def toggle_advanced_mode(advanced_mode): - # Show all elements if advanced mode is on - if advanced_mode: - return {elem: gr.update(visible=True) for elem in all_elements} - else: - # Show only specific elements if advanced mode is off - return {elem: gr.update(visible=elem in simple_mode_elements) for elem in all_elements} - - # Top-Level Gradio Tab #2 - 'Search / Detailed View' - with gr.Blocks() as search_interface: - with gr.Tab("Search Ingested Materials / Detailed Entry View / Prompts"): - search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...") - search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", - label="Search By") - - search_button = gr.Button("Search") - items_output = gr.Dropdown(label="Select Item", choices=[]) - item_mapping = gr.State({}) - - search_button.click(fn=update_dropdown, - inputs=[search_query_input, search_type_input], - outputs=[items_output, item_mapping] - ) - - prompt_summary_output = gr.HTML(label="Prompt & Summary", visible=True) - # FIXME - temp change; see if markdown works nicer... - content_output = gr.Markdown(label="Content", visible=True) - items_output.change(fn=update_detailed_view, - inputs=[items_output, item_mapping], - outputs=[prompt_summary_output, content_output] - ) - # sub-tab #2 for Search / Detailed view - with gr.Tab("View Prompts"): - with gr.Column(): - prompt_dropdown = gr.Dropdown( - label="Select Prompt (Thanks to the 'Fabric' project for this initial set: https://github.com/danielmiessler/fabric", - choices=[]) - prompt_details_output = gr.HTML() - - prompt_dropdown.change( - fn=display_prompt_details, - inputs=prompt_dropdown, - outputs=prompt_details_output - ) - - prompt_list_button = gr.Button("List Prompts") - prompt_list_button.click( - fn=update_prompt_dropdown, - outputs=prompt_dropdown - ) - - # FIXME - # Sub-tab #3 for Search / Detailed view - with gr.Tab("Search Prompts"): - with gr.Column(): - search_query_input = gr.Textbox(label="Search Query (It's broken)", - placeholder="Enter your search query...") - search_results_output = gr.Markdown() - - search_button = gr.Button("Search Prompts") - search_button.click( - fn=display_search_results, - inputs=[search_query_input], - outputs=[search_results_output] - ) - - search_query_input.change( - fn=display_search_results, - inputs=[search_query_input], - outputs=[search_results_output] - ) - - def add_prompt(name, details, system, user=None): - try: - conn = sqlite3.connect('prompts.db') - cursor = conn.cursor() - cursor.execute(''' - INSERT INTO Prompts (name, details, system, user) - VALUES (?, ?, ?, ?) - ''', (name, details, system, user)) - conn.commit() - conn.close() - return "Prompt added successfully." - except sqlite3.IntegrityError: - return "Prompt with this name already exists." - except sqlite3.Error as e: - return f"Database error: {e}" - - # Sub-tab #4 for Search / Detailed view - with gr.Tab("Add Prompts"): - gr.Markdown("### Add Prompt") - title_input = gr.Textbox(label="Title", placeholder="Enter the prompt title") - description_input = gr.Textbox(label="Description", placeholder="Enter the prompt description", lines=3) - system_prompt_input = gr.Textbox(label="System Prompt", placeholder="Enter the system prompt", lines=3) - user_prompt_input = gr.Textbox(label="User Prompt", placeholder="Enter the user prompt", lines=3) - add_prompt_button = gr.Button("Add Prompt") - add_prompt_output = gr.HTML() - - add_prompt_button.click( - fn=add_prompt, - inputs=[title_input, description_input, system_prompt_input, user_prompt_input], - outputs=add_prompt_output - ) - - # Top-Level Gradio Tab #3 - with gr.Blocks() as llamafile_interface: - with gr.Tab("Llamafile Settings"): - gr.Markdown("Settings for Llamafile") - - # Toggle switch for Advanced/Simple mode - am_noob = gr.Checkbox( - label="Check this to enable sane defaults and then download(if not already downloaded) a model, click 'Start Llamafile' and then go to --> 'Llamafile Chat Interface')\n\n", - value=False, visible=True) - advanced_mode_toggle = gr.Checkbox( - label="Advanced Mode - Enable to show all settings\n\n", - value=False) - - # Simple mode elements - model_checked = gr.Checkbox(label="Enable Setting Local LLM Model Path", value=False, visible=True) - model_value = gr.Textbox(label="Path to Local Model File", value="", visible=True) - ngl_checked = gr.Checkbox(label="Enable Setting GPU Layers", value=False, visible=True) - ngl_value = gr.Number(label="Number of GPU Layers", value=None, precision=0, visible=True) - - # Advanced mode elements - verbose_checked = gr.Checkbox(label="Enable Verbose Output", value=False, visible=False) - threads_checked = gr.Checkbox(label="Set CPU Threads", value=False, visible=False) - threads_value = gr.Number(label="Number of CPU Threads", value=None, precision=0, visible=False) - http_threads_checked = gr.Checkbox(label="Set HTTP Server Threads", value=False, visible=False) - http_threads_value = gr.Number(label="Number of HTTP Server Threads", value=None, precision=0, - visible=False) - hf_repo_checked = gr.Checkbox(label="Use Huggingface Repo Model", value=False, visible=False) - hf_repo_value = gr.Textbox(label="Huggingface Repo Name", value="", visible=False) - hf_file_checked = gr.Checkbox(label="Set Huggingface Model File", value=False, visible=False) - hf_file_value = gr.Textbox(label="Huggingface Model File", value="", visible=False) - ctx_size_checked = gr.Checkbox(label="Set Prompt Context Size", value=False, visible=False) - ctx_size_value = gr.Number(label="Prompt Context Size", value=8124, precision=0, visible=False) - host_checked = gr.Checkbox(label="Set IP to Listen On", value=False, visible=False) - host_value = gr.Textbox(label="Host IP Address", value="", visible=False) - port_checked = gr.Checkbox(label="Set Server Port", value=False, visible=False) - port_value = gr.Number(label="Port Number", value=None, precision=0, visible=False) - - # Start and Stop buttons - start_button = gr.Button("Start Llamafile") - stop_button = gr.Button("Stop Llamafile") - output_display = gr.Markdown() - - all_elements = [ - verbose_checked, threads_checked, threads_value, http_threads_checked, http_threads_value, - model_checked, model_value, hf_repo_checked, hf_repo_value, hf_file_checked, hf_file_value, - ctx_size_checked, ctx_size_value, ngl_checked, ngl_value, host_checked, host_value, port_checked, - port_value - ] - - simple_mode_elements = [model_checked, model_value, ngl_checked, ngl_value] - - advanced_mode_toggle.change( - fn=toggle_advanced_mode, - inputs=[advanced_mode_toggle], - outputs=all_elements - ) - - # Function call with the new inputs - start_button.click( - fn=start_llamafile, - inputs=[am_noob, verbose_checked, threads_checked, threads_value, http_threads_checked, - http_threads_value, - model_checked, model_value, hf_repo_checked, hf_repo_value, hf_file_checked, hf_file_value, - ctx_size_checked, ctx_size_value, ngl_checked, ngl_value, host_checked, host_value, - port_checked, port_value], - outputs=output_display - ) - - # Second sub-tab for Llamafile - with gr.Tab("Llamafile Chat Interface"): - gr.Markdown("Page to interact with Llamafile Server (iframe to Llamafile server port)") - # Define the HTML content with the iframe - html_content = """ - - - - - - Llama.cpp Server Chat Interface - Loaded from http://127.0.0.1:8080 - - - - - - - """ - gr.HTML(html_content) - - # Third sub-tab for Llamafile - # https://github.com/lmg-anon/mikupad/releases - with gr.Tab("Mikupad Chat Interface"): - gr.Markdown("Not implemented. Have to wait until I get rid of Gradio") - gr.HTML(html_content) - - def export_keywords_to_csv(): - try: - keywords = fetch_all_keywords() - if not keywords: - return None, "No keywords found in the database." - - filename = "keywords.csv" - with open(filename, 'w', newline='', encoding='utf-8') as file: - writer = csv.writer(file) - writer.writerow(["Keyword"]) - for keyword in keywords: - writer.writerow([keyword]) - - return filename, f"Keywords exported to {filename}" - except Exception as e: - logger.error(f"Error exporting keywords to CSV: {e}") - return None, f"Error exporting keywords: {e}" - - - # Top-Level Gradio Tab #4 - Don't ask me how this is tabbed, but it is... #FIXME - export_keywords_interface = gr.Interface( - fn=export_keywords_to_csv, - inputs=[], - outputs=[gr.File(label="Download Exported Keywords"), gr.Textbox(label="Status")], - title="Export Keywords", - description="Export all keywords in the database to a CSV file." - ) - - # Gradio interface for importing data - def import_data(file): - # Placeholder for actual import functionality - return "Data imported successfully" - - # Top-Level Gradio Tab #5 - Export/Import - Same deal as above, not sure why this is auto-tabbed - import_interface = gr.Interface( - fn=import_data, - inputs=gr.File(label="Upload file for import"), - outputs="text", - title="Import Data", - description="Import data into the database from a CSV file." - ) - - # # Top-Level Gradio Tab #6 - Export/Import - Same deal as above, not sure why this is auto-tabbed - # import_export_tab = gr.TabbedInterface( - # [gr.TabbedInterface( - # [gr.Interface( - # fn=export_to_csv, - # inputs=[ - # gr.Textbox(label="Search Query", placeholder="Enter your search query here..."), - # gr.CheckboxGroup(label="Search Fields", choices=["Title", "Content"], value=["Title"]), - # gr.Textbox(label="Keyword (Match ALL, can use multiple keywords, separated by ',' (comma) )", - # placeholder="Enter keywords here..."), - # gr.Number(label="Page", value=1, precision=0), - # gr.Number(label="Results per File", value=1000, precision=0) - # ], - # outputs="text", - # title="Export Search Results to CSV", - # description="Export the search results to a CSV file." - # ), - # export_keywords_interface], - # ["Export Search Results", "Export Keywords"] - # ), - # import_interface], - # ["Export", "Import"] - # ) - - def ensure_dir_exists(path): - if not os.path.exists(path): - os.makedirs(path) - - def gradio_download_youtube_video(url): - """Download video using yt-dlp with specified options.""" - # Determine ffmpeg path based on the operating system. - ffmpeg_path = './Bin/ffmpeg.exe' if os.name == 'nt' else 'ffmpeg' - - # Extract information about the video - with yt_dlp.YoutubeDL({'quiet': True}) as ydl: - info_dict = ydl.extract_info(url, download=False) - sanitized_title = sanitize_filename(info_dict['title']) - original_ext = info_dict['ext'] - - # Setup the final directory and filename - download_dir = Path(f"results/{sanitized_title}") - download_dir.mkdir(parents=True, exist_ok=True) - output_file_path = download_dir / f"{sanitized_title}.{original_ext}" - - # Initialize yt-dlp with generic options and the output template - ydl_opts = { - 'format': 'bestvideo+bestaudio/best', - 'ffmpeg_location': ffmpeg_path, - 'outtmpl': str(output_file_path), - 'noplaylist': True, 'quiet': True - } - - # Execute yt-dlp to download the video - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - ydl.download([url]) - - # Final check to ensure file exists - if not output_file_path.exists(): - raise FileNotFoundError(f"Expected file was not found: {output_file_path}") - - return str(output_file_path) - - # FIXME - example to use for rest of gradio theming, just stuff in HTML/Markdown - # <-- set description variable with HTML --> - desc = "

Youtube Video Downloader

This Input takes a Youtube URL as input and creates " \ - "a webm file for you to download.
If you want a full-featured one: " \ - "https://github.com/StefanLobbenmeier/youtube-dl-gui or https://github.com/yt-dlg/yt-dlg

" - - # Sixth Top Tab - Download Video/Audio Files - download_videos_interface = gr.Interface( - fn=gradio_download_youtube_video, - inputs=gr.Textbox(label="YouTube URL", placeholder="Enter YouTube video URL here"), - outputs=gr.File(label="Download Video"), - title="YouTube Video Downloader", - description=desc, - allow_flagging="never" - ) - - # Combine interfaces into a tabbed interface - tabbed_interface = gr.TabbedInterface( - [iface, search_interface, llamafile_interface, download_videos_interface], - ["Transcription / Summarization / Ingestion", "Search / Detailed View", - "Local LLM with Llamafile", "Export/Import", "Download Video/Audio Files"]) - - # Launch the interface - server_port_variable = 7860 - global server_mode, share_public - - if share_public == True: - tabbed_interface.launch(share=True, ) - elif server_mode == True and share_public is False: - tabbed_interface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable) - else: - tabbed_interface.launch(share=False, ) - #tabbed_interface.launch(share=True, ) - - -def clean_youtube_url(url): - parsed_url = urlparse(url) - query_params = parse_qs(parsed_url.query) - if 'list' in query_params: - query_params.pop('list') - cleaned_query = urlencode(query_params, doseq=True) - cleaned_url = urlunparse(parsed_url._replace(query=cleaned_query)) - return cleaned_url - -def extract_video_info(url): - info_dict = get_youtube(url) - title = info_dict.get('title', 'Untitled') - return info_dict, title - - -def download_audio_file(url, save_path): - response = requests.get(url, stream=True) - file_size = int(response.headers.get('content-length', 0)) - if file_size > 500 * 1024 * 1024: # 500 MB limit - raise ValueError("File size exceeds the 500MB limit.") - with open(save_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - return save_path - -def process_audio_file(audio_url, audio_file): - progress = [] - transcriptions = [] - - def update_progress(stage, message): - progress.append(f"{stage}: {message}") - return "\n".join(progress), "\n".join(transcriptions) - - try: - if audio_url: - # Process audio file from URL - save_path = Path("downloaded_audio_file.wav") - download_audio_file(audio_url, save_path) - elif audio_file: - # Process uploaded audio file - audio_file_size = os.path.getsize(audio_file.name) - if audio_file_size > 500 * 1024 * 1024: # 500 MB limit - return update_progress("Error", "File size exceeds the 500MB limit.") - save_path = Path(audio_file.name) - else: - return update_progress("Error", "No audio file provided.") - - # Perform transcription and summarization - transcription, summary, json_file_path, summary_file_path, _, _ = process_url( - url=None, - num_speakers=2, - whisper_model="small.en", - custom_prompt_input=None, - offset=0, - api_name=None, - api_key=None, - vad_filter=False, - download_video_flag=False, - download_audio=False, - rolling_summarization=False, - detail_level=0.01, - question_box=None, - keywords="default,no_keyword_set", - chunk_text_by_words=False, - max_words=0, - chunk_text_by_sentences=False, - max_sentences=0, - chunk_text_by_paragraphs=False, - max_paragraphs=0, - chunk_text_by_tokens=False, - max_tokens=0, - local_file_path=str(save_path) - ) - transcriptions.append(transcription) - progress.append("Processing complete.") - except Exception as e: - progress.append(f"Error: {str(e)}") - - return "\n".join(progress), "\n".join(transcriptions) - - -def process_url( - url, - num_speakers, - whisper_model, - custom_prompt_input, - offset, - api_name, - api_key, - vad_filter, - download_video_flag, - download_audio, - rolling_summarization, - detail_level, - # It's for the asking a question about a returned prompt - needs to be removed #FIXME - question_box, - keywords, - chunk_text_by_words, - max_words, - chunk_text_by_sentences, - max_sentences, - chunk_text_by_paragraphs, - max_paragraphs, - chunk_text_by_tokens, - max_tokens, - local_file_path=None -): - # Handle the chunk summarization options - set_chunk_txt_by_words = chunk_text_by_words - set_max_txt_chunk_words = max_words - set_chunk_txt_by_sentences = chunk_text_by_sentences - set_max_txt_chunk_sentences = max_sentences - set_chunk_txt_by_paragraphs = chunk_text_by_paragraphs - set_max_txt_chunk_paragraphs = max_paragraphs - set_chunk_txt_by_tokens = chunk_text_by_tokens - set_max_txt_chunk_tokens = max_tokens - - progress = [] - success_message = "All videos processed successfully. Transcriptions and summaries have been ingested into the database." - - - # Validate input - if not url and not local_file_path: - return "Process_URL: No URL provided.", "No URL provided.", None, None, None, None, None, None - - # FIXME - Chatgpt again? - if isinstance(url, str): - urls = url.strip().split('\n') - if len(urls) > 1: - return process_video_urls(urls, num_speakers, whisper_model, custom_prompt_input, offset, api_name, api_key, vad_filter, - download_video_flag, download_audio, rolling_summarization, detail_level, question_box, - keywords, chunk_text_by_words, max_words, chunk_text_by_sentences, max_sentences, - chunk_text_by_paragraphs, max_paragraphs, chunk_text_by_tokens, max_tokens) - else: - urls = [url] - - if url and not is_valid_url(url): - return "Process_URL: Invalid URL format.", "Invalid URL format.", None, None, None, None, None, None - - if url: - # Clean the URL to remove playlist parameters if any - url = clean_youtube_url(url) - logging.info(f"Process_URL: Processing URL: {url}") - - if api_name: - print("Process_URL: API Name received:", api_name) # Debugging line - - video_file_path = None - global info_dict - - # FIXME - need to handle local audio file processing - # If Local audio file is provided - if local_file_path: - try: - pass - # # insert code to process local audio file - # # Need to be able to add a title/author/etc for ingestion into the database - # # Also want to be able to optionally _just_ ingest it, and not ingest. - # # FIXME - # #download_path = create_download_directory(title) - # #audio_path = download_video(url, download_path, info_dict, download_video_flag) - # - # audio_file_path = local_file_path - # global segments - # audio_file_path, segments = perform_transcription(audio_file_path, offset, whisper_model, vad_filter) - # - # if audio_file_path is None or segments is None: - # logging.error("Process_URL: Transcription failed or segments not available.") - # return "Process_URL: Transcription failed.", "Transcription failed.", None, None, None, None - # - # logging.debug(f"Process_URL: Transcription audio_file: {audio_file_path}") - # logging.debug(f"Process_URL: Transcription segments: {segments}") - # - # transcription_text = {'audio_file': audio_file_path, 'transcription': segments} - # logging.debug(f"Process_URL: Transcription text: {transcription_text}") - # - # if rolling_summarization: - # text = extract_text_from_segments(segments) - # summary_text = rolling_summarize_function( - # transcription_text, - # detail=detail_level, - # api_name=api_name, - # api_key=api_key, - # custom_prompt=custom_prompt, - # chunk_by_words=chunk_text_by_words, - # max_words=max_words, - # chunk_by_sentences=chunk_text_by_sentences, - # max_sentences=max_sentences, - # chunk_by_paragraphs=chunk_text_by_paragraphs, - # max_paragraphs=max_paragraphs, - # chunk_by_tokens=chunk_text_by_tokens, - # max_tokens=max_tokens - # ) - # if api_name: - # summary_text = perform_summarization(api_name, segments_json_path, custom_prompt, api_key, config) - # if summary_text is None: - # logging.error("Summary text is None. Check summarization function.") - # summary_file_path = None # Set summary_file_path to None if summary is not generated - # else: - # summary_text = 'Summary not available' - # summary_file_path = None # Set summary_file_path to None if summary is not generated - # - # json_file_path, summary_file_path = save_transcription_and_summary(transcription_text, summary_text, download_path) - # - # add_media_to_database(url, info_dict, segments, summary_text, keywords, custom_prompt, whisper_model) - # - # return transcription_text, summary_text, json_file_path, summary_file_path, None, None - - except Exception as e: - logging.error(f": {e}") - return str(e), 'process_url: Error processing the request.', None, None, None, None - - - # If URL/Local video file is provided - try: - info_dict, title = extract_video_info(url) - download_path = create_download_directory(title) - video_path = download_video(url, download_path, info_dict, download_video_flag) - global segments - audio_file_path, segments = perform_transcription(video_path, offset, whisper_model, vad_filter) - - if audio_file_path is None or segments is None: - logging.error("Process_URL: Transcription failed or segments not available.") - return "Process_URL: Transcription failed.", "Transcription failed.", None, None, None, None - - logging.debug(f"Process_URL: Transcription audio_file: {audio_file_path}") - logging.debug(f"Process_URL: Transcription segments: {segments}") - - transcription_text = {'audio_file': audio_file_path, 'transcription': segments} - logging.debug(f"Process_URL: Transcription text: {transcription_text}") - - if rolling_summarization: - text = extract_text_from_segments(segments) - summary_text = rolling_summarize_function( - transcription_text, - detail=detail_level, - api_name=api_name, - api_key=api_key, - custom_prompt_input=custom_prompt_input, - chunk_by_words=chunk_text_by_words, - max_words=max_words, - chunk_by_sentences=chunk_text_by_sentences, - max_sentences=max_sentences, - chunk_by_paragraphs=chunk_text_by_paragraphs, - max_paragraphs=max_paragraphs, - chunk_by_tokens=chunk_text_by_tokens, - max_tokens=max_tokens - ) - if api_name: - summary_text = perform_summarization(api_name, segments_json_path, custom_prompt_input, api_key) - if summary_text is None: - logging.error("Summary text is None. Check summarization function.") - summary_file_path = None # Set summary_file_path to None if summary is not generated - else: - summary_text = 'Summary not available' - summary_file_path = None # Set summary_file_path to None if summary is not generated - - json_file_path, summary_file_path = save_transcription_and_summary(transcription_text, summary_text, download_path) - - add_media_to_database(url, info_dict, segments, summary_text, keywords, custom_prompt_input, whisper_model) - - return transcription_text, summary_text, json_file_path, summary_file_path, None, None - - except Exception as e: - logging.error(f": {e}") - return str(e), 'process_url: Error processing the request.', None, None, None, None - -# Handle multiple videos as input -# Handle multiple videos as input -def process_video_urls(url_list, num_speakers, whisper_model, custom_prompt_input, offset, api_name, api_key, vad_filter, - download_video_flag, download_audio, rolling_summarization, detail_level, question_box, - keywords, chunk_text_by_words, max_words, chunk_text_by_sentences, max_sentences, - chunk_text_by_paragraphs, max_paragraphs, chunk_text_by_tokens, max_tokens): - global current_progress - progress = [] # This must always be a list - status = [] # This must always be a list - - def update_progress(index, url, message): - progress.append(f"Processing {index + 1}/{len(url_list)}: {url}") # Append to list - status.append(message) # Append to list - return "\n".join(progress), "\n".join(status) # Return strings for display - - - for index, url in enumerate(url_list): - try: - transcription, summary, json_file_path, summary_file_path, _, _ = process_url( - url=url, - num_speakers=num_speakers, - whisper_model=whisper_model, - custom_prompt_input=custom_prompt_input, - offset=offset, - api_name=api_name, - api_key=api_key, - vad_filter=vad_filter, - download_video_flag=download_video_flag, - download_audio=download_audio, - rolling_summarization=rolling_summarization, - detail_level=detail_level, - question_box=question_box, - keywords=keywords, - chunk_text_by_words=chunk_text_by_words, - max_words=max_words, - chunk_text_by_sentences=max_sentences, - max_sentences=max_sentences, - chunk_text_by_paragraphs=chunk_text_by_paragraphs, - max_paragraphs=max_paragraphs, - chunk_text_by_tokens=chunk_text_by_tokens, - max_tokens=max_tokens - ) - # Update progress and transcription properly - current_progress, current_status = update_progress(index, url, "Video processed and ingested into the database.") - except Exception as e: - current_progress, current_status = update_progress(index, url, f"Error: {str(e)}") - - success_message = "All videos have been transcribed, summarized, and ingested into the database successfully." - return current_progress, success_message, None, None, None, None - - -# FIXME - Prompt sample box - -# Sample data -prompts_category_1 = [ - "What are the key points discussed in the video?", - "Summarize the main arguments made by the speaker.", - "Describe the conclusions of the study presented." -] - -prompts_category_2 = [ - "How does the proposed solution address the problem?", - "What are the implications of the findings?", - "Can you explain the theory behind the observed phenomenon?" -] - -all_prompts = prompts_category_1 + prompts_category_2 - - -# Search function -def search_prompts(query): - filtered_prompts = [prompt for prompt in all_prompts if query.lower() in prompt.lower()] - return "\n".join(filtered_prompts) - - -# Handle prompt selection -def handle_prompt_selection(prompt): - return f"You selected: {prompt}" - - -# -# -####################################################################################################################### - - -####################################################################################################################### -# Local LLM Setup / Running -# -# Function List -# 1. download_latest_llamafile(repo, asset_name_prefix, output_filename) -# 2. download_file(url, dest_path, expected_checksum=None, max_retries=3, delay=5) -# 3. verify_checksum(file_path, expected_checksum) -# 4. cleanup_process() -# 5. signal_handler(sig, frame) -# 6. local_llm_function() -# 7. launch_in_new_terminal_windows(executable, args) -# 8. launch_in_new_terminal_linux(executable, args) -# 9. launch_in_new_terminal_mac(executable, args) -# -# -####################################################################################################################### - - -####################################################################################################################### -# Helper Functions for Main() & process_url() -# - -def perform_transcription(video_path, offset, whisper_model, vad_filter): - global segments_json_path - audio_file_path = convert_to_wav(video_path, offset) - segments_json_path = audio_file_path.replace('.wav', '.segments.json') - - # Check if segments JSON already exists - if os.path.exists(segments_json_path): - logging.info(f"Segments file already exists: {segments_json_path}") - try: - with open(segments_json_path, 'r') as file: - segments = json.load(file) - if not segments: # Check if the loaded JSON is empty - logging.warning(f"Segments JSON file is empty, re-generating: {segments_json_path}") - raise ValueError("Empty segments JSON file") - logging.debug(f"Loaded segments from {segments_json_path}") - except (json.JSONDecodeError, ValueError) as e: - logging.error(f"Failed to read or parse the segments JSON file: {e}") - # Remove the corrupted file - os.remove(segments_json_path) - # Re-generate the transcription - logging.info(f"Re-generating transcription for {audio_file_path}") - audio_file, segments = re_generate_transcription(audio_file_path, whisper_model, vad_filter) - if segments is None: - return None, None - else: - # Perform speech to text transcription - audio_file, segments = re_generate_transcription(audio_file_path, whisper_model, vad_filter) - - return audio_file_path, segments - - -def re_generate_transcription(audio_file_path, whisper_model, vad_filter): - try: - segments = speech_to_text(audio_file_path, whisper_model=whisper_model, vad_filter=vad_filter) - # Save segments to JSON - segments_json_path = audio_file_path.replace('.wav', '.segments.json') - with open(segments_json_path, 'w') as file: - json.dump(segments, file, indent=2) - logging.debug(f"Transcription segments saved to {segments_json_path}") - return audio_file_path, segments - except Exception as e: - logging.error(f"Error in re-generating transcription: {str(e)}") - return None, None - - -def save_transcription_and_summary(transcription_text, summary_text, download_path): - video_title = sanitize_filename(info_dict.get('title', 'Untitled')) - - json_file_path = os.path.join(download_path, f"{video_title}.segments.json") - summary_file_path = os.path.join(download_path, f"{video_title}_summary.txt") - - with open(json_file_path, 'w') as json_file: - json.dump(transcription_text['transcription'], json_file, indent=2) - - if summary_text is not None: - with open(summary_file_path, 'w') as file: - file.write(summary_text) - else: - logging.warning("Summary text is None. Skipping summary file creation.") - summary_file_path = None - - return json_file_path, summary_file_path - -def add_media_to_database(url, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model): - content = ' '.join([segment['Text'] for segment in segments if 'Text' in segment]) - add_media_with_keywords( - url=url, - title=info_dict.get('title', 'Untitled'), - media_type='video', - content=content, - keywords=','.join(keywords), - prompt=custom_prompt_input or 'No prompt provided', - summary=summary or 'No summary provided', - transcription_model=whisper_model, - author=info_dict.get('uploader', 'Unknown'), - ingestion_date=datetime.now().strftime('%Y-%m-%d') - ) - - -def perform_summarization(api_name, json_file_path, custom_prompt_input, api_key): - # Load Config - loaded_config_data = load_and_log_configs() - - if custom_prompt_input is None: - # FIXME - Setup proper default prompt & extract said prompt from config file or prompts.db file. - #custom_prompt_input = config.get('Prompts', 'video_summarize_prompt', fallback="Above is the transcript of a video. Please read through the transcript carefully. Identify the main topics that are discussed over the course of the transcript. Then, summarize the key points about each main topic in bullet points. The bullet points should cover the key information conveyed about each topic in the video, but should be much shorter than the full transcript. Please output your bullet point summary inside tags. Do not repeat yourself while writing the summary.") - custom_prompt_input = "Above is the transcript of a video. Please read through the transcript carefully. Identify the main topics that are discussed over the course of the transcript. Then, summarize the key points about each main topic in bullet points. The bullet points should cover the key information conveyed about each topic in the video, but should be much shorter than the full transcript. Please output your bullet point summary inside tags. Do not repeat yourself while writing the summary." - summary = None - try: - if not json_file_path or not os.path.exists(json_file_path): - logging.error(f"JSON file does not exist: {json_file_path}") - return None - - with open(json_file_path, 'r') as file: - data = json.load(file) - - segments = data - if not isinstance(segments, list): - logging.error(f"Segments is not a list: {type(segments)}") - return None - - text = extract_text_from_segments(segments) - - if api_name.lower() == 'openai': - #def summarize_with_openai(api_key, input_data, custom_prompt_arg) - summary = summarize_with_openai(api_key, text, custom_prompt_input) - - elif api_name.lower() == "anthropic": - # def summarize_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5): - summary = summarize_with_anthropic(api_key, text, custom_prompt_input) - elif api_name.lower() == "cohere": - # def summarize_with_cohere(api_key, input_data, model, custom_prompt_arg) - summary = summarize_with_cohere(api_key, text, custom_prompt_input) - - elif api_name.lower() == "groq": - logging.debug(f"MAIN: Trying to summarize with groq") - # def summarize_with_groq(api_key, input_data, model, custom_prompt_arg): - summary = summarize_with_groq(api_key, text, custom_prompt_input) - - elif api_name.lower() == "openrouter": - logging.debug(f"MAIN: Trying to summarize with OpenRouter") - # def summarize_with_openrouter(api_key, input_data, custom_prompt_arg): - summary = summarize_with_openrouter(api_key, text, custom_prompt_input) - - elif api_name.lower() == "deepseek": - logging.debug(f"MAIN: Trying to summarize with DeepSeek") - # def summarize_with_deepseek(api_key, input_data, custom_prompt_arg): - summary = summarize_with_deepseek(api_key, text, custom_prompt_input) - - elif api_name.lower() == "llama.cpp": - logging.debug(f"MAIN: Trying to summarize with Llama.cpp") - # def summarize_with_llama(api_url, file_path, token, custom_prompt) - summary = summarize_with_llama(text, custom_prompt_input) - - elif api_name.lower() == "kobold": - logging.debug(f"MAIN: Trying to summarize with Kobold.cpp") - # def summarize_with_kobold(input_data, kobold_api_token, custom_prompt_input, api_url): - summary = summarize_with_kobold(text, api_key, custom_prompt_input) - - elif api_name.lower() == "ooba": - # def summarize_with_oobabooga(input_data, api_key, custom_prompt, api_url): - summary = summarize_with_oobabooga(text, api_key, custom_prompt_input) - - elif api_name.lower() == "tabbyapi": - # def summarize_with_tabbyapi(input_data, tabby_model, custom_prompt_input, api_key=None, api_IP): - summary = summarize_with_tabbyapi(text, custom_prompt_input) - - elif api_name.lower() == "vllm": - logging.debug(f"MAIN: Trying to summarize with VLLM") - # def summarize_with_vllm(api_key, input_data, custom_prompt_input): - summary = summarize_with_vllm(text, custom_prompt_input) - - elif api_name.lower() == "local-llm": - logging.debug(f"MAIN: Trying to summarize with Local LLM") - summary = summarize_with_local_llm(text, custom_prompt_input) - - elif api_name.lower() == "huggingface": - logging.debug(f"MAIN: Trying to summarize with huggingface") - # def summarize_with_huggingface(api_key, input_data, custom_prompt_arg): - summarize_with_huggingface(api_key, text, custom_prompt_input) - # Add additional API handlers here... - - else: - logging.warning(f"Unsupported API: {api_name}") - - if summary is None: - logging.debug("Summarization did not return valid text.") - - if summary: - logging.info(f"Summary generated using {api_name} API") - # Save the summary file in the same directory as the JSON file - summary_file_path = json_file_path.replace('.json', '_summary.txt') - with open(summary_file_path, 'w') as file: - file.write(summary) - else: - logging.warning(f"Failed to generate summary using {api_name} API") - return summary - - except requests.exceptions.ConnectionError: - logging.error("Connection error while summarizing") - except Exception as e: - logging.error(f"Error summarizing with {api_name}: {str(e)}") - - return summary - -# -# -####################################################################################################################### - - -###################################################################################################################### -# Main() -# - -def main(input_path, api_name=None, api_key=None, - num_speakers=2, - whisper_model="small.en", - offset=0, - vad_filter=False, - download_video_flag=False, - custom_prompt=None, - overwrite=False, - rolling_summarization=False, - detail=0.01, - keywords=None, - llm_model=None, - time_based=False, - set_chunk_txt_by_words=False, - set_max_txt_chunk_words=0, - set_chunk_txt_by_sentences=False, - set_max_txt_chunk_sentences=0, - set_chunk_txt_by_paragraphs=False, - set_max_txt_chunk_paragraphs=0, - set_chunk_txt_by_tokens=False, - set_max_txt_chunk_tokens=0, - ): - global detail_level_number, summary, audio_file, transcription_text, info_dict - - detail_level = detail - - print(f"Keywords: {keywords}") - - if not input_path: - return [] - - start_time = time.monotonic() - paths = [input_path] if not os.path.isfile(input_path) else read_paths_from_file(input_path) - results = [] - - for path in paths: - try: - if path.startswith('http'): - info_dict, title = extract_video_info(path) - download_path = create_download_directory(title) - video_path = download_video(path, download_path, info_dict, download_video_flag) - - if video_path: - audio_file, segments = perform_transcription(video_path, offset, whisper_model, vad_filter) - transcription_text = {'audio_file': audio_file, 'transcription': segments} - # FIXME - V1 - #transcription_text = {'video_path': path, 'audio_file': audio_file, 'transcription': segments} - - if rolling_summarization == True: - text = extract_text_from_segments(segments) - detail = detail_level - additional_instructions = custom_prompt_input - chunk_text_by_words = set_chunk_txt_by_words - max_words = set_max_txt_chunk_words - chunk_text_by_sentences = set_chunk_txt_by_sentences - max_sentences = set_max_txt_chunk_sentences - chunk_text_by_paragraphs = set_chunk_txt_by_paragraphs - max_paragraphs = set_max_txt_chunk_paragraphs - chunk_text_by_tokens = set_chunk_txt_by_tokens - max_tokens = set_max_txt_chunk_tokens - # FIXME - summarize_recursively = rolling_summarization - verbose = False - model = None - summary = rolling_summarize_function(text, detail, api_name, api_key, model, custom_prompt_input, - chunk_text_by_words, - max_words, chunk_text_by_sentences, - max_sentences, chunk_text_by_paragraphs, - max_paragraphs, chunk_text_by_tokens, - max_tokens, summarize_recursively, verbose - ) - - elif api_name: - summary = perform_summarization(api_name, transcription_text, custom_prompt_input, api_key) - else: - summary = None - - if summary: - # Save the summary file in the download_path directory - summary_file_path = os.path.join(download_path, f"{transcription_text}_summary.txt") - with open(summary_file_path, 'w') as file: - file.write(summary) - - add_media_to_database(path, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model) - else: - logging.error(f"Failed to download video: {path}") - else: - download_path, info_dict, urls_or_media_file = process_local_file(path) - if isinstance(urls_or_media_file, list): - # Text file containing URLs - for url in urls_or_media_file: - info_dict, title = extract_video_info(url) - download_path = create_download_directory(title) - video_path = download_video(url, download_path, info_dict, download_video_flag) - - if video_path: - audio_file, segments = perform_transcription(video_path, offset, whisper_model, vad_filter) - # FIXME - V1 - #transcription_text = {'video_path': url, 'audio_file': audio_file, 'transcription': segments} - transcription_text = {'audio_file': audio_file, 'transcription': segments} - if rolling_summarization: - text = extract_text_from_segments(segments) - summary = summarize_with_detail_openai(text, detail=detail) - elif api_name: - summary = perform_summarization(api_name, transcription_text, custom_prompt_input, api_key) - else: - summary = None - - if summary: - # Save the summary file in the download_path directory - summary_file_path = os.path.join(download_path, f"{transcription_text}_summary.txt") - with open(summary_file_path, 'w') as file: - file.write(summary) - - add_media_to_database(url, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model) - else: - logging.error(f"Failed to download video: {url}") - else: - # Video or audio file - media_path = urls_or_media_file - - if media_path.lower().endswith(('.mp4', '.avi', '.mov')): - # Video file - audio_file, segments = perform_transcription(media_path, offset, whisper_model, vad_filter) - elif media_path.lower().endswith(('.wav', '.mp3', '.m4a')): - # Audio file - segments = speech_to_text(media_path, whisper_model=whisper_model, vad_filter=vad_filter) - else: - logging.error(f"Unsupported media file format: {media_path}") - continue - - transcription_text = {'media_path': path, 'audio_file': media_path, 'transcription': segments} - - if rolling_summarization: - text = extract_text_from_segments(segments) - summary = summarize_with_detail_openai(text, detail=detail) - elif api_name: - summary = perform_summarization(api_name, transcription_text, custom_prompt_input, api_key) - else: - summary = None - - if summary: - # Save the summary file in the download_path directory - summary_file_path = os.path.join(download_path, f"{transcription_text}_summary.txt") - with open(summary_file_path, 'w') as file: - file.write(summary) - - add_media_to_database(path, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model) - - except Exception as e: - logging.error(f"Error processing {path}: {str(e)}") - continue - - return transcription_text - - -def signal_handler(sig, frame): - logging.info('Signal handler called with signal: %s', sig) - cleanup_process() - sys.exit(0) - - -############################## MAIN ############################## -# -# - -if __name__ == "__main__": - # Register signal handlers - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - # Logging setup - logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') - - # Load Config - loaded_config_data = load_and_log_configs() - - if loaded_config_data: - logging.info("Main: Configuration loaded successfully") - # You can access the configuration data like this: - # print(f"OpenAI API Key: {config_data['api_keys']['openai']}") - # print(f"Anthropic Model: {config_data['models']['anthropic']}") - # print(f"Kobold API IP: {config_data['local_apis']['kobold']['ip']}") - # print(f"Output Path: {config_data['output_path']}") - # print(f"Processing Choice: {config_data['processing_choice']}") - else: - print("Failed to load configuration") - - # Print ascii_art - print_hello() - - transcription_text = None - - parser = argparse.ArgumentParser( - description='Transcribe and summarize videos.', - epilog=''' -Sample commands: - 1. Simple Sample command structure: - summarize.py -api openai -k tag_one tag_two tag_three - - 2. Rolling Summary Sample command structure: - summarize.py -api openai -prompt "custom_prompt_goes_here-is-appended-after-transcription" -roll -detail 0.01 -k tag_one tag_two tag_three - - 3. FULL Sample command structure: - summarize.py -api openai -ns 2 -wm small.en -off 0 -vad -log INFO -prompt "custom_prompt" -overwrite -roll -detail 0.01 -k tag_one tag_two tag_three - - 4. Sample command structure for UI: - summarize.py -gui -log DEBUG - ''', - formatter_class=argparse.RawTextHelpFormatter - ) - parser.add_argument('input_path', type=str, help='Path or URL of the video', nargs='?') - parser.add_argument('-v', '--video', action='store_true', help='Download the video instead of just the audio') - parser.add_argument('-api', '--api_name', type=str, help='API name for summarization (optional)') - parser.add_argument('-key', '--api_key', type=str, help='API key for summarization (optional)') - parser.add_argument('-ns', '--num_speakers', type=int, default=2, help='Number of speakers (default: 2)') - parser.add_argument('-wm', '--whisper_model', type=str, default='small', - help='Whisper model (default: small)| Options: tiny.en, tiny, base.en, base, small.en, small, medium.en, ' - 'medium, large-v1, large-v2, large-v3, large, distil-large-v2, distil-medium.en, ' - 'distil-small.en, distil-large-v3 ') - parser.add_argument('-off', '--offset', type=int, default=0, help='Offset in seconds (default: 0)') - parser.add_argument('-vad', '--vad_filter', action='store_true', help='Enable VAD filter') - parser.add_argument('-log', '--log_level', type=str, default='INFO', - choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Log level (default: INFO)') - parser.add_argument('-gui', '--user_interface', action='store_true', help="Launch the Gradio user interface") - parser.add_argument('-demo', '--demo_mode', action='store_true', help='Enable demo mode') - parser.add_argument('-prompt', '--custom_prompt', type=str, - help='Pass in a custom prompt to be used in place of the existing one.\n (Probably should just ' - 'modify the script itself...)') - parser.add_argument('-overwrite', '--overwrite', action='store_true', help='Overwrite existing files') - parser.add_argument('-roll', '--rolling_summarization', action='store_true', help='Enable rolling summarization') - parser.add_argument('-detail', '--detail_level', type=float, help='Mandatory if rolling summarization is enabled, ' - 'defines the chunk size.\n Default is 0.01(lots ' - 'of chunks) -> 1.00 (few chunks)\n Currently ' - 'only OpenAI works. ', - default=0.01, ) - parser.add_argument('-model', '--llm_model', type=str, default='', - help='Model to use for LLM summarization (only used for vLLM/TabbyAPI)') - parser.add_argument('-k', '--keywords', nargs='+', default=['cli_ingest_no_tag'], - help='Keywords for tagging the media, can use multiple separated by spaces (default: cli_ingest_no_tag)') - parser.add_argument('--log_file', type=str, help='Where to save logfile (non-default)') - parser.add_argument('--local_llm', action='store_true', - help="Use a local LLM from the script(Downloads llamafile from github and 'mistral-7b-instruct-v0.2.Q8' - 8GB model from Huggingface)") - parser.add_argument('--server_mode', action='store_true', - help='Run in server mode (This exposes the GUI/Server to the network)') - parser.add_argument('--share_public', type=int, default=7860, - help="This will use Gradio's built-in ngrok tunneling to share the server publicly on the internet. Specify the port to use (default: 7860)") - parser.add_argument('--port', type=int, default=7860, help='Port to run the server on') - # parser.add_argument('--offload', type=int, default=20, help='Numbers of layers to offload to GPU for Llamafile usage') - # parser.add_argument('-o', '--output_path', type=str, help='Path to save the output file') - - args = parser.parse_args() - - # Set Chunking values/variables - set_chunk_txt_by_words = False - set_max_txt_chunk_words = 0 - set_chunk_txt_by_sentences = False - set_max_txt_chunk_sentences = 0 - set_chunk_txt_by_paragraphs = False - set_max_txt_chunk_paragraphs = 0 - set_chunk_txt_by_tokens = False - set_max_txt_chunk_tokens = 0 - - global server_mode - - if args.share_public: - share_public = args.share_public - else: - share_public = None - if args.server_mode: - - server_mode = args.server_mode - else: - server_mode = None - if args.server_mode is True: - server_mode = True - if args.port: - server_port = args.port - else: - server_port = None - - ########## Logging setup - logger = logging.getLogger() - logger.setLevel(getattr(logging, args.log_level)) - - # Create console handler - console_handler = logging.StreamHandler() - console_handler.setLevel(getattr(logging, args.log_level)) - console_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - console_handler.setFormatter(console_formatter) - - if args.log_file: - # Create file handler - file_handler = logging.FileHandler(args.log_file) - file_handler.setLevel(getattr(logging, args.log_level)) - file_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - file_handler.setFormatter(file_formatter) - logger.addHandler(file_handler) - logger.info(f"Log file created at: {args.log_file}") - - ########## Custom Prompt setup - custom_prompt_input = args.custom_prompt - - if not args.custom_prompt: - logging.debug("No custom prompt defined, will use default") - args.custom_prompt_input = ( - "\n\nabove is the transcript of a video. " - "Please read through the transcript carefully. Identify the main topics that are " - "discussed over the course of the transcript. Then, summarize the key points about each " - "main topic in a concise bullet point. The bullet points should cover the key " - "information conveyed about each topic in the video, but should be much shorter than " - "the full transcript. Please output your bullet point summary inside " - "tags." - ) - print("No custom prompt defined, will use default") - - custom_prompt_input = args.custom_prompt - else: - logging.debug(f"Custom prompt defined, will use \n\nf{custom_prompt_input} \n\nas the prompt") - print(f"Custom Prompt has been defined. Custom prompt: \n\n {args.custom_prompt}") - - # Check if the user wants to use the local LLM from the script - local_llm = args.local_llm - logging.info(f'Local LLM flag: {local_llm}') - - - - if not args.user_interface: - if local_llm: - local_llm_function() - time.sleep(2) - webbrowser.open_new_tab('http://127.0.0.1:7860') - launch_ui(demo_mode=False) - elif not args.input_path: - parser.print_help() - sys.exit(1) - - else: - logging.info('Starting the transcription and summarization process.') - logging.info(f'Input path: {args.input_path}') - logging.info(f'API Name: {args.api_name}') - logging.info(f'Number of speakers: {args.num_speakers}') - logging.info(f'Whisper model: {args.whisper_model}') - logging.info(f'Offset: {args.offset}') - logging.info(f'VAD filter: {args.vad_filter}') - logging.info(f'Log Level: {args.log_level}') - logging.info(f'Demo Mode: {args.demo_mode}') - logging.info(f'Custom Prompt: {args.custom_prompt}') - logging.info(f'Overwrite: {args.overwrite}') - logging.info(f'Rolling Summarization: {args.rolling_summarization}') - logging.info(f'User Interface: {args.user_interface}') - logging.info(f'Video Download: {args.video}') - # logging.info(f'Save File location: {args.output_path}') - # logging.info(f'Log File location: {args.log_file}') - - # Don't care we're running on HF - launch_ui(demo_mode=False) - - global api_name - api_name = args.api_name - - summary = None # Initialize to ensure it's always defined - if args.detail_level == None: - args.detail_level = 0.01 - - # FIXME - # if args.api_name and args.rolling_summarization and any( - # key.startswith(args.api_name) and value is not None for key, value in api_keys.items()): - # logging.info(f'MAIN: API used: {args.api_name}') - # logging.info('MAIN: Rolling Summarization will be performed.') - - elif args.api_name: - logging.info(f'MAIN: API used: {args.api_name}') - logging.info('MAIN: Summarization (not rolling) will be performed.') - - else: - logging.info('No API specified. Summarization will not be performed.') - - logging.debug("Platform check being performed...") - platform_check() - logging.debug("CUDA check being performed...") - cuda_check() - processing_choice = "cpu" - logging.debug("ffmpeg check being performed...") - check_ffmpeg() - # download_ffmpeg() - - llm_model = args.llm_model or None - # FIXME - dirty hack - args.time_based = False - - try: - results = main(args.input_path, api_name=args.api_name, api_key=args.api_key, - num_speakers=args.num_speakers, whisper_model=args.whisper_model, offset=args.offset, - vad_filter=args.vad_filter, download_video_flag=args.video, custom_prompt=args.custom_prompt_input, - overwrite=args.overwrite, rolling_summarization=args.rolling_summarization, - detail=args.detail_level, keywords=args.keywords, llm_model=args.llm_model, - time_based=args.time_based, set_chunk_txt_by_words=set_chunk_txt_by_words, - set_max_txt_chunk_words=set_max_txt_chunk_words, - set_chunk_txt_by_sentences=set_chunk_txt_by_sentences, - set_max_txt_chunk_sentences=set_max_txt_chunk_sentences, - set_chunk_txt_by_paragraphs=set_chunk_txt_by_paragraphs, - set_max_txt_chunk_paragraphs=set_max_txt_chunk_paragraphs, - set_chunk_txt_by_tokens=set_chunk_txt_by_tokens, - set_max_txt_chunk_tokens=set_max_txt_chunk_tokens) - - logging.info('Transcription process completed.') - atexit.register(cleanup_process) - except Exception as e: - logging.error('An error occurred during the transcription process.') - logging.error(str(e)) - sys.exit(1) - - finally: - cleanup_process() - -######### Words-per-second Chunking ######### -# FIXME - WHole section needs to be re-written -def chunk_transcript(transcript: str, chunk_duration: int, words_per_second) -> list[str]: - words = transcript.split() - words_per_chunk = chunk_duration * words_per_second - chunks = [' '.join(words[i:i + words_per_chunk]) for i in range(0, len(words), words_per_chunk)] - return chunks - - -def summarize_chunks(api_name: str, api_key: str, transcript: List[dict], chunk_duration: int, - words_per_second: int) -> str: - - - if not transcript: - logging.error("Empty or None transcript provided to summarize_chunks") - return "Error: Empty or None transcript provided" - - text = extract_text_from_segments(transcript) - chunks = chunk_transcript(text, chunk_duration, words_per_second) - - custom_prompt = args.custom_prompt - - summaries = [] - for chunk in chunks: - if api_name == 'openai': - # Ensure the correct model and prompt are passed - summaries.append(summarize_with_openai(api_key, chunk, custom_prompt)) - elif api_name == 'anthropic': - summaries.append(summarize_with_anthropic(api_key, chunk,custom_prompt)) - elif api_name == 'cohere': - summaries.append(summarize_with_cohere(api_key, chunk, custom_prompt)) - elif api_name == 'groq': - summaries.append(summarize_with_groq(api_key, chunk, custom_prompt)) - elif api_name == 'llama': - summaries.append(summarize_with_llama(chunk, api_key, custom_prompt)) - elif api_name == 'kobold': - summaries.append(summarize_with_kobold(chunk, api_key, custom_prompt)) - elif api_name == 'ooba': - summaries.append(summarize_with_oobabooga(chunk, api_key, custom_prompt)) - elif api_name == 'tabbyapi': - summaries.append(summarize_with_vllm(chunk, llm_model, custom_prompt)) - elif api_name == 'local-llm': - summaries.append(summarize_with_local_llm(chunk, custom_prompt)) - else: - return f"Unsupported API: {api_name}" - - return "\n\n".join(summaries) - -# FIXME - WHole section needs to be re-written -def summarize_with_detail_openai(text, detail, verbose=False): - summary_with_detail_variable = rolling_summarize(text, detail=detail, verbose=True) - print(len(openai_tokenize(summary_with_detail_variable))) - return summary_with_detail_variable - - -def summarize_with_detail_recursive_openai(text, detail, verbose=False): - summary_with_recursive_summarization = rolling_summarize(text, detail=detail, summarize_recursively=True) - print(summary_with_recursive_summarization) - -# -# -################################################################################# - - - -import csv -import logging -import os -import re -import sqlite3 -import time -from contextlib import contextmanager -from datetime import datetime -from typing import List, Tuple - -import gradio as gr -import pandas as pd - -# Import Local - - -# Set up logging -#logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -#logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - - -# Custom exceptions -class DatabaseError(Exception): - pass - - -class InputError(Exception): - pass - - -# Database connection function with connection pooling -class Database: - def __init__(self, db_name=None): - self.db_name = db_name or os.getenv('DB_NAME', 'media_summary.db') - self.pool = [] - self.pool_size = 10 - - @contextmanager - def get_connection(self): - retry_count = 5 - retry_delay = 1 - conn = None - while retry_count > 0: - try: - conn = self.pool.pop() if self.pool else sqlite3.connect(self.db_name, check_same_thread=False) - yield conn - self.pool.append(conn) - return - except sqlite3.OperationalError as e: - if 'database is locked' in str(e): - logging.warning(f"Database is locked, retrying in {retry_delay} seconds...") - retry_count -= 1 - time.sleep(retry_delay) - else: - raise DatabaseError(f"Database error: {e}") - except Exception as e: - raise DatabaseError(f"Unexpected error: {e}") - finally: - # Ensure the connection is returned to the pool even on failure - if conn: - self.pool.append(conn) - raise DatabaseError("Database is locked and retries have been exhausted") - - def execute_query(self, query: str, params: Tuple = ()) -> None: - with self.get_connection() as conn: - try: - cursor = conn.cursor() - cursor.execute(query, params) - conn.commit() - except sqlite3.Error as e: - raise DatabaseError(f"Database error: {e}, Query: {query}") - -db = Database() - - -# Function to create tables with the new media schema -def create_tables() -> None: - table_queries = [ - ''' - CREATE TABLE IF NOT EXISTS Media ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT, - title TEXT NOT NULL, - type TEXT NOT NULL, - content TEXT, - author TEXT, - ingestion_date TEXT, - prompt TEXT, - summary TEXT, - transcription_model TEXT - ) - ''', - ''' - CREATE TABLE IF NOT EXISTS Keywords ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - keyword TEXT NOT NULL UNIQUE - ) - ''', - ''' - CREATE TABLE IF NOT EXISTS MediaKeywords ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - media_id INTEGER NOT NULL, - keyword_id INTEGER NOT NULL, - FOREIGN KEY (media_id) REFERENCES Media(id), - FOREIGN KEY (keyword_id) REFERENCES Keywords(id) - ) - ''', - ''' - CREATE TABLE IF NOT EXISTS MediaVersion ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - media_id INTEGER NOT NULL, - version INTEGER NOT NULL, - prompt TEXT, - summary TEXT, - created_at TEXT NOT NULL, - FOREIGN KEY (media_id) REFERENCES Media(id) - ) - ''', - ''' - CREATE TABLE IF NOT EXISTS MediaModifications ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - media_id INTEGER NOT NULL, - prompt TEXT, - summary TEXT, - modification_date TEXT, - FOREIGN KEY (media_id) REFERENCES Media(id) - ) - ''', - ''' - CREATE VIRTUAL TABLE IF NOT EXISTS media_fts USING fts5(title, content); - ''', - ''' - CREATE VIRTUAL TABLE IF NOT EXISTS keyword_fts USING fts5(keyword); - ''', - ''' - CREATE INDEX IF NOT EXISTS idx_media_title ON Media(title); - ''', - ''' - CREATE INDEX IF NOT EXISTS idx_media_type ON Media(type); - ''', - ''' - CREATE INDEX IF NOT EXISTS idx_media_author ON Media(author); - ''', - ''' - CREATE INDEX IF NOT EXISTS idx_media_ingestion_date ON Media(ingestion_date); - ''', - ''' - CREATE INDEX IF NOT EXISTS idx_keywords_keyword ON Keywords(keyword); - ''', - ''' - CREATE INDEX IF NOT EXISTS idx_mediakeywords_media_id ON MediaKeywords(media_id); - ''', - ''' - CREATE INDEX IF NOT EXISTS idx_mediakeywords_keyword_id ON MediaKeywords(keyword_id); - ''', - ''' - CREATE INDEX IF NOT EXISTS idx_media_version_media_id ON MediaVersion(media_id); - ''' - ] - for query in table_queries: - db.execute_query(query) - -create_tables() - - -####################################################################################################################### -# Keyword-related Functions -# - -# Function to add a keyword -def add_keyword(keyword: str) -> int: - keyword = keyword.strip().lower() - with db.get_connection() as conn: - cursor = conn.cursor() - try: - cursor.execute('INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)', (keyword,)) - cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,)) - keyword_id = cursor.fetchone()[0] - cursor.execute('INSERT OR IGNORE INTO keyword_fts (rowid, keyword) VALUES (?, ?)', (keyword_id, keyword)) - logging.info(f"Keyword '{keyword}' added to keyword_fts with ID: {keyword_id}") - conn.commit() - return keyword_id - except sqlite3.IntegrityError as e: - logging.error(f"Integrity error adding keyword: {e}") - raise DatabaseError(f"Integrity error adding keyword: {e}") - except sqlite3.Error as e: - logging.error(f"Error adding keyword: {e}") - raise DatabaseError(f"Error adding keyword: {e}") - - -# Function to delete a keyword -def delete_keyword(keyword: str) -> str: - keyword = keyword.strip().lower() - with db.get_connection() as conn: - cursor = conn.cursor() - try: - cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,)) - keyword_id = cursor.fetchone() - if keyword_id: - cursor.execute('DELETE FROM Keywords WHERE keyword = ?', (keyword,)) - cursor.execute('DELETE FROM keyword_fts WHERE rowid = ?', (keyword_id[0],)) - conn.commit() - return f"Keyword '{keyword}' deleted successfully." - else: - return f"Keyword '{keyword}' not found." - except sqlite3.Error as e: - raise DatabaseError(f"Error deleting keyword: {e}") - - - -# Function to add media with keywords -def add_media_with_keywords(url, title, media_type, content, keywords, prompt, summary, transcription_model, author, ingestion_date): - # Set default values for missing fields - url = url or 'Unknown' - title = title or 'Untitled' - media_type = media_type or 'Unknown' - content = content or 'No content available' - keywords = keywords or 'default' - prompt = prompt or 'No prompt available' - summary = summary or 'No summary available' - transcription_model = transcription_model or 'Unknown' - author = author or 'Unknown' - ingestion_date = ingestion_date or datetime.now().strftime('%Y-%m-%d') - - # Ensure URL is valid - if not is_valid_url(url): - url = 'localhost' - - if media_type not in ['document', 'video', 'article']: - raise InputError("Invalid media type. Allowed types: document, video, article.") - - if ingestion_date and not is_valid_date(ingestion_date): - raise InputError("Invalid ingestion date format. Use YYYY-MM-DD.") - - if not ingestion_date: - ingestion_date = datetime.now().strftime('%Y-%m-%d') - - # Split keywords correctly by comma - keyword_list = [keyword.strip().lower() for keyword in keywords.split(',')] - - logging.info(f"URL: {url}") - logging.info(f"Title: {title}") - logging.info(f"Media Type: {media_type}") - logging.info(f"Keywords: {keywords}") - logging.info(f"Content: {content}") - logging.info(f"Prompt: {prompt}") - logging.info(f"Summary: {summary}") - logging.info(f"Author: {author}") - logging.info(f"Ingestion Date: {ingestion_date}") - logging.info(f"Transcription Model: {transcription_model}") - - try: - with db.get_connection() as conn: - cursor = conn.cursor() - - # Initialize keyword_list - keyword_list = [keyword.strip().lower() for keyword in keywords.split(',')] - - # Check if media already exists - cursor.execute('SELECT id FROM Media WHERE url = ?', (url,)) - existing_media = cursor.fetchone() - - if existing_media: - media_id = existing_media[0] - logger.info(f"Existing media found with ID: {media_id}") - - # Insert new prompt and summary into MediaModifications - cursor.execute(''' - INSERT INTO MediaModifications (media_id, prompt, summary, modification_date) - VALUES (?, ?, ?, ?) - ''', (media_id, prompt, summary, ingestion_date)) - logger.info("New summary and prompt added to MediaModifications") - else: - logger.info("New media entry being created") - - # Insert new media item - cursor.execute(''' - INSERT INTO Media (url, title, type, content, author, ingestion_date, transcription_model) - VALUES (?, ?, ?, ?, ?, ?, ?) - ''', (url, title, media_type, content, author, ingestion_date, transcription_model)) - media_id = cursor.lastrowid - - # Insert keywords and associate with media item - for keyword in keyword_list: - keyword = keyword.strip().lower() - cursor.execute('INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)', (keyword,)) - cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,)) - keyword_id = cursor.fetchone()[0] - cursor.execute('INSERT OR IGNORE INTO MediaKeywords (media_id, keyword_id) VALUES (?, ?)', (media_id, keyword_id)) - cursor.execute('INSERT INTO media_fts (rowid, title, content) VALUES (?, ?, ?)', (media_id, title, content)) - - # Also insert the initial prompt and summary into MediaModifications - cursor.execute(''' - INSERT INTO MediaModifications (media_id, prompt, summary, modification_date) - VALUES (?, ?, ?, ?) - ''', (media_id, prompt, summary, ingestion_date)) - - conn.commit() - - # Insert initial version of the prompt and summary - add_media_version(media_id, prompt, summary) - - return f"Media '{title}' added successfully with keywords: {', '.join(keyword_list)}" - except sqlite3.IntegrityError as e: - logger.error(f"Integrity Error: {e}") - raise DatabaseError(f"Integrity error adding media with keywords: {e}") - except sqlite3.Error as e: - logger.error(f"SQL Error: {e}") - raise DatabaseError(f"Error adding media with keywords: {e}") - except Exception as e: - logger.error(f"Unexpected Error: {e}") - raise DatabaseError(f"Unexpected error: {e}") - - -def fetch_all_keywords() -> list[str]: - try: - with db.get_connection() as conn: - cursor = conn.cursor() - cursor.execute('SELECT keyword FROM Keywords') - keywords = [row[0] for row in cursor.fetchall()] - return keywords - except sqlite3.Error as e: - raise DatabaseError(f"Error fetching keywords: {e}") - -def keywords_browser_interface(): - keywords = fetch_all_keywords() - return gr.Markdown("\n".join(f"- {keyword}" for keyword in keywords)) - -def display_keywords(): - try: - keywords = fetch_all_keywords() - return "\n".join(keywords) if keywords else "No keywords found." - except DatabaseError as e: - return str(e) - - -# Function to fetch items based on search query and type -def browse_items(search_query, search_type): - try: - with db.get_connection() as conn: - cursor = conn.cursor() - if search_type == 'Title': - cursor.execute("SELECT id, title, url FROM Media WHERE title LIKE ?", (f'%{search_query}%',)) - elif search_type == 'URL': - cursor.execute("SELECT id, title, url FROM Media WHERE url LIKE ?", (f'%{search_query}%',)) - results = cursor.fetchall() - return results - except sqlite3.Error as e: - raise Exception(f"Error fetching items by {search_type}: {e}") - - -# Function to fetch item details -def fetch_item_details(media_id: int): - try: - with db.get_connection() as conn: - cursor = conn.cursor() - cursor.execute("SELECT prompt, summary FROM MediaModifications WHERE media_id = ?", (media_id,)) - prompt_summary_results = cursor.fetchall() - cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,)) - content_result = cursor.fetchone() - content = content_result[0] if content_result else "" - return prompt_summary_results, content - except sqlite3.Error as e: - raise Exception(f"Error fetching item details: {e}") - -# -# -####################################################################################################################### - - - - -# Function to add a version of a prompt and summary -def add_media_version(media_id: int, prompt: str, summary: str) -> None: - try: - with db.get_connection() as conn: - cursor = conn.cursor() - - # Get the current version number - cursor.execute('SELECT MAX(version) FROM MediaVersion WHERE media_id = ?', (media_id,)) - current_version = cursor.fetchone()[0] or 0 - - # Insert the new version - cursor.execute(''' - INSERT INTO MediaVersion (media_id, version, prompt, summary, created_at) - VALUES (?, ?, ?, ?, ?) - ''', (media_id, current_version + 1, prompt, summary, datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) - conn.commit() - except sqlite3.Error as e: - raise DatabaseError(f"Error adding media version: {e}") - - -# Function to search the database with advanced options, including keyword search and full-text search -def search_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 10): - if page < 1: - raise ValueError("Page number must be 1 or greater.") - - # Prepare keywords by splitting and trimming - keywords = [keyword.strip().lower() for keyword in keywords.split(',') if keyword.strip()] - - with db.get_connection() as conn: - cursor = conn.cursor() - offset = (page - 1) * results_per_page - - # Prepare the search conditions for general fields - search_conditions = [] - params = [] - - for field in search_fields: - if search_query: # Ensure there's a search query before adding this condition - search_conditions.append(f"Media.{field} LIKE ?") - params.append(f'%{search_query}%') - - # Prepare the conditions for keywords filtering - keyword_conditions = [] - for keyword in keywords: - keyword_conditions.append( - f"EXISTS (SELECT 1 FROM MediaKeywords mk JOIN Keywords k ON mk.keyword_id = k.id WHERE mk.media_id = Media.id AND k.keyword LIKE ?)") - params.append(f'%{keyword}%') - - # Combine all conditions - where_clause = " AND ".join( - search_conditions + keyword_conditions) if search_conditions or keyword_conditions else "1=1" - - # Complete the query - query = f''' - SELECT DISTINCT Media.url, Media.title, Media.type, Media.content, Media.author, Media.ingestion_date, Media.prompt, Media.summary - FROM Media - WHERE {where_clause} - LIMIT ? OFFSET ? - ''' - params.extend([results_per_page, offset]) - - cursor.execute(query, params) - results = cursor.fetchall() - - return results - - -# Gradio function to handle user input and display results with pagination, with better feedback -def search_and_display(search_query, search_fields, keywords, page): - results = search_db(search_query, search_fields, keywords, page) - - if isinstance(results, pd.DataFrame): - # Convert DataFrame to a list of tuples or lists - processed_results = results.values.tolist() # This converts DataFrame rows to lists - elif isinstance(results, list): - # Ensure that each element in the list is itself a list or tuple (not a dictionary) - processed_results = [list(item.values()) if isinstance(item, dict) else item for item in results] - else: - raise TypeError("Unsupported data type for results") - - return processed_results - - -def display_details(index, results): - if index is None or results is None: - return "Please select a result to view details." - - try: - # Ensure the index is an integer and access the row properly - index = int(index) - if isinstance(results, pd.DataFrame): - if index >= len(results): - return "Index out of range. Please select a valid index." - selected_row = results.iloc[index] - else: - # If results is not a DataFrame, but a list (assuming list of dicts) - selected_row = results[index] - except ValueError: - return "Index must be an integer." - except IndexError: - return "Index out of range. Please select a valid index." - - # Build HTML output safely - details_html = f""" -

{selected_row.get('Title', 'No Title')}

-

URL: {selected_row.get('URL', 'No URL')}

-

Type: {selected_row.get('Type', 'No Type')}

-

Author: {selected_row.get('Author', 'No Author')}

-

Ingestion Date: {selected_row.get('Ingestion Date', 'No Date')}

-

Prompt: {selected_row.get('Prompt', 'No Prompt')}

-

Summary: {selected_row.get('Summary', 'No Summary')}

-

Content: {selected_row.get('Content', 'No Content')}

- """ - return details_html - - -def get_details(index, dataframe): - if index is None or dataframe is None or index >= len(dataframe): - return "Please select a result to view details." - row = dataframe.iloc[index] - details = f""" -

{row['Title']}

-

URL: {row['URL']}

-

Type: {row['Type']}

-

Author: {row['Author']}

-

Ingestion Date: {row['Ingestion Date']}

-

Prompt: {row['Prompt']}

-

Summary: {row['Summary']}

-

Content:

- 
{row['Content']}
- """ - return details - - -def format_results(results): - if not results: - return pd.DataFrame(columns=['URL', 'Title', 'Type', 'Content', 'Author', 'Ingestion Date', 'Prompt', 'Summary']) - - df = pd.DataFrame(results, columns=['URL', 'Title', 'Type', 'Content', 'Author', 'Ingestion Date', 'Prompt', 'Summary']) - logging.debug(f"Formatted DataFrame: {df}") - - return df - -# Function to export search results to CSV with pagination -def export_to_csv(search_query: str, search_fields: List[str], keyword: str, page: int = 1, results_per_file: int = 1000): - try: - results = search_db(search_query, search_fields, keyword, page, results_per_file) - df = format_results(results) - filename = f'search_results_page_{page}.csv' - df.to_csv(filename, index=False) - return f"Results exported to {filename}" - except (DatabaseError, InputError) as e: - return str(e) - - -# Helper function to validate URL format -def is_valid_url(url: str) -> bool: - regex = re.compile( - r'^(?:http|ftp)s?://' # http:// or https:// - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... - r'localhost|' # localhost... - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4 - r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6 - r'(?::\d+)?' # optional port - r'(?:/?|[/?]\S+)$', re.IGNORECASE) - return re.match(regex, url) is not None - - -# Helper function to validate date format -def is_valid_date(date_string: str) -> bool: - try: - datetime.strptime(date_string, '%Y-%m-%d') - return True - except ValueError: - return False - -# -# -####################################################################################################################### - - - - -####################################################################################################################### -# Functions to manage prompts DB -# - -def create_prompts_db(): - conn = sqlite3.connect('prompts.db') - cursor = conn.cursor() - cursor.execute(''' - CREATE TABLE IF NOT EXISTS Prompts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL UNIQUE, - details TEXT, - system TEXT, - user TEXT - ) - ''') - conn.commit() - conn.close() - -create_prompts_db() - - -def add_prompt(name, details, system, user=None): - try: - conn = sqlite3.connect('prompts.db') - cursor = conn.cursor() - cursor.execute(''' - INSERT INTO Prompts (name, details, system, user) - VALUES (?, ?, ?, ?) - ''', (name, details, system, user)) - conn.commit() - conn.close() - return "Prompt added successfully." - except sqlite3.IntegrityError: - return "Prompt with this name already exists." - except sqlite3.Error as e: - return f"Database error: {e}" - -def fetch_prompt_details(name): - conn = sqlite3.connect('prompts.db') - cursor = conn.cursor() - cursor.execute(''' - SELECT details, system, user - FROM Prompts - WHERE name = ? - ''', (name,)) - result = cursor.fetchone() - conn.close() - return result - -def list_prompts(): - conn = sqlite3.connect('prompts.db') - cursor = conn.cursor() - cursor.execute(''' - SELECT name - FROM Prompts - ''') - results = cursor.fetchall() - conn.close() - return [row[0] for row in results] - -def insert_prompt_to_db(title, description, system_prompt, user_prompt): - result = add_prompt(title, description, system_prompt, user_prompt) - return result - - - - -# -# -####################################################################################################################### - - - -# Summarization_General_Lib.py -######################################### -# General Summarization Library -# This library is used to perform summarization. -# -#### -import configparser -#################### -# Function List -# -# 1. extract_text_from_segments(segments: List[Dict]) -> str -# 2. summarize_with_openai(api_key, file_path, custom_prompt_arg) -# 3. summarize_with_anthropic(api_key, file_path, model, custom_prompt_arg, max_retries=3, retry_delay=5) -# 4. summarize_with_cohere(api_key, file_path, model, custom_prompt_arg) -# 5. summarize_with_groq(api_key, file_path, model, custom_prompt_arg) -# -# -#################### - - -# Import necessary libraries -import os -import logging -import time -import requests -from typing import List, Dict -import json -import configparser -from requests import RequestException - -####################################################################################################################### -# Function Definitions -# - -# def extract_text_from_segments(segments): -# text = ' '.join([segment['Text'] for segment in segments if 'Text' in segment]) -# return text -def extract_text_from_segments(segments): - logging.debug(f"Segments received: {segments}") - logging.debug(f"Type of segments: {type(segments)}") - - text = "" - - if isinstance(segments, list): - for segment in segments: - logging.debug(f"Current segment: {segment}") - logging.debug(f"Type of segment: {type(segment)}") - if 'Text' in segment: - text += segment['Text'] + " " - else: - logging.warning(f"Skipping segment due to missing 'Text' key: {segment}") - else: - logging.warning(f"Unexpected type of 'segments': {type(segments)}") - - return text.strip() - # FIXME - Dead code? - # if isinstance(segments, dict): - # if 'segments' in segments: - # segment_list = segments['segments'] - # if isinstance(segment_list, list): - # for segment in segment_list: - # logging.debug(f"Current segment: {segment}") - # logging.debug(f"Type of segment: {type(segment)}") - # if 'Text' in segment: - # text += segment['Text'] + " " - # else: - # logging.warning(f"Skipping segment due to missing 'Text' key: {segment}") - # else: - # logging.warning(f"Unexpected type of 'segments' value: {type(segment_list)}") - # else: - # logging.warning("'segments' key not found in the dictionary") - # else: - # logging.warning(f"Unexpected type of 'segments': {type(segments)}") - # - # return text.strip() - - -def summarize_with_openai(api_key, input_data, custom_prompt_arg): - loaded_config_data = summarize.load_and_log_configs() - try: - # API key validation - if api_key is None or api_key.strip() == "": - logging.info("OpenAI: API key not provided as parameter") - logging.info("OpenAI: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['openai'] - - if api_key is None or api_key.strip() == "": - logging.error("OpenAI: API key not found or is empty") - return "OpenAI: API Key Not Provided/Found in Config file or is empty" - - logging.debug(f"OpenAI: Using API Key: {api_key[:5]}...{api_key[-5:]}") - - # Input data handling - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("OpenAI: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("OpenAI: Using provided string data for summarization") - data = input_data - - logging.debug(f"OpenAI: Loaded data: {data}") - logging.debug(f"OpenAI: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("OpenAI: Summary already exists in the loaded data") - return data['summary'] - - # Text extraction - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("OpenAI: Invalid input data format") - - openai_model = loaded_config_data['models']['openai'] or "gpt-4o" - - headers = { - 'Authorization': f'Bearer {api_key}', - 'Content-Type': 'application/json' - } - - logging.debug( - f"OpenAI API Key: {openai_api_key[:5]}...{openai_api_key[-5:] if openai_api_key else None}") - logging.debug("openai: Preparing data + prompt for submittal") - openai_prompt = f"{text} \n\n\n\n{custom_prompt_arg}" - data = { - "model": openai_model, - "messages": [ - {"role": "system", "content": "You are a professional summarizer."}, - {"role": "user", "content": openai_prompt} - ], - "max_tokens": 4096, - "temperature": 0.1 - } - - logging.debug("openai: Posting request") - response = requests.post('https://api.openai.com/v1/chat/completions', headers=headers, json=data) - - if response.status_code == 200: - response_data = response.json() - if 'choices' in response_data and len(response_data['choices']) > 0: - summary = response_data['choices'][0]['message']['content'].strip() - logging.debug("openai: Summarization successful") - return summary - else: - logging.warning("openai: Summary not found in the response data") - return "openai: Summary not available" - else: - logging.error(f"openai: Summarization failed with status code {response.status_code}") - logging.error(f"openai: Error response: {response.text}") - return f"openai: Failed to process summary. Status code: {response.status_code}" - except Exception as e: - logging.error(f"openai: Error in processing: {str(e)}", exc_info=True) - return f"openai: Error occurred while processing summary: {str(e)}" - - -def summarize_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5): - try: - loaded_config_data = summarize.load_and_log_configs() - global anthropic_api_key - # API key validation - if api_key is None: - logging.info("Anthropic: API key not provided as parameter") - logging.info("Anthropic: Attempting to use API key from config file") - anthropic_api_key = loaded_config_data['api_keys']['anthropic'] - - if api_key is None or api_key.strip() == "": - logging.error("Anthropic: API key not found or is empty") - return "Anthropic: API Key Not Provided/Found in Config file or is empty" - - logging.debug(f"Anthropic: Using API Key: {api_key[:5]}...{api_key[-5:]}") - - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("AnthropicAI: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("AnthropicAI: Using provided string data for summarization") - data = input_data - - logging.debug(f"AnthropicAI: Loaded data: {data}") - logging.debug(f"AnthropicAI: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("Anthropic: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("Anthropic: Invalid input data format") - - anthropic_model = loaded_config_data['models']['anthropic'] - - headers = { - 'x-api-key': anthropic_api_key, - 'anthropic-version': '2023-06-01', - 'Content-Type': 'application/json' - } - - anthropic_prompt = custom_prompt_arg - logging.debug(f"Anthropic: Prompt is {anthropic_prompt}") - user_message = { - "role": "user", - "content": f"{text} \n\n\n\n{anthropic_prompt}" - } - - data = { - "model": model, - "max_tokens": 4096, # max _possible_ tokens to return - "messages": [user_message], - "stop_sequences": ["\n\nHuman:"], - "temperature": 0.1, - "top_k": 0, - "top_p": 1.0, - "metadata": { - "user_id": "example_user_id", - }, - "stream": False, - "system": "You are a professional summarizer." - } - - for attempt in range(max_retries): - try: - logging.debug("anthropic: Posting request to API") - response = requests.post('https://api.anthropic.com/v1/messages', headers=headers, json=data) - - # Check if the status code indicates success - if response.status_code == 200: - logging.debug("anthropic: Post submittal successful") - response_data = response.json() - try: - summary = response_data['content'][0]['text'].strip() - logging.debug("anthropic: Summarization successful") - print("Summary processed successfully.") - return summary - except (IndexError, KeyError) as e: - logging.debug("anthropic: Unexpected data in response") - print("Unexpected response format from Anthropic API:", response.text) - return None - elif response.status_code == 500: # Handle internal server error specifically - logging.debug("anthropic: Internal server error") - print("Internal server error from API. Retrying may be necessary.") - time.sleep(retry_delay) - else: - logging.debug( - f"anthropic: Failed to summarize, status code {response.status_code}: {response.text}") - print(f"Failed to process summary, status code {response.status_code}: {response.text}") - return None - - except RequestException as e: - logging.error(f"anthropic: Network error during attempt {attempt + 1}/{max_retries}: {str(e)}") - if attempt < max_retries - 1: - time.sleep(retry_delay) - else: - return f"anthropic: Network error: {str(e)}" - except FileNotFoundError as e: - logging.error(f"anthropic: File not found: {input_data}") - return f"anthropic: File not found: {input_data}" - except json.JSONDecodeError as e: - logging.error(f"anthropic: Invalid JSON format in file: {input_data}") - return f"anthropic: Invalid JSON format in file: {input_data}" - except Exception as e: - logging.error(f"anthropic: Error in processing: {str(e)}") - return f"anthropic: Error occurred while processing summary with Anthropic: {str(e)}" - - -# Summarize with Cohere -def summarize_with_cohere(api_key, input_data, model, custom_prompt_arg): - loaded_config_data = summarize.load_and_log_configs() - try: - # API key validation - if api_key is None: - logging.info("cohere: API key not provided as parameter") - logging.info("cohere: Attempting to use API key from config file") - cohere_api_key = loaded_config_data['api_keys']['cohere'] - - if api_key is None or api_key.strip() == "": - logging.error("cohere: API key not found or is empty") - return "cohere: API Key Not Provided/Found in Config file or is empty" - - logging.debug(f"cohere: Using API Key: {api_key[:5]}...{api_key[-5:]}") - - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("Cohere: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("Cohere: Using provided string data for summarization") - data = input_data - - logging.debug(f"Cohere: Loaded data: {data}") - logging.debug(f"Cohere: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("Cohere: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("Invalid input data format") - - cohere_model = loaded_config_data['models']['cohere'] - - headers = { - 'accept': 'application/json', - 'content-type': 'application/json', - 'Authorization': f'Bearer {cohere_api_key}' - } - - cohere_prompt = f"{text} \n\n\n\n{custom_prompt_arg}" - logging.debug("cohere: Prompt being sent is {cohere_prompt}") - - data = { - "chat_history": [ - {"role": "USER", "message": cohere_prompt} - ], - "message": "Please provide a summary.", - "model": model, - "connectors": [{"id": "web-search"}] - } - - logging.debug("cohere: Submitting request to API endpoint") - print("cohere: Submitting request to API endpoint") - response = requests.post('https://api.cohere.ai/v1/chat', headers=headers, json=data) - response_data = response.json() - logging.debug("API Response Data: %s", response_data) - - if response.status_code == 200: - if 'text' in response_data: - summary = response_data['text'].strip() - logging.debug("cohere: Summarization successful") - print("Summary processed successfully.") - return summary - else: - logging.error("Expected data not found in API response.") - return "Expected data not found in API response." - else: - logging.error(f"cohere: API request failed with status code {response.status_code}: {response.text}") - print(f"Failed to process summary, status code {response.status_code}: {response.text}") - return f"cohere: API request failed: {response.text}" - - except Exception as e: - logging.error("cohere: Error in processing: %s", str(e)) - return f"cohere: Error occurred while processing summary with Cohere: {str(e)}" - - -# https://console.groq.com/docs/quickstart -def summarize_with_groq(api_key, input_data, custom_prompt_arg): - loaded_config_data = summarize.load_and_log_configs() - try: - # API key validation - if api_key is None: - logging.info("groq: API key not provided as parameter") - logging.info("groq: Attempting to use API key from config file") - groq_api_key = loaded_config_data['api_keys']['groq'] - - if api_key is None or api_key.strip() == "": - logging.error("groq: API key not found or is empty") - return "groq: API Key Not Provided/Found in Config file or is empty" - - logging.debug(f"groq: Using API Key: {api_key[:5]}...{api_key[-5:]}") - - # Transcript data handling & Validation - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("Groq: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("Groq: Using provided string data for summarization") - data = input_data - - logging.debug(f"Groq: Loaded data: {data}") - logging.debug(f"Groq: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("Groq: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("Groq: Invalid input data format") - - # Set the model to be used - groq_model = loaded_config_data['models']['groq'] - - headers = { - 'Authorization': f'Bearer {api_key}', - 'Content-Type': 'application/json' - } - - groq_prompt = f"{text} \n\n\n\n{custom_prompt_arg}" - logging.debug("groq: Prompt being sent is {groq_prompt}") - - data = { - "messages": [ - { - "role": "user", - "content": groq_prompt - } - ], - "model": groq_model - } - - logging.debug("groq: Submitting request to API endpoint") - print("groq: Submitting request to API endpoint") - response = requests.post('https://api.groq.com/openai/v1/chat/completions', headers=headers, json=data) - - response_data = response.json() - logging.debug("API Response Data: %s", response_data) - - if response.status_code == 200: - if 'choices' in response_data and len(response_data['choices']) > 0: - summary = response_data['choices'][0]['message']['content'].strip() - logging.debug("groq: Summarization successful") - print("Summarization successful.") - return summary - else: - logging.error("Expected data not found in API response.") - return "Expected data not found in API response." - else: - logging.error(f"groq: API request failed with status code {response.status_code}: {response.text}") - return f"groq: API request failed: {response.text}" - - except Exception as e: - logging.error("groq: Error in processing: %s", str(e)) - return f"groq: Error occurred while processing summary with groq: {str(e)}" - - -def summarize_with_openrouter(api_key, input_data, custom_prompt_arg): - loaded_config_data = summarize.load_and_log_configs() - import requests - import json - global openrouter_model, openrouter_api_key - # API key validation - if api_key is None: - logging.info("openrouter: API key not provided as parameter") - logging.info("openrouter: Attempting to use API key from config file") - openrouter_api_key = loaded_config_data['api_keys']['openrouter'] - - if api_key is None or api_key.strip() == "": - logging.error("openrouter: API key not found or is empty") - return "openrouter: API Key Not Provided/Found in Config file or is empty" - - logging.debug(f"openai: Using API Key: {api_key[:5]}...{api_key[-5:]}") - - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("openrouter: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("openrouter: Using provided string data for summarization") - data = input_data - - logging.debug(f"openrouter: Loaded data: {data}") - logging.debug(f"openrouter: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("openrouter: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("Invalid input data format") - - config = configparser.ConfigParser() - file_path = 'config.txt' - - # Check if the file exists in the specified path - if os.path.exists(file_path): - config.read(file_path) - elif os.path.exists('config.txt'): # Check in the current directory - config.read('../config.txt') - else: - print("config.txt not found in the specified path or current directory.") - - openrouter_prompt = f"{input_data} \n\n\n\n{custom_prompt_arg}" - - try: - logging.debug("openrouter: Submitting request to API endpoint") - print("openrouter: Submitting request to API endpoint") - response = requests.post( - url="https://openrouter.ai/api/v1/chat/completions", - headers={ - "Authorization": f"Bearer {openrouter_api_key}", - }, - data=json.dumps({ - "model": f"{openrouter_model}", - "messages": [ - {"role": "user", "content": openrouter_prompt} - ] - }) - ) - - response_data = response.json() - logging.debug("API Response Data: %s", response_data) - - if response.status_code == 200: - if 'choices' in response_data and len(response_data['choices']) > 0: - summary = response_data['choices'][0]['message']['content'].strip() - logging.debug("openrouter: Summarization successful") - print("openrouter: Summarization successful.") - return summary - else: - logging.error("openrouter: Expected data not found in API response.") - return "openrouter: Expected data not found in API response." - else: - logging.error(f"openrouter: API request failed with status code {response.status_code}: {response.text}") - return f"openrouter: API request failed: {response.text}" - except Exception as e: - logging.error("openrouter: Error in processing: %s", str(e)) - return f"openrouter: Error occurred while processing summary with openrouter: {str(e)}" - -def summarize_with_huggingface(api_key, input_data, custom_prompt_arg): - loaded_config_data = summarize.load_and_log_configs() - global huggingface_api_key - logging.debug(f"huggingface: Summarization process starting...") - try: - # API key validation - if api_key is None: - logging.info("HuggingFace: API key not provided as parameter") - logging.info("HuggingFace: Attempting to use API key from config file") - huggingface_api_key = loaded_config_data['api_keys']['openai'] - - if api_key is None or api_key.strip() == "": - logging.error("HuggingFace: API key not found or is empty") - return "HuggingFace: API Key Not Provided/Found in Config file or is empty" - - logging.debug(f"HuggingFace: Using API Key: {api_key[:5]}...{api_key[-5:]}") - - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("HuggingFace: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("HuggingFace: Using provided string data for summarization") - data = input_data - - logging.debug(f"HuggingFace: Loaded data: {data}") - logging.debug(f"HuggingFace: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("HuggingFace: Summary already exists in the loaded data") - return data['summary'] - - # If the loaded data is a list of segment dictionaries or a string, proceed with summarization - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("HuggingFace: Invalid input data format") - - print(f"HuggingFace: lets make sure the HF api key exists...\n\t {api_key}") - headers = { - "Authorization": f"Bearer {api_key}" - } - - huggingface_model = loaded_config_data['models']['huggingface'] - API_URL = f"https://api-inference.huggingface.co/models/{huggingface_model}" - - huggingface_prompt = f"{text}\n\n\n\n{custom_prompt_arg}" - logging.debug("huggingface: Prompt being sent is {huggingface_prompt}") - data = { - "inputs": text, - "parameters": {"max_length": 512, "min_length": 100} # You can adjust max_length and min_length as needed - } - - print(f"huggingface: lets make sure the HF api key is the same..\n\t {huggingface_api_key}") - - logging.debug("huggingface: Submitting request...") - - response = requests.post(API_URL, headers=headers, json=data) - - if response.status_code == 200: - summary = response.json()[0]['summary_text'] - logging.debug("huggingface: Summarization successful") - print("Summarization successful.") - return summary - else: - logging.error(f"huggingface: Summarization failed with status code {response.status_code}: {response.text}") - return f"Failed to process summary, status code {response.status_code}: {response.text}" - except Exception as e: - logging.error("huggingface: Error in processing: %s", str(e)) - print(f"Error occurred while processing summary with huggingface: {str(e)}") - return None - - -def summarize_with_deepseek(api_key, input_data, custom_prompt_arg): - loaded_config_data = summarize.load_and_log_configs() - try: - # API key validation - if api_key is None or api_key.strip() == "": - logging.info("DeepSeek: API key not provided as parameter") - logging.info("DeepSeek: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['deepseek'] - - if api_key is None or api_key.strip() == "": - logging.error("DeepSeek: API key not found or is empty") - return "DeepSeek: API Key Not Provided/Found in Config file or is empty" - - logging.debug(f"DeepSeek: Using API Key: {api_key[:5]}...{api_key[-5:]}") - - # Input data handling - if isinstance(input_data, str) and os.path.isfile(input_data): - logging.debug("DeepSeek: Loading json data for summarization") - with open(input_data, 'r') as file: - data = json.load(file) - else: - logging.debug("DeepSeek: Using provided string data for summarization") - data = input_data - - logging.debug(f"DeepSeek: Loaded data: {data}") - logging.debug(f"DeepSeek: Type of data: {type(data)}") - - if isinstance(data, dict) and 'summary' in data: - # If the loaded data is a dictionary and already contains a summary, return it - logging.debug("DeepSeek: Summary already exists in the loaded data") - return data['summary'] - - # Text extraction - if isinstance(data, list): - segments = data - text = extract_text_from_segments(segments) - elif isinstance(data, str): - text = data - else: - raise ValueError("DeepSeek: Invalid input data format") - - deepseek_model = loaded_config_data['models']['deepseek'] or "deepseek-chat" - - headers = { - 'Authorization': f'Bearer {api_key}', - 'Content-Type': 'application/json' - } - - logging.debug( - f"Deepseek API Key: {api_key[:5]}...{api_key[-5:] if api_key else None}") - logging.debug("openai: Preparing data + prompt for submittal") - deepseek_prompt = f"{text} \n\n\n\n{custom_prompt_arg}" - data = { - "model": deepseek_model, - "messages": [ - {"role": "system", "content": "You are a professional summarizer."}, - {"role": "user", "content": deepseek_prompt} - ], - "stream": False, - "temperature": 0.8 - } - - logging.debug("DeepSeek: Posting request") - response = requests.post('https://api.deepseek.com/chat/completions', headers=headers, json=data) - - if response.status_code == 200: - response_data = response.json() - if 'choices' in response_data and len(response_data['choices']) > 0: - summary = response_data['choices'][0]['message']['content'].strip() - logging.debug("DeepSeek: Summarization successful") - return summary - else: - logging.warning("DeepSeek: Summary not found in the response data") - return "DeepSeek: Summary not available" - else: - logging.error(f"DeepSeek: Summarization failed with status code {response.status_code}") - logging.error(f"DeepSeek: Error response: {response.text}") - return f"DeepSeek: Failed to process summary. Status code: {response.status_code}" - except Exception as e: - logging.error(f"DeepSeek: Error in processing: {str(e)}", exc_info=True) - return f"DeepSeek: Error occurred while processing summary: {str(e)}" - - -# -# -####################################################################################################################### - - - - -# System_Checks_Lib.py -######################################### -# System Checks Library -# This library is used to check the system for the necessary dependencies to run the script. -# It checks for the OS, the availability of the GPU, and the availability of the ffmpeg executable. -# If the GPU is available, it asks the user if they would like to use it for processing. -# If ffmpeg is not found, it asks the user if they would like to download it. -# The script will exit if the user chooses not to download ffmpeg. -#### - -#################### -# Function List -# -# 1. platform_check() -# 2. cuda_check() -# 3. decide_cpugpu() -# 4. check_ffmpeg() -# 5. download_ffmpeg() -# -#################### - - - - -# Import necessary libraries -import os -import platform -import subprocess -import shutil -import zipfile -import logging - - -####################################################################################################################### -# Function Definitions -# - -def platform_check(): - global userOS - if platform.system() == "Linux": - print("Linux OS detected \n Running Linux appropriate commands") - userOS = "Linux" - elif platform.system() == "Windows": - print("Windows OS detected \n Running Windows appropriate commands") - userOS = "Windows" - else: - print("Other OS detected \n Maybe try running things manually?") - exit() - - -# Check for NVIDIA GPU and CUDA availability -def cuda_check(): - global processing_choice - try: - # Run nvidia-smi to capture its output - nvidia_smi_output = subprocess.check_output("nvidia-smi", shell=True).decode() - - # Look for CUDA version in the output - if "CUDA Version" in nvidia_smi_output: - cuda_version = next( - (line.split(":")[-1].strip() for line in nvidia_smi_output.splitlines() if "CUDA Version" in line), - "Not found") - print(f"NVIDIA GPU with CUDA Version {cuda_version} is available.") - processing_choice = "cuda" - else: - print("CUDA is not installed or configured correctly.") - processing_choice = "cpu" - - except subprocess.CalledProcessError as e: - print(f"Failed to run 'nvidia-smi': {str(e)}") - processing_choice = "cpu" - except Exception as e: - print(f"An error occurred: {str(e)}") - processing_choice = "cpu" - - # Optionally, check for the CUDA_VISIBLE_DEVICES env variable as an additional check - if "CUDA_VISIBLE_DEVICES" in os.environ: - print("CUDA_VISIBLE_DEVICES is set:", os.environ["CUDA_VISIBLE_DEVICES"]) - else: - print("CUDA_VISIBLE_DEVICES not set.") - - -# Ask user if they would like to use either their GPU or their CPU for transcription -def decide_cpugpu(): - global processing_choice - processing_input = input("Would you like to use your GPU or CPU for transcription? (1/cuda)GPU/(2/cpu)CPU): ") - if processing_choice == "cuda" and (processing_input.lower() == "cuda" or processing_input == "1"): - print("You've chosen to use the GPU.") - logging.debug("GPU is being used for processing") - processing_choice = "cuda" - elif processing_input.lower() == "cpu" or processing_input == "2": - print("You've chosen to use the CPU.") - logging.debug("CPU is being used for processing") - processing_choice = "cpu" - else: - print("Invalid choice. Please select either GPU or CPU.") - - -# check for existence of ffmpeg -def check_ffmpeg(): - if shutil.which("ffmpeg") or (os.path.exists("Bin") and os.path.isfile(".\\Bin\\ffmpeg.exe")): - logging.debug("ffmpeg found installed on the local system, in the local PATH, or in the './Bin' folder") - pass - else: - logging.debug("ffmpeg not installed on the local system/in local PATH") - print( - "ffmpeg is not installed.\n\n You can either install it manually, or through your package manager of " - "choice.\n Windows users, builds are here: https://www.gyan.dev/ffmpeg/builds/") - if userOS == "Windows": - download_ffmpeg() - elif userOS == "Linux": - print( - "You should install ffmpeg using your platform's appropriate package manager, 'apt install ffmpeg'," - "'dnf install ffmpeg' or 'pacman', etc.") - else: - logging.debug("running an unsupported OS") - print("You're running an unspported/Un-tested OS") - exit_script = input("Let's exit the script, unless you're feeling lucky? (y/n)") - if exit_script == "y" or "yes" or "1": - exit() - - -# Download ffmpeg -def download_ffmpeg(): - user_choice = input("Do you want to download ffmpeg? (y)Yes/(n)No: ") - if user_choice.lower() in ['yes', 'y', '1']: - print("Downloading ffmpeg") - url = "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.zip" - response = requests.get(url) - - if response.status_code == 200: - print("Saving ffmpeg zip file") - logging.debug("Saving ffmpeg zip file") - zip_path = "ffmpeg-release-essentials.zip" - with open(zip_path, 'wb') as file: - file.write(response.content) - - logging.debug("Extracting the 'ffmpeg.exe' file from the zip") - print("Extracting ffmpeg.exe from zip file to '/Bin' folder") - with zipfile.ZipFile(zip_path, 'r') as zip_ref: - # Find the ffmpeg.exe file within the zip - ffmpeg_path = None - for file_info in zip_ref.infolist(): - if file_info.filename.endswith("ffmpeg.exe"): - ffmpeg_path = file_info.filename - break - - if ffmpeg_path is None: - logging.error("ffmpeg.exe not found in the zip file.") - print("ffmpeg.exe not found in the zip file.") - return - - logging.debug("checking if the './Bin' folder exists, creating if not") - bin_folder = "Bin" - if not os.path.exists(bin_folder): - logging.debug("Creating a folder for './Bin', it didn't previously exist") - os.makedirs(bin_folder) - - logging.debug("Extracting 'ffmpeg.exe' to the './Bin' folder") - zip_ref.extract(ffmpeg_path, path=bin_folder) - - logging.debug("Moving 'ffmpeg.exe' to the './Bin' folder") - src_path = os.path.join(bin_folder, ffmpeg_path) - dst_path = os.path.join(bin_folder, "ffmpeg.exe") - shutil.move(src_path, dst_path) - - logging.debug("Removing ffmpeg zip file") - print("Deleting zip file (we've already extracted ffmpeg.exe, no worries)") - os.remove(zip_path) - - logging.debug("ffmpeg.exe has been downloaded and extracted to the './Bin' folder.") - print("ffmpeg.exe has been successfully downloaded and extracted to the './Bin' folder.") - else: - logging.error("Failed to download the zip file.") - print("Failed to download the zip file.") - else: - logging.debug("User chose to not download ffmpeg") - print("ffmpeg will not be downloaded.") - -# -# -####################################################################################################################### -import tiktoken -def openai_tokenize(text: str) -> list[str]: - encoding = tiktoken.encoding_for_model('gpt-4-turbo') - return encoding.encode(text) - - - -# Video_DL_Ingestion_Lib.py -######################################### -# Video Downloader and Ingestion Library -# This library is used to handle downloading videos from YouTube and other platforms. -# It also handles the ingestion of the videos into the database. -# It uses yt-dlp to extract video information and download the videos. -#### - -#################### -# Function List -# -# 1. get_video_info(url) -# 2. create_download_directory(title) -# 3. sanitize_filename(title) -# 4. normalize_title(title) -# 5. get_youtube(video_url) -# 6. get_playlist_videos(playlist_url) -# 7. download_video(video_url, download_path, info_dict, download_video_flag) -# 8. save_to_file(video_urls, filename) -# 9. save_summary_to_file(summary, file_path) -# 10. process_url(url, num_speakers, whisper_model, custom_prompt, offset, api_name, api_key, vad_filter, download_video, download_audio, rolling_summarization, detail_level, question_box, keywords, chunk_summarization, chunk_duration_input, words_per_second_input) -# -# -#################### - - -# Import necessary libraries to run solo for testing -from datetime import datetime -import json -import logging -import os -import re -import subprocess -import sys -import unicodedata -# 3rd-Party Imports -import yt_dlp - - - -####################################################################################################################### -# Function Definitions -# - -def get_video_info(url: str) -> dict: - ydl_opts = { - 'quiet': True, - 'no_warnings': True, - 'skip_download': True, - } - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - try: - info_dict = ydl.extract_info(url, download=False) - return info_dict - except Exception as e: - logging.error(f"Error extracting video info: {e}") - return None - - -def create_download_directory(title): - base_dir = "Results" - # Remove characters that are illegal in Windows filenames and normalize - safe_title = normalize_title(title) - logging.debug(f"{title} successfully normalized") - session_path = os.path.join(base_dir, safe_title) - if not os.path.exists(session_path): - os.makedirs(session_path, exist_ok=True) - logging.debug(f"Created directory for downloaded video: {session_path}") - else: - logging.debug(f"Directory already exists for downloaded video: {session_path}") - return session_path - - -def sanitize_filename(filename): - # Remove invalid characters and replace spaces with underscores - sanitized = re.sub(r'[<>:"/\\|?*]', '', filename) - sanitized = re.sub(r'\s+', ' ', sanitized).strip() - return sanitized - - -def normalize_title(title): - # Normalize the string to 'NFKD' form and encode to 'ascii' ignoring non-ascii characters - title = unicodedata.normalize('NFKD', title).encode('ascii', 'ignore').decode('ascii') - title = title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('"', '').replace('*', '').replace('?', - '').replace( - '<', '').replace('>', '').replace('|', '') - return title - - -def get_youtube(video_url): - ydl_opts = { - 'format': 'bestaudio[ext=m4a]', - 'noplaylist': False, - 'quiet': True, - 'extract_flat': True - } - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - logging.debug("About to extract youtube info") - info_dict = ydl.extract_info(video_url, download=False) - logging.debug("Youtube info successfully extracted") - return info_dict - - -def get_playlist_videos(playlist_url): - ydl_opts = { - 'extract_flat': True, - 'skip_download': True, - 'quiet': True - } - - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - info = ydl.extract_info(playlist_url, download=False) - - if 'entries' in info: - video_urls = [entry['url'] for entry in info['entries']] - playlist_title = info['title'] - return video_urls, playlist_title - else: - print("No videos found in the playlist.") - return [], None - - -def download_video(video_url, download_path, info_dict, download_video_flag): - global video_file_path, ffmpeg_path - global audio_file_path - - # Normalize Video Title name - logging.debug("About to normalize downloaded video title") - if 'title' not in info_dict or 'ext' not in info_dict: - logging.error("info_dict is missing 'title' or 'ext'") - return None - - normalized_video_title = normalize_title(info_dict['title']) - video_file_path = os.path.join(download_path, f"{normalized_video_title}.{info_dict['ext']}") - - # Check for existence of video file - if os.path.exists(video_file_path): - logging.info(f"Video file already exists: {video_file_path}") - return video_file_path - - # Setup path handling for ffmpeg on different OSs - if sys.platform.startswith('win'): - ffmpeg_path = os.path.join(os.getcwd(), 'Bin', 'ffmpeg.exe') - elif sys.platform.startswith('linux'): - ffmpeg_path = 'ffmpeg' - elif sys.platform.startswith('darwin'): - ffmpeg_path = 'ffmpeg' - - if download_video_flag: - video_file_path = os.path.join(download_path, f"{normalized_video_title}.mp4") - ydl_opts_video = { - 'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', - 'outtmpl': video_file_path, - 'ffmpeg_location': ffmpeg_path - } - - try: - with yt_dlp.YoutubeDL(ydl_opts_video) as ydl: - logging.debug("yt_dlp: About to download video with youtube-dl") - ydl.download([video_url]) - logging.debug("yt_dlp: Video successfully downloaded with youtube-dl") - if os.path.exists(video_file_path): - return video_file_path - else: - logging.error("yt_dlp: Video file not found after download") - return None - except Exception as e: - logging.error(f"yt_dlp: Error downloading video: {e}") - return None - elif not download_video_flag: - video_file_path = os.path.join(download_path, f"{normalized_video_title}.mp4") - # Set options for video and audio - ydl_opts = { - 'format': 'bestaudio[ext=m4a]', - 'quiet': True, - 'outtmpl': video_file_path - } + results = main(args.input_path, api_name=args.api_name, api_key=args.api_key, + num_speakers=args.num_speakers, whisper_model=args.whisper_model, offset=args.offset, + vad_filter=args.vad_filter, download_video_flag=args.video, custom_prompt=args.custom_prompt_input, + overwrite=args.overwrite, rolling_summarization=args.rolling_summarization, + detail=args.detail_level, keywords=args.keywords, llm_model=args.llm_model, + time_based=args.time_based, set_chunk_txt_by_words=set_chunk_txt_by_words, + set_max_txt_chunk_words=set_max_txt_chunk_words, + set_chunk_txt_by_sentences=set_chunk_txt_by_sentences, + set_max_txt_chunk_sentences=set_max_txt_chunk_sentences, + set_chunk_txt_by_paragraphs=set_chunk_txt_by_paragraphs, + set_max_txt_chunk_paragraphs=set_max_txt_chunk_paragraphs, + set_chunk_txt_by_tokens=set_chunk_txt_by_tokens, + set_max_txt_chunk_tokens=set_max_txt_chunk_tokens) - try: - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - logging.debug("yt_dlp: About to download video with youtube-dl") - ydl.download([video_url]) - logging.debug("yt_dlp: Video successfully downloaded with youtube-dl") - if os.path.exists(video_file_path): - return video_file_path - else: - logging.error("yt_dlp: Video file not found after download") - return None + logging.info('Transcription process completed.') + atexit.register(cleanup_process) except Exception as e: - logging.error(f"yt_dlp: Error downloading video: {e}") - return None - - else: - logging.debug("download_video: Download video flag is set to False and video file path is not found") - return None - - - - -def save_to_file(video_urls, filename): - with open(filename, 'w') as file: - file.write('\n'.join(video_urls)) - print(f"Video URLs saved to {filename}") + logging.error('An error occurred during the transcription process.') + logging.error(str(e)) + sys.exit(1) -# -# -####################################################################################################################### + finally: + cleanup_process()