Spaces:

chinmayjha
/

context-ai

Sleeping

File size: 38,467 Bytes

b27eb78

<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>

<meta charset="utf-8">
<meta name="generator" content="quarto-1.7.29">

<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">


<title>readme</title>
<style>
code{white-space: pre-wrap;}
span.smallcaps{font-variant: small-caps;}
div.columns{display: flex; gap: min(4vw, 1.5em);}
div.column{flex: auto; overflow-x: auto;}
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
ul.task-list{list-style: none;}
ul.task-list li input[type="checkbox"] {
  width: 0.8em;
  margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */ 
  vertical-align: middle;
}
/* CSS for syntax highlighting */
html { -webkit-text-size-adjust: 100%; }
pre > code.sourceCode { white-space: pre; position: relative; }
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
pre > code.sourceCode > span:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode > span { color: inherit; text-decoration: inherit; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
pre > code.sourceCode { white-space: pre-wrap; }
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
}
pre.numberSource code
  { counter-reset: source-line 0; }
pre.numberSource code > span
  { position: relative; left: -4em; counter-increment: source-line; }
pre.numberSource code > span > a:first-child::before
  { content: counter(source-line);
    position: relative; left: -1em; text-align: right; vertical-align: baseline;
    border: none; display: inline-block;
    -webkit-touch-callout: none; -webkit-user-select: none;
    -khtml-user-select: none; -moz-user-select: none;
    -ms-user-select: none; user-select: none;
    padding: 0 4px; width: 4em;
  }
pre.numberSource { margin-left: 3em;  padding-left: 4px; }
div.sourceCode
  {   }
@media screen {
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
}
</style>


<script src="README_files/libs/clipboard/clipboard.min.js"></script>
<script src="README_files/libs/quarto-html/quarto.js" type="module"></script>
<script src="README_files/libs/quarto-html/tabsets/tabsets.js" type="module"></script>
<script src="README_files/libs/quarto-html/popper.min.js"></script>
<script src="README_files/libs/quarto-html/tippy.umd.min.js"></script>
<script src="README_files/libs/quarto-html/anchor.min.js"></script>
<link href="README_files/libs/quarto-html/tippy.css" rel="stylesheet">
<link href="README_files/libs/quarto-html/quarto-syntax-highlighting-0815c480559380816a4d1ea211a47e91.css" rel="stylesheet" id="quarto-text-highlighting-styles">
<script src="README_files/libs/bootstrap/bootstrap.min.js"></script>
<link href="README_files/libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
<link href="README_files/libs/bootstrap/bootstrap-81267100e462c21b3d6c0d5bf76a3417.min.css" rel="stylesheet" append-hash="true" id="quarto-bootstrap" data-mode="light">


</head>

<body class="fullcontent quarto-light">

<div id="quarto-content" class="page-columns page-rows-contents page-layout-article">

<main class="content" id="quarto-document-content">




<table style="border-collapse: collapse; border: none;">
<tbody><tr style="border: none;">
<td width="20%" style="border: none;">
<a href="https://decodingml.substack.com/" aria-label="Decoding ML"> <img src="https://github.com/user-attachments/assets/f2f2f9c0-54b7-4ae3-bf8d-23a359c86982" alt="Decoding ML Logo" width="150"> </a>
</td>
<td width="80%" style="border: none;">
<div>
<pre><code>    &lt;h2&gt;📬 Stay Updated&lt;/h2&gt;
    &lt;p&gt;&lt;b&gt;&lt;a href="https://decodingml.substack.com/"&gt;Join Decoding ML&lt;/a&gt;&lt;/b&gt; for proven content on designing, coding, and deploying production-grade AI systems with software engineering and MLOps best practices to help you ship AI applications. Every week, straight to your inbox.&lt;/p&gt;
  &lt;/div&gt;
&lt;/td&gt;</code></pre>
</div></td></tr>
</tbody></table>
<p align="center">
<a href="https://decodingml.substack.com/"> <img src="https://img.shields.io/static/v1?label&amp;logo=substack&amp;message=Subscribe%20Now&amp;style=for-the-badge&amp;color=black&amp;scale=2" alt="Subscribe Now" height="40"> </a>
</p>
<hr>
<section id="installation-and-usage-guide-for-the-second-brain-online-module" class="level1">
<h1>🚀 Installation and Usage Guide for the Second Brain Online Module</h1>
<p>This guide will help you set up and run the Second Brain Online Module, which contains the code for <strong>Module 6: the RAG agentic app and LLMOps layer</strong>.</p>
<p>Note that this module is entirely independent of the offline ML pipelines. Thus, it comes with its own set of dependencies and requirements. Also, to run this module, you must first go through the steps from the <a href="../second-brain-offline">Offline ML Pipelines</a> part of the course, which will populate the vector database and other required resources.</p>
</section>
<section id="table-of-contents" class="level1">
<h1>📑 Table of Contents</h1>
<ul>
<li><a href="#-prerequisites">📋 Prerequisites</a></li>
<li><a href="#-getting-started">🎯 Getting Started</a></li>
<li><a href="#-project-structure">📁 Project Structure</a></li>
<li><a href="#-set-up-your-local-infrastructure">🏗️ Set Up Your Local Infrastructure</a></li>
<li><a href="#️-running-the-code-for-each-module">⚡️ Running the Code for Each Module</a></li>
<li><a href="#-utility-commands">🔧 Utlity Commands</a></li>
</ul>
</section>
<section id="prerequisites" class="level1">
<h1>📋 Prerequisites</h1>
<p>We depend on the same prerequisites as the offline ML pipelines. If modules 1 to 5 are working, you are good to go. Just make sure to fill in the <code>.env</code> file with the correct credentials.</p>
<section id="local-tools" class="level2">
<h2 class="anchored" data-anchor-id="local-tools">Local Tools</h2>
<p>For all the modules, you’ll need the following tools installed locally:</p>
<table class="caption-top table">
<thead>
<tr class="header">
<th>Tool</th>
<th>Version</th>
<th>Purpose</th>
<th>Installation Link</th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td>Python</td>
<td>3.11</td>
<td>Programming language runtime</td>
<td><a href="https://www.python.org/downloads/">Download</a></td>
</tr>
<tr class="even">
<td>uv</td>
<td>≥ 0.4.30</td>
<td>Python package installer and virtual environment manager</td>
<td><a href="https://github.com/astral-sh/uv">Download</a></td>
</tr>
<tr class="odd">
<td>GNU Make</td>
<td>≥ 3.81</td>
<td>Build automation tool</td>
<td><a href="https://www.gnu.org/software/make/">Download</a></td>
</tr>
<tr class="even">
<td>Git</td>
<td>≥2.44.0</td>
<td>Version control</td>
<td><a href="https://git-scm.com/downloads">Download</a></td>
</tr>
<tr class="odd">
<td>Docker</td>
<td>≥27.4.0</td>
<td>Containerization platform</td>
<td><a href="https://www.docker.com/get-started/">Download</a></td>
</tr>
</tbody>
</table>
<details>
<summary>
<b>📌 Windows users also need to install WSL (Click to expand)</b>
</summary>
<p>We will be using Unix commands across the course, so if you are using Windows, you will need to <strong>install WSL</strong>, which will install a Linux kernel on your Windows machine and allow you to use the Unix commands from our course (this is the recommended way to write software on Windows).</p>
🔗 <a href="https://www.youtube.com/watch?v=YByZ_sOOWsQ">Follow this guide to install WSL</a>.
</details>
</section>
<section id="cloud-services" class="level2">
<h2 class="anchored" data-anchor-id="cloud-services">Cloud Services</h2>
<p>Also, the course requires access to these cloud services. The authentication to these services is done by adding the corresponding environment variables to the <code>.env</code> file:</p>
<table class="caption-top table">
<thead>
<tr class="header">
<th>Service</th>
<th>Purpose</th>
<th>Cost</th>
<th>Environment Variable</th>
<th>Setup Guide</th>
<th>Starting with Module</th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td><a href="https://openai.com/index/openai-api/">OpenAI API</a></td>
<td>LLM API</td>
<td>Pay-per-use</td>
<td><code>OPENAI_API_KEY</code></td>
<td><a href="https://platform.openai.com/docs/quickstart">Quick Start Guide</a></td>
<td>Module 2</td>
</tr>
<tr class="even">
<td><a href="https://huggingface.com/">Hugging Face</a></td>
<td>MLOps</td>
<td>Free tier</td>
<td><code>HUGGINGFACE_ACCESS_TOKEN</code></td>
<td><a href="https://huggingface.co/docs/hub/en/security-tokens">Quick Start Guide</a></td>
<td>Module 3</td>
</tr>
<tr class="odd">
<td><a href="https://rebrand.ly/second-brain-course-comet">Comet</a></td>
<td>Experiment tracking</td>
<td>Free tier</td>
<td><code>COMET_API_KEY</code></td>
<td><a href="https://rebrand.ly/second-brain-course-comet-quickstart">Quick Start Guide</a></td>
<td>Module 4</td>
</tr>
<tr class="even">
<td><a href="https://rebrand.ly/second-brain-course-opik">Opik</a></td>
<td>LLM evaluation and prompt monitoring</td>
<td>Free tier (Hosted on Comet - same API Key)</td>
<td><code>COMET_API_KEY</code></td>
<td><a href="https://rebrand.ly/second-brain-course-comet-quickstart">Quick Start Guide</a></td>
<td>Module 6</td>
</tr>
</tbody>
</table>
<p>When working locally, the infrastructure is set up using Docker. Thus, you can use the default values found in the <a href="src/second_brain_online/config.py">config.py</a> file for all the infrastructure-related environment variables.</p>
<p>But, in case you want to deploy the code, you’ll need to setup the following services with their corresponding environment variables:</p>
<table class="caption-top table">
<thead>
<tr class="header">
<th>Service</th>
<th>Purpose</th>
<th>Cost</th>
<th>Required Credentials</th>
<th>Setup Guide</th>
</tr>
</thead>
<tbody>
<tr class="odd">
<td><a href="https://rebrand.ly/second-brain-course-mongodb">MongoDB</a></td>
<td>document database (with vector search)</td>
<td>Free tier</td>
<td><code>MONGODB_URI</code></td>
<td>1. <a href="https://rebrand.ly/second-brain-course-mongodb-setup-1">Create a free MongoDB Atlas account</a> <br> 2. <a href="https://rebrand.ly/second-brain-course-mongodb-setup-2">Create a Cluster</a> <br> 3. <a href="https://rebrand.ly/second-brain-course-mongodb-setup-3">Add a Database User</a> <br> 4. <a href="https://rebrand.ly/second-brain-course-mongodb-setup-4">Configure a Network Connection</a></td>
</tr>
</tbody>
</table>
</section>
</section>
<section id="getting-started" class="level1">
<h1>🎯 Getting Started</h1>
<section id="clone-the-repository" class="level2">
<h2 class="anchored" data-anchor-id="clone-the-repository">1. Clone the Repository</h2>
<p>Start by cloning the repository and navigating to the project directory:</p>
<div class="sourceCode" id="cb2"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="fu">git</span> clone https://github.com/decodingml/second-brain-ai-assistant-course.git</span>
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="bu">cd</span> second-brain-ai-assistant-course </span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
</section>
<section id="installation" class="level2">
<h2 class="anchored" data-anchor-id="installation">2. Installation</h2>
<p>First deactivate any active virtual environment and move to the <code>second-brain-online</code> directory:</p>
<div class="sourceCode" id="cb3"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="ex">deactivate</span></span>
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="bu">cd</span> apps/second-brain-online</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<p>To install the dependencies and activate the virtual environment, run the following commands:</p>
<div class="sourceCode" id="cb4"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="ex">uv</span> venv .venv-online</span>
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a><span class="bu">.</span> ./.venv-online/bin/activate</span>
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a><span class="ex">uv</span> pip install <span class="at">-e</span> .</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<blockquote class="blockquote">
<p>[!NOTE] The online application uses a different set of dependencies than the offline ML pipelines.</p>
</blockquote>
</section>
<section id="environment-configuration" class="level2">
<h2 class="anchored" data-anchor-id="environment-configuration">3. Environment Configuration</h2>
<p>Before running any command, you have to set up your environment: 1. Create your environment file: <code>bash    cp .env.example .env</code> 2. Open <code>.env</code> and configure the required credentials following the inline comments and the recommendations from the <a href="#-prerequisites">Cloud Services</a> section.</p>
</section>
</section>
<section id="project-structure" class="level1">
<h1>📁 Project Structure</h1>
<p>At Decoding ML we teach how to build production ML systems. Thus, instead of splitting the code into separate modules, the course follows the structure of a real-world Python project:</p>
<div class="sourceCode" id="cb5"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="bu">.</span></span>
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="ex">├──</span> configs/                   <span class="co"># ZenML configuration files</span></span>
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="ex">├──</span> src/second_brain_online/  <span class="co"># Main package directory</span></span>
<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a><span class="ex">│</span>   ├── application/           <span class="co"># Application layer</span></span>
<span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a><span class="ex">│</span>   ├── config.py              <span class="co"># Configuration settings</span></span>
<span id="cb5-6"><a href="#cb5-6" aria-hidden="true" tabindex="-1"></a><span class="ex">│</span>   └── opik_utils.py          <span class="co"># Opik utility functions</span></span>
<span id="cb5-7"><a href="#cb5-7" aria-hidden="true" tabindex="-1"></a><span class="ex">├──</span> tools/                     <span class="co"># Entrypoint scripts that use the Python package</span></span>
<span id="cb5-8"><a href="#cb5-8" aria-hidden="true" tabindex="-1"></a><span class="ex">├──</span> .env.example               <span class="co"># Environment variables template</span></span>
<span id="cb5-9"><a href="#cb5-9" aria-hidden="true" tabindex="-1"></a><span class="ex">├──</span> .python-version            <span class="co"># Python version specification</span></span>
<span id="cb5-10"><a href="#cb5-10" aria-hidden="true" tabindex="-1"></a><span class="ex">├──</span> Makefile                   <span class="co"># Project commands</span></span>
<span id="cb5-11"><a href="#cb5-11" aria-hidden="true" tabindex="-1"></a><span class="ex">└──</span> pyproject.toml             <span class="co"># Project dependencies</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
</section>
<section id="set-up-your-local-infrastructure" class="level1">
<h1>🏗️ Set Up Your Local Infrastructure</h1>
<p>We use Docker to setup the local infrastructure (MongoDB).</p>
<blockquote class="blockquote">
<p>[!WARNING] Before running the command below, ensure you do not have any processes running on port <code>27017</code> (MongoDB).</p>
</blockquote>
<p>To start the Docker infrastructure, run:</p>
<div class="sourceCode" id="cb6"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="fu">make</span> local-infrastructure-up</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<p>To stop the Docker infrastructure, run:</p>
<div class="sourceCode" id="cb7"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="fu">make</span> local-infrastructure-down</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<blockquote class="blockquote">
<p>[!NOTE] To visualize the raw and RAG data from MongoDB, we recommend using <a href="https://rebrand.ly/second-brain-course-mongodb-compass">MongoDB Compass</a> or Mongo’s official IDE plugin (e.g., <code>MongoDB for VS Code</code>). To connect to the working MongoDB instance, use the <code>MONGODB_URI</code> value from the <code>.env</code> file or found inside the <a href="src/second_brain_online/config.py">config.py</a> file.</p>
</blockquote>
<div class="quarto-figure quarto-figure-center">
<figure class="figure">
<p><img src="../../static/mongodb_atlas_example.png" class="img-fluid figure-img"></p>
<figcaption>mongodb_atlas_example.png</figcaption>
</figure>
</div>
</section>
<section id="running-the-code-for-each-module" class="level1">
<h1>⚡️ Running the Code for Each Module</h1>
<p><strong>NOTE:</strong> To run these modules, you must first go through the steps from the <a href="../second-brain-offline">Offline ML Pipelines</a> part of the course.</p>
<p>To simulate the course modules, we split the CLI commands and offline ML pipelines you must run per module so you know exactly where you are in the course.</p>
<section id="module-6-running-the-rag-agentic-app-and-llmops-layer" class="level2">
<h2 class="anchored" data-anchor-id="module-6-running-the-rag-agentic-app-and-llmops-layer">Module 6: Running the RAG Agentic App and LLMOps Layer</h2>
<p>Quickly test the agent from the CLI with a predefined query:</p>
<div class="sourceCode" id="cb8"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="fu">make</span> run_agent_query RETRIEVER_CONFIG=configs/compute_rag_vector_index_openai_parent.yaml</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<p>You should see something like this:</p>
<pre class="console"><code>Vector databases and vector indices are related concepts in the field of data storage and retrieval, particularly in contexts where high-dimensional vector representations of data are used, such as in machine learning and AI. Here are the key differences:

1. **Vector Databases**:
   - A vector database ...</code></pre>
<blockquote class="blockquote">
<p>[!IMPORTANT] Be sure that the retriever config is the exact same one as the one used in Module 5 during the RAG feature pipeline to populate the vector database. If they don’t match, the used retriever will use different settings resulting in errors or unexpected results. Here is a quick reminder of when to use which config: 1. Parent Retrieval with OpenAI models: <code>configs/compute_rag_vector_index_openai_parent.yaml</code> 2. Simple Contextual Retrieval with OpenAI models: <code>configs/compute_rag_vector_index_openai_contextual_simple.yaml</code> 3. Simple Contextual Retrieval with Hugging Face models: <code>configs/compute_rag_vector_index_huggingface_contextual_simple.yaml</code> 4. Full-fledged Contextual Retrieval with OpenAI models: <code>configs/compute_rag_vector_index_openai_contextual.yaml</code></p>
</blockquote>
<p>You can also spin-up a Gradio UI to test the agent with custom queries similar to any other chatbot:</p>
<div class="sourceCode" id="cb10"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="fu">make</span> run_agent_app RETRIEVER_CONFIG=configs/compute_rag_vector_index_openai_parent.yaml</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<p>You should see something like this:</p>
<video src="https://github.com/user-attachments/assets/bfea8e24-6d52-4a33-8857-5d05154ab69e"></video>

<p>Evaluate the agent with our predefined evaluation queries (found under <code>tools/evaluate_app.py</code>):</p>
<div class="sourceCode" id="cb11"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="fu">make</span> evaluate_agent RETRIEVER_CONFIG=configs/compute_rag_vector_index_openai_parent.yaml</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<p>After running the evaluation, open <a href="https://rebrand.ly/second-brain-course-opik-dashboard">Opik</a> to see the evaluation results, as seen in the image below:</p>
<div class="quarto-figure quarto-figure-center">
<figure class="figure">
<p><img src="../../static/opik_evaluation_results_example.png" class="img-fluid figure-img"></p>
<figcaption>Opik Evaluation Results</figcaption>
</figure>
</div>
<p>For running the evaluation, plus playing around with the agent (~20 queries), the costs and running time are: - Running costs OpenAI: ~$0.5 - Running costs Hugging Face Dedicated Endpoints (optional - you can use only the OpenAI models for summarization): ~$1 (the deployment costs $1 / hour) - Running time evaluation: ~15 minutes (the AI Assistant runs in real-time)</p>
</section>
</section>
<section id="utlity-commands" class="level1">
<h1>🔧 Utlity Commands</h1>
<section id="formatting" class="level2">
<h2 class="anchored" data-anchor-id="formatting">Formatting</h2>
<pre><code>make format-check
make format-fix</code></pre>
</section>
<section id="linting" class="level2">
<h2 class="anchored" data-anchor-id="linting">Linting</h2>
<div class="sourceCode" id="cb13"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb13-1"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a><span class="fu">make</span> lint-check</span>
<span id="cb13-2"><a href="#cb13-2" aria-hidden="true" tabindex="-1"></a><span class="fu">make</span> lint-fix</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
</section>
</section>
</main></div>


<!-- /main column -->
<script id="quarto-html-after-body" type="application/javascript">
  window.document.addEventListener("DOMContentLoaded", function (event) {
    const icon = "";
    const anchorJS = new window.AnchorJS();
    anchorJS.options = {
      placement: 'right',
      icon: icon
    };
    anchorJS.add('.anchored');
    const isCodeAnnotation = (el) => {
      for (const clz of el.classList) {
        if (clz.startsWith('code-annotation-')) {                     
          return true;
        }
      }
      return false;
    }
    const onCopySuccess = function(e) {
      // button target
      const button = e.trigger;
      // don't keep focus
      button.blur();
      // flash "checked"
      button.classList.add('code-copy-button-checked');
      var currentTitle = button.getAttribute("title");
      button.setAttribute("title", "Copied!");
      let tooltip;
      if (window.bootstrap) {
        button.setAttribute("data-bs-toggle", "tooltip");
        button.setAttribute("data-bs-placement", "left");
        button.setAttribute("data-bs-title", "Copied!");
        tooltip = new bootstrap.Tooltip(button, 
          { trigger: "manual", 
            customClass: "code-copy-button-tooltip",
            offset: [0, -8]});
        tooltip.show();    
      }
      setTimeout(function() {
        if (tooltip) {
          tooltip.hide();
          button.removeAttribute("data-bs-title");
          button.removeAttribute("data-bs-toggle");
          button.removeAttribute("data-bs-placement");
        }
        button.setAttribute("title", currentTitle);
        button.classList.remove('code-copy-button-checked');
      }, 1000);
      // clear code selection
      e.clearSelection();
    }
    const getTextToCopy = function(trigger) {
        const codeEl = trigger.previousElementSibling.cloneNode(true);
        for (const childEl of codeEl.children) {
          if (isCodeAnnotation(childEl)) {
            childEl.remove();
          }
        }
        return codeEl.innerText;
    }
    const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
      text: getTextToCopy
    });
    clipboard.on('success', onCopySuccess);
    if (window.document.getElementById('quarto-embedded-source-code-modal')) {
      const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
        text: getTextToCopy,
        container: window.document.getElementById('quarto-embedded-source-code-modal')
      });
      clipboardModal.on('success', onCopySuccess);
    }
      var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
      var mailtoRegex = new RegExp(/^mailto:/);
        var filterRegex = new RegExp('/' + window.location.host + '/');
      var isInternal = (href) => {
          return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
      }
      // Inspect non-navigation links and adorn them if external
     var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
      for (var i=0; i<links.length; i++) {
        const link = links[i];
        if (!isInternal(link.href)) {
          // undo the damage that might have been done by quarto-nav.js in the case of
          // links that we want to consider external
          if (link.dataset.originalHref !== undefined) {
            link.href = link.dataset.originalHref;
          }
        }
      }
    function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
      const config = {
        allowHTML: true,
        maxWidth: 500,
        delay: 100,
        arrow: false,
        appendTo: function(el) {
            return el.parentElement;
        },
        interactive: true,
        interactiveBorder: 10,
        theme: 'quarto',
        placement: 'bottom-start',
      };
      if (contentFn) {
        config.content = contentFn;
      }
      if (onTriggerFn) {
        config.onTrigger = onTriggerFn;
      }
      if (onUntriggerFn) {
        config.onUntrigger = onUntriggerFn;
      }
      window.tippy(el, config); 
    }
    const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
    for (var i=0; i<noterefs.length; i++) {
      const ref = noterefs[i];
      tippyHover(ref, function() {
        // use id or data attribute instead here
        let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
        try { href = new URL(href).hash; } catch {}
        const id = href.replace(/^#\/?/, "");
        const note = window.document.getElementById(id);
        if (note) {
          return note.innerHTML;
        } else {
          return "";
        }
      });
    }
    const xrefs = window.document.querySelectorAll('a.quarto-xref');
    const processXRef = (id, note) => {
      // Strip column container classes
      const stripColumnClz = (el) => {
        el.classList.remove("page-full", "page-columns");
        if (el.children) {
          for (const child of el.children) {
            stripColumnClz(child);
          }
        }
      }
      stripColumnClz(note)
      if (id === null || id.startsWith('sec-')) {
        // Special case sections, only their first couple elements
        const container = document.createElement("div");
        if (note.children && note.children.length > 2) {
          container.appendChild(note.children[0].cloneNode(true));
          for (let i = 1; i < note.children.length; i++) {
            const child = note.children[i];
            if (child.tagName === "P" && child.innerText === "") {
              continue;
            } else {
              container.appendChild(child.cloneNode(true));
              break;
            }
          }
          if (window.Quarto?.typesetMath) {
            window.Quarto.typesetMath(container);
          }
          return container.innerHTML
        } else {
          if (window.Quarto?.typesetMath) {
            window.Quarto.typesetMath(note);
          }
          return note.innerHTML;
        }
      } else {
        // Remove any anchor links if they are present
        const anchorLink = note.querySelector('a.anchorjs-link');
        if (anchorLink) {
          anchorLink.remove();
        }
        if (window.Quarto?.typesetMath) {
          window.Quarto.typesetMath(note);
        }
        if (note.classList.contains("callout")) {
          return note.outerHTML;
        } else {
          return note.innerHTML;
        }
      }
    }
    for (var i=0; i<xrefs.length; i++) {
      const xref = xrefs[i];
      tippyHover(xref, undefined, function(instance) {
        instance.disable();
        let url = xref.getAttribute('href');
        let hash = undefined; 
        if (url.startsWith('#')) {
          hash = url;
        } else {
          try { hash = new URL(url).hash; } catch {}
        }
        if (hash) {
          const id = hash.replace(/^#\/?/, "");
          const note = window.document.getElementById(id);
          if (note !== null) {
            try {
              const html = processXRef(id, note.cloneNode(true));
              instance.setContent(html);
            } finally {
              instance.enable();
              instance.show();
            }
          } else {
            // See if we can fetch this
            fetch(url.split('#')[0])
            .then(res => res.text())
            .then(html => {
              const parser = new DOMParser();
              const htmlDoc = parser.parseFromString(html, "text/html");
              const note = htmlDoc.getElementById(id);
              if (note !== null) {
                const html = processXRef(id, note);
                instance.setContent(html);
              } 
            }).finally(() => {
              instance.enable();
              instance.show();
            });
          }
        } else {
          // See if we can fetch a full url (with no hash to target)
          // This is a special case and we should probably do some content thinning / targeting
          fetch(url)
          .then(res => res.text())
          .then(html => {
            const parser = new DOMParser();
            const htmlDoc = parser.parseFromString(html, "text/html");
            const note = htmlDoc.querySelector('main.content');
            if (note !== null) {
              // This should only happen for chapter cross references
              // (since there is no id in the URL)
              // remove the first header
              if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
                note.children[0].remove();
              }
              const html = processXRef(null, note);
              instance.setContent(html);
            } 
          }).finally(() => {
            instance.enable();
            instance.show();
          });
        }
      }, function(instance) {
      });
    }
        let selectedAnnoteEl;
        const selectorForAnnotation = ( cell, annotation) => {
          let cellAttr = 'data-code-cell="' + cell + '"';
          let lineAttr = 'data-code-annotation="' +  annotation + '"';
          const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
          return selector;
        }
        const selectCodeLines = (annoteEl) => {
          const doc = window.document;
          const targetCell = annoteEl.getAttribute("data-target-cell");
          const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
          const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
          const lines = annoteSpan.getAttribute("data-code-lines").split(",");
          const lineIds = lines.map((line) => {
            return targetCell + "-" + line;
          })
          let top = null;
          let height = null;
          let parent = null;
          if (lineIds.length > 0) {
              //compute the position of the single el (top and bottom and make a div)
              const el = window.document.getElementById(lineIds[0]);
              top = el.offsetTop;
              height = el.offsetHeight;
              parent = el.parentElement.parentElement;
            if (lineIds.length > 1) {
              const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
              const bottom = lastEl.offsetTop + lastEl.offsetHeight;
              height = bottom - top;
            }
            if (top !== null && height !== null && parent !== null) {
              // cook up a div (if necessary) and position it 
              let div = window.document.getElementById("code-annotation-line-highlight");
              if (div === null) {
                div = window.document.createElement("div");
                div.setAttribute("id", "code-annotation-line-highlight");
                div.style.position = 'absolute';
                parent.appendChild(div);
              }
              div.style.top = top - 2 + "px";
              div.style.height = height + 4 + "px";
              div.style.left = 0;
              let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
              if (gutterDiv === null) {
                gutterDiv = window.document.createElement("div");
                gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
                gutterDiv.style.position = 'absolute';
                const codeCell = window.document.getElementById(targetCell);
                const gutter = codeCell.querySelector('.code-annotation-gutter');
                gutter.appendChild(gutterDiv);
              }
              gutterDiv.style.top = top - 2 + "px";
              gutterDiv.style.height = height + 4 + "px";
            }
            selectedAnnoteEl = annoteEl;
          }
        };
        const unselectCodeLines = () => {
          const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
          elementsIds.forEach((elId) => {
            const div = window.document.getElementById(elId);
            if (div) {
              div.remove();
            }
          });
          selectedAnnoteEl = undefined;
        };
          // Handle positioning of the toggle
      window.addEventListener(
        "resize",
        throttle(() => {
          elRect = undefined;
          if (selectedAnnoteEl) {
            selectCodeLines(selectedAnnoteEl);
          }
        }, 10)
      );
      function throttle(fn, ms) {
      let throttle = false;
      let timer;
        return (...args) => {
          if(!throttle) { // first call gets through
              fn.apply(this, args);
              throttle = true;
          } else { // all the others get throttled
              if(timer) clearTimeout(timer); // cancel #2
              timer = setTimeout(() => {
                fn.apply(this, args);
                timer = throttle = false;
              }, ms);
          }
        };
      }
        // Attach click handler to the DT
        const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
        for (const annoteDlNode of annoteDls) {
          annoteDlNode.addEventListener('click', (event) => {
            const clickedEl = event.target;
            if (clickedEl !== selectedAnnoteEl) {
              unselectCodeLines();
              const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
              if (activeEl) {
                activeEl.classList.remove('code-annotation-active');
              }
              selectCodeLines(clickedEl);
              clickedEl.classList.add('code-annotation-active');
            } else {
              // Unselect the line
              unselectCodeLines();
              clickedEl.classList.remove('code-annotation-active');
            }
          });
        }
    const findCites = (el) => {
      const parentEl = el.parentElement;
      if (parentEl) {
        const cites = parentEl.dataset.cites;
        if (cites) {
          return {
            el,
            cites: cites.split(' ')
          };
        } else {
          return findCites(el.parentElement)
        }
      } else {
        return undefined;
      }
    };
    var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
    for (var i=0; i<bibliorefs.length; i++) {
      const ref = bibliorefs[i];
      const citeInfo = findCites(ref);
      if (citeInfo) {
        tippyHover(citeInfo.el, function() {
          var popup = window.document.createElement('div');
          citeInfo.cites.forEach(function(cite) {
            var citeDiv = window.document.createElement('div');
            citeDiv.classList.add('hanging-indent');
            citeDiv.classList.add('csl-entry');
            var biblioDiv = window.document.getElementById('ref-' + cite);
            if (biblioDiv) {
              citeDiv.innerHTML = biblioDiv.innerHTML;
            }
            popup.appendChild(citeDiv);
          });
          return popup.innerHTML;
        });
      }
    }
  });
  </script>
 <!-- /content -->




</body></html>