Commit
·
5b3cd33
1
Parent(s):
8381f2e
declauding the documentation some more
Browse files- documentation.html +72 -69
documentation.html
CHANGED
|
@@ -100,28 +100,28 @@
|
|
| 100 |
<body>
|
| 101 |
<div class="header">
|
| 102 |
<h1>MagentaRT Research API</h1>
|
| 103 |
-
<p class="muted"><strong>AI Music Generation API</strong> •
|
| 104 |
-
<span class="badge">
|
| 105 |
</div>
|
| 106 |
|
| 107 |
<div class="section">
|
| 108 |
<h2>what this is</h2>
|
| 109 |
-
<p>
|
| 110 |
|
| 111 |
-
<p>
|
| 112 |
|
| 113 |
-
<p>
|
| 114 |
|
| 115 |
<div class="info">
|
| 116 |
-
<strong>
|
| 117 |
</div>
|
| 118 |
</div>
|
| 119 |
|
| 120 |
<section id="env-vars" style="margin-top: 24px;">
|
| 121 |
<h3>environment variables (optional, but helpful)</h3>
|
| 122 |
<p>
|
| 123 |
-
|
| 124 |
-
<em>Settings → Variables and secrets → Variables</em>.
|
| 125 |
select models at runtime using <code>/model/select</code> from the frontend/API.
|
| 126 |
</p>
|
| 127 |
|
|
@@ -132,48 +132,48 @@
|
|
| 132 |
<li><code>MRT_CKPT_STEP</code> → <code>1863001</code></li>
|
| 133 |
<li><code>MRT_SIZE</code> → <code>large</code></li>
|
| 134 |
</ul>
|
| 135 |
-
<p style="margin:8px 0 0 0;"><small>
|
| 136 |
</div>
|
| 137 |
|
| 138 |
<table class="var-table" style="width:100%;border-collapse:collapse;margin:12px 0;">
|
| 139 |
<thead>
|
| 140 |
<tr>
|
| 141 |
-
<th style="text-align:left;border-bottom:1px solid #ddd;padding:8px;">
|
| 142 |
-
<th style="text-align:left;border-bottom:1px solid #ddd;padding:8px;">
|
| 143 |
-
<th style="text-align:left;border-bottom:1px solid #ddd;padding:8px;">
|
| 144 |
-
<th style="text-align:left;border-bottom:1px solid #ddd;padding:8px;">
|
| 145 |
</tr>
|
| 146 |
</thead>
|
| 147 |
<tbody>
|
| 148 |
<tr>
|
| 149 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>MRT_CKPT_REPO</code></td>
|
| 150 |
-
<td style="padding:8px;border-bottom:1px solid #eee;">
|
| 151 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>thepatch/magenta-ft</code></td>
|
| 152 |
-
<td style="padding:8px;border-bottom:1px solid #eee;">
|
| 153 |
</tr>
|
| 154 |
<tr>
|
| 155 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>MRT_CKPT_STEP</code></td>
|
| 156 |
-
<td style="padding:8px;border-bottom:1px solid #eee;">
|
| 157 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>1863001</code></td>
|
| 158 |
-
<td style="padding:8px;border-bottom:1px solid #eee;">
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>MRT_SIZE</code></td>
|
| 162 |
-
<td style="padding:8px;border-bottom:1px solid #eee;">
|
| 163 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>large</code></td>
|
| 164 |
-
<td style="padding:8px;border-bottom:1px solid #eee;">
|
| 165 |
</tr>
|
| 166 |
<tr>
|
| 167 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>SPACE_MODE</code></td>
|
| 168 |
-
<td style="padding:8px;border-bottom:1px solid #eee;">
|
| 169 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>serve</code> or <code>template</code></td>
|
| 170 |
-
<td style="padding:8px;border-bottom:1px solid #eee;">
|
| 171 |
</tr>
|
| 172 |
</tbody>
|
| 173 |
</table>
|
| 174 |
|
| 175 |
<details style="margin-top:12px;">
|
| 176 |
-
<summary><strong>
|
| 177 |
<pre style="background:#111;color:#eee;padding:12px;border-radius:8px;overflow:auto;margin-top:8px;"><code style="background: transparent; color: inherit; padding: 0; border: 0; box-shadow: none; display: block;">curl -X POST https://<your-space>.hf.space/model/select \
|
| 178 |
-H 'Content-Type: application/json' \
|
| 179 |
-d '{
|
|
@@ -182,7 +182,7 @@
|
|
| 182 |
"size": "large",
|
| 183 |
"prewarm": true
|
| 184 |
}'</code></pre>
|
| 185 |
-
<p style="margin:8px 0 0 0;"><small>
|
| 186 |
</details>
|
| 187 |
</section>
|
| 188 |
|
|
@@ -190,7 +190,7 @@
|
|
| 190 |
<a class="btn" href="/tester" target="_blank" style="
|
| 191 |
display:inline-block; padding:10px 14px; border-radius:8px;
|
| 192 |
background:#111; color:#eee; text-decoration:none; border:1px solid #444;">
|
| 193 |
-
|
| 194 |
</a>
|
| 195 |
</p>
|
| 196 |
|
|
@@ -205,7 +205,7 @@
|
|
| 205 |
|
| 206 |
<div class="section">
|
| 207 |
<h2>overview</h2>
|
| 208 |
-
<p>
|
| 209 |
</div>
|
| 210 |
|
| 211 |
<div class="section">
|
|
@@ -250,66 +250,66 @@
|
|
| 250 |
<h2>API endpoints</h2>
|
| 251 |
|
| 252 |
<div class="endpoint">
|
| 253 |
-
<strong>POST /generate</strong> -
|
| 254 |
</div>
|
| 255 |
|
| 256 |
<div class="endpoint">
|
| 257 |
-
<strong>POST /generate_style</strong> -
|
| 258 |
</div>
|
| 259 |
|
| 260 |
<div class="endpoint">
|
| 261 |
-
<strong>POST /jam/start</strong> -
|
| 262 |
</div>
|
| 263 |
|
| 264 |
<div class="endpoint">
|
| 265 |
-
<strong>GET /jam/next</strong> -
|
| 266 |
</div>
|
| 267 |
|
| 268 |
<div class="endpoint">
|
| 269 |
-
<strong>POST /jam/consume</strong> -
|
| 270 |
</div>
|
| 271 |
|
| 272 |
<div class="endpoint">
|
| 273 |
-
<strong>POST /jam/stop</strong> -
|
| 274 |
</div>
|
| 275 |
|
| 276 |
<div class="endpoint">
|
| 277 |
-
<strong>WEBSOCKET /ws/jam</strong> -
|
| 278 |
</div>
|
| 279 |
|
| 280 |
<div class="endpoint">
|
| 281 |
-
<strong>POST /model/select</strong> -
|
| 282 |
</div>
|
| 283 |
</div>
|
| 284 |
|
| 285 |
<div class="section">
|
| 286 |
<h2>custom fine-tuning</h2>
|
| 287 |
-
<p>
|
| 288 |
|
| 289 |
<div class="grid">
|
| 290 |
<div class="card">
|
| 291 |
<h3>1. train your model</h3>
|
| 292 |
-
<p>
|
| 293 |
-
<p><a href="https://
|
| 294 |
<p>This will create checkpoint folders like:</p>
|
| 295 |
<ul>
|
| 296 |
<li><code>checkpoint_1861001/</code></li>
|
| 297 |
<li><code>checkpoint_1862001/</code></li>
|
| 298 |
-
<li>
|
| 299 |
</ul>
|
| 300 |
</div>
|
| 301 |
|
| 302 |
<div class="card">
|
| 303 |
<h3>2. package checkpoints</h3>
|
| 304 |
-
<p>
|
| 305 |
<div class="warning">
|
| 306 |
-
<strong>
|
| 307 |
</div>
|
| 308 |
</div>
|
| 309 |
</div>
|
| 310 |
|
| 311 |
<h3>checkpoint packaging script</h3>
|
| 312 |
-
<p>
|
| 313 |
<pre><button class="copy-btn" onclick="copyCode(this)">Copy</button># Mount Drive to access your trained checkpoints
|
| 314 |
from google.colab import drive
|
| 315 |
drive.mount('/content/drive')
|
|
@@ -335,7 +335,7 @@ from google.colab import files
|
|
| 335 |
files.download('/content/checkpoint_1862001.tgz')</pre>
|
| 336 |
|
| 337 |
<h3>3. upload to hugging face</h3>
|
| 338 |
-
<p>
|
| 339 |
<ul>
|
| 340 |
<li>Your <code>.tgz</code> checkpoint files</li>
|
| 341 |
<li><code>cluster_centroids.npy</code> (for steering)</li>
|
|
@@ -344,70 +344,73 @@ files.download('/content/checkpoint_1862001.tgz')</pre>
|
|
| 344 |
|
| 345 |
<div class="info">
|
| 346 |
<strong>Example Repository:</strong> <a href="https://huggingface.co/thepatch/magenta-ft" target="_blank">thepatch/magenta-ft</a><br>
|
| 347 |
-
|
| 348 |
</div>
|
| 349 |
|
| 350 |
<h3>4. use in the app</h3>
|
| 351 |
-
<p>
|
| 352 |
</div>
|
| 353 |
|
| 354 |
<div class="section">
|
| 355 |
<h2>technical specifications</h2>
|
| 356 |
<ul>
|
| 357 |
-
<li><strong>
|
| 358 |
-
<li><strong>
|
| 359 |
-
<li><strong>
|
| 360 |
-
<li><strong>
|
| 361 |
-
<li><strong>Memory Requirements:</strong>
|
| 362 |
</ul>
|
| 363 |
|
| 364 |
-
|
| 365 |
-
<strong>Note:</strong> The <code>/generate_style</code> endpoint is experimental and may not properly adhere to BPM without additional context (considering metronome-based context instead of silence).
|
| 366 |
-
</div>
|
| 367 |
-
</div>
|
| 368 |
|
| 369 |
<div class="section">
|
| 370 |
-
<h2>
|
| 371 |
-
<p>
|
| 372 |
<ul>
|
| 373 |
-
<li>
|
| 374 |
-
<li>
|
| 375 |
-
<li>
|
| 376 |
-
<li>
|
| 377 |
</ul>
|
| 378 |
</div>
|
| 379 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 380 |
<div class="section">
|
| 381 |
<h2>deployment</h2>
|
| 382 |
<p>To run your own instance:</p>
|
| 383 |
<ol>
|
| 384 |
-
<li>
|
| 385 |
-
<li>
|
| 386 |
-
<li>
|
| 387 |
-
<li>
|
|
|
|
| 388 |
</ol>
|
| 389 |
</div>
|
| 390 |
|
| 391 |
<div class="section">
|
| 392 |
<h2>support & contact</h2>
|
| 393 |
-
<p>
|
| 394 |
-
<p><strong>
|
| 395 |
|
| 396 |
<div class="info">
|
| 397 |
-
<strong>
|
| 398 |
</div>
|
| 399 |
</div>
|
| 400 |
|
| 401 |
<div class="section">
|
| 402 |
<h2>licensing</h2>
|
| 403 |
-
<p>
|
| 404 |
-
<p><a href="/docs">API
|
| 405 |
</div>
|
| 406 |
|
| 407 |
-
<div class="section">
|
| 408 |
<h2>contributors</h2>
|
| 409 |
<p>Kevin Griffing and Andrew Luck</p>
|
| 410 |
-
</div>
|
| 411 |
|
| 412 |
<script>
|
| 413 |
function copyCode(button) {
|
|
|
|
| 100 |
<body>
|
| 101 |
<div class="header">
|
| 102 |
<h1>MagentaRT Research API</h1>
|
| 103 |
+
<p class="muted"><strong>AI Music Generation API</strong> • real-time streaming with http/ws • custom fine-tune model-switching support</p>
|
| 104 |
+
<span class="badge">research project</span>
|
| 105 |
</div>
|
| 106 |
|
| 107 |
<div class="section">
|
| 108 |
<h2>what this is</h2>
|
| 109 |
+
<p>this API serves google's <a href="https://huggingface.co/google/magenta-realtime" target="_blank">magentaRT</a> in two distinct ways. first, as a backend for our iOS app (the untitled jamming app) where users create initial loops with stability ai's <a href="https://huggingface.co/stabilityai/stable-audio-open-small" target="_blank">stable-audio-open-small</a> and then MagentaRT jams on top of that audio context. Second, as a standalone web interface that connects directly to MagentaRT via WebSockets without any audio context.</p>
|
| 110 |
|
| 111 |
+
<p>both modes support switching between base models and custom fine-tunes hosted on Hugging Face. this is designed as a template space for duplication, letting you experiment with real-time music generation outside of google colab.</p>
|
| 112 |
|
| 113 |
+
<p>this is meant to be duplicated to your own GPU-enabled space since the iOS app is still in active development and doesn't have funding to support multiple concurrent users yet.</p>
|
| 114 |
|
| 115 |
<div class="info">
|
| 116 |
+
<strong>hardware requirements:</strong> optimal performance requires an L40S GPU (48GB VRAM) for real-time streaming. L4 24GB almost works but will not achieve real-time performance (if someone knows an optimization that will solve this, please let me know).
|
| 117 |
</div>
|
| 118 |
</div>
|
| 119 |
|
| 120 |
<section id="env-vars" style="margin-top: 24px;">
|
| 121 |
<h3>environment variables (optional, but helpful)</h3>
|
| 122 |
<p>
|
| 123 |
+
you can boot this Space directly into your own finetune by setting the variables below in
|
| 124 |
+
<em>Settings → Variables and secrets → Variables</em>. if you don't set them, you can still
|
| 125 |
select models at runtime using <code>/model/select</code> from the frontend/API.
|
| 126 |
</p>
|
| 127 |
|
|
|
|
| 132 |
<li><code>MRT_CKPT_STEP</code> → <code>1863001</code></li>
|
| 133 |
<li><code>MRT_SIZE</code> → <code>large</code></li>
|
| 134 |
</ul>
|
| 135 |
+
<p style="margin:8px 0 0 0;"><small>those values correspond to the example finetune in this repo (checkpoint_1863001.tgz on top of the <em>large</em> base).</small></p>
|
| 136 |
</div>
|
| 137 |
|
| 138 |
<table class="var-table" style="width:100%;border-collapse:collapse;margin:12px 0;">
|
| 139 |
<thead>
|
| 140 |
<tr>
|
| 141 |
+
<th style="text-align:left;border-bottom:1px solid #ddd;padding:8px;">name</th>
|
| 142 |
+
<th style="text-align:left;border-bottom:1px solid #ddd;padding:8px;">what it does</th>
|
| 143 |
+
<th style="text-align:left;border-bottom:1px solid #ddd;padding:8px;">example</th>
|
| 144 |
+
<th style="text-align:left;border-bottom:1px solid #ddd;padding:8px;">when to set</th>
|
| 145 |
</tr>
|
| 146 |
</thead>
|
| 147 |
<tbody>
|
| 148 |
<tr>
|
| 149 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>MRT_CKPT_REPO</code></td>
|
| 150 |
+
<td style="padding:8px;border-bottom:1px solid #eee;">huggingface repo ID that hosts your finetune checkpoints/assets.</td>
|
| 151 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>thepatch/magenta-ft</code></td>
|
| 152 |
+
<td style="padding:8px;border-bottom:1px solid #eee;">set to make this finetune the default on boot.</td>
|
| 153 |
</tr>
|
| 154 |
<tr>
|
| 155 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>MRT_CKPT_STEP</code></td>
|
| 156 |
+
<td style="padding:8px;border-bottom:1px solid #eee;">checkpoint step number to load on boot.</td>
|
| 157 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>1863001</code></td>
|
| 158 |
+
<td style="padding:8px;border-bottom:1px solid #eee;">set if you want a specific checkpoint preselected.</td>
|
| 159 |
</tr>
|
| 160 |
<tr>
|
| 161 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>MRT_SIZE</code></td>
|
| 162 |
+
<td style="padding:8px;border-bottom:1px solid #eee;">base model family used by the finetune (e.g., <em>large</em>).</td>
|
| 163 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>large</code></td>
|
| 164 |
+
<td style="padding:8px;border-bottom:1px solid #eee;">set to match the base you finetuned from.</td>
|
| 165 |
</tr>
|
| 166 |
<tr>
|
| 167 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>SPACE_MODE</code></td>
|
| 168 |
+
<td style="padding:8px;border-bottom:1px solid #eee;">controls readiness behavior: <code>serve</code> (GPU, ready to generate) vs <code>template</code> (CPU template for duplication). If unset, the server auto-detects.</td>
|
| 169 |
<td style="padding:8px;border-bottom:1px solid #eee;"><code>serve</code> or <code>template</code></td>
|
| 170 |
+
<td style="padding:8px;border-bottom:1px solid #eee;">set for explicit behavior; otherwise it falls back to auto-detection.</td>
|
| 171 |
</tr>
|
| 172 |
</tbody>
|
| 173 |
</table>
|
| 174 |
|
| 175 |
<details style="margin-top:12px;">
|
| 176 |
+
<summary><strong>alternative: select a model at runtime via API</strong></summary>
|
| 177 |
<pre style="background:#111;color:#eee;padding:12px;border-radius:8px;overflow:auto;margin-top:8px;"><code style="background: transparent; color: inherit; padding: 0; border: 0; box-shadow: none; display: block;">curl -X POST https://<your-space>.hf.space/model/select \
|
| 178 |
-H 'Content-Type: application/json' \
|
| 179 |
-d '{
|
|
|
|
| 182 |
"size": "large",
|
| 183 |
"prewarm": true
|
| 184 |
}'</code></pre>
|
| 185 |
+
<p style="margin:8px 0 0 0;"><small>when you call <code>prewarm:true</code>, the backend performs a warmup before returning, so the first jam starts hot.</small></p>
|
| 186 |
</details>
|
| 187 |
</section>
|
| 188 |
|
|
|
|
| 190 |
<a class="btn" href="/tester" target="_blank" style="
|
| 191 |
display:inline-block; padding:10px 14px; border-radius:8px;
|
| 192 |
background:#111; color:#eee; text-decoration:none; border:1px solid #444;">
|
| 193 |
+
open realtime web tester
|
| 194 |
</a>
|
| 195 |
</p>
|
| 196 |
|
|
|
|
| 205 |
|
| 206 |
<div class="section">
|
| 207 |
<h2>overview</h2>
|
| 208 |
+
<p>this API revolves around google's magentaRT, designed for real-time audio streaming using finetunes hosted on HF. built for iOS app integration with webSocket streaming support for web applications (and potentially VSTs inside the DAW).</p>
|
| 209 |
</div>
|
| 210 |
|
| 211 |
<div class="section">
|
|
|
|
| 250 |
<h2>API endpoints</h2>
|
| 251 |
|
| 252 |
<div class="endpoint">
|
| 253 |
+
<strong>POST /generate</strong> - generate 4–8 bars of music with input audio
|
| 254 |
</div>
|
| 255 |
|
| 256 |
<div class="endpoint">
|
| 257 |
+
<strong>POST /generate_style</strong> - generate music from style prompts only (experimental)
|
| 258 |
</div>
|
| 259 |
|
| 260 |
<div class="endpoint">
|
| 261 |
+
<strong>POST /jam/start</strong> - start continuous jamming session
|
| 262 |
</div>
|
| 263 |
|
| 264 |
<div class="endpoint">
|
| 265 |
+
<strong>GET /jam/next</strong> - get next audio chunk from session
|
| 266 |
</div>
|
| 267 |
|
| 268 |
<div class="endpoint">
|
| 269 |
+
<strong>POST /jam/consume</strong> - mark chunk as consumed
|
| 270 |
</div>
|
| 271 |
|
| 272 |
<div class="endpoint">
|
| 273 |
+
<strong>POST /jam/stop</strong> - end jamming session
|
| 274 |
</div>
|
| 275 |
|
| 276 |
<div class="endpoint">
|
| 277 |
+
<strong>WEBSOCKET /ws/jam</strong> - real-time streaming interface
|
| 278 |
</div>
|
| 279 |
|
| 280 |
<div class="endpoint">
|
| 281 |
+
<strong>POST /model/select</strong> - switch between base and fine-tuned models
|
| 282 |
</div>
|
| 283 |
</div>
|
| 284 |
|
| 285 |
<div class="section">
|
| 286 |
<h2>custom fine-tuning</h2>
|
| 287 |
+
<p>train your own MagentaRT models and use them in the web app demo and the iOS app.</p>
|
| 288 |
|
| 289 |
<div class="grid">
|
| 290 |
<div class="card">
|
| 291 |
<h3>1. train your model</h3>
|
| 292 |
+
<p>use the official MagentaRT fine-tuning notebook:</p>
|
| 293 |
+
<p><a href="https://colab.research.google.com/github/magenta/magenta-realtime/blob/main/notebooks/Magenta_RT_Finetune.ipynb" target="_blank">MagentaRT Fine-tuning Colab</a></p>
|
| 294 |
<p>This will create checkpoint folders like:</p>
|
| 295 |
<ul>
|
| 296 |
<li><code>checkpoint_1861001/</code></li>
|
| 297 |
<li><code>checkpoint_1862001/</code></li>
|
| 298 |
+
<li>and steering assets: <code>cluster_centroids.npy</code>, <code>mean_style_embed.npy</code></li>
|
| 299 |
</ul>
|
| 300 |
</div>
|
| 301 |
|
| 302 |
<div class="card">
|
| 303 |
<h3>2. package checkpoints</h3>
|
| 304 |
+
<p>checkpoints must be compressed as .tgz files to preserve .zarray files correctly.</p>
|
| 305 |
<div class="warning">
|
| 306 |
+
<strong>important:</strong> do not download checkpoint folders directly from Google Drive - the .zarray files won't transfer properly.
|
| 307 |
</div>
|
| 308 |
</div>
|
| 309 |
</div>
|
| 310 |
|
| 311 |
<h3>checkpoint packaging script</h3>
|
| 312 |
+
<p>use this in a Colab cell to properly package your checkpoints:</p>
|
| 313 |
<pre><button class="copy-btn" onclick="copyCode(this)">Copy</button># Mount Drive to access your trained checkpoints
|
| 314 |
from google.colab import drive
|
| 315 |
drive.mount('/content/drive')
|
|
|
|
| 335 |
files.download('/content/checkpoint_1862001.tgz')</pre>
|
| 336 |
|
| 337 |
<h3>3. upload to hugging face</h3>
|
| 338 |
+
<p>create a model repository and upload:</p>
|
| 339 |
<ul>
|
| 340 |
<li>Your <code>.tgz</code> checkpoint files</li>
|
| 341 |
<li><code>cluster_centroids.npy</code> (for steering)</li>
|
|
|
|
| 344 |
|
| 345 |
<div class="info">
|
| 346 |
<strong>Example Repository:</strong> <a href="https://huggingface.co/thepatch/magenta-ft" target="_blank">thepatch/magenta-ft</a><br>
|
| 347 |
+
shows the correct file structure with .tgz files and .npy steering assets in the root directory.
|
| 348 |
</div>
|
| 349 |
|
| 350 |
<h3>4. use in the app</h3>
|
| 351 |
+
<p>in the iOS app's model selector, point to your hf repository URL. the app will automatically discover available checkpoints and allow switching between them.</p>
|
| 352 |
</div>
|
| 353 |
|
| 354 |
<div class="section">
|
| 355 |
<h2>technical specifications</h2>
|
| 356 |
<ul>
|
| 357 |
+
<li><strong>audio format:</strong> 48 kHz stereo, ~2.0s chunks with ~40ms crossfade. the 4/8 bar chunks will have varying length due to input bpm</li>
|
| 358 |
+
<li><strong>model sizes:</strong> 'base' and 'large' variants available (we didn't notice any speedup in generation time using 'base' rather than 'large')</li>
|
| 359 |
+
<li><strong>steering:</strong> support for text prompts, audio embeddings, and centroid-based fine-tune steering</li>
|
| 360 |
+
<li><strong>real-time performance:</strong> L40S recommended; L4 will experience slight delays</li>
|
| 361 |
+
<li><strong>Memory Requirements:</strong> 30+GB VRAM for sustained real-time streaming</li>
|
| 362 |
</ul>
|
| 363 |
|
| 364 |
+
|
|
|
|
|
|
|
|
|
|
| 365 |
|
| 366 |
<div class="section">
|
| 367 |
+
<h2>a little more about the ios app</h2>
|
| 368 |
+
<p>uses http requests</p>
|
| 369 |
<ul>
|
| 370 |
+
<li>the reseed endpoints are still under development... the idea is to re-inject the initial context with token splicing</li>
|
| 371 |
+
<li>single-shot generation endpoints (one_shot_generation.py)</li>
|
| 372 |
+
<li>the stable-audio-open-small backend is hosted by me. it generates with just 2gb GPU RAM</li>
|
| 373 |
+
<li>gradual style embed changes to try and avoid abrupt genre switches</li>
|
| 374 |
</ul>
|
| 375 |
</div>
|
| 376 |
|
| 377 |
+
<div class="warning">
|
| 378 |
+
<strong>Note:</strong> The <code>/generate_style</code> endpoint is experimental and may not properly adhere to BPM without additional context (considering metronome-based context instead of silence).
|
| 379 |
+
</div>
|
| 380 |
+
</div>
|
| 381 |
+
|
| 382 |
<div class="section">
|
| 383 |
<h2>deployment</h2>
|
| 384 |
<p>To run your own instance:</p>
|
| 385 |
<ol>
|
| 386 |
+
<li>duplicate this huggingface space by clicking the three dots in the top right</li>
|
| 387 |
+
<li>select 'run locally' if you got a 5090 or something</li>
|
| 388 |
+
<li>ensure you have access to an L40S GPU by enabling billing</li>
|
| 389 |
+
<li>point your iOS app to the new space URL (e.g., <code>https://your-username-magenta-retry.hf.space</code>)</li>
|
| 390 |
+
<li>upload your fine-tuned models to hf as described above </li>
|
| 391 |
</ol>
|
| 392 |
</div>
|
| 393 |
|
| 394 |
<div class="section">
|
| 395 |
<h2>support & contact</h2>
|
| 396 |
+
<p>this is an active research project. for questions, technical support, or collaboration:</p>
|
| 397 |
+
<p><strong>email:</strong> <a href="mailto:kev@thecollabagepatch.com">kev@thecollabagepatch.com</a></p>
|
| 398 |
|
| 399 |
<div class="info">
|
| 400 |
+
<strong>research Status:</strong> this project is under very active development. features and API may change. We welcome feedback and contributions from the research community. im just a vibe coder.
|
| 401 |
</div>
|
| 402 |
</div>
|
| 403 |
|
| 404 |
<div class="section">
|
| 405 |
<h2>licensing</h2>
|
| 406 |
+
<p>built on google's magentaRT (Apache 2.0 + CC-BY 4.0). users are responsible for their generated outputs and ensuring compliance with applicable laws and platform policies.</p>
|
| 407 |
+
<p><a href="/docs">auto-generated API docs (for all the http requests)</a></p>
|
| 408 |
</div>
|
| 409 |
|
| 410 |
+
<!-- <div class="section">
|
| 411 |
<h2>contributors</h2>
|
| 412 |
<p>Kevin Griffing and Andrew Luck</p>
|
| 413 |
+
</div> -->
|
| 414 |
|
| 415 |
<script>
|
| 416 |
function copyCode(button) {
|