fix: only import node-llama-cpp if needed and skip for huggingchat image

.github/workflows/deploy-prod.yml

@@ -49,6 +49,7 @@ jobs:
             APP_BASE=/chat
             PUBLIC_APP_COLOR=yellow
             PUBLIC_COMMIT_SHA=${{ env.GITHUB_SHA_SHORT }}
+            SKIP_LLAMA_CPP_BUILD=true
   deploy:
     name: Deploy on prod
     runs-on: ubuntu-latest

README.md

@@ -1104,9 +1104,9 @@ docker build -t huggingchat:latest --build-arg INCLUDE_DB=false --build-arg APP_
 If you want to run the images with your local .env.local you have two options
 
 ```bash
-DOTENV_LOCAL=$(<.env.local)  docker run --rm --network=host -e DOTENV_LOCAL -p 3000:3000 chat-ui
+DOTENV_LOCAL=$(<.env.local)  docker run --network=host -e DOTENV_LOCAL chat-ui-db
 ```
 
 ```bash
-docker run --rm --network=host --mount type=bind,source="$(pwd)/.env.local",target=/app/.env.local -p 3000:3000 chat-ui
+docker run --network=host --mount type=bind,source="$(pwd)/.env.local",target=/app/.env.local chat-ui-db
 ```

src/lib/server/endpoints/local/endpointLocal.ts

@@ -10,14 +10,11 @@ import {
 	makeImageProcessor,
 	type ImageProcessor,
 } from "../images";
-
-import { LlamaChatSession, LlamaContextSequence, resolveModelFile } from "node-llama-cpp";
 import { findRepoRoot } from "$lib/server/findRepoRoot";
 import { fileURLToPath } from "url";
 import { dirname, join } from "path";
 import { logger } from "$lib/server/logger";
-import { llama } from "./utilsLocal";
-
+import type { LlamaContextSequence } from "node-llama-cpp";
 export const endpointLocalParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
 	model: z.any(),
@@ -55,7 +52,37 @@ export async function endpointLocal(
 
 	// Initialize Llama model
 
+	const { getLlama, LlamaChatSession, resolveModelFile } = await import("node-llama-cpp");
+
 	const modelPath = await resolveModelFile(path, modelFolder);
+
+	const llama = await getLlama({
+		logger: (level, message) => {
+			switch (level) {
+				case "fatal":
+					logger.fatal(message);
+					break;
+				case "error":
+					logger.error(message);
+					break;
+				case "warn":
+					logger.warn(message);
+					break;
+				case "info":
+					logger.info(message);
+					break;
+				case "log":
+					logger.info(message); // Map 'log' to 'info' since pino doesn't have a 'log' level
+					break;
+				case "debug":
+					logger.debug(message);
+					break;
+				default:
+					break;
+			}
+		},
+	});
+
 	if (!llama) {
 		throw new Error("Failed to initialize llama.cpp build.");
 	}

src/lib/server/endpoints/local/utilsLocal.ts

@@ -1,37 +0,0 @@
-import { getLlama } from "node-llama-cpp";
-import { logger } from "$lib/server/logger";
-
-export const llama = await getLlama({
-	logger: (level, message) => {
-		// Log messages based on their level
-		switch (level) {
-			case "fatal":
-				logger.fatal(message);
-				break;
-			case "error":
-				logger.error(message);
-				break;
-			case "warn":
-				logger.warn(message);
-				break;
-			case "info":
-				logger.info(message);
-				break;
-			case "log":
-				logger.info(message); // Map 'log' to 'info' since pino doesn't have a 'log' level
-				break;
-			case "debug":
-				logger.debug(message);
-				break;
-			default:
-				// For 'disabled' or any other unexpected levels
-				break;
-		}
-	},
-}).catch((e) => {
-	logger.warn(
-		e,
-		"Failed to initialize llama.cpp. This won't break anything if you're not using the \"local\" endpoint."
-	);
-	return undefined;
-});

src/lib/server/models.ts

@@ -14,7 +14,6 @@ import { getTokenizer } from "$lib/utils/getTokenizer";
 import { logger } from "$lib/server/logger";
 import { type ToolInput } from "$lib/types/Tool";
 import { join, dirname } from "path";
-import { resolveModelFile, readGgufFileInfo } from "node-llama-cpp";
 import { fileURLToPath } from "url";
 import { findRepoRoot } from "./findRepoRoot";
 import { Template } from "@huggingface/jinja";
@@ -151,6 +150,8 @@ async function getChatPromptRender(
 		const endpoint = m.endpoints?.find((e) => e.type === "local");
 		const path = endpoint?.modelPath ?? `hf:${m.id ?? m.name}`;
 
+		const { resolveModelFile, readGgufFileInfo } = await import("node-llama-cpp");
+
 		const modelPath = await resolveModelFile(path, MODELS_FOLDER);
 
 		const info = await readGgufFileInfo(modelPath, {