fix: build local llama (#1805)

* fix: build local llama

* fix: node build local

* fix: docker build

* feat: use group for CI?

* fix: github actions

.github/workflows/build-image.yml

@@ -19,7 +19,8 @@ on:
 
 jobs:
   build-and-publish-image-with-db:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: aws-general-8-plus
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -79,7 +80,8 @@ jobs:
             INCLUDE_DB=true
             PUBLIC_COMMIT_SHA=${{ env.GITHUB_SHA_SHORT }}
   build-and-publish-image-nodb:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: aws-general-8-plus
     steps:
       - name: Checkout
         uses: actions/checkout@v4

Dockerfile

@@ -101,6 +101,5 @@ ENV MODELS_STORAGE_PATH=/data/models
 #import the build & dependencies
 COPY --from=builder --chown=1000 /app/build /app/build
 COPY --from=builder --chown=1000 /app/node_modules /app/node_modules
-COPY --from=builder --chown=1000 /app/node_modules/node-llama-cpp/llama /app/build/server/llama
 
 CMD ["/bin/bash", "-c", "/app/entrypoint.sh"]

package-lock.json

@@ -21,7 +21,7 @@
 				"aws-sigv4-fetch": "^4.0.1",
 				"aws4": "^1.13.0",
 				"date-fns": "^2.29.3",
-				"dotenv": "^16.0.3",
+				"dotenv": "^16.5.0",
 				"express": "^4.21.2",
 				"file-type": "^19.4.1",
 				"google-auth-library": "^9.13.0",
@@ -66,6 +66,7 @@
 				"@tailwindcss/typography": "^0.5.9",
 				"@types/dompurify": "^3.0.5",
 				"@types/express": "^4.17.21",
+				"@types/fs-extra": "^11.0.4",
 				"@types/js-yaml": "^4.0.9",
 				"@types/jsdom": "^21.1.1",
 				"@types/jsonpath": "^0.2.4",
@@ -82,6 +83,7 @@
 				"eslint": "^8.28.0",
 				"eslint-config-prettier": "^8.5.0",
 				"eslint-plugin-svelte": "^2.45.1",
+				"fs-extra": "^11.3.0",
 				"isomorphic-dompurify": "^2.13.0",
 				"js-yaml": "^4.1.0",
 				"minimist": "^1.2.8",
@@ -5053,6 +5055,17 @@
 			"resolved": "https://registry.npmjs.org/@types/firefox-webext-browser/-/firefox-webext-browser-120.0.4.tgz",
 			"integrity": "sha512-lBrpf08xhiZBigrtdQfUaqX1UauwZ+skbFiL8u2Tdra/rklkKadYmIzTwkNZSWtuZ7OKpFqbE2HHfDoFqvZf6w=="
 		},
+		"node_modules/@types/fs-extra": {
+			"version": "11.0.4",
+			"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz",
+			"integrity": "sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"@types/jsonfile": "*",
+				"@types/node": "*"
+			}
+		},
 		"node_modules/@types/har-format": {
 			"version": "1.2.15",
 			"resolved": "https://registry.npmjs.org/@types/har-format/-/har-format-1.2.15.tgz",
@@ -5087,6 +5100,16 @@
 			"integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==",
 			"dev": true
 		},
+		"node_modules/@types/jsonfile": {
+			"version": "6.1.4",
+			"resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.4.tgz",
+			"integrity": "sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"@types/node": "*"
+			}
+		},
 		"node_modules/@types/jsonpath": {
 			"version": "0.2.4",
 			"resolved": "https://registry.npmjs.org/@types/jsonpath/-/jsonpath-0.2.4.tgz",
@@ -7167,9 +7190,10 @@
 			}
 		},
 		"node_modules/dotenv": {
-			"version": "16.4.5",
-			"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz",
-			"integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==",
+			"version": "16.5.0",
+			"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz",
+			"integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==",
+			"license": "BSD-2-Clause",
 			"engines": {
 				"node": ">=12"
 			},

package.json

@@ -27,6 +27,7 @@
 		"@tailwindcss/typography": "^0.5.9",
 		"@types/dompurify": "^3.0.5",
 		"@types/express": "^4.17.21",
+		"@types/fs-extra": "^11.0.4",
 		"@types/js-yaml": "^4.0.9",
 		"@types/jsdom": "^21.1.1",
 		"@types/jsonpath": "^0.2.4",
@@ -43,6 +44,7 @@
 		"eslint": "^8.28.0",
 		"eslint-config-prettier": "^8.5.0",
 		"eslint-plugin-svelte": "^2.45.1",
+		"fs-extra": "^11.3.0",
 		"isomorphic-dompurify": "^2.13.0",
 		"js-yaml": "^4.1.0",
 		"minimist": "^1.2.8",
@@ -79,7 +81,7 @@
 		"aws-sigv4-fetch": "^4.0.1",
 		"aws4": "^1.13.0",
 		"date-fns": "^2.29.3",
-		"dotenv": "^16.0.3",
+		"dotenv": "^16.5.0",
 		"express": "^4.21.2",
 		"file-type": "^19.4.1",
 		"google-auth-library": "^9.13.0",

src/lib/server/endpoints/local/utilsLocal.ts

@@ -28,7 +28,6 @@ export const llama = await getLlama({
 				break;
 		}
 	},
-	build: "never",
 }).catch((e) => {
 	logger.warn(
 		e,

src/lib/server/logger.ts

@@ -15,4 +15,4 @@ if (dev) {
 	};
 }
 
-export const logger = pino({ ...options, level: config.LOG_LEVEL ?? "info" });
+export const logger = pino({ ...options, level: config.LOG_LEVEL || "info" });

vite.config.ts

@@ -2,6 +2,10 @@ import { sveltekit } from "@sveltejs/kit/vite";
 import Icons from "unplugin-icons/vite";
 import { promises } from "fs";
 import { defineConfig } from "vitest/config";
+import { resolve } from "path";
+import fs from "fs-extra";
+import { spawn } from "child_process";
+import type { Plugin } from "vite";
 
 // used to load fonts server side for thumbnail generation
 function loadTTFAsArrayBuffer() {
@@ -16,6 +20,65 @@ function loadTTFAsArrayBuffer() {
 		},
 	};
 }
+const isViteNode = process.argv.some((arg) => arg.includes("vite-node")) || !!process.env.VITE_NODE;
+const shouldCopyLlama = process.env.npm_lifecycle_event === "build" && !isViteNode; // Copy node-llama-cpp/llama files to build output
+
+function copyLlamaFiles() {
+	return {
+		name: "copy-llama-files",
+		apply: "build" as const,
+		closeBundle: async () => {
+			try {
+				// Run npx command first and pipe IO
+				console.log("Running node-llama-cpp source download...");
+
+				await new Promise((resolve, reject) => {
+					const npxProcess = spawn("npx", ["--no", "node-llama-cpp", "source", "download"], {
+						stdio: "inherit", // Pipe all IO to parent process
+						shell: true,
+					});
+
+					npxProcess.on("close", (code) => {
+						if (code === 0) {
+							console.log("✓ Successfully downloaded llama source files");
+							resolve(code);
+						} else {
+							reject(new Error(`npx command failed with code ${code}`));
+						}
+					});
+
+					npxProcess.on("error", (err) => {
+						reject(err);
+					});
+				});
+
+				const sourcePath = resolve("node_modules/node-llama-cpp/llama");
+				const destPath = resolve("build/server/llama");
+
+				// Ensure destination directory exists
+				await fs.ensureDir(destPath);
+
+				// Copy files - using a filter to prevent copying files to subdirectories of themselves
+				await fs.copy(sourcePath, destPath, {
+					filter: (src, dest) => {
+						// Skip if source path is inside destination path or vice versa
+						if (src.includes(destPath) || dest.includes(sourcePath)) {
+							console.log(`Skipping problematic copy: ${src} -> ${dest}`);
+							return false;
+						}
+						return true;
+					},
+					overwrite: true,
+					dereference: true,
+				});
+
+				console.log("✓ Successfully copied llama files to build output");
+			} catch (error) {
+				console.error("Error in llama files process:", error);
+			}
+		},
+	} satisfies Plugin;
+}
 
 export default defineConfig({
 	plugins: [
@@ -24,6 +87,7 @@ export default defineConfig({
 			compiler: "svelte",
 		}),
 		loadTTFAsArrayBuffer(),
+		...(shouldCopyLlama ? [copyLlamaFiles()] : []),
 	],
 	optimizeDeps: {
 		include: ["uuid", "@huggingface/transformers", "sharp", "@gradio/client", "clsx"],