Upd ingest js local loadup #1

.gitignore

@@ -14,3 +14,6 @@ node_modules
 
 # Resources
 exefiles
+
+# Local Netlify folder
+.netlify

ingestion_js/.gitignore

@@ -1 +1,2 @@
 .vercel
+.env

ingestion_js/app/api/debug/route.ts

@@ -15,7 +15,7 @@ export async function GET(req: NextRequest) {
 
     switch (action) {
       case 'status':
-        if (!job_id) {
+  if (!job_id) {
           return NextResponse.json({ error: 'job_id required for status check' }, { status: 400 })
         }
         return await debugJobStatus(job_id)

ingestion_js/app/api/upload/route.ts

@@ -14,6 +14,9 @@ export async function POST(req: NextRequest) {
     return NextResponse.json({ error: 'Server not configured: EMBED_BASE_URL missing' }, { status: 500 })
   }
 
+  // Treat anything not running on Vercel production as local/dev
+  const isLocal = (process.env.VERCEL !== '1') && (process.env.NODE_ENV !== 'production')
+
   const job_id = randomUUID()
   const formData = await req.formData()
 
@@ -33,11 +36,14 @@ export async function POST(req: NextRequest) {
   const MAX_FILES_PER_UPLOAD = parseInt(process.env.MAX_FILES_PER_UPLOAD || '15')
   const MAX_FILE_MB = parseInt(process.env.MAX_FILE_MB || '50')
 
-  if (files.length > MAX_FILES_PER_UPLOAD) {
-    return NextResponse.json(
-      { error: `Too many files. Max ${MAX_FILES_PER_UPLOAD} allowed per upload.` },
-      { status: 400 }
-    )
+  // Skip per-upload count limits when running locally to aid debugging
+  if (!isLocal) {
+    if (files.length > MAX_FILES_PER_UPLOAD) {
+      return NextResponse.json(
+        { error: `Too many files. Max ${MAX_FILES_PER_UPLOAD} allowed per upload.` },
+        { status: 400 }
+      )
+    }
   }
 
   let replace_set = new Set<string>()
@@ -61,11 +67,14 @@ export async function POST(req: NextRequest) {
   const preloaded_files: { fname: string; buf: Buffer }[] = []
   for (const file of files) {
     const raw = Buffer.from(await file.arrayBuffer())
-    if (raw.length > MAX_FILE_MB * 1024 * 1024) {
-      return NextResponse.json(
-        { error: `${file.name} exceeds ${MAX_FILE_MB} MB limit` },
-        { status: 400 }
-      )
+    // Skip per-file size limits when running locally to aid debugging
+    if (!isLocal) {
+      if (raw.length > MAX_FILE_MB * 1024 * 1024) {
+        return NextResponse.json(
+          { error: `${file.name} exceeds ${MAX_FILE_MB} MB limit` },
+          { status: 400 }
+        )
+      }
     }
     const eff_name = rename_map[file.name] || file.name
     preloaded_files.push({ fname: eff_name, buf: raw })
@@ -145,21 +154,32 @@ async function processFilesInBackground(
       const cards = await imports.buildCardsFromPages(pages, fname, user_id, project_id)
       console.log(`[UPLOAD_DEBUG] Built ${cards.length} cards`)
 
-      console.log(`[UPLOAD_DEBUG] Generating embeddings for ${cards.length} cards`)
-      const vectors = await imports.embedRemote(cards.map((c: any) => c.content))
-      if (vectors.length !== cards.length) {
-        throw new Error(`Embedding mismatch: got ${vectors.length} for ${cards.length} cards`)
-      }
-      for (let j = 0; j < cards.length; j++) {
-        cards[j].embedding = vectors[j]
+      // Batch embeddings to avoid high memory usage
+      console.log(`[UPLOAD_DEBUG] Generating embeddings for ${cards.length} cards in batches`)
+      const BATCH = 32
+      for (let start = 0; start < cards.length; start += BATCH) {
+        const end = Math.min(start + BATCH, cards.length)
+        const batch = cards.slice(start, end)
+        const vectors = await imports.embedRemote(batch.map((c: any) => c.content))
+        if (vectors.length !== batch.length) {
+          throw new Error(`Embedding mismatch: got ${vectors.length} for ${batch.length} cards`)
+        }
+        for (let j = 0; j < batch.length; j++) {
+          batch[j].embedding = vectors[j]
+        }
       }
-      console.log(`[UPLOAD_DEBUG] Generated embeddings`)
+      console.log(`[UPLOAD_DEBUG] Generated embeddings (batched)`)
 
       console.log(`[UPLOAD_DEBUG] Storing ${cards.length} cards in MongoDB`)
       await imports.storeCards(cards)
       console.log(`[UPLOAD_DEBUG] Stored cards`)
 
-      const full_text = pages.map((p: any) => p.text).join('\n\n')
+      // Cap text length to reduce memory pressure for summarization
+      const full_text_raw = pages.map((p: any) => p.text).join('\n\n')
+      const MAX_SUMMARY_CHARS = 200_000
+      const full_text = full_text_raw.length > MAX_SUMMARY_CHARS
+        ? full_text_raw.slice(0, MAX_SUMMARY_CHARS)
+        : full_text_raw
       const file_summary = await imports.cheapSummarize(full_text, 6)
       await imports.upsertFileSummary(user_id, project_id, fname, file_summary)
       console.log(`[UPLOAD_DEBUG] Upserted file summary for ${fname}`)

ingestion_js/lib/embedder.ts

@@ -1,36 +1,49 @@
 export async function embedRemote(texts: string[]): Promise<number[][]> {
-  console.log(`[EMBEDDER_DEBUG] Embedding ${texts.length} texts`)
-  
+  const total = texts?.length || 0
+  console.log(`[EMBEDDER_DEBUG] Embedding ${total} texts`)
+
   if (!texts || texts.length === 0) return []
   const base = (process.env.EMBED_BASE_URL || '').replace(/\/$/, '')
   if (!base) {
     console.error('[EMBEDDER_DEBUG] EMBED_BASE_URL is required')
     throw new Error('EMBED_BASE_URL is required')
   }
-  
-  console.log(`[EMBEDDER_DEBUG] Calling ${base}/embed`)
-  
-  const res = await fetch(`${base}/embed`, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({ texts }),
-    // 60s like Python client
-    next: { revalidate: 0 }
-  })
-  
-  if (!res.ok) {
-    console.warn(`[EMBEDDER_DEBUG] Embedding failed with status ${res.status}, using zero vectors`)
-    // Fail closed with zeros to avoid crashes (parity with Python fallback)
-    return Array.from({ length: texts.length }, () => Array(384).fill(0))
+
+  // Memory-safe batching to avoid large payloads in Node/Vercel
+  const batchSize = Math.max(1, parseInt(process.env.EMBED_BATCH_SIZE || '32', 10))
+  const results: number[][] = []
+
+  for (let start = 0; start < texts.length; start += batchSize) {
+    const end = Math.min(start + batchSize, texts.length)
+    const batch = texts.slice(start, end)
+    console.log(`[EMBEDDER_DEBUG] Calling ${base}/embed for batch ${start}..${end - 1} (size=${batch.length})`)
+
+    const res = await fetch(`${base}/embed`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ texts: batch }),
+      next: { revalidate: 0 }
+    })
+
+    if (!res.ok) {
+      console.warn(`[EMBEDDER_DEBUG] Batch embedding failed with status ${res.status}, using zero vectors`)
+      const zeros = Array.from({ length: batch.length }, () => Array(384).fill(0))
+      results.push(...zeros)
+      continue
+    }
+
+    const data = (await res.json()) as any
+    const vectors = Array.isArray(data?.vectors) ? data.vectors : []
+    if (!Array.isArray(vectors) || vectors.length !== batch.length) {
+      console.warn('[EMBEDDER_DEBUG] Invalid vectors format/length, using zero vectors for this batch')
+      const zeros = Array.from({ length: batch.length }, () => Array(384).fill(0))
+      results.push(...zeros)
+      continue
+    }
+
+    results.push(...vectors)
   }
-  
-  const data = await res.json() as any
-  const vectors = Array.isArray(data?.vectors) ? data.vectors : []
-  if (!Array.isArray(vectors)) {
-    console.warn('[EMBEDDER_DEBUG] Invalid vectors format, using zero vectors')
-    return Array.from({ length: texts.length }, () => Array(384).fill(0))
-  }
-  
-  console.log(`[EMBEDDER_DEBUG] Successfully embedded ${vectors.length} vectors`)
-  return vectors
+
+  console.log(`[EMBEDDER_DEBUG] Successfully embedded ${results.length}/${total} vectors`)
+  return results
 }

ingestion_js/lib/parser.ts

@@ -31,41 +31,40 @@ function extractTextFromPdfBuffer(buffer: Buffer): string {
 export async function parsePdfBytes(buf: Buffer): Promise<Page[]> {
   console.log(`[PARSER_DEBUG] Parsing PDF with ${buf.length} bytes`)
   
-  // First, try robust per-page extraction using pdfjs-dist (Node-safe, no canvas rendering)
-  try {
-    const pdfjs: any = await import('pdfjs-dist/legacy/build/pdf')
-    // In Node, we don't use a worker
-    if (pdfjs.GlobalWorkerOptions) {
-      pdfjs.GlobalWorkerOptions.workerSrc = undefined
-    }
-
-    const loadingTask = pdfjs.getDocument({
-      data: new Uint8Array(buf),
-      disableFontFace: true,
-      useSystemFonts: true,
-      isEvalSupported: false
-    })
-    const pdf = await loadingTask.promise
-    const pageCount: number = pdf.numPages
-    console.log(`[PARSER_DEBUG] pdfjs-dist loaded. Pages: ${pageCount}`)
+  // Optional heavy parser (guarded by env to avoid OOM locally)
+  if (process.env.PARSER_USE_PDFJS === '1') {
+    try {
+      const pdfjs: any = await import('pdfjs-dist/legacy/build/pdf')
+      if (pdfjs.GlobalWorkerOptions) {
+        pdfjs.GlobalWorkerOptions.workerSrc = undefined
+      }
+      const loadingTask = pdfjs.getDocument({
+        data: new Uint8Array(buf),
+        disableFontFace: true,
+        useSystemFonts: true,
+        isEvalSupported: false
+      })
+      const pdf = await loadingTask.promise
+      const pageCount: number = pdf.numPages
+      console.log(`[PARSER_DEBUG] pdfjs-dist loaded. Pages: ${pageCount}`)
 
-    const pages: Page[] = []
-    for (let i = 1; i <= pageCount; i++) {
-      const page = await pdf.getPage(i)
-      const textContent = await page.getTextContent()
-      const text = (textContent.items || [])
-        .map((it: any) => (typeof it.str === 'string' ? it.str : ''))
-        .join(' ')
-        .replace(/\s+/g, ' ')
-        .trim()
+      const pages: Page[] = []
+      for (let i = 1; i <= pageCount; i++) {
+        const page = await pdf.getPage(i)
+        const textContent = await page.getTextContent()
+        const text = (textContent.items || [])
+          .map((it: any) => (typeof it.str === 'string' ? it.str : ''))
+          .join(' ')
+          .replace(/\s+/g, ' ')
+          .trim()
 
-      pages.push({ page_num: i, text, images: [] })
+        pages.push({ page_num: i, text, images: [] })
+      }
+      console.log(`[PARSER_DEBUG] Parsed PDF with ${pages.length} pages via pdfjs-dist`)
+      return pages
+    } catch (err) {
+      console.warn('[PARSER_DEBUG] pdfjs-dist extraction failed, falling back to basic extractor:', err)
     }
-
-    console.log(`[PARSER_DEBUG] Parsed PDF with ${pages.length} pages via pdfjs-dist`)
-    return pages
-  } catch (err) {
-    console.warn('[PARSER_DEBUG] pdfjs-dist extraction failed, falling back to basic extractor:', err)
   }
 
   // Fallback: use lightweight extractor and page-count from pdf-lib

ingestion_js/test_upload_js.sh

@@ -6,7 +6,8 @@ echo "🚀 Testing Ingestion JS API on Vercel"
 echo "======================================"
 
 # Configuration
-BACKEND_URL="https://study-buddy-ingestion1.vercel.app/api"
+# BACKEND_URL="https://study-buddy-ingestion1.vercel.app/api"
+BACKEND_URL="http://localhost:3001/api"
 USER_ID="44e65346-8eaa-4f95-b17a-f6219953e7a8"
 PROJECT_ID="496e2fad-ec7e-4562-b06a-ea2491f2460"