Optimize data fetching: remove date range query and per-agent DB queries for better performance

src/lib/dataService.js

@@ -140,26 +140,39 @@ class DataService {
   async _fetchAllFromRemote() {
     // First, get the max date from DB to know what we're aiming for
     let dbMaxDate = null
+    let dbMinDate = null
     try {
-      const { data: maxDateData, error: maxDateError } = await supabase
-        .from('trading_decisions')
-        .select('date')
-        .order('date', { ascending: false })
-        .limit(1)
-      if (!maxDateError && maxDateData && maxDateData.length > 0) {
-        dbMaxDate = maxDateData[0].date
-        console.log(`[DataService] Target max date from DB: ${dbMaxDate}`)
+      const [minDateResult, maxDateResult] = await Promise.all([
+        supabase
+          .from('trading_decisions')
+          .select('date')
+          .order('date', { ascending: true })
+          .limit(1),
+        supabase
+          .from('trading_decisions')
+          .select('date')
+          .order('date', { ascending: false })
+          .limit(1)
+      ])
+      
+      if (!minDateResult.error && minDateResult.data && minDateResult.data.length > 0) {
+        dbMinDate = minDateResult.data[0].date
+      }
+      if (!maxDateResult.error && maxDateResult.data && maxDateResult.data.length > 0) {
+        dbMaxDate = maxDateResult.data[0].date
       }
+      console.log(`[DataService] DB date range: min = ${dbMinDate}, max = ${dbMaxDate}`)
     } catch (e) {
-      console.error('[DataService] Error getting DB max date:', e)
+      console.error('[DataService] Error getting DB date range:', e)
     }
 
+    const all = []
+    
+    // Fetch all data using pagination (faster than date range query)
     const pageSize = 1000
     let from = 0
-    const all = []
     let pageCount = 0
-    let lastFetchedDate = null
-
+    
     while (true) {
       const to = from + pageSize - 1
       pageCount++
@@ -187,7 +200,7 @@ class DataService {
       // Track the last date in this page
       const pageDates = pageData.map(r => r && r.date).filter(Boolean).sort()
       if (pageDates.length > 0) {
-        lastFetchedDate = pageDates[pageDates.length - 1]
+        const lastFetchedDate = pageDates[pageDates.length - 1]
         console.log(`[DataService] Page ${pageCount}: ${pageData.length} rows, date range: ${pageDates[0]} to ${lastFetchedDate}`)
       }
       
@@ -197,33 +210,29 @@ class DataService {
         break
       }
 
-      // Continue to next page - don't stop early based on date comparison
-      // because pagination with order('date', ascending: true) might have
-      // multiple pages with the same date, and we need to fetch all pages
       from += pageSize
     }
 
-    // Log the final date range
+    // Log the final date range and verify - if missing data, fetch it
     if (all.length > 0) {
       const dates = all.map(r => r && r.date).filter(Boolean).sort()
       const fetchedMinDate = dates[0]
       const fetchedMaxDate = dates[dates.length - 1]
-      console.log(`[DataService] Total fetched: ${all.length} rows across ${pageCount} pages, date range: ${fetchedMinDate} to ${fetchedMaxDate}`)
+      console.log(`[DataService] Total fetched: ${all.length} rows, date range: ${fetchedMinDate} to ${fetchedMaxDate}`)
       console.log(`[DataService] Last 10 dates:`, dates.slice(-10))
       
-      // Verify we got the max date - if not, fetch missing data by date range
+      // Verify we got the max date - if not, fetch missing data
       if (dbMaxDate) {
         const fetchedMaxDateStr = typeof fetchedMaxDate === 'string' ? fetchedMaxDate.split('T')[0] : fetchedMaxDate
         const dbMaxDateStr = typeof dbMaxDate === 'string' ? dbMaxDate.split('T')[0] : dbMaxDate
         console.log(`[DataService] Verification: DB max date = ${dbMaxDateStr}, Fetched max date = ${fetchedMaxDateStr}`)
         
         if (fetchedMaxDateStr !== dbMaxDateStr && fetchedMaxDateStr < dbMaxDateStr) {
-          console.warn(`[DataService] Missing data detected! DB has ${dbMaxDateStr} but we only fetched up to ${fetchedMaxDateStr}`)
+          console.warn(`[DataService] Missing data detected! DB has ${dbMaxDateStr} but we fetched up to ${fetchedMaxDateStr}`)
           console.log(`[DataService] Fetching missing data from ${fetchedMaxDateStr} to ${dbMaxDateStr}...`)
           
-          // Fetch all data from the day after fetchedMaxDate to dbMaxDate
-          // Use gte (greater than or equal) to include the next day
-          const nextDay = new Date(fetchedMaxDateStr)
+          // Fetch missing data by date range
+          const nextDay = new Date(fetchedMaxDateStr + 'T00:00:00.000Z')
           nextDay.setUTCDate(nextDay.getUTCDate() + 1)
           const nextDayStr = nextDay.toISOString().split('T')[0]
           
@@ -236,13 +245,12 @@ class DataService {
           
           if (!missingError && missingData && missingData.length > 0) {
             console.log(`[DataService] Fetched ${missingData.length} missing rows`)
-            // Merge missing data, avoiding duplicates by id
             const existingIds = new Set(all.map(r => r.id))
             const newRows = missingData.filter(r => !existingIds.has(r.id))
             all.push(...newRows)
-            console.log(`[DataService] Added ${newRows.length} new rows (${missingData.length - newRows.length} duplicates skipped)`)
+            console.log(`[DataService] Added ${newRows.length} new rows`)
             
-            // Re-sort all data by date
+            // Re-sort
             all.sort((a, b) => {
               const dateA = typeof a.date === 'string' ? a.date.split('T')[0] : a.date
               const dateB = typeof b.date === 'string' ? b.date.split('T')[0] : b.date
@@ -251,10 +259,6 @@ class DataService {
             
             const finalDates = all.map(r => r && r.date).filter(Boolean).sort()
             console.log(`[DataService] After fetching missing data: ${all.length} total rows, date range: ${finalDates[0]} to ${finalDates[finalDates.length - 1]}`)
-          } else if (missingError) {
-            console.error(`[DataService] Failed to fetch missing data:`, missingError)
-          } else {
-            console.warn(`[DataService] No missing data found (this might indicate a data inconsistency)`)
           }
         } else if (fetchedMaxDateStr === dbMaxDateStr) {
           console.log(`[DataService] ✓ Successfully fetched all data up to ${dbMaxDateStr}`)
@@ -323,7 +327,7 @@ class DataService {
         .sort()
       const start_date = dates[0] || '-'
       
-      // 计算 end_date：先从已获取的数据计算，然后查询数据库获取该 agent 的最新日期
+      // 计算 end_date：从已获取的数据中提取，因为数据已经完整
       let end_date = '-'
       if (seqFiltered.length > 0) {
         const lastRow = seqFiltered[seqFiltered.length - 1]
@@ -332,31 +336,6 @@ class DataService {
         end_date = dates[dates.length - 1]
       }
       
-      // 查询数据库获取该 agent 的最新日期，确保 end_date 是最新的
-      try {
-        const { data: agentMaxDateData, error: agentMaxDateError } = await supabase
-          .from('trading_decisions')
-          .select('date')
-          .eq('agent_name', row.agent_name)
-          .eq('asset', row.asset)
-          .eq('model', row.model)
-          .order('date', { ascending: false })
-          .limit(1)
-        
-        if (!agentMaxDateError && agentMaxDateData && agentMaxDateData.length > 0) {
-          const dbAgentMaxDate = agentMaxDateData[0].date
-          const dbAgentMaxDateStr = typeof dbAgentMaxDate === 'string' ? dbAgentMaxDate.split('T')[0] : dbAgentMaxDate
-          
-          // 如果数据库中的日期更新，使用数据库的日期
-          if (dbAgentMaxDateStr && (!end_date || end_date === '-' || dbAgentMaxDateStr > end_date)) {
-            console.log(`[DataService] STEP3.5 - ${row.agent_name}|${row.asset}|${row.model}: DB has newer date ${dbAgentMaxDateStr} than calculated ${end_date}, using DB date`)
-            end_date = dbAgentMaxDateStr
-          }
-        }
-      } catch (e) {
-        console.error(`[DataService] Error querying DB for agent max date:`, e)
-      }
-      
       console.log(`[DataService] STEP3 - ${row.agent_name}|${row.asset}|${row.model}: Final end_date = ${end_date}, dates array length = ${dates.length}, seqFiltered length = ${seqFiltered.length}`)
       if (seqFiltered.length > 0) {
         const lastRowDate = seqFiltered[seqFiltered.length - 1].dateNormalized || (typeof seqFiltered[seqFiltered.length - 1].date === 'string' ? seqFiltered[seqFiltered.length - 1].date.split('T')[0] : seqFiltered[seqFiltered.length - 1].date)