Improve data fetching: track max date and auto-fetch missing pages

src/lib/dataService.js

@@ -138,12 +138,33 @@ class DataService {
    * 从 Supabase 拉取所有数据
    */
   async _fetchAllFromRemote() {
+    // First, get the max date from DB to know what we're aiming for
+    let dbMaxDate = null
+    try {
+      const { data: maxDateData, error: maxDateError } = await supabase
+        .from('trading_decisions')
+        .select('date')
+        .order('date', { ascending: false })
+        .limit(1)
+      if (!maxDateError && maxDateData && maxDateData.length > 0) {
+        dbMaxDate = maxDateData[0].date
+        console.log(`[DataService] Target max date from DB: ${dbMaxDate}`)
+      }
+    } catch (e) {
+      console.error('[DataService] Error getting DB max date:', e)
+    }
+
     const pageSize = 1000
     let from = 0
     const all = []
+    let pageCount = 0
+    let lastFetchedDate = null
 
     while (true) {
       const to = from + pageSize - 1
+      pageCount++
+      console.log(`[DataService] Fetching page ${pageCount}: range(${from}, ${to})`)
+      
       const { data, error } = await supabase
         .from('trading_decisions')
         .select('id, agent_name, asset, model, date, price, recommended_action, news_count, sentiment, created_at, updated_at')
@@ -155,16 +176,71 @@ class DataService {
         break
       }
 
-      all.push(...(data || []))
+      const pageData = data || []
+      if (pageData.length === 0) {
+        console.log(`[DataService] Page ${pageCount}: 0 rows, stopping`)
+        break
+      }
+
+      all.push(...pageData)
+      
+      // Track the last date in this page
+      const pageDates = pageData.map(r => r && r.date).filter(Boolean).sort()
+      if (pageDates.length > 0) {
+        lastFetchedDate = pageDates[pageDates.length - 1]
+        console.log(`[DataService] Page ${pageCount}: ${pageData.length} rows, date range: ${pageDates[0]} to ${lastFetchedDate}`)
+      }
       
-      if (!data || data.length < pageSize) break
+      // If we got less than pageSize, we're done
+      if (pageData.length < pageSize) {
+        console.log(`[DataService] Last page reached (got ${pageData.length} rows, expected ${pageSize})`)
+        break
+      }
+
+      // If we've reached or passed the max date, we're done
+      if (dbMaxDate && lastFetchedDate) {
+        const lastFetchedDateStr = typeof lastFetchedDate === 'string' ? lastFetchedDate.split('T')[0] : lastFetchedDate
+        const dbMaxDateStr = typeof dbMaxDate === 'string' ? dbMaxDate.split('T')[0] : dbMaxDate
+        if (lastFetchedDateStr >= dbMaxDateStr) {
+          console.log(`[DataService] Reached target max date ${dbMaxDateStr}, stopping`)
+          break
+        }
+      }
+
       from += pageSize
     }
 
-    // Log the date range to help debug
+    // Log the final date range
     if (all.length > 0) {
       const dates = all.map(r => r && r.date).filter(Boolean).sort()
-      console.log('[DataService] Fetched data date range:', dates[0], 'to', dates[dates.length - 1], 'total:', all.length)
+      console.log(`[DataService] Total fetched: ${all.length} rows across ${pageCount} pages, date range: ${dates[0]} to ${dates[dates.length - 1]}`)
+      console.log(`[DataService] Last 10 dates:`, dates.slice(-10))
+      
+      // Verify we got the max date
+      const fetchedMaxDate = dates[dates.length - 1]
+      if (dbMaxDate) {
+        const fetchedMaxDateStr = typeof fetchedMaxDate === 'string' ? fetchedMaxDate.split('T')[0] : fetchedMaxDate
+        const dbMaxDateStr = typeof dbMaxDate === 'string' ? dbMaxDate.split('T')[0] : dbMaxDate
+        console.log(`[DataService] Verification: DB max date = ${dbMaxDateStr}, Fetched max date = ${fetchedMaxDateStr}`)
+        if (fetchedMaxDateStr !== dbMaxDateStr) {
+          console.error(`[DataService] ERROR: Still missing data! DB has ${dbMaxDateStr} but we only fetched up to ${fetchedMaxDateStr}`)
+          // Try one more page to see if we can get the missing data
+          console.log(`[DataService] Attempting to fetch one more page starting from ${from}...`)
+          const { data: extraData, error: extraError } = await supabase
+            .from('trading_decisions')
+            .select('id, agent_name, asset, model, date, price, recommended_action, news_count, sentiment, created_at, updated_at')
+            .order('date', { ascending: true })
+            .range(from, from + pageSize - 1)
+          if (!extraError && extraData && extraData.length > 0) {
+            console.log(`[DataService] Extra page fetched: ${extraData.length} rows`)
+            all.push(...extraData)
+            const newDates = all.map(r => r && r.date).filter(Boolean).sort()
+            console.log(`[DataService] After extra page: date range: ${newDates[0]} to ${newDates[newDates.length - 1]}`)
+          }
+        }
+      }
+    } else {
+      console.warn('[DataService] No data fetched!')
     }
 
     return all