Enhance MT5 MarketDataSender with robust symbol matching and incremental candle uploads

- Rewrote FindSymbolId function for exact symbol matching in JSON responses with improved parsing and debug logging
- Added GetLatestCandleTime function to fetch latest stored candle time from API
- Modified SendAllHistoricalCandles to perform incremental uploads, sending only new candles after the latest stored time
- Enhanced SendLivePrices with better price validation, fallback mechanisms, and market depth integration for bid/ask sizes
- Improved error handling and logging throughout for better debugging

MT5/Experts/MarketDataSender.mq5

@@ -135,22 +135,63 @@ bool SyncSymbolsWithDatabase()
    return true;
 }
 
+//+------------------------------------------------------------------+
+//+------------------------------------------------------------------+
+//| Find exact symbolId from JSON response                           |
+//+------------------------------------------------------------------+
+//+------------------------------------------------------------------+
+//| Robust JSON search: matches exact symbol only                    |
 //+------------------------------------------------------------------+
 int FindSymbolId(string response, string symbolName)
 {
-   string searchPattern = StringFormat("\"symbol\":\"%s\"", symbolName);
-   int symbolPos = StringFind(response, searchPattern);
-   if(symbolPos < 0) return -1;
+   int pos = 0;
+   string patternSymbol = "\"symbol\":\"";
+   string patternId = "\"id\":";
 
-   int idPos = StringFind(response, "\"id\":", symbolPos);
-   if(idPos < 0) return -1;
+   while(true)
+   {
+      // find each symbol occurrence
+      int symPos = StringFind(response, patternSymbol, pos);
+      if(symPos < 0)
+         break;
 
-   int startPos = idPos + 5;
-   int endPos = StringFind(response, ",", startPos);
-   if(endPos < 0) endPos = StringFind(response, "}", startPos);
+      // extract actual symbol value
+      int symStart = symPos + StringLen(patternSymbol);
+      int symEnd = StringFind(response, "\"", symStart);
+      if(symEnd < 0) break;
 
-   string idStr = StringSubstr(response, startPos, endPos - startPos);
-   return (int)StringToInteger(idStr);
+      string foundSymbol = StringSubstr(response, symStart, symEnd - symStart);
+
+      // 🟩 DEBUG LOG: show all symbols found
+      // ✅ exact match check (case-sensitive)
+      if(foundSymbol == symbolName)
+      {
+         // find id that comes *before* this symbol entry
+         int blockStart = StringFind(response, patternId, symPos - 100);
+         if(blockStart < 0)
+            blockStart = StringFind(response, patternId, symPos);
+
+         if(blockStart >= 0)
+         {
+            int idStart = blockStart + StringLen(patternId);
+            int idEnd = StringFind(response, ",", idStart);
+            if(idEnd < 0) idEnd = StringFind(response, "}", idStart);
+
+            string idStr = StringSubstr(response, idStart, idEnd - idStart);
+            int id = (int)StringToInteger(idStr);
+
+            // 🟩 Success log
+            Print("✅ Exact match found → symbol='", symbolName, "' | ID=", id);
+            return id;
+         }
+      }
+
+      // move to next
+      pos = symEnd + 1;
+   }
+
+   Print("⚠️ No exact match for symbol ", symbolName);
+   return -1;
 }
 
 //+------------------------------------------------------------------+
@@ -204,6 +245,54 @@ int CreateSymbolInDatabase(string symbolName)
    string idStr = StringSubstr(createResponse, startPos, endPos - startPos);
    return (int)StringToInteger(idStr);
 }
+//+------------------------------------------------------------------+
+//| Fetch latest stored candle openTime from API                     |
+//+------------------------------------------------------------------+
+datetime GetLatestCandleTime(int symbolId)
+{
+   string url = ApiBaseUrl + "/api/candles/" + IntegerToString(symbolId) + "/latest";
+   string headers = "Content-Type: application/json\r\n";
+   string resultHeaders = "";
+   char result[];
+   char emptyData[];
+
+   ResetLastError();
+   int res = WebRequest("GET", url, headers, 10000, emptyData, result, resultHeaders);
+
+   if(res != 200)
+   {
+      Print("⚠️ Could not fetch latest candle for symbolId=", symbolId, " (HTTP ", res, ")");
+      return 0;
+   }
+
+   string response = CharArrayToString(result);
+   int pos = StringFind(response, "\"openTime\":\"");
+   if(pos < 0)
+   {
+      Print("⚠️ No openTime found in response for symbolId=", symbolId);
+      return 0;
+   }
+
+   pos += StringLen("\"openTime\":\"");
+   int end = StringFind(response, "\"", pos);
+   string openTimeStr = StringSubstr(response, pos, end - pos);
+
+   // --- Parse ISO8601 to datetime ---
+   int year  = (int)StringToInteger(StringSubstr(openTimeStr, 0, 4));
+   int month = (int)StringToInteger(StringSubstr(openTimeStr, 5, 2));
+   int day   = (int)StringToInteger(StringSubstr(openTimeStr, 8, 2));
+   int hour  = (int)StringToInteger(StringSubstr(openTimeStr, 11, 2));
+   int min   = (int)StringToInteger(StringSubstr(openTimeStr, 14, 2));
+   int sec   = (int)StringToInteger(StringSubstr(openTimeStr, 17, 2));
+
+   MqlDateTime t;
+   t.year = year; t.mon = month; t.day = day;
+   t.hour = hour; t.min = min; t.sec = sec;
+
+   datetime dt = StructToTime(t);
+   Print("🕓 Latest stored candle openTime for symbolId=", symbolId, " → ", TimeToString(dt, TIME_DATE|TIME_SECONDS));
+   return dt;
+}
 
 //+------------------------------------------------------------------+
 //+------------------------------------------------------------------+
@@ -216,6 +305,11 @@ void SendAllHistoricalCandles()
    for(int i = 0; i < ArraySize(symbols); i++)
    {
       string sym = symbols[i];
+      int symbolId = symbolIds[i];
+      if(symbolId <= 0) continue;
+
+      // --- Get last stored candle time ---
+      datetime latestApiTime = GetLatestCandleTime(symbolId);
 
       // --- Ensure data is ready ---
       Sleep(300);
@@ -227,42 +321,43 @@ void SendAllHistoricalCandles()
          tries++;
       }
 
-      // --- Now copy candles ---
       MqlRates rates[];
       ResetLastError();
       int copied = CopyRates(sym, HistoricalTimeframe, 0, HistoricalCandleCount, rates);
-      int err = GetLastError();
-
       if(copied <= 0)
       {
-         Print("⚠️ Failed to copy candles for ", sym, " (copied=", copied, ", err=", err, ")");
+         Print("⚠️ Failed to copy candles for ", sym);
          continue;
       }
 
       Print("✅ Copied ", copied, " candles for ", sym);
 
-      // --- Print a few sample candles ---
-      int sampleCount = MathMin(5, copied); // show up to 5 examples
-      for(int j = 0; j < sampleCount; j++)
+      // --- Filter new candles ---
+      int startIndex = 0;
+      for(int j = 0; j < copied; j++)
       {
-         MqlRates r = rates[j];
-         string openTime = TimeToString(r.time, TIME_DATE|TIME_SECONDS);
-         string closeTime = TimeToString(r.time + PeriodSeconds(HistoricalTimeframe), TIME_DATE|TIME_SECONDS);
-         PrintFormat(
-            "🕒 [%s] %s → %s | O=%.5f H=%.5f L=%.5f C=%.5f | Vol=%.2f",
-            sym, openTime, closeTime, r.open, r.high, r.low, r.close, r.tick_volume
-         );
+         if(rates[j].time > latestApiTime)
+         {
+            startIndex = j;
+            break;
+         }
       }
 
-      // --- Send candles in batches to API ---
-      int sentTotal = 0;
+      int newCount = copied - startIndex;
+      if(newCount <= 0)
+      {
+         Print("ℹ️ No new candles to send for ", sym);
+         continue;
+      }
+
+      Print("🆕 Sending ", newCount, " new candles for ", sym, " after ", TimeToString(latestApiTime, TIME_DATE|TIME_SECONDS));
+
+      // --- Send new candles in batches ---
       int batchSize = 200;
-      for(int start = 0; start < copied; start += batchSize)
+      int sentTotal = 0;
+      for(int start = startIndex; start < copied; start += batchSize)
       {
          int size = MathMin(batchSize, copied - start);
-         int symbolId = symbolIds[i];
-         if(symbolId <= 0) continue;
-
          string json = BuildCandleJSONFromRates(symbolId, rates, start, size);
          string url = ApiBaseUrl + "/api/candles/bulk";
          string response;
@@ -273,33 +368,116 @@ void SendAllHistoricalCandles()
             break;
          }
          sentTotal += size;
-         Print("📤 Sent candles for ", sym, ": ", sentTotal, "/", copied);
+         Print("📤 Sent ", sentTotal, "/", newCount, " new candles for ", sym);
       }
    }
-   Print("✅ Historical upload finished.");
+   Print("✅ Incremental candle upload finished.");
 }
 
-
+//+------------------------------------------------------------------+
+//| Send live prices of all active symbols                           |
 //+------------------------------------------------------------------+
 void SendLivePrices()
 {
    bool firstItem = true;
    string json = "{\"prices\":[";
-
    int sentCount = 0;
+
    for(int i = 0; i < ArraySize(symbols); i++)
    {
       string sym = symbols[i];
+      int symId = symbolIds[i];
+      if(symId <= 0) continue;
+
+      // Ensure symbol is visible in Market Watch
+      if(!SymbolSelect(sym, true))
+      {
+         Print("⚠️ Failed to select symbol: ", sym);
+         continue;
+      }
+
+      // Read primary prices
       double bid = SymbolInfoDouble(sym, SYMBOL_BID);
       double ask = SymbolInfoDouble(sym, SYMBOL_ASK);
       double last = SymbolInfoDouble(sym, SYMBOL_LAST);
-      if(bid <= 0 || ask <= 0 || last <= 0) continue;
 
-      int symId = symbolIds[i];
-      if(symId <= 0) continue;
+      // If last = 0 (some providers), use midprice as fallback
+      if(last <= 0 && bid > 0 && ask > 0)
+         last = (bid + ask) / 2.0;
 
-      string item = StringFormat("{\"symbolId\":%d,\"price\":%.8f,\"bid\":%.8f,\"ask\":%.8f,\"bidSize\":%.8f,\"askSize\":%.8f}",
-                                 symId, last, bid, ask, 0.0, 0.0);
+      // Skip if prices are still invalid
+      if(bid <= 0 || ask <= 0 || last <= 0)
+      {
+         Print("⚠️ Skipping symbol ", sym, " — invalid bid/ask/last (", DoubleToString(bid,8), "/", DoubleToString(ask,8), "/", DoubleToString(last,8), ")");
+         continue;
+      }
+
+      // Initialize sizes
+      double bidSize = 0.0;
+      double askSize = 0.0;
+
+      // Try to fetch market depth (book) and classify volumes by price vs bid/ask
+      MqlBookInfo book[];
+      if(MarketBookGet(sym, book) && ArraySize(book) > 0)
+      {
+         for(int j = 0; j < ArraySize(book); j++)
+         {
+            double p = book[j].price;
+            double v = book[j].volume;
+
+            // If price is >= ask => ask side
+            if(p >= ask) askSize += v;
+            // If price is <= bid => bid side
+            else if(p <= bid) bidSize += v;
+            else
+            {
+               // price in-between -> assign to nearer side
+               double distToBid = MathAbs(p - bid);
+               double distToAsk = MathAbs(ask - p);
+               if(distToBid <= distToAsk) bidSize += v; else askSize += v;
+            }
+         }
+         PrintFormat("ℹ️ MarketBook for %s → bid=%.8f ask=%.8f bidSize=%.2f askSize=%.2f (book entries=%d)", sym, bid, ask, bidSize, askSize, ArraySize(book));
+      }
+      else
+      {
+         // MarketBook not available or empty
+         // Try SymbolInfoTick as fallback
+         MqlTick tick;
+         if(SymbolInfoTick(sym, tick))
+         {
+            // tick.volume is aggregated tick volume — not exact bid/ask sizes but better than zero
+            double tickVol = (double)tick.volume;
+            if(tickVol > 0.0)
+            {
+               // assign tick volume to both sides conservatively
+               if(bidSize <= 0) bidSize = tickVol;
+               if(askSize <= 0) askSize = tickVol;
+               PrintFormat("ℹ️ tick fallback for %s → tick.volume=%.2f", sym, tickVol);
+            }
+            else
+            {
+               Print("ℹ️ tick available but volume zero for ", sym);
+            }
+         }
+         else
+         {
+            Print("ℹ️ MarketBook and tick not available for ", sym);
+         }
+      }
+
+      // Final safety: ensure API-required positive numbers
+      // If a side is zero or negative, set to minimal positive 1.0
+      if(bidSize <= 0.0) bidSize = 1.0;
+      if(askSize <= 0.0) askSize = 1.0;
+
+      // Build JSON item for this symbol
+      string item = StringFormat(
+         "{\"symbolId\":%d,\"price\":%.8f,\"bid\":%.8f,\"ask\":%.8f,\"bidSize\":%.8f,\"askSize\":%.8f}",
+         symId, last, bid, ask, bidSize, askSize
+      );
+
+      // Add to aggregate payload
       if(!firstItem) json += ",";
       json += item;
       firstItem = false;
@@ -310,17 +488,25 @@ void SendLivePrices()
 
    if(sentCount == 0)
    {
-      Print("No valid live prices to send right now.");
+      Print("⚠️ No valid live prices to send right now. Check if market is open and symbols have tick data.");
       return;
    }
 
+   // Log URL and truncated payload for debugging
    string url = ApiBaseUrl + "/api/live-prices/bulk";
+   int maxShow = 1000;
+   string payloadLog = (StringLen(json) > maxShow) ? StringSubstr(json, 0, maxShow) + "...(truncated)" : json;
+   Print("📤 Calling API: ", url);
+   Print("📦 Payload (truncated): ", payloadLog);
+
+   // Send and report
    string response;
    bool ok = SendJSON(url, json, response);
+
    if(ok)
-      Print("✅ Sent ", sentCount, " live prices.");
+      Print("✅ Successfully sent ", sentCount, " live prices to API.");
    else
-      Print("❌ Failed to send live prices (sent ", sentCount, " items). Response: ", response);
+      Print("❌ Failed to send live prices (", sentCount, " items). API response: ", response);
 }
 
 //+------------------------------------------------------------------+
@@ -407,4 +593,4 @@ bool SendJSON(string url, string json, string &response)
    return false;
 }
 
-//+------------------------------------------------------------------+
+//+------------------------------------------------------------------+