feat: enhance MT5 expert advisor with timer-based sync and improved symbol parsing

- Add timer-based periodic candle synchronization every 60 seconds
- Improve symbol parsing to handle suffixes and better detect base/quote assets
- Enhance error handling and logging throughout the code
- Add cleanup functionality to remove old candles (keep last 1000)
- Improve history loading checks with timeout safety
- Better fallback for instrument type detection (crypto, metal, index, forex)

MT5/Experts/MarketDataSender.mq5

@@ -15,7 +15,7 @@ input int LivePriceIntervalSeconds = 5;
 string symbols[];
 int symbolIds[];
 datetime lastSend = 0;
-
+datetime lastCandleSync = 0;
 //+------------------------------------------------------------------+
 int OnInit()
 {
@@ -28,6 +28,8 @@ int OnInit()
 
    Print("✅ Symbols initialized: ", ArraySize(symbols));
    SendAllHistoricalCandles();
+   EventSetTimer(60);  // ⏱️ Trigger OnTimer() every 30 minutes
+   Print("✅ Timer set: SendAllHistoricalCandles() will run every 30 minutes.");
    return(INIT_SUCCEEDED);
 }
 
@@ -202,11 +204,53 @@ int CreateSymbolInDatabase(string symbolName)
    string exchange = "MT5";
    string instrumentType = "forex";
 
+   // --- Clean suffixes like ".pro", ".m", ".r", "_i" ---
+   int dotPos = StringFind(symbolName, ".");
+   if(dotPos > 0)
+      symbolName = StringSubstr(symbolName, 0, dotPos);
+
+   // --- Try basic split for 6-char pairs like EURUSD, GBPJPY, BTCUSD ---
    if(StringLen(symbolName) >= 6)
    {
       baseAsset = StringSubstr(symbolName, 0, 3);
       quoteAsset = StringSubstr(symbolName, 3, 3);
    }
+   else
+   {
+      // --- Try alternate detection ---
+      if(StringFind(symbolName, "USD") >= 0)
+      {
+         int pos = StringFind(symbolName, "USD");
+         baseAsset = StringSubstr(symbolName, 0, pos);
+         quoteAsset = "USD";
+      }
+      else if(StringFind(symbolName, "EUR") >= 0)
+      {
+         int pos = StringFind(symbolName, "EUR");
+         baseAsset = StringSubstr(symbolName, 0, pos);
+         quoteAsset = "EUR";
+      }
+      else
+      {
+         // Fallback safe defaults
+         baseAsset = symbolName;
+         quoteAsset = "USD";
+      }
+   }
+
+   // --- Final safety: ensure no empty fields ---
+   if(StringLen(baseAsset) == 0) baseAsset = "UNKNOWN";
+   if(StringLen(quoteAsset) == 0) quoteAsset = "USD";
+
+   // --- Decide instrument type ---
+   if(StringFind(symbolName, "BTC") == 0 || StringFind(symbolName, "ETH") == 0)
+      instrumentType = "crypto";
+   else if(StringFind(symbolName, "XAU") == 0 || StringFind(symbolName, "XAG") == 0)
+      instrumentType = "metal";
+   else if(StringFind(symbolName, "US30") == 0 || StringFind(symbolName, "NAS") == 0)
+      instrumentType = "index";
+   else
+      instrumentType = "forex";
 
    string json = StringFormat(
       "{\"symbol\":\"%s\",\"baseAsset\":\"%s\",\"quoteAsset\":\"%s\",\"exchange\":\"%s\",\"instrumentType\":\"%s\",\"isActive\":true}",
@@ -219,18 +263,15 @@ int CreateSymbolInDatabase(string symbolName)
 
    char postData[];
    StringToCharArray(json, postData, 0, CP_UTF8);
-
-   // ✅ FIX: Remove trailing null terminator from JSON
    if(ArraySize(postData) > 0 && postData[ArraySize(postData) - 1] == 0)
       ArrayResize(postData, ArraySize(postData) - 1);
 
    char result[];
-
    int res = WebRequest("POST", url, headers, 5000, postData, result, resultHeaders);
 
    if(res != 201 && res != 200)
    {
-      Print("❌ Failed to create symbol: ", res, " Response: ", CharArrayToString(result));
+      PrintFormat("❌ Failed to create symbol %s | HTTP %d | Response: %s", symbolName, res, CharArrayToString(result));
       return -1;
    }
 
@@ -245,6 +286,7 @@ int CreateSymbolInDatabase(string symbolName)
    string idStr = StringSubstr(createResponse, startPos, endPos - startPos);
    return (int)StringToInteger(idStr);
 }
+
 //+------------------------------------------------------------------+
 //| Fetch latest stored candle openTime from API                     |
 //+------------------------------------------------------------------+
@@ -298,6 +340,9 @@ datetime GetLatestCandleTime(int symbolId)
 //+------------------------------------------------------------------+
 //| Send all historical candles to the API (Fixed Version)           |
 //+------------------------------------------------------------------+
+//+------------------------------------------------------------------+
+//| Send all historical candles to the API (Fixed + Timeout Safe)    |
+//+------------------------------------------------------------------+
 void SendAllHistoricalCandles()
 {
    Print("Starting historical upload for ", ArraySize(symbols), " symbols...");
@@ -311,26 +356,42 @@ void SendAllHistoricalCandles()
       // --- Get last stored candle time ---
       datetime latestApiTime = GetLatestCandleTime(symbolId);
 
-      // --- Ensure data is ready ---
+      // --- Ensure history data is available ---
       Sleep(300);
       int tries = 0;
-      while(!SeriesInfoInteger(sym, HistoricalTimeframe, SERIES_SYNCHRONIZED) && tries < 10)
+      bool historyReady = false;
+
+      while(tries < 10)
       {
-         Print("⏳ Waiting for ", sym, " history to load...");
+         if(SeriesInfoInteger(sym, HistoricalTimeframe, SERIES_SYNCHRONIZED))
+         {
+            historyReady = true;
+            break;
+         }
+         PrintFormat("⏳ Waiting for %s history to load... (try %d/10)", sym, tries + 1);
          Sleep(500);
          tries++;
       }
 
+      if(!historyReady)
+      {
+         PrintFormat("⚠️ Skipping %s — history not loaded after 10 tries (~5s timeout).", sym);
+         continue;
+      }
+
+      // --- Copy rates ---
       MqlRates rates[];
       ResetLastError();
       int copied = CopyRates(sym, HistoricalTimeframe, 0, HistoricalCandleCount, rates);
+
       if(copied <= 0)
       {
-         Print("⚠️ Failed to copy candles for ", sym);
+         int err = GetLastError();
+         PrintFormat("⚠️ Failed to copy candles for %s (error %d)", sym, err);
          continue;
       }
 
-      Print("✅ Copied ", copied, " candles for ", sym);
+      PrintFormat("✅ Copied %d candles for %s", copied, sym);
 
       // --- Filter new candles ---
       int startIndex = 0;
@@ -346,34 +407,39 @@ void SendAllHistoricalCandles()
       int newCount = copied - startIndex;
       if(newCount <= 0)
       {
-         Print("ℹ️ No new candles to send for ", sym);
+         PrintFormat("ℹ️ No new candles to send for %s", sym);
          continue;
       }
 
-      Print("🆕 Sending ", newCount, " new candles for ", sym, " after ", TimeToString(latestApiTime, TIME_DATE|TIME_SECONDS));
+      PrintFormat("🆕 Sending %d new candles for %s after %s", newCount, sym, TimeToString(latestApiTime, TIME_DATE|TIME_SECONDS));
 
       // --- Send new candles in batches ---
       int batchSize = 200;
       int sentTotal = 0;
+
       for(int start = startIndex; start < copied; start += batchSize)
       {
          int size = MathMin(batchSize, copied - start);
          string json = BuildCandleJSONFromRates(symbolId, rates, start, size);
          string url = ApiBaseUrl + "/api/candles/bulk";
          string response;
+
          bool ok = SendJSON(url, json, response);
          if(!ok)
          {
-            Print("❌ Failed to send candle batch for ", sym, " start=", start);
+            PrintFormat("❌ Failed to send candle batch for %s (start=%d)", sym, start);
             break;
          }
+
          sentTotal += size;
-         Print("📤 Sent ", sentTotal, "/", newCount, " new candles for ", sym);
+         PrintFormat("📤 Sent %d/%d new candles for %s", sentTotal, newCount, sym);
       }
    }
+
    Print("✅ Incremental candle upload finished.");
 }
 
+
 //+------------------------------------------------------------------+
 //| Send live prices of all active symbols                           |
 //+------------------------------------------------------------------+
@@ -593,4 +659,58 @@ bool SendJSON(string url, string json, string &response)
    return false;
 }
 
-//+------------------------------------------------------------------+
+//+------------------------------------------------------------------+
+//+------------------------------------------------------------------+
+//| Cleanup old candles (keep only last 1000)                        |
+//+------------------------------------------------------------------+
+void CleanupOldCandles(int symbolId)
+{
+   string url = ApiBaseUrl + "/api/candles/cleanup/" + IntegerToString(symbolId) + "?keep=1000";
+   string headers = "Content-Type: application/json\r\n";
+   string resultHeaders = "";
+   char result[];
+   char emptyData[];
+
+   ResetLastError();
+   int res = WebRequest("DELETE", url, headers, 10000, emptyData, result, resultHeaders);
+
+   string response = CharArrayToString(result);
+   if(res == 200 || res == 204)
+      Print("🧹 Cleanup successful for symbolId=", symbolId, " → kept last 1000 candles.");
+   else
+      Print("⚠️ Cleanup failed for symbolId=", symbolId, " HTTP=", res, " Response=", response);
+}
+
+//+------------------------------------------------------------------+
+//| Timer event: runs every 60 seconds                               |
+//+------------------------------------------------------------------+
+void OnTimer()
+{
+   datetime now = TimeCurrent();
+
+   // ✅ Run full candle sync only once every minute
+   if(now - lastCandleSync >= 600)
+   {
+      Print("⏰ Running scheduled candle sync and cleanup...");
+      SendAllHistoricalCandles();
+
+      // ✅ After uploading candles, clean up old ones
+      for(int i = 0; i < ArraySize(symbols); i++)
+      {
+         int symId = symbolIds[i];
+         if(symId <= 0) continue;
+         CleanupOldCandles(symId);
+         Sleep(500); // small delay to avoid API overload
+      }
+
+      lastCandleSync = now;
+      Print("✅ Candle sync + cleanup cycle completed.");
+   }
+}
+
+
+
+void OnDeinit(const int reason)
+{
+   EventKillTimer();
+}