feat: Add MT5 symbol synchronization and historical candle fetching

- Implement MQL5 Expert Advisor for symbol management
- Add symbol synchronization with database API
- Fetch and store last 1000 historical candles per symbol
- Add bulk candle upload functionality to API endpoints
- Include basic error handling for data operations

- Create Symbol Controller with full CRUD operations
- Add symbol search and filtering capabilities
- Implement pagination for symbol listing
- Add soft delete functionality for symbol management

MT5/Experts/MarketDataSender.mq5

@@ -1,313 +1,410 @@
 //+------------------------------------------------------------------+
-//|                                 MarketDataSender.mq5             |
-//|                        Copyright 2025, MetaQuotes Software Corp. |
-//|                                             https://www.mql5.com |
+//|                   MarketDataSender.mq5 (Final Fixed Version)     |
 //+------------------------------------------------------------------+
-#property copyright "Copyright 2025, MetaQuotes Software Corp."
-#property link      "https://www.mql5.com"
-#property version   "1.00"
-#property description "Sends historical candles and live prices to REST API"
-#property script_show_inputs
+#property strict
+#property description "Fetches all symbols' candles and live prices, sends to API."
 
 #include <Trade\SymbolInfo.mqh>
-#include <JAson.mqh> // Include JSON library
 
-//--- REST API Configuration
-input string ApiBaseUrl = "http://localhost:3000"; // Base URL of your API
-input string ApiKey = ""; // Optional API key if required
+input string ApiBaseUrl = "http://market-price.insightbull.io";
+input int HistoricalCandleCount = 1000;
+input ENUM_TIMEFRAMES HistoricalTimeframe = PERIOD_H1;
+input int LivePriceIntervalSeconds = 5;
 
-//--- Historical Data Configuration
-input int    HistoricalCandleCount = 1000; // Number of historical candles to send
-input ENUM_TIMEFRAMES HistoricalTimeframe = PERIOD_H1; // Timeframe for historical data
-
-//--- Global variables
-CJAson json;
-CSymbolInfo symbolInfo;
+// Globals
 string symbols[];
-datetime lastSentTime = 0;
+int symbolIds[];
+datetime lastSend = 0;
 
-//+------------------------------------------------------------------+
-//| Expert initialization function                                   |
 //+------------------------------------------------------------------+
 int OnInit()
 {
-   // Get all symbols
-   int count = SymbolsTotal(true);
-   ArrayResize(symbols, count);
-   
-   for(int i = 0; i < count; i++)
+   Print("Initializing MarketDataSender EA...");
+   if(!InitializeSymbols())
    {
-      symbols[i] = SymbolName(i, true);
+      Print("❌ Failed to initialize symbols.");
+      return(INIT_FAILED);
    }
-   
-   // Send initial historical data
-   SendHistoricalData();
-   
+
+   Print("✅ Symbols initialized: ", ArraySize(symbols));
+   SendAllHistoricalCandles();
    return(INIT_SUCCEEDED);
 }
 
-//+------------------------------------------------------------------+
-//| Expert tick function                                             |
 //+------------------------------------------------------------------+
 void OnTick()
 {
-   // Send live prices every second
-   if(TimeCurrent() - lastSentTime >= 1)
+   if(TimeCurrent() - lastSend >= LivePriceIntervalSeconds)
    {
       SendLivePrices();
-      lastSentTime = TimeCurrent();
+      lastSend = TimeCurrent();
    }
 }
 
 //+------------------------------------------------------------------+
-//| Send historical candle data                                      |
-//+------------------------------------------------------------------+
-void SendHistoricalData()
+bool InitializeSymbols()
 {
-   for(int s = 0; s < ArraySize(symbols); s++)
+   int total = SymbolsTotal(true);
+   if(total <= 0)
    {
-      string symbol = symbols[s];
-      
-      // Get historical candles
-      MqlRates rates[];
-      int copied = CopyRates(symbol, HistoricalTimeframe, 0, HistoricalCandleCount, rates);
-      
-      if(copied <= 0) continue;
-      
-      // Prepare JSON payload
-      CJAsonArray candlesArray;
-      
-      for(int i = 0; i < copied; i++)
-      {
-         CJAson candleObj;
-         candleObj.Add("symbolId", GetSymbolId(symbol)); // You need to implement GetSymbolId()
-         candleObj.Add("openTime", TimeToString(rates[i].time, TIME_DATE|TIME_MINUTES|TIME_SECONDS));
-         candleObj.Add("closeTime", TimeToString(rates[i].time + PeriodSeconds(HistoricalTimeframe), TIME_DATE|TIME_MINUTES|TIME_SECONDS));
-         candleObj.Add("open", rates[i].open);
-         candleObj.Add("high", rates[i].high);
-         candleObj.Add("low", rates[i].low);
-         candleObj.Add("close", rates[i].close);
-         candleObj.Add("volume", rates[i].tick_volume);
-         
-         candlesArray.Add(candleObj);
-      }
-      
-      // Create final payload
-      CJAson payload;
-      payload.Add("candles", candlesArray);
-      
-      // Send to API
-      string url = ApiBaseUrl + "/api/candles/bulk";
-      string result;
-      string headers = "Content-Type: application/json";
-      if(StringLen(ApiKey) > 0) headers += "\r\nAuthorization: Bearer " + ApiKey;
-      
-      // Implement retry logic with exponential backoff
-      int retries = 3;
-      int delayMs = 1000;
-      int res = -1;
-      
-      for(int attempt = 0; attempt < retries; attempt++) {
-         res = WebRequest("POST", url, headers, 5000, payload.GetJson(), result);
-         
-         if(res == 200) break;
-         
-         Print("Attempt ", attempt+1, " failed (", res, "). Retrying in ", delayMs, "ms");
-         Sleep(delayMs);
-         delayMs *= 2; // Exponential backoff
-      }
-      
-      if(res == 200) {
-         Print("Successfully sent historical data for ", symbol);
-      } else {
-         Print("Permanent failure sending historical data for ", symbol, ": ", res, " - ", result);
-         // TODO: Implement dead letter queue storage
-      }
+      Print("❌ No symbols found!");
+      return false;
    }
-}
 
-//+------------------------------------------------------------------+
-//| Send live prices                                                 |
-//+------------------------------------------------------------------+
-void SendLivePrices()
-{
-   CJAsonArray pricesArray;
-   
-   for(int s = 0; s < ArraySize(symbols); s++)
+   ArrayResize(symbols, total);
+   ArrayResize(symbolIds, total);
+
+   for(int i = 0; i < total; i++)
    {
-      string symbol = symbols[s];
-      
-      if(!symbolInfo.Name(symbol)) continue;
-      symbolInfo.RefreshRates();
-      
-      CJAson priceObj;
-      priceObj.Add("symbolId", GetSymbolId(symbol));
-      priceObj.Add("price", symbolInfo.Last());
-      priceObj.Add("bid", symbolInfo.Bid());
-      priceObj.Add("ask", symbolInfo.Ask());
-      priceObj.Add("bidSize", symbolInfo.VolumeBid());
-      priceObj.Add("askSize", symbolInfo.VolumeAsk());
-      
-      pricesArray.Add(priceObj);
+      symbols[i] = SymbolName(i, true);
+      symbolIds[i] = -1;
    }
-   
-   // Create final payload
-   CJAson payload;
-   payload.Add("prices", pricesArray);
-   
-   // Send to API
-   string url = ApiBaseUrl + "/api/live-prices/bulk";
-   string result;
-   string headers = "Content-Type: application/json";
-   if(StringLen(ApiKey) > 0) headers += "\r\nAuthorization: Bearer " + ApiKey;
-   
-      // Implement retry logic with exponential backoff
-      int retries = 3;
-      int delayMs = 1000;
-      int res = -1;
-      
-      for(int attempt = 0; attempt < retries; attempt++) {
-         res = WebRequest("POST", url, headers, 5000, payload.GetJson(), result);
-         
-         if(res == 200) break;
-         
-         Print("Attempt ", attempt+1, " failed (", res, "). Retrying in ", delayMs, "ms");
-         Sleep(delayMs);
-         delayMs *= 2; // Exponential backoff
-      }
-      
-      if(res == 200) {
-         Print("Successfully sent live prices");
-      } else {
-         Print("Permanent failure sending live prices: ", res, " - ", result);
-         // TODO: Implement dead letter queue storage
-      }
+
+   if(!SyncSymbolsWithDatabase())
+   {
+      Print("❌ Failed to sync symbols with database");
+      return false;
+   }
+
+   return true;
 }
 
 //+------------------------------------------------------------------+
-//| Initialize symbol map from database                              |
-//+------------------------------------------------------------------+
-bool InitializeSymbolMap()
+bool SyncSymbolsWithDatabase()
 {
-   // Fetch existing symbols from API
-   string url = ApiBaseUrl + "/api/symbols";
-   string result;
-   string headers = "Content-Type: application/json";
-   if(StringLen(ApiKey) > 0) headers += "\r\nAuthorization: Bearer " + ApiKey;
-   
-   int res = WebRequest("GET", url, headers, 5000, "", result);
-   
-   if(res != 200)
+   Print("Syncing symbols with database...");
+
+     string url = ApiBaseUrl + "/api/symbols";
+   string headers = "Content-Type: application/json\r\n";
+   string resultHeaders = "";
+   char result[];
+   char emptyData[];   // ✅ required placeholder for GET request
+
+   ResetLastError();
+
+   // ✅ Correct GET request signature: includes empty data[]
+   int res = WebRequest("GET", url, headers, 5000, emptyData, result, resultHeaders);
+
+   if(res == -1)
    {
-      Print("Failed to fetch symbols: ", res, " - ", result);
+      int err = GetLastError();
+      Print("❌ WebRequest connection error: ", err, " URL=", url);
       return false;
    }
-   
-   // Parse response
-   CJAson parser;
-   if(!parser.Parse(result))
+
+   if(res != 200)
    {
-      Print("Failed to parse symbols response");
+      Print("❌ Failed to fetch symbols from API: HTTP ", res, " Response: ", CharArrayToString(result));
       return false;
    }
-   
-   // Create lookup table
-   CJAsonArray symbolsArray = parser.GetArray("data");
-   int dbSymbolCount = symbolsArray.Size();
-   
-   for(int s = 0; s < ArraySize(symbols); s++)
+
+   string symbolsResponse = CharArrayToString(result);
+
+   if(StringFind(symbolsResponse, "\"data\"") < 0)
    {
-      string mt5Symbol = symbols[s];
-      bool found = false;
-      
-      // Search for matching symbol in database
-      for(int i = 0; i < dbSymbolCount; i++)
+      Print("⚠️ Unexpected response format from symbols API: ", symbolsResponse);
+   }
+
+   for(int i = 0; i < ArraySize(symbols); i++)
+   {
+      string symbolName = symbols[i];
+      int symbolId = FindSymbolId(symbolsResponse, symbolName);
+
+      if(symbolId > 0)
       {
-         CJAson dbSymbol = symbolsArray.GetObject(i);
-         string dbSymbolName = dbSymbol.GetString("symbol");
-         
-         if(dbSymbolName == mt5Symbol)
-         {
-            symbolIdMap[s] = (int)dbSymbol.GetInt("id");
-            found = true;
-            break;
-         }
+         symbolIds[i] = symbolId;
+         Print("✅ Found existing symbol: ", symbolName, " (ID: ", symbolId, ")");
       }
-      
-      // Create symbol if not found
-      if(!found)
+      else
       {
-         int newId = CreateSymbol(mt5Symbol);
-         if(newId > 0)
+         Sleep(300); // prevent overload (0.3 second delay)
+         symbolId = CreateSymbolInDatabase(symbolName);
+         if(symbolId > 0)
          {
-            symbolIdMap[s] = newId;
-            Print("Created new symbol: ", mt5Symbol, " (ID: ", newId, ")");
+            symbolIds[i] = symbolId;
+            Print("✅ Created new symbol: ", symbolName, " (ID: ", symbolId, ")");
          }
          else
          {
-            Print("Failed to create symbol: ", mt5Symbol);
-            return false;
+            Print("❌ Failed to create symbol: ", symbolName," (ID: ", symbolId, ")");
+            symbolIds[i] = -1;
          }
       }
    }
-   
+
    return true;
 }
 
 //+------------------------------------------------------------------+
-//| Create new symbol in database                                    |
+int FindSymbolId(string response, string symbolName)
+{
+   string searchPattern = StringFormat("\"symbol\":\"%s\"", symbolName);
+   int symbolPos = StringFind(response, searchPattern);
+   if(symbolPos < 0) return -1;
+
+   int idPos = StringFind(response, "\"id\":", symbolPos);
+   if(idPos < 0) return -1;
+
+   int startPos = idPos + 5;
+   int endPos = StringFind(response, ",", startPos);
+   if(endPos < 0) endPos = StringFind(response, "}", startPos);
+
+   string idStr = StringSubstr(response, startPos, endPos - startPos);
+   return (int)StringToInteger(idStr);
+}
+
 //+------------------------------------------------------------------+
-int CreateSymbol(string symbol)
+int CreateSymbolInDatabase(string symbolName)
 {
-   string url = ApiBaseUrl + "/api/symbols";
-   string result;
-   string headers = "Content-Type: application/json";
-   if(StringLen(ApiKey) > 0) headers += "\r\nAuthorization: Bearer " + ApiKey;
-   
-   // Extract exchange and instrument type from symbol name
-   string parts[];
-   StringSplit(symbol, '_', parts);
-   string exchange = (ArraySize(parts) > 1) ? parts[0] : "MT5";
-   string instrumentType = "forex"; // Default, can be improved
-   
-   // Prepare payload
-   CJAson payload;
-   payload.Add("symbol", symbol);
-   payload.Add("exchange", exchange);
-   payload.Add("instrumentType", instrumentType);
-   payload.Add("isActive", true);
-   
-   int res = WebRequest("POST", url, headers, 5000, payload.GetJson(), result);
-   
-   if(res != 201)
+   string baseAsset = "";
+   string quoteAsset = "";
+   string exchange = "MT5";
+   string instrumentType = "forex";
+
+   if(StringLen(symbolName) >= 6)
    {
-      Print("Error creating symbol: ", res, " - ", result);
-      return -1;
+      baseAsset = StringSubstr(symbolName, 0, 3);
+      quoteAsset = StringSubstr(symbolName, 3, 3);
    }
-   
-   // Parse response to get new ID
-   CJAson parser;
-   if(!parser.Parse(result))
+
+   string json = StringFormat(
+      "{\"symbol\":\"%s\",\"baseAsset\":\"%s\",\"quoteAsset\":\"%s\",\"exchange\":\"%s\",\"instrumentType\":\"%s\",\"isActive\":true}",
+      symbolName, baseAsset, quoteAsset, exchange, instrumentType
+   );
+
+   string url = ApiBaseUrl + "/api/symbols";
+   string headers = "Content-Type: application/json\r\n";
+   string resultHeaders = "";
+
+   char postData[];
+   StringToCharArray(json, postData, 0, CP_UTF8);
+
+   // ✅ FIX: Remove trailing null terminator from JSON
+   if(ArraySize(postData) > 0 && postData[ArraySize(postData) - 1] == 0)
+      ArrayResize(postData, ArraySize(postData) - 1);
+
+   char result[];
+
+   int res = WebRequest("POST", url, headers, 5000, postData, result, resultHeaders);
+
+   if(res != 201 && res != 200)
    {
-      Print("Failed to parse create symbol response");
+      Print("❌ Failed to create symbol: ", res, " Response: ", CharArrayToString(result));
       return -1;
    }
-   
-   return (int)parser.GetObject("data").GetInt("id");
+
+   string createResponse = CharArrayToString(result);
+   int idPos = StringFind(createResponse, "\"id\":");
+   if(idPos < 0) return -1;
+
+   int startPos = idPos + 5;
+   int endPos = StringFind(createResponse, ",", startPos);
+   if(endPos < 0) endPos = StringFind(createResponse, "}", startPos);
+
+   string idStr = StringSubstr(createResponse, startPos, endPos - startPos);
+   return (int)StringToInteger(idStr);
 }
 
 //+------------------------------------------------------------------+
-//| Get symbol ID from map                                           |
 //+------------------------------------------------------------------+
-int GetSymbolId(string symbol)
+//| Send all historical candles to the API (Fixed Version)           |
+//+------------------------------------------------------------------+
+void SendAllHistoricalCandles()
 {
+   Print("Starting historical upload for ", ArraySize(symbols), " symbols...");
+
    for(int i = 0; i < ArraySize(symbols); i++)
    {
-      if(symbols[i] == symbol)
+      string sym = symbols[i];
+
+      // --- Ensure data is ready ---
+      Sleep(300);
+      int tries = 0;
+      while(!SeriesInfoInteger(sym, HistoricalTimeframe, SERIES_SYNCHRONIZED) && tries < 10)
+      {
+         Print("⏳ Waiting for ", sym, " history to load...");
+         Sleep(500);
+         tries++;
+      }
+
+      // --- Now copy candles ---
+      MqlRates rates[];
+      ResetLastError();
+      int copied = CopyRates(sym, HistoricalTimeframe, 0, HistoricalCandleCount, rates);
+      int err = GetLastError();
+
+      if(copied <= 0)
+      {
+         Print("⚠️ Failed to copy candles for ", sym, " (copied=", copied, ", err=", err, ")");
+         continue;
+      }
+
+      Print("✅ Copied ", copied, " candles for ", sym);
+
+      // --- Print a few sample candles ---
+      int sampleCount = MathMin(5, copied); // show up to 5 examples
+      for(int j = 0; j < sampleCount; j++)
       {
-         return symbolIdMap[i];
+         MqlRates r = rates[j];
+         string openTime = TimeToString(r.time, TIME_DATE|TIME_SECONDS);
+         string closeTime = TimeToString(r.time + PeriodSeconds(HistoricalTimeframe), TIME_DATE|TIME_SECONDS);
+         PrintFormat(
+            "🕒 [%s] %s → %s | O=%.5f H=%.5f L=%.5f C=%.5f | Vol=%.2f",
+            sym, openTime, closeTime, r.open, r.high, r.low, r.close, r.tick_volume
+         );
       }
+
+      // --- Send candles in batches to API ---
+      int sentTotal = 0;
+      int batchSize = 200;
+      for(int start = 0; start < copied; start += batchSize)
+      {
+         int size = MathMin(batchSize, copied - start);
+         int symbolId = symbolIds[i];
+         if(symbolId <= 0) continue;
+
+         string json = BuildCandleJSONFromRates(symbolId, rates, start, size);
+         string url = ApiBaseUrl + "/api/candles/bulk";
+         string response;
+         bool ok = SendJSON(url, json, response);
+         if(!ok)
+         {
+            Print("❌ Failed to send candle batch for ", sym, " start=", start);
+            break;
+         }
+         sentTotal += size;
+         Print("📤 Sent candles for ", sym, ": ", sentTotal, "/", copied);
+      }
+   }
+   Print("✅ Historical upload finished.");
+}
+
+
+//+------------------------------------------------------------------+
+void SendLivePrices()
+{
+   bool firstItem = true;
+   string json = "{\"prices\":[";
+
+   int sentCount = 0;
+   for(int i = 0; i < ArraySize(symbols); i++)
+   {
+      string sym = symbols[i];
+      double bid = SymbolInfoDouble(sym, SYMBOL_BID);
+      double ask = SymbolInfoDouble(sym, SYMBOL_ASK);
+      double last = SymbolInfoDouble(sym, SYMBOL_LAST);
+      if(bid <= 0 || ask <= 0 || last <= 0) continue;
+
+      int symId = symbolIds[i];
+      if(symId <= 0) continue;
+
+      string item = StringFormat("{\"symbolId\":%d,\"price\":%.8f,\"bid\":%.8f,\"ask\":%.8f,\"bidSize\":%.8f,\"askSize\":%.8f}",
+                                 symId, last, bid, ask, 0.0, 0.0);
+      if(!firstItem) json += ",";
+      json += item;
+      firstItem = false;
+      sentCount++;
+   }
+
+   json += "]}";
+
+   if(sentCount == 0)
+   {
+      Print("No valid live prices to send right now.");
+      return;
+   }
+
+   string url = ApiBaseUrl + "/api/live-prices/bulk";
+   string response;
+   bool ok = SendJSON(url, json, response);
+   if(ok)
+      Print("✅ Sent ", sentCount, " live prices.");
+   else
+      Print("❌ Failed to send live prices (sent ", sentCount, " items). Response: ", response);
+}
+
+//+------------------------------------------------------------------+
+string ToISO8601(datetime t)
+{
+   MqlDateTime st;
+   TimeToStruct(t, st);
+   return StringFormat("%04d-%02d-%02dT%02d:%02d:%02d.000Z", st.year, st.mon, st.day, st.hour, st.min, st.sec);
+}
+
+string BuildCandleJSONFromRates(int symbolId, MqlRates &rates[], int startIndex, int count)
+{
+   string json = "{\"candles\":[";
+   bool first = true;
+   int ratesSize = ArraySize(rates);
+
+   for(int i = startIndex; i < startIndex + count && i < ratesSize; i++)
+   {
+      MqlRates r = rates[i];
+      if(r.time <= 0) continue;
+
+      datetime open_dt  = (datetime)r.time;
+      datetime close_dt = (datetime)(r.time + (datetime)PeriodSeconds(HistoricalTimeframe));
+
+      string openTime  = ToISO8601(open_dt);
+      string closeTime = ToISO8601(close_dt);
+
+      double volume       = (r.tick_volume > 0 ? r.tick_volume : 1);
+      double quoteVolume  = (r.real_volume > 0 ? r.real_volume : volume);
+
+      string one = StringFormat(
+         "{\"symbolId\":%d,\"openTime\":\"%s\",\"closeTime\":\"%s\",\"open\":%.5f,\"high\":%.5f,\"low\":%.5f,\"close\":%.5f,\"volume\":%.5f,\"tradesCount\":%d,\"quoteVolume\":%.5f}",
+         symbolId, openTime, closeTime,
+         r.open, r.high, r.low, r.close,
+         volume, (int)volume, quoteVolume
+      );
+
+      if(!first) json += ",";
+      json += one;
+      first = false;
    }
-   return -1;
+
+   json += "]}";
+   return json;
 }
+
+//+------------------------------------------------------------------+
+bool SendJSON(string url, string json, string &response)
+{
+   ResetLastError();
+
+   char postData[];
+   StringToCharArray(json, postData, 0, CP_UTF8);
+
+   // ✅ Remove trailing null terminator
+   if(ArraySize(postData) > 0 && postData[ArraySize(postData) - 1] == 0)
+      ArrayResize(postData, ArraySize(postData) - 1);
+
+   if(ArraySize(postData) <= 0)
+   {
+      Print("❌ Empty postData for URL: ", url);
+      return false;
+   }
+
+   char result[];
+   string headers = "Content-Type: application/json\r\n";
+   string resultHeaders = "";
+   int timeout = 15000;
+
+   int res = WebRequest("POST", url, headers, timeout, postData, result, resultHeaders);
+
+   if(res == -1)
+   {
+      int err = GetLastError();
+      Print("WebRequest error: ", err, " url=", url);
+      return false;
+   }
+
+   response = CharArrayToString(result);
+   if(res == 200 || res == 201)
+      return true;
+
+   Print("HTTP status ", res, " response: ", response);
+   return false;
+}
+
 //+------------------------------------------------------------------+

src/controllers/symbolController.js

@@ -8,15 +8,17 @@ class SymbolController {
       const {
         exchange,
         instrumentType,
-        isActive = true,
-        limit = 100,
+        isActive,
+        limit = 1000,
         offset = 0
       } = req.query;
 
       const where = {};
       if (exchange) where.exchange = exchange;
       if (instrumentType) where.instrumentType = instrumentType;
-      if (isActive !== undefined) where.isActive = isActive === 'true';
+      if (isActive !== undefined && isActive !== '') {
+        where.isActive = isActive === 'true' || isActive === true;
+      }
 
       const symbols = await Symbol.findAndCountAll({
         where,