Merge branch 'feature/cherry-picked-to-master' of MQL-Development/market-data-service into master

README.md

@@ -48,7 +48,7 @@ The service includes an MT5 Expert Advisor (EA) that automatically sends histori
 - Error recovery mechanisms  
 - Comprehensive test coverage  
 
-- **Multi-Asset Support**: Handles cryptocurrencies, stocks, forex, and commodities
+- **Multi-Asset Support**: Handles cryptocurrencies, stocks, forex, commodities, and indices
 - **Real-time Data**: Live price feeds with bid/ask spreads
 - **Historical Data**: OHLCV candle data with flexible timeframes
 - **RESTful API**: Well-structured endpoints for all operations

migrations/20251027075914-add-index-to-instrument-type.js

@@ -0,0 +1,16 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up (queryInterface, Sequelize) {
+    // Drop the existing CHECK constraint and add a new one with 'index'
+    await queryInterface.sequelize.query("ALTER TABLE symbols DROP CONSTRAINT symbols_instrument_type_check;");
+    await queryInterface.sequelize.query("ALTER TABLE symbols ADD CONSTRAINT symbols_instrument_type_check CHECK (instrument_type IN ('crypto', 'stock', 'forex', 'commodity', 'index'));");
+  },
+
+  async down (queryInterface, Sequelize) {
+    // Revert the CHECK constraint without 'index'
+    await queryInterface.sequelize.query("ALTER TABLE symbols DROP CONSTRAINT symbols_instrument_type_check;");
+    await queryInterface.sequelize.query("ALTER TABLE symbols ADD CONSTRAINT symbols_instrument_type_check CHECK (instrument_type IN ('crypto', 'stock', 'forex', 'commodity'));");
+  }
+};

src/controllers/candleController.js

@@ -88,9 +88,11 @@ class CandleController {
       });
 
       if (!candle) {
-        const error = new Error('No candle data found for this symbol');
-        error.statusCode = 404;
-        return next(error);
+        return res.json({
+          success: true,
+          data: null,
+          message: 'No candle data found for this symbol'
+        });
       }
 
       res.json({
@@ -165,7 +167,12 @@ class CandleController {
       }
 
       // Verify all symbols exist
-      const symbolIds = [...new Set(candles.map(c => c.symbolId))];
+      const processedCandles = candles.map(candle => ({
+        ...candle,
+        symbolId: parseInt(candle.symbolId)
+      }));
+
+      const symbolIds = [...new Set(processedCandles.map(c => c.symbolId))];
       const existingSymbols = await Symbol.findAll({
         where: { id: symbolIds },
         attributes: ['id']
@@ -180,7 +187,35 @@ class CandleController {
         return next(error);
       }
 
-      const createdCandles = await Candle1h.bulkCreate(candles);
+      // Check for existing candles to identify duplicates
+      const existingCandles = await Candle1h.findAll({
+        where: {
+          [Op.or]: processedCandles.map(candle => ({
+            symbolId: candle.symbolId,
+            openTime: candle.openTime
+          }))
+        },
+        attributes: ['symbolId', 'openTime']
+      });
+
+      // Create a set of existing keys for quick lookup
+      const existingKeys = new Set(
+        existingCandles.map(c => `${c.symbolId}-${c.openTime.toISOString()}`)
+      );
+
+      // Filter out duplicates
+      const newCandles = processedCandles.filter(candle =>
+        !existingKeys.has(`${candle.symbolId}-${candle.openTime.toISOString()}`)
+      );
+
+      const duplicateCount = processedCandles.length - newCandles.length;
+
+      // Log duplicates if any
+      if (duplicateCount > 0) {
+        console.log(`Bulk create candles: ${duplicateCount} duplicates skipped`);
+      }
+
+      const createdCandles = await Candle1h.bulkCreate(newCandles);
 
       // Emit WebSocket events for real-time updates
       const io = req.app.get('io');
@@ -226,7 +261,7 @@ class CandleController {
       res.status(201).json({
         success: true,
         data: createdCandles,
-        message: `${createdCandles.length} candles created successfully`
+        message: `${createdCandles.length} candles created successfully${duplicateCount > 0 ? `, ${duplicateCount} duplicates skipped` : ''}`
       });
     } catch (error) {
       next(error);

src/controllers/symbolController.js

@@ -67,13 +67,24 @@ class SymbolController {
   // Create new symbol
   async createSymbol(req, res, next) {
     try {
-      const symbol = await Symbol.create(req.body);
-
-      res.status(201).json({
-        success: true,
-        data: symbol,
-        message: 'Symbol created successfully'
+      const [symbol, created] = await Symbol.findOrCreate({
+        where: { symbol: req.body.symbol },
+        defaults: req.body
       });
+
+      if (created) {
+        res.status(201).json({
+          success: true,
+          data: symbol,
+          message: 'Symbol created successfully'
+        });
+      } else {
+        res.status(200).json({
+          success: true,
+          data: symbol,
+          message: 'Symbol already exists'
+        });
+      }
     } catch (error) {
       next(error);
     }

src/middleware/validation.js

@@ -6,7 +6,7 @@ const symbolSchema = Joi.object({
   baseAsset: Joi.string().max(50),
   quoteAsset: Joi.string().max(50),
   exchange: Joi.string().max(50),
-  instrumentType: Joi.string().valid('crypto', 'stock', 'forex', 'commodity').required(),
+  instrumentType: Joi.string().valid('crypto', 'stock', 'forex', 'commodity', 'index').required(),
   isActive: Joi.boolean().default(true)
 });
 

src/models/Symbol.js

@@ -24,7 +24,7 @@ const Symbol = sequelize.define('Symbol', {
     type: DataTypes.STRING(50)
   },
   instrumentType: {
-    type: DataTypes.ENUM('crypto', 'stock', 'forex', 'commodity'),
+    type: DataTypes.ENUM('crypto', 'stock', 'forex', 'commodity', 'index'),
     field: 'instrument_type',
     allowNull: false
   },

src/routes/symbols.js

@@ -6,7 +6,7 @@ const { validate, validateQuery, validateParams, symbolSchema, symbolIdSchema }
 // GET /api/symbols - Get all symbols with optional filtering
 router.get('/', validateQuery(require('joi').object({
   exchange: require('joi').string().max(50),
-  instrumentType: require('joi').string().valid('crypto', 'stock', 'forex', 'commodity'),
+  instrumentType: require('joi').string().valid('crypto', 'stock', 'forex', 'commodity', 'index'),
   isActive: require('joi').string().valid('true', 'false'),
   limit: require('joi').number().integer().min(1).max(1000).default(100),
   offset: require('joi').number().integer().min(0).default(0)