diff --git a/QuantConnect.ThetaData.Tests/TestHelpers.cs b/QuantConnect.ThetaData.Tests/TestHelpers.cs index 0b7b459..89cd8fc 100644 --- a/QuantConnect.ThetaData.Tests/TestHelpers.cs +++ b/QuantConnect.ThetaData.Tests/TestHelpers.cs @@ -174,11 +174,11 @@ public static void AssertTradeBars(IEnumerable tradeBars, Symbol symbo } public static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateTime, DateTime endDateTime, - SecurityExchangeHours exchangeHours = null, DateTimeZone dataTimeZone = null) + SecurityExchangeHours exchangeHours = null, DateTimeZone dataTimeZone = null, bool includeExtendedMarketHours = true) { if (exchangeHours == null) { - exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork); + exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); } if (dataTimeZone == null) @@ -188,15 +188,15 @@ public static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution reso var dataType = LeanData.GetDataType(resolution, tickType); return new HistoryRequest( - startDateTime, - endDateTime, + startDateTime.ConvertToUtc(exchangeHours.TimeZone), + endDateTime.ConvertToUtc(exchangeHours.TimeZone), dataType, symbol, resolution, exchangeHours, dataTimeZone, - null, - true, + resolution, + includeExtendedMarketHours, false, DataNormalizationMode.Raw, tickType diff --git a/QuantConnect.ThetaData.Tests/ThetaDataAdditionalTests.cs b/QuantConnect.ThetaData.Tests/ThetaDataAdditionalTests.cs new file mode 100644 index 0000000..af95244 --- /dev/null +++ b/QuantConnect.ThetaData.Tests/ThetaDataAdditionalTests.cs @@ -0,0 +1,87 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using NUnit.Framework; +using System.Collections.Generic; + +namespace QuantConnect.Lean.DataSource.ThetaData.Tests; + +[TestFixture] +public class ThetaDataAdditionalTests +{ + [Test] + public void GenerateDateRangesWithNinetyDaysInterval() + { + var intervalDays = 90; + var startDate = new DateTime(2020, 07, 18); + var endDate = new DateTime(2021, 01, 14); + + var expectedRanges = new List<(DateTime startDate, DateTime endDate)> + { + (new DateTime(2020, 07, 18), new DateTime(2020, 10, 16)), + (new DateTime(2020, 10, 17), new DateTime(2021, 01, 14)), + }; + + var actualRanges = new List<(DateTime startDate, DateTime endDate)>(ThetaDataExtensions.GenerateDateRangesWithInterval(startDate, endDate, intervalDays)); + + Assert.AreEqual(expectedRanges.Count, actualRanges.Count, "The number of ranges should match."); + + for (int i = 0; i < expectedRanges.Count; i++) + { + Assert.AreEqual(expectedRanges[i].startDate, actualRanges[i].startDate, $"Start date mismatch at index {i}"); + Assert.AreEqual(expectedRanges[i].endDate, actualRanges[i].endDate, $"End date mismatch at index {i}"); + } + } + + [Test] + public void GenerateDateRangesWithOneDayInterval() + { + var intervalDays = 1; + + var startDate = new DateTime(2024, 07, 26); + var endDate = new DateTime(2024, 07, 30); + + var expectedRanges = new List<(DateTime startDate, DateTime endDate)> + { + (new DateTime(2024, 07, 26), new DateTime(2024, 07, 27)), + (new DateTime(2024, 07, 28), new DateTime(2024, 07, 29)), + (new DateTime(2024, 07, 30), new DateTime(2024, 07, 30)) + }; + + var actualRanges = new List<(DateTime startDate, DateTime endDate)>(ThetaDataExtensions.GenerateDateRangesWithInterval(startDate, endDate, intervalDays)); + + Assert.AreEqual(expectedRanges.Count, actualRanges.Count, "The number of ranges should match."); + + for (int i = 0; i < expectedRanges.Count; i++) + { + Assert.AreEqual(expectedRanges[i].startDate, actualRanges[i].startDate, $"Start date mismatch at index {i}"); + Assert.AreEqual(expectedRanges[i].endDate, actualRanges[i].endDate, $"End date mismatch at index {i}"); + } + } + + [Test] + public void GenerateDateRangesWithInterval_ShouldHandleSameStartEndDate() + { + DateTime startDate = new DateTime(2025, 2, 1); + DateTime endDate = new DateTime(2025, 2, 1); + + var ranges = new List<(DateTime startDate, DateTime endDate)>( + ThetaDataExtensions.GenerateDateRangesWithInterval(startDate, endDate, 1) + ); + + Assert.AreEqual(1, ranges.Count, "There should be no date ranges generated."); + } +} diff --git a/QuantConnect.ThetaData.Tests/ThetaDataHistoryProviderTests..cs b/QuantConnect.ThetaData.Tests/ThetaDataHistoryProviderTests..cs index 7d9a027..bbec97b 100644 --- a/QuantConnect.ThetaData.Tests/ThetaDataHistoryProviderTests..cs +++ b/QuantConnect.ThetaData.Tests/ThetaDataHistoryProviderTests..cs @@ -16,6 +16,11 @@ using System; using System.Linq; using NUnit.Framework; +using QuantConnect.Data; +using System.Diagnostics; +using QuantConnect.Logging; +using Microsoft.CodeAnalysis; +using System.Collections.Generic; namespace QuantConnect.Lean.DataSource.ThetaData.Tests { @@ -114,5 +119,109 @@ public void GetHistoryTickTradeValidateOnDistinctData(string ticker, Resolution Assert.That(history.Count, Is.EqualTo(distinctHistory.Count)); } + + [TestCase("SPY", SecurityType.Equity, Resolution.Hour, "1998/01/02", "2025/02/16", new[] { TickType.Quote, TickType.Trade })] + [TestCase("SPY", SecurityType.Equity, Resolution.Daily, "1998/01/02", "2025/02/16", new[] { TickType.Quote, TickType.Trade })] + [TestCase("SPY", SecurityType.Equity, Resolution.Minute, "2025/01/02", "2025/02/16", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "1998/01/02", "2025/02/16", new[] { TickType.Quote, TickType.Trade })] + public void GetHistoryRequestWithLongRange(string ticker, SecurityType securityType, Resolution resolution, DateTime startDate, DateTime endDate, TickType[] tickTypes) + { + var symbol = TestHelpers.CreateSymbol(ticker, securityType); + + var historyRequests = new List(); + foreach (var tickType in tickTypes) + { + historyRequests.Add(TestHelpers.CreateHistoryRequest(symbol, resolution, tickType, startDate, endDate)); + } + + foreach (var historyRequest in historyRequests) + { + var stopwatch = Stopwatch.StartNew(); + var history = _thetaDataProvider.GetHistory(historyRequest).ToList(); + stopwatch.Stop(); + + Assert.IsNotEmpty(history); + + var firstDate = history.First().Time; + var lastDate = history.Last().Time; + + Log.Trace($"[{nameof(ThetaDataHistoryProviderTests)}] Execution completed in {stopwatch.Elapsed.TotalMinutes:F2} min | " + + $"Symbol: {historyRequest.Symbol}, Resolution: {resolution}, TickType: {historyRequest.TickType}, Count: {history.Count}, " + + $"First Date: {firstDate:yyyy-MM-dd HH:mm:ss}, Last Date: {lastDate:yyyy-MM-dd HH:mm:ss}"); + + // Ensure historical data is returned in chronological order + for (var i = 1; i < history.Count; i++) + { + if (history[i].Time < history[i - 1].Time) + Assert.Fail("Historical data is not in chronological order."); + } + } + } + + [TestCase("AAPL", SecurityType.Equity, Resolution.Minute, "2025/02/19", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Minute, "2025/02/18", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Minute, "2025/02/15", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Minute, "2025/02/10", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "2025/02/19", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "2025/02/18", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "2025/02/10", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "2025/02/01", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Hour, "2025/01/01", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Daily, "2025/02/19", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Daily, "2025/02/18", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Daily, "2025/02/15", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Daily, "2025/02/10", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + [TestCase("AAPL", SecurityType.Equity, Resolution.Daily, "2025/01/01", "2025/02/20", new[] { TickType.Quote, TickType.Trade })] + public void GetHistoryRequestWithCalculateAmountReturnsData(string ticker, SecurityType securityType, Resolution resolution, DateTime startDate, DateTime endDate, TickType[] tickTypes) + { + var symbol = TestHelpers.CreateSymbol(ticker, securityType); + + var historyRequests = new List(); + foreach (var tickType in tickTypes) + { + historyRequests.Add(TestHelpers.CreateHistoryRequest(symbol, resolution, tickType, startDate, endDate, includeExtendedMarketHours: false)); + } + + foreach (var historyRequest in historyRequests) + { + var history = _thetaDataProvider.GetHistory(historyRequest).ToList(); + //Log.Trace(string.Join("\n", history.Select(x => new { Time = x.Time, EndTime = x.EndTime, Data = x }))); + + int expectedAmount = CalculateExpectedHistoryAmount(historyRequest); + + Assert.AreEqual(expectedAmount, history.Count, "History data count does not match expected amount."); + } + } + + private int CalculateExpectedHistoryAmount(HistoryRequest request) + { + var endTime = request.EndTimeUtc.ConvertFromUtc(request.DataTimeZone); + var currentDate = request.StartTimeUtc.ConvertFromUtc(request.DataTimeZone); + int totalDataPoints = 0; + + while (currentDate < endTime) + { + if (request.ExchangeHours.IsDateOpen(currentDate, request.IncludeExtendedMarketHours)) + { + int dataPointsPerDay = GetDataPointsPerDay(request.Resolution); + totalDataPoints += dataPointsPerDay; + } + + currentDate = currentDate.AddDays(1); + } + + return totalDataPoints; + } + + private int GetDataPointsPerDay(Resolution resolution) + { + return resolution switch + { + Resolution.Minute => 390, // 720 minutes from 9:30 AM to 4:00 PM (Trading Hours) + Resolution.Hour => 7, + Resolution.Daily => 1, // 1 bar per day + _ => throw new ArgumentOutOfRangeException(nameof(resolution), "Unsupported resolution") + }; + } } } diff --git a/QuantConnect.ThetaData/Models/Interfaces/ISubscriptionPlan.cs b/QuantConnect.ThetaData/Models/Interfaces/ISubscriptionPlan.cs index de63398..c499167 100644 --- a/QuantConnect.ThetaData/Models/Interfaces/ISubscriptionPlan.cs +++ b/QuantConnect.ThetaData/Models/Interfaces/ISubscriptionPlan.cs @@ -20,8 +20,16 @@ namespace QuantConnect.Lean.DataSource.ThetaData.Models.Interfaces; /// /// The ISubscriptionPlan interface defines the base structure for different price plans offered by ThetaData for users. /// For detailed documentation on ThetaData subscription plans, refer to the following links: -/// -/// +/// +/// +/// https://www.thetadata.net/subscribe +/// Institutional Data Retail Pricing +/// +/// +/// https://http-docs.thetadata.us/Articles/Getting-Started/Subscriptions.html#options-data +/// Initial Access Date Based on Subscription Plan +/// +/// /// public interface ISubscriptionPlan { diff --git a/QuantConnect.ThetaData/Models/Rest/RequestParameters.cs b/QuantConnect.ThetaData/Models/Rest/RequestParameters.cs new file mode 100644 index 0000000..8ee4656 --- /dev/null +++ b/QuantConnect.ThetaData/Models/Rest/RequestParameters.cs @@ -0,0 +1,40 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.ThetaData.Models.Rest; + +/// +/// Contains constant values for various request parameters used in API queries. +/// +public static class RequestParameters +{ + /// + /// Represents the time interval in milliseconds since midnight Eastern Time (ET). + /// Example values: + /// - 09:30:00 ET = 34_200_000 ms + /// - 16:00:00 ET = 57_600_000 ms + /// + public const string IntervalInMilliseconds = "ivl"; + + /// + /// Represents the start date for a query or request. + /// + public const string StartDate = "start_date"; + + /// + /// Represents the end date for a query or request. + /// + public const string EndDate = "end_date"; +} diff --git a/QuantConnect.ThetaData/Models/Wrappers/StopwatchWrapper.cs b/QuantConnect.ThetaData/Models/Wrappers/StopwatchWrapper.cs new file mode 100644 index 0000000..0c93f65 --- /dev/null +++ b/QuantConnect.ThetaData/Models/Wrappers/StopwatchWrapper.cs @@ -0,0 +1,67 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Diagnostics; +using QuantConnect.Logging; +using QuantConnect.Lean.DataSource.ThetaData.Models.Common; + +namespace QuantConnect.Lean.DataSource.ThetaData.Models.Wrappers; + +/// +/// A utility class that conditionally starts a stopwatch for measuring execution time +/// when debugging is enabled. Implements to ensure +/// automatic logging upon completion. +/// +public class StopwatchWrapper : IDisposable +{ + private readonly Stopwatch? _stopwatch; + private readonly string _message; + + /// + /// Initializes a new instance of the class + /// and starts a stopwatch to measure execution time. + /// + /// A descriptive message to include in the log output. + private StopwatchWrapper(string message) + { + _message = message; + _stopwatch = Stopwatch.StartNew(); + } + + /// + /// Starts a stopwatch if debugging is enabled and returns an appropriate disposable instance. + /// + /// A descriptive message to include in the log output. + /// + /// A instance if debugging is enabled, + /// otherwise a no-op instance. + /// + public static IDisposable? StartIfEnabled(string message) + { + return Log.DebuggingEnabled ? new StopwatchWrapper(message) : null; + } + + /// + /// Stops the stopwatch and logs the elapsed time if debugging is enabled. + /// + public void Dispose() + { + if (_stopwatch != null) + { + _stopwatch.Stop(); + Log.Debug($"{_message} completed in {_stopwatch.ElapsedMilliseconds} ms"); + } + } +} diff --git a/QuantConnect.ThetaData/ThetaDataExtensions.cs b/QuantConnect.ThetaData/ThetaDataExtensions.cs index c5fe9f3..5963bee 100644 --- a/QuantConnect.ThetaData/ThetaDataExtensions.cs +++ b/QuantConnect.ThetaData/ThetaDataExtensions.cs @@ -48,7 +48,7 @@ public static class ThetaDataExtensions /// The number of days each date range should cover. /// An enumerable sequence of tuples where each tuple contains a start date (inclusive) and end date (exclusive) covering the specified number of days. /// - /// + /// /// /// Best Practices for request interval size and duration: /// Making large requests is often times inefficient and can lead to out of memory errors. @@ -60,13 +60,17 @@ public static class ThetaDataExtensions { DateTime currentDate = startDate; - while (currentDate < endDate) + while (currentDate <= endDate) { DateTime nextDate = currentDate.AddDays(intervalDays); + + if (nextDate > endDate) + nextDate = endDate; + yield return (currentDate, nextDate); // Move to the next interval - currentDate = nextDate; + currentDate = nextDate.AddDays(1); } } diff --git a/QuantConnect.ThetaData/ThetaDataHistoryProvider.cs b/QuantConnect.ThetaData/ThetaDataHistoryProvider.cs index 5e023ec..a3af7be 100644 --- a/QuantConnect.ThetaData/ThetaDataHistoryProvider.cs +++ b/QuantConnect.ThetaData/ThetaDataHistoryProvider.cs @@ -109,12 +109,20 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } + var startDateTimeUtc = historyRequest.StartTimeUtc; if (_userSubscriptionPlan.FirstAccessDate.Date > historyRequest.StartTimeUtc.Date) { if (!_invalidStartDateInCurrentSubscriptionWarningFired) { _invalidStartDateInCurrentSubscriptionWarningFired = true; - Log.Trace($"{nameof(ThetaDataProvider)}.{nameof(GetHistory)}: The requested start time ({historyRequest.StartTimeUtc.Date}) exceeds the maximum available date ({_userSubscriptionPlan.FirstAccessDate.Date}) allowed by the user's subscription."); + Log.Trace($"{nameof(ThetaDataProvider)}.{nameof(GetHistory)}: The requested start time ({historyRequest.StartTimeUtc.Date}) exceeds the maximum available date ({_userSubscriptionPlan.FirstAccessDate.Date}) allowed by the user's subscription. Using the new adjusted start date: {_userSubscriptionPlan.FirstAccessDate.Date}."); + } + // Ensures efficient data retrieval by blocking requests outside the user's subscription period, which reduces processing overhead and avoids unnecessary data requests. + startDateTimeUtc = _userSubscriptionPlan.FirstAccessDate.Date; + + if (startDateTimeUtc >= historyRequest.EndTimeUtc) + { + return null; } } @@ -128,7 +136,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } - if (historyRequest.StartTimeUtc >= historyRequest.EndTimeUtc) + if (startDateTimeUtc >= historyRequest.EndTimeUtc) { if (!_invalidStartTimeWarningFired) { @@ -151,8 +159,12 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) var restRequest = new RestRequest(Method.GET); restRequest = GetSymbolHistoryQueryParametersBySymbol(restRequest, historyRequest.Symbol); - restRequest.AddQueryParameter("start_date", historyRequest.StartTimeUtc.ConvertFromUtc(TimeZoneThetaData).ConvertToThetaDataDateFormat()); - restRequest.AddQueryParameter("end_date", historyRequest.EndTimeUtc.ConvertFromUtc(TimeZoneThetaData).ConvertToThetaDataDateFormat()); + + var startDateTimeLocal = startDateTimeUtc.ConvertFromUtc(TimeZoneThetaData); + var endDateTimeLocal = historyRequest.EndTimeUtc.ConvertFromUtc(TimeZoneThetaData); + + restRequest.AddQueryParameter(RequestParameters.StartDate, startDateTimeLocal.ConvertToThetaDataDateFormat()); + restRequest.AddQueryParameter(RequestParameters.EndDate, endDateTimeLocal.ConvertToThetaDataDateFormat()); restRequest.AddOrUpdateParameter("start_time", "0", ParameterType.QueryString); restRequest.Resource = GetResourceUrlHistoryData(historyRequest.Symbol.SecurityType, historyRequest.TickType, historyRequest.Resolution); @@ -168,24 +180,50 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return GetHistoricalOpenInterestData(restRequest, historyRequest.Symbol, symbolExchangeTimeZone); } + var history = default(IEnumerable); switch (historyRequest.Resolution) { case Resolution.Tick: - return GetTickHistoryData(restRequest, historyRequest.Symbol, Resolution.Tick, historyRequest.TickType, historyRequest.StartTimeUtc, historyRequest.EndTimeUtc, symbolExchangeTimeZone); + history = GetTickHistoryData(restRequest, historyRequest.Symbol, Resolution.Tick, historyRequest.TickType, startDateTimeUtc, historyRequest.EndTimeUtc, symbolExchangeTimeZone); + break; case Resolution.Second: case Resolution.Minute: case Resolution.Hour: - return GetIntradayHistoryData(restRequest, historyRequest.Symbol, historyRequest.Resolution, historyRequest.TickType, symbolExchangeTimeZone); + history = GetIntradayHistoryData(restRequest, historyRequest.Symbol, historyRequest.Resolution, historyRequest.TickType, symbolExchangeTimeZone); + break; case Resolution.Daily: - return GetDailyHistoryData(restRequest, historyRequest.Symbol, historyRequest.Resolution, historyRequest.TickType, symbolExchangeTimeZone); + history = GetDailyHistoryData(restRequest, historyRequest.Symbol, historyRequest.Resolution, historyRequest.TickType, symbolExchangeTimeZone); + break; default: throw new ArgumentException($"{nameof(ThetaDataProvider)}.{nameof(GetHistory)}: Invalid resolution: {historyRequest.Resolution}. Supported resolutions are Tick, Second, Minute, Hour, and Daily."); } + + if (history == null) + { + return null; + } + + return FilterHistory(history, historyRequest, startDateTimeLocal, endDateTimeLocal); + } + + private IEnumerable FilterHistory(IEnumerable history, HistoryRequest request, DateTime startTimeLocal, DateTime endTimeLocal) + { + // cleaning the data before returning it back to user + foreach (var bar in history) + { + if (bar.Time >= startTimeLocal && bar.EndTime <= endTimeLocal) + { + if (request.ExchangeHours.IsOpen(bar.Time, bar.EndTime, request.IncludeExtendedMarketHours)) + { + yield return bar; + } + } + } } public IEnumerable? GetIndexIntradayHistoryData(RestRequest request, Symbol symbol, Resolution resolution, DateTimeZone symbolExchangeTimeZone) { - request.AddQueryParameter("ivl", GetIntervalsInMilliseconds(resolution)); + request.AddQueryParameter(RequestParameters.IntervalInMilliseconds, GetIntervalsInMilliseconds(resolution)); var period = resolution.ToTimeSpan(); foreach (var prices in _restApiClient.ExecuteRequest>(request)) @@ -220,7 +258,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) case TickType.Trade: return GetHistoricalTickTradeDataByOneDayInterval(request, symbol, startDateTimeUtc, endDateTimeUtc, symbolExchangeTimeZone); case TickType.Quote: - request.AddQueryParameter("ivl", GetIntervalsInMilliseconds(resolution)); + request.AddQueryParameter(RequestParameters.IntervalInMilliseconds, GetIntervalsInMilliseconds(resolution)); Func quoteCallback = (quote) => new Tick(ConvertThetaDataTimeZoneToSymbolExchangeTimeZone(quote.DateTimeMilliseconds, symbolExchangeTimeZone), symbol, quote.AskCondition, quote.AskExchange.TryGetExchangeOrDefault(), quote.BidSize, quote.BidPrice, quote.AskSize, quote.AskPrice); @@ -233,7 +271,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) public IEnumerable? GetIntradayHistoryData(RestRequest request, Symbol symbol, Resolution resolution, TickType tickType, DateTimeZone symbolExchangeTimeZone) { - request.AddQueryParameter("ivl", GetIntervalsInMilliseconds(resolution)); + request.AddQueryParameter(RequestParameters.IntervalInMilliseconds, GetIntervalsInMilliseconds(resolution)); var period = resolution.ToTimeSpan(); @@ -265,14 +303,14 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return GetHistoryEndOfDay(request, // If OHLC prices zero, low trading activity, empty result, low volatility. (eof) => eof.Open == 0 || eof.High == 0 || eof.Low == 0 || eof.Close == 0, - (tradeDateTime, eof) => new TradeBar(ConvertThetaDataTimeZoneToSymbolExchangeTimeZone(tradeDateTime, symbolExchangeTimeZone), symbol, eof.Open, eof.High, eof.Low, eof.Close, eof.Volume, period)); + (tradeDateTime, eof) => new TradeBar(ConvertThetaDataTimeZoneToSymbolExchangeTimeZone(tradeDateTime.Date, symbolExchangeTimeZone), symbol, eof.Open, eof.High, eof.Low, eof.Close, eof.Volume, period)); case TickType.Quote: return GetHistoryEndOfDay(request, // If Ask/Bid - prices/sizes zero, low quote activity, empty result, low volatility. (eof) => eof.AskPrice == 0 || eof.AskSize == 0 || eof.BidPrice == 0 || eof.BidSize == 0, (quoteDateTime, eof) => { - var bar = new QuoteBar(ConvertThetaDataTimeZoneToSymbolExchangeTimeZone(quoteDateTime, symbolExchangeTimeZone), symbol, null, decimal.Zero, null, decimal.Zero, period); + var bar = new QuoteBar(ConvertThetaDataTimeZoneToSymbolExchangeTimeZone(quoteDateTime.Date, symbolExchangeTimeZone), symbol, null, decimal.Zero, null, decimal.Zero, period); bar.UpdateQuote(eof.BidPrice, eof.BidSize, eof.AskPrice, eof.AskSize); return bar; }); @@ -299,8 +337,8 @@ private IEnumerable GetHistoricalTickTradeDataByOneDayInterval(RestRequest foreach (var dateRange in ThetaDataExtensions.GenerateDateRangesWithInterval(startDateTimeET, endDateTimeET)) { - request.AddOrUpdateParameter("start_date", dateRange.startDate.ConvertToThetaDataDateFormat(), ParameterType.QueryString); - request.AddOrUpdateParameter("end_date", dateRange.endDate.ConvertToThetaDataDateFormat(), ParameterType.QueryString); + request.AddOrUpdateParameter(RequestParameters.StartDate, dateRange.startDate.ConvertToThetaDataDateFormat(), ParameterType.QueryString); + request.AddOrUpdateParameter(RequestParameters.EndDate, dateRange.endDate.ConvertToThetaDataDateFormat(), ParameterType.QueryString); foreach (var trades in _restApiClient.ExecuteRequest>(request)) { diff --git a/QuantConnect.ThetaData/ThetaDataOptionChainProvider.cs b/QuantConnect.ThetaData/ThetaDataOptionChainProvider.cs index 77de606..c3b5b68 100644 --- a/QuantConnect.ThetaData/ThetaDataOptionChainProvider.cs +++ b/QuantConnect.ThetaData/ThetaDataOptionChainProvider.cs @@ -79,7 +79,7 @@ public IEnumerable GetOptionContractList(Symbol symbol, DateTime date) // just using quote, which is the most inclusive var request = new RestRequest($"/list/contracts/option/quote", Method.GET); - request.AddQueryParameter("start_date", date.ConvertToThetaDataDateFormat()); + request.AddQueryParameter(RequestParameters.StartDate, date.ConvertToThetaDataDateFormat()); request.AddQueryParameter("root", underlying.Value); foreach (var option in _restApiClient.ExecuteRequest>(request).SelectMany(x => x.Response)) diff --git a/QuantConnect.ThetaData/ThetaDataRestApiClient.cs b/QuantConnect.ThetaData/ThetaDataRestApiClient.cs index 757def6..15ea36f 100644 --- a/QuantConnect.ThetaData/ThetaDataRestApiClient.cs +++ b/QuantConnect.ThetaData/ThetaDataRestApiClient.cs @@ -18,6 +18,9 @@ using QuantConnect.Util; using QuantConnect.Logging; using QuantConnect.Configuration; +using System.Collections.Concurrent; +using QuantConnect.Lean.DataSource.ThetaData.Models.Rest; +using QuantConnect.Lean.DataSource.ThetaData.Models.Wrappers; using QuantConnect.Lean.DataSource.ThetaData.Models.Interfaces; namespace QuantConnect.Lean.DataSource.ThetaData @@ -27,6 +30,11 @@ namespace QuantConnect.Lean.DataSource.ThetaData /// public class ThetaDataRestApiClient { + /// + /// The maximum number of times a failed request will be retried. + /// + private const int MaxRequestRetries = 2; + /// /// Represents the API version used in the REST API endpoints. /// @@ -61,6 +69,40 @@ public ThetaDataRestApiClient(RateGate rateGate) _rateGate = rateGate; } + /// + /// Executes a REST request in parallel and returns the results synchronously. + /// + /// The type of object that implements the interface. + /// The REST request to execute. + /// A collection of objects that implement the interface. + public IEnumerable ExecuteRequest(RestRequest? request) where T : IBaseResponse + { + var parameters = GetSpecificQueryParameters(request.Parameters, RequestParameters.IntervalInMilliseconds, RequestParameters.StartDate, RequestParameters.EndDate); + + if (parameters.Count != 3) + { + return ExecuteRequestAsync(request).SynchronouslyAwaitTaskResult(); + } + + var intervalInDay = parameters[RequestParameters.IntervalInMilliseconds] switch + { + "0" => 1, + "1000" or "60000" => 30, + "3600000" => 90, + _ => throw new NotImplementedException($"{nameof(ThetaDataRestApiClient)}.{nameof(ExecuteRequestParallelAsync)}: The interval '{parameters[RequestParameters.IntervalInMilliseconds]}' is not supported.") + }; + + var startDate = parameters[RequestParameters.StartDate].ConvertFromThetaDataDateFormat(); + var endDate = parameters[RequestParameters.EndDate].ConvertFromThetaDataDateFormat(); + + if ((endDate - startDate).TotalDays <= intervalInDay) + { + return ExecuteRequestAsync(request).SynchronouslyAwaitTaskResult(); + } + + return ExecuteRequestParallelAsync(request, startDate, endDate, intervalInDay).SynchronouslyAwaitTaskResult(); + } + /// /// Executes a REST request and deserializes the response content into an object. /// @@ -68,41 +110,149 @@ public ThetaDataRestApiClient(RateGate rateGate) /// The REST request to execute. /// An enumerable collection of objects that implement the specified base response interface. /// Thrown when an error occurs during the execution of the request or when the response is invalid. - public IEnumerable ExecuteRequest(RestRequest? request) where T : IBaseResponse + private async IAsyncEnumerable ExecuteRequestWithPaginationAsync(RestRequest? request) where T : IBaseResponse { + var retryCount = 0; while (request != null) { Log.Debug($"{nameof(ThetaDataRestApiClient)}.{nameof(ExecuteRequest)}: URI: {_restClient.BuildUri(request)}"); _rateGate?.WaitToProceed(); - var response = _restClient.Execute(request); - - // docs: https://http-docs.thetadata.us/docs/theta-data-rest-api-v2/3ucp87xxgy8d3-error-codes - if ((int)response.StatusCode == 472) + using (StopwatchWrapper.StartIfEnabled($"{nameof(ThetaDataRestApiClient)}.{nameof(ExecuteRequest)}: Executed request to {request.Resource}")) { - Log.Trace($"{nameof(ThetaDataRestApiClient)}.{nameof(ExecuteRequest)}:No data found for the specified request (Status Code: 472)."); - yield break; - } + var response = await _restClient.ExecuteAsync(request).ConfigureAwait(false); - if (response == null || response.StatusCode != System.Net.HttpStatusCode.OK) - { - throw new Exception($"{nameof(ThetaDataRestApiClient)}.{nameof(ExecuteRequest)}: No response received for request to {request.Resource}. Error message: {response?.ErrorMessage ?? "No error message available."}"); + // docs: https://http-docs.thetadata.us/docs/theta-data-rest-api-v2/3ucp87xxgy8d3-error-codes + if ((int)response.StatusCode == 472) + { + Log.Debug($"{nameof(ThetaDataRestApiClient)}.{nameof(ExecuteRequest)}:No data found for the specified request (Status Code: 472) by {response.ResponseUri}"); + yield break; + } + + if (response == null || response.StatusCode != System.Net.HttpStatusCode.OK) + { + if (retryCount < MaxRequestRetries) + { + retryCount++; + await Task.Delay(1000 * retryCount).ConfigureAwait(false); + continue; + } + throw new Exception($"{nameof(ThetaDataRestApiClient)}.{nameof(ExecuteRequest)}: No response received for request to {request.Resource}. Error message: {response?.ErrorMessage ?? "No error message available."}"); + } + + var res = JsonConvert.DeserializeObject(response.Content); + + yield return res; + + if (res?.Header.NextPage != null) + { + request = new RestRequest(Method.GET) { Resource = new Uri(res.Header.NextPage).AbsolutePath.Replace(ApiVersion, string.Empty) }; + } + else + { + request = null; + + } } + } + } + + /// + /// Executes a REST request asynchronously and retrieves a paginated response. + /// + /// The type of response that implements . + /// The REST request to execute. + /// + /// A task that represents the asynchronous operation, returning an + /// containing the responses received from paginated requests. + /// + private async Task> ExecuteRequestAsync(RestRequest? request) where T : IBaseResponse + { + var responses = new List(); + await foreach (var response in ExecuteRequestWithPaginationAsync(request)) + { + responses.Add(response); + } + return responses; + } + + /// + /// Executes a REST request in parallel over multiple date ranges, ensuring efficient batch processing. + /// A maximum of 4 parallel requests are made at a time to avoid excessive API load. + /// + /// The type of response that implements . + /// The REST request to execute. + /// The start date of the data range. + /// The end date of the data range. + /// + /// The interval in days for splitting the date range into smaller requests. + /// + /// + /// A task representing the asynchronous operation, returning an + /// containing the aggregated responses from all parallel requests. + /// + private async Task> ExecuteRequestParallelAsync(RestRequest? request, DateTime startDate, DateTime endDate, int intervalInDay) where T : IBaseResponse + { + var resultDict = new ConcurrentDictionary>(); - var res = JsonConvert.DeserializeObject(response.Content); + var dateRanges = ThetaDataExtensions.GenerateDateRangesWithInterval(startDate, endDate, intervalInDay).Select((range, index) => (range, index)).ToList(); - yield return res; + await Parallel.ForEachAsync(dateRanges, async (item, _) => + { + var (dateRange, index) = item; + var requestClone = new RestRequest(request.Resource, request.Method); - var nextPage = res?.Header.NextPage == null ? null : new Uri(res.Header.NextPage); + foreach (var param in request.Parameters) + { + switch (param.Name) + { + case RequestParameters.StartDate: + requestClone.AddOrUpdateParameter(RequestParameters.StartDate, dateRange.startDate.ConvertToThetaDataDateFormat(), ParameterType.QueryString); + break; + case RequestParameters.EndDate: + requestClone.AddOrUpdateParameter(RequestParameters.EndDate, dateRange.endDate.ConvertToThetaDataDateFormat(), ParameterType.QueryString); + break; + default: + requestClone.AddParameter(param); + break; + } + } - request = null; + var results = new List(); + await foreach (var response in ExecuteRequestWithPaginationAsync(requestClone)) + { + results.Add(response); + } + resultDict[index] = results; + }).ConfigureAwait(false); + return resultDict.OrderBy(kvp => kvp.Key).SelectMany(kvp => kvp.Value); + } - if (nextPage != null) + /// + /// Extracts specific query parameters from a collection of request parameters. + /// + /// The collection of request parameters. + /// The parameter names to find. + /// A dictionary of the matching query parameters and their values. + /// Thrown when a required parameter is missing or has an invalid value. + private Dictionary GetSpecificQueryParameters(IReadOnlyCollection requestParameters, params string[] findingParamNames) + { + var parameters = new Dictionary(findingParamNames.Length); + foreach (var parameter in requestParameters) + { + if (parameter?.Name != null && findingParamNames.Contains(parameter.Name, StringComparer.InvariantCultureIgnoreCase)) { - request = new RestRequest(Method.GET) { Resource = nextPage.AbsolutePath.Replace(ApiVersion, string.Empty) }; + var value = parameter.Value?.ToString(); + if (string.IsNullOrEmpty(value)) + { + throw new ArgumentException($"The value for the parameter '{parameter.Name}' is null or empty. Ensure that this parameter has a valid value.", nameof(requestParameters)); + } + + parameters[parameter.Name] = value; } - }; + } + return parameters; } } }