From 79369a5e40317ded337ed0637227ef1508456b9a Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 18 Sep 2024 18:47:38 -0400 Subject: [PATCH 01/38] Fix pandas converter to handle list of data with different symbols --- .../Data/UniverseSelection/OptionUniverse.cs | 8 ++++ Common/Python/PandasConverter.cs | 46 +++++++++++-------- Common/Python/PandasData.cs | 8 ++-- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/Common/Data/UniverseSelection/OptionUniverse.cs b/Common/Data/UniverseSelection/OptionUniverse.cs index 735ae4c92752..8cc3280f0454 100644 --- a/Common/Data/UniverseSelection/OptionUniverse.cs +++ b/Common/Data/UniverseSelection/OptionUniverse.cs @@ -384,6 +384,14 @@ public PreCalculatedGreeks(string csvLine) { _csvLine = csvLine; } + + /// + /// Gets a string representation of the greeks values + /// + public override string ToString() + { + return $"D: {Delta}, G: {Gamma}, V: {Vega}, T: {Theta}, R: {Rho}"; + } } } } diff --git a/Common/Python/PandasConverter.cs b/Common/Python/PandasConverter.cs index 1db45d05ce73..12c235d8f998 100644 --- a/Common/Python/PandasConverter.cs +++ b/Common/Python/PandasConverter.cs @@ -70,6 +70,11 @@ public PyObject GetDataFrame(IEnumerable data, Type dataType = null) AddSliceDataTypeDataToDict(slice, requestedTick, requestedTradeBar, requestedQuoteBar, sliceDataDict, ref maxLevels, dataType); } + return CreateDataFrame(sliceDataDict, maxLevels); + } + + private static PyObject CreateDataFrame(Dictionary sliceDataDict, int maxLevels, bool sort = true) + { using (Py.GIL()) { if (sliceDataDict.Count == 0) @@ -77,9 +82,13 @@ public PyObject GetDataFrame(IEnumerable data, Type dataType = null) return _pandas.DataFrame(); } using var dataFrames = sliceDataDict.Select(x => x.Value.ToPandasDataFrame(maxLevels)).ToPyListUnSafe(); - using var sortDic = Py.kw("sort", true); + using var sortDic = Py.kw("sort", sort); var result = _concat.Invoke(new[] { dataFrames }, sortDic); + using var replaceValue = 0.ToPython(); + using var inplace = Py.kw("inplace", true); + result.GetAttr("fillna").Invoke(new[] { replaceValue }, inplace); + foreach (var df in dataFrames) { df.Dispose(); @@ -97,27 +106,22 @@ public PyObject GetDataFrame(IEnumerable data, Type dataType = null) public PyObject GetDataFrame(IEnumerable data) where T : IBaseData { - PandasData sliceData = null; + var pandasDataBySymbol = new Dictionary(); + var maxLevels = 0; + foreach (var datum in data) { - if (sliceData == null) + if (!pandasDataBySymbol.TryGetValue(datum.Symbol.ID, out var pandasData)) { - sliceData = new PandasData(datum); + pandasData = new PandasData(datum); + pandasDataBySymbol[datum.Symbol.ID] = pandasData; + maxLevels = Math.Max(maxLevels, pandasData.Levels); } - sliceData.Add(datum); + pandasData.Add(datum); } - using (Py.GIL()) - { - // If sliceData is still null, data is an empty enumerable - // returns an empty pandas.DataFrame - if (sliceData == null) - { - return _pandas.DataFrame(); - } - return sliceData.ToPandasDataFrame(); - } + return CreateDataFrame(pandasDataBySymbol, maxLevels, sort: false); } /// @@ -187,9 +191,15 @@ public PyObject GetIndicatorDataFrame(PyObject data) /// public override string ToString() { - return _pandas == null - ? Messages.PandasConverter.PandasModuleNotImported - : _pandas.Repr(); + if (_pandas == null) + { + return Messages.PandasConverter.PandasModuleNotImported; + } + + using (Py.GIL()) + { + return _pandas.Repr(); + } } /// diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 3a31290b53d0..7cf511f3941d 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -25,7 +25,6 @@ using System.Globalization; using System.Linq; using System.Reflection; -using QuantConnect.Util; namespace QuantConnect.Python { @@ -139,7 +138,7 @@ public PandasData(object data) IsCustomData = Extensions.IsCustomDataType(_symbol, type); if (_symbol.SecurityType == SecurityType.Future) Levels = 3; - if (_symbol.SecurityType.IsOption()) Levels = 5; + if (_symbol.SecurityType.IsOption() && _symbol.IsCanonical()) Levels = 5; IEnumerable columns = _standardColumns; @@ -158,7 +157,10 @@ public PandasData(object data) } else { - var members = type.GetMembers().Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property).ToList(); + var members = type + .GetMembers(BindingFlags.Instance | BindingFlags.Public) + .Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) + .ToList(); var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key); foreach (var duplicateKey in duplicateKeys) From 11e3b9fe4a6c5742660f6218501b66e2a290a760 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 19 Sep 2024 17:08:02 -0400 Subject: [PATCH 02/38] Properly convert list of data into dataframe Take into consideration data for multiple symbols in the same list --- .../OptionChainFullDataRegressionAlgorithm.py | 15 +++-- Common/Python/PandasConverter.cs | 62 ++++++++++--------- Common/Python/PandasData.cs | 8 +-- Tests/Python/PandasConverterTests.cs | 35 +++++++++++ 4 files changed, 83 insertions(+), 37 deletions(-) diff --git a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py index 976d69469d01..4799362ded68 100644 --- a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py +++ b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py @@ -27,14 +27,19 @@ def initialize(self): goog = self.add_equity("GOOG").symbol + option_chain = self.option_chain(goog) + + # Demonstration using data frame: # Get contracts expiring within 10 days, with an implied volatility greater than 0.5 and a delta less than 0.5 - contracts = [ - contract_data - for contract_data in self.option_chain(goog) - if contract_data.id.date - self.time <= timedelta(days=10) and contract_data.implied_volatility > 0.5 and contract_data.greeks.delta < 0.5 + contracts_ids = [ + contract_data["id"] + for index, contract_data in option_chain.data_frame.iterrows() + if contract_data["id"].date - self.time <= timedelta(days=10) and contract_data["impliedvolatility"] > 0.5 and contract_data["greeks"].delta < 0.5 ] + # Get the contract with the latest expiration date - self._option_contract = sorted(contracts, key=lambda x: x.id.date, reverse=True)[0] + option_contract_id = sorted(contracts_ids, key=lambda id: id.date, reverse=True)[0] + self._option_contract = [x.symbol for x in option_chain if x.symbol.id == option_contract_id][0] self.add_option_contract(self._option_contract) diff --git a/Common/Python/PandasConverter.cs b/Common/Python/PandasConverter.cs index 12c235d8f998..753c131b7008 100644 --- a/Common/Python/PandasConverter.cs +++ b/Common/Python/PandasConverter.cs @@ -73,30 +73,6 @@ public PyObject GetDataFrame(IEnumerable data, Type dataType = null) return CreateDataFrame(sliceDataDict, maxLevels); } - private static PyObject CreateDataFrame(Dictionary sliceDataDict, int maxLevels, bool sort = true) - { - using (Py.GIL()) - { - if (sliceDataDict.Count == 0) - { - return _pandas.DataFrame(); - } - using var dataFrames = sliceDataDict.Select(x => x.Value.ToPandasDataFrame(maxLevels)).ToPyListUnSafe(); - using var sortDic = Py.kw("sort", sort); - var result = _concat.Invoke(new[] { dataFrames }, sortDic); - - using var replaceValue = 0.ToPython(); - using var inplace = Py.kw("inplace", true); - result.GetAttr("fillna").Invoke(new[] { replaceValue }, inplace); - - foreach (var df in dataFrames) - { - df.Dispose(); - } - return result; - } - } - /// /// Converts an enumerable of in a pandas.DataFrame /// @@ -107,21 +83,22 @@ public PyObject GetDataFrame(IEnumerable data) where T : IBaseData { var pandasDataBySymbol = new Dictionary(); - var maxLevels = 0; - foreach (var datum in data) { if (!pandasDataBySymbol.TryGetValue(datum.Symbol.ID, out var pandasData)) { pandasData = new PandasData(datum); pandasDataBySymbol[datum.Symbol.ID] = pandasData; - maxLevels = Math.Max(maxLevels, pandasData.Levels); } pandasData.Add(datum); } - return CreateDataFrame(pandasDataBySymbol, maxLevels, sort: false); + return CreateDataFrame(pandasDataBySymbol, + sort: false, + // Multiple data frames (one for each symbol) will be concatenated, + // so make sure rows with missing values only are not filtered out before concatenation + filterMissingValueColumns: pandasDataBySymbol.Count <= 1); } /// @@ -202,6 +179,35 @@ public override string ToString() } } + /// + /// Create a data frame by concatenated the resulting data frames from the given data + /// + private static PyObject CreateDataFrame(Dictionary dataBySymbol, int maxLevels = 2, bool sort = true, + bool filterMissingValueColumns = true) + { + using (Py.GIL()) + { + if (dataBySymbol.Count == 0) + { + return _pandas.DataFrame(); + } + + using var dataFrames = dataBySymbol.Select(x => x.Value.ToPandasDataFrame(maxLevels, filterMissingValueColumns)).ToPyListUnSafe(); + using var sortDic = Py.kw("sort", sort); + var result = _concat.Invoke(new[] { dataFrames }, sortDic); + + // Drop columns with only NaN or None values + using var kwargs = Py.kw("axis", 1, "inplace", true, "how", "all"); + result.GetAttr("dropna").Invoke(Array.Empty(), kwargs); + + foreach (var df in dataFrames) + { + df.Dispose(); + } + return result; + } + } + /// /// Creates a series from a list of and adds it to the /// as the value of the given diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 7cf511f3941d..fd0769a3d817 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -181,8 +181,7 @@ public PandasData(object data) } } - var customColumns = new HashSet(columns); - customColumns.Add("value"); + var customColumns = new HashSet(columns) { "value" }; customColumns.UnionWith(keys); columns = customColumns; @@ -344,8 +343,9 @@ public void AddTick(Tick tick) /// Get the pandas.DataFrame of the current state /// /// Number of levels of the multi index + /// If false, make sure columns with "missing" values only are still added to the dataframe /// pandas.DataFrame object - public PyObject ToPandasDataFrame(int levels = 2) + public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns = true) { List list; var symbol = _symbol.ID.ToString().ToPython(); @@ -385,7 +385,7 @@ public PyObject ToPandasDataFrame(int levels = 2) using var pyDict = new PyDict(); foreach (var kvp in _series) { - if (kvp.Value.ShouldFilter) continue; + if (filterMissingValueColumns && kvp.Value.ShouldFilter) continue; if (!indexCache.TryGetValue(kvp.Value.Times, out var index)) { diff --git a/Tests/Python/PandasConverterTests.cs b/Tests/Python/PandasConverterTests.cs index fb3c25bf6ff0..1d9a6a92d006 100644 --- a/Tests/Python/PandasConverterTests.cs +++ b/Tests/Python/PandasConverterTests.cs @@ -104,6 +104,41 @@ public void HandlesEnumerableDataType() } } + [Test] + public void HandlesEnumerableWithMultipleSymbols() + { + var converter = new PandasConverter(); + var data = new List + { + new TradeBar(new DateTime(2020, 1, 2), Symbols.IBM, 101m, 102m, 100m, 101m, 10m), + new TradeBar(new DateTime(2020, 1, 3), Symbols.IBM, 101m, 102m, 100m, 101m, 20m), + new TradeBar(new DateTime(2020, 1, 2), Symbols.SPY_C_192_Feb19_2016, 51m, 52m, 50m, 51m, 100m), + new TradeBar(new DateTime(2020, 1, 3), Symbols.SPY_C_192_Feb19_2016, 51m, 52m, 50m, 51m, 200m), + }; + + dynamic dataFrame = converter.GetDataFrame(data); + + using (Py.GIL()) + { + Assert.Multiple(() => + { + foreach (var symbol in data.Select(x => x.Symbol).Distinct()) + { + Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)), $"Unexpected empty sub dataframe for {symbol}"); + + var subDataFrame = dataFrame.loc[symbol]; + Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); + + var count = subDataFrame.__len__().AsManagedObject(typeof(int)); + Assert.AreEqual(2, count, $"Unexpected rows count for {symbol} sub dataframe"); + + var dataCount = subDataFrame.values.__len__().AsManagedObject(typeof(int)); + Assert.AreEqual(2, dataCount, $"Unexpected rows count for {symbol} sub dataframe"); + } + }); + } + } + [Test] public void HandlesEmptyEnumerable() { From be9d68c8f565a4729d6977ee31a67b707c63a143 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 19 Sep 2024 17:16:17 -0400 Subject: [PATCH 03/38] Cleanup --- Common/Python/PandasData.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index fd0769a3d817..a6d1e4e0fd1c 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -138,7 +138,7 @@ public PandasData(object data) IsCustomData = Extensions.IsCustomDataType(_symbol, type); if (_symbol.SecurityType == SecurityType.Future) Levels = 3; - if (_symbol.SecurityType.IsOption() && _symbol.IsCanonical()) Levels = 5; + if (_symbol.SecurityType.IsOption()) Levels = 5; IEnumerable columns = _standardColumns; From 862dc8d099dbfbdbb5fe8b4a78e27ae5e90fb863 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 20 Sep 2024 15:16:28 -0400 Subject: [PATCH 04/38] Index dataframes by symbol object instead of SID string --- Common/PandasMapper.py | 10 ++--- Common/Python/PandasData.cs | 2 +- Tests/Algorithm/AlgorithmChainsTests.cs | 56 +++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index f20b76fc8aad..4c28896594b4 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -30,15 +30,13 @@ def mapper(key): Symbol SecurityIdentifier.If cannot map, returns the object ''' keyType = type(key) - if keyType is Symbol: - return str(key.ID) if keyType is str: reserved = ['high', 'low', 'open', 'close'] if key in reserved: return key kvp = SymbolCache.TryGetSymbol(key, None) if kvp[0]: - return str(kvp[1].ID) + return kvp[1] if keyType is list: return [mapper(x) for x in key] if keyType is tuple: @@ -49,7 +47,7 @@ def mapper(key): def wrap_keyerror_function(f): '''Wraps function f with wrapped_function, used for functions that throw KeyError when not found. - wrapped_function converts the args / kwargs to use alternative index keys and then calls the function. + wrapped_function converts the args / kwargs to use alternative index keys and then calls the function. If this fails we fall back to the original key and try it as well, if they both fail we throw our error. ''' def wrapped_function(*args, **kwargs): @@ -111,7 +109,7 @@ def wrapped_function(*args, **kwargs): pd.core.indexes.base.Index.get_loc = wrap_keyerror_function(pd.core.indexes.base.Index.get_loc) # Wrap our DF _getitem__ as well, even though most pathways go through the above functions -# There are cases like indexing with an array that need to be mapped earlier to stop KeyError from arising +# There are cases like indexing with an array that need to be mapped earlier to stop KeyError from arising pd.core.frame.DataFrame.__getitem__ = wrap_keyerror_function(pd.core.frame.DataFrame.__getitem__) # For older version of pandas we may need to wrap extra functions @@ -119,7 +117,7 @@ def wrapped_function(*args, **kwargs): pd.core.indexes.base.Index.get_value = wrap_keyerror_function(pd.core.indexes.base.Index.get_value) # Special cases where we need to wrap a function that won't throw a keyerror when not found but instead returns true or false -# Wrap __contains__ to support Python syntax like 'SPY' in DataFrame +# Wrap __contains__ to support Python syntax like 'SPY' in DataFrame pd.core.indexes.base.Index.__contains__ = wrap_bool_function(pd.core.indexes.base.Index.__contains__) # For compatibility with PandasData.cs usage of this module (Previously wrapped classes) diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index a6d1e4e0fd1c..f78acbd0a5c3 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -348,7 +348,7 @@ public void AddTick(Tick tick) public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns = true) { List list; - var symbol = _symbol.ID.ToString().ToPython(); + var symbol = _symbol.ToPython(); // Create the index labels var names = _defaultNames; diff --git a/Tests/Algorithm/AlgorithmChainsTests.cs b/Tests/Algorithm/AlgorithmChainsTests.cs index af02e9aafd9b..179cb47be80a 100644 --- a/Tests/Algorithm/AlgorithmChainsTests.cs +++ b/Tests/Algorithm/AlgorithmChainsTests.cs @@ -14,8 +14,10 @@ */ using System; +using System.Collections.Generic; using System.Linq; using NUnit.Framework; +using Python.Runtime; using QuantConnect.Algorithm; using QuantConnect.Data; using QuantConnect.Interfaces; @@ -71,5 +73,59 @@ public void GetsFullDataOptionChain(Symbol symbol, DateTime date) CollectionAssert.AreEquivalent(optionContractsSymbols, optionContractsData.Select(x => x.Symbol)); } + + [TestCaseSource(nameof(OptionChainTestCases))] + public void GetsFullDataOptionChainAsDataFrame(Symbol symbol, DateTime date) + { + _algorithm.SetPandasConverter(); + _algorithm.SetDateTime(date.ConvertToUtc(_algorithm.TimeZone)); + + using var _ = Py.GIL(); + + var module = PyModule.FromString(nameof(GetsFullDataOptionChainAsDataFrame), @" +def get_option_chain_data_from_dataframe(algorithm, canonical): + option_chain_df = algorithm.option_chain(canonical).data_frame + + # Will make it more complex than it needs to be, + # just so that we can test indexing by symbol using df.loc[] + for (symbol, date) in option_chain_df.index: + symbol_data = option_chain_df.loc[(symbol)] + + if symbol_data.shape[0] != 1: + raise ValueError(f'Expected 1 row for {symbol}, got {symbol_data.shape[0]}') + + yield { + 'symbol': symbol, + 'date': date, + 'open': symbol_data['open'].values[0], + 'high': symbol_data['high'].values[0], + 'low': symbol_data['low'].values[0], + 'close': symbol_data['close'].values[0], + 'openinterest': symbol_data['openinterest'].values[0], + 'impliedvolatility': symbol_data['impliedvolatility'].values[0], + 'greeks': symbol_data['greeks'].values[0] + } +"); + + using var pyAlgorithm = _algorithm.ToPython(); + using var pySymbol = symbol.ToPython(); + + using var pyOptionChainData = module.GetAttr("get_option_chain_data_from_dataframe").Invoke(pyAlgorithm, pySymbol); + var optionChain = new List(); + + Assert.DoesNotThrow(() => + { + foreach (PyObject item in pyOptionChainData.GetIterator()) + { + var contractSymbol = item["symbol"].GetAndDispose(); + optionChain.Add(contractSymbol); + item.DisposeSafely(); + } + }); + + var optionContractsSymbols = _optionChainProvider.GetOptionContractList(symbol, date.Date).ToList(); + + CollectionAssert.AreEquivalent(optionContractsSymbols, optionChain); + } } } From c2e3f7e8b17dd71e5d18424e5db25daf4ad10129 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 23 Sep 2024 11:21:44 -0400 Subject: [PATCH 05/38] Add symbol equality operator to compare against object --- Common/PandasMapper.py | 4 +-- Common/Symbol.cs | 49 ++++++++++++++++++++++++++++- Tests/Python/PandasIndexingTests.cs | 4 +-- 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index 4c28896594b4..3800ccd29d3d 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -63,14 +63,14 @@ def wrapped_function(*args, **kwargs): return f(*newargs, **newkwargs) except KeyError as e: - mKey = [arg for arg in newargs if isinstance(arg, str)] + mKey = [str(arg) for arg in newargs if isinstance(arg, str) or isinstance(arg, Symbol)] # Execute original # Allows for df, Series, etc indexing for keys like 'SPY' if they exist try: return f(*args, **kwargs) except KeyError as e: - oKey = [arg for arg in args if isinstance(arg, str)] + oKey = [str(arg) for arg in args if isinstance(arg, str) or isinstance(arg, Symbol)] raise KeyError(f"No key found for either mapped or original key. Mapped Key: {mKey}; Original Key: {oKey}") wrapped_function.__name__ = f.__name__ diff --git a/Common/Symbol.cs b/Common/Symbol.cs index 346f477550bf..9ca0a46d3d5e 100644 --- a/Common/Symbol.cs +++ b/Common/Symbol.cs @@ -19,7 +19,6 @@ using Python.Runtime; using Newtonsoft.Json; using QuantConnect.Securities; -using QuantConnect.Securities.IndexOption; namespace QuantConnect { @@ -741,6 +740,32 @@ public bool Equals(Symbol other) return left.Equals(right); } + /// + /// Equals operator + /// + /// The left operand + /// The right operand + /// True if both symbols are equal, otherwise false + /// This is necessary in cases like Pythonnet passing a string + /// as an object instead of using the implicit conversion + public static bool operator ==(Symbol left, object right) + { + return left.Equals(right); + } + + /// + /// Equals operator + /// + /// The left operand + /// The right operand + /// True if both symbols are equal, otherwise false + /// This is necessary in cases like Pythonnet passing a string + /// as an object instead of using the implicit conversion + public static bool operator ==(object left, Symbol right) + { + return right.Equals(left); + } + /// /// Not equals operator /// @@ -752,6 +777,28 @@ public bool Equals(Symbol other) return !(left == right); } + /// + /// Not equals operator + /// + /// The left operand + /// The right operand + /// True if both symbols are not equal, otherwise false + public static bool operator !=(Symbol left, object right) + { + return !(left == right); + } + + /// + /// Not equals operator + /// + /// The left operand + /// The right operand + /// True if both symbols are not equal, otherwise false + public static bool operator !=(object left, Symbol right) + { + return !(left == right); + } + #endregion #region Implicit operators diff --git a/Tests/Python/PandasIndexingTests.cs b/Tests/Python/PandasIndexingTests.cs index b00d9ba54868..425a01f3b948 100644 --- a/Tests/Python/PandasIndexingTests.cs +++ b/Tests/Python/PandasIndexingTests.cs @@ -53,7 +53,7 @@ public void IndexingDataFrameWithList() public void ContainsUserMappedTickers() { using (Py.GIL()) - { + { PyObject result = _pandasDataFrameTests.test_contains_user_mapped_ticker(); var test = result.As(); @@ -82,7 +82,7 @@ public void ExpectedException() PyObject result = _pandasDataFrameTests.test_expected_exception(); var exception = result.As(); - Assert.IsTrue(exception.Contains("No key found for either mapped or original key. Mapped Key: ['AAPL R735QTJ8XC9X']; Original Key: ['aapl']", StringComparison.InvariantCulture)); + Assert.IsTrue(exception.Contains("No key found for either mapped or original key. Mapped Key: ['AAPL']; Original Key: ['aapl']", StringComparison.InvariantCulture)); } } } From 8123c612299b40c1ecb1dc3045beda19064db9a8 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 23 Sep 2024 11:51:48 -0400 Subject: [PATCH 06/38] Exclude "ID" from option chain dataframe --- .../OptionChainFullDataRegressionAlgorithm.py | 15 +++++++++------ Common/Python/PandasData.cs | 5 ++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py index 4799362ded68..22241cd01e4d 100644 --- a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py +++ b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py @@ -31,17 +31,20 @@ def initialize(self): # Demonstration using data frame: # Get contracts expiring within 10 days, with an implied volatility greater than 0.5 and a delta less than 0.5 - contracts_ids = [ - contract_data["id"] + contracts = [ + # Index is a tuple (symbol, date) + index[0] for index, contract_data in option_chain.data_frame.iterrows() - if contract_data["id"].date - self.time <= timedelta(days=10) and contract_data["impliedvolatility"] > 0.5 and contract_data["greeks"].delta < 0.5 + if index[0].id.date - self.time <= timedelta(days=10) and contract_data["impliedvolatility"] > 0.5 and contract_data["greeks"].delta < 0.5 ] # Get the contract with the latest expiration date - option_contract_id = sorted(contracts_ids, key=lambda id: id.date, reverse=True)[0] - self._option_contract = [x.symbol for x in option_chain if x.symbol.id == option_contract_id][0] + option_contract = sorted(contracts, key=lambda x: x.id.date, reverse=True)[0] - self.add_option_contract(self._option_contract) + # Can use the symbol instance to index the data frame + self.debug(f"Option contract data:\n{option_chain.data_frame.loc[(option_contract)]}") + + self._option_contract = self.add_option_contract(option_contract) def on_data(self, data): # Do some trading with the selected contract for sample purposes diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index f78acbd0a5c3..e44634b8776f 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -17,6 +17,7 @@ using QuantConnect.Data; using QuantConnect.Data.Fundamental; using QuantConnect.Data.Market; +using QuantConnect.Data.UniverseSelection; using QuantConnect.Util; using System; using System.Collections; @@ -157,9 +158,11 @@ public PandasData(object data) } else { + var isOptionUniverse = type == typeof(OptionUniverse); var members = type .GetMembers(BindingFlags.Instance | BindingFlags.Public) - .Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) + .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) && + (!isOptionUniverse || x.Name != nameof(OptionUniverse.ID))) .ToList(); var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key); From 5138b1c71c8a60faa58397e7ad55d848ec565c24 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 23 Sep 2024 12:01:06 -0400 Subject: [PATCH 07/38] Minor fix --- Common/Symbol.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Common/Symbol.cs b/Common/Symbol.cs index 9ca0a46d3d5e..a81e5af8197b 100644 --- a/Common/Symbol.cs +++ b/Common/Symbol.cs @@ -750,7 +750,20 @@ public bool Equals(Symbol other) /// as an object instead of using the implicit conversion public static bool operator ==(Symbol left, object right) { - return left.Equals(right); + if (ReferenceEquals(left, right)) + { + // this is a performance shortcut + return true; + } + + var rightStr = right as string; + // Use the implicit conversion for strings + if (rightStr != null) + { + return left == (Symbol)rightStr; + } + + return left == (right as Symbol); } /// From b122ff4cd46d6e037a9563189c09b774a5ca0d29 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 23 Sep 2024 12:48:30 -0400 Subject: [PATCH 08/38] Add greeks columns directly in option chain dataframe. Also add pass-through properties for greek values in OptionUniverse --- .../OptionChainFullDataRegressionAlgorithm.cs | 6 +- .../OptionChainFullDataRegressionAlgorithm.py | 2 +- .../Data/UniverseSelection/OptionUniverse.cs | 25 ++++++ Common/Python/PandasData.cs | 79 ++++++++++++++----- 4 files changed, 90 insertions(+), 22 deletions(-) diff --git a/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs b/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs index ce1fe0705ab6..bee130277f25 100644 --- a/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs +++ b/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs @@ -42,11 +42,11 @@ public override void Initialize() _optionContract = OptionChain(goog) // Get contracts expiring within 10 days, with an implied volatility greater than 0.5 and a delta less than 0.5 - .Where(contractData => contractData.ID.Date - Time <= TimeSpan.FromDays(10) && + .Where(contractData => contractData.Symbol.ID.Date - Time <= TimeSpan.FromDays(10) && contractData.ImpliedVolatility > 0.5m && - contractData.Greeks.Delta < 0.5m) + contractData.Delta < 0.5m) // Get the contract with the latest expiration date - .OrderByDescending(x => x.ID.Date) + .OrderByDescending(x => x.Symbol.ID.Date) .First(); AddOptionContract(_optionContract); diff --git a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py index 22241cd01e4d..a10f8671602c 100644 --- a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py +++ b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py @@ -35,7 +35,7 @@ def initialize(self): # Index is a tuple (symbol, date) index[0] for index, contract_data in option_chain.data_frame.iterrows() - if index[0].id.date - self.time <= timedelta(days=10) and contract_data["impliedvolatility"] > 0.5 and contract_data["greeks"].delta < 0.5 + if index[0].id.date - self.time <= timedelta(days=10) and contract_data["impliedvolatility"] > 0.5 and contract_data["delta"] < 0.5 ] # Get the contract with the latest expiration date diff --git a/Common/Data/UniverseSelection/OptionUniverse.cs b/Common/Data/UniverseSelection/OptionUniverse.cs index 8cc3280f0454..46a1c97c6892 100644 --- a/Common/Data/UniverseSelection/OptionUniverse.cs +++ b/Common/Data/UniverseSelection/OptionUniverse.cs @@ -134,6 +134,31 @@ public BaseGreeks Greeks } } + /// + /// Gets the delta. Pass-through for + /// + public decimal Delta => Greeks.Delta; + + /// + /// Gets the gamma. Pass-through for + /// + public decimal Gamma => Greeks.Gamma; + + /// + /// Gets the vega. Pass-through for + /// + public decimal Vega => Greeks.Vega; + + /// + /// Gets the theta. Pass-through for + /// + public decimal Theta => Greeks.Theta; + + /// + /// Gets the rho. Pass-through for + /// + public decimal Rho => Greeks.Rho; + /// /// Time that the data became available to use /// diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index e44634b8776f..bace8162e956 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -72,7 +72,7 @@ public class PandasData private static PyList _level3Names; private readonly static HashSet _baseDataProperties = typeof(BaseData).GetProperties().ToHashSet(x => x.Name.ToLowerInvariant()); - private readonly static ConcurrentDictionary> _membersByType = new (); + private readonly static ConcurrentDictionary> _membersByType = new (); private readonly static IReadOnlyList _standardColumns = new string [] { Open, High, Low, Close, LastPrice, Volume, @@ -84,7 +84,7 @@ public class PandasData private readonly bool _isFundamentalType; private readonly Dictionary _series; - private readonly IEnumerable _members = Enumerable.Empty(); + private readonly IEnumerable _members = Enumerable.Empty(); /// /// Gets true if this is a custom data request, false for normal QC data @@ -154,7 +154,7 @@ public PandasData(object data) { if (_membersByType.TryGetValue(type, out _members)) { - keys = _members.ToHashSet(x => x.Name.ToLowerInvariant()); + keys = _members.ToHashSet(x => x[^1].Name.ToLowerInvariant()); } else { @@ -163,23 +163,51 @@ public PandasData(object data) .GetMembers(BindingFlags.Instance | BindingFlags.Public) .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) && (!isOptionUniverse || x.Name != nameof(OptionUniverse.ID))) + .Select(x => new[] { x }) .ToList(); - var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key); + var duplicateKeys = members.GroupBy(x => x[^1].Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key); foreach (var duplicateKey in duplicateKeys) { throw new ArgumentException($"PandasData.ctor(): {Messages.PandasData.DuplicateKey(duplicateKey, type.FullName)}"); } + if (isOptionUniverse) + { + // Let's expand the greeks + members = members.SelectMany(x => + { + var member = x[^1]; + if (member.Name == nameof(OptionUniverse.Greeks)) + { + var type = (member as PropertyInfo).PropertyType; + var greeksNames = new[] + { + nameof(OptionUniverse.Greeks.Delta), + nameof(OptionUniverse.Greeks.Gamma), + nameof(OptionUniverse.Greeks.Vega), + nameof(OptionUniverse.Greeks.Theta), + nameof(OptionUniverse.Greeks.Rho) + }; + + return type.GetProperties(BindingFlags.Instance | BindingFlags.Public) + .Where(y => greeksNames.Contains(y.Name)) + .Select(y => new[] { member, y }); + } + + return new[] { x }; + }).ToList(); + } + // If the custom data derives from a Market Data (e.g. Tick, TradeBar, QuoteBar), exclude its keys - keys = members.ToHashSet(x => x.Name.ToLowerInvariant()); + keys = members.ToHashSet(x => x[^1].Name.ToLowerInvariant()); keys.ExceptWith(_baseDataProperties); keys.ExceptWith(GetPropertiesNames(typeof(QuoteBar), type)); keys.ExceptWith(GetPropertiesNames(typeof(TradeBar), type)); keys.ExceptWith(GetPropertiesNames(typeof(Tick), type)); keys.Add("value"); - _members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant())).ToList(); + _members = members.Where(x => keys.Contains(x[^1].Name.ToLowerInvariant())).ToList(); _membersByType.TryAdd(type, _members); } } @@ -203,25 +231,40 @@ public void Add(object baseData) foreach (var member in _members) { // TODO field/property.GetValue is expensive - var key = member.Name.ToLowerInvariant(); - var propertyMember = member as PropertyInfo; - if (propertyMember != null) + var key = member[^1].Name.ToLowerInvariant(); + + var value = baseData; + var memberInfo = (MemberInfo)null; + for (var i = 0; i < member.Length; i++) { - var propertyValue = propertyMember.GetValue(baseData); - if (_isFundamentalType && propertyMember.PropertyType.IsAssignableTo(typeof(FundamentalTimeDependentProperty))) + memberInfo = member[i]; + if (memberInfo is PropertyInfo propertyInfo) { - propertyValue = ((FundamentalTimeDependentProperty)propertyValue).Clone(new FixedTimeProvider(endTime)); + value = propertyInfo.GetValue(value); + } + else + { + value = (memberInfo as FieldInfo)?.GetValue(value); + } + + if (value == null) + { + break; } - AddToSeries(key, endTime, propertyValue); - continue; } - else + + if (memberInfo is PropertyInfo propertyMember) { - var fieldMember = member as FieldInfo; - if (fieldMember != null) + if (_isFundamentalType && propertyMember.PropertyType.IsAssignableTo(typeof(FundamentalTimeDependentProperty))) { - AddToSeries(key, endTime, fieldMember.GetValue(baseData)); + value = ((FundamentalTimeDependentProperty)value).Clone(new FixedTimeProvider(endTime)); } + AddToSeries(key, endTime, value); + } + else if (memberInfo is FieldInfo fieldMember) + { + AddToSeries(key, endTime, fieldMember.GetValue(baseData)); + } } From aa35111566a39c4ecacd635b8aea959cebed5a26 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 23 Sep 2024 13:00:54 -0400 Subject: [PATCH 09/38] Some cleanup --- Common/Python/PandasData.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index bace8162e956..efb342e9129f 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -60,6 +60,15 @@ public class PandasData private const string Suspicious = "suspicious"; private const string OpenInterest = "openinterest"; + private static readonly string[] _optionUniverseGreeksNames = new[] + { + nameof(OptionUniverse.Greeks.Delta), + nameof(OptionUniverse.Greeks.Gamma), + nameof(OptionUniverse.Greeks.Vega), + nameof(OptionUniverse.Greeks.Theta), + nameof(OptionUniverse.Greeks.Rho) + }; + // we keep these so we don't need to ask for them each time private static PyString _empty; private static PyObject _pandas; @@ -181,17 +190,8 @@ public PandasData(object data) if (member.Name == nameof(OptionUniverse.Greeks)) { var type = (member as PropertyInfo).PropertyType; - var greeksNames = new[] - { - nameof(OptionUniverse.Greeks.Delta), - nameof(OptionUniverse.Greeks.Gamma), - nameof(OptionUniverse.Greeks.Vega), - nameof(OptionUniverse.Greeks.Theta), - nameof(OptionUniverse.Greeks.Rho) - }; - return type.GetProperties(BindingFlags.Instance | BindingFlags.Public) - .Where(y => greeksNames.Contains(y.Name)) + .Where(y => _optionUniverseGreeksNames.Contains(y.Name)) .Select(y => new[] { member, y }); } From 3b649c30929b9a7e9b2b8933e4218fbfbb83ad30 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 23 Sep 2024 15:21:28 -0400 Subject: [PATCH 10/38] Minor fix --- Algorithm/QCAlgorithm.cs | 6 ++- Common/Python/PandasData.cs | 81 ++++++++++--------------------------- 2 files changed, 26 insertions(+), 61 deletions(-) diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index c1deaf6f1b24..65aa6a8d9101 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3383,8 +3383,10 @@ public DataHistory OptionChain(Symbol symbol) .Select(contractSymbol => new OptionUniverse() { Symbol = contractSymbol, - EndTime = Time.Date - }); + EndTime = Time.Date, + Data = null, + }) + .ToList(); } return new DataHistory(optionChain, new Lazy(() => PandasConverter.GetDataFrame(optionChain))); diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index efb342e9129f..8eaedde541f7 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -60,13 +60,10 @@ public class PandasData private const string Suspicious = "suspicious"; private const string OpenInterest = "openinterest"; - private static readonly string[] _optionUniverseGreeksNames = new[] + private static readonly string[] _optionUniverseExcludedProperties = new[] { - nameof(OptionUniverse.Greeks.Delta), - nameof(OptionUniverse.Greeks.Gamma), - nameof(OptionUniverse.Greeks.Vega), - nameof(OptionUniverse.Greeks.Theta), - nameof(OptionUniverse.Greeks.Rho) + nameof(OptionUniverse.ID), + nameof(OptionUniverse.Greeks) }; // we keep these so we don't need to ask for them each time @@ -81,7 +78,7 @@ public class PandasData private static PyList _level3Names; private readonly static HashSet _baseDataProperties = typeof(BaseData).GetProperties().ToHashSet(x => x.Name.ToLowerInvariant()); - private readonly static ConcurrentDictionary> _membersByType = new (); + private readonly static ConcurrentDictionary> _membersByType = new (); private readonly static IReadOnlyList _standardColumns = new string [] { Open, High, Low, Close, LastPrice, Volume, @@ -93,7 +90,7 @@ public class PandasData private readonly bool _isFundamentalType; private readonly Dictionary _series; - private readonly IEnumerable _members = Enumerable.Empty(); + private readonly IEnumerable _members = Enumerable.Empty(); /// /// Gets true if this is a custom data request, false for normal QC data @@ -163,7 +160,7 @@ public PandasData(object data) { if (_membersByType.TryGetValue(type, out _members)) { - keys = _members.ToHashSet(x => x[^1].Name.ToLowerInvariant()); + keys = _members.ToHashSet(x => x.Name.ToLowerInvariant()); } else { @@ -171,43 +168,24 @@ public PandasData(object data) var members = type .GetMembers(BindingFlags.Instance | BindingFlags.Public) .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) && - (!isOptionUniverse || x.Name != nameof(OptionUniverse.ID))) - .Select(x => new[] { x }) + (!isOptionUniverse || !_optionUniverseExcludedProperties.Contains(x.Name))) .ToList(); - var duplicateKeys = members.GroupBy(x => x[^1].Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key); + var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key); foreach (var duplicateKey in duplicateKeys) { throw new ArgumentException($"PandasData.ctor(): {Messages.PandasData.DuplicateKey(duplicateKey, type.FullName)}"); } - if (isOptionUniverse) - { - // Let's expand the greeks - members = members.SelectMany(x => - { - var member = x[^1]; - if (member.Name == nameof(OptionUniverse.Greeks)) - { - var type = (member as PropertyInfo).PropertyType; - return type.GetProperties(BindingFlags.Instance | BindingFlags.Public) - .Where(y => _optionUniverseGreeksNames.Contains(y.Name)) - .Select(y => new[] { member, y }); - } - - return new[] { x }; - }).ToList(); - } - // If the custom data derives from a Market Data (e.g. Tick, TradeBar, QuoteBar), exclude its keys - keys = members.ToHashSet(x => x[^1].Name.ToLowerInvariant()); + keys = members.ToHashSet(x => x.Name.ToLowerInvariant()); keys.ExceptWith(_baseDataProperties); keys.ExceptWith(GetPropertiesNames(typeof(QuoteBar), type)); keys.ExceptWith(GetPropertiesNames(typeof(TradeBar), type)); keys.ExceptWith(GetPropertiesNames(typeof(Tick), type)); keys.Add("value"); - _members = members.Where(x => keys.Contains(x[^1].Name.ToLowerInvariant())).ToList(); + _members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant())).ToList(); _membersByType.TryAdd(type, _members); } } @@ -231,40 +209,25 @@ public void Add(object baseData) foreach (var member in _members) { // TODO field/property.GetValue is expensive - var key = member[^1].Name.ToLowerInvariant(); - - var value = baseData; - var memberInfo = (MemberInfo)null; - for (var i = 0; i < member.Length; i++) + var key = member.Name.ToLowerInvariant(); + var propertyMember = member as PropertyInfo; + if (propertyMember != null) { - memberInfo = member[i]; - if (memberInfo is PropertyInfo propertyInfo) - { - value = propertyInfo.GetValue(value); - } - else - { - value = (memberInfo as FieldInfo)?.GetValue(value); - } - - if (value == null) + var propertyValue = propertyMember.GetValue(baseData); + if (_isFundamentalType && propertyMember.PropertyType.IsAssignableTo(typeof(FundamentalTimeDependentProperty))) { - break; + propertyValue = ((FundamentalTimeDependentProperty)propertyValue).Clone(new FixedTimeProvider(endTime)); } + AddToSeries(key, endTime, propertyValue); + continue; } - - if (memberInfo is PropertyInfo propertyMember) + else { - if (_isFundamentalType && propertyMember.PropertyType.IsAssignableTo(typeof(FundamentalTimeDependentProperty))) + var fieldMember = member as FieldInfo; + if (fieldMember != null) { - value = ((FundamentalTimeDependentProperty)value).Clone(new FixedTimeProvider(endTime)); + AddToSeries(key, endTime, fieldMember.GetValue(baseData)); } - AddToSeries(key, endTime, value); - } - else if (memberInfo is FieldInfo fieldMember) - { - AddToSeries(key, endTime, fieldMember.GetValue(baseData)); - } } From 407688deb40bda8711539c3c9ee05f3efd4a2d89 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 24 Sep 2024 17:20:16 -0400 Subject: [PATCH 11/38] Add new QCAlgorithm.OptionChains() method - Use OptionChains as output - Add DataFrame to OptionChain and OptionChains - Rename Greeks classes - Add ISymbolProvider for classes that have a symbol (IBaseData, OptionContract) --- ...lForOptionStylesBaseRegressionAlgorithm.cs | 7 +- Algorithm/QCAlgorithm.cs | 51 ++++- Common/Data/IBaseData.cs | 24 +-- Common/Data/ISymbolProvider.cs | 32 +++ Common/Data/Market/BaseGreeks.cs | 101 ---------- Common/Data/Market/Greeks.cs | 187 ++++++------------ Common/Data/Market/ModeledGreeks.cs | 158 +++++++++++++++ Common/Data/Market/OptionChain.cs | 39 +++- Common/Data/Market/OptionChains.cs | 31 ++- Common/Data/Market/OptionContract.cs | 185 ++++++++++++++--- .../Data/UniverseSelection/OptionUniverse.cs | 16 +- Common/Python/PandasConverter.cs | 86 +++++--- Common/Python/PandasData.cs | 102 +++++++--- Common/QuantConnect.csproj | 2 +- .../Option/CurrentPriceOptionPriceModel.cs | 4 +- .../Option/OptionPriceModelResult.cs | 14 +- .../Securities/Option/QLOptionPriceModel.cs | 2 +- Common/Symbol.cs | 6 +- Engine/DataFeeds/TimeSliceFactory.cs | 55 +----- .../Options/OptionFilterUniverseTests.cs | 2 +- .../OptionPriceModelPriceGeneratorTests.cs | 2 +- 21 files changed, 708 insertions(+), 398 deletions(-) create mode 100644 Common/Data/ISymbolProvider.cs delete mode 100644 Common/Data/Market/BaseGreeks.cs create mode 100644 Common/Data/Market/ModeledGreeks.cs diff --git a/Algorithm.CSharp/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.cs b/Algorithm.CSharp/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.cs index 13d35e5a6f71..37852f18056a 100644 --- a/Algorithm.CSharp/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.cs +++ b/Algorithm.CSharp/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.cs @@ -86,7 +86,7 @@ public void CheckGreeks(OptionChain contracts) foreach (var contract in contracts) { - Greeks greeks = new Greeks(); + Greeks greeks = null; try { greeks = contract.Greeks; @@ -110,6 +110,11 @@ public void CheckGreeks(OptionChain contracts) // Greeks should be valid if they were successfuly accessed for supported option style if (_optionStyleIsSupported) { + if (greeks == null) + { + greeks = new ModeledGreeks(); + } + if (greeks.Delta == 0m && greeks.Gamma == 0m && greeks.Theta == 0m && greeks.Vega == 0m && greeks.Rho == 0m) { throw new RegressionTestException($"Expected greeks to not be zero simultaneously for {contract.Symbol.Value}, an {_option.Style} style option, using {_option?.PriceModel.GetType().Name}, but they were"); diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 65aa6a8d9101..00f2322d63fa 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3356,11 +3356,7 @@ public List Fundamentals(List symbols) /// The symbol for which the option chain is asked for. /// It can be either the canonical option or the underlying symbol. /// - /// - /// The option chain as an enumerable of , - /// each containing the contract symbol along with additional data, including daily price data, - /// implied volatility and greeks. - /// + /// The option chain /// /// As of 2024/09/11, future options chain will not contain any additional data (e.g. daily price data, implied volatility and greeks), /// it will be populated with the contract symbol only. This is expected to change in the future. @@ -3368,6 +3364,7 @@ public List Fundamentals(List symbols) [DocumentationAttribute(AddingData)] public DataHistory OptionChain(Symbol symbol) { + // TODO: Use OptionChains() var canonicalSymbol = GetCanonicalOptionSymbol(symbol); IEnumerable optionChain; @@ -3392,6 +3389,50 @@ public DataHistory OptionChain(Symbol symbol) return new DataHistory(optionChain, new Lazy(() => PandasConverter.GetDataFrame(optionChain))); } + /// + /// Get the option chains for the specified symbols at the current time () + /// + /// + /// The symbols for which the option chain is asked for. + /// It can be either the canonical options or the underlying symbols. + /// + /// The option chains + [DocumentationAttribute(AddingData)] + public OptionChains OptionChains(IEnumerable symbols) + { + // TODO: Future options???????????? + var canonicalSymbols = symbols.Select(GetCanonicalOptionSymbol); + var history = History(canonicalSymbols, 1); + + var time = Time.Date; + var chains = new OptionChains(time); + var dataFrames = new Dictionary(); + foreach (var chainData in history.GetUniverseData()) + { + var symbol = chainData.Keys.Single(); + var contracts = chainData.Values.Single().Cast().ToList(); + + var optionChain = new OptionChain(symbol, time, contracts); + chains.Add(symbol, optionChain); + } + + return chains; + } + + /// + /// Get the option chains for the specified symbols at the current time () + /// + /// + /// The symbols for which the option chain is asked for. + /// It can be either the canonical options or the underlying symbols. + /// + /// The option chains + [DocumentationAttribute(AddingData)] + public OptionChains OptionChains(PyObject symbols) + { + return OptionChains(symbols.ConvertToSymbolEnumerable()); + } + /// /// Register a command type to be used /// diff --git a/Common/Data/IBaseData.cs b/Common/Data/IBaseData.cs index 728f5f20b724..9eaf56a9ff75 100644 --- a/Common/Data/IBaseData.cs +++ b/Common/Data/IBaseData.cs @@ -1,11 +1,11 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); + * + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,7 +21,7 @@ namespace QuantConnect.Data /// /// Base Data Class: Type, Timestamp, Key -- Base Features. /// - public interface IBaseData + public interface IBaseData : ISymbolProvider { /// /// Market Data Type of this data - does it come in individual price packets or is it grouped into OHLC. @@ -31,7 +31,7 @@ MarketDataType DataType get; set; } - + /// /// Time keeper of data -- all data is timeseries based. /// @@ -49,16 +49,6 @@ DateTime EndTime get; set; } - - - /// - /// Symbol for underlying Security - /// - Symbol Symbol - { - get; - set; - } /// @@ -112,7 +102,7 @@ decimal Price BaseData Reader(SubscriptionDataConfig config, StreamReader stream, DateTime date, bool isLiveMode); /// - /// Return the URL string source of the file. This will be converted to a stream + /// Return the URL string source of the file. This will be converted to a stream /// /// Type of datafeed we're reqesting - backtest or live /// Configuration object diff --git a/Common/Data/ISymbolProvider.cs b/Common/Data/ISymbolProvider.cs new file mode 100644 index 000000000000..e2a329ee9366 --- /dev/null +++ b/Common/Data/ISymbolProvider.cs @@ -0,0 +1,32 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Data +{ + /// + /// Base data with a symbol + /// + public interface ISymbolProvider + { + /// + /// Gets the Symbol + /// + Symbol Symbol + { + get; + set; + } + } +} diff --git a/Common/Data/Market/BaseGreeks.cs b/Common/Data/Market/BaseGreeks.cs deleted file mode 100644 index 6374d014ed5e..000000000000 --- a/Common/Data/Market/BaseGreeks.cs +++ /dev/null @@ -1,101 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -namespace QuantConnect.Data.Market -{ - /// - /// Defines the greeks - /// - public abstract class BaseGreeks - { - /// - /// Gets the delta. - /// - /// Delta measures the rate of change of the option value with respect to changes in - /// the underlying asset'sprice. (∂V/∂S) - /// - /// - public abstract decimal Delta { get; protected set; } - - /// - /// Gets the gamma. - /// - /// Gamma measures the rate of change of Delta with respect to changes in - /// the underlying asset'sprice. (∂²V/∂S²) - /// - /// - public abstract decimal Gamma { get; protected set; } - - /// - /// Gets the vega. - /// - /// Vega measures the rate of change of the option value with respect to changes in - /// the underlying's volatility. (∂V/∂σ) - /// - /// - public abstract decimal Vega { get; protected set; } - - /// - /// Gets the theta. - /// - /// Theta measures the rate of change of the option value with respect to changes in - /// time. This is commonly known as the 'time decay.' (∂V/∂τ) - /// - /// - public abstract decimal Theta { get; protected set; } - - /// - /// Gets the rho. - /// - /// Rho measures the rate of change of the option value with respect to changes in - /// the risk free interest rate. (∂V/∂r) - /// - /// - public abstract decimal Rho { get; protected set; } - - /// - /// Gets the lambda. - /// - /// Lambda is the percentage change in option value per percentage change in the - /// underlying's price, a measure of leverage. Sometimes referred to as gearing. - /// (∂V/∂S ✕ S/V) - /// - /// - public abstract decimal Lambda { get; protected set; } - - /// - /// Gets the lambda. - /// - /// Lambda is the percentage change in option value per percentage change in the - /// underlying's price, a measure of leverage. Sometimes referred to as gearing. - /// (∂V/∂S ✕ S/V) - /// - /// - /// - /// Alias for required for compatibility with Python when - /// PEP8 API is used (lambda is a reserved keyword in Python). - /// - public virtual decimal Lambda_ => Lambda; - - /// - /// Gets the theta per day. - /// - /// Theta measures the rate of change of the option value with respect to changes in - /// time. This is commonly known as the 'time decay.' (∂V/∂τ) - /// - /// - public virtual decimal ThetaPerDay => Theta / 365m; - } -} diff --git a/Common/Data/Market/Greeks.cs b/Common/Data/Market/Greeks.cs index fca5b43ec902..25ff19c6bb85 100644 --- a/Common/Data/Market/Greeks.cs +++ b/Common/Data/Market/Greeks.cs @@ -13,146 +13,89 @@ * limitations under the License. */ -using System; - namespace QuantConnect.Data.Market { /// /// Defines the greeks /// - public class Greeks : BaseGreeks + public abstract class Greeks { - private Lazy _delta; - private Lazy _gamma; - private Lazy _vega; - private Lazy _theta; - private Lazy _rho; - private Lazy _lambda; - - // _deltagamma stores gamma and delta combined and is done - // for optimization purposes (approximation of delta and gamma is very similar) - private Lazy> _deltaGamma; - - /// - public override decimal Delta - { - get - { - return _delta != null ? _delta.Value : _deltaGamma.Value.Item1; - } - protected set - { - _delta = new Lazy(() => value); - } - } - - /// - public override decimal Gamma - { - get - { - return _gamma != null ? _gamma.Value : _deltaGamma.Value.Item2; - } - protected set - { - _gamma = new Lazy(() => value); - } - } - - /// - public override decimal Vega - { - get - { - return _vega.Value; - } - protected set - { - _vega = new Lazy(() => value); - } - } + /// + /// Gets the delta. + /// + /// Delta measures the rate of change of the option value with respect to changes in + /// the underlying asset'sprice. (∂V/∂S) + /// + /// + public abstract decimal Delta { get; protected set; } - /// - public override decimal Theta - { - get - { - return _theta.Value; - } - protected set - { - _theta = new Lazy(() => value); - } - } + /// + /// Gets the gamma. + /// + /// Gamma measures the rate of change of Delta with respect to changes in + /// the underlying asset'sprice. (∂²V/∂S²) + /// + /// + public abstract decimal Gamma { get; protected set; } - /// - public override decimal Rho - { - get - { - return _rho.Value; - } - protected set - { - _rho = new Lazy(() => value); - } - } + /// + /// Gets the vega. + /// + /// Vega measures the rate of change of the option value with respect to changes in + /// the underlying's volatility. (∂V/∂σ) + /// + /// + public abstract decimal Vega { get; protected set; } - /// - public override decimal Lambda - { - get - { - return _lambda.Value; - } - protected set - { - _lambda = new Lazy(() => value); - } - } + /// + /// Gets the theta. + /// + /// Theta measures the rate of change of the option value with respect to changes in + /// time. This is commonly known as the 'time decay.' (∂V/∂τ) + /// + /// + public abstract decimal Theta { get; protected set; } /// - /// Initializes a new default instance of the class + /// Gets the rho. + /// + /// Rho measures the rate of change of the option value with respect to changes in + /// the risk free interest rate. (∂V/∂r) + /// /// - public Greeks() - : this(0m, 0m, 0m, 0m, 0m, 0m) - { - } + public abstract decimal Rho { get; protected set; } /// - /// Initializes a new instance of the class + /// Gets the lambda. + /// + /// Lambda is the percentage change in option value per percentage change in the + /// underlying's price, a measure of leverage. Sometimes referred to as gearing. + /// (∂V/∂S ✕ S/V) + /// /// - public Greeks(decimal delta, decimal gamma, decimal vega, decimal theta, decimal rho, decimal lambda) - { - Delta = delta; - Gamma = gamma; - Vega = vega; - Theta = theta; - Rho = rho; - Lambda = lambda; - } + public abstract decimal Lambda { get; protected set; } + /// - /// Initializes a new instance of the class + /// Gets the lambda. + /// + /// Lambda is the percentage change in option value per percentage change in the + /// underlying's price, a measure of leverage. Sometimes referred to as gearing. + /// (∂V/∂S ✕ S/V) + /// /// - public Greeks(Func delta, Func gamma, Func vega, Func theta, Func rho, Func lambda) - { - _delta = new Lazy(delta); - _gamma = new Lazy(gamma); - _vega = new Lazy(vega); - _theta = new Lazy(theta); - _rho = new Lazy(rho); - _lambda = new Lazy(lambda); - } + /// + /// Alias for required for compatibility with Python when + /// PEP8 API is used (lambda is a reserved keyword in Python). + /// + public virtual decimal Lambda_ => Lambda; + /// - /// Initializes a new instance of the class + /// Gets the theta per day. + /// + /// Theta measures the rate of change of the option value with respect to changes in + /// time. This is commonly known as the 'time decay.' (∂V/∂τ) + /// /// - public Greeks(Func> deltaGamma, Func vega, Func theta, Func rho, Func lambda) - { - _deltaGamma = new Lazy>(deltaGamma); - _vega = new Lazy(vega); - _theta = new Lazy(theta); - _rho = new Lazy(rho); - _lambda = new Lazy(lambda); - } + public virtual decimal ThetaPerDay => Theta / 365m; } } diff --git a/Common/Data/Market/ModeledGreeks.cs b/Common/Data/Market/ModeledGreeks.cs new file mode 100644 index 000000000000..8ba57a08d5d1 --- /dev/null +++ b/Common/Data/Market/ModeledGreeks.cs @@ -0,0 +1,158 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; + +namespace QuantConnect.Data.Market +{ + /// + /// Defines the greeks + /// + public class ModeledGreeks : Greeks + { + private Lazy _delta; + private Lazy _gamma; + private Lazy _vega; + private Lazy _theta; + private Lazy _rho; + private Lazy _lambda; + + // _deltagamma stores gamma and delta combined and is done + // for optimization purposes (approximation of delta and gamma is very similar) + private Lazy> _deltaGamma; + + /// + public override decimal Delta + { + get + { + return _delta != null ? _delta.Value : _deltaGamma.Value.Item1; + } + protected set + { + _delta = new Lazy(() => value); + } + } + + /// + public override decimal Gamma + { + get + { + return _gamma != null ? _gamma.Value : _deltaGamma.Value.Item2; + } + protected set + { + _gamma = new Lazy(() => value); + } + } + + /// + public override decimal Vega + { + get + { + return _vega.Value; + } + protected set + { + _vega = new Lazy(() => value); + } + } + + /// + public override decimal Theta + { + get + { + return _theta.Value; + } + protected set + { + _theta = new Lazy(() => value); + } + } + + /// + public override decimal Rho + { + get + { + return _rho.Value; + } + protected set + { + _rho = new Lazy(() => value); + } + } + + /// + public override decimal Lambda + { + get + { + return _lambda.Value; + } + protected set + { + _lambda = new Lazy(() => value); + } + } + + /// + /// Initializes a new default instance of the class + /// + public ModeledGreeks() + : this(0m, 0m, 0m, 0m, 0m, 0m) + { + } + + /// + /// Initializes a new instance of the class + /// + public ModeledGreeks(decimal delta, decimal gamma, decimal vega, decimal theta, decimal rho, decimal lambda) + { + Delta = delta; + Gamma = gamma; + Vega = vega; + Theta = theta; + Rho = rho; + Lambda = lambda; + } + /// + /// Initializes a new instance of the class + /// + public ModeledGreeks(Func delta, Func gamma, Func vega, Func theta, Func rho, Func lambda) + { + _delta = new Lazy(delta); + _gamma = new Lazy(gamma); + _vega = new Lazy(vega); + _theta = new Lazy(theta); + _rho = new Lazy(rho); + _lambda = new Lazy(lambda); + } + /// + /// Initializes a new instance of the class + /// + public ModeledGreeks(Func> deltaGamma, Func vega, Func theta, Func rho, Func lambda) + { + _deltaGamma = new Lazy>(deltaGamma); + _vega = new Lazy(vega); + _theta = new Lazy(theta); + _rho = new Lazy(rho); + _lambda = new Lazy(lambda); + } + } +} diff --git a/Common/Data/Market/OptionChain.cs b/Common/Data/Market/OptionChain.cs index b4b54e47e801..6845e3feac62 100644 --- a/Common/Data/Market/OptionChain.cs +++ b/Common/Data/Market/OptionChain.cs @@ -17,6 +17,9 @@ using System.Collections; using System.Collections.Generic; using System.Linq; +using Python.Runtime; +using QuantConnect.Data.UniverseSelection; +using QuantConnect.Python; using QuantConnect.Securities.Option; using QuantConnect.Util; @@ -29,6 +32,7 @@ namespace QuantConnect.Data.Market public class OptionChain : BaseData, IEnumerable { private readonly Dictionary>> _auxiliaryData = new Dictionary>>(); + private readonly Lazy _dataframe; /// /// Gets the most recent trade information for the underlying. This may @@ -79,12 +83,18 @@ public HashSet FilteredContracts get; private set; } + /// + /// The data frame representation of the option chain + /// + public PyObject DataFrame => _dataframe.Value; + /// /// Initializes a new default instance of the class /// private OptionChain() { DataType = MarketDataType.OptionChain; + _dataframe = new Lazy(() => new PandasConverter().GetDataFrame(this, symbolOnlyIndex: true)); } /// @@ -93,6 +103,7 @@ private OptionChain() /// The symbol for this chain. /// The time of this chain public OptionChain(Symbol canonicalOptionSymbol, DateTime time) + : this() { Time = time; Symbol = canonicalOptionSymbol; @@ -116,6 +127,7 @@ public OptionChain(Symbol canonicalOptionSymbol, DateTime time) /// All contracts for this option chain /// The filtered list of contracts for this option chain public OptionChain(Symbol canonicalOptionSymbol, DateTime time, BaseData underlying, IEnumerable trades, IEnumerable quotes, IEnumerable contracts, IEnumerable filteredContracts) + : this() { Time = time; Underlying = underlying; @@ -176,6 +188,31 @@ public OptionChain(Symbol canonicalOptionSymbol, DateTime time, BaseData underly } } + /// + /// Initializes a new option chain for a list of contracts as instances + /// + /// The canonical option symbol + /// The time of this chain + /// The list of contracts data + public OptionChain(Symbol canonicalOptionSymbol, DateTime time, List contracts) + : this(canonicalOptionSymbol, time) + { + Time = time; + Underlying = contracts.FirstOrDefault()?.Underlying; + Symbol = canonicalOptionSymbol; + DataType = MarketDataType.OptionChain; + + Ticks = new Ticks(time); + TradeBars = new TradeBars(time); + QuoteBars = new QuoteBars(time); + Contracts = new OptionContracts(time); + + foreach (var contractData in contracts) + { + Contracts[contractData.Symbol] = OptionContract.Create(contractData); + } + } + /// /// Gets the auxiliary data with the specified type and symbol /// @@ -315,4 +352,4 @@ internal void AddAuxData(BaseData baseData) list.Add(baseData); } } -} \ No newline at end of file +} diff --git a/Common/Data/Market/OptionChains.cs b/Common/Data/Market/OptionChains.cs index c1ad7422af19..f73800f75ce3 100644 --- a/Common/Data/Market/OptionChains.cs +++ b/Common/Data/Market/OptionChains.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -13,7 +13,10 @@ * limitations under the License. */ +using Python.Runtime; +using QuantConnect.Python; using System; +using System.Linq; namespace QuantConnect.Data.Market { @@ -22,11 +25,14 @@ namespace QuantConnect.Data.Market /// public class OptionChains : DataDictionary { + private readonly Lazy _dataframe; + /// /// Creates a new instance of the dictionary /// public OptionChains() { + _dataframe = new Lazy(InitializeDataFrame); } /// @@ -35,8 +41,14 @@ public OptionChains() public OptionChains(DateTime time) : base(time) { + _dataframe = new Lazy(InitializeDataFrame); } + /// + /// The data frame representation of the option chains + /// + public PyObject DataFrame => _dataframe.Value; + /// /// Gets or sets the OptionChain with the specified ticker. /// @@ -56,5 +68,20 @@ public OptionChains(DateTime time) /// The Symbol of the element to get or set. /// Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations public new OptionChain this[Symbol symbol] { get { return base[symbol]; } set { base[symbol] = value; } } + + private PyObject InitializeDataFrame() + { + var dataFrames = this.Select(kvp => kvp.Value.DataFrame).ToList(); + var canonicalSymbols = this.Select(kvp => kvp.Key); + + var result = PandasConverter.ConcatDataFrames(dataFrames, keys: canonicalSymbols, names: new[] { "canonical", "symbol" }); + + foreach (var df in dataFrames) + { + df.Dispose(); + } + + return result; + } } -} \ No newline at end of file +} diff --git a/Common/Data/Market/OptionContract.cs b/Common/Data/Market/OptionContract.cs index 1c4bdef042fc..2c98669b9de1 100644 --- a/Common/Data/Market/OptionContract.cs +++ b/Common/Data/Market/OptionContract.cs @@ -13,26 +13,34 @@ * limitations under the License. */ +using QuantConnect.Data.UniverseSelection; using QuantConnect.Interfaces; using QuantConnect.Securities; using QuantConnect.Securities.Option; using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; namespace QuantConnect.Data.Market { /// /// Defines a single option contract at a specific expiration and strike price /// - public class OptionContract + public class OptionContract : ISymbolProvider { + private static readonly SymbolPropertiesDatabase _symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder(); + private Lazy _optionPriceModelResult = new(() => OptionPriceModelResult.None); + private readonly List _data = new(); + /// /// Gets the option contract's symbol /// public Symbol Symbol { - get; private set; + get; set; } /// @@ -98,33 +106,45 @@ public DateTime Time /// /// Gets the open interest /// - public decimal OpenInterest - { - get; set; - } + public decimal OpenInterest => GetLastOpenInterest()?.Value ?? decimal.Zero; /// /// Gets the last price this contract traded at /// - public decimal LastPrice + public decimal LastPrice => GetLastTrades().LastOrDefault() switch { - get; set; - } + Tick tick => tick.LastPrice, + TradeBar tradeBar => tradeBar.Close, + _ => decimal.Zero + }; /// /// Gets the last volume this contract traded at /// - public long Volume - { - get; set; - } + public long Volume => (long)(GetLastTradeBar()?.Volume ?? 0L); /// /// Gets the current bid price /// public decimal BidPrice { - get; set; + get + { + foreach (var data in GetLastQuotes()) + { + if (data is Tick tick && tick.BidPrice != 0) + { + return tick.BidPrice; + } + + if (data is QuoteBar quoteBar && quoteBar.Bid != null && quoteBar.Bid.Close != 0) + { + return quoteBar.Bid.Close; + } + } + + return decimal.Zero; + } } /// @@ -132,7 +152,23 @@ public decimal BidPrice /// public long BidSize { - get; set; + get + { + foreach (var data in GetLastQuotes()) + { + if (data is Tick tick && tick.BidPrice != 0) + { + return (long)tick.BidSize; + } + + if (data is QuoteBar quoteBar && quoteBar.Bid != null && quoteBar.Bid.Close != 0) + { + return (long)quoteBar.LastBidSize; + } + } + + return 0; + } } /// @@ -140,7 +176,23 @@ public long BidSize /// public decimal AskPrice { - get; set; + get + { + foreach (var data in GetLastQuotes()) + { + if (data is Tick tick && tick.AskPrice != 0) + { + return tick.AskPrice; + } + + if (data is QuoteBar quoteBar && quoteBar.Ask != null && quoteBar.Ask.Close != 0) + { + return quoteBar.Ask.Close; + } + } + + return decimal.Zero; + } } /// @@ -148,7 +200,23 @@ public decimal AskPrice /// public long AskSize { - get; set; + get + { + foreach (var data in GetLastQuotes()) + { + if (data is Tick tick && tick.AskPrice != 0) + { + return (long)tick.AskSize; + } + + if (data is QuoteBar quoteBar && quoteBar.Ask != null && quoteBar.Ask.Close != 0) + { + return (long)quoteBar.LastAskSize; + } + } + + return 0; + } } /// @@ -171,6 +239,25 @@ public OptionContract(ISecurityPrice security, Symbol underlyingSymbol) ScaledStrike = Strike * security.SymbolProperties.StrikeMultiplier; } + /// + /// Initializes a new option contract from a given instance + /// + /// The option universe contract data to use as source for this contract + public OptionContract(OptionUniverse contractData) + { + Symbol = contractData.Symbol; + UnderlyingSymbol = contractData.Symbol.Underlying; + + // TODO: What about the strike multiplier if no security is provided? Should we access the spdb directly? + var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties( + contractData.Symbol.ID.Market, + contractData.Symbol, + contractData.Symbol.SecurityType, + // What should the default be? We don't have access to the account currency here + Currencies.USD); + ScaledStrike = Strike * symbolProperties.StrikeMultiplier; + } + /// /// Sets the option price model evaluator function to be used for this contract /// @@ -189,7 +276,7 @@ internal void SetOptionPriceModel(Func optionPriceModelE public override string ToString() => Symbol.Value; /// - /// Creates a + /// Creates a /// /// /// provides price properties for a @@ -211,15 +298,63 @@ public static OptionContract Create(Symbol underlyingSymbol, DateTime endTime, I return new OptionContract(security, underlyingSymbol) { Time = endTime, - LastPrice = security.Close, - Volume = (long)security.Volume, - BidPrice = security.BidPrice, - BidSize = (long)security.BidSize, - AskPrice = security.AskPrice, - AskSize = (long)security.AskSize, - OpenInterest = security.OpenInterest, UnderlyingLastPrice = underlyingLastPrice }; } + + /// + /// Creates a new option contract from a given instance, + /// using its data to form a quote bar to source pricing data + /// + /// The option universe contract data to use as source for this contract + public static OptionContract Create(OptionUniverse contractData) + { + var contract = new OptionContract(contractData) + { + Time = contractData.EndTime, + }; + + var bar = new Bar(contractData.Open, contractData.High, contractData.Low, contractData.Close); + var quoteBar = new QuoteBar(contractData.Time, contractData.Symbol, bar, 0, bar, 0) + { + EndTime = contractData.EndTime, + }; + + contract.Update(quoteBar); + + return contract; + } + + /// + /// Updates the option contract with the new data, which can be a or or + /// + internal void Update(BaseData data) + { + _data.Add(data); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private IEnumerable GetLastTrades() + { + return _data.Where(x => x is TradeBar || (x is Tick tick && tick.TickType == TickType.Trade)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private TradeBar GetLastTradeBar() + { + return _data.LastOrDefault(x => x is TradeBar) as TradeBar; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private IEnumerable GetLastQuotes() + { + return _data.Where(x => x is QuoteBar || (x is Tick tick && tick.TickType == TickType.Quote)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private BaseData GetLastOpenInterest() + { + return _data.LastOrDefault(x => x is Tick tick && tick.TickType == TickType.OpenInterest); + } } } diff --git a/Common/Data/UniverseSelection/OptionUniverse.cs b/Common/Data/UniverseSelection/OptionUniverse.cs index 46a1c97c6892..9952e6b38444 100644 --- a/Common/Data/UniverseSelection/OptionUniverse.cs +++ b/Common/Data/UniverseSelection/OptionUniverse.cs @@ -125,7 +125,7 @@ public decimal ImpliedVolatility /// /// Greeks values of the option /// - public BaseGreeks Greeks + public Greeks Greeks { get { @@ -135,27 +135,27 @@ public BaseGreeks Greeks } /// - /// Gets the delta. Pass-through for + /// Gets the delta. Pass-through for /// public decimal Delta => Greeks.Delta; /// - /// Gets the gamma. Pass-through for + /// Gets the gamma. Pass-through for /// public decimal Gamma => Greeks.Gamma; /// - /// Gets the vega. Pass-through for + /// Gets the vega. Pass-through for /// public decimal Vega => Greeks.Vega; /// - /// Gets the theta. Pass-through for + /// Gets the theta. Pass-through for /// public decimal Theta => Greeks.Theta; /// - /// Gets the rho. Pass-through for + /// Gets the rho. Pass-through for /// public decimal Rho => Greeks.Rho; @@ -315,7 +315,7 @@ public override Resolution DefaultResolution() /// Gets the CSV string representation of this universe entry /// public static string ToCsv(Symbol symbol, decimal open, decimal high, decimal low, decimal close, decimal volume, decimal? openInterest, - decimal? impliedVolatility, BaseGreeks greeks) + decimal? impliedVolatility, Greeks greeks) { return $"{symbol.ID},{symbol.Value},{open},{high},{low},{close},{volume}," + $"{openInterest},{impliedVolatility},{greeks?.Delta},{greeks?.Gamma},{greeks?.Vega},{greeks?.Theta},{greeks?.Rho}"; @@ -356,7 +356,7 @@ private void ThrowIfNotAnOption(string propertyName) /// Pre-calculated greeks lazily parsed from csv line. /// It parses the greeks values from the csv line only when they are requested to avoid holding decimals in memory. /// - private class PreCalculatedGreeks : BaseGreeks + private class PreCalculatedGreeks : Greeks { private readonly string _csvLine; diff --git a/Common/Python/PandasConverter.cs b/Common/Python/PandasConverter.cs index 753c131b7008..af27405846bc 100644 --- a/Common/Python/PandasConverter.cs +++ b/Common/Python/PandasConverter.cs @@ -35,17 +35,14 @@ public class PandasConverter /// /// Creates an instance of . /// - public PandasConverter() + static PandasConverter() { - if (_pandas == null) + using (Py.GIL()) { - using (Py.GIL()) - { - var pandas = Py.Import("pandas"); - _pandas = pandas; - // keep it so we don't need to ask for it each time - _concat = pandas.GetAttr("concat"); - } + var pandas = Py.Import("pandas"); + _pandas = pandas; + // keep it so we don't need to ask for it each time + _concat = pandas.GetAttr("concat"); } } @@ -77,24 +74,23 @@ public PyObject GetDataFrame(IEnumerable data, Type dataType = null) /// Converts an enumerable of in a pandas.DataFrame /// /// Enumerable of + /// Whether to make the index only the symbol, without time or any other index levels /// containing a pandas.DataFrame /// Helper method for testing - public PyObject GetDataFrame(IEnumerable data) - where T : IBaseData + public PyObject GetDataFrame(IEnumerable data, bool symbolOnlyIndex = false) + where T : ISymbolProvider { var pandasDataBySymbol = new Dictionary(); + var maxLevels = 0; foreach (var datum in data) { - if (!pandasDataBySymbol.TryGetValue(datum.Symbol.ID, out var pandasData)) - { - pandasData = new PandasData(datum); - pandasDataBySymbol[datum.Symbol.ID] = pandasData; - } - + var pandasData = GetPandasDataValue(pandasDataBySymbol, datum.Symbol, datum, ref maxLevels); pandasData.Add(datum); } return CreateDataFrame(pandasDataBySymbol, + // Use 2 instead of maxLevels for backwards compatibility + maxLevels: symbolOnlyIndex ? 1 : 2, sort: false, // Multiple data frames (one for each symbol) will be concatenated, // so make sure rows with missing values only are not filtered out before concatenation @@ -192,18 +188,60 @@ private static PyObject CreateDataFrame(Dictionary x.Value.ToPandasDataFrame(maxLevels, filterMissingValueColumns)).ToPyListUnSafe(); - using var sortDic = Py.kw("sort", sort); - var result = _concat.Invoke(new[] { dataFrames }, sortDic); - - // Drop columns with only NaN or None values - using var kwargs = Py.kw("axis", 1, "inplace", true, "how", "all"); - result.GetAttr("dropna").Invoke(Array.Empty(), kwargs); + var dataFrames = dataBySymbol.Select(x => x.Value.ToPandasDataFrame(maxLevels, filterMissingValueColumns)); + var result = ConcatDataFrames(dataFrames, sort: sort, dropna: true); foreach (var df in dataFrames) { df.Dispose(); } + + return result; + } + } + + /// + /// Concatenates multiple data frames + /// + /// The data frames to concatenate + /// + /// Optional new keys for a new multi-index level that would be added + /// to index each individual data frame in the resulting one + /// + /// The optional names of the new index level (and the existing ones if they need to be changed) + /// Whether to sort the resulting data frame + /// Whether to drop columns containing NA values only (Nan, None, etc) + /// A new data frame result from concatenating the input + public static PyObject ConcatDataFrames(IEnumerable dataFrames, IEnumerable keys = null, IEnumerable names = null, + bool sort = true, bool dropna = true) + { + using (Py.GIL()) + { + using var pyDataFrames = dataFrames.ToPyListUnSafe(); + using var kwargs = Py.kw("sort", sort); + PyList pyKeys = null; + PyList pyNames = null; + + if (keys != null && names != null) + { + pyKeys = keys.ToPyListUnSafe(); + pyNames = names.ToPyListUnSafe(); + kwargs.SetItem("keys", pyKeys); + kwargs.SetItem("names", pyNames); + } + + var result = _concat.Invoke(new[] { pyDataFrames }, kwargs); + + // Drop columns with only NaN or None values + if (dropna) + { + using var dropnaKwargs = Py.kw("axis", 1, "inplace", true, "how", "all"); + result.GetAttr("dropna").Invoke(Array.Empty(), dropnaKwargs); + } + + pyKeys?.Dispose(); + pyNames?.Dispose(); + return result; } } diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 8eaedde541f7..8633f8e3d333 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -60,6 +60,10 @@ public class PandasData private const string Suspicious = "suspicious"; private const string OpenInterest = "openinterest"; + private const string Expiry = "expiry"; + private const string Strike = "strike"; + private const string Right = "right"; + private static readonly string[] _optionUniverseExcludedProperties = new[] { nameof(OptionUniverse.ID), @@ -74,12 +78,13 @@ public class PandasData private static PyObject _multiIndexFactory; private static PyList _defaultNames; + private static PyList _level1Names; private static PyList _level2Names; private static PyList _level3Names; private readonly static HashSet _baseDataProperties = typeof(BaseData).GetProperties().ToHashSet(x => x.Name.ToLowerInvariant()); - private readonly static ConcurrentDictionary> _membersByType = new (); - private readonly static IReadOnlyList _standardColumns = new string [] + private readonly static ConcurrentDictionary> _membersByType = new(); + private readonly static IReadOnlyList _standardColumns = new string[] { Open, High, Low, Close, LastPrice, Volume, AskOpen, AskHigh, AskLow, AskClose, AskPrice, AskSize, Quantity, Suspicious, @@ -123,14 +128,17 @@ public PandasData(object data) var symbol = new PyString("symbol"); var expiry = new PyString("expiry"); _defaultNames = new PyList(new PyObject[] { expiry, new PyString("strike"), new PyString("type"), symbol, time }); + _level1Names = new PyList(new PyObject[] { symbol }); _level2Names = new PyList(new PyObject[] { symbol, time }); _level3Names = new PyList(new PyObject[] { expiry, symbol, time }); } } + var baseData = data as IBaseData; + // in the case we get a list/collection of data we take the first data point to determine the type // but it's also possible to get a data which supports enumerating we don't care about those cases - if (data is not IBaseData && data is IEnumerable enumerable) + if (baseData == null && data is IEnumerable enumerable) { foreach (var item in enumerable) { @@ -141,15 +149,25 @@ public PandasData(object data) var type = data.GetType(); _isFundamentalType = type == typeof(Fundamental); - _symbol = ((IBaseData)data).Symbol; + _symbol = ((ISymbolProvider)data).Symbol; IsCustomData = Extensions.IsCustomDataType(_symbol, type); - if (_symbol.SecurityType == SecurityType.Future) Levels = 3; - if (_symbol.SecurityType.IsOption()) Levels = 5; + if (baseData == null) + { + Levels = 1; + } + else if (_symbol.SecurityType == SecurityType.Future) + { + Levels = 3; + } + else if (_symbol.SecurityType.IsOption()) + { + Levels = 5; + } IEnumerable columns = _standardColumns; - if (IsCustomData || ((IBaseData)data).DataType == MarketDataType.Auxiliary) + if (IsCustomData || baseData == null || baseData.DataType == MarketDataType.Auxiliary) { var keys = (data as DynamicData)?.GetStorageDictionary() // if this is a PythonData instance we add in '__typename' which we don't want into the data frame @@ -158,13 +176,14 @@ public PandasData(object data) // C# types that are not DynamicData type if (keys == null) { + var isOptionUniverse = type == typeof(OptionUniverse); + if (_membersByType.TryGetValue(type, out _members)) { keys = _members.ToHashSet(x => x.Name.ToLowerInvariant()); } else { - var isOptionUniverse = type == typeof(OptionUniverse); var members = type .GetMembers(BindingFlags.Instance | BindingFlags.Public) .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) && @@ -188,6 +207,13 @@ public PandasData(object data) _members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant())).ToList(); _membersByType.TryAdd(type, _members); } + + if (isOptionUniverse) + { + keys.Add(Expiry); + keys.Add(Strike); + keys.Add(Right); + } } var customColumns = new HashSet(columns) { "value" }; @@ -205,7 +231,12 @@ public PandasData(object data) /// object that contains security data public void Add(object baseData) { - var endTime = ((IBaseData)baseData).EndTime; + var endTime = default(DateTime); + if (baseData is IBaseData iBaseData) + { + endTime = iBaseData.EndTime; + } + foreach (var member in _members) { // TODO field/property.GetValue is expensive @@ -234,7 +265,7 @@ public void Add(object baseData) var storage = (baseData as DynamicData)?.GetStorageDictionary(); if (storage != null) { - var value = ((IBaseData) baseData).Value; + var value = ((IBaseData)baseData).Value; AddToSeries("value", endTime, value); foreach (var kvp in storage.Where(x => x.Key != "value" @@ -246,18 +277,27 @@ public void Add(object baseData) } else { - var tick = baseData as Tick; - if (tick != null) - { - AddTick(tick); - } - else - { - var tradeBar = baseData as TradeBar; - var quoteBar = baseData as QuoteBar; - Add(tradeBar, quoteBar); - } + AddTick(baseData as Tick); + + var tradeBar = baseData as TradeBar; + var quoteBar = baseData as QuoteBar; + Add(tradeBar, quoteBar); + + AddOptionData(baseData as OptionUniverse); + } + } + + private void AddOptionData(OptionUniverse optionUniverse) + { + if (optionUniverse == null) + { + return; } + + var time = optionUniverse.EndTime; + GetSerie(Expiry).Add(time, optionUniverse.Symbol.ID.Date); + GetSerie(Strike).Add(time, optionUniverse.Symbol.ID.StrikePrice); + GetSerie(Right).Add(time, optionUniverse.Symbol.ID.OptionRight); } /// @@ -311,6 +351,11 @@ public void Add(TradeBar tradeBar, QuoteBar quoteBar) /// object that contains tick information of the security public void AddTick(Tick tick) { + if (tick == null) + { + return; + } + var time = tick.EndTime; // We will fill some series with null for tick types that don't have a value for that series, so that we make sure @@ -361,7 +406,13 @@ public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns // Create the index labels var names = _defaultNames; - if (levels == 2) + + if (levels == 1) + { + names = _level1Names; + list = new List { symbol }; + } + else if (levels == 2) { // symbol, time names = _level2Names; @@ -457,8 +508,11 @@ private static void DisposeIfNotEmpty(PyObject pyObject) /// private static PyTuple CreateTupleIndex(DateTime index, List list) { - DisposeIfNotEmpty(list[list.Count - 1]); - list[list.Count - 1] = index.ToPython(); + if (list.Count > 1) + { + DisposeIfNotEmpty(list[list.Count - 1]); + list[list.Count - 1] = index.ToPython(); + } return new PyTuple(list.ToArray()); } diff --git a/Common/QuantConnect.csproj b/Common/QuantConnect.csproj index 262200725a5a..4218bf22f991 100644 --- a/Common/QuantConnect.csproj +++ b/Common/QuantConnect.csproj @@ -35,7 +35,7 @@ - + diff --git a/Common/Securities/Option/CurrentPriceOptionPriceModel.cs b/Common/Securities/Option/CurrentPriceOptionPriceModel.cs index 572aa7a50f58..e337a0acbd20 100644 --- a/Common/Securities/Option/CurrentPriceOptionPriceModel.cs +++ b/Common/Securities/Option/CurrentPriceOptionPriceModel.cs @@ -28,7 +28,7 @@ public class CurrentPriceOptionPriceModel : IOptionPriceModel { /// /// Creates a new containing the current - /// and a default, empty instance of first Order + /// and a default, empty instance of first Order /// /// The option security object /// The current data slice. This can be used to access other information @@ -38,7 +38,7 @@ public class CurrentPriceOptionPriceModel : IOptionPriceModel /// price of the specified option contract public OptionPriceModelResult Evaluate(Security security, Slice slice, OptionContract contract) { - return new OptionPriceModelResult(security.Price, new Greeks()); + return new OptionPriceModelResult(security.Price, new ModeledGreeks()); } } } \ No newline at end of file diff --git a/Common/Securities/Option/OptionPriceModelResult.cs b/Common/Securities/Option/OptionPriceModelResult.cs index 53e8239ecd88..f01e0cb880c8 100644 --- a/Common/Securities/Option/OptionPriceModelResult.cs +++ b/Common/Securities/Option/OptionPriceModelResult.cs @@ -26,9 +26,9 @@ public class OptionPriceModelResult /// /// Represents the zero option price and greeks. /// - public static OptionPriceModelResult None { get; } = new(0, new Greeks()); + public static OptionPriceModelResult None { get; } = new(0, new ModeledGreeks()); - private readonly Lazy _greeks; + private readonly Lazy _greeks; private readonly Lazy _impliedVolatility; /// @@ -53,7 +53,7 @@ public decimal ImpliedVolatility /// /// Gets the various sensitivities as computed by the /// - public Greeks Greeks + public ModeledGreeks Greeks { get { @@ -66,11 +66,11 @@ public Greeks Greeks /// /// The theoretical price computed by the price model /// The sensitivities (greeks) computed by the price model - public OptionPriceModelResult(decimal theoreticalPrice, Greeks greeks) + public OptionPriceModelResult(decimal theoreticalPrice, ModeledGreeks greeks) { TheoreticalPrice = theoreticalPrice; _impliedVolatility = new Lazy(() => 0m); - _greeks = new Lazy(() => greeks); + _greeks = new Lazy(() => greeks); } /// @@ -79,11 +79,11 @@ public OptionPriceModelResult(decimal theoreticalPrice, Greeks greeks) /// The theoretical price computed by the price model /// The calculated implied volatility /// The sensitivities (greeks) computed by the price model - public OptionPriceModelResult(decimal theoreticalPrice, Func impliedVolatility, Func greeks) + public OptionPriceModelResult(decimal theoreticalPrice, Func impliedVolatility, Func greeks) { TheoreticalPrice = theoreticalPrice; _impliedVolatility = new Lazy(impliedVolatility); - _greeks = new Lazy(greeks); + _greeks = new Lazy(greeks); } } } diff --git a/Common/Securities/Option/QLOptionPriceModel.cs b/Common/Securities/Option/QLOptionPriceModel.cs index 41793277c892..456627c53ac8 100644 --- a/Common/Securities/Option/QLOptionPriceModel.cs +++ b/Common/Securities/Option/QLOptionPriceModel.cs @@ -289,7 +289,7 @@ decimal tryGetGreekOrReevaluate(Func greek, Func impliedVol.IsNaNOrInfinity() ? 0m : impliedVol.SafeDecimalCast(), - () => new Greeks(() => tryGetGreekOrReevaluate(() => option.delta(), (black) => black.delta(spot)), + () => new ModeledGreeks(() => tryGetGreekOrReevaluate(() => option.delta(), (black) => black.delta(spot)), () => tryGetGreekOrReevaluate(() => option.gamma(), (black) => black.gamma(spot)), () => tryGetGreekOrReevaluate(() => option.vega(), (black) => black.vega(maturity)) / 100, // per cent () => tryGetGreekOrReevaluate(() => option.theta(), (black) => black.theta(spot, maturity)), diff --git a/Common/Symbol.cs b/Common/Symbol.cs index a81e5af8197b..709d5d786fb2 100644 --- a/Common/Symbol.cs +++ b/Common/Symbol.cs @@ -756,9 +756,8 @@ public bool Equals(Symbol other) return true; } - var rightStr = right as string; // Use the implicit conversion for strings - if (rightStr != null) + if (right is not Symbol && right is string rightStr) { return left == (Symbol)rightStr; } @@ -776,7 +775,8 @@ public bool Equals(Symbol other) /// as an object instead of using the implicit conversion public static bool operator ==(object left, Symbol right) { - return right.Equals(left); + // We already have an implementation for (Symbol left, object left), we can reuse it by inverting the operands + return right == left; } /// diff --git a/Engine/DataFeeds/TimeSliceFactory.cs b/Engine/DataFeeds/TimeSliceFactory.cs index 9a831cb664fc..b33a353ac15b 100644 --- a/Engine/DataFeeds/TimeSliceFactory.cs +++ b/Engine/DataFeeds/TimeSliceFactory.cs @@ -481,19 +481,19 @@ private bool HandleOptionData(DateTime algorithmTime, BaseData baseData, OptionC case MarketDataType.Tick: var tick = (Tick)baseData; chain.Ticks.Add(tick.Symbol, tick); - UpdateContract(contract, tick); + contract.Update(tick); break; case MarketDataType.TradeBar: var tradeBar = (TradeBar)baseData; chain.TradeBars[symbol] = tradeBar; - UpdateContract(contract, tradeBar); + contract.Update(tradeBar); break; case MarketDataType.QuoteBar: var quote = (QuoteBar)baseData; chain.QuoteBars[symbol] = quote; - UpdateContract(contract, quote); + contract.Update(quote); break; case MarketDataType.Base: @@ -572,55 +572,6 @@ private bool HandleFuturesData(DateTime algorithmTime, BaseData baseData, Future return true; } - private static void UpdateContract(OptionContract contract, QuoteBar quote) - { - if (quote.Ask != null && quote.Ask.Close != 0m) - { - contract.AskPrice = quote.Ask.Close; - contract.AskSize = (long)quote.LastAskSize; - } - if (quote.Bid != null && quote.Bid.Close != 0m) - { - contract.BidPrice = quote.Bid.Close; - contract.BidSize = (long)quote.LastBidSize; - } - } - - private static void UpdateContract(OptionContract contract, Tick tick) - { - if (tick.TickType == TickType.Trade) - { - contract.LastPrice = tick.Price; - } - else if (tick.TickType == TickType.Quote) - { - if (tick.AskPrice != 0m) - { - contract.AskPrice = tick.AskPrice; - contract.AskSize = (long)tick.AskSize; - } - if (tick.BidPrice != 0m) - { - contract.BidPrice = tick.BidPrice; - contract.BidSize = (long)tick.BidSize; - } - } - else if (tick.TickType == TickType.OpenInterest) - { - if (tick.Value != 0m) - { - contract.OpenInterest = tick.Value; - } - } - } - - private static void UpdateContract(OptionContract contract, TradeBar tradeBar) - { - if (tradeBar.Close == 0m) return; - contract.LastPrice = tradeBar.Close; - contract.Volume = (long)tradeBar.Volume; - } - private static void UpdateContract(FuturesContract contract, QuoteBar quote) { if (quote.Ask != null && quote.Ask.Close != 0m) diff --git a/Tests/Common/Securities/Options/OptionFilterUniverseTests.cs b/Tests/Common/Securities/Options/OptionFilterUniverseTests.cs index 2fa62bf0f12c..d367deb03aea 100644 --- a/Tests/Common/Securities/Options/OptionFilterUniverseTests.cs +++ b/Tests/Common/Securities/Options/OptionFilterUniverseTests.cs @@ -196,7 +196,7 @@ private OptionUniverse GetContractData(Symbol contract) return _testOptionsData.Single(x => x.Symbol == contract); } - private BaseGreeks GetGreeks(Symbol contract) + private Greeks GetGreeks(Symbol contract) { return GetContractData(contract).Greeks; } diff --git a/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs b/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs index 3c8f85eb6fcd..359d759e10f3 100644 --- a/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs +++ b/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs @@ -97,7 +97,7 @@ public void ReturnsNewPrice() var priceModelMock = new Mock(); priceModelMock .Setup(s => s.Evaluate(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new OptionPriceModelResult(1000, new Greeks())); + .Returns(new OptionPriceModelResult(1000, new ModeledGreeks())); _option.PriceModel = priceModelMock.Object; var randomPriceGenerator = new OptionPriceModelPriceGenerator(_option); From b96a9792ca6902f54942a43fe5cdd630091daba3 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 25 Sep 2024 12:07:31 -0400 Subject: [PATCH 12/38] Unify QCAlgorithmOptionChain API Also refactor OptionContract to handle: (1) Actual market data and option price model data, and (2) OptionUniverse data --- ...ContractFromUniverseRegressionAlgorithm.cs | 2 +- .../OptionChainFullDataRegressionAlgorithm.cs | 5 +- ...lForOptionStylesBaseRegressionAlgorithm.py | 9 +- Algorithm/QCAlgorithm.cs | 55 ++- Common/Data/Market/OptionChain.cs | 2 +- Common/Data/Market/OptionContract.cs | 317 ++++++++++-------- Engine/DataFeeds/TimeSliceFactory.cs | 4 +- .../Securities/OptionPriceModelTests.cs | 10 +- .../FedRateQLRiskFreeRateEstimatorTests.cs | 6 +- .../OptionPriceModelPriceGenerator.cs | 3 +- 10 files changed, 217 insertions(+), 196 deletions(-) diff --git a/Algorithm.CSharp/AddOptionContractFromUniverseRegressionAlgorithm.cs b/Algorithm.CSharp/AddOptionContractFromUniverseRegressionAlgorithm.cs index 88c72be803f0..320fef26f12c 100644 --- a/Algorithm.CSharp/AddOptionContractFromUniverseRegressionAlgorithm.cs +++ b/Algorithm.CSharp/AddOptionContractFromUniverseRegressionAlgorithm.cs @@ -117,7 +117,7 @@ public override void OnSecuritiesChanged(SecurityChanges changes) && optionContract.ID.OptionStyle == OptionStyle.American); AddOptionContract(option); - foreach (var symbol in new[] { option.Symbol, option.Underlying.Symbol }) + foreach (var symbol in new[] { option.Symbol, option.UnderlyingSymbol }) { var config = SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(symbol).ToList(); diff --git a/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs b/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs index bee130277f25..23dce11319e0 100644 --- a/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs +++ b/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs @@ -44,10 +44,11 @@ public override void Initialize() // Get contracts expiring within 10 days, with an implied volatility greater than 0.5 and a delta less than 0.5 .Where(contractData => contractData.Symbol.ID.Date - Time <= TimeSpan.FromDays(10) && contractData.ImpliedVolatility > 0.5m && - contractData.Delta < 0.5m) + contractData.Greeks.Delta < 0.5m) // Get the contract with the latest expiration date .OrderByDescending(x => x.Symbol.ID.Date) - .First(); + .First() + .Symbol; AddOptionContract(_optionContract); } diff --git a/Algorithm.Python/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.py b/Algorithm.Python/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.py index 37907f3ca573..de43f774c95f 100644 --- a/Algorithm.Python/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.py +++ b/Algorithm.Python/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.py @@ -53,7 +53,7 @@ def check_greeks(self, contracts): self._tried_greeks_calculation = True for contract in contracts: - greeks = Greeks() + greeks = None try: greeks = contract.greeks @@ -70,9 +70,10 @@ def check_greeks(self, contracts): # Delta can be {-1, 0, 1} if the price is too wild, rho can be 0 if risk free rate is 0 # Vega can be 0 if the price is very off from theoretical price, Gamma = 0 if Delta belongs to {-1, 1} if (self._option_style_is_supported - and ((contract.right == OptionRight.CALL and (greeks.delta < 0.0 or greeks.delta > 1.0 or greeks.rho < 0.0)) - or (contract.right == OptionRight.PUT and (greeks.delta < -1.0 or greeks.delta > 0.0 or greeks.rho > 0.0)) - or greeks.theta == 0.0 or greeks.vega < 0.0 or greeks.gamma < 0.0)): + and (greeks is None + or ((contract.right == OptionRight.CALL and (greeks.delta < 0.0 or greeks.delta > 1.0 or greeks.rho < 0.0)) + or (contract.right == OptionRight.PUT and (greeks.delta < -1.0 or greeks.delta > 0.0 or greeks.rho > 0.0)) + or greeks.theta == 0.0 or greeks.vega < 0.0 or greeks.gamma < 0.0))): raise Exception(f'Expected greeks to have valid values. Greeks were: Delta: {greeks.delta}, Rho: {greeks.rho}, Theta: {greeks.theta}, Vega: {greeks.vega}, Gamma: {greeks.gamma}') diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 00f2322d63fa..02e15b7c6366 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3362,31 +3362,9 @@ public List Fundamentals(List symbols) /// it will be populated with the contract symbol only. This is expected to change in the future. /// [DocumentationAttribute(AddingData)] - public DataHistory OptionChain(Symbol symbol) + public OptionChain OptionChain(Symbol symbol) { - // TODO: Use OptionChains() - var canonicalSymbol = GetCanonicalOptionSymbol(symbol); - IEnumerable optionChain; - - // TODO: Until future options are supported by OptionUniverse, we need to fall back to the OptionChainProvider for them - if (canonicalSymbol.SecurityType != SecurityType.FutureOption) - { - var history = History(canonicalSymbol, 1); - optionChain = history?.SingleOrDefault()?.Data?.Cast() ?? Enumerable.Empty(); - } - else - { - optionChain = OptionChainProvider.GetOptionContractList(canonicalSymbol, Time) - .Select(contractSymbol => new OptionUniverse() - { - Symbol = contractSymbol, - EndTime = Time.Date, - Data = null, - }) - .ToList(); - } - - return new DataHistory(optionChain, new Lazy(() => PandasConverter.GetDataFrame(optionChain))); + return OptionChains(new[] { symbol }).Values.SingleOrDefault() ?? new OptionChain(GetCanonicalOptionSymbol(symbol), Time.Date); } /// @@ -3400,18 +3378,31 @@ public DataHistory OptionChain(Symbol symbol) [DocumentationAttribute(AddingData)] public OptionChains OptionChains(IEnumerable symbols) { - // TODO: Future options???????????? - var canonicalSymbols = symbols.Select(GetCanonicalOptionSymbol); - var history = History(canonicalSymbols, 1); + var canonicalSymbols = symbols.Select(GetCanonicalOptionSymbol).ToList(); + var optionCanonicalSymbols = canonicalSymbols.Where(x => x.SecurityType != SecurityType.FutureOption); + var futureOptionCanonicalSymbols = canonicalSymbols.Where(x => x.SecurityType == SecurityType.FutureOption); + + // TODO: Resolution.Daily should not be necessary. Remove when GH#8343 is resolved + var optionChainsData = History(optionCanonicalSymbols, 1, Resolution.Daily).GetUniverseData() + .Select(x => (x.Keys.Single(), x.Values.Single().Cast())); + + // TODO: For FOPs, we fall back to the option chain provider until OptionUniverse supports them + var futureOptionChainsData = futureOptionCanonicalSymbols.Select(symbol => + { + var optionChainData = OptionChainProvider.GetOptionContractList(symbol, Time) + .Select(contractSymbol => new OptionUniverse() + { + Symbol = contractSymbol, + EndTime = Time.Date, + Data = null, + }); + return (symbol, optionChainData); + }); var time = Time.Date; var chains = new OptionChains(time); - var dataFrames = new Dictionary(); - foreach (var chainData in history.GetUniverseData()) + foreach (var (symbol, contracts) in optionChainsData.Concat(futureOptionChainsData)) { - var symbol = chainData.Keys.Single(); - var contracts = chainData.Values.Single().Cast().ToList(); - var optionChain = new OptionChain(symbol, time, contracts); chains.Add(symbol, optionChain); } diff --git a/Common/Data/Market/OptionChain.cs b/Common/Data/Market/OptionChain.cs index 6845e3feac62..f033c0764b5f 100644 --- a/Common/Data/Market/OptionChain.cs +++ b/Common/Data/Market/OptionChain.cs @@ -194,7 +194,7 @@ public OptionChain(Symbol canonicalOptionSymbol, DateTime time, BaseData underly /// The canonical option symbol /// The time of this chain /// The list of contracts data - public OptionChain(Symbol canonicalOptionSymbol, DateTime time, List contracts) + public OptionChain(Symbol canonicalOptionSymbol, DateTime time, IEnumerable contracts) : this(canonicalOptionSymbol, time) { Time = time; diff --git a/Common/Data/Market/OptionContract.cs b/Common/Data/Market/OptionContract.cs index 2c98669b9de1..dfd4c495a4a4 100644 --- a/Common/Data/Market/OptionContract.cs +++ b/Common/Data/Market/OptionContract.cs @@ -18,22 +18,17 @@ using QuantConnect.Securities; using QuantConnect.Securities.Option; using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; namespace QuantConnect.Data.Market { /// /// Defines a single option contract at a specific expiration and strike price /// - public class OptionContract : ISymbolProvider + public class OptionContract : ISymbolProvider, ISymbol { private static readonly SymbolPropertiesDatabase _symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder(); - private Lazy _optionPriceModelResult = new(() => OptionPriceModelResult.None); - - private readonly List _data = new(); + private IOptionData _optionData = new OptionPriceModelResultData(() => OptionPriceModelResult.None); /// /// Gets the option contract's symbol @@ -43,13 +38,15 @@ public Symbol Symbol get; set; } + /// + /// The security identifier of the option symbol + /// + public SecurityIdentifier ID => Symbol.ID; + /// /// Gets the underlying security's symbol /// - public Symbol UnderlyingSymbol - { - get; private set; - } + public Symbol UnderlyingSymbol => Symbol.Underlying; /// /// Gets the strike price @@ -83,17 +80,17 @@ public decimal ScaledStrike /// /// Gets the theoretical price of this option contract as computed by the /// - public decimal TheoreticalPrice => _optionPriceModelResult.Value.TheoreticalPrice; + public decimal TheoreticalPrice => _optionData.TheoreticalPrice; /// /// Gets the implied volatility of the option contract as computed by the /// - public decimal ImpliedVolatility => _optionPriceModelResult.Value.ImpliedVolatility; + public decimal ImpliedVolatility => _optionData.ImpliedVolatility; /// /// Gets the greeks for this contract /// - public Greeks Greeks => _optionPriceModelResult.Value.Greeks; + public Greeks Greeks => _optionData.Greeks; /// /// Gets the local date time this contract's data was last updated @@ -106,136 +103,50 @@ public DateTime Time /// /// Gets the open interest /// - public decimal OpenInterest => GetLastOpenInterest()?.Value ?? decimal.Zero; + public decimal OpenInterest => _optionData.OpenInterest; /// /// Gets the last price this contract traded at /// - public decimal LastPrice => GetLastTrades().LastOrDefault() switch - { - Tick tick => tick.LastPrice, - TradeBar tradeBar => tradeBar.Close, - _ => decimal.Zero - }; + public decimal LastPrice => _optionData.LastPrice; /// /// Gets the last volume this contract traded at /// - public long Volume => (long)(GetLastTradeBar()?.Volume ?? 0L); + public long Volume => _optionData.Volume; /// /// Gets the current bid price /// - public decimal BidPrice - { - get - { - foreach (var data in GetLastQuotes()) - { - if (data is Tick tick && tick.BidPrice != 0) - { - return tick.BidPrice; - } - - if (data is QuoteBar quoteBar && quoteBar.Bid != null && quoteBar.Bid.Close != 0) - { - return quoteBar.Bid.Close; - } - } - - return decimal.Zero; - } - } + public decimal BidPrice => _optionData.BidPrice; /// /// Get the current bid size /// - public long BidSize - { - get - { - foreach (var data in GetLastQuotes()) - { - if (data is Tick tick && tick.BidPrice != 0) - { - return (long)tick.BidSize; - } - - if (data is QuoteBar quoteBar && quoteBar.Bid != null && quoteBar.Bid.Close != 0) - { - return (long)quoteBar.LastBidSize; - } - } - - return 0; - } - } + public long BidSize => _optionData.BidSize; /// /// Gets the ask price /// - public decimal AskPrice - { - get - { - foreach (var data in GetLastQuotes()) - { - if (data is Tick tick && tick.AskPrice != 0) - { - return tick.AskPrice; - } - - if (data is QuoteBar quoteBar && quoteBar.Ask != null && quoteBar.Ask.Close != 0) - { - return quoteBar.Ask.Close; - } - } - - return decimal.Zero; - } - } + public decimal AskPrice => _optionData.AskPrice; /// /// Gets the current ask size /// - public long AskSize - { - get - { - foreach (var data in GetLastQuotes()) - { - if (data is Tick tick && tick.AskPrice != 0) - { - return (long)tick.AskSize; - } - - if (data is QuoteBar quoteBar && quoteBar.Ask != null && quoteBar.Ask.Close != 0) - { - return (long)quoteBar.LastAskSize; - } - } - - return 0; - } - } + public long AskSize => _optionData.AskSize; /// /// Gets the last price the underlying security traded at /// - public decimal UnderlyingLastPrice - { - get; set; - } + public decimal UnderlyingLastPrice => _optionData.UnderlyingLastPrice; /// /// Initializes a new instance of the class /// /// The option contract security - /// The symbol of the underlying security - public OptionContract(ISecurityPrice security, Symbol underlyingSymbol) + public OptionContract(ISecurityPrice security) { Symbol = security.Symbol; - UnderlyingSymbol = underlyingSymbol; ScaledStrike = Strike * security.SymbolProperties.StrikeMultiplier; } @@ -246,8 +157,6 @@ public OptionContract(ISecurityPrice security, Symbol underlyingSymbol) public OptionContract(OptionUniverse contractData) { Symbol = contractData.Symbol; - UnderlyingSymbol = contractData.Symbol.Underlying; - // TODO: What about the strike multiplier if no security is provided? Should we access the spdb directly? var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties( contractData.Symbol.ID.Market, @@ -256,6 +165,8 @@ public OptionContract(OptionUniverse contractData) // What should the default be? We don't have access to the account currency here Currencies.USD); ScaledStrike = Strike * symbolProperties.StrikeMultiplier; + + _optionData = new OptionUniverseData(contractData); } /// @@ -264,7 +175,7 @@ public OptionContract(OptionUniverse contractData) /// Function delegate used to evaluate the option price model internal void SetOptionPriceModel(Func optionPriceModelEvaluator) { - _optionPriceModelResult = new Lazy(optionPriceModelEvaluator); + _optionData = new OptionPriceModelResultData(optionPriceModelEvaluator, _optionData as OptionPriceModelResultData); } /// @@ -279,27 +190,28 @@ internal void SetOptionPriceModel(Func optionPriceModelE /// Creates a /// /// - /// provides price properties for a - /// last price the underlying security traded at + /// Provides price properties for a + /// Last underlying security trade data /// Option contract - public static OptionContract Create(BaseData baseData, ISecurityPrice security, decimal underlyingLastPrice) - => Create(baseData.Symbol.Underlying, baseData.EndTime, security, underlyingLastPrice); + public static OptionContract Create(BaseData baseData, ISecurityPrice security, BaseData underlying) + => Create(baseData.EndTime, security, underlying); /// /// Creates a /// - /// The symbol of the underlying security /// local date time this contract's data was last updated /// provides price properties for a - /// last price the underlying security traded at + /// last underlying security trade data /// Option contract - public static OptionContract Create(Symbol underlyingSymbol, DateTime endTime, ISecurityPrice security, decimal underlyingLastPrice) + public static OptionContract Create(DateTime endTime, ISecurityPrice security, BaseData underlying) { - return new OptionContract(security, underlyingSymbol) + var contract = new OptionContract(security) { Time = endTime, - UnderlyingLastPrice = underlyingLastPrice }; + contract._optionData.SetUnderlying(underlying); + + return contract; } /// @@ -314,47 +226,164 @@ public static OptionContract Create(OptionUniverse contractData) Time = contractData.EndTime, }; - var bar = new Bar(contractData.Open, contractData.High, contractData.Low, contractData.Close); - var quoteBar = new QuoteBar(contractData.Time, contractData.Symbol, bar, 0, bar, 0) - { - EndTime = contractData.EndTime, - }; - - contract.Update(quoteBar); - return contract; } /// - /// Updates the option contract with the new data, which can be a or or + /// Implicit conversion into /// - internal void Update(BaseData data) + /// The option contract to be converted + public static implicit operator Symbol(OptionContract contract) { - _data.Add(data); + return contract.Symbol; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private IEnumerable GetLastTrades() + /// + /// Updates the option contract with the new data, which can be a or or + /// + internal void Update(BaseData data) { - return _data.Where(x => x is TradeBar || (x is Tick tick && tick.TickType == TickType.Trade)); + if (data.Symbol == Symbol) + { + _optionData.Update(data); + } + else if (data.Symbol == UnderlyingSymbol) + { + _optionData.SetUnderlying(data); + } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private TradeBar GetLastTradeBar() + #region Option Contract Data Handlers + + private interface IOptionData { - return _data.LastOrDefault(x => x is TradeBar) as TradeBar; + decimal LastPrice { get; } + decimal UnderlyingLastPrice { get; } + long Volume { get; } + decimal BidPrice { get; } + long BidSize { get; } + decimal AskPrice { get; } + long AskSize { get; } + decimal OpenInterest { get; } + decimal TheoreticalPrice { get; } + decimal ImpliedVolatility { get; } + Greeks Greeks { get; } + + void Update(BaseData data); + + void SetUnderlying(BaseData data); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private IEnumerable GetLastQuotes() + /// + /// Handles option data for a contract from actual price data (trade, quote, open interest) and theoretical price model results + /// + private class OptionPriceModelResultData : IOptionData { - return _data.Where(x => x is QuoteBar || (x is Tick tick && tick.TickType == TickType.Quote)); + private readonly Lazy _optionPriceModelResult; + private TradeBar _tradeBar; + private QuoteBar _quoteBar; + private OpenInterest _openInterest; + private BaseData _underlying; + + public decimal LastPrice => _tradeBar?.Close ?? decimal.Zero; + + public decimal UnderlyingLastPrice => _underlying?.Price ?? decimal.Zero; + + public long Volume => (long)(_tradeBar?.Volume ?? 0L); + + public decimal BidPrice => _quoteBar?.Bid?.Close ?? decimal.Zero; + + public long BidSize => (long)(_quoteBar?.LastBidSize ?? 0L); + + public decimal AskPrice => _quoteBar?.Ask?.Close ?? decimal.Zero; + + public long AskSize => (long)(_quoteBar?.LastAskSize ?? 0L); + + public decimal OpenInterest => _openInterest?.Value ?? decimal.Zero; + + public decimal TheoreticalPrice => _optionPriceModelResult.Value.TheoreticalPrice; + public decimal ImpliedVolatility => _optionPriceModelResult.Value.ImpliedVolatility; + public Greeks Greeks => _optionPriceModelResult.Value.Greeks; + + public OptionPriceModelResultData(Func optionPriceModelEvaluator, + OptionPriceModelResultData previousOptionData = null) + { + _optionPriceModelResult = new(optionPriceModelEvaluator); + + if (previousOptionData != null) + { + _tradeBar = previousOptionData._tradeBar; + _quoteBar = previousOptionData._quoteBar; + _openInterest = previousOptionData._openInterest; + _underlying = previousOptionData._underlying; + } + } + + public void Update(BaseData data) + { + switch (data) + { + case TradeBar tradeBar: + _tradeBar = tradeBar; + break; + case QuoteBar quoteBar: + _quoteBar = quoteBar; + break; + case OpenInterest openInterest: + _openInterest = openInterest; + break; + } + } + + public void SetUnderlying(BaseData data) + { + _underlying = data; + } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private BaseData GetLastOpenInterest() + /// + /// Handles option data for a contract from a instance + /// + private class OptionUniverseData : IOptionData { - return _data.LastOrDefault(x => x is Tick tick && tick.TickType == TickType.OpenInterest); + private readonly OptionUniverse _contractData; + + public decimal LastPrice => _contractData.Close; + + public decimal UnderlyingLastPrice => _contractData.Underlying.Price; + + public long Volume => (long)_contractData.Volume; + + public decimal BidPrice => _contractData.Close; + + public long BidSize => 0; + + public decimal AskPrice => _contractData.Close; + + public long AskSize => 0; + + public decimal OpenInterest => _contractData.OpenInterest; + + public decimal TheoreticalPrice => decimal.Zero; + + public decimal ImpliedVolatility => _contractData.ImpliedVolatility; + + public Greeks Greeks => _contractData.Greeks; + + public OptionUniverseData(OptionUniverse contractData) + { + _contractData = contractData; + } + + public void Update(BaseData data) + { + } + + public void SetUnderlying(BaseData data) + { + } } + + #endregion } } diff --git a/Engine/DataFeeds/TimeSliceFactory.cs b/Engine/DataFeeds/TimeSliceFactory.cs index b33a353ac15b..000759f2e69b 100644 --- a/Engine/DataFeeds/TimeSliceFactory.cs +++ b/Engine/DataFeeds/TimeSliceFactory.cs @@ -452,7 +452,7 @@ private bool HandleOptionData(DateTime algorithmTime, BaseData baseData, OptionC { foreach (var addedContract in chain.Contracts) { - addedContract.Value.UnderlyingLastPrice = chain.Underlying.Price; + addedContract.Value.Update(chain.Underlying); } } foreach (var contractSymbol in universeData.FilteredContracts ?? Enumerable.Empty()) @@ -465,7 +465,7 @@ private bool HandleOptionData(DateTime algorithmTime, BaseData baseData, OptionC OptionContract contract; if (!chain.Contracts.TryGetValue(baseData.Symbol, out contract)) { - contract = OptionContract.Create(baseData, security, chain.Underlying.Price); + contract = OptionContract.Create(baseData, security, chain.Underlying); chain.Contracts[baseData.Symbol] = contract; diff --git a/Tests/Common/Securities/OptionPriceModelTests.cs b/Tests/Common/Securities/OptionPriceModelTests.cs index 0229f5580b17..df2aa38c36cd 100644 --- a/Tests/Common/Securities/OptionPriceModelTests.cs +++ b/Tests/Common/Securities/OptionPriceModelTests.cs @@ -212,7 +212,7 @@ public void BaroneAdesiWhaleyPortfolioTest() var equity = GetEquity(spy, underlyingPrice, underlyingVol, tz); - var contract = new OptionContract(option, Symbols.SPY) { Time = evaluationDate }; + var contract = new OptionContract(option) { Time = evaluationDate }; var optionCall = GetOption(SPY_C_192_Feb19_2016E, equity, tz); optionCall.SetMarketPrice(new Tick { Value = price }); @@ -515,7 +515,7 @@ public void MatchesIBGreeksFarATMCall(OptionStyle style, string qlModelName, dou [TestCase(OptionStyle.European, "CrankNicolsonFD", 0.01d, 0.01d, 0.01d, 0.33d, 642d)] [TestCase(OptionStyle.European, "Integral", 0.01d, 0.12d, 0.01d, 0.33d, 4622d)] public void MatchesIBGreeksNearATMPut(OptionStyle style, string qlModelName, double errorIV, double errorDelta, double errorGamma, double errorVega, double errorTheta) - { + { var filename = style == OptionStyle.American ? "SPY230811P00450000" : "SPX230811P04500000"; var symbol = Symbols.SPY; // dummy var strike = Parse.Decimal(filename[10..]) / 1000m; @@ -812,7 +812,7 @@ public void MatchesIBGreeksFarOTMPut(OptionStyle style, string qlModelName, doub MatchesIBGreeksTest(symbol, optionSymbol, filename, qlModelName, errorIV, errorDelta, errorGamma, errorVega, errorTheta); } - private void MatchesIBGreeksTest(Symbol symbol, Symbol optionSymbol, string filename, string qlModelName, + private void MatchesIBGreeksTest(Symbol symbol, Symbol optionSymbol, string filename, string qlModelName, double errorIV, double errorDelta, double errorGamma, double errorVega, double errorTheta) { var tz = TimeZones.NewYork; @@ -879,7 +879,7 @@ public void ImpliedVolatilityEstimator(OptionRight optionRight, decimal strike, // Expect minor error due to interest rate and dividend yield used in IB Assert.AreEqual(impliedVolEstimate, ibImpliedVol, 0.001); } - + [Test] public void PriceModelEvaluateSpeedTest() { @@ -971,7 +971,7 @@ public static Equity GetEquity(Symbol symbol, decimal underlyingPrice, decimal u public OptionContract GetOptionContract(Symbol symbol, Symbol underlying, DateTime evaluationDate) { var option = CreateOption(symbol); - return new OptionContract(option, underlying) { Time = evaluationDate }; + return new OptionContract(option) { Time = evaluationDate }; } public static Option GetOption(Symbol symbol, Equity underlying, NodaTime.DateTimeZone tz) diff --git a/Tests/Common/Securities/Options/FedRateQLRiskFreeRateEstimatorTests.cs b/Tests/Common/Securities/Options/FedRateQLRiskFreeRateEstimatorTests.cs index dec6003ec015..86c02a132028 100644 --- a/Tests/Common/Securities/Options/FedRateQLRiskFreeRateEstimatorTests.cs +++ b/Tests/Common/Securities/Options/FedRateQLRiskFreeRateEstimatorTests.cs @@ -44,9 +44,9 @@ public void Estimate(string dateString, decimal rate) // get the risk free rate var estimator = new FedRateQLRiskFreeRateEstimator(); - var result = estimator.Estimate(option, - new Slice(evaluationDate, new List { tick }, evaluationDate), - new OptionContract(option, spx)); + var result = estimator.Estimate(option, + new Slice(evaluationDate, new List { tick }, evaluationDate), + new OptionContract(option)); Assert.AreEqual(rate, result); } diff --git a/ToolBox/RandomDataGenerator/OptionPriceModelPriceGenerator.cs b/ToolBox/RandomDataGenerator/OptionPriceModelPriceGenerator.cs index ddbeeb1b7f4a..540f617f0f68 100644 --- a/ToolBox/RandomDataGenerator/OptionPriceModelPriceGenerator.cs +++ b/ToolBox/RandomDataGenerator/OptionPriceModelPriceGenerator.cs @@ -66,10 +66,9 @@ public decimal NextValue(decimal maximumPercentDeviation, DateTime referenceDate _option, null, OptionContract.Create( - _option.Symbol.Underlying, referenceDate, _option, - _option.Underlying.Price + new Tick(referenceDate, _option.Underlying.Symbol, _option.Underlying.Price, _option.Underlying.Price) )) .TheoreticalPrice; } From 7b0db104b99a081f0ff5b64fb3ee975f37ce669c Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 25 Sep 2024 12:43:09 -0400 Subject: [PATCH 13/38] Pass symbol properties to OptionUniverse option chain from algorithm --- Algorithm/QCAlgorithm.cs | 3 ++- Common/Data/Market/OptionChain.cs | 6 ++++-- Common/Data/Market/OptionContract.cs | 15 +++++---------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 02e15b7c6366..31c1a8d12098 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3403,7 +3403,8 @@ public OptionChains OptionChains(IEnumerable symbols) var chains = new OptionChains(time); foreach (var (symbol, contracts) in optionChainsData.Concat(futureOptionChainsData)) { - var optionChain = new OptionChain(symbol, time, contracts); + var symbolProperties = SymbolPropertiesDatabase.GetSymbolProperties(symbol.ID.Market, symbol, symbol.SecurityType, AccountCurrency); + var optionChain = new OptionChain(symbol, time, contracts, symbolProperties); chains.Add(symbol, optionChain); } diff --git a/Common/Data/Market/OptionChain.cs b/Common/Data/Market/OptionChain.cs index f033c0764b5f..170702660470 100644 --- a/Common/Data/Market/OptionChain.cs +++ b/Common/Data/Market/OptionChain.cs @@ -20,6 +20,7 @@ using Python.Runtime; using QuantConnect.Data.UniverseSelection; using QuantConnect.Python; +using QuantConnect.Securities; using QuantConnect.Securities.Option; using QuantConnect.Util; @@ -194,7 +195,8 @@ public OptionChain(Symbol canonicalOptionSymbol, DateTime time, BaseData underly /// The canonical option symbol /// The time of this chain /// The list of contracts data - public OptionChain(Symbol canonicalOptionSymbol, DateTime time, IEnumerable contracts) + /// The option symbol properties + public OptionChain(Symbol canonicalOptionSymbol, DateTime time, IEnumerable contracts, SymbolProperties symbolProperties) : this(canonicalOptionSymbol, time) { Time = time; @@ -209,7 +211,7 @@ public OptionChain(Symbol canonicalOptionSymbol, DateTime time, IEnumerable instance /// /// The option universe contract data to use as source for this contract - public OptionContract(OptionUniverse contractData) + /// The contract symbol properties + public OptionContract(OptionUniverse contractData, SymbolProperties symbolProperties) { Symbol = contractData.Symbol; - // TODO: What about the strike multiplier if no security is provided? Should we access the spdb directly? - var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties( - contractData.Symbol.ID.Market, - contractData.Symbol, - contractData.Symbol.SecurityType, - // What should the default be? We don't have access to the account currency here - Currencies.USD); ScaledStrike = Strike * symbolProperties.StrikeMultiplier; _optionData = new OptionUniverseData(contractData); @@ -219,9 +213,10 @@ public static OptionContract Create(DateTime endTime, ISecurityPrice security, B /// using its data to form a quote bar to source pricing data /// /// The option universe contract data to use as source for this contract - public static OptionContract Create(OptionUniverse contractData) + /// The contract symbol properties + public static OptionContract Create(OptionUniverse contractData, SymbolProperties symbolProperties) { - var contract = new OptionContract(contractData) + var contract = new OptionContract(contractData, symbolProperties) { Time = contractData.EndTime, }; From 1d632a9a54347e0427d1e51dd2abbcfb4f9b1794 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 25 Sep 2024 17:43:40 -0400 Subject: [PATCH 14/38] Format OptionContract for dataframe --- Common/Data/Market/OptionContract.cs | 25 +++++++++++++++++++++++++ Common/Python/PandasData.cs | 24 ++++++++++++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/Common/Data/Market/OptionContract.cs b/Common/Data/Market/OptionContract.cs index ff13ab384937..9b268b5b828b 100644 --- a/Common/Data/Market/OptionContract.cs +++ b/Common/Data/Market/OptionContract.cs @@ -92,6 +92,31 @@ public decimal ScaledStrike /// public Greeks Greeks => _optionData.Greeks; + /// + /// Gets the delta. Pass-through for + /// + public decimal Delta => Greeks.Delta; + + /// + /// Gets the gamma. Pass-through for + /// + public decimal Gamma => Greeks.Gamma; + + /// + /// Gets the vega. Pass-through for + /// + public decimal Vega => Greeks.Vega; + + /// + /// Gets the theta. Pass-through for + /// + public decimal Theta => Greeks.Theta; + + /// + /// Gets the rho. Pass-through for + /// + public decimal Rho => Greeks.Rho; + /// /// Gets the local date time this contract's data was last updated /// diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 8633f8e3d333..0a1bfd867376 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -64,12 +64,18 @@ public class PandasData private const string Strike = "strike"; private const string Right = "right"; - private static readonly string[] _optionUniverseExcludedProperties = new[] + private static readonly string[] _optionUniverseExcludedMembers = new[] { nameof(OptionUniverse.ID), nameof(OptionUniverse.Greeks) }; + private static readonly string[] _optionContractExcludedMembers = new[] + { + nameof(OptionContract.ID), + nameof(OptionContract.Greeks) + }; + // we keep these so we don't need to ask for them each time private static PyString _empty; private static PyObject _pandas; @@ -177,6 +183,7 @@ public PandasData(object data) if (keys == null) { var isOptionUniverse = type == typeof(OptionUniverse); + var isOptionContract = type == typeof(OptionContract); if (_membersByType.TryGetValue(type, out _members)) { @@ -186,9 +193,18 @@ public PandasData(object data) { var members = type .GetMembers(BindingFlags.Instance | BindingFlags.Public) - .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) && - (!isOptionUniverse || !_optionUniverseExcludedProperties.Contains(x.Name))) - .ToList(); + .Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property); + + if (isOptionUniverse) + { + members = members.Where(x => !_optionUniverseExcludedMembers.Contains(x.Name)); + } + else if (isOptionContract) + { + members = members.Where(x => !_optionContractExcludedMembers.Contains(x.Name)); + } + + members = members.ToList(); var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key); foreach (var duplicateKey in duplicateKeys) From 6eb1df3108c525c99ff008c7dc13975d5e1e5525 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 25 Sep 2024 18:03:34 -0400 Subject: [PATCH 15/38] Minor fix --- .../OptionChainFullDataRegressionAlgorithm.py | 8 ++++---- Common/QuantConnect.csproj | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py index a10f8671602c..d230e517f7dd 100644 --- a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py +++ b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py @@ -32,10 +32,10 @@ def initialize(self): # Demonstration using data frame: # Get contracts expiring within 10 days, with an implied volatility greater than 0.5 and a delta less than 0.5 contracts = [ - # Index is a tuple (symbol, date) - index[0] - for index, contract_data in option_chain.data_frame.iterrows() - if index[0].id.date - self.time <= timedelta(days=10) and contract_data["impliedvolatility"] > 0.5 and contract_data["delta"] < 0.5 + symbol + # Index is a tuple and the first element is the symbol + for (symbol,), contract_data in option_chain.data_frame.iterrows() + if symbol.id.date - self.time <= timedelta(days=10) and contract_data["impliedvolatility"] > 0.5 and contract_data["delta"] < 0.5 ] # Get the contract with the latest expiration date diff --git a/Common/QuantConnect.csproj b/Common/QuantConnect.csproj index 4218bf22f991..262200725a5a 100644 --- a/Common/QuantConnect.csproj +++ b/Common/QuantConnect.csproj @@ -35,7 +35,7 @@ - + From 80ef6a55f306470eb242c1dc151f40a2de650f29 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 26 Sep 2024 11:25:16 -0400 Subject: [PATCH 16/38] Add multiple option chains api regression algorithms and other minor changes --- ...ainsMultipleFullDataRegressionAlgorithm.cs | 143 ++++++++++++++++++ ...lForOptionStylesBaseRegressionAlgorithm.cs | 8 +- .../OptionChainFullDataRegressionAlgorithm.py | 19 +-- ...ainsMultipleFullDataRegressionAlgorithm.py | 62 ++++++++ Algorithm/QCAlgorithm.Python.cs | 15 ++ Algorithm/QCAlgorithm.cs | 17 +-- 6 files changed, 230 insertions(+), 34 deletions(-) create mode 100644 Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs create mode 100644 Algorithm.Python/OptionChainsMultipleFullDataRegressionAlgorithm.py diff --git a/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs b/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs new file mode 100644 index 000000000000..b457a6662eb4 --- /dev/null +++ b/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs @@ -0,0 +1,143 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Data; +using QuantConnect.Data.Market; +using QuantConnect.Interfaces; +using QuantConnect.Securities; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm illustrating the usage of the method + /// to get multiple option chains, which contains additional data besides the symbols, including prices, implied volatility and greeks. + /// It also shows how this data can be used to filter the contracts based on certain criteria. + /// + public class OptionChainsMultipleFullDataRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + private Symbol _googOptionContract; + private Symbol _spxOptionContract; + + public override void Initialize() + { + SetStartDate(2015, 12, 24); + SetEndDate(2015, 12, 24); + SetCash(100000); + + var goog = AddEquity("GOOG").Symbol; + var spx = AddIndex("SPX").Symbol; + + var chains = OptionChains(new[] { goog, spx }); + + _googOptionContract = GetContract(chains, goog, TimeSpan.FromDays(10)); + _spxOptionContract = GetContract(chains, spx, TimeSpan.FromDays(60)); + + AddOptionContract(_googOptionContract); + AddIndexOptionContract(_spxOptionContract); + } + + private Symbol GetContract(OptionChains chains, Symbol underlying, TimeSpan expirySpan) + { + return chains + .Where(kvp => kvp.Key.Underlying == underlying) + .Select(kvp => kvp.Value) + .Single() + // Get contracts expiring within a given span, with an implied volatility greater than 0.5 and a delta less than 0.5 + .Where(contractData => contractData.Symbol.ID.Date - Time <= expirySpan && + contractData.ImpliedVolatility > 0.5m && + contractData.Delta < 0.5m) + // Get the contract with the latest expiration date + .OrderByDescending(x => x.Symbol.ID.Date) + .First() + .Symbol; + } + + public override void OnData(Slice slice) + { + // Do some trading with the selected contract for sample purposes + if (!Portfolio.Invested) + { + MarketOrder(_googOptionContract, 1); + } + else + { + Liquidate(); + } + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public virtual List Languages { get; } = new() { Language.CSharp, Language.Python }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 1059; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 2; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "210"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "96041"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "0"}, + {"Tracking Error", "0"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "$209.00"}, + {"Estimated Strategy Capacity", "$0"}, + {"Lowest Capacity Asset", "GOOCV W6U7PD1F2WYU|GOOCV VP83T1ZUHROL"}, + {"Portfolio Turnover", "85.46%"}, + {"OrderListHash", "a7ab1a9e64fe9ba76ea33a40a78a4e3b"} + }; + } +} diff --git a/Algorithm.CSharp/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.cs b/Algorithm.CSharp/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.cs index 37852f18056a..2157c9a7316e 100644 --- a/Algorithm.CSharp/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.cs +++ b/Algorithm.CSharp/OptionPriceModelForOptionStylesBaseRegressionAlgorithm.cs @@ -110,12 +110,8 @@ public void CheckGreeks(OptionChain contracts) // Greeks should be valid if they were successfuly accessed for supported option style if (_optionStyleIsSupported) { - if (greeks == null) - { - greeks = new ModeledGreeks(); - } - - if (greeks.Delta == 0m && greeks.Gamma == 0m && greeks.Theta == 0m && greeks.Vega == 0m && greeks.Rho == 0m) + if (greeks == null || + (greeks.Delta == 0m && greeks.Gamma == 0m && greeks.Theta == 0m && greeks.Vega == 0m && greeks.Rho == 0m)) { throw new RegressionTestException($"Expected greeks to not be zero simultaneously for {contract.Symbol.Value}, an {_option.Style} style option, using {_option?.PriceModel.GetType().Name}, but they were"); } diff --git a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py index d230e517f7dd..53c5fb1a310e 100644 --- a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py +++ b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py @@ -12,6 +12,7 @@ # limitations under the License. from AlgorithmImports import * +from datetime import timedelta ### ### Regression algorithm illustrating the usage of the method @@ -30,19 +31,13 @@ def initialize(self): option_chain = self.option_chain(goog) # Demonstration using data frame: + df = option_chain.data_frame # Get contracts expiring within 10 days, with an implied volatility greater than 0.5 and a delta less than 0.5 - contracts = [ - symbol - # Index is a tuple and the first element is the symbol - for (symbol,), contract_data in option_chain.data_frame.iterrows() - if symbol.id.date - self.time <= timedelta(days=10) and contract_data["impliedvolatility"] > 0.5 and contract_data["delta"] < 0.5 - ] - - # Get the contract with the latest expiration date - option_contract = sorted(contracts, key=lambda x: x.id.date, reverse=True)[0] - - # Can use the symbol instance to index the data frame - self.debug(f"Option contract data:\n{option_chain.data_frame.loc[(option_contract)]}") + contracts = df.loc[(df['expiry'] <= self.time + timedelta(days=10)) & (df['impliedvolatility'] > 0.5) & (df['delta'] < 0.5)] + + # Get the contract with the latest expiration date. + # Note: the result of df.loc[] is a series, and its name is a tuple with a single element (contract symbol) + option_contract = contracts.loc[contracts['expiry'].idxmax()].name[0] self._option_contract = self.add_option_contract(option_contract) diff --git a/Algorithm.Python/OptionChainsMultipleFullDataRegressionAlgorithm.py b/Algorithm.Python/OptionChainsMultipleFullDataRegressionAlgorithm.py new file mode 100644 index 000000000000..8cf04c969a91 --- /dev/null +++ b/Algorithm.Python/OptionChainsMultipleFullDataRegressionAlgorithm.py @@ -0,0 +1,62 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * +from datetime import timedelta + +### +### Regression algorithm illustrating the usage of the method +### to get multiple option chains, which contains additional data besides the symbols, including prices, implied volatility and greeks. +### It also shows how this data can be used to filter the contracts based on certain criteria. +### +class OptionChainsMultipleFullDataRegressionAlgorithm(QCAlgorithm): + + def initialize(self): + self.set_start_date(2015, 12, 24) + self.set_end_date(2015, 12, 24) + self.set_cash(100000) + + goog = self.add_equity("GOOG").symbol + spx = self.add_index("SPX").symbol + + chains = self.option_chains([goog, spx]) + + self._goog_option_contract = self.get_contract(chains, goog, timedelta(days=10)) + self._spx_option_contract = self.get_contract(chains, spx, timedelta(days=60)) + + self.add_option_contract(self._goog_option_contract) + self.add_index_option_contract(self._spx_option_contract) + + def get_contract(self, chains: OptionChains, underlying: Symbol, expiry_span: timedelta) -> Symbol: + df = chains.data_frame + + # Index by the requested underlying, by getting all data with canonicals which underlying is the requested underlying symbol: + canonicals = df.index.get_level_values('canonical') + condition = [canonical for canonical in canonicals if getattr(canonical, 'underlying') == underlying] + df = df.loc[condition] + + # Get contracts expiring in the next 10 days with an implied volatility greater than 0.5 and a delta less than 0.5 + contracts = df.loc[(df['expiry'] <= self.time + expiry_span) & (df['impliedvolatility'] > 0.5) & (df['delta'] < 0.5)] + + # Select the contract with the latest expiry date + contracts.sort_values(by='expiry', ascending=False, inplace=True) + + # Get the symbol: the resulting series name is a tuple (canonical symbol, contract symbol) + return contracts.iloc[0].name[1] + + def on_data(self, data): + # Do some trading with the selected contract for sample purposes + if not self.portfolio.invested: + self.market_order(self._goog_option_contract, 1) + else: + self.liquidate() diff --git a/Algorithm/QCAlgorithm.Python.cs b/Algorithm/QCAlgorithm.Python.cs index dbca952c68b3..607f42ad8f0f 100644 --- a/Algorithm/QCAlgorithm.Python.cs +++ b/Algorithm/QCAlgorithm.Python.cs @@ -1638,6 +1638,21 @@ public void AddCommand(PyObject type) }; } + + /// + /// Get the option chains for the specified symbols at the current time () + /// + /// + /// The symbols for which the option chain is asked for. + /// It can be either the canonical options or the underlying symbols. + /// + /// The option chains + [DocumentationAttribute(AddingData)] + public OptionChains OptionChains(PyObject symbols) + { + return OptionChains(symbols.ConvertToSymbolEnumerable()); + } + /// /// Gets indicator base type /// diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 31c1a8d12098..1741ea787858 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3382,8 +3382,7 @@ public OptionChains OptionChains(IEnumerable symbols) var optionCanonicalSymbols = canonicalSymbols.Where(x => x.SecurityType != SecurityType.FutureOption); var futureOptionCanonicalSymbols = canonicalSymbols.Where(x => x.SecurityType == SecurityType.FutureOption); - // TODO: Resolution.Daily should not be necessary. Remove when GH#8343 is resolved - var optionChainsData = History(optionCanonicalSymbols, 1, Resolution.Daily).GetUniverseData() + var optionChainsData = History(optionCanonicalSymbols, 1).GetUniverseData() .Select(x => (x.Keys.Single(), x.Values.Single().Cast())); // TODO: For FOPs, we fall back to the option chain provider until OptionUniverse supports them @@ -3411,20 +3410,6 @@ public OptionChains OptionChains(IEnumerable symbols) return chains; } - /// - /// Get the option chains for the specified symbols at the current time () - /// - /// - /// The symbols for which the option chain is asked for. - /// It can be either the canonical options or the underlying symbols. - /// - /// The option chains - [DocumentationAttribute(AddingData)] - public OptionChains OptionChains(PyObject symbols) - { - return OptionChains(symbols.ConvertToSymbolEnumerable()); - } - /// /// Register a command type to be used /// From 8839ea0b14782dcc03616ec7a41376eeff3b6e98 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 26 Sep 2024 11:49:47 -0400 Subject: [PATCH 17/38] Address peer review Add NullGreeks class: keep ModeledGreeks as internal as possible --- .../OptionChainFullDataRegressionAlgorithm.cs | 7 +- ...ainsMultipleFullDataRegressionAlgorithm.cs | 7 +- .../OptionChainFullDataRegressionAlgorithm.py | 4 +- Common/Data/Market/Greeks.cs | 12 +- Common/Data/Market/ModeledGreeks.cs | 107 +----------------- Common/Data/Market/NullGreeks.cs | 41 +++++++ Common/Data/Market/OptionChain.cs | 2 +- Common/Data/Market/OptionChains.cs | 9 +- Common/Data/Market/OptionContract.cs | 18 ++- .../Data/UniverseSelection/OptionUniverse.cs | 36 +----- .../Option/CurrentPriceOptionPriceModel.cs | 16 +-- .../Option/OptionPriceModelResult.cs | 20 ++-- Common/Symbol.cs | 5 +- .../OptionPriceModelPriceGeneratorTests.cs | 2 +- 14 files changed, 98 insertions(+), 188 deletions(-) create mode 100644 Common/Data/Market/NullGreeks.cs diff --git a/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs b/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs index 23dce11319e0..ce1fe0705ab6 100644 --- a/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs +++ b/Algorithm.CSharp/OptionChainFullDataRegressionAlgorithm.cs @@ -42,13 +42,12 @@ public override void Initialize() _optionContract = OptionChain(goog) // Get contracts expiring within 10 days, with an implied volatility greater than 0.5 and a delta less than 0.5 - .Where(contractData => contractData.Symbol.ID.Date - Time <= TimeSpan.FromDays(10) && + .Where(contractData => contractData.ID.Date - Time <= TimeSpan.FromDays(10) && contractData.ImpliedVolatility > 0.5m && contractData.Greeks.Delta < 0.5m) // Get the contract with the latest expiration date - .OrderByDescending(x => x.Symbol.ID.Date) - .First() - .Symbol; + .OrderByDescending(x => x.ID.Date) + .First(); AddOptionContract(_optionContract); } diff --git a/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs b/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs index b457a6662eb4..33dafd99d242 100644 --- a/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs +++ b/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs @@ -59,13 +59,12 @@ private Symbol GetContract(OptionChains chains, Symbol underlying, TimeSpan expi .Select(kvp => kvp.Value) .Single() // Get contracts expiring within a given span, with an implied volatility greater than 0.5 and a delta less than 0.5 - .Where(contractData => contractData.Symbol.ID.Date - Time <= expirySpan && + .Where(contractData => contractData.ID.Date - Time <= expirySpan && contractData.ImpliedVolatility > 0.5m && contractData.Delta < 0.5m) // Get the contract with the latest expiration date - .OrderByDescending(x => x.Symbol.ID.Date) - .First() - .Symbol; + .OrderByDescending(x => x.ID.Date) + .First(); } public override void OnData(Slice slice) diff --git a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py index 53c5fb1a310e..f3bf850a5a77 100644 --- a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py +++ b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py @@ -37,9 +37,9 @@ def initialize(self): # Get the contract with the latest expiration date. # Note: the result of df.loc[] is a series, and its name is a tuple with a single element (contract symbol) - option_contract = contracts.loc[contracts['expiry'].idxmax()].name[0] + self._option_contract = contracts.loc[contracts['expiry'].idxmax()].name[0] - self._option_contract = self.add_option_contract(option_contract) + self.add_option_contract(self._option_contract) def on_data(self, data): # Do some trading with the selected contract for sample purposes diff --git a/Common/Data/Market/Greeks.cs b/Common/Data/Market/Greeks.cs index 25ff19c6bb85..216f45c2eed1 100644 --- a/Common/Data/Market/Greeks.cs +++ b/Common/Data/Market/Greeks.cs @@ -27,7 +27,7 @@ public abstract class Greeks /// the underlying asset'sprice. (∂V/∂S) /// /// - public abstract decimal Delta { get; protected set; } + public abstract decimal Delta { get; } /// /// Gets the gamma. @@ -36,7 +36,7 @@ public abstract class Greeks /// the underlying asset'sprice. (∂²V/∂S²) /// /// - public abstract decimal Gamma { get; protected set; } + public abstract decimal Gamma { get; } /// /// Gets the vega. @@ -45,7 +45,7 @@ public abstract class Greeks /// the underlying's volatility. (∂V/∂σ) /// /// - public abstract decimal Vega { get; protected set; } + public abstract decimal Vega { get; } /// /// Gets the theta. @@ -54,7 +54,7 @@ public abstract class Greeks /// time. This is commonly known as the 'time decay.' (∂V/∂τ) /// /// - public abstract decimal Theta { get; protected set; } + public abstract decimal Theta { get; } /// /// Gets the rho. @@ -63,7 +63,7 @@ public abstract class Greeks /// the risk free interest rate. (∂V/∂r) /// /// - public abstract decimal Rho { get; protected set; } + public abstract decimal Rho { get; } /// /// Gets the lambda. @@ -73,7 +73,7 @@ public abstract class Greeks /// (∂V/∂S ✕ S/V) /// /// - public abstract decimal Lambda { get; protected set; } + public abstract decimal Lambda { get; } /// /// Gets the lambda. diff --git a/Common/Data/Market/ModeledGreeks.cs b/Common/Data/Market/ModeledGreeks.cs index 8ba57a08d5d1..03951d65dce2 100644 --- a/Common/Data/Market/ModeledGreeks.cs +++ b/Common/Data/Market/ModeledGreeks.cs @@ -29,108 +29,24 @@ public class ModeledGreeks : Greeks private Lazy _rho; private Lazy _lambda; - // _deltagamma stores gamma and delta combined and is done - // for optimization purposes (approximation of delta and gamma is very similar) - private Lazy> _deltaGamma; - /// - public override decimal Delta - { - get - { - return _delta != null ? _delta.Value : _deltaGamma.Value.Item1; - } - protected set - { - _delta = new Lazy(() => value); - } - } + public override decimal Delta => _delta.Value; /// - public override decimal Gamma - { - get - { - return _gamma != null ? _gamma.Value : _deltaGamma.Value.Item2; - } - protected set - { - _gamma = new Lazy(() => value); - } - } + public override decimal Gamma => _gamma.Value; /// - public override decimal Vega - { - get - { - return _vega.Value; - } - protected set - { - _vega = new Lazy(() => value); - } - } + public override decimal Vega => _vega.Value; /// - public override decimal Theta - { - get - { - return _theta.Value; - } - protected set - { - _theta = new Lazy(() => value); - } - } + public override decimal Theta => _theta.Value; /// - public override decimal Rho - { - get - { - return _rho.Value; - } - protected set - { - _rho = new Lazy(() => value); - } - } + public override decimal Rho => _rho.Value; /// - public override decimal Lambda - { - get - { - return _lambda.Value; - } - protected set - { - _lambda = new Lazy(() => value); - } - } - - /// - /// Initializes a new default instance of the class - /// - public ModeledGreeks() - : this(0m, 0m, 0m, 0m, 0m, 0m) - { - } + public override decimal Lambda => _lambda.Value; - /// - /// Initializes a new instance of the class - /// - public ModeledGreeks(decimal delta, decimal gamma, decimal vega, decimal theta, decimal rho, decimal lambda) - { - Delta = delta; - Gamma = gamma; - Vega = vega; - Theta = theta; - Rho = rho; - Lambda = lambda; - } /// /// Initializes a new instance of the class /// @@ -143,16 +59,5 @@ public ModeledGreeks(Func delta, Func gamma, Func veg _rho = new Lazy(rho); _lambda = new Lazy(lambda); } - /// - /// Initializes a new instance of the class - /// - public ModeledGreeks(Func> deltaGamma, Func vega, Func theta, Func rho, Func lambda) - { - _deltaGamma = new Lazy>(deltaGamma); - _vega = new Lazy(vega); - _theta = new Lazy(theta); - _rho = new Lazy(rho); - _lambda = new Lazy(lambda); - } } } diff --git a/Common/Data/Market/NullGreeks.cs b/Common/Data/Market/NullGreeks.cs new file mode 100644 index 000000000000..5eec585322c2 --- /dev/null +++ b/Common/Data/Market/NullGreeks.cs @@ -0,0 +1,41 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Data.Market +{ + /// + /// Defines greeks that are all zero + /// + public class NullGreeks : Greeks + { + /// + public override decimal Delta => decimal.Zero; + + /// + public override decimal Gamma => decimal.Zero; + + /// + public override decimal Vega => decimal.Zero; + + /// + public override decimal Theta => decimal.Zero; + + /// + public override decimal Rho => decimal.Zero; + + /// + public override decimal Lambda => decimal.Zero; + } +} diff --git a/Common/Data/Market/OptionChain.cs b/Common/Data/Market/OptionChain.cs index 170702660470..b03244c9877d 100644 --- a/Common/Data/Market/OptionChain.cs +++ b/Common/Data/Market/OptionChain.cs @@ -200,7 +200,6 @@ public OptionChain(Symbol canonicalOptionSymbol, DateTime time, IEnumerable kvp.Value.DataFrame).ToList(); var canonicalSymbols = this.Select(kvp => kvp.Key); - var result = PandasConverter.ConcatDataFrames(dataFrames, keys: canonicalSymbols, names: new[] { "canonical", "symbol" }); - - foreach (var df in dataFrames) - { - df.Dispose(); - } - - return result; + return PandasConverter.ConcatDataFrames(dataFrames, keys: canonicalSymbols, names: new[] { "canonical", "symbol" }); } } } diff --git a/Common/Data/Market/OptionContract.cs b/Common/Data/Market/OptionContract.cs index 9b268b5b828b..63aaeedb55e6 100644 --- a/Common/Data/Market/OptionContract.cs +++ b/Common/Data/Market/OptionContract.cs @@ -26,9 +26,8 @@ namespace QuantConnect.Data.Market /// public class OptionContract : ISymbolProvider, ISymbol { - private static readonly SymbolPropertiesDatabase _symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder(); - - private IOptionData _optionData = new OptionPriceModelResultData(() => OptionPriceModelResult.None); + private IOptionData _optionData = OptionPriceModelResultData.Null; + private readonly SymbolProperties _symbolProperties; /// /// Gets the option contract's symbol @@ -56,11 +55,7 @@ public Symbol Symbol /// /// Gets the strike price multiplied by the strike multiplier /// - public decimal ScaledStrike - { - get; - private set; - } + public decimal ScaledStrike => Strike * _symbolProperties.StrikeMultiplier; /// /// Gets the expiration date @@ -172,7 +167,7 @@ public DateTime Time public OptionContract(ISecurityPrice security) { Symbol = security.Symbol; - ScaledStrike = Strike * security.SymbolProperties.StrikeMultiplier; + _symbolProperties = security.SymbolProperties; } /// @@ -183,8 +178,7 @@ public OptionContract(ISecurityPrice security) public OptionContract(OptionUniverse contractData, SymbolProperties symbolProperties) { Symbol = contractData.Symbol; - ScaledStrike = Strike * symbolProperties.StrikeMultiplier; - + _symbolProperties = symbolProperties; _optionData = new OptionUniverseData(contractData); } @@ -299,6 +293,8 @@ private interface IOptionData /// private class OptionPriceModelResultData : IOptionData { + public static readonly OptionPriceModelResultData Null = new(() => OptionPriceModelResult.None); + private readonly Lazy _optionPriceModelResult; private TradeBar _tradeBar; private QuoteBar _quoteBar; diff --git a/Common/Data/UniverseSelection/OptionUniverse.cs b/Common/Data/UniverseSelection/OptionUniverse.cs index 9952e6b38444..5b84be9e08b2 100644 --- a/Common/Data/UniverseSelection/OptionUniverse.cs +++ b/Common/Data/UniverseSelection/OptionUniverse.cs @@ -361,46 +361,22 @@ private class PreCalculatedGreeks : Greeks private readonly string _csvLine; /// - public override decimal Delta - { - get => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex); - protected set => throw new InvalidOperationException("Delta is read-only."); - } + public override decimal Delta => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex); /// - public override decimal Gamma - { - get => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 1); - protected set => throw new InvalidOperationException("Gamma is read-only."); - } + public override decimal Gamma => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 1); /// - public override decimal Vega - { - get => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 2); - protected set => throw new InvalidOperationException("Vega is read-only."); - } + public override decimal Vega => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 2); /// - public override decimal Theta - { - get => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 3); - protected set => throw new InvalidOperationException("Theta is read-only."); - } + public override decimal Theta => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 3); /// - public override decimal Rho - { - get => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 4); - protected set => throw new InvalidOperationException("Rho is read-only."); - } + public override decimal Rho => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 4); /// - public override decimal Lambda - { - get => decimal.Zero; - protected set => throw new InvalidOperationException("Lambda is read-only."); - } + public override decimal Lambda => decimal.Zero; /// /// Initializes a new default instance of the class diff --git a/Common/Securities/Option/CurrentPriceOptionPriceModel.cs b/Common/Securities/Option/CurrentPriceOptionPriceModel.cs index e337a0acbd20..29dc71cde6e5 100644 --- a/Common/Securities/Option/CurrentPriceOptionPriceModel.cs +++ b/Common/Securities/Option/CurrentPriceOptionPriceModel.cs @@ -1,11 +1,11 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); + * + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,14 +21,14 @@ namespace QuantConnect.Securities.Option { /// /// Provides a default implementation of that does not compute any - /// greeks and uses the current price for the theoretical price. + /// greeks and uses the current price for the theoretical price. /// This is a stub implementation until the real models are implemented /// public class CurrentPriceOptionPriceModel : IOptionPriceModel { /// /// Creates a new containing the current - /// and a default, empty instance of first Order + /// and a default, empty instance of first Order /// /// The option security object /// The current data slice. This can be used to access other information @@ -38,7 +38,7 @@ public class CurrentPriceOptionPriceModel : IOptionPriceModel /// price of the specified option contract public OptionPriceModelResult Evaluate(Security security, Slice slice, OptionContract contract) { - return new OptionPriceModelResult(security.Price, new ModeledGreeks()); + return new OptionPriceModelResult(security.Price, new NullGreeks()); } } -} \ No newline at end of file +} diff --git a/Common/Securities/Option/OptionPriceModelResult.cs b/Common/Securities/Option/OptionPriceModelResult.cs index f01e0cb880c8..5f8427bd716c 100644 --- a/Common/Securities/Option/OptionPriceModelResult.cs +++ b/Common/Securities/Option/OptionPriceModelResult.cs @@ -1,11 +1,11 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); + * + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,9 +26,9 @@ public class OptionPriceModelResult /// /// Represents the zero option price and greeks. /// - public static OptionPriceModelResult None { get; } = new(0, new ModeledGreeks()); + public static OptionPriceModelResult None { get; } = new(0, new NullGreeks()); - private readonly Lazy _greeks; + private readonly Lazy _greeks; private readonly Lazy _impliedVolatility; /// @@ -53,7 +53,7 @@ public decimal ImpliedVolatility /// /// Gets the various sensitivities as computed by the /// - public ModeledGreeks Greeks + public Greeks Greeks { get { @@ -66,11 +66,11 @@ public ModeledGreeks Greeks /// /// The theoretical price computed by the price model /// The sensitivities (greeks) computed by the price model - public OptionPriceModelResult(decimal theoreticalPrice, ModeledGreeks greeks) + public OptionPriceModelResult(decimal theoreticalPrice, Greeks greeks) { TheoreticalPrice = theoreticalPrice; _impliedVolatility = new Lazy(() => 0m); - _greeks = new Lazy(() => greeks); + _greeks = new Lazy(() => greeks); } /// @@ -79,11 +79,11 @@ public OptionPriceModelResult(decimal theoreticalPrice, ModeledGreeks greeks) /// The theoretical price computed by the price model /// The calculated implied volatility /// The sensitivities (greeks) computed by the price model - public OptionPriceModelResult(decimal theoreticalPrice, Func impliedVolatility, Func greeks) + public OptionPriceModelResult(decimal theoreticalPrice, Func impliedVolatility, Func greeks) { TheoreticalPrice = theoreticalPrice; _impliedVolatility = new Lazy(impliedVolatility); - _greeks = new Lazy(greeks); + _greeks = new Lazy(greeks); } } } diff --git a/Common/Symbol.cs b/Common/Symbol.cs index 709d5d786fb2..07a311ce33d6 100644 --- a/Common/Symbol.cs +++ b/Common/Symbol.cs @@ -756,13 +756,14 @@ public bool Equals(Symbol other) return true; } + var rightSymbol = right as Symbol; // Use the implicit conversion for strings - if (right is not Symbol && right is string rightStr) + if (rightSymbol == null && right is string rightStr) { return left == (Symbol)rightStr; } - return left == (right as Symbol); + return left == rightSymbol; } /// diff --git a/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs b/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs index 359d759e10f3..3ec4937bc2dc 100644 --- a/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs +++ b/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs @@ -97,7 +97,7 @@ public void ReturnsNewPrice() var priceModelMock = new Mock(); priceModelMock .Setup(s => s.Evaluate(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new OptionPriceModelResult(1000, new ModeledGreeks())); + .Returns(new OptionPriceModelResult(1000, new NullGreeks())); _option.PriceModel = priceModelMock.Object; var randomPriceGenerator = new OptionPriceModelPriceGenerator(_option); From 52a9b8965f8acf6e0b1bb07a50f7f2df6db62a0a Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 26 Sep 2024 17:41:36 -0400 Subject: [PATCH 18/38] Minor fix and add PandasConverter unit tests --- .../OptionChainFullDataRegressionAlgorithm.py | 4 +- ...ainsMultipleFullDataRegressionAlgorithm.py | 4 +- Common/Data/Market/OptionChains.cs | 7 +- .../Data/UniverseSelection/OptionUniverse.cs | 25 ----- Common/Python/PandasConverter.cs | 23 ++-- Common/Python/PandasData.cs | 42 +------- Tests/Python/PandasConverterTests.cs | 102 ++++++++++++++++++ 7 files changed, 128 insertions(+), 79 deletions(-) diff --git a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py index f3bf850a5a77..49330c2b333c 100644 --- a/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py +++ b/Algorithm.Python/OptionChainFullDataRegressionAlgorithm.py @@ -33,11 +33,11 @@ def initialize(self): # Demonstration using data frame: df = option_chain.data_frame # Get contracts expiring within 10 days, with an implied volatility greater than 0.5 and a delta less than 0.5 - contracts = df.loc[(df['expiry'] <= self.time + timedelta(days=10)) & (df['impliedvolatility'] > 0.5) & (df['delta'] < 0.5)] + contracts = df.loc[(df.expiry <= self.time + timedelta(days=10)) & (df.impliedvolatility > 0.5) & (df.delta < 0.5)] # Get the contract with the latest expiration date. # Note: the result of df.loc[] is a series, and its name is a tuple with a single element (contract symbol) - self._option_contract = contracts.loc[contracts['expiry'].idxmax()].name[0] + self._option_contract = contracts.loc[contracts.expiry.idxmax()].name[0] self.add_option_contract(self._option_contract) diff --git a/Algorithm.Python/OptionChainsMultipleFullDataRegressionAlgorithm.py b/Algorithm.Python/OptionChainsMultipleFullDataRegressionAlgorithm.py index 8cf04c969a91..a53e55ff76ff 100644 --- a/Algorithm.Python/OptionChainsMultipleFullDataRegressionAlgorithm.py +++ b/Algorithm.Python/OptionChainsMultipleFullDataRegressionAlgorithm.py @@ -42,11 +42,11 @@ def get_contract(self, chains: OptionChains, underlying: Symbol, expiry_span: ti # Index by the requested underlying, by getting all data with canonicals which underlying is the requested underlying symbol: canonicals = df.index.get_level_values('canonical') - condition = [canonical for canonical in canonicals if getattr(canonical, 'underlying') == underlying] + condition = [canonical for canonical in canonicals if canonical.underlying == underlying] df = df.loc[condition] # Get contracts expiring in the next 10 days with an implied volatility greater than 0.5 and a delta less than 0.5 - contracts = df.loc[(df['expiry'] <= self.time + expiry_span) & (df['impliedvolatility'] > 0.5) & (df['delta'] < 0.5)] + contracts = df.loc[(df.expiry <= self.time + expiry_span) & (df.impliedvolatility > 0.5) & (df.delta < 0.5)] # Select the contract with the latest expiry date contracts.sort_values(by='expiry', ascending=False, inplace=True) diff --git a/Common/Data/Market/OptionChains.cs b/Common/Data/Market/OptionChains.cs index c6661baf94d1..ccd6a207b91b 100644 --- a/Common/Data/Market/OptionChains.cs +++ b/Common/Data/Market/OptionChains.cs @@ -16,6 +16,7 @@ using Python.Runtime; using QuantConnect.Python; using System; +using System.Collections.Generic; using System.Linq; namespace QuantConnect.Data.Market @@ -25,14 +26,16 @@ namespace QuantConnect.Data.Market /// public class OptionChains : DataDictionary { + private static readonly IEnumerable _indexNames = new[] { "canonical", "symbol" }; + private readonly Lazy _dataframe; /// /// Creates a new instance of the dictionary /// public OptionChains() + : this(default) { - _dataframe = new Lazy(InitializeDataFrame); } /// @@ -74,7 +77,7 @@ private PyObject InitializeDataFrame() var dataFrames = this.Select(kvp => kvp.Value.DataFrame).ToList(); var canonicalSymbols = this.Select(kvp => kvp.Key); - return PandasConverter.ConcatDataFrames(dataFrames, keys: canonicalSymbols, names: new[] { "canonical", "symbol" }); + return PandasConverter.ConcatDataFrames(dataFrames, keys: canonicalSymbols, names: _indexNames); } } } diff --git a/Common/Data/UniverseSelection/OptionUniverse.cs b/Common/Data/UniverseSelection/OptionUniverse.cs index 5b84be9e08b2..b3afe4bf1c69 100644 --- a/Common/Data/UniverseSelection/OptionUniverse.cs +++ b/Common/Data/UniverseSelection/OptionUniverse.cs @@ -134,31 +134,6 @@ public Greeks Greeks } } - /// - /// Gets the delta. Pass-through for - /// - public decimal Delta => Greeks.Delta; - - /// - /// Gets the gamma. Pass-through for - /// - public decimal Gamma => Greeks.Gamma; - - /// - /// Gets the vega. Pass-through for - /// - public decimal Vega => Greeks.Vega; - - /// - /// Gets the theta. Pass-through for - /// - public decimal Theta => Greeks.Theta; - - /// - /// Gets the rho. Pass-through for - /// - public decimal Rho => Greeks.Rho; - /// /// Time that the data became available to use /// diff --git a/Common/Python/PandasConverter.cs b/Common/Python/PandasConverter.cs index af27405846bc..d9746e1f7eef 100644 --- a/Common/Python/PandasConverter.cs +++ b/Common/Python/PandasConverter.cs @@ -35,14 +35,17 @@ public class PandasConverter /// /// Creates an instance of . /// - static PandasConverter() + public PandasConverter() { - using (Py.GIL()) + if (_pandas == null) { - var pandas = Py.Import("pandas"); - _pandas = pandas; - // keep it so we don't need to ask for it each time - _concat = pandas.GetAttr("concat"); + using (Py.GIL()) + { + var pandas = Py.Import("pandas"); + _pandas = pandas; + // keep it so we don't need to ask for it each time + _concat = pandas.GetAttr("concat"); + } } } @@ -215,9 +218,15 @@ private static PyObject CreateDataFrame(Dictionary dataFrames, IEnumerable keys = null, IEnumerable names = null, bool sort = true, bool dropna = true) { + var dataFramesList = dataFrames.ToList(); + if (dataFramesList.Count == 0) + { + return _pandas.DataFrame(); + } + using (Py.GIL()) { - using var pyDataFrames = dataFrames.ToPyListUnSafe(); + using var pyDataFrames = dataFramesList.ToPyListUnSafe(); using var kwargs = Py.kw("sort", sort); PyList pyKeys = null; PyList pyNames = null; diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 0a1bfd867376..b55147400713 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -17,7 +17,6 @@ using QuantConnect.Data; using QuantConnect.Data.Fundamental; using QuantConnect.Data.Market; -using QuantConnect.Data.UniverseSelection; using QuantConnect.Util; using System; using System.Collections; @@ -60,16 +59,6 @@ public class PandasData private const string Suspicious = "suspicious"; private const string OpenInterest = "openinterest"; - private const string Expiry = "expiry"; - private const string Strike = "strike"; - private const string Right = "right"; - - private static readonly string[] _optionUniverseExcludedMembers = new[] - { - nameof(OptionUniverse.ID), - nameof(OptionUniverse.Greeks) - }; - private static readonly string[] _optionContractExcludedMembers = new[] { nameof(OptionContract.ID), @@ -182,9 +171,6 @@ public PandasData(object data) // C# types that are not DynamicData type if (keys == null) { - var isOptionUniverse = type == typeof(OptionUniverse); - var isOptionContract = type == typeof(OptionContract); - if (_membersByType.TryGetValue(type, out _members)) { keys = _members.ToHashSet(x => x.Name.ToLowerInvariant()); @@ -195,11 +181,7 @@ public PandasData(object data) .GetMembers(BindingFlags.Instance | BindingFlags.Public) .Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property); - if (isOptionUniverse) - { - members = members.Where(x => !_optionUniverseExcludedMembers.Contains(x.Name)); - } - else if (isOptionContract) + if (type == typeof(OptionContract)) { members = members.Where(x => !_optionContractExcludedMembers.Contains(x.Name)); } @@ -223,13 +205,6 @@ public PandasData(object data) _members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant())).ToList(); _membersByType.TryAdd(type, _members); } - - if (isOptionUniverse) - { - keys.Add(Expiry); - keys.Add(Strike); - keys.Add(Right); - } } var customColumns = new HashSet(columns) { "value" }; @@ -298,24 +273,9 @@ public void Add(object baseData) var tradeBar = baseData as TradeBar; var quoteBar = baseData as QuoteBar; Add(tradeBar, quoteBar); - - AddOptionData(baseData as OptionUniverse); } } - private void AddOptionData(OptionUniverse optionUniverse) - { - if (optionUniverse == null) - { - return; - } - - var time = optionUniverse.EndTime; - GetSerie(Expiry).Add(time, optionUniverse.Symbol.ID.Date); - GetSerie(Strike).Add(time, optionUniverse.Symbol.ID.StrikePrice); - GetSerie(Right).Add(time, optionUniverse.Symbol.ID.OptionRight); - } - /// /// Adds Lean data objects to the end of the lists /// diff --git a/Tests/Python/PandasConverterTests.cs b/Tests/Python/PandasConverterTests.cs index 1d9a6a92d006..af99d8369200 100644 --- a/Tests/Python/PandasConverterTests.cs +++ b/Tests/Python/PandasConverterTests.cs @@ -3755,6 +3755,108 @@ public void RunDataFrameFromTickHistoryRegressionAlgorithm() parameter.ExpectedFinalStatus); } + [Test] + public void ConcatenatesDataFrames() + { + using (Py.GIL()) + { + var test = PyModule.FromString("ConcatenatesDataFrames", + @" +import pandas as pd + +index1 = pd.Index(['X', 'Y'], name=""Class"") +df1 = pd.DataFrame([[1, 2], [3, 4]], index=index1, columns=[""A"", ""B""]) + +index2 = pd.Index(['L', 'M'], name=""Class"") +df2 = pd.DataFrame([[5, 6], [7, 8]], index=index2, columns=[""A"", ""B""]) + +index3 = pd.Index(['R', 'S'], name=""Class"") +df3 = pd.DataFrame([[9, 10], [11, 12]], index=index3, columns=[""A"", ""B""]) + +concatenated = pd.concat([df1, df2, df3]) +"); + + using var df1 = test.GetAttr("df1"); + using var df2 = test.GetAttr("df2"); + using var df3 = test.GetAttr("df3"); + using var expected = test.GetAttr("concatenated"); + + using var concatenated = PandasConverter.ConcatDataFrames(new[] { df1, df2, df3 }, sort: false, dropna: false); + + Assert.AreEqual(expected.GetAttr("to_string").Invoke().GetAndDispose(), + concatenated.GetAttr("to_string").Invoke().GetAndDispose()); + } + } + + [Test] + public void ConcatenatesDataFramesWithAddedIndexLevel() + { + using (Py.GIL()) + { + var test = PyModule.FromString("ConcatenatesDataFramesWithAddedIndexLevel", + @" +import pandas as pd + +index1 = pd.Index(['X', 'Y'], name=""Class"") +df1 = pd.DataFrame([[1, 2], [3, 4]], index=index1, columns=[""A"", ""B""]) + +index2 = pd.Index(['L', 'M'], name=""Class"") +df2 = pd.DataFrame([[5, 6], [7, 8]], index=index2, columns=[""A"", ""B""]) + +index3 = pd.Index(['R', 'S'], name=""Class"") +df3 = pd.DataFrame([[9, 10], [11, 12]], index=index3, columns=[""A"", ""B""]) + +concatenated = pd.concat([df1, df2, df3], keys=['df1', 'df2', 'df3'], names=['source_df']) +"); + + using var df1 = test.GetAttr("df1"); + using var df2 = test.GetAttr("df2"); + using var df3 = test.GetAttr("df3"); + using var expected = test.GetAttr("concatenated"); + + using var concatenated = PandasConverter.ConcatDataFrames(new[] { df1, df2, df3 }, + keys: new[] { "df1", "df2", "df3" }, + names: new[] { "source_df" }, + sort: false, + dropna: false); + + Assert.AreEqual(expected.GetAttr("to_string").Invoke().GetAndDispose(), + concatenated.GetAttr("to_string").Invoke().GetAndDispose()); + } + } + + [Test] + public void ConcatenateReturnsEmptyDataFrameIfInputListIsEmpty() + { + using (Py.GIL()) + { + var test = PyModule.FromString("ConcatenateReturnsEmptyDataFrameIfInputListIsEmpty", + @" +import pandas as pd + +index1 = pd.Index(['X', 'Y'], name=""Class"") +df1 = pd.DataFrame([[1, 2], [3, 4]], index=index1, columns=[""A"", ""B""]) + +index2 = pd.Index(['L', 'M'], name=""Class"") +df2 = pd.DataFrame([[5, 6], [7, 8]], index=index2, columns=[""A"", ""B""]) + +index3 = pd.Index(['R', 'S'], name=""Class"") +df3 = pd.DataFrame([[9, 10], [11, 12]], index=index3, columns=[""A"", ""B""]) + +concatenated = pd.concat([df1, df2, df3], keys=['df1', 'df2', 'df3'], names=['source_df']) +"); + + using var df1 = test.GetAttr("df1"); + using var df2 = test.GetAttr("df2"); + using var df3 = test.GetAttr("df3"); + using var expected = test.GetAttr("concatenated"); + + using var concatenated = PandasConverter.ConcatDataFrames(Array.Empty()); + + Assert.IsTrue(concatenated.GetAttr("empty").GetAndDispose()); + } + } + public IEnumerable GetHistory(Symbol symbol, Resolution resolution, IEnumerable data) where T : IBaseData { From 5ef230256ee0ab3df5c7c396a71183fd773a399a Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 26 Sep 2024 17:55:14 -0400 Subject: [PATCH 19/38] Peer review: Non-thread-safe Lazy for Python --- Algorithm/QCAlgorithm.Indicators.cs | 6 ++++-- Common/Data/Market/OptionChain.cs | 2 +- Common/Data/Market/OptionChains.cs | 2 +- Common/Python/PandasData.cs | 1 + Common/Securities/Option/OptionPriceModelResult.cs | 8 ++++---- Research/FutureHistory.cs | 3 ++- Research/OptionHistory.cs | 3 ++- 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Algorithm/QCAlgorithm.Indicators.cs b/Algorithm/QCAlgorithm.Indicators.cs index 01a9814d7354..48b22e5813bc 100644 --- a/Algorithm/QCAlgorithm.Indicators.cs +++ b/Algorithm/QCAlgorithm.Indicators.cs @@ -498,7 +498,7 @@ public ChandeKrollStop CKS(Symbol symbol, int atrPeriod, decimal atrMult, int pe InitializeIndicator(indicator, resolution, selector, symbol); return indicator; } - + /// /// Creates a new ChaikinMoneyFlow indicator. /// @@ -4004,7 +4004,9 @@ void consumeLastPoint(IndicatorDataPoint newInputPoint) indicator.Updated -= callback; return new IndicatorHistory(indicatorsDataPointsByTime, indicatorsDataPointPerProperty, - new Lazy(() => PandasConverter.GetIndicatorDataFrame(indicatorsDataPointPerProperty.Select(x => new KeyValuePair>(x.Name, x.Values))))); + new Lazy( + () => PandasConverter.GetIndicatorDataFrame(indicatorsDataPointPerProperty.Select(x => new KeyValuePair>(x.Name, x.Values))), + isThreadSafe: false)); } private Type GetDataTypeFromSelector(Func selector) diff --git a/Common/Data/Market/OptionChain.cs b/Common/Data/Market/OptionChain.cs index b03244c9877d..8f3d5dc521d4 100644 --- a/Common/Data/Market/OptionChain.cs +++ b/Common/Data/Market/OptionChain.cs @@ -95,7 +95,7 @@ public HashSet FilteredContracts private OptionChain() { DataType = MarketDataType.OptionChain; - _dataframe = new Lazy(() => new PandasConverter().GetDataFrame(this, symbolOnlyIndex: true)); + _dataframe = new Lazy(() => new PandasConverter().GetDataFrame(this, symbolOnlyIndex: true), isThreadSafe: false); } /// diff --git a/Common/Data/Market/OptionChains.cs b/Common/Data/Market/OptionChains.cs index ccd6a207b91b..109d16aa7a2b 100644 --- a/Common/Data/Market/OptionChains.cs +++ b/Common/Data/Market/OptionChains.cs @@ -44,7 +44,7 @@ public OptionChains() public OptionChains(DateTime time) : base(time) { - _dataframe = new Lazy(InitializeDataFrame); + _dataframe = new Lazy(InitializeDataFrame, isThreadSafe: false); } /// diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index b55147400713..9f76503170ab 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -181,6 +181,7 @@ public PandasData(object data) .GetMembers(BindingFlags.Instance | BindingFlags.Public) .Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property); + // TODO: Instead of this special case, we could have attributes to mark class members as excluded for pandas if (type == typeof(OptionContract)) { members = members.Where(x => !_optionContractExcludedMembers.Contains(x.Name)); diff --git a/Common/Securities/Option/OptionPriceModelResult.cs b/Common/Securities/Option/OptionPriceModelResult.cs index 5f8427bd716c..0e19e272c67b 100644 --- a/Common/Securities/Option/OptionPriceModelResult.cs +++ b/Common/Securities/Option/OptionPriceModelResult.cs @@ -69,8 +69,8 @@ public Greeks Greeks public OptionPriceModelResult(decimal theoreticalPrice, Greeks greeks) { TheoreticalPrice = theoreticalPrice; - _impliedVolatility = new Lazy(() => 0m); - _greeks = new Lazy(() => greeks); + _impliedVolatility = new Lazy(() => 0m, isThreadSafe: false); + _greeks = new Lazy(() => greeks, isThreadSafe: false); } /// @@ -82,8 +82,8 @@ public OptionPriceModelResult(decimal theoreticalPrice, Greeks greeks) public OptionPriceModelResult(decimal theoreticalPrice, Func impliedVolatility, Func greeks) { TheoreticalPrice = theoreticalPrice; - _impliedVolatility = new Lazy(impliedVolatility); - _greeks = new Lazy(greeks); + _impliedVolatility = new Lazy(impliedVolatility, isThreadSafe: false); + _greeks = new Lazy(greeks, isThreadSafe: false); } } } diff --git a/Research/FutureHistory.cs b/Research/FutureHistory.cs index a069afe703c2..a8d69aab8dd3 100644 --- a/Research/FutureHistory.cs +++ b/Research/FutureHistory.cs @@ -31,7 +31,8 @@ public class FutureHistory : DataHistory /// Create a new instance of . /// /// - public FutureHistory(IEnumerable data) : base(data, new Lazy(() => new PandasConverter().GetDataFrame(data))) + public FutureHistory(IEnumerable data) + : base(data, new Lazy(() => new PandasConverter().GetDataFrame(data), isThreadSafe: false)) { } diff --git a/Research/OptionHistory.cs b/Research/OptionHistory.cs index 83b6bd98f37a..b6f3b5f4c247 100644 --- a/Research/OptionHistory.cs +++ b/Research/OptionHistory.cs @@ -31,7 +31,8 @@ public class OptionHistory : DataHistory /// Create a new instance of . /// /// - public OptionHistory(IEnumerable data) : base(data, new Lazy(() => new PandasConverter().GetDataFrame(data))) + public OptionHistory(IEnumerable data) + : base(data, new Lazy(() => new PandasConverter().GetDataFrame(data), isThreadSafe: false)) { } From 5d0e18d3e46040cab8bc92deb335db8bf8780eaa Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 26 Sep 2024 19:17:39 -0400 Subject: [PATCH 20/38] Handle Greeks unwrapping by PandasData --- ...ainsMultipleFullDataRegressionAlgorithm.cs | 2 +- Common/Data/Market/OptionContract.cs | 25 ---- Common/Python/PandasData.cs | 108 ++++++++++++++---- 3 files changed, 89 insertions(+), 46 deletions(-) diff --git a/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs b/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs index 33dafd99d242..0e5484e5f0fc 100644 --- a/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs +++ b/Algorithm.CSharp/OptionChainsMultipleFullDataRegressionAlgorithm.cs @@ -61,7 +61,7 @@ private Symbol GetContract(OptionChains chains, Symbol underlying, TimeSpan expi // Get contracts expiring within a given span, with an implied volatility greater than 0.5 and a delta less than 0.5 .Where(contractData => contractData.ID.Date - Time <= expirySpan && contractData.ImpliedVolatility > 0.5m && - contractData.Delta < 0.5m) + contractData.Greeks.Delta < 0.5m) // Get the contract with the latest expiration date .OrderByDescending(x => x.ID.Date) .First(); diff --git a/Common/Data/Market/OptionContract.cs b/Common/Data/Market/OptionContract.cs index 63aaeedb55e6..e422711518e8 100644 --- a/Common/Data/Market/OptionContract.cs +++ b/Common/Data/Market/OptionContract.cs @@ -87,31 +87,6 @@ public Symbol Symbol /// public Greeks Greeks => _optionData.Greeks; - /// - /// Gets the delta. Pass-through for - /// - public decimal Delta => Greeks.Delta; - - /// - /// Gets the gamma. Pass-through for - /// - public decimal Gamma => Greeks.Gamma; - - /// - /// Gets the vega. Pass-through for - /// - public decimal Vega => Greeks.Vega; - - /// - /// Gets the theta. Pass-through for - /// - public decimal Theta => Greeks.Theta; - - /// - /// Gets the rho. Pass-through for - /// - public decimal Rho => Greeks.Rho; - /// /// Gets the local date time this contract's data was last updated /// diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 9f76503170ab..258325bbefc2 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -62,9 +62,22 @@ public class PandasData private static readonly string[] _optionContractExcludedMembers = new[] { nameof(OptionContract.ID), - nameof(OptionContract.Greeks) }; + private static readonly string[] _greeksMemberNames = new[] + { + nameof(Greeks.Delta), + nameof(Greeks.Gamma), + nameof(Greeks.Vega), + nameof(Greeks.Theta), + nameof(Greeks.Rho), + }; + + private static readonly MemberInfo[] _greeksMembers = typeof(Greeks) + .GetMembers(BindingFlags.Instance | BindingFlags.Public) + .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) && _greeksMemberNames.Contains(x.Name)) + .ToArray(); + // we keep these so we don't need to ask for them each time private static PyString _empty; private static PyObject _pandas; @@ -138,6 +151,7 @@ public PandasData(object data) foreach (var item in enumerable) { data = item; + baseData = data as IBaseData; break; } } @@ -206,6 +220,26 @@ public PandasData(object data) _members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant())).ToList(); _membersByType.TryAdd(type, _members); } + + // Make sure to add the greeks member names to the series so they can be added to the data frame + if (_members.Any(member => + { + var memberType = member switch + { + PropertyInfo property => property.PropertyType, + FieldInfo field => field.FieldType, + // Should not happen + _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") + }; + + return memberType.IsAssignableTo(typeof(Greeks)); + })) + { + foreach (var greek in _greeksMemberNames) + { + keys.Add(greek.ToLowerInvariant()); + } + } } var customColumns = new HashSet(columns) { "value" }; @@ -231,25 +265,31 @@ public void Add(object baseData) foreach (var member in _members) { - // TODO field/property.GetValue is expensive - var key = member.Name.ToLowerInvariant(); - var propertyMember = member as PropertyInfo; - if (propertyMember != null) + var memberType = member switch { - var propertyValue = propertyMember.GetValue(baseData); - if (_isFundamentalType && propertyMember.PropertyType.IsAssignableTo(typeof(FundamentalTimeDependentProperty))) - { - propertyValue = ((FundamentalTimeDependentProperty)propertyValue).Clone(new FixedTimeProvider(endTime)); - } - AddToSeries(key, endTime, propertyValue); - continue; + PropertyInfo property => property.PropertyType, + FieldInfo field => field.FieldType, + // Should not happen + _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") + }; + + if (!memberType.IsAssignableTo(typeof(Greeks))) + { + AddMemberToSeries(baseData, endTime, member); } else { - var fieldMember = member as FieldInfo; - if (fieldMember != null) + var greeks = member switch + { + PropertyInfo property => property.GetValue(baseData), + FieldInfo field => field.GetValue(baseData), + // Should not happen + _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") + }; + + foreach (var greekMember in _greeksMembers) { - AddToSeries(key, endTime, fieldMember.GetValue(baseData)); + AddMemberToSeries(greeks, endTime, greekMember); } } } @@ -267,13 +307,41 @@ public void Add(object baseData) AddToSeries(kvp.Key, endTime, kvp.Value); } } - else + else if (baseData is Tick tick) + { + AddTick(tick); + } + else if (baseData is TradeBar tradeBar) + { + Add(tradeBar, null); + } + else if (baseData is QuoteBar quoteBar) { - AddTick(baseData as Tick); + Add(null, quoteBar); + } + } - var tradeBar = baseData as TradeBar; - var quoteBar = baseData as QuoteBar; - Add(tradeBar, quoteBar); + private void AddMemberToSeries(object baseData, DateTime endTime, MemberInfo member) + { + // TODO field/property.GetValue is expensive + var key = member.Name.ToLowerInvariant(); + var propertyMember = member as PropertyInfo; + if (propertyMember != null) + { + var propertyValue = propertyMember.GetValue(baseData); + if (_isFundamentalType && propertyMember.PropertyType.IsAssignableTo(typeof(FundamentalTimeDependentProperty))) + { + propertyValue = ((FundamentalTimeDependentProperty)propertyValue).Clone(new FixedTimeProvider(endTime)); + } + AddToSeries(key, endTime, propertyValue); + } + else + { + var fieldMember = member as FieldInfo; + if (fieldMember != null) + { + AddToSeries(key, endTime, fieldMember.GetValue(baseData)); + } } } From 77b74f25a566cbdf9435f2dfd2eaba2c2dfb88fb Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 27 Sep 2024 10:42:45 -0400 Subject: [PATCH 21/38] PandasData cleanup --- Common/Python/PandasData.cs | 65 ++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 258325bbefc2..3611f9389246 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -59,6 +59,12 @@ public class PandasData private const string Suspicious = "suspicious"; private const string OpenInterest = "openinterest"; + #region OptionContract Members Handling + + // TODO: In the future, excluding, adding, renaming and unwrapping members (like the Greeks case) + // should be handled generically: we could define attributes so that class members can be marked as + // excluded, or to be renamed and/ or unwrapped (much like how Json attributes work) + private static readonly string[] _optionContractExcludedMembers = new[] { nameof(OptionContract.ID), @@ -78,6 +84,8 @@ public class PandasData .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) && _greeksMemberNames.Contains(x.Name)) .ToArray(); + #endregion + // we keep these so we don't need to ask for them each time private static PyString _empty; private static PyObject _pandas; @@ -195,8 +203,9 @@ public PandasData(object data) .GetMembers(BindingFlags.Instance | BindingFlags.Public) .Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property); - // TODO: Instead of this special case, we could have attributes to mark class members as excluded for pandas - if (type == typeof(OptionContract)) + // TODO: Avoid hard-coded especial cases by using something like attributes to change + // pandas conversion behavior + if (type.IsAssignableTo(typeof(OptionContract))) { members = members.Where(x => !_optionContractExcludedMembers.Contains(x.Name)); } @@ -222,22 +231,11 @@ public PandasData(object data) } // Make sure to add the greeks member names to the series so they can be added to the data frame - if (_members.Any(member => + if (_members.Any(x => GetMemberType(x).IsAssignableTo(typeof(Greeks)))) { - var memberType = member switch + foreach (var greekMemberName in _greeksMemberNames) { - PropertyInfo property => property.PropertyType, - FieldInfo field => field.FieldType, - // Should not happen - _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") - }; - - return memberType.IsAssignableTo(typeof(Greeks)); - })) - { - foreach (var greek in _greeksMemberNames) - { - keys.Add(greek.ToLowerInvariant()); + keys.Add(greekMemberName.ToLowerInvariant()); } } } @@ -265,14 +263,7 @@ public void Add(object baseData) foreach (var member in _members) { - var memberType = member switch - { - PropertyInfo property => property.PropertyType, - FieldInfo field => field.FieldType, - // Should not happen - _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") - }; - + var memberType = GetMemberType(member); if (!memberType.IsAssignableTo(typeof(Greeks))) { AddMemberToSeries(baseData, endTime, member); @@ -321,27 +312,33 @@ public void Add(object baseData) } } + private static Type GetMemberType(MemberInfo member) + { + return member switch + { + PropertyInfo property => property.PropertyType, + FieldInfo field => field.FieldType, + // Should not happen + _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") + }; + } + private void AddMemberToSeries(object baseData, DateTime endTime, MemberInfo member) { // TODO field/property.GetValue is expensive var key = member.Name.ToLowerInvariant(); - var propertyMember = member as PropertyInfo; - if (propertyMember != null) + if (member is PropertyInfo property) { - var propertyValue = propertyMember.GetValue(baseData); - if (_isFundamentalType && propertyMember.PropertyType.IsAssignableTo(typeof(FundamentalTimeDependentProperty))) + var propertyValue = property.GetValue(baseData); + if (_isFundamentalType && property.PropertyType.IsAssignableTo(typeof(FundamentalTimeDependentProperty))) { propertyValue = ((FundamentalTimeDependentProperty)propertyValue).Clone(new FixedTimeProvider(endTime)); } AddToSeries(key, endTime, propertyValue); } - else + else if (member is FieldInfo field) { - var fieldMember = member as FieldInfo; - if (fieldMember != null) - { - AddToSeries(key, endTime, fieldMember.GetValue(baseData)); - } + AddToSeries(key, endTime, field.GetValue(baseData)); } } From d58d78f7fba9f9ce4723460d483540877c669c7a Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 27 Sep 2024 12:06:30 -0400 Subject: [PATCH 22/38] Add data and other minor changes --- Common/Data/Market/OptionContract.cs | 5 +- Common/Python/PandasData.cs | 7 +- .../usa/universes/spx/20151222.csv | 234 +++++++++++++++ .../usa/universes/spx/20151223.csv | 273 ++++++++++++++++++ Tests/Algorithm/AlgorithmChainsTests.cs | 22 +- 5 files changed, 531 insertions(+), 10 deletions(-) create mode 100644 Data/indexoption/usa/universes/spx/20151222.csv create mode 100644 Data/indexoption/usa/universes/spx/20151223.csv diff --git a/Common/Data/Market/OptionContract.cs b/Common/Data/Market/OptionContract.cs index e422711518e8..26b85561bc42 100644 --- a/Common/Data/Market/OptionContract.cs +++ b/Common/Data/Market/OptionContract.cs @@ -341,7 +341,10 @@ private class OptionUniverseData : IOptionData public decimal LastPrice => _contractData.Close; - public decimal UnderlyingLastPrice => _contractData.Underlying.Price; + // TODO: Null check required for FOPs: since OptionUniverse does not support FOPs, + // these instances will by "synthetic" and will not have underlying data. + // Can be removed after FOPs are supported by OptionUniverse + public decimal UnderlyingLastPrice => _contractData?.Underlying?.Price ?? decimal.Zero; public long Volume => (long)_contractData.Volume; diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 3611f9389246..c6bf86ccadeb 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -278,9 +278,12 @@ public void Add(object baseData) _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") }; - foreach (var greekMember in _greeksMembers) + if (greeks != null) { - AddMemberToSeries(greeks, endTime, greekMember); + foreach (var greekMember in _greeksMembers) + { + AddMemberToSeries(greeks, endTime, greekMember); + } } } } diff --git a/Data/indexoption/usa/universes/spx/20151222.csv b/Data/indexoption/usa/universes/spx/20151222.csv new file mode 100644 index 000000000000..2379c86405b5 --- /dev/null +++ b/Data/indexoption/usa/universes/spx/20151222.csv @@ -0,0 +1,234 @@ +#symbol_id,symbol_value,open,high,low,close,volume,open_interest,implied_volatility,delta,gamma,vega,theta,rho +SPX WLF4IPHQKPF2|SPX 31,SPX 170616C00300000,1645.4000,1680.2000,1645.4000,1679.5000,0,,0.5645710,0.9991909,0.0000020,0.0688230,-0.0116314,4.3540311 +SPX W87JLJTDKMY6|SPX 31,SPX 160219C00400000,1599.4000,1633.6500,1599.4000,1633.0500,0,,1.2263960,0.9998213,0.0000007,0.0055627,-0.0167744,0.6379273 +SPX WHEDSZ06EEQ6|SPX 31,SPX 170120C00650000,1316.1500,1350.2000,1316.1500,1349.7500,0,,0.4460003,0.9967469,0.0000104,0.2085762,-0.0292000,6.8641754 +SPX W87JLK5489AM|SPX 31,SPX 160219C00750000,1249.9500,1284.1500,1249.9500,1283.6000,0,,0.8045122,0.9994710,0.0000029,0.0152662,-0.0310045,1.1954716 +SPX W8Z446DUDDSE|SPX 31,SPX 160318C00750000,1246.3000,1281.3500,1246.3000,1280.0500,0,375,0.7097078,0.9989528,0.0000050,0.0348394,-0.0347495,1.7648360 +SPX WHEDSZ1TXR7Y|SPX 31,SPX 170120C00750000,1218.2500,1252.3500,1218.2500,1251.3000,0,,0.4216917,0.9942234,0.0000184,0.3484318,-0.0385812,7.8685124 +SPX WQEBVQED3Z0U|SPX 31,SPX 171215C00750000,1194.5500,1230.4000,1194.5500,1228.0500,0,2225,0.3770008,0.9856338,0.0000337,1.0473987,-0.0464553,13.8605867 +SPX WHEDSYQR3H2M|SPX 31,SPX 170120C00800000,1169.5000,1203.5000,1169.5000,1202.8500,0,,0.4111890,0.9924080,0.0000240,0.4436537,-0.0443187,8.3564435 +SPX W87JLNL5H566|SPX 31,SPX 160219C00825000,1175.0500,1209.6000,1175.0500,1208.7000,0,,0.7563814,0.9991781,0.0000046,0.0229242,-0.0373710,1.3141723 +SPX WDYBH707U1HQ|SPX 31,SPX 160916C00850000,1129.1000,1162.8000,1129.1000,1162.4500,0,35,0.4329086,0.9948040,0.0000198,0.2616087,-0.0438890,6.1156183 +SPX W8Z44A25B04U|SPX 31,SPX 160318C00875000,1121.7000,1157.1500,1121.7000,1155.4500,0,,0.6397557,0.9980221,0.0000099,0.0622530,-0.0468518,2.0550019 +SPX WQEBVU2O1LDA|SPX 31,SPX 171215C00875000,1077.6000,1112.8500,1077.6000,1110.5500,0,,0.3564302,0.9759429,0.0000553,1.6243412,-0.0618721,15.8078513 +SPX WDYBH6NNEQWE|SPX 31,SPX 160916C00900000,1079.9500,1113.5500,1079.9500,1113.1500,0,35,0.4199418,0.9932746,0.0000256,0.3288676,-0.0497828,6.4546762 +SPX WQEBVQ3G80XA|SPX 31,SPX 171215C00900000,1054.4000,1089.6500,1054.4000,1087.3000,0,6,0.3522833,0.9736241,0.0000604,1.7545569,-0.0650898,16.1785666 +SPX WHEDT2YEK41A|SPX 31,SPX 170120C00925000,1048.2500,1081.9500,1048.2500,1081.3000,0,,0.3843426,0.9862823,0.0000430,0.7431784,-0.0603953,9.5360099 +SPX WDYBH71VDDZI|SPX 31,SPX 160916C00950000,1031.2000,1064.4500,1031.2000,1064.0500,0,33,0.4082785,0.9912878,0.0000331,0.4130281,-0.0567045,6.7859533 +SPX W792YEH1UXYM|SPX 31,SPX 160115C00975000,1030.4000,1064.4000,1030.4000,1064.1000,0,,0.8994549,0.9996082,0.0000031,0.0073128,-0.0407479,0.6231797 +SPX WBGPST1BBV5A|SPX 31,SPX 160617C01025000,964.3500,999.2500,964.3500,997.6500,0,14,0.4292979,0.9931313,0.0000314,0.2723833,-0.0604854,4.8812934 +SPX W87JLOQHWTIM|SPX 31,SPX 160219C01075000,925.5000,959.9000,925.5000,959.1500,0,,0.5770101,0.9981138,0.0000128,0.0490153,-0.0535211,1.7096209 +SPX W9QOMSBZJ8A6|SPX 31,SPX 160415C01100000,896.0500,931.2000,896.0500,929.3000,0,,0.4440084,0.9956028,0.0000254,0.1471616,-0.0583435,3.4053797 +SPX W792YF5UX5DA|SPX 31,SPX 160115C01125000,880.4500,914.7500,880.4500,914.2000,0,4,0.7570591,0.9993217,0.0000060,0.0121530,-0.0504513,0.7187471 +SPX WHEDSZ8G3572|SPX 31,SPX 170120C01150000,833.7000,865.9500,833.7000,865.6500,0,,0.3424358,0.9654098,0.0001054,1.6219049,-0.0993420,11.4070635 +SPX W87JLP71AAGE|SPX 31,SPX 160219C01175000,825.8500,860.4000,825.8500,859.3500,0,,0.5144098,0.9973656,0.0000194,0.0663447,-0.0612208,1.8667091 +SPX W8Z446M4249A|SPX 31,SPX 160318C01250000,748.1500,783.6500,748.1500,781.6500,0,91,0.4205737,0.9938876,0.0000414,0.1714339,-0.0755422,2.9191418 +SPX WHEDT4KAD9BI|SPX 31,SPX 170120C01275000,717.4500,748.7000,717.4500,748.3000,0,,0.3198755,0.9463878,0.0001608,2.3104699,-0.1247837,12.2571484 +SPX W87JLPVUCHV2|SPX 31,SPX 160219C01325000,676.3500,710.8500,676.3500,709.8500,0,,0.4289266,0.9954479,0.0000380,0.1084310,-0.0758182,2.0996481 +SPX WGFX5PIUHFKE|SPX 31,SPX 161216C01350000,647.5500,678.1500,647.5500,677.7500,0,6,0.3124158,0.9353554,0.0001997,2.5543624,-0.1434911,11.6644756 +SPX W792YGB7CTPQ|SPX 31,SPX 160115C01375000,630.8000,665.1500,630.8000,664.4500,0,2,0.5385865,0.9984825,0.0000177,0.0254412,-0.0668818,0.8775487 +SPX W792YAMMMXSE|SPX 31,SPX 160115C01380000,625.8000,660.1500,625.8000,659.5000,0,1,0.5339671,0.9984706,0.0000180,0.0256227,-0.0669738,0.8807319 +SPX W87JLQ92N9TA|SPX 31,SPX 160219C01405000,596.9500,632.3500,596.9500,629.8500,0,,0.3903544,0.9933209,0.0000587,0.1524337,-0.0890141,2.2201115 +SPX W87JLKG16CJY|SPX 31,SPX 160219C01410000,591.9000,627.3500,591.9000,624.9500,0,,0.3886194,0.9930834,0.0000609,0.1572128,-0.0905014,2.2272795 +SPX W792YGHTI7OU|SPX 31,SPX 160115C01415000,590.7500,625.5500,590.7500,624.5000,0,,0.5184349,0.9978520,0.0000253,0.0348963,-0.0773331,0.9022825 +SPX W9QOMYTU07SE|SPX 31,SPX 160415C01425000,575.9000,609.8500,575.9000,607.8000,0,,0.3358098,0.9781339,0.0001362,0.5959515,-0.1251014,4.3008465 +SPX W87JLKGD30NI|SPX 31,SPX 160219C01430000,572.0500,607.4500,572.0500,605.0500,0,,0.3783131,0.9924785,0.0000673,0.1692770,-0.0935352,2.2571571 +SPX W792YGOFNLNY|SPX 31,SPX 160115C01455000,550.9000,585.3000,550.9000,584.5500,0,,0.4819829,0.9976969,0.0000290,0.0371715,-0.0780442,0.9276687 +SPX WGFX5VOGFSKU|SPX 31,SPX 161216C01475000,535.4000,564.4500,535.4000,564.0000,0,4,0.2897630,0.9039086,0.0002907,3.4483957,-0.1728517,12.1559167 +SPX WQEBVQ4FY17Y|SPX 31,SPX 171215C01500000,535.4500,563.3000,535.4500,561.5500,0,1976,0.2740688,0.8508117,0.0002953,6.6684104,-0.1562808,21.6678011 +SPX W87JLQR9K38U|SPX 31,SPX 160219C01515000,488.1000,522.6500,488.1000,520.8000,0,,0.3409151,0.9880499,0.0001120,0.2537433,-0.1148475,2.3777795 +SPX W87JLQSX3FQM|SPX 31,SPX 160219C01525000,478.2000,513.3000,478.2000,510.6500,0,1,0.3362004,0.9874406,0.0001185,0.2649169,-0.1172789,2.3916774 +SPX W8Z44D1N8K8E|SPX 31,SPX 160318C01525000,477.5500,512.2500,477.5500,509.2000,0,50,0.3196344,0.9750631,0.0001840,0.5784418,-0.1472430,3.4720851 +SPX W792YH1NYDM6|SPX 31,SPX 160115C01535000,471.2000,506.2500,471.2000,504.7000,0,,0.4312298,0.9961236,0.0000518,0.0594213,-0.0966394,0.9767285 +SPX WBGPSP1FMTCE|SPX 31,SPX 160617C01550000,458.5000,490.0500,458.5000,487.7500,0,2,0.2924378,0.9293262,0.0003254,1.9225134,-0.1964160,6.7264627 +SPX W792YAPLSYOE|SPX 31,SPX 160115C01560000,446.1500,480.3500,446.1500,479.7000,0,,0.4089793,0.9959227,0.0000571,0.0621721,-0.0968909,0.9924591 +SPX WQEBVXAFNVXQ|SPX 31,SPX 171215C01575000,477.7000,504.9500,477.7000,502.9500,0,1,0.2664059,0.8235028,0.0003389,7.4392316,-0.1670299,21.7307405 +SPX W8Z446RKJ5VY|SPX 31,SPX 160318C01580000,424.4500,457.9000,424.4500,455.6000,0,,0.3036111,0.9653636,0.0002544,0.7598357,-0.1746899,3.5522891 +SPX W792YAQFKMXA|SPX 31,SPX 160115C01610000,396.3500,430.3500,396.3500,429.8000,0,,0.3752410,0.9944929,0.0000814,0.0812880,-0.1090394,1.0225308 +SPX W792YHEW95KE|SPX 31,SPX 160115C01615000,391.3500,425.7500,391.3500,424.8500,0,,0.3677265,0.9947505,0.0000796,0.0778979,-0.1052164,1.0260815 +SPX W87JLR7SXK6M|SPX 31,SPX 160219C01615000,389.6500,424.5500,389.6500,421.6000,0,,0.2970814,0.9790806,0.0002077,0.4100980,-0.1473258,2.5069858 +SPX W792YAQLIYZ2|SPX 31,SPX 160115C01620000,386.4500,420.5500,386.4500,419.8500,0,,0.3693423,0.9940498,0.0000886,0.0870680,-0.1128301,1.0283337 +SPX W8Z446S8CI32|SPX 31,SPX 160318C01620000,386.4000,419.1500,386.4000,416.7500,0,,0.2920030,0.9560963,0.0003208,0.9216154,-0.1974771,3.5993014 +SPX W9QOMZQWR5NY|SPX 31,SPX 160415C01625000,384.9000,416.8000,384.9000,414.4500,0,,0.2827285,0.9373480,0.0003818,1.4063101,-0.2144573,4.6461259 +SPX WHEDT6666ELQ|SPX 31,SPX 170120C01625000,410.9500,437.0000,410.9500,436.7000,0,,0.2633631,0.8426314,0.0004312,5.1009916,-0.2041412,13.3354292 +SPX WQEBVXIPCMEM|SPX 31,SPX 171215C01625000,440.0000,465.2000,440.0000,464.6500,0,1,0.2588607,0.8049213,0.0003711,7.9159018,-0.1717398,21.7746986 +SPX W87JLRB40966|SPX 31,SPX 160219C01635000,370.2500,404.1000,370.2500,402.0000,0,,0.2925643,0.9749739,0.0002452,0.4769439,-0.1627994,2.5248841 +SPX WDYBH7DG4TFY|SPX 31,SPX 160916C01650000,377.4000,405.5000,377.4000,403.3500,0,1,0.2646369,0.8596318,0.0004819,3.8983561,-0.2282365,9.6681473 +SPX W87JLKK60PSE|SPX 31,SPX 160219C01660000,346.1000,379.6000,346.1000,377.2500,0,,0.2858663,0.9692561,0.0002979,0.5661490,-0.1822581,2.5452730 +SPX W792YARFAN7Y|SPX 31,SPX 160115C01670000,336.4500,372.0500,336.4500,369.8000,0,,0.3349806,0.9919198,0.0001280,0.1140731,-0.1270337,1.0574753 +SPX W8Z446T246BY|SPX 31,SPX 160318C01670000,339.2000,371.4500,339.2000,368.9000,0,,0.2787353,0.9403281,0.0004290,1.1764203,-0.2319473,3.6365396 +SPX W9QOMZZ6FW4U|SPX 31,SPX 160415C01675000,338.9000,369.9500,338.9000,367.4000,0,,0.2708628,0.9182459,0.0004888,1.7247810,-0.2450943,4.6711559 +SPX WLF4IX6941NY|SPX 31,SPX 170616C01675000,386.5500,411.3000,386.5500,410.2000,0,206,0.2559680,0.7979195,0.0004432,6.9952647,-0.1969952,17.1180194 +SPX WQEBVXQZ1CVI|SPX 31,SPX 171215C01675000,403.6500,428.1500,403.6500,427.3500,0,,0.2536030,0.7837099,0.0004028,8.4161832,-0.1774755,21.6610361 +SPX W792YARL8Z9Q|SPX 31,SPX 160115C01680000,326.6000,362.3500,326.6000,359.7000,0,,0.3282050,0.9913742,0.0001383,0.1208094,-0.1303536,1.0631443 +SPX W87JLJVIYZLA|SPX 31,SPX 160219C01700000,307.9000,340.7500,307.9000,338.2500,2,100,0.2710299,0.9596416,0.0003928,0.7077202,-0.2084670,2.5764236 +SPX W87JLRMOROMM|SPX 31,SPX 160219C01705000,303.1500,336.0000,303.1500,333.4000,0,5,0.2695824,0.9580145,0.0004078,0.7307676,-0.2129538,2.5788553 +SPX W87JLKKZSE1A|SPX 31,SPX 160219C01710000,298.3500,330.6000,298.3500,328.7000,0,391,0.2671723,0.9568556,0.0004206,0.7470356,-0.2152381,2.5829192 +SPX W87JLKLHNE6M|SPX 31,SPX 160219C01740000,270.3000,302.3000,270.3000,299.8000,0,1,0.2586189,0.9452439,0.0005257,0.9038196,-0.2446421,2.5911932 +SPX W9QOMT33VZ7Y|SPX 31,SPX 160415C01750000,271.8500,301.3000,271.8500,298.4000,0,1,0.2529977,0.8794222,0.0006951,2.2908550,-0.2939513,4.6385652 +SPX W792YI21S0HA|SPX 31,SPX 160115C01755000,252.8000,287.7000,252.8000,285.3500,0,30,0.2809842,0.9843871,0.0002704,0.2021996,-0.1687400,1.1017952 +SPX W87JLKLTK2A6|SPX 31,SPX 160219C01760000,251.8500,283.2500,251.8500,280.6500,0,2,0.2534856,0.9354897,0.0006099,1.0277508,-0.2675560,2.5897937 +SPX WGFX5X22K7E6|SPX 31,SPX 161216C01775000,288.5000,312.8000,288.5000,311.1000,0,208,0.2415411,0.7701669,0.0006211,6.1408160,-0.2396644,11.9639122 +SPX WLF4IXV2692M|SPX 31,SPX 170616C01825000,278.7500,301.0500,278.7500,299.5000,0,,0.2370116,0.7189723,0.0005730,8.3738196,-0.2135787,16.3893229 +SPX W792YILW86EM|SPX 31,SPX 160115C01875000,138.2000,168.1000,138.2000,167.7500,0,311,0.2206215,0.9383794,0.0010687,0.6274656,-0.3440539,1.1170559 +SPX W792YIQUU7VY|SPX 31,SPX 160115C01905000,111.5000,139.4500,111.5000,139.4500,0,43,0.2052160,0.9110505,0.0015206,0.8304474,-0.4117672,1.0997141 +SPX WDYBHFGINR72|SPX 31,SPX 160916C01925000,167.1000,186.6500,167.1000,185.2000,0,1248,0.2168822,0.6706878,0.0009543,6.3263057,-0.2870967,8.4434270 +SPX WGFX5XQVMESU|SPX 31,SPX 161216C01925000,184.2500,203.4500,184.2500,201.9500,0,6050,0.2202923,0.6618350,0.0008204,7.3976920,-0.2569692,10.8628904 +SPX W792YIVTG9DA|SPX 31,SPX 160115C01935000,86.9000,111.9500,86.9000,111.9500,0,37,0.1942960,0.8648286,0.0021680,1.1210321,-0.5113530,1.0569459 +SPX W792YIXGZLV2|SPX 31,SPX 160115C01945000,78.9500,103.2000,78.9500,103.0000,0,25,0.1920156,0.8434360,0.0024216,1.2374284,-0.5527540,1.0345885 +SPX W792YJ2FLNCE|SPX 31,SPX 160115C01975000,56.7000,77.7500,56.7000,77.6500,3,6213,0.1793074,0.7701806,0.0032812,1.5657520,-0.6416950,0.9553907 +SPX W8Z44F5PYIY6|SPX 31,SPX 160318C01985000,79.9500,97.2000,79.9500,96.3000,1,1144,0.1918408,0.6403834,0.0019655,3.7092035,-0.4447822,2.8345966 +SPX W8Z446YCMVWU|SPX 31,SPX 160318C01990000,76.7500,93.6000,76.7500,92.8000,0,1746,0.1905150,0.6308826,0.0019966,3.7418845,-0.4451046,2.7970127 +SPX W792YJ5QOCBY|SPX 31,SPX 160115C01995000,43.4500,61.8000,43.4500,61.7000,2,3807,0.1720926,0.7044251,0.0038900,1.7815384,-0.6937457,0.8796007 +SPX W87JLSYNCQY6|SPX 31,SPX 160219C01995000,61.3500,78.0000,61.3500,77.6000,0,3,0.1797382,0.6411392,0.0025501,3.0469899,-0.5026754,1.9554142 +SPX W792YAX1Q0WE|SPX 31,SPX 160115C02010000,34.4500,50.7500,34.4500,50.5500,40,1496,0.1654320,0.6473269,0.0043521,1.9160549,-0.7130899,0.8122092 +SPX W8Z44FAOKKFI|SPX 31,SPX 160318C02015000,61.6500,76.9000,61.6500,76.0000,6473,6757,0.1826994,0.5808721,0.0021562,3.8752027,-0.4399529,2.5964864 +SPX W8Z44FDZN9F2|SPX 31,SPX 160318C02035000,50.6000,64.5000,50.6000,63.5000,20,7510,0.1771730,0.5371121,0.0022605,3.9396506,-0.4320663,2.4152438 +SPX WDYBH7K2A7F2|SPX 31,SPX 160916C02050000,92.7500,106.4500,92.7500,105.6500,0,3123,0.1480432,0.5314403,0.0015365,6.9532140,-0.2185950,7.1943522 +SPX W87JLKQS63RI|SPX 31,SPX 160219C02060000,26.9000,37.6500,26.9000,36.9500,41,3466,0.1385793,0.4488341,0.0035017,3.2259496,-0.4070410,1.4042845 +SPX W8Z446ZIB89A|SPX 31,SPX 160318C02060000,38.3000,50.0000,38.3000,49.1000,0,3158,0.1441809,0.4691114,0.0027814,3.9449177,-0.3541393,2.1460035 +SPX W87JLTBVNIWE|SPX 31,SPX 160219C02075000,21.0500,30.1000,21.0500,29.4500,202,8591,0.1335993,0.3931053,0.0035301,3.1352633,-0.3799933,1.2347037 +SPX WQEBVZL4J8MM|SPX 31,SPX 171215C02075000,159.5000,174.0500,159.5000,173.9000,0,2280,0.1502318,0.5464233,0.0009189,11.3736365,-0.1438674,18.6344625 +SPX W87JLKR42RV2|SPX 31,SPX 160219C02080000,19.2000,27.8000,19.2000,27.2000,6,615,0.1318164,0.3739989,0.0035252,3.0891213,-0.3689916,1.1761985 +SPX WDYBH7KK57KE|SPX 31,SPX 160916C02080000,77.7000,90.0500,77.7000,89.3000,0,,0.1449510,0.4844475,0.0015730,6.9695845,-0.2128389,6.6060019 +SPX W87JLKRRW426|SPX 31,SPX 160219C02120000,8.4000,13.1000,7.9500,12.4500,78,449,0.1202474,0.2256706,0.0030642,2.4494331,-0.2645901,0.7157904 +SPX W87JLTK5C9DA|SPX 31,SPX 160219C02125000,7.4500,11.5000,6.8500,11.1000,1934,10590,0.1186198,0.2078700,0.0029620,2.3357530,-0.2486666,0.6599968 +SPX W8Z44FUJ0QCU|SPX 31,SPX 160318C02135000,12.3500,18.2000,11.9000,17.0000,2,522,0.1231776,0.2423887,0.0025583,3.0999410,-0.2341397,1.1292115 +SPX W792YAZ74DJI|SPX 31,SPX 160115C02140000,0.8500,1.1750,0.6250,1.1250,139,6129,0.1120017,0.0474727,0.0017125,0.5104426,-0.1249653,0.0612411 +SPX W8Z44FXU3FCE|SPX 31,SPX 160318C02155000,8.4500,12.7000,7.8000,11.7000,0,41,0.1185776,0.1868202,0.0022835,2.6635791,-0.1929683,0.8735870 +SPX W792YAZJ11N2|SPX 31,SPX 160115C02160000,0.5250,0.9000,0.4750,0.5750,280,11869,0.1149455,0.0258133,0.0010124,0.3096968,-0.0776062,0.0333233 +SPX W87JLTQRHNCE|SPX 31,SPX 160219C02165000,2.6500,5.1500,2.1000,3.7500,138,3735,0.1083499,0.0924711,0.0018755,1.3508982,-0.1304557,0.2955089 +SPX W792YAZOZDOU|SPX 31,SPX 160115C02170000,0.4750,0.7000,0.3250,0.4500,59,8098,0.1181081,0.0202848,0.0008043,0.2527893,-0.0650134,0.0261875 +SPX W87JLKSRM4CU|SPX 31,SPX 160219C02180000,1.7500,3.9250,0.9250,2.3000,262,623,0.1051642,0.0628441,0.0014405,1.0070383,-0.0941741,0.2012153 +SPX W792YK15VXPQ|SPX 31,SPX 160115C02185000,0.4250,0.5750,0.1250,0.3750,17,1679,0.1257968,0.0162874,0.0006263,0.2096652,-0.0573423,0.0210184 +SPX W792YB00W1SE|SPX 31,SPX 160115C02190000,0.2250,0.5250,0.1500,0.3500,252,1852,0.1281431,0.0150769,0.0005754,0.1962236,-0.0546421,0.0194543 +SPX WBGPSPD0E8SU|SPX 31,SPX 160617C02250000,7.8500,11.0500,6.9500,9.8000,65,13213,0.1134352,0.1262099,0.0012854,2.9456098,-0.1009768,1.2029193 +SPX WDYBHHAO5MY6|SPX 31,SPX 160916C02325000,7.9000,10.8000,6.8500,9.4000,0,4776,0.1110559,0.1049692,0.0009363,3.1784400,-0.0713729,1.5047417 +SPX W87JLKVKTT72|SPX 31,SPX 160219C02350000,0.2500,0.2750,0.1750,0.2000,0,1549,0.1371840,0.0056798,0.0001448,0.1320484,-0.0158306,0.0181985 +SPX WBGPSOFRR41A|SPX 31,SPX 160617C02400000,0.5500,0.9750,0.5250,0.8750,2,6075,0.1049012,0.0167364,0.0002790,0.5912789,-0.0183964,0.1615706 +SPX WHEDT9UH663Y|SPX 31,SPX 170120C02425000,7.7500,10.2000,6.8500,9.0000,0,2,0.1103905,0.0869899,0.0006767,3.3555799,-0.0515788,1.8196500 +SPX WQEBW170CDWU|SPX 31,SPX 171215C02425000,38.2500,43.1000,38.2500,42.9500,0,285,0.1271522,0.2211194,0.0008136,8.5238328,-0.0860999,8.0904228 +SPX W87JLKX8D5OU|SPX 31,SPX 160219C02450000,0.2000,0.2250,0.1250,0.1500,5,5,0.1675611,0.0036438,0.0000798,0.0888465,-0.0129531,0.0116403 +SPX WGFX5Q11E8ZY|SPX 31,SPX 161216C02450000,4.4500,6.5000,3.2000,5.1000,0,1555,0.1073466,0.0572345,0.0005286,2.3224112,-0.0377441,1.0987418 +SPX WHEDSZVLM03Y|SPX 31,SPX 170120C02550000,1.6000,3.5500,1.0750,2.4500,0,60,0.1053714,0.0294781,0.0003003,1.4212355,-0.0205668,0.6229424 +SPX WQEBVR84S7WU|SPX 31,SPX 171215C02550000,18.4500,21.1500,18.2000,20.9500,0,1002,0.1197581,0.1303540,0.0006166,6.0836361,-0.0570684,4.8529920 +SPX W9QOMSEGU90U|SPX 31,SPX 160415C02600000,0.4000,0.4000,0.2000,0.2000,0,,0.1574953,0.0037085,0.0000616,0.1263486,-0.0089016,0.0230657 +SPX WLF4IQP2E972|SPX 31,SPX 170616C02650000,3.0250,4.3500,2.4250,3.3500,0,124,0.1071022,0.0335799,0.0002808,1.8545378,-0.0201287,0.9657172 +SPX WHEDSYU26626|SPX 31,SPX 170120C02800000,0.7500,1.4500,0.1750,0.7500,0,42,0.1214194,0.0090302,0.0000947,0.5166668,-0.0084376,0.1908371 +SPX WLF4IPLVF2NI|SPX 31,SPX 170616C02800000,1.1750,2.6500,0.8750,1.7000,0,32,0.1139066,0.0174242,0.0001522,1.0691848,-0.0121749,0.5017116 +SPX WGFX5OX0PJDA|SPX 31,SPX 161216C03000000,0.2000,0.2000,0.1750,0.1750,0,3407,0.1309221,0.0023137,0.0000273,0.1463881,-0.0027910,0.0447262 +SPX WQEBVQ2GK5SE|SPX 31,SPX 171215C03000000,1.0500,1.9250,0.8500,1.8500,0,619,0.1171266,0.0162029,0.0001204,1.1614247,-0.0102573,0.6180910 +SPX WHEDSYV1W6CU|SPX 31,SPX 170120C03400000,1.1000,2.4000,0.2000,1.1000,0,,0.1937197,0.0085389,0.0000565,0.4919384,-0.0125294,0.1762280 +SPX WQEBVQ7R0Q7I|SPX 31,SPX 171215C03500000,0.8000,0.8000,0.8000,0.8000,0,855,0.1427460,0.0063931,0.0000439,0.5162667,-0.0054291,0.2424916 +SPX 30GIDYEOYNA5Q|SPX 31,SPX 170120P00200000,0.6250,2.0250,0.5500,0.6250,0,2500,0.8443776,-0.0009787,0.0000018,0.0699676,-0.0074186,-0.0283154 +SPX 30GIDYEP4LM7I|SPX 31,SPX 170120P00300000,0.6750,1.3750,0.2750,0.6750,0,3,0.7005073,-0.0013026,0.0000029,0.0909172,-0.0079835,-0.0359899 +SPX 308349M1AZKTQ|SPX 31,SPX 160318P00400000,0.1000,0.1500,0.0750,0.0750,2200,2449,1.0398633,-0.0002530,0.0000009,0.0093601,-0.0056187,-0.0013980 +SPX 30PIC1625MU5Q|SPX 31,SPX 171215P00600000,2.5500,2.5500,2.0750,2.1000,0,4598,0.3838042,-0.0050724,0.0000133,0.4203998,-0.0108120,-0.2465638 +SPX 307BJR02NYQ6M|SPX 31,SPX 160219P00650000,0.1500,0.3750,0.1000,0.1000,0,5594,0.9151450,-0.0004642,0.0000022,0.0135268,-0.0105762,-0.0016734 +SPX 308349M1SUKZ2|SPX 31,SPX 160318P00700000,0.3250,0.5000,0.2500,0.3250,0,140,0.7828174,-0.0013311,0.0000056,0.0433964,-0.0195842,-0.0071909 +SPX 30PIC162BL67I|SPX 31,SPX 171215P00700000,5.7000,6.6500,3.2000,5.2250,0,762,0.3857143,-0.0113956,0.0000270,0.8572045,-0.0220744,-0.5639604 +SPX 307BJR04BI2OE|SPX 31,SPX 160219P00750000,0.1750,0.4000,0.0750,0.1000,200,5105,0.8045122,-0.0005290,0.0000029,0.0152662,-0.0104893,-0.0018845 +SPX 30FJXB584J5ZI|SPX 31,SPX 161216P00750000,1.5500,1.8750,1.3500,1.4500,3,729,0.4265415,-0.0046894,0.0000158,0.2762895,-0.0160952,-0.1084097 +SPX 308349M1YSX0U|SPX 31,SPX 160318P00800000,0.2500,0.2500,0.2000,0.2250,0,850,0.6660016,-0.0011170,0.0000056,0.0369614,-0.0141828,-0.0059213 +SPX 30GIDYF2OQX3I|SPX 31,SPX 170120P00850000,3.6750,4.2500,1.8750,3.1750,0,7,0.3998820,-0.0096719,0.0000305,0.5486448,-0.0271888,-0.2473751 +SPX 308UOS8AUWDKE|SPX 31,SPX 160415P00900000,0.4250,0.5000,0.2500,0.3500,0,,0.5318746,-0.0018288,0.0000096,0.0667115,-0.0154010,-0.0127802 +SPX 30D2BMMMUOKA6|SPX 31,SPX 160916P00900000,2.3000,3.6250,1.1500,1.8000,0,29,0.4199418,-0.0067254,0.0000256,0.3288676,-0.0253059,-0.1140584 +SPX 30GIDYIXLTXF2|SPX 31,SPX 170120P00925000,5.1500,5.5000,2.9250,4.5000,0,13,0.3843426,-0.0137177,0.0000430,0.7431784,-0.0353252,-0.3507602 +SPX 30KJ4O9PF2U0E|SPX 31,SPX 170616P00925000,9.1000,9.7500,6.4000,8.1000,0,12,0.3637835,-0.0207223,0.0000552,1.2390154,-0.0402500,-0.7467432 +SPX 306D33QEPWD1Q|SPX 31,SPX 160115P00950000,0.0750,0.2000,0.0750,0.2000,0,8518,1.0431385,-0.0012301,0.0000076,0.0210001,-0.0468044,-0.0017335 +SPX 30PIC16GVGHE6|SPX 31,SPX 171215P00950000,16.2000,16.2000,12.8000,14.7000,0,4,0.3444228,-0.0315524,0.0000718,2.0363875,-0.0463168,-1.5649583 +SPX 306D33UG94RCE|SPX 31,SPX 160115P00975000,0.0750,0.2000,0.0500,0.0500,0,1343,0.8994549,-0.0003918,0.0000031,0.0073128,-0.0140527,-0.0005434 +SPX 30FJXB4VW2ONI|SPX 31,SPX 161216P01000000,6.2000,7.1500,3.8000,5.3000,0,19663,0.3764142,-0.0169775,0.0000553,0.8519946,-0.0435273,-0.3929798 +SPX 30PIC16IIZTVY|SPX 31,SPX 171215P01050000,22.8500,22.8500,18.9000,20.7000,0,344,0.3303574,-0.0443633,0.0000988,2.6897160,-0.0583724,-2.2038466 +SPX 308349QYFBRE6|SPX 31,SPX 160318P01075000,0.6500,0.8000,0.3500,0.5750,0,73,0.5123902,-0.0034421,0.0000204,0.1026910,-0.0302546,-0.0179677 +SPX 30PIC1B77R33I|SPX 31,SPX 171215P01125000,28.9000,28.9000,25.5000,26.3500,0,40,0.3206743,-0.0562694,0.0001231,3.2511130,-0.0681918,-2.7998683 +SPX 30GIDYKB7YC8E|SPX 31,SPX 170120P01225000,17.1500,17.4500,13.5500,14.7500,0,1,0.3283348,-0.0450194,0.0001363,2.0096120,-0.0807390,-1.1506805 +SPX 30GIDYF9AWB2M|SPX 31,SPX 170120P01250000,18.7000,18.9000,15.0000,16.1000,0,6050,0.3241007,-0.0491715,0.0001482,2.1569074,-0.0854434,-1.2569280 +SPX 307BJR0CR5572|SPX 31,SPX 160219P01260000,0.7000,0.8000,0.3250,0.4500,0,13,0.4663332,-0.0036481,0.0000287,0.0889417,-0.0353161,-0.0126139 +SPX 306D33QK0F2MM|SPX 31,SPX 160115P01270000,0.2000,0.3250,0.0750,0.0750,0,932,0.6111681,-0.0008357,0.0000091,0.0147294,-0.0192157,-0.0011388 +SPX 306D33Q15R25Q|SPX 31,SPX 160115P01300000,0.1750,0.3000,0.0750,0.0750,0,19034,0.5829447,-0.0008749,0.0000099,0.0153634,-0.0191148,-0.0011899 +SPX 30PIC163BB6I6|SPX 31,SPX 171215P01300000,47.6500,47.6500,42.9500,43.5000,0,3169,0.2969671,-0.0912619,0.0001925,4.7096920,-0.0903860,-4.5491285 +SPX 308349S3RRFQM|SPX 31,SPX 160318P01325000,1.7750,1.7750,0.7250,1.1250,2,3552,0.3872169,-0.0083068,0.0000590,0.2247105,-0.0498798,-0.0427379 +SPX 30GIDYFAYFNKE|SPX 31,SPX 170120P01350000,26.2000,26.2000,21.6000,22.5000,0,3,0.3069835,-0.0686883,0.0002034,2.8045133,-0.1047024,-1.7560280 +SPX 308349MNGQAA6|SPX 31,SPX 160318P01380000,2.3250,2.3250,0.7500,1.4500,1,24,0.3667209,-0.0109883,0.0000795,0.2869682,-0.0602726,-0.0564429 +SPX 306D33QMNOFF2|SPX 31,SPX 160115P01430000,0.3000,0.4750,0.1750,0.2000,0,552,0.5113301,-0.0024623,0.0000290,0.0394902,-0.0430688,-0.0033419 +SPX 306D33QMTMRGU|SPX 31,SPX 160115P01440000,0.3250,0.4750,0.1500,0.1750,0,561,0.4955332,-0.0022421,0.0000275,0.0362796,-0.0383422,-0.0030384 +SPX 30D2BMN9CBXU6|SPX 31,SPX 160916P01450000,22.4000,22.4000,17.4000,18.3500,0,1071,0.3028158,-0.0706492,0.0002554,2.3641894,-0.1289329,-1.1946741 +SPX 30GIDYFCLZ026|SPX 31,SPX 170120P01450000,35.7500,35.7500,29.9500,30.8500,0,201,0.2902959,-0.0940248,0.0002726,3.5550322,-0.1247479,-2.4044279 +SPX 30PIC16P557V2|SPX 31,SPX 171215P01450000,70.1000,70.1000,63.9000,63.9000,0,574,0.2782710,-0.1320766,0.0002678,6.1390476,-0.1089578,-6.5997131 +SPX 307BJR6GJLTNY|SPX 31,SPX 160219P01455000,1.5250,1.5250,0.6500,0.8750,0,2,0.3659108,-0.0084182,0.0000768,0.1869014,-0.0580937,-0.0288457 +SPX 306D33QNHG3NY|SPX 31,SPX 160115P01480000,0.3750,0.5250,0.1750,0.2000,0,580,0.4657679,-0.0026908,0.0000345,0.0427875,-0.0424922,-0.0036400 +SPX 308349MP49MRY|SPX 31,SPX 160318P01480000,3.8500,3.8500,1.4500,2.4500,0,10,0.3334475,-0.0191770,0.0001413,0.4633849,-0.0883165,-0.0983126 +SPX 308349MPA7YTQ|SPX 31,SPX 160318P01490000,4.0500,4.0500,1.5750,2.5750,0,3,0.3300468,-0.0202416,0.0001494,0.4851356,-0.0914973,-0.1037477 +SPX 307BJR6QGTWMM|SPX 31,SPX 160219P01515000,2.1250,3.3000,0.8750,1.2000,0,273,0.3409151,-0.0119501,0.0001120,0.2537433,-0.0734070,-0.0408799 +SPX 308349MPS2YZ2|SPX 31,SPX 160318P01520000,4.8000,4.8000,1.9500,3.0500,3,10,0.3208775,-0.0241289,0.0001783,0.5626700,-0.1030953,-0.1236270 +SPX 306D33WZ7OUI6|SPX 31,SPX 160115P01525000,0.4250,0.5000,0.1750,0.2250,699,5304,0.4310307,-0.0032271,0.0000439,0.0504036,-0.0463059,-0.0043561 +SPX 30AKPYBB7H17Y|SPX 31,SPX 160617P01525000,16.6500,16.8000,11.9000,12.7000,0,1685,0.2976933,-0.0634037,0.0002939,1.7676770,-0.1444567,-0.6897765 +SPX 30D2BMNAZVABY|SPX 31,SPX 160916P01550000,31.6500,31.6500,24.9500,26.1000,0,6313,0.2828323,-0.0998917,0.0003546,3.0658873,-0.1552669,-1.6884516 +SPX 306D33X5TU8HA|SPX 31,SPX 160115P01565000,0.4750,1.9000,0.2000,0.3000,0,836,0.4084018,-0.0044194,0.0000615,0.0668168,-0.0581412,-0.0059602 +SPX 307BJR70E1ZLA|SPX 31,SPX 160219P01575000,3.0500,4.0500,1.2000,1.6250,122,590,0.3152644,-0.0168458,0.0001627,0.3410775,-0.0911333,-0.0575203 +SPX 307BJR721LC32|SPX 31,SPX 160219P01585000,3.2500,4.2000,1.0500,1.7250,120,180,0.3114357,-0.0179751,0.0001741,0.3605102,-0.0951334,-0.0613636 +SPX 30D2BMMO0CWMM|SPX 31,SPX 160916P01600000,37.5500,37.5500,29.9000,31.0000,45,6697,0.2735601,-0.1185646,0.0004147,3.4678194,-0.1692781,-2.0046058 +SPX 308349THDVUJY|SPX 31,SPX 160318P01625000,8.7500,8.7500,4.4750,5.8000,0,3101,0.2920030,-0.0459574,0.0003329,0.9561637,-0.1589159,-0.2354242 +SPX 30AKPY52AFZ7Y|SPX 31,SPX 160617P01650000,27.8000,27.8000,20.6000,21.5000,0,13620,0.2707476,-0.1071860,0.0004796,2.6232658,-0.1936445,-1.1664779 +SPX 307BJR0JV5JBI|SPX 31,SPX 160219P01690000,6.4500,6.4500,3.4000,3.4000,0,71,0.2729638,-0.0368083,0.0003618,0.6565590,-0.1513829,-0.1254444 +SPX 307BJQZUQ8SZ2|SPX 31,SPX 160219P01700000,6.9000,6.9000,3.4000,3.7500,6158,18442,0.2710299,-0.0403584,0.0003928,0.7077202,-0.1619661,-0.1375836 +SPX 306D33XSZD3E6|SPX 31,SPX 160115P01705000,1.1750,1.1750,0.5000,0.6000,2000,2775,0.3133436,-0.0106094,0.0001736,0.1447693,-0.0964663,-0.0142313 +SPX 308UOSG6NEFZI|SPX 31,SPX 160415P01725000,22.3000,22.3000,14.9500,15.9000,0,79,0.2588558,-0.1059929,0.0006197,2.0895153,-0.2301147,-0.7271863 +SPX 30GIDYMLWTOXA|SPX 31,SPX 170120P01725000,79.4000,79.4000,68.5500,69.3000,0,92,0.2483602,-0.2064883,0.0005421,6.0476693,-0.1769886,-5.3019957 +SPX 307BJR0KOX7KE|SPX 31,SPX 160219P01740000,9.1500,9.5500,4.5000,5.1000,5,242,0.2586189,-0.0547561,0.0005257,0.9038196,-0.1970470,-0.1866730 +SPX 308349MTF2C26|SPX 31,SPX 160318P01740000,17.0500,17.0500,10.3000,11.4000,0,640,0.2595655,-0.0905001,0.0006334,1.6172631,-0.2376634,-0.4635874 +SPX 307BJR0L0TVNY|SPX 31,SPX 160219P01760000,10.5500,10.8000,5.3000,6.0500,21,125,0.2534856,-0.0645103,0.0006099,1.0277508,-0.2194139,-0.2200020 +SPX 307BJR0LCQJRI|SPX 31,SPX 160219P01780000,12.1500,12.1500,6.4500,7.0500,2,771,0.2472524,-0.0750139,0.0007023,1.1543217,-0.2401216,-0.2558441 +SPX 306D33Y7V77U6|SPX 31,SPX 160115P01795000,2.8250,3.5750,1.1500,1.1750,7,708,0.2580984,-0.0230741,0.0004104,0.2818623,-0.1543594,-0.0308682 +SPX 307BJR82FEYY6|SPX 31,SPX 160219P01805000,14.4000,14.4000,8.0500,8.5000,78,705,0.2391202,-0.0902500,0.0008346,1.3267678,-0.2665147,-0.3078272 +SPX 306D33QSXX5AM|SPX 31,SPX 160115P01810000,3.3500,3.3500,1.0750,1.3750,5,4542,0.2502900,-0.0272511,0.0004865,0.3240449,-0.1720044,-0.0364482 +SPX 307BJR0LULJWU|SPX 31,SPX 160219P01810000,14.9500,14.9500,7.8000,8.8500,90,657,0.2377142,-0.0938400,0.0008642,1.3656482,-0.2726242,-0.3200978 +SPX 308349MUKQOEM|SPX 31,SPX 160318P01810000,25.3000,25.3000,16.6500,17.3000,10,66,0.2402146,-0.1360801,0.0009163,2.1652120,-0.2930407,-0.6974356 +SPX 30GIDYN2G75V2|SPX 31,SPX 170120P01825000,104.3500,104.3500,92.1500,92.1500,0,1293,0.2341101,-0.2670830,0.0006628,6.9695491,-0.1894292,-6.8779061 +SPX 308349MV8K0LQ|SPX 31,SPX 160318P01850000,31.5500,31.5500,20.5500,21.9000,72,45165,0.2287598,-0.1708180,0.0011188,2.5175600,-0.3232809,-0.8758051 +SPX 307BJR8CCN1WU|SPX 31,SPX 160219P01865000,22.0000,22.0000,12.9500,13.3000,20,56,0.2192485,-0.1402251,0.0012465,1.8167400,-0.3330349,-0.4784354 +SPX 308349UMQBIWE|SPX 31,SPX 160318P01875000,36.2000,36.2000,24.3000,25.4000,0,12666,0.2222305,-0.1972509,0.0012596,2.7536321,-0.3425611,-1.0119070 +SPX 307BJR8FNPQWE|SPX 31,SPX 160219P01885000,25.3000,25.3000,15.1500,15.6000,0,54,0.2132588,-0.1630226,0.0014165,2.0081747,-0.3573504,-0.5564415 +SPX 30D2BMMOI7WRY|SPX 31,SPX 160916P01900000,99.1500,99.1500,84.8000,84.8500,2,11412,0.2222040,-0.3070082,0.0009043,6.1418876,-0.2347999,-5.2256279 +SPX 30KJ4O5JL3VLA|SPX 31,SPX 170616P01900000,158.4500,158.4500,142.9500,143.3000,0,2138,0.2286266,-0.3277939,0.0006362,8.9685613,-0.1671467,-12.0352779 +SPX 307BJR8IYSFVY|SPX 31,SPX 160219P01905000,29.1000,29.1000,17.8000,18.3500,5,1327,0.2075139,-0.1894886,0.0016012,2.2088872,-0.3815945,-0.6471338 +SPX 30AKPYD5CYWZ2|SPX 31,SPX 160617P01925000,80.7500,80.7500,65.2500,65.8500,2,13145,0.2127640,-0.3103932,0.0011673,5.0171623,-0.2817946,-3.3931857 +SPX 30PIC1EVIQULQ|SPX 31,SPX 171215P01925000,199.0000,199.0000,183.0000,183.4500,0,52,0.2296949,-0.3442662,0.0005584,10.5672544,-0.1435190,-17.5456483 +SPX 30GIDYFKVNQJ2|SPX 31,SPX 170120P01950000,145.2500,145.2500,127.3000,128.4500,0,911,0.2168368,-0.3599549,0.0008140,7.9287154,-0.1943503,-9.3170030 +SPX 307BJR8SW0IUM|SPX 31,SPX 160219P01965000,44.5000,44.5000,29.2000,29.6000,0,56,0.1901740,-0.2928870,0.0022178,2.8038710,-0.4396276,-1.0024895 +SPX 306D33QVR4U4U|SPX 31,SPX 160115P01980000,27.3500,27.3500,13.3500,13.6000,116,8380,0.1774544,-0.2449790,0.0034339,1.6216499,-0.6017653,-0.3284500 +SPX 30GIDYEOYPFBI|SPX 31,SPX 170120P02000000,165.2000,165.2000,145.8500,146.5500,0,1406,0.2106717,-0.4024716,0.0008667,8.2012553,-0.1925597,-10.4498772 +SPX 306D33Z890UPA|SPX 31,SPX 160115P02015000,40.6500,40.6500,22.1000,22.2000,132,1911,0.1624016,-0.3729958,0.0045185,1.9528468,-0.6572466,-0.5009800 +SPX 306D33QWEY6BY|SPX 31,SPX 160115P02020000,43.0500,43.0500,23.4000,23.8000,203,4859,0.1607398,-0.3951520,0.0046440,1.9865465,-0.6606104,-0.5309671 +SPX 308349VGHZRSE|SPX 31,SPX 160318P02055000,96.7500,96.7500,73.8500,74.4000,0,3194,0.1436139,-0.5172686,0.0027982,3.9530763,-0.2980601,-2.6439984 +SPX 306D33ZI68XNY|SPX 31,SPX 160115P02075000,76.6000,76.6000,48.4500,48.6500,138,15372,0.1302403,-0.6899382,0.0052513,1.8201273,-0.4674408,-0.9311358 +SPX 30GIDYO7SMU7I|SPX 31,SPX 170120P02075000,199.6500,199.6500,176.9000,178.1000,0,459,0.1443269,-0.4879257,0.0013036,8.4513182,-0.1238483,-12.1462246 +SPX 308349VS2R78U|SPX 31,SPX 160318P02125000,141.1000,141.1000,111.6500,112.7000,0,2005,0.1255134,-0.7282786,0.0026645,3.2898223,-0.1956009,-3.7528466 +SPX 30AKPY5AK4POU|SPX 31,SPX 160617P02150000,186.4500,186.4500,159.3000,159.3000,0,2938,0.1302054,-0.6862551,0.0019162,5.0405047,-0.1429440,-7.4618434 +SPX 30FJXBEVFBWJ2|SPX 31,SPX 161216P02175000,248.5000,248.5000,221.8500,222.9000,0,206,0.1372965,-0.6306911,0.0013584,7.6342048,-0.1056634,-14.4357949 +SPX 30PIC1G0V6IY6|SPX 31,SPX 171215P02175000,317.5000,317.5000,293.8500,295.5500,0,2,0.1444271,-0.5471694,0.0009556,11.3711186,-0.0769349,-26.4628765 +SPX 307BJR0RYVXQM|SPX 31,SPX 160219P02180000,180.2500,180.2500,145.4500,146.9500,0,2,0.1051642,-0.9371559,0.0014405,1.0070383,-0.0345435,-3.2790998 +SPX 306D34020P3LA|SPX 31,SPX 160115P02195000,189.1500,189.1500,154.4000,155.4500,0,2,0.1274493,-0.9877998,0.0004819,0.1634542,0.0148439,-1.3884289 +SPX 307BJQZVK0H7Y|SPX 31,SPX 160219P02200000,199.2500,199.2500,164.3500,165.9000,0,462,0.1055729,-0.9592075,0.0010172,0.7138919,-0.0066290,-3.3814846 +SPX 307BJR0SGQXVY|SPX 31,SPX 160219P02210000,208.8500,208.8500,174.3500,175.6500,0,,0.1030093,-0.9710228,0.0007871,0.5390252,0.0112908,-3.4352116 +SPX 30FJXBF3P0MZY|SPX 31,SPX 161216P02225000,281.2000,281.2000,251.6500,253.7000,0,53,0.1291852,-0.7054864,0.0013191,6.9753209,-0.0802204,-16.2270317 +SPX 306D33R01XJF2|SPX 31,SPX 160115P02240000,234.0000,234.0000,199.5500,200.3000,0,,0.1556804,-0.9906471,0.0003130,0.1296842,0.0176101,-1.4209162 +SPX 30KJ4O6HNIOLQ|SPX 31,SPX 170616P02250000,332.8500,332.8500,306.6500,306.6500,0,1,0.1354007,-0.6645271,0.0010841,9.0515885,-0.0693236,-23.7502506 +SPX 307BJRA4ULL66|SPX 31,SPX 160219P02255000,253.2500,253.2500,218.8000,219.9000,0,27,0.1072147,-0.9889949,0.0003314,0.2362197,0.0393797,-3.5646897 +SPX 307BJR0TGGY6M|SPX 31,SPX 160219P02270000,268.2000,268.2000,233.7500,234.6500,0,42,0.1097526,-0.9915136,0.0002580,0.1882298,0.0439272,-3.5967280 +SPX 306D340F8ZVJI|SPX 31,SPX 160115P02275000,268.9000,268.9000,234.8500,235.2000,1,989,0.1502243,-0.9977869,0.0000897,0.0358539,0.0506404,-1.4524972 +SPX 307BJRA85OA5Q|SPX 31,SPX 160219P02275000,273.1500,273.1500,238.6500,239.8000,0,,0.1165480,-0.9890666,0.0003032,0.2348800,0.0381765,-3.5968935 +SPX 307BJRA9T7MNI|SPX 31,SPX 160219P02285000,283.2500,283.2500,248.5000,249.8000,0,,0.1157191,-0.9918963,0.0002350,0.1807532,0.0441380,-3.6219236 +SPX 308349M4G3XRI|SPX 31,SPX 160318P02300000,300.9000,300.9000,265.5500,267.3500,0,8,0.1035636,-0.9898057,0.0002639,0.2688163,0.0461874,-5.3809359 +SPX 308349N3CAF0U|SPX 31,SPX 160318P02340000,341.1000,341.1000,306.1500,307.4500,0,,0.1132381,-0.9924401,0.0001857,0.2068420,0.0499831,-5.4878901 +SPX 306D33R1VF7YM|SPX 31,SPX 160115P02350000,344.0000,344.0000,309.1000,310.1000,0,1886,0.1973292,-0.9974908,0.0000765,0.0401692,0.0472415,-1.5001055 +SPX 30GIDYFRHT4I6|SPX 31,SPX 170120P02350000,381.8500,381.8500,349.5500,351.0000,0,,0.1170745,-0.8454231,0.0009586,5.0411630,-0.0192518,-21.9173430 +SPX 30D2BMXI546SU|SPX 31,SPX 160916P02375000,391.1500,391.1500,355.9500,357.9000,0,,0.1071692,-0.9375024,0.0006564,2.1501176,0.0183033,-16.4332837 +SPX 30D2BMMPBZL0U|SPX 31,SPX 160916P02400000,414.8000,414.8000,379.5500,381.4500,0,1,0.1056372,-0.9528673,0.0005327,1.7200003,0.0288819,-16.8356699 +SPX 306D33R3IYKGE|SPX 31,SPX 160115P02450000,443.8500,443.8500,409.2000,410.1000,0,139,0.3519454,-0.9762871,0.0003038,0.2923100,-0.1485294,-1.5791527 +SPX 30PIC177C21AM|SPX 31,SPX 171215P02550000,586.6000,586.6000,556.1000,556.1000,0,,0.1197581,-0.8696460,0.0006166,6.0836361,0.0114237,-44.6917555 +SPX 306D33Q3H3QUM|SPX 31,SPX 160115P02700000,693.6000,693.6000,658.9500,659.9000,0,15,0.3536146,-0.9989883,0.0000187,0.0175562,0.0605847,-1.7259446 +SPX 30GIDYG1F17GU|SPX 31,SPX 170120P02950000,959.0500,959.0500,924.9500,925.3500,0,32,0.1368213,-0.9928582,0.0000684,0.4203798,0.0722799,-31.3804655 +SPX 30AKPY4BI21JI|SPX 31,SPX 160617P03000000,1003.7000,1003.7000,969.6500,970.1500,0,1,0.1885837,-0.9977076,0.0000268,0.1019835,0.0762484,-14.4853822 +SPX 30KJ4O5LKJW6M|SPX 31,SPX 170616P03100000,1108.9500,1108.9500,1074.8000,1075.0000,0,,0.1352842,-0.9911172,0.0000715,0.5966810,0.0757556,-45.0474478 diff --git a/Data/indexoption/usa/universes/spx/20151223.csv b/Data/indexoption/usa/universes/spx/20151223.csv new file mode 100644 index 000000000000..27f860dd5333 --- /dev/null +++ b/Data/indexoption/usa/universes/spx/20151223.csv @@ -0,0 +1,273 @@ +#symbol_id,symbol_value,open,high,low,close,volume,open_interest,implied_volatility,delta,gamma,vega,theta,rho +SPX X0JMQES2VX66|SPX 31,SPX 181221C00300000,1631.1500,1646.4000,1626.5000,1646.4000,0,3,0.5074832,0.9961985,0.0000062,0.4044020,-0.0170694,8.4016626 +SPX W8Z44629O3HQ|SPX 31,SPX 160318C00500000,1529.5000,1554.7000,1529.5000,1551.9500,0,44,0.9496134,0.9995536,0.0000017,0.0159735,-0.0225226,1.1641986 +SPX WHEDSYQ98GXA|SPX 31,SPX 170120C00500000,1497.4000,1524.2000,1497.4000,1519.9000,0,,0.5337475,0.9978515,0.0000059,0.1449944,-0.0232440,5.2738365 +SPX W8Z446AJAOSU|SPX 31,SPX 160318C00550000,1479.6000,1504.8000,1479.6000,1502.1000,0,32,0.9009002,0.9994409,0.0000022,0.0196735,-0.0253789,1.2802607 +SPX W9QOMSB5RK1A|SPX 31,SPX 160415C00600000,1427.3500,1455.3500,1427.3500,1450.0000,0,,0.7585941,0.9991593,0.0000033,0.0330273,-0.0273840,1.8516596 +SPX WDYBH6WWRCI6|SPX 31,SPX 160916C00650000,1360.1000,1387.3000,1360.1000,1382.7000,0,,0.5110809,0.9979943,0.0000070,0.1123068,-0.0282824,4.6918128 +SPX WBGPSOO7C1E6|SPX 31,SPX 160617C00750000,1270.4000,1297.6000,1270.4000,1293.0500,0,1832,0.5279093,0.9984403,0.0000067,0.0725575,-0.0312097,3.5890924 +SPX WQEBVTUECUWE|SPX 31,SPX 171215C00825000,1157.3500,1183.8000,1157.3500,1179.2000,0,,0.3640240,0.9814946,0.0000429,1.3162543,-0.0540443,15.0822757 +SPX WLF4IPIQAPPQ|SPX 31,SPX 170616C00900000,1096.2500,1123.7000,1096.2500,1118.4000,0,,0.3680870,0.9825946,0.0000465,1.0804654,-0.0599242,12.4956358 +SPX WDYBH6MBU7NY|SPX 31,SPX 160916C01000000,1015.0000,1042.1500,1015.0000,1037.4500,0,250,0.3992342,0.9896513,0.0000389,0.4851699,-0.0627592,7.0947658 +SPX W87JLOI8831Q|SPX 31,SPX 160219C01025000,1009.0500,1035.5500,1009.0500,1031.4500,0,,0.6201123,0.9985451,0.0000094,0.0388357,-0.0489404,1.6033199 +SPX WLF4IPYLSPAM|SPX 31,SPX 170616C01050000,954.6000,981.3000,954.6000,976.2000,0,,0.3423531,0.9690246,0.0000813,1.7549149,-0.0818528,14.1903006 +SPX WGFX5OYCA2LQ|SPX 31,SPX 161216C01100000,913.3000,939.5000,913.3000,935.2000,0,400,0.3578960,0.9761501,0.0000767,1.1491187,-0.0856704,10.1379875 +SPX W87JLKBQDN9Q|SPX 31,SPX 160219C01150000,884.2500,910.6000,884.2500,906.6500,0,,0.5345471,0.9979737,0.0000147,0.0525059,-0.0557988,1.7975328 +SPX WLF4IUVK8OZ2|SPX 31,SPX 170616C01175000,838.6000,863.7500,838.6000,859.7500,0,,0.3228105,0.9524202,0.0001223,2.4897469,-0.1028839,15.4089756 +SPX W8Z4463FCFU6|SPX 31,SPX 160318C01200000,831.4500,859.0000,831.4500,853.8500,0,3,0.4518739,0.9953701,0.0000299,0.1347870,-0.0681934,2.7759611 +SPX WGFX5OYI8ENI|SPX 31,SPX 161216C01200000,818.0500,844.0000,818.0500,839.6500,0,415,0.3411811,0.9642814,0.0001126,1.6070758,-0.1067282,10.8324282 +SPX W87JLPFAZ0XA|SPX 31,SPX 160219C01225000,809.4500,835.9000,809.4500,831.7500,0,,0.4962165,0.9971019,0.0000218,0.0725777,-0.0647254,1.9122346 +SPX WBGPSTYE2T0U|SPX 31,SPX 160617C01225000,800.5500,828.6000,800.5500,822.6500,0,1,0.3713871,0.9849667,0.0000712,0.5444489,-0.0897895,5.7256255 +SPX WHEDT4C0OIUM|SPX 31,SPX 170120C01225000,795.0000,820.5500,795.0000,816.3000,0,,0.3286970,0.9583192,0.0001265,1.9094129,-0.1102476,11.9865846 +SPX W792YAKBA93I|SPX 31,SPX 160115C01240000,799.3000,825.3000,799.3000,821.3000,0,,0.6686491,0.9992268,0.0000078,0.0135822,-0.0542099,0.7582937 +SPX W792YFUNZCRY|SPX 31,SPX 160115C01275000,764.4000,790.3000,764.4000,786.3000,0,16,0.6344248,0.9991855,0.0000086,0.0142485,-0.0550716,0.7796738 +SPX WGFX5URDOUPA|SPX 31,SPX 161216C01275000,747.5000,773.0000,747.5000,768.7000,0,,0.3247342,0.9543778,0.0001443,1.9603408,-0.1204412,11.3323806 +SPX W8Z446MLX4EM|SPX 31,SPX 160318C01280000,751.7000,779.6000,751.7000,774.2000,0,,0.4149083,0.9936816,0.0000430,0.1776970,-0.0777887,2.9540047 +SPX WHEDSYRKV5BI|SPX 31,SPX 170120C01300000,725.1500,750.2500,725.1500,746.1000,0,,0.3130411,0.9472322,0.0001605,2.3069748,-0.1235802,12.5031701 +SPX W87JLKEDN026|SPX 31,SPX 160219C01310000,724.7000,751.7000,724.7000,746.9500,0,,0.4459521,0.9962025,0.0000310,0.0925426,-0.0715782,2.0425778 +SPX W9QOMYDAMQUM|SPX 31,SPX 160415C01325000,706.2000,733.5000,706.2000,728.4500,0,,0.3698967,0.9883636,0.0000715,0.3498595,-0.0925584,4.0222549 +SPX WQEBVW5387LA|SPX 31,SPX 171215C01325000,706.4000,730.0500,706.4000,726.1000,0,,0.2923231,0.9085509,0.0001936,4.7721211,-0.1257910,21.1168754 +SPX W87JLKFDD0CU|SPX 31,SPX 160219C01370000,664.9000,691.4000,664.9000,687.1500,0,,0.4105680,0.9954710,0.0000394,0.1083394,-0.0759756,2.1342755 +SPX W792YGB7CTPQ|SPX 31,SPX 160115C01375000,664.4500,690.4000,664.4500,686.4000,0,2,0.5410294,0.9990506,0.0000116,0.0164045,-0.0574348,0.8407385 +SPX W87JLKFJBCEM|SPX 31,SPX 160219C01380000,655.0500,683.3500,655.0500,677.2000,0,,0.4083245,0.9950368,0.0000430,0.1175581,-0.0792923,2.1484801 +SPX W87JLQ92N9TA|SPX 31,SPX 160219C01405000,629.8500,658.4000,629.8500,652.3000,0,,0.4013579,0.9939208,0.0000524,0.1407725,-0.0873169,2.1838920 +SPX WHEDT593FGQ6|SPX 31,SPX 170120C01425000,611.0000,635.2000,611.0000,631.1000,0,,0.2934548,0.9198775,0.0002367,3.1896020,-0.1523536,13.1303125 +SPX W87JLQE19BAM|SPX 31,SPX 160219C01435000,600.2000,625.8000,600.2000,622.4000,0,,0.3847895,0.9932495,0.0000599,0.1544442,-0.0906506,2.2287107 +SPX W9QOMSY59XQM|SPX 31,SPX 160415C01450000,583.5000,610.5500,583.5000,605.3000,0,,0.3338572,0.9776867,0.0001383,0.6110601,-0.1282258,4.3359258 +SPX W792YAO47Y8E|SPX 31,SPX 160115C01470000,569.6000,595.5000,569.6000,591.5000,0,,0.4875229,0.9980046,0.0000254,0.0323302,-0.0753704,0.8975430 +SPX W8Z44CTDJTRI|SPX 31,SPX 160318C01475000,558.6000,585.4000,558.6000,580.8000,0,,0.3400749,0.9838327,0.0001190,0.4031742,-0.1196535,3.3587000 +SPX WHEDT5HD4772|SPX 31,SPX 170120C01475000,566.3500,590.2000,566.3500,586.0000,0,,0.2853202,0.9064639,0.0002733,3.5808133,-0.1637199,13.3177065 +SPX X0JMQLJIOBJI|SPX 31,SPX 181221C01475000,609.0500,618.3500,605.9500,618.3500,0,,0.2696868,0.8455862,0.0002467,8.4922175,-0.1325455,30.4086411 +SPX W8Z4463X7FZI|SPX 31,SPX 160318C01500000,534.0000,560.2500,534.0000,556.3000,0,1504,0.3326526,0.9812327,0.0001381,0.4579276,-0.1290964,3.4037238 +SPX WLF4IPJQ0Q0E|SPX 31,SPX 170616C01500000,551.8000,574.3500,551.8000,570.0000,0,1,0.2755165,0.8778523,0.0002927,5.0875098,-0.1619495,17.4271506 +SPX W87JLQPM0QR2|SPX 31,SPX 160219C01505000,530.5500,557.2000,530.5500,552.6500,0,,0.3500967,0.9907999,0.0000865,0.2027820,-0.1025015,2.3302907 +SPX W8Z446QEUTJI|SPX 31,SPX 160318C01510000,524.0500,550.2000,524.0500,546.3500,0,,0.3288294,0.9803218,0.0001455,0.4767526,-0.1319158,3.4224222 +SPX W792YH00F14E|SPX 31,SPX 160115C01525000,514.6500,540.7000,514.6500,536.5000,0,5,0.4455192,0.9974946,0.0000342,0.0397341,-0.0811854,0.9305621 +SPX WQEBVX25Z5GU|SPX 31,SPX 171215C01525000,542.1500,563.1000,542.1500,559.2500,0,1,0.2697516,0.8507823,0.0002966,6.7474371,-0.1565204,22.0581851 +SPX W87JLKI0MD5A|SPX 31,SPX 160219C01530000,505.7500,531.9500,505.7500,527.9000,0,,0.3437577,0.9886221,0.0001060,0.2440627,-0.1143044,2.3621018 +SPX W792YAP9WAKU|SPX 31,SPX 160115C01540000,499.7000,525.6000,499.7000,521.6000,0,,0.4381645,0.9971220,0.0000394,0.0450340,-0.0861127,0.9392608 +SPX WHEDSZF28J66|SPX 31,SPX 170120C01550000,500.6000,523.6000,500.6000,519.5500,0,,0.2726148,0.8833753,0.0003356,4.2012595,-0.1799592,13.5232913 +SPX W8Z446R8MHSE|SPX 31,SPX 160318C01560000,475.1000,501.2000,475.1000,497.2500,0,,0.3126946,0.9740120,0.0001934,0.6027087,-0.1514686,3.5068689 +SPX W87JLKIOFPCE|SPX 31,SPX 160219C01570000,466.4000,492.2500,466.4000,488.2000,0,,0.3222835,0.9867387,0.0001291,0.2787013,-0.1204507,2.4185607 +SPX WGFX5W4ZT9IM|SPX 31,SPX 161216C01575000,475.8500,499.0000,475.8500,494.7500,0,17,0.2715159,0.8803158,0.0003595,4.0841130,-0.1896667,12.5247352 +SPX X0JMQM021SHA|SPX 31,SPX 181221C01575000,535.0500,543.9500,532.4500,543.8500,0,,0.2571128,0.8153407,0.0002902,9.5258955,-0.1402562,30.8896301 +SPX W792YAQ3NYTQ|SPX 31,SPX 160115C01590000,449.8000,475.8000,449.8000,471.6000,0,,0.4037884,0.9961837,0.0000551,0.0580393,-0.0957067,0.9686610 +SPX W87JLKJ0CDFY|SPX 31,SPX 160219C01590000,446.2500,472.4000,446.2500,468.3000,0,,0.3142690,0.9850053,0.0001472,0.3098241,-0.1274740,2.4440843 +SPX W792YA2GC8XA|SPX 31,SPX 160115C01600000,439.9000,465.8000,439.9000,461.6500,0,106,0.3906100,0.9964248,0.0000537,0.0547397,-0.0914034,0.9751048 +SPX W792YAQFKMXA|SPX 31,SPX 160115C01610000,429.8000,457.6000,429.8000,451.7000,0,,0.3902630,0.9957001,0.0000634,0.0645818,-0.1001796,0.9802810 +SPX W87JLR7SXK6M|SPX 31,SPX 160219C01615000,421.6000,447.9500,421.6000,443.7000,0,,0.3045453,0.9824146,0.0001741,0.3551425,-0.1374089,2.4745558 +SPX WQEBVXIPCMEM|SPX 31,SPX 171215C01625000,464.6500,484.3500,464.6500,480.4500,0,1,0.2558983,0.8158444,0.0003582,7.7297159,-0.1676970,22.2373311 +SPX X0JMQM8BQIY6|SPX 31,SPX 181221C01625000,499.9000,507.9500,497.1000,507.9500,0,,0.2518610,0.7983074,0.0003127,10.0531464,-0.1441027,30.9570128 +SPX W8Z446SQ7I8E|SPX 31,SPX 160318C01650000,388.1000,413.5500,388.1000,409.7000,1,71,0.2893290,0.9542610,0.0003325,0.9587400,-0.2048303,3.6156888 +SPX W792YHLIEJJI|SPX 31,SPX 160115C01655000,384.6000,410.8000,384.6000,406.7500,0,,0.3544613,0.9949072,0.0000812,0.0751068,-0.1045376,1.0068188 +SPX W87JLREF2Y5Q|SPX 31,SPX 160219C01655000,382.3500,408.4000,382.3500,404.2500,0,,0.2908948,0.9765686,0.0002325,0.4529576,-0.1587335,2.5175301 +SPX W8Z446SW5UA6|SPX 31,SPX 160318C01660000,378.3500,404.0500,378.3500,400.0500,0,,0.2846467,0.9524647,0.0003486,0.9888375,-0.2073862,3.6299733 +SPX W792YHOTH8J2|SPX 31,SPX 160115C01675000,364.8000,391.0500,364.8000,386.8000,0,34,0.3402366,0.9943077,0.0000933,0.0829177,-0.1086036,1.0182923 +SPX W9QOMZZ6FW4U|SPX 31,SPX 160415C01675000,367.4000,392.9000,367.4000,388.3500,0,,0.2738891,0.9286392,0.0004325,1.5676932,-0.2307990,4.6957227 +SPX WGFX5WLJ6QGE|SPX 31,SPX 161216C01675000,391.1500,412.9500,391.1500,408.7500,0,,0.2563563,0.8385066,0.0004668,5.0070798,-0.2140187,12.5169081 +SPX WHEDT6EFV52M|SPX 31,SPX 170120C01675000,395.3500,416.8500,395.3500,412.6500,0,,0.2528824,0.8335597,0.0004606,5.3496063,-0.2064510,13.5694331 +SPX W8Z446TE0UFI|SPX 31,SPX 160318C01690000,350.1000,375.9000,350.1000,371.2500,0,1,0.2761911,0.9431858,0.0004139,1.1393316,-0.2270985,3.6525066 +SPX W792YA2MAKZ2|SPX 31,SPX 160115C01700000,339.9000,365.9000,339.9000,361.8500,0,311,0.3214053,0.9935754,0.0001100,0.0923050,-0.1125005,1.0326595 +SPX X0JMQEUE8LV2|SPX 31,SPX 181221C01700000,448.8000,455.8500,445.7500,455.8000,0,1,0.2450907,0.7704791,0.0003464,10.8360306,-0.1496605,30.8430248 +SPX W9QON07G4MLQ|SPX 31,SPX 160415C01725000,321.1500,346.0500,321.1500,341.4500,0,,0.2614368,0.9076727,0.0005502,1.9039095,-0.2610512,4.7063562 +SPX WHEDT6MPJVJI|SPX 31,SPX 170120C01725000,355.1000,375.6500,355.1000,371.6000,0,1,0.2454425,0.8090500,0.0005175,5.8331907,-0.2162178,13.4667924 +SPX X0JMQMOV3ZVY|SPX 31,SPX 181221C01725000,432.0000,439.2000,428.9500,438.7500,0,,0.2433516,0.7605382,0.0003571,11.0932312,-0.1515622,30.7349475 +SPX W8Z446UDQUQ6|SPX 31,SPX 160318C01750000,294.0500,319.0500,294.0500,314.3500,7,45,0.2593627,0.9189735,0.0005797,1.4985060,-0.2706208,3.6686071 +SPX X0JMQFKIVCI6|SPX 31,SPX 181221C01750000,415.8500,422.7500,412.4000,422.1000,0,,0.2426309,0.7500428,0.0003665,11.3524393,-0.1539045,30.5511238 +SPX W9QON0FPTD2M|SPX 31,SPX 160415C01775000,276.0500,300.3000,276.0500,295.6000,0,,0.2489763,0.8809810,0.0006944,2.2880519,-0.2925166,4.6774919 +SPX WBGPSWHCMW6M|SPX 31,SPX 160617C01775000,285.5000,308.6500,285.5000,303.9500,0,11,0.2454818,0.8409723,0.0006880,3.4774442,-0.2805412,6.7950711 +SPX WDYBHERPLJSE|SPX 31,SPX 160916C01775000,298.5000,320.0000,298.3000,315.4000,0,2,0.2412837,0.8078139,0.0006411,4.8281103,-0.2538920,9.6345019 +SPX WBGPSOES13QM|SPX 31,SPX 160617C01800000,264.5000,285.5500,264.5000,282.4500,0,212,0.2404605,0.8243601,0.0007488,3.7074586,-0.2909177,6.7327094 +SPX WHEDSYSEMTKE|SPX 31,SPX 170120C01800000,297.2500,316.4500,296.5500,312.3500,0,152,0.2363250,0.7657780,0.0006057,6.5733972,-0.2308410,13.1317596 +SPX W87JLS4VOI26|SPX 31,SPX 160219C01815000,228.9000,253.6000,228.9000,249.5000,0,,0.2398703,0.9218293,0.0007442,1.1955025,-0.2949797,2.5831120 +SPX W8Z446VJF72M|SPX 31,SPX 160318C01820000,230.3000,254.4000,230.3000,249.6500,0,,0.2400233,0.8776343,0.0008464,2.0246241,-0.3270445,3.6201698 +SPX W87JLS6J7UJY|SPX 31,SPX 160219C01825000,219.6000,243.7000,219.6000,240.0500,2,15,0.2354393,0.9167474,0.0007951,1.2537416,-0.3022888,2.5816670 +SPX WBGPSWPMBMNI|SPX 31,SPX 160617C01825000,243.9500,264.4500,243.8500,261.3000,0,32,0.2349557,0.8066513,0.0008136,3.9357835,-0.2999205,6.6586146 +SPX WLF4IQBU3H8U|SPX 31,SPX 170616C01850000,281.8500,299.5500,281.2500,295.2000,0,25,0.2319993,0.7196672,0.0005780,8.4598352,-0.2124570,16.6704526 +SPX W792YAUKF05Q|SPX 31,SPX 160115C01860000,182.3000,206.1500,182.3000,203.1000,0,7,0.2297447,0.9693723,0.0005896,0.3536994,-0.2308289,1.0993640 +SPX X0JMQNDO67AM|SPX 31,SPX 181221C01875000,339.6500,344.8500,333.8500,342.5500,0,,0.2328493,0.6965429,0.0004201,12.4864643,-0.1600990,29.6733916 +SPX W87JLJVUVNOU|SPX 31,SPX 160219C01900000,152.6000,174.0000,152.6000,171.2000,453,409,0.2124098,0.8521335,0.0013285,1.8899103,-0.3931311,2.4800267 +SPX WBGPSOEXZFSE|SPX 31,SPX 160617C01900000,184.8000,203.4000,184.8000,200.4500,2,4677,0.2200827,0.7422108,0.0010226,4.6338883,-0.3248972,6.3084196 +SPX WQEBVYWBH17Y|SPX 31,SPX 171215C01925000,258.0500,273.3000,257.6000,269.4000,0,54,0.2268889,0.6702006,0.0005495,10.5143773,-0.1934728,20.4781233 +SPX WDYBH7IEQUXA|SPX 31,SPX 160916C01950000,168.0500,183.8000,168.0500,181.0000,0,3366,0.2126691,0.6715419,0.0009620,6.3862373,-0.2859906,8.5559844 +SPX W792YAWDWOPA|SPX 31,SPX 160115C01970000,81.7500,101.2000,81.7500,98.7000,4,471,0.1844062,0.8555364,0.0024128,1.1617374,-0.5245266,1.0192411 +SPX W8Z44F42F6GE|SPX 31,SPX 160318C01975000,103.5500,120.6500,103.5500,118.4500,0,3416,0.1973444,0.7038633,0.0017545,3.4505923,-0.4350236,3.0878533 +SPX WBGPSXEFDU26|SPX 31,SPX 160617C01975000,130.7500,147.1000,130.7500,144.3000,0,3770,0.2039944,0.6615905,0.0012495,5.2482595,-0.3362820,5.7774594 +SPX WLF4IYJV8GHA|SPX 31,SPX 170616C01975000,201.4000,214.6000,201.4000,212.0000,0,250,0.2183557,0.6387036,0.0006830,9.4083285,-0.2186447,15.4297552 +SPX WQEBVZ4L5ROU|SPX 31,SPX 171215C01975000,228.3500,240.7000,228.3500,238.7500,0,718,0.2227069,0.6409332,0.0005779,10.8547759,-0.1948605,19.8886486 +SPX W8Z446Y6OJV2|SPX 31,SPX 160318C01980000,99.9500,116.7500,99.9500,114.6500,0,30,0.1958987,0.6957117,0.0017892,3.4932184,-0.4366364,3.0574413 +SPX W8Z44F7DHVFY|SPX 31,SPX 160318C01995000,89.4000,105.6500,89.4000,103.4000,60,3770,0.1922501,0.6696214,0.0018878,3.6170085,-0.4419366,2.9572184 +SPX W9QOMSAI0CZY|SPX 31,SPX 160415C02000000,96.2000,111.7500,96.2000,109.6000,0,,0.1925584,0.6470722,0.0016771,4.2740542,-0.3961182,3.7555243 +SPX W792YAX1Q0WE|SPX 31,SPX 160115C02010000,50.5500,66.8000,50.5500,64.7000,20,1503,0.1681608,0.7504179,0.0036949,1.6223361,-0.6504843,0.9071438 +SPX W87JLT59I4XA|SPX 31,SPX 160219C02035000,51.1500,65.2500,51.1500,63.3000,0,30,0.1714312,0.6054112,0.0027438,3.1501563,-0.5029366,1.8489926 +SPX W87JLKQG9FNY|SPX 31,SPX 160219C02040000,48.2000,61.9000,48.2000,60.0000,5,962,0.1699414,0.5920046,0.0027919,3.1775868,-0.5022421,1.8107291 +SPX W87JLTBVNIWE|SPX 31,SPX 160219C02075000,29.4500,40.2500,29.4500,38.5500,40,8696,0.1287903,0.4820367,0.0037813,3.2614874,-0.3923286,1.5033698 +SPX W8Z446ZU7WCU|SPX 31,SPX 160318C02080000,38.7500,49.9000,38.7500,47.9500,0,104,0.1326167,0.4801887,0.0030096,3.9777480,-0.3348289,2.2067355 +SPX WDYBH7KK57KE|SPX 31,SPX 160916C02080000,89.3000,100.4000,89.3000,98.2000,0,,0.1397427,0.5229886,0.0016132,7.0366255,-0.2107788,7.1889247 +SPX W9QOMSDN2KRY|SPX 31,SPX 160415C02100000,39.1500,49.1000,39.1500,47.3500,1,23,0.1310734,0.4381153,0.0026138,4.5342506,-0.2856155,2.6627046 +SPX WHEDSYSWHTPQ|SPX 31,SPX 170120C02100000,105.1500,114.4000,105.1500,113.0000,0,1766,0.1366404,0.5103678,0.0013620,8.5464406,-0.1742698,10.1621199 +SPX W792YAYV7PFY|SPX 31,SPX 160115C02120000,2.4750,4.6500,1.9500,4.4000,472,9459,0.1039753,0.1591405,0.0045635,1.2389010,-0.2968691,0.1985974 +SPX W87JLTLSVLV2|SPX 31,SPX 160219C02135000,8.7000,14.0500,7.7000,12.5000,0,49,0.1107916,0.2391383,0.0034220,2.5390820,-0.2583766,0.7562169 +SPX W87JLTNGEYCU|SPX 31,SPX 160219C02145000,6.6500,11.1500,5.7500,9.7000,1,271,0.1077472,0.2005290,0.0031801,2.2947455,-0.2265855,0.6353636 +SPX X0JMQFR50QHA|SPX 31,SPX 181221C02150000,193.6500,198.4000,189.8500,197.8000,1,1,0.1435104,0.5322814,0.0007755,14.2066581,-0.1180188,27.0974021 +SPX W8Z44FXU3FCE|SPX 31,SPX 160318C02155000,11.7000,17.5500,10.9500,15.5000,5,39,0.1137538,0.2383234,0.0027270,3.0915948,-0.2190418,1.1143824 +SPX W792YJXUT8Q6|SPX 31,SPX 160115C02165000,0.5000,2.7750,0.4000,0.7250,48,1321,0.1039807,0.0348387,0.0014488,0.3933415,-0.0933895,0.0436212 +SPX X0JMQORAAM3Y|SPX 31,SPX 181221C02175000,180.8500,187.0000,178.3500,186.4000,0,1,0.1469106,0.5157464,0.0007595,14.2422423,-0.1196811,26.2383014 +SPX W87JLKSXKGEM|SPX 31,SPX 160219C02190000,1.7500,4.4750,0.9750,2.4250,3698,4836,0.0971003,0.0700238,0.0016901,1.0990872,-0.0969144,0.2233679 +SPX W8Z44652VSBY|SPX 31,SPX 160318C02200000,4.2000,7.3500,3.3000,5.5500,216,71439,0.1027410,0.1132262,0.0018716,1.9163968,-0.1215767,0.5337181 +SPX WBGPSOFFUFXQ|SPX 31,SPX 160617C02200000,19.9000,25.6500,19.0000,23.8000,5,30602,0.1167701,0.2468895,0.0018839,4.5295401,-0.1632652,2.3484533 +SPX WHEDSYT2G5RI|SPX 31,SPX 170120C02200000,60.7000,67.3500,60.5000,65.8500,0,751,0.1269072,0.3685821,0.0013866,8.0811590,-0.1494453,7.5086265 +SPX W8Z44725KL1Q|SPX 31,SPX 160318C02220000,2.5500,5.0750,2.2500,3.3000,152,1065,0.0995576,0.0752523,0.0014279,1.4167777,-0.0867827,0.3556164 +SPX W792YK7S1BOU|SPX 31,SPX 160115C02225000,0.1250,1.6750,0.1250,0.1250,77,19922,0.1198492,0.0063583,0.0002923,0.0914618,-0.0248628,0.0079658 +SPX WDYBHGU4S60E|SPX 31,SPX 160916C02225000,29.3000,35.1500,28.6500,33.2000,0,4225,0.1187515,0.2693588,0.0015741,5.8347416,-0.1439110,3.8324835 +SPX WLF4IZP7O4TQ|SPX 31,SPX 170616C02225000,77.3500,83.1000,77.3500,81.9000,0,3,0.1321118,0.3844090,0.0011514,9.5966239,-0.1368132,10.5376231 +SPX WHEDSZQMZYMM|SPX 31,SPX 170120C02250000,43.3500,48.3500,42.8000,47.2500,0,8,0.1243573,0.3017962,0.0013082,7.4711338,-0.1338522,6.1982258 +SPX W87JLU5NBRSE|SPX 31,SPX 160219C02255000,0.3250,2.4500,0.2000,0.3750,0,414,0.0981084,0.0134813,0.0004302,0.2826900,-0.0249259,0.0431490 +SPX W87JLKU38SR2|SPX 31,SPX 160219C02260000,0.2250,2.4250,0.2000,0.3500,0,298,0.0993784,0.0125615,0.0003996,0.2659361,-0.0237362,0.0402023 +SPX W792YKJCSR5A|SPX 31,SPX 160115C02295000,0.0500,2.2750,0.0500,0.2500,0,142,0.1764869,0.0084405,0.0002548,0.1174045,-0.0467946,0.0105228 +SPX WDYBH6PYRFLA|SPX 31,SPX 160916C02300000,12.8500,16.2500,11.5500,14.6000,500,5004,0.1093811,0.1515255,0.0012147,4.1471893,-0.0930024,2.1843531 +SPX WGFX5P0BQ372|SPX 31,SPX 161216C02300000,24.6000,28.6000,23.8000,27.1000,200,21960,0.1158902,0.2122446,0.0012234,5.9318313,-0.1071760,4.0360413 +SPX W87JLKUX0GZY|SPX 31,SPX 160219C02310000,0.2500,2.3500,0.2000,0.3000,0,1,0.1178357,0.0093764,0.0002613,0.2061776,-0.0216981,0.0299484 +SPX WHEDT9DXSP66|SPX 31,SPX 170120C02325000,23.8500,27.3500,23.0500,26.0500,0,,0.1155299,0.1999425,0.0011306,5.9985432,-0.0986796,4.1665165 +SPX WHEDT9M7HFN2|SPX 31,SPX 170120C02375000,15.0500,17.9000,13.9000,16.3500,0,11,0.1103924,0.1418318,0.0009492,4.8120632,-0.0750964,2.9794942 +SPX WHEDT9UH663Y|SPX 31,SPX 170120C02425000,9.0000,11.5500,7.5000,9.7500,0,2,0.1059794,0.0949607,0.0007440,3.6209592,-0.0538833,2.0084387 +SPX WBGPSPGBGXSE|SPX 31,SPX 160617C02450000,0.4500,0.4750,0.2750,0.4000,0,926,0.0991231,0.0086194,0.0001645,0.3358187,-0.0099136,0.0840392 +SPX WGFX5P0NMRAM|SPX 31,SPX 161216C02500000,2.9500,5.0000,2.4000,3.0750,1,6152,0.1007874,0.0384180,0.0004045,1.7056840,-0.0260743,0.7485320 +SPX X0JMQQD63RE6|SPX 31,SPX 181221C02525000,62.8000,67.8500,61.3000,64.4500,2,3,0.1296722,0.2572209,0.0006962,11.5245654,-0.0811240,13.9773667 +SPX WGFX5Q2OXLHQ|SPX 31,SPX 161216C02550000,1.7500,3.8750,0.9750,1.8000,0,7907,0.1003838,0.0240803,0.0002758,1.1584400,-0.0175375,0.4704008 +SPX WLF4IPLJIEJY|SPX 31,SPX 170616C02600000,5.2000,7.3500,3.3000,5.4000,0,21,0.1046483,0.0513295,0.0004007,2.6451709,-0.0283685,1.4886942 +SPX WLF4IPLPGQLQ|SPX 31,SPX 170616C02700000,2.2500,4.2500,1.1500,2.3250,0,72,0.1029224,0.0248259,0.0002248,1.4593851,-0.0152388,0.7243138 +SPX X0JMQG128TFY|SPX 31,SPX 181221C02750000,24.4000,29.7500,23.1500,25.9500,3,3,0.1208248,0.1305622,0.0004915,7.5806486,-0.0485609,7.2982136 +SPX WHEDT00K81LA|SPX 31,SPX 170120C02850000,0.7250,2.5250,0.7000,0.7250,0,2,0.1227461,0.0085897,0.0000887,0.5000141,-0.0082672,0.1832839 +SPX WLF4IPMDA2SU|SPX 31,SPX 170616C03100000,0.9500,2.8250,0.2000,0.9500,0,114,0.1316807,0.0090069,0.0000735,0.6109159,-0.0079270,0.2611785 +SPX WLF4IPMP6QWE|SPX 31,SPX 170616C03300000,1.2250,2.9250,1.2250,1.2250,0,2,0.1540328,0.0098909,0.0000682,0.6630438,-0.0099760,0.2841633 +SPX 30D2BMMLUYJZI|SPX 31,SPX 160916P00300000,0.1750,0.1750,0.1750,0.1750,0,210,0.7504850,-0.0004259,0.0000012,0.0270621,-0.0037692,-0.0077220 +SPX 30KJ4O5GXUISU|SPX 31,SPX 170616P00300000,0.3750,0.4000,0.3250,0.3500,0,18,0.5643011,-0.0007518,0.0000018,0.0650577,-0.0033449,-0.0281577 +SPX 308349M1GXWVI|SPX 31,SPX 160318P00500000,0.1250,0.3250,0.1250,0.1250,64,11573,0.9496134,-0.0004464,0.0000017,0.0159735,-0.0088559,-0.0024476 +SPX 30AKPY4BTWKHA|SPX 31,SPX 160617P00500000,0.2000,0.2250,0.1750,0.1750,25,793,0.6808133,-0.0005914,0.0000021,0.0297829,-0.0057102,-0.0067436 +SPX 30ZNMVURM2ENI|SPX 31,SPX 181221P00500000,5.8000,7.5000,3.5500,5.0000,0,,0.4081407,-0.0081330,0.0000153,0.7945726,-0.0142334,-0.6525862 +SPX 308UOS8AD1DF2|SPX 31,SPX 160415P00600000,0.2250,0.2250,0.2250,0.2250,0,,0.7585941,-0.0008407,0.0000033,0.0330273,-0.0109966,-0.0060889 +SPX 307BJR02NYQ6M|SPX 31,SPX 160219P00650000,0.1000,0.3250,0.1000,0.1000,0,5594,0.9317693,-0.0004545,0.0000021,0.0133137,-0.0107842,-0.0016317 +SPX 30KJ4O5HLNUZY|SPX 31,SPX 170616P00700000,2.1000,3.7500,1.4750,2.0500,0,228,0.3921103,-0.0056009,0.0000162,0.4017594,-0.0142037,-0.2015160 +SPX 30KJ4O5SUGH72|SPX 31,SPX 170616P00750000,3.6750,3.6750,2.6500,3.5000,0,763,0.3969730,-0.0089347,0.0000242,0.6066250,-0.0216813,-0.3248661 +SPX 307BJQZT8NSJ2|SPX 31,SPX 160219P00800000,0.1250,0.2750,0.1000,0.1500,0,1179,0.7967114,-0.0007777,0.0000041,0.0218694,-0.0151386,-0.0027589 +SPX 308349M1YSX0U|SPX 31,SPX 160318P00800000,0.2250,0.2500,0.2000,0.2250,5050,5800,0.6774949,-0.0010914,0.0000054,0.0364224,-0.0143854,-0.0057954 +SPX 307BJR05Z1F66|SPX 31,SPX 160219P00850000,0.1500,0.4500,0.1500,0.1500,0,2407,0.7480620,-0.0008288,0.0000046,0.0231884,-0.0150684,-0.0029247 +SPX 30AKPY4P2579Q|SPX 31,SPX 160617P00850000,0.5500,0.7750,0.4250,0.6000,0,182,0.4889727,-0.0025974,0.0000114,0.1152787,-0.0158172,-0.0288079 +SPX 30ZNMVV4UB1FY|SPX 31,SPX 181221P00850000,21.9000,23.0500,19.0500,19.9000,0,,0.3433168,-0.0328259,0.0000598,2.6188366,-0.0387140,-2.6260548 +SPX 30ZNMVYRHPBAM|SPX 31,SPX 181221P00875000,21.6500,23.2000,20.6000,21.5000,0,,0.3390035,-0.0353579,0.0000643,2.7838346,-0.0405718,-2.8267284 +SPX 308UOS8P2V0NI|SPX 31,SPX 160415P00950000,0.4250,0.4750,0.2750,0.3750,0,,0.5114475,-0.0020125,0.0000108,0.0733538,-0.0164228,-0.0140677 +SPX 30D2BMR2LVLNY|SPX 31,SPX 160916P00975000,2.6000,4.4250,1.5500,2.4750,0,29,0.4052078,-0.0092009,0.0000346,0.4378176,-0.0325888,-0.1572684 +SPX 30KJ4O9XORKHA|SPX 31,SPX 170616P00975000,9.9000,11.1000,7.0500,9.2000,0,12,0.3550765,-0.0234891,0.0000622,1.3930885,-0.0441901,-0.8538664 +SPX 306D33UOITHTA|SPX 31,SPX 160115P01025000,0.0500,2.2750,0.0500,0.2000,0,6700,0.9811261,-0.0013208,0.0000087,0.0221988,-0.0486124,-0.0017931 +SPX 30AKPY4SD7W9A|SPX 31,SPX 160617P01050000,1.4500,3.4000,0.9500,1.3750,0,68,0.4184210,-0.0064553,0.0000302,0.2603040,-0.0304755,-0.0710316 +SPX 30D2BMN2Q6JV2|SPX 31,SPX 160916P01050000,3.7000,5.3500,2.1000,3.4750,0,18,0.3876036,-0.0130172,0.0000489,0.5921010,-0.0420876,-0.2222980 +SPX 30ZNMVV85DQFI|SPX 31,SPX 181221P01050000,36.6000,38.3000,34.9000,35.7000,0,,0.3154613,-0.0586771,0.0001039,4.1822590,-0.0560383,-4.6967459 +SPX 308UOSD75GVVY|SPX 31,SPX 160415P01075000,0.6750,0.7500,0.5000,0.6250,0,,0.4592157,-0.0035773,0.0000203,0.1233070,-0.0247551,-0.0248769 +SPX 30D2BMMN6L8DQ|SPX 31,SPX 160916P01100000,4.6500,6.0000,2.4500,4.3000,2,4000,0.3759760,-0.0161831,0.0000608,0.7141191,-0.0491778,-0.2761688 +SPX 308UOSDNOUCTQ|SPX 31,SPX 160415P01175000,1.0250,1.1500,0.6000,0.9250,0,2,0.4200148,-0.0055788,0.0000330,0.1833738,-0.0336291,-0.0386411 +SPX 30D2BMRZOMJJI|SPX 31,SPX 160916P01175000,6.4000,7.9500,4.2000,5.8500,0,182,0.3593222,-0.0222168,0.0000834,0.9349968,-0.0614122,-0.3788122 +SPX 30PIC1BFHFTKE|SPX 31,SPX 171215P01175000,30.5500,30.5500,27.3500,28.5000,0,11,0.3119575,-0.0607422,0.0001328,3.4928229,-0.0712045,-3.0450064 +SPX 307BJQZTWH4Q6|SPX 31,SPX 160219P01200000,0.3750,1.9500,0.2250,0.3250,0,9217,0.5067796,-0.0024897,0.0000186,0.0632766,-0.0278007,-0.0085883 +SPX 30FJXBAIB9XM6|SPX 31,SPX 161216P01225000,12.9500,14.0500,9.9500,11.9000,0,783,0.3340339,-0.0382134,0.0001215,1.6982736,-0.0766621,-0.8910613 +SPX 308349MLBBXN2|SPX 31,SPX 160318P01250000,0.8750,2.9000,0.7000,0.8250,0,6222,0.4289796,-0.0056537,0.0000377,0.1610195,-0.0401159,-0.0292247 +SPX 30AKPY4VOAL8U|SPX 31,SPX 160617P01250000,3.6000,5.4500,2.1250,3.4250,0,3671,0.3644051,-0.0167467,0.0000796,0.5972442,-0.0606603,-0.1836113 +SPX 307BJR0CR5572|SPX 31,SPX 160219P01260000,0.4500,0.7250,0.2500,0.3750,0,13,0.4715169,-0.0030452,0.0000240,0.0758902,-0.0310070,-0.0104691 +SPX 307BJR0D31TAM|SPX 31,SPX 160219P01280000,0.4500,0.7250,0.2500,0.4000,0,22,0.4608201,-0.0033034,0.0000265,0.0816552,-0.0325998,-0.0113458 +SPX 30ZNMVUSXP31Q|SPX 31,SPX 181221P01300000,67.8000,69.7500,65.7000,65.8500,0,,0.2855017,-0.1068397,0.0001805,6.5786591,-0.0780517,-8.5719073 +SPX 306D33QLC1R0U|SPX 31,SPX 160115P01350000,0.1250,0.3000,0.0750,0.0750,0,13112,0.5637653,-0.0009124,0.0000107,0.0158153,-0.0198799,-0.0011999 +SPX 30ZNMVVD3ZRWU|SPX 31,SPX 181221P01350000,76.1000,77.9500,73.6000,73.7000,0,,0.2811332,-0.1194678,0.0001985,7.1246469,-0.0828113,-9.6058041 +SPX 308349SC1G67I|SPX 31,SPX 160318P01375000,1.3250,3.0500,0.6500,1.2250,2,2278,0.3730344,-0.0092479,0.0000668,0.2484955,-0.0537384,-0.0475144 +SPX 306D33QLTWR66|SPX 31,SPX 160115P01380000,0.1250,0.4250,0.0750,0.0750,340,959,0.5365255,-0.0009571,0.0000118,0.0165262,-0.0197669,-0.0012565 +SPX 308349MNGQAA6|SPX 31,SPX 160318P01380000,1.4500,3.0750,0.6750,1.2750,2,26,0.3719807,-0.0096026,0.0000693,0.2568109,-0.0553754,-0.0493402 +SPX 306D33WC25ZLA|SPX 31,SPX 160115P01385000,0.1250,0.4250,0.1000,0.1000,387,1062,0.5454214,-0.0012308,0.0000146,0.0208121,-0.0253057,-0.0016181 +SPX 306D33WFD8OKU|SPX 31,SPX 160115P01405000,0.1500,0.4250,0.1250,0.1250,0,990,0.5378942,-0.0015340,0.0000181,0.0254498,-0.0305146,-0.0020169 +SPX 307BJR689X372|SPX 31,SPX 160219P01405000,0.7250,2.8000,0.5500,0.6750,0,1,0.4013579,-0.0060792,0.0000524,0.1407725,-0.0488842,-0.0207839 +SPX 308349MO4JMHA|SPX 31,SPX 160318P01420000,1.7000,3.4000,0.8750,1.5500,0,12,0.3579567,-0.0118708,0.0000866,0.3088943,-0.0640500,-0.0609345 +SPX 307BJR6BKZS6M|SPX 31,SPX 160219P01425000,0.7750,2.8250,0.4000,0.7250,0,29,0.3914255,-0.0066420,0.0000581,0.1522469,-0.0515471,-0.0226880 +SPX 308349SKB4WOE|SPX 31,SPX 160318P01425000,1.7250,3.4750,1.0000,1.5750,0,880,0.3557919,-0.0121117,0.0000886,0.3143244,-0.0647757,-0.0621564 +SPX 307BJR0G27U6M|SPX 31,SPX 160219P01460000,0.8750,2.9000,0.7000,0.8000,0,78,0.3728973,-0.0076031,0.0000687,0.1715284,-0.0552989,-0.0259242 +SPX 308349MOSCYOE|SPX 31,SPX 160318P01460000,2.1500,3.8000,1.2250,1.8250,2,174,0.3424546,-0.0143259,0.0001065,0.3633979,-0.0720314,-0.0734315 +SPX 306D33WU92T0U|SPX 31,SPX 160115P01495000,0.2000,0.4750,0.1500,0.1500,64,945,0.4652147,-0.0020861,0.0000277,0.0336612,-0.0348879,-0.0027306 +SPX 30PIC163N7ULQ|SPX 31,SPX 171215P01500000,72.2000,72.2000,66.3500,67.4000,0,6111,0.2711097,-0.1401088,0.0002829,6.4671284,-0.1115899,-7.0564656 +SPX 307BJR6S4D94E|SPX 31,SPX 160219P01525000,1.2500,3.2750,0.7500,1.1250,5110,5937,0.3460727,-0.0110965,0.0001030,0.2388071,-0.0713761,-0.0377694 +SPX 30D2BMTLKFOTQ|SPX 31,SPX 160916P01525000,23.9000,24.4000,19.4500,21.5500,0,305,0.2876333,-0.0833589,0.0003018,2.7092849,-0.1404313,-1.4176620 +SPX 30ZNMW1QZMVE6|SPX 31,SPX 181221P01525000,111.3000,113.4000,105.6500,106.0500,0,,0.2622195,-0.1685015,0.0002686,8.9898151,-0.0953793,-13.5809293 +SPX 306D33QON4G0E|SPX 31,SPX 160115P01550000,0.2500,1.5750,0.1750,0.1750,61,11827,0.4236083,-0.0026284,0.0000375,0.0414934,-0.0391430,-0.0034318 +SPX 30KJ4O662R95A|SPX 31,SPX 170616P01550000,60.5000,60.5000,54.4000,55.8500,0,501,0.2704491,-0.1400122,0.0003277,5.5907883,-0.1304404,-5.1158323 +SPX 306D33QOZ143Y|SPX 31,SPX 160115P01570000,0.2750,1.7750,0.2500,0.2500,0,909,0.4215045,-0.0036659,0.0000509,0.0559837,-0.0525419,-0.0047900 +SPX 307BJR75CO12M|SPX 31,SPX 160219P01605000,1.9750,3.9000,1.0000,1.5750,0,394,0.3089999,-0.0166487,0.0001638,0.3389127,-0.0902946,-0.0564882 +SPX 307BJR0IPH6Z2|SPX 31,SPX 160219P01620000,2.1500,4.0250,1.0250,1.7000,0,107,0.3026600,-0.0181748,0.0001802,0.3652703,-0.0952852,-0.0616376 +SPX 307BJR7ABA2JY|SPX 31,SPX 160219P01635000,2.3500,2.3500,1.0750,1.8250,240,246,0.2959842,-0.0197481,0.0001978,0.3919977,-0.0999627,-0.0669370 +SPX 306D33QQ4PGGE|SPX 31,SPX 160115P01640000,0.4000,0.5500,0.3000,0.3000,0,1190,0.3675918,-0.0049267,0.0000760,0.0729203,-0.0596403,-0.0064154 +SPX 308349MRXHBM6|SPX 31,SPX 160318P01650000,6.5500,8.1000,5.3500,5.7500,1139,21871,0.2893290,-0.0457390,0.0003325,0.9587400,-0.1597304,-0.2342436 +SPX 308UOSFYDPPIM|SPX 31,SPX 160415P01675000,12.2500,13.4000,8.5500,10.6500,0,,0.2738891,-0.0713608,0.0004325,1.5676932,-0.1850508,-0.4904919 +SPX 307BJR7P746ZY|SPX 31,SPX 160219P01725000,4.5000,6.1500,2.5250,3.6000,52,1885,0.2677540,-0.0392484,0.0003873,0.6944198,-0.1597448,-0.1329913 +SPX 30GIDYMLWTOXA|SPX 31,SPX 170120P01725000,69.3000,69.3000,61.6000,63.2000,0,92,0.2454425,-0.1909500,0.0005175,5.8331907,-0.1694642,-4.9244493 +SPX 30KJ4ODDQ2LIM|SPX 31,SPX 170616P01725000,94.5000,94.5000,86.0000,87.4000,0,609,0.2454746,-0.2118306,0.0004698,7.2756837,-0.1509142,-7.7511093 +SPX 30PIC1DYFZWQ6|SPX 31,SPX 171215P01725000,120.8000,120.8000,111.7000,112.9500,0,1795,0.2463032,-0.2270563,0.0004215,8.7544838,-0.1333196,-11.5074903 +SPX 30D2BMUQWVD66|SPX 31,SPX 160916P01775000,55.9000,55.9000,48.3500,50.3000,0,2749,0.2412837,-0.1921861,0.0006411,4.8281103,-0.2056167,-3.2725822 +SPX 306D33QSG255A|SPX 31,SPX 160115P01780000,1.1000,3.0250,0.6250,0.7000,84,1099,0.2770400,-0.0137652,0.0002484,0.1797130,-0.1105115,-0.0178396 +SPX 306D33Y9IQKBY|SPX 31,SPX 160115P01805000,1.2500,3.1500,0.7500,0.8250,34,3528,0.2605199,-0.0168698,0.0003146,0.2140160,-0.1236745,-0.0218432 +SPX 30D2BMNFYHBTA|SPX 31,SPX 160916P01850000,71.7000,71.7000,62.1500,64.6500,30,1419,0.2294411,-0.2439605,0.0007737,5.5413440,-0.2221946,-4.1636278 +SPX 30GIDYFJ84E1A|SPX 31,SPX 170120P01850000,98.1500,98.1500,88.2500,89.7000,0,415,0.2280445,-0.2653782,0.0006707,7.0243384,-0.1861497,-6.8684036 +SPX 306D33YHSFASU|SPX 31,SPX 160115P01855000,2.2000,3.8750,0.9250,1.3000,56,3097,0.2309732,-0.0281261,0.0005464,0.3295382,-0.1685411,-0.0363719 +SPX 306D33QULGHSE|SPX 31,SPX 160115P01910000,4.6500,6.0000,1.9500,2.8500,157,3543,0.2079109,-0.0609224,0.0011347,0.6160057,-0.2828092,-0.0788043 +SPX 306D33YRPNDRI|SPX 31,SPX 160115P01915000,5.1500,6.3000,1.8000,3.0500,80,2430,0.2055810,-0.0651837,0.0012097,0.6493142,-0.2946630,-0.0843170 +SPX 306D33QURETU6|SPX 31,SPX 160115P01920000,5.6500,6.7500,2.8000,3.3250,160,2738,0.2041169,-0.0705736,0.0012954,0.6904026,-0.3109756,-0.0913032 +SPX 307BJR0NO38GE|SPX 31,SPX 160219P01920000,20.6500,20.6500,14.5500,16.2500,7,1973,0.2069531,-0.1727444,0.0015095,2.0922062,-0.3671840,-0.5860724 +SPX 30FJXBDQ2W86M|SPX 31,SPX 161216P01925000,113.3500,113.3500,102.1000,103.6500,30,6420,0.2187866,-0.3170605,0.0007960,7.2863506,-0.2016481,-7.4442610 +SPX 307BJR0NZZWJY|SPX 31,SPX 160219P01940000,24.2500,24.2500,17.4500,19.1000,234,915,0.2011153,-0.2008926,0.0017056,2.2972420,-0.3908089,-0.6819380 +SPX 30D2BMNHM0OB2|SPX 31,SPX 160916P01950000,99.3000,99.3000,87.8500,89.7000,0,3489,0.2126691,-0.3284581,0.0009620,6.3862373,-0.2329558,-5.6236291 +SPX 30ZNMVVN17UVI|SPX 31,SPX 181221P01950000,238.0000,242.2500,226.3000,230.6500,2,3,0.2222438,-0.3378726,0.0004603,13.0594785,-0.1074988,-27.5840726 +SPX 306D33Z1MVGQ6|SPX 31,SPX 160115P01975000,12.6500,12.6500,7.6000,7.7500,3604,51558,0.1828379,-0.1554880,0.0025558,1.2201338,-0.4897439,-0.2014163 +SPX 308UOSHBZU4BY|SPX 31,SPX 160415P01975000,57.5000,57.5000,47.3500,49.4000,0,,0.1993533,-0.3150786,0.0015491,4.0872195,-0.3401933,-2.1741605 +SPX 30AKPYDDMNNFY|SPX 31,SPX 160617P01975000,80.3000,80.3000,69.2500,71.3000,0,9022,0.2039944,-0.3384095,0.0012495,5.2482595,-0.2824332,-3.7195560 +SPX 306D33Z9WK772|SPX 31,SPX 160115P02025000,25.5000,25.5000,15.0000,16.3000,3814,29961,0.1614736,-0.3028958,0.0042322,1.7843410,-0.6265764,-0.3930779 +SPX 308349MYDODJI|SPX 31,SPX 160318P02040000,68.0500,68.0500,55.2500,57.2500,686,13944,0.1785631,-0.4178762,0.0021904,3.8979745,-0.3824880,-2.1507932 +SPX 306D33ZD7MW6M|SPX 31,SPX 160115P02045000,33.1000,33.1000,20.6500,21.9500,201,3205,0.1533605,-0.3888042,0.0048918,1.9588121,-0.6490181,-0.5052018 +SPX 30PIC16Z2DATQ|SPX 31,SPX 171215P02050000,233.6500,233.6500,218.3500,220.1000,0,2141,0.2177865,-0.4049872,0.0006128,11.2552980,-0.1406896,-20.9333879 +SPX 30ZNMVVOOR7DA|SPX 31,SPX 181221P02050000,279.3500,284.2500,273.1000,273.7000,325,325,0.2272899,-0.3857177,0.0004710,13.6644455,-0.1125423,-32.2398016 +SPX 306D33ZGIPL66|SPX 31,SPX 160115P02065000,42.8500,42.8500,27.6000,29.2000,239,1247,0.1259150,-0.4903427,0.0061987,2.0379216,-0.5452585,-0.6357648 +SPX 307BJR0Q5E972|SPX 31,SPX 160219P02070000,66.6000,66.6000,51.6000,53.7500,169,371,0.1300598,-0.4989275,0.0037482,3.2647858,-0.3406889,-1.6873901 +SPX 307BJR0QT7LE6|SPX 31,SPX 160219P02110000,90.1500,90.1500,71.5000,73.7000,0,50,0.1182678,-0.6591567,0.0037894,3.0014068,-0.2703705,-2.2387955 +SPX 306D33QY8FUVI|SPX 31,SPX 160115P02130000,92.0000,92.0000,68.8500,71.0500,0,82,0.1010510,-0.8879592,0.0036899,0.9735601,-0.1678844,-1.1642975 +SPX 308349MZV9DZI|SPX 31,SPX 160318P02130000,115.9500,115.9500,96.8500,99.0500,0,2,0.1206240,-0.6800760,0.0029694,3.5696921,-0.2114156,-3.4821263 +SPX 307BJR9L05F8U|SPX 31,SPX 160219P02135000,108.4000,108.4000,87.3500,89.8500,0,14,0.1107916,-0.7608617,0.0034220,2.5390820,-0.1999753,-2.5939489 +SPX 308349VX1D8Q6|SPX 31,SPX 160318P02155000,134.2000,134.2000,112.7500,115.3500,0,,0.1137538,-0.7616766,0.0027270,3.0915948,-0.1601387,-3.9138625 +SPX 308349N0J2Q6M|SPX 31,SPX 160318P02170000,146.0500,146.0500,123.1000,126.2000,0,,0.1097928,-0.8078418,0.0024930,2.7278814,-0.1266688,-4.1620575 +SPX 306D33ZYPMELQ|SPX 31,SPX 160115P02175000,135.6000,135.6000,111.0500,113.7500,0,352,0.1071506,-0.9734009,0.0011243,0.3145482,-0.0172881,-1.2985608 +SPX 30AKPYEAPELBI|SPX 31,SPX 160617P02175000,175.7000,175.7000,155.1500,158.2000,3,530,0.1220085,-0.6972861,0.0019941,5.0094064,-0.1302085,-7.5953487 +SPX 30FJXBEVFBWJ2|SPX 31,SPX 161216P02175000,222.9000,222.9000,203.4000,206.2500,0,206,0.1303749,-0.6040359,0.0014448,7.8810604,-0.1048367,-13.8048730 +SPX 30PIC1G0V6IY6|SPX 31,SPX 171215P02175000,295.5500,295.5500,275.5500,278.9000,0,2,0.1412592,-0.5254904,0.0009705,11.5617837,-0.0777832,-25.4725807 +SPX 30ZNMVUUFA3HQ|SPX 31,SPX 181221P02200000,351.0000,360.0000,343.5000,346.2500,31,31,0.1459020,-0.5028949,0.0007653,14.2529712,-0.0598557,-38.5327238 +SPX 306D3406ZB52M|SPX 31,SPX 160115P02225000,185.3000,185.3000,160.3500,163.4000,0,140,0.1198492,-0.9936417,0.0002923,0.0914618,0.0360587,-1.3545196 +SPX 30AKPYEIZ3BSE|SPX 31,SPX 160617P02225000,212.5500,212.5500,189.2500,192.9000,0,140,0.1127631,-0.8038957,0.0017098,3.9698130,-0.0768704,-8.8251965 +SPX 308349N1ISQHA|SPX 31,SPX 160318P02230000,199.3500,199.3500,174.2000,177.1500,0,,0.0978264,-0.9405065,0.0012117,1.1813385,-0.0100340,-4.9217535 +SPX 308349N1UPEKU|SPX 31,SPX 160318P02250000,218.2000,218.2000,192.4500,196.1500,1,93,0.0958760,-0.9628105,0.0008484,0.8106909,0.0139091,-5.0736312 +SPX 30AKPY5C7O26M|SPX 31,SPX 160617P02250000,232.9500,232.9500,209.0000,212.5000,0,1340,0.1086362,-0.8505992,0.0014917,3.3367370,-0.0495508,-9.3855166 +SPX 30D2BMNMKMPSE|SPX 31,SPX 160916P02250000,251.8000,251.8000,228.6000,232.4500,0,,0.1163369,-0.7709011,0.0014741,5.3529300,-0.0675084,-13.0877984 +SPX 306D33R0DU7IM|SPX 31,SPX 160115P02260000,220.2500,220.2500,195.3500,198.3500,0,,0.1556556,-0.9897124,0.0003435,0.1395984,0.0127259,-1.3710739 +SPX 307BJR0TAIM4U|SPX 31,SPX 160219P02260000,224.9000,224.9000,199.2500,202.5500,0,,0.0993784,-0.9874385,0.0003996,0.2659361,0.0380844,-3.5061091 +SPX 306D33R0JSJKE|SPX 31,SPX 160115P02270000,230.2000,230.2000,204.9500,208.3000,85,87,0.1444830,-0.9956174,0.0001741,0.0656906,0.0406894,-1.3845592 +SPX 308349WGVTENI|SPX 31,SPX 160318P02275000,242.5000,242.5000,216.7000,220.4000,0,317,0.0953412,-0.9790341,0.0005294,0.5030698,0.0329241,-5.2087166 +SPX 30PIC164YUIZY|SPX 31,SPX 171215P02300000,369.3500,369.3500,347.3000,350.3000,0,1,0.1317910,-0.6494106,0.0009684,10.7631144,-0.0539887,-31.9245900 +SPX 308349N30DQXA|SPX 31,SPX 160318P02320000,287.3500,287.3500,260.5500,264.9500,0,,0.1002823,-0.9902418,0.0002606,0.2604427,0.0475724,-5.3668876 +SPX 30ZNMW5FAMMWE|SPX 31,SPX 181221P02325000,421.1500,427.5500,412.2000,415.2000,0,,0.1390319,-0.5984965,0.0007785,13.8166897,-0.0453172,-46.4998419 +SPX 30PIC1740ZCB2|SPX 31,SPX 171215P02350000,402.6500,402.6500,379.8500,382.7500,0,250,0.1286517,-0.6972093,0.0009345,10.1393045,-0.0424925,-34.5268268 +SPX 30PIC1GXXXGTQ|SPX 31,SPX 171215P02375000,419.6500,419.6500,396.8000,399.8500,0,1,0.1269808,-0.7206556,0.0009118,9.7645585,-0.0362304,-35.8240089 +SPX 30GIDYFT5CGZY|SPX 31,SPX 170120P02450000,437.7500,437.7500,411.8500,415.7500,0,,0.1044499,-0.9230939,0.0006446,3.0920581,0.0212060,-24.4899650 +SPX 30ZNMVVVAWLCE|SPX 31,SPX 181221P02450000,500.4000,508.0000,490.9500,493.6000,0,,0.1321086,-0.6928202,0.0007445,12.5541861,-0.0258106,-54.7053839 +SPX 30FJXBGHB51TA|SPX 31,SPX 161216P02525000,507.6000,507.6000,480.4000,485.1000,0,,0.0994484,-0.9709857,0.0003253,1.3536882,0.0481387,-23.9823052 +SPX 30ZNMW6CDDKRY|SPX 31,SPX 181221P02525000,558.2500,561.0500,543.2500,546.0000,0,,0.1296722,-0.7427791,0.0006962,11.5245654,-0.0139875,-59.4273800 +SPX 306D33R56HWY6|SPX 31,SPX 160115P02550000,510.0000,510.0000,483.5000,488.1000,0,19,0.3927130,-0.9815923,0.0002217,0.2338126,-0.1307973,-1.5827232 +SPX 30FJXB61W7EVI|SPX 31,SPX 161216P02550000,531.8500,531.8500,504.5500,509.3500,0,,0.1003838,-0.9759197,0.0002758,1.1584400,0.0516429,-24.3214189 +SPX 30GIDYFUSVTHQ|SPX 31,SPX 170120P02550000,532.1000,532.1000,505.2000,509.6000,0,,0.1005625,-0.9691639,0.0003230,1.4917295,0.0483706,-26.5278914 +SPX 306D33R6U19FY|SPX 31,SPX 160115P02650000,609.9000,609.9000,583.5000,588.0000,0,,0.4535594,-0.9834688,0.0001751,0.2133032,-0.1386578,-1.6481540 +SPX 308349M53X9YM|SPX 31,SPX 160318P02700000,666.3000,666.3000,639.4000,643.9500,0,,0.1908919,-0.9977202,0.0000375,0.0712754,0.0657050,-6.2891749 +SPX 30ZNMVW09IMTQ|SPX 31,SPX 181221P02750000,735.2500,737.5000,720.6500,721.8000,0,,0.1208248,-0.8694378,0.0004915,7.5806486,0.0245582,-72.6475502 +SPX 30AKPY4FMU9M6|SPX 31,SPX 160617P02800000,771.0000,771.0000,744.7500,748.5500,0,,0.1562682,-0.9966103,0.0000455,0.1465340,0.0696652,-13.4314000 +SPX 30FJXB50CRKTQ|SPX 31,SPX 161216P02800000,777.8000,777.8000,750.6500,755.2000,0,17,0.2344222,-0.8754691,0.0004278,4.2032848,-0.0675783,-25.0065368 +SPX 30KJ4O5L8N832|SPX 31,SPX 170616P02900000,878.7000,878.7000,851.7000,856.0000,0,2,0.1158120,-0.9873498,0.0001124,0.8211936,0.0688026,-41.9336798 +SPX 30KJ4O6T8A426|SPX 31,SPX 170616P02950000,927.7000,927.7000,900.6000,905.0500,0,,0.1193518,-0.9888118,0.0000980,0.7382080,0.0708791,-42.7058646 +SPX 30KJ4O5LWGKA6|SPX 31,SPX 170616P03300000,1271.5500,1271.5500,1244.5500,1248.9000,0,,0.1540328,-0.9901091,0.0000682,0.6630438,0.0791063,-47.8527718 diff --git a/Tests/Algorithm/AlgorithmChainsTests.cs b/Tests/Algorithm/AlgorithmChainsTests.cs index 179cb47be80a..c9457f0b3816 100644 --- a/Tests/Algorithm/AlgorithmChainsTests.cs +++ b/Tests/Algorithm/AlgorithmChainsTests.cs @@ -88,7 +88,7 @@ def get_option_chain_data_from_dataframe(algorithm, canonical): # Will make it more complex than it needs to be, # just so that we can test indexing by symbol using df.loc[] - for (symbol, date) in option_chain_df.index: + for (symbol,) in option_chain_df.index: symbol_data = option_chain_df.loc[(symbol)] if symbol_data.shape[0] != 1: @@ -96,14 +96,22 @@ def get_option_chain_data_from_dataframe(algorithm, canonical): yield { 'symbol': symbol, - 'date': date, - 'open': symbol_data['open'].values[0], - 'high': symbol_data['high'].values[0], - 'low': symbol_data['low'].values[0], - 'close': symbol_data['close'].values[0], + 'expiry': symbol_data['expiry'].values[0], + 'strike': symbol_data['strike'].values[0], + 'right': symbol_data['right'].values[0], + 'style': symbol_data['style'].values[0], + 'lastprice': symbol_data['lastprice'].values[0], + 'askprice': symbol_data['askprice'].values[0], + 'bidprice': symbol_data['bidprice'].values[0], 'openinterest': symbol_data['openinterest'].values[0], 'impliedvolatility': symbol_data['impliedvolatility'].values[0], - 'greeks': symbol_data['greeks'].values[0] + 'delta': symbol_data['delta'].values[0], + 'gamma': symbol_data['gamma'].values[0], + 'vega': symbol_data['vega'].values[0], + 'theta': symbol_data['theta'].values[0], + 'rho': symbol_data['rho'].values[0], + 'underlyingsymbol': symbol_data['underlyingsymbol'].values[0], + 'underlyinglastprice': symbol_data['underlyinglastprice'].values[0], } "); From 3dbcc4b41d15b1856e65036fc2aa886be9c964c5 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 27 Sep 2024 12:41:13 -0400 Subject: [PATCH 23/38] Unit test fix --- Tests/Engine/DataFeeds/TimeSliceTests.cs | 92 +++++++++++++++++++----- 1 file changed, 73 insertions(+), 19 deletions(-) diff --git a/Tests/Engine/DataFeeds/TimeSliceTests.cs b/Tests/Engine/DataFeeds/TimeSliceTests.cs index 30eab2e49c6e..9d867b5621e9 100644 --- a/Tests/Engine/DataFeeds/TimeSliceTests.cs +++ b/Tests/Engine/DataFeeds/TimeSliceTests.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -26,6 +26,9 @@ using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Securities; using QuantConnect.Tests.Common.Data.UniverseSelection; +using QuantConnect.Securities.Option; +using QuantConnect.Securities.Future; +using QuantConnect.Securities.Equity; namespace QuantConnect.Tests.Engine.DataFeeds { @@ -217,15 +220,7 @@ public void SuspiciousTicksAreNotAddedToConsolidatorUpdateData() private IEnumerable GetSlices(Symbol symbol, int initialVolume) { var subscriptionDataConfig = new SubscriptionDataConfig(typeof(ZipEntryName), symbol, Resolution.Second, TimeZones.Utc, TimeZones.Utc, true, true, false); - var security = new Security( - SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), - subscriptionDataConfig, - new Cash(Currencies.USD, 0, 1m), - SymbolProperties.GetDefault(Currencies.USD), - ErrorCurrencyConverter.Instance, - RegisteredSecurityDataTypesProvider.Null, - new SecurityCache() - ); + var security = GetSecurity(subscriptionDataConfig); var refTime = DateTime.UtcNow; return Enumerable @@ -237,20 +232,79 @@ private IEnumerable GetSlices(Symbol symbol, int initialVolume) var ask = new Bar(110, 110, 110, 110); var volume = (i + 1) * initialVolume; + var packets = new List(); + var packet = new DataFeedPacket(security, subscriptionDataConfig, new List + { + new QuoteBar(time, symbol, bid, i*10, ask, (i + 1) * 11), + new TradeBar(time, symbol, 100, 100, 110, 106, volume) + }); + + if (symbol.SecurityType == SecurityType.Option) + { + var underlying = (security as Option).Underlying; + packets.Add(new DataFeedPacket(underlying, underlying.SubscriptionDataConfig, new List + { + new QuoteBar(time, underlying.Symbol, bid, i*10, ask, (i + 1) * 11), + new TradeBar(time, underlying.Symbol, 100, 100, 110, 106, volume) + })); + } + + packets.Add(packet); + return _timeSliceFactory.Create( time, - new List - { - new DataFeedPacket(security, subscriptionDataConfig, new List - { - new QuoteBar(time, symbol, bid, i*10, ask, (i + 1) * 11), - new TradeBar(time, symbol, 100, 100, 110, 106, volume) - }), - }, + packets, SecurityChangesTests.CreateNonInternal(Enumerable.Empty(), Enumerable.Empty()), new Dictionary()) .Slice; }); } + + private Security GetSecurity(SubscriptionDataConfig config) + { + if (config.Symbol.SecurityType == SecurityType.Option) + { + var option = new Option( + SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), + config, + new Cash(Currencies.USD, 0, 1m), + new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), + ErrorCurrencyConverter.Instance, + RegisteredSecurityDataTypesProvider.Null); + var underlyingConfig = new SubscriptionDataConfig(typeof(ZipEntryName), config.Symbol.Underlying, Resolution.Second, + TimeZones.Utc, TimeZones.Utc, true, true, false); + var equity = new Equity( + SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), + underlyingConfig, + new Cash(Currencies.USD, 0, 1m), + SymbolProperties.GetDefault(Currencies.USD), + ErrorCurrencyConverter.Instance, + RegisteredSecurityDataTypesProvider.Null); + option.Underlying = equity; + + return option; + } + + if (config.Symbol.SecurityType == SecurityType.Future) + { + return new Future( + SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), + config, + new Cash(Currencies.USD, 0, 1m), + SymbolProperties.GetDefault(Currencies.USD), + ErrorCurrencyConverter.Instance, + RegisteredSecurityDataTypesProvider.Null); + } + + return new Security( + SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), + config, + new Cash(Currencies.USD, 0, 1m), + SymbolProperties.GetDefault(Currencies.USD), + ErrorCurrencyConverter.Instance, + RegisteredSecurityDataTypesProvider.Null, + new SecurityCache() + ); + } } -} \ No newline at end of file +} From 801b0f5add8c8971c73f752d45a4abc6fa84d4c4 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 27 Sep 2024 15:40:42 -0400 Subject: [PATCH 24/38] Update Pythonnet to 2.0.39 --- Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj | 2 +- Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj | 2 +- Algorithm.Python/QuantConnect.Algorithm.Python.csproj | 2 +- Algorithm/QuantConnect.Algorithm.csproj | 2 +- AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj | 2 +- Common/QuantConnect.csproj | 2 +- Engine/QuantConnect.Lean.Engine.csproj | 2 +- Indicators/QuantConnect.Indicators.csproj | 2 +- Report/QuantConnect.Report.csproj | 2 +- Research/QuantConnect.Research.csproj | 2 +- Tests/QuantConnect.Tests.csproj | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj b/Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj index 9bd55cda326c..75ace3e4ab70 100644 --- a/Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj +++ b/Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj @@ -34,7 +34,7 @@ portable - + diff --git a/Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj b/Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj index 8f4de2012ea2..cd7e54e89dda 100644 --- a/Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj +++ b/Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj @@ -30,7 +30,7 @@ LICENSE - + diff --git a/Algorithm.Python/QuantConnect.Algorithm.Python.csproj b/Algorithm.Python/QuantConnect.Algorithm.Python.csproj index 5b199146b18e..1713557d4efa 100644 --- a/Algorithm.Python/QuantConnect.Algorithm.Python.csproj +++ b/Algorithm.Python/QuantConnect.Algorithm.Python.csproj @@ -39,7 +39,7 @@ - + diff --git a/Algorithm/QuantConnect.Algorithm.csproj b/Algorithm/QuantConnect.Algorithm.csproj index 2a59f8f6235b..d2fe1d7621db 100644 --- a/Algorithm/QuantConnect.Algorithm.csproj +++ b/Algorithm/QuantConnect.Algorithm.csproj @@ -30,7 +30,7 @@ LICENSE - + diff --git a/AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj b/AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj index 6a59f8ca8c6d..f92733aaeff6 100644 --- a/AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj +++ b/AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj @@ -29,7 +29,7 @@ LICENSE - + diff --git a/Common/QuantConnect.csproj b/Common/QuantConnect.csproj index 262200725a5a..4218bf22f991 100644 --- a/Common/QuantConnect.csproj +++ b/Common/QuantConnect.csproj @@ -35,7 +35,7 @@ - + diff --git a/Engine/QuantConnect.Lean.Engine.csproj b/Engine/QuantConnect.Lean.Engine.csproj index 9b94f153d79e..8f90895f0634 100644 --- a/Engine/QuantConnect.Lean.Engine.csproj +++ b/Engine/QuantConnect.Lean.Engine.csproj @@ -43,7 +43,7 @@ - + diff --git a/Indicators/QuantConnect.Indicators.csproj b/Indicators/QuantConnect.Indicators.csproj index 33fbd23c8736..8b2f8612e1c3 100644 --- a/Indicators/QuantConnect.Indicators.csproj +++ b/Indicators/QuantConnect.Indicators.csproj @@ -32,7 +32,7 @@ - + diff --git a/Report/QuantConnect.Report.csproj b/Report/QuantConnect.Report.csproj index abddd475832d..7ea9c799fd36 100644 --- a/Report/QuantConnect.Report.csproj +++ b/Report/QuantConnect.Report.csproj @@ -41,7 +41,7 @@ LICENSE - + diff --git a/Research/QuantConnect.Research.csproj b/Research/QuantConnect.Research.csproj index b3365f346939..73b27052c640 100644 --- a/Research/QuantConnect.Research.csproj +++ b/Research/QuantConnect.Research.csproj @@ -34,7 +34,7 @@ - + diff --git a/Tests/QuantConnect.Tests.csproj b/Tests/QuantConnect.Tests.csproj index 1944cfd4475d..c92d8746dd66 100644 --- a/Tests/QuantConnect.Tests.csproj +++ b/Tests/QuantConnect.Tests.csproj @@ -33,7 +33,7 @@ - + From 5b357b0fe7e6cf3c0b0e2d7525482dd99b09f490 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 27 Sep 2024 15:43:12 -0400 Subject: [PATCH 25/38] Cleanup --- Algorithm/QCAlgorithm.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 1741ea787858..2d5e5d1422d9 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -3393,7 +3393,6 @@ public OptionChains OptionChains(IEnumerable symbols) { Symbol = contractSymbol, EndTime = Time.Date, - Data = null, }); return (symbol, optionChainData); }); From c8787d7ca6b890d3f5ded98768dc5d7c274d48cb Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 27 Sep 2024 17:51:17 -0400 Subject: [PATCH 26/38] PandasData handling children class members Address peer review --- Common/Data/Market/ModeledGreeks.cs | 36 +++-- Common/Data/Market/NullGreeks.cs | 35 +++- Common/Data/Market/OptionContract.cs | 2 +- .../Data/UniverseSelection/OptionUniverse.cs | 6 - Common/Python/PandasData.cs | 150 +++++++++++------- .../Option/CurrentPriceOptionPriceModel.cs | 2 +- .../Option/OptionPriceModelResult.cs | 2 +- .../OptionPriceModelPriceGeneratorTests.cs | 2 +- 8 files changed, 149 insertions(+), 86 deletions(-) diff --git a/Common/Data/Market/ModeledGreeks.cs b/Common/Data/Market/ModeledGreeks.cs index 03951d65dce2..d93be9fd54c9 100644 --- a/Common/Data/Market/ModeledGreeks.cs +++ b/Common/Data/Market/ModeledGreeks.cs @@ -29,22 +29,34 @@ public class ModeledGreeks : Greeks private Lazy _rho; private Lazy _lambda; - /// + /// + /// Gets the delta + /// public override decimal Delta => _delta.Value; - /// + /// + /// Gets the gamma + /// public override decimal Gamma => _gamma.Value; - /// + /// + /// Gets the vega + /// public override decimal Vega => _vega.Value; - /// + /// + /// Gets the theta + /// public override decimal Theta => _theta.Value; - /// + /// + /// Gets the rho + /// public override decimal Rho => _rho.Value; - /// + /// + /// Gets the lambda + /// public override decimal Lambda => _lambda.Value; /// @@ -52,12 +64,12 @@ public class ModeledGreeks : Greeks /// public ModeledGreeks(Func delta, Func gamma, Func vega, Func theta, Func rho, Func lambda) { - _delta = new Lazy(delta); - _gamma = new Lazy(gamma); - _vega = new Lazy(vega); - _theta = new Lazy(theta); - _rho = new Lazy(rho); - _lambda = new Lazy(lambda); + _delta = new Lazy(delta, isThreadSafe: false); + _gamma = new Lazy(gamma, isThreadSafe: false); + _vega = new Lazy(vega, isThreadSafe: false); + _theta = new Lazy(theta, isThreadSafe: false); + _rho = new Lazy(rho, isThreadSafe: false); + _lambda = new Lazy(lambda, isThreadSafe: false); } } } diff --git a/Common/Data/Market/NullGreeks.cs b/Common/Data/Market/NullGreeks.cs index 5eec585322c2..695bbf412880 100644 --- a/Common/Data/Market/NullGreeks.cs +++ b/Common/Data/Market/NullGreeks.cs @@ -18,24 +18,45 @@ namespace QuantConnect.Data.Market /// /// Defines greeks that are all zero /// - public class NullGreeks : Greeks + internal class NullGreeks : Greeks { - /// + /// + /// Singleton instance of + /// + public static readonly NullGreeks Instance = new NullGreeks(); + + /// + /// Gets the delta + /// public override decimal Delta => decimal.Zero; - /// + /// + /// Gets the gamma + /// public override decimal Gamma => decimal.Zero; - /// + /// + /// Gets the vega + /// public override decimal Vega => decimal.Zero; - /// + /// + /// Gets the theta + /// public override decimal Theta => decimal.Zero; - /// + /// + /// Gets the rho + /// public override decimal Rho => decimal.Zero; - /// + /// + /// Gets the lambda + /// public override decimal Lambda => decimal.Zero; + + private NullGreeks() + { + } } } diff --git a/Common/Data/Market/OptionContract.cs b/Common/Data/Market/OptionContract.cs index 26b85561bc42..b5b4cd693367 100644 --- a/Common/Data/Market/OptionContract.cs +++ b/Common/Data/Market/OptionContract.cs @@ -299,7 +299,7 @@ private class OptionPriceModelResultData : IOptionData public OptionPriceModelResultData(Func optionPriceModelEvaluator, OptionPriceModelResultData previousOptionData = null) { - _optionPriceModelResult = new(optionPriceModelEvaluator); + _optionPriceModelResult = new(optionPriceModelEvaluator, isThreadSafe: false); if (previousOptionData != null) { diff --git a/Common/Data/UniverseSelection/OptionUniverse.cs b/Common/Data/UniverseSelection/OptionUniverse.cs index b3afe4bf1c69..dee24e34300c 100644 --- a/Common/Data/UniverseSelection/OptionUniverse.cs +++ b/Common/Data/UniverseSelection/OptionUniverse.cs @@ -335,22 +335,16 @@ private class PreCalculatedGreeks : Greeks { private readonly string _csvLine; - /// public override decimal Delta => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex); - /// public override decimal Gamma => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 1); - /// public override decimal Vega => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 2); - /// public override decimal Theta => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 3); - /// public override decimal Rho => _csvLine.GetDecimalFromCsv(StartingGreeksCsvIndex + 4); - /// public override decimal Lambda => decimal.Zero; /// diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index c6bf86ccadeb..83adf26dc358 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -72,16 +72,17 @@ public class PandasData private static readonly string[] _greeksMemberNames = new[] { - nameof(Greeks.Delta), - nameof(Greeks.Gamma), - nameof(Greeks.Vega), - nameof(Greeks.Theta), - nameof(Greeks.Rho), + nameof(Greeks.Delta).ToLowerInvariant(), + nameof(Greeks.Gamma).ToLowerInvariant(), + nameof(Greeks.Vega).ToLowerInvariant(), + nameof(Greeks.Theta).ToLowerInvariant(), + nameof(Greeks.Rho).ToLowerInvariant(), }; private static readonly MemberInfo[] _greeksMembers = typeof(Greeks) .GetMembers(BindingFlags.Instance | BindingFlags.Public) - .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) && _greeksMemberNames.Contains(x.Name)) + .Where(x => (x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property) && + _greeksMemberNames.Contains(x.Name.ToLowerInvariant())) .ToArray(); #endregion @@ -99,7 +100,7 @@ public class PandasData private static PyList _level3Names; private readonly static HashSet _baseDataProperties = typeof(BaseData).GetProperties().ToHashSet(x => x.Name.ToLowerInvariant()); - private readonly static ConcurrentDictionary> _membersByType = new(); + private readonly static ConcurrentDictionary> _membersByType = new(); private readonly static IReadOnlyList _standardColumns = new string[] { Open, High, Low, Close, LastPrice, Volume, @@ -109,9 +110,10 @@ public class PandasData private readonly Symbol _symbol; private readonly bool _isFundamentalType; + private readonly bool _isBaseData; private readonly Dictionary _series; - private readonly IEnumerable _members = Enumerable.Empty(); + private readonly IEnumerable _members = Enumerable.Empty(); /// /// Gets true if this is a custom data request, false for normal QC data @@ -166,7 +168,8 @@ public PandasData(object data) var type = data.GetType(); _isFundamentalType = type == typeof(Fundamental); - _symbol = ((ISymbolProvider)data).Symbol; + _isBaseData = baseData != null; + _symbol = _isBaseData ? baseData.Symbol : ((ISymbolProvider)data).Symbol; IsCustomData = Extensions.IsCustomDataType(_symbol, type); if (baseData == null) @@ -184,7 +187,7 @@ public PandasData(object data) IEnumerable columns = _standardColumns; - if (IsCustomData || baseData == null || baseData.DataType == MarketDataType.Auxiliary) + if (IsCustomData || !_isBaseData || baseData.DataType == MarketDataType.Auxiliary) { var keys = (data as DynamicData)?.GetStorageDictionary() // if this is a PythonData instance we add in '__typename' which we don't want into the data frame @@ -195,7 +198,7 @@ public PandasData(object data) { if (_membersByType.TryGetValue(type, out _members)) { - keys = _members.ToHashSet(x => x.Name.ToLowerInvariant()); + keys = _members.SelectMany(x => x.GetMemberNames()).ToHashSet(); } else { @@ -210,34 +213,33 @@ public PandasData(object data) members = members.Where(x => !_optionContractExcludedMembers.Contains(x.Name)); } - members = members.ToList(); + var dataTypeMembers = members.Select(x => + { + if (!DataTypeMember.GetMemberType(x).IsAssignableTo(typeof(Greeks))) + { + return new DataTypeMember(x); + } + + return new DataTypeMember(x, _greeksMembers); + }).ToList(); - var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key); + var duplicateKeys = dataTypeMembers.GroupBy(x => x.Member.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key); foreach (var duplicateKey in duplicateKeys) { throw new ArgumentException($"PandasData.ctor(): {Messages.PandasData.DuplicateKey(duplicateKey, type.FullName)}"); } // If the custom data derives from a Market Data (e.g. Tick, TradeBar, QuoteBar), exclude its keys - keys = members.ToHashSet(x => x.Name.ToLowerInvariant()); + keys = dataTypeMembers.SelectMany(x => x.GetMemberNames()).ToHashSet(); keys.ExceptWith(_baseDataProperties); keys.ExceptWith(GetPropertiesNames(typeof(QuoteBar), type)); keys.ExceptWith(GetPropertiesNames(typeof(TradeBar), type)); keys.ExceptWith(GetPropertiesNames(typeof(Tick), type)); keys.Add("value"); - _members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant())).ToList(); + _members = dataTypeMembers.Where(x => x.GetMemberNames().All(name => keys.Contains(name))).ToList(); _membersByType.TryAdd(type, _members); } - - // Make sure to add the greeks member names to the series so they can be added to the data frame - if (_members.Any(x => GetMemberType(x).IsAssignableTo(typeof(Greeks)))) - { - foreach (var greekMemberName in _greeksMemberNames) - { - keys.Add(greekMemberName.ToLowerInvariant()); - } - } } var customColumns = new HashSet(columns) { "value" }; @@ -255,43 +257,31 @@ public PandasData(object data) /// object that contains security data public void Add(object baseData) { - var endTime = default(DateTime); - if (baseData is IBaseData iBaseData) - { - endTime = iBaseData.EndTime; - } - + var endTime = _isBaseData ? ((IBaseData)baseData).EndTime : default; foreach (var member in _members) { - var memberType = GetMemberType(member); - if (!memberType.IsAssignableTo(typeof(Greeks))) + if (!member.ShouldBeUnwrapped) { - AddMemberToSeries(baseData, endTime, member); + AddMemberToSeries(baseData, endTime, member.Member); } else { - var greeks = member switch - { - PropertyInfo property => property.GetValue(baseData), - FieldInfo field => field.GetValue(baseData), - // Should not happen - _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") - }; - - if (greeks != null) + var memberValue = member.GetMemberValue(baseData); + if (memberValue != null) { - foreach (var greekMember in _greeksMembers) + foreach (var childMember in member.Children) { - AddMemberToSeries(greeks, endTime, greekMember); + AddMemberToSeries(memberValue, endTime, childMember); } } } } - var storage = (baseData as DynamicData)?.GetStorageDictionary(); + var dynamicData = baseData as DynamicData; + var storage = dynamicData?.GetStorageDictionary(); if (storage != null) { - var value = ((IBaseData)baseData).Value; + var value = dynamicData.Value; AddToSeries("value", endTime, value); foreach (var kvp in storage.Where(x => x.Key != "value" @@ -315,17 +305,6 @@ public void Add(object baseData) } } - private static Type GetMemberType(MemberInfo member) - { - return member switch - { - PropertyInfo property => property.PropertyType, - FieldInfo field => field.FieldType, - // Should not happen - _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") - }; - } - private void AddMemberToSeries(object baseData, DateTime endTime, MemberInfo member) { // TODO field/property.GetValue is expensive @@ -661,5 +640,62 @@ public FixedTimeProvider(DateTime time) _time = time; } } + + private class DataTypeMember + { + public MemberInfo Member { get; } + + public MemberInfo[] Children { get; } + + public bool ShouldBeUnwrapped => Children != null && Children.Length > 0; + + public DataTypeMember(MemberInfo member, MemberInfo[] children = null) + { + Member = member; + Children = children; + } + + public IEnumerable GetMemberNames() + { + // If there are no children, return the name of the member. Else ignore the member and return the children names + if (ShouldBeUnwrapped) + { + foreach (var child in Children) + { + yield return child.Name.ToLowerInvariant(); + } + yield break; + } + + yield return Member.Name.ToLowerInvariant(); + } + + public Type GetMemberType() + { + return GetMemberType(Member); + } + + public object GetMemberValue(object instance) + { + return Member switch + { + PropertyInfo property => property.GetValue(instance), + FieldInfo field => field.GetValue(instance), + // Should not happen + _ => throw new InvalidOperationException($"Unexpected member type: {Member.MemberType}") + }; + } + + public static Type GetMemberType(MemberInfo member) + { + return member switch + { + PropertyInfo property => property.PropertyType, + FieldInfo field => field.FieldType, + // Should not happen + _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") + }; + } + } } } diff --git a/Common/Securities/Option/CurrentPriceOptionPriceModel.cs b/Common/Securities/Option/CurrentPriceOptionPriceModel.cs index 29dc71cde6e5..35dad2800b9e 100644 --- a/Common/Securities/Option/CurrentPriceOptionPriceModel.cs +++ b/Common/Securities/Option/CurrentPriceOptionPriceModel.cs @@ -38,7 +38,7 @@ public class CurrentPriceOptionPriceModel : IOptionPriceModel /// price of the specified option contract public OptionPriceModelResult Evaluate(Security security, Slice slice, OptionContract contract) { - return new OptionPriceModelResult(security.Price, new NullGreeks()); + return new OptionPriceModelResult(security.Price, NullGreeks.Instance); } } } diff --git a/Common/Securities/Option/OptionPriceModelResult.cs b/Common/Securities/Option/OptionPriceModelResult.cs index 0e19e272c67b..417d0f6efc74 100644 --- a/Common/Securities/Option/OptionPriceModelResult.cs +++ b/Common/Securities/Option/OptionPriceModelResult.cs @@ -26,7 +26,7 @@ public class OptionPriceModelResult /// /// Represents the zero option price and greeks. /// - public static OptionPriceModelResult None { get; } = new(0, new NullGreeks()); + public static OptionPriceModelResult None { get; } = new(0, NullGreeks.Instance); private readonly Lazy _greeks; private readonly Lazy _impliedVolatility; diff --git a/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs b/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs index 3ec4937bc2dc..979f07a37ace 100644 --- a/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs +++ b/Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs @@ -97,7 +97,7 @@ public void ReturnsNewPrice() var priceModelMock = new Mock(); priceModelMock .Setup(s => s.Evaluate(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(new OptionPriceModelResult(1000, new NullGreeks())); + .Returns(new OptionPriceModelResult(1000, NullGreeks.Instance)); _option.PriceModel = priceModelMock.Object; var randomPriceGenerator = new OptionPriceModelPriceGenerator(_option); From f261e7b187093378b8e829eb4a9e0e389d418573 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 30 Sep 2024 19:16:13 -0400 Subject: [PATCH 27/38] Fix: indexing symbol conversion in pandas mapper --- Common/Data/Market/ModeledGreeks.cs | 2 +- Common/PandasMapper.py | 24 +++++++++++++++++++----- Common/Python/PandasData.cs | 5 ----- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Common/Data/Market/ModeledGreeks.cs b/Common/Data/Market/ModeledGreeks.cs index d93be9fd54c9..e1feb8fb5726 100644 --- a/Common/Data/Market/ModeledGreeks.cs +++ b/Common/Data/Market/ModeledGreeks.cs @@ -20,7 +20,7 @@ namespace QuantConnect.Data.Market /// /// Defines the greeks /// - public class ModeledGreeks : Greeks + internal class ModeledGreeks : Greeks { private Lazy _delta; private Lazy _gamma; diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index 3800ccd29d3d..ebb76e8ee995 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -25,24 +25,38 @@ AddReference("QuantConnect.Common") from QuantConnect import * -def mapper(key): +def mapper(key, force_symbol_conversion = False): '''Maps a Symbol object or a Symbol Ticker (string) to the string representation of Symbol SecurityIdentifier.If cannot map, returns the object ''' keyType = type(key) if keyType is str: reserved = ['high', 'low', 'open', 'close'] - if key in reserved: + if not force_symbol_conversion or key in reserved: return key + kvp = SymbolCache.TryGetSymbol(key, None) if kvp[0]: return kvp[1] if keyType is list: - return [mapper(x) for x in key] + return [mapper(x, force_symbol_conversion) for x in key] if keyType is tuple: - return tuple([mapper(x) for x in key]) + # If 'self' (the first arg) is an index and it contains symbols, we need to force conversion of string keys to symbols + if not force_symbol_conversion and (type(key[0]) is pd.MultiIndex or type(key[0]) is pd.Index): + for index_row in key[0]: + values = index_row + if type(values) is not tuple: + values = [values] + for x in values: + if type(x) is Symbol: + force_symbol_conversion = True + break + if force_symbol_conversion: + break + + return tuple([mapper(x, force_symbol_conversion) for x in key]) if keyType is dict: - return { k: mapper(v) for k, v in key.items()} + return { k: mapper(v, force_symbol_conversion) for k, v in key.items()} return key def wrap_keyerror_function(f): diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 83adf26dc358..dc247d24f1f5 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -670,11 +670,6 @@ public IEnumerable GetMemberNames() yield return Member.Name.ToLowerInvariant(); } - public Type GetMemberType() - { - return GetMemberType(Member); - } - public object GetMemberValue(object instance) { return Member switch From a27cea6f72a701d60b69849abea7f58431b8a5d8 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 1 Oct 2024 16:22:39 -0400 Subject: [PATCH 28/38] Fix pandas mapper to convert string keys to symbol only when necessary --- Common/PandasMapper.py | 47 +++++++++++------ Common/Python/PandasConverter.cs | 23 +++++---- Common/Python/PandasData.cs | 77 ++++++++++++++++++++-------- Common/Symbol.cs | 43 +++++++--------- Tests/Python/PandasConverterTests.cs | 6 +-- Tests/Python/PandasIndexingTests.cs | 2 +- 6 files changed, 122 insertions(+), 76 deletions(-) diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index ebb76e8ee995..94b30a8a0049 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -31,28 +31,45 @@ def mapper(key, force_symbol_conversion = False): ''' keyType = type(key) if keyType is str: + if force_symbol_conversion: + kvp = SymbolCache.try_get_symbol(key, None) + if kvp[0]: + return kvp[1] + else: reserved = ['high', 'low', 'open', 'close'] - if not force_symbol_conversion or key in reserved: + if key in reserved: return key - kvp = SymbolCache.TryGetSymbol(key, None) - if kvp[0]: - return kvp[1] if keyType is list: return [mapper(x, force_symbol_conversion) for x in key] if keyType is tuple: - # If 'self' (the first arg) is an index and it contains symbols, we need to force conversion of string keys to symbols - if not force_symbol_conversion and (type(key[0]) is pd.MultiIndex or type(key[0]) is pd.Index): - for index_row in key[0]: - values = index_row - if type(values) is not tuple: - values = [values] - for x in values: - if type(x) is Symbol: + # If 'self' (the first arg) is an index and it contains symbols, we try to convert string keys into symbols + if not force_symbol_conversion: + self_value = key[0] + self_type = type(self_value) + if self_type is pd.MultiIndex or self_type is pd.Index: + # We add the __has_symbols__ attribute to the index to avoid checking for symbols in the future + if hasattr(self_value, '__has_symbols__'): + if getattr(self_value, '__has_symbols__'): force_symbol_conversion = True - break - if force_symbol_conversion: - break + # Check whether the index contains symbols, if it does we add the __has_symbols__ attribute and force conversion + else: + has_symbols = False + for index_row in self_value: + values = index_row + if type(values) is not tuple: + values = [values] + + for x in values: + if type(x) is Symbol: + has_symbols = True + break + if has_symbols: + break + + force_symbol_conversion = has_symbols + setattr(self_value, '__has_symbols__', has_symbols) + return tuple([mapper(x, force_symbol_conversion) for x in key]) if keyType is dict: diff --git a/Common/Python/PandasConverter.cs b/Common/Python/PandasConverter.cs index d9746e1f7eef..e27627371eae 100644 --- a/Common/Python/PandasConverter.cs +++ b/Common/Python/PandasConverter.cs @@ -33,19 +33,16 @@ public class PandasConverter private static PyObject _concat; /// - /// Creates an instance of . + /// Initializes the class /// - public PandasConverter() + static PandasConverter() { - if (_pandas == null) + using (Py.GIL()) { - using (Py.GIL()) - { - var pandas = Py.Import("pandas"); - _pandas = pandas; - // keep it so we don't need to ask for it each time - _concat = pandas.GetAttr("concat"); - } + var pandas = Py.Import("pandas"); + _pandas = pandas; + // keep it so we don't need to ask for it each time + _concat = pandas.GetAttr("concat"); } } @@ -248,6 +245,8 @@ public static PyObject ConcatDataFrames(IEnumerable dataFrames, IEnume result.GetAttr("dropna").Invoke(Array.Empty(), dropnaKwargs); } + PandasData.SetUpIndex(result); + pyKeys?.Dispose(); pyNames?.Dispose(); @@ -282,7 +281,9 @@ private void AddSeriesToPyDict(string key, List points, PyDi /// containing a pandas.DataFrame private PyObject MakeIndicatorDataFrame(PyDict pyDict) { - return _pandas.DataFrame(pyDict, columns: pyDict.Keys().Select(x => x.As().ToLowerInvariant()).OrderBy(x => x)); + var result = _pandas.DataFrame(pyDict, columns: pyDict.Keys().Select(x => x.As().ToLowerInvariant()).OrderBy(x => x)); + PandasData.SetUpIndex(result); + return result; } /// diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index dc247d24f1f5..79e90e7b6a63 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -93,6 +93,10 @@ public class PandasData private static PyObject _seriesFactory; private static PyObject _dataFrameFactory; private static PyObject _multiIndexFactory; + private static PyObject _multiIndex; + private static PyObject _isinstance; + private static PyObject _pyTrue; + private static PyObject _pyFalse; private static PyList _defaultNames; private static PyList _level1Names; @@ -126,32 +130,40 @@ public class PandasData public int Levels { get; } = 2; /// - /// Initializes an instance of + /// Initializes the static members of the class /// - public PandasData(object data) + static PandasData() { - if (_pandas == null) - { - using (Py.GIL()) - { - // Use our PandasMapper class that modifies pandas indexing to support tickers, symbols and SIDs - _pandas = Py.Import("PandasMapper"); - _seriesFactory = _pandas.GetAttr("Series"); - _dataFrameFactory = _pandas.GetAttr("DataFrame"); - using var multiIndex = _pandas.GetAttr("MultiIndex"); - _multiIndexFactory = multiIndex.GetAttr("from_tuples"); - _empty = new PyString(string.Empty); - - var time = new PyString("time"); - var symbol = new PyString("symbol"); - var expiry = new PyString("expiry"); - _defaultNames = new PyList(new PyObject[] { expiry, new PyString("strike"), new PyString("type"), symbol, time }); - _level1Names = new PyList(new PyObject[] { symbol }); - _level2Names = new PyList(new PyObject[] { symbol, time }); - _level3Names = new PyList(new PyObject[] { expiry, symbol, time }); - } + using (Py.GIL()) + { + // Use our PandasMapper class that modifies pandas indexing to support tickers, symbols and SIDs + _pandas = Py.Import("PandasMapper"); + _seriesFactory = _pandas.GetAttr("Series"); + _dataFrameFactory = _pandas.GetAttr("DataFrame"); + _multiIndex = _pandas.GetAttr("MultiIndex"); + _multiIndexFactory = _multiIndex.GetAttr("from_tuples"); + _empty = new PyString(string.Empty); + + using var builtins = Py.Import("builtins"); + _isinstance = builtins.GetAttr("isinstance"); + _pyTrue = builtins.GetAttr("True"); + _pyFalse = builtins.GetAttr("False"); + + var time = new PyString("time"); + var symbol = new PyString("symbol"); + var expiry = new PyString("expiry"); + _defaultNames = new PyList(new PyObject[] { expiry, new PyString("strike"), new PyString("type"), symbol, time }); + _level1Names = new PyList(new PyObject[] { symbol }); + _level2Names = new PyList(new PyObject[] { symbol, time }); + _level3Names = new PyList(new PyObject[] { expiry, symbol, time }); } + } + /// + /// Initializes an instance of + /// + public PandasData(object data) + { var baseData = data as IBaseData; // in the case we get a list/collection of data we take the first data point to determine the type @@ -507,6 +519,7 @@ public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns // Create the DataFrame var result = _dataFrameFactory.Invoke(pyDict); + SetUpIndex(result); foreach (var item in pyDict) { @@ -516,6 +529,26 @@ public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns return result; } + /// + /// Adds an internal "__has_symbols__" attribute to the columns and index of the DataFrame + /// to be used as metadata to determine if the DataFrame has symbols in the columns or index, + /// so that we map string keys to symbols when needed for data frame indexing. + /// + /// + internal static void SetUpIndex(PyObject result) + { + result.GetAttr("columns").SetAttr("__has_symbols__", _pyFalse); + var dfIndex = result.GetAttr("index"); + dfIndex.SetAttr("__has_symbols__", _pyTrue); + if (_isinstance.Invoke(dfIndex, _multiIndex).IsTrue()) + { + foreach (PyObject level in dfIndex.GetAttr("levels").GetIterator()) + { + level.SetAttr("__has_symbols__", _pyTrue); + } + } + } + /// /// Only dipose of the PyObject if it was set to something different than empty /// diff --git a/Common/Symbol.cs b/Common/Symbol.cs index 07a311ce33d6..317a037c64fe 100644 --- a/Common/Symbol.cs +++ b/Common/Symbol.cs @@ -620,20 +620,15 @@ public override bool Equals(object obj) if (ReferenceEquals(this, obj)) return true; // compare strings just as you would a symbol object - var sidString = obj as string; - if (sidString != null) + if (obj is string stringSymbol) { - SecurityIdentifier sid; - if (SecurityIdentifier.TryParse(sidString, out sid)) - { - return ID.Equals(sid); - } + return Equals((Symbol)stringSymbol); } // compare a sid just as you would a symbol object - if (obj is SecurityIdentifier) + if (obj is SecurityIdentifier sid) { - return ID.Equals((SecurityIdentifier) obj); + return ID.Equals(sid); } if (obj.GetType() != GetType()) return false; @@ -756,14 +751,7 @@ public bool Equals(Symbol other) return true; } - var rightSymbol = right as Symbol; - // Use the implicit conversion for strings - if (rightSymbol == null && right is string rightStr) - { - return left == (Symbol)rightStr; - } - - return left == rightSymbol; + return left.Equals(right); } /// @@ -776,8 +764,21 @@ public bool Equals(Symbol other) /// as an object instead of using the implicit conversion public static bool operator ==(object left, Symbol right) { + if (ReferenceEquals(left, right)) + { + // this is a performance shortcut + return true; + } + + var leftSymbol = left as Symbol; + // Use the implicit conversion for strings + if (leftSymbol == null && left is string leftStr) + { + return leftStr == (string)right; + } + // We already have an implementation for (Symbol left, object left), we can reuse it by inverting the operands - return right == left; + return leftSymbol == right; } /// @@ -842,12 +843,6 @@ public static implicit operator Symbol(string ticker) return symbol; } - SecurityIdentifier sid; - if (SecurityIdentifier.TryParse(ticker, out sid)) - { - return new Symbol(sid, sid.Symbol); - } - return new Symbol(new SecurityIdentifier(ticker, 0), ticker); } diff --git a/Tests/Python/PandasConverterTests.cs b/Tests/Python/PandasConverterTests.cs index af99d8369200..ca8808f5749b 100644 --- a/Tests/Python/PandasConverterTests.cs +++ b/Tests/Python/PandasConverterTests.cs @@ -385,8 +385,9 @@ def Test2(dataFrame): # Bad accessor, expected to throw data = dataFrame.LOW def Test3(dataFrame): - # Bad key, expected to throw + # Should not throw, access all LOW ticker data data = dataFrame.loc['low'] + def Test4(dataFrame): # Should not throw, access data column low for all tickers data = dataFrame.low @@ -399,9 +400,8 @@ def Test4(dataFrame): Assert.DoesNotThrow(() => test1(dataFrame)); Assert.Throws(() => test2(dataFrame)); - Assert.Throws(() => test3(dataFrame)); + Assert.DoesNotThrow(() => test3(dataFrame)); Assert.DoesNotThrow(() => test4(dataFrame)); - } } diff --git a/Tests/Python/PandasIndexingTests.cs b/Tests/Python/PandasIndexingTests.cs index 425a01f3b948..6bc3690c2d02 100644 --- a/Tests/Python/PandasIndexingTests.cs +++ b/Tests/Python/PandasIndexingTests.cs @@ -82,7 +82,7 @@ public void ExpectedException() PyObject result = _pandasDataFrameTests.test_expected_exception(); var exception = result.As(); - Assert.IsTrue(exception.Contains("No key found for either mapped or original key. Mapped Key: ['AAPL']; Original Key: ['aapl']", StringComparison.InvariantCulture)); + Assert.IsTrue(exception.Contains("No key found for either mapped or original key. Mapped Key: ['aapl']; Original Key: ['aapl']", StringComparison.InvariantCulture), exception); } } } From 8c21d2a890c9458c7bb816ffda3465f726da5b5b Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 1 Oct 2024 18:06:13 -0400 Subject: [PATCH 29/38] Cleanup --- Common/PandasMapper.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index 94b30a8a0049..c7a429ff453a 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -36,9 +36,9 @@ def mapper(key, force_symbol_conversion = False): if kvp[0]: return kvp[1] else: - reserved = ['high', 'low', 'open', 'close'] + reserved = ['high', 'low', 'open', 'close'] if key in reserved: - return key + return key if keyType is list: return [mapper(x, force_symbol_conversion) for x in key] @@ -46,8 +46,7 @@ def mapper(key, force_symbol_conversion = False): # If 'self' (the first arg) is an index and it contains symbols, we try to convert string keys into symbols if not force_symbol_conversion: self_value = key[0] - self_type = type(self_value) - if self_type is pd.MultiIndex or self_type is pd.Index: + if type(self_value) is pd.Index: # We add the __has_symbols__ attribute to the index to avoid checking for symbols in the future if hasattr(self_value, '__has_symbols__'): if getattr(self_value, '__has_symbols__'): @@ -55,16 +54,9 @@ def mapper(key, force_symbol_conversion = False): # Check whether the index contains symbols, if it does we add the __has_symbols__ attribute and force conversion else: has_symbols = False - for index_row in self_value: - values = index_row - if type(values) is not tuple: - values = [values] - - for x in values: - if type(x) is Symbol: - has_symbols = True - break - if has_symbols: + for index_value in self_value: + if type(index_value) is Symbol: + has_symbols = True break force_symbol_conversion = has_symbols From fa129c097699d44f0b3f3e30e5df1f121da404b5 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 1 Oct 2024 18:38:21 -0400 Subject: [PATCH 30/38] Cleanup --- Common/PandasMapper.py | 6 ++---- Common/Python/PandasConverter.cs | 12 ++++++------ Common/Python/PandasData.cs | 10 ++++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index c7a429ff453a..186eb13e9d09 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -53,14 +53,12 @@ def mapper(key, force_symbol_conversion = False): force_symbol_conversion = True # Check whether the index contains symbols, if it does we add the __has_symbols__ attribute and force conversion else: - has_symbols = False for index_value in self_value: if type(index_value) is Symbol: - has_symbols = True + force_symbol_conversion = True break - force_symbol_conversion = has_symbols - setattr(self_value, '__has_symbols__', has_symbols) + setattr(self_value, '__has_symbols__', force_symbol_conversion) return tuple([mapper(x, force_symbol_conversion) for x in key]) diff --git a/Common/Python/PandasConverter.cs b/Common/Python/PandasConverter.cs index e27627371eae..e9b4c840f438 100644 --- a/Common/Python/PandasConverter.cs +++ b/Common/Python/PandasConverter.cs @@ -215,14 +215,14 @@ private static PyObject CreateDataFrame(Dictionary dataFrames, IEnumerable keys = null, IEnumerable names = null, bool sort = true, bool dropna = true) { - var dataFramesList = dataFrames.ToList(); - if (dataFramesList.Count == 0) - { - return _pandas.DataFrame(); - } - using (Py.GIL()) { + var dataFramesList = dataFrames.ToList(); + if (dataFramesList.Count == 0) + { + return _pandas.DataFrame(); + } + using var pyDataFrames = dataFramesList.ToPyListUnSafe(); using var kwargs = Py.kw("sort", sort); PyList pyKeys = null; diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 79e90e7b6a63..7c70cbb92076 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -537,14 +537,16 @@ public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns /// internal static void SetUpIndex(PyObject result) { - result.GetAttr("columns").SetAttr("__has_symbols__", _pyFalse); - var dfIndex = result.GetAttr("index"); + using var dfIndex = result.GetAttr("index"); dfIndex.SetAttr("__has_symbols__", _pyTrue); - if (_isinstance.Invoke(dfIndex, _multiIndex).IsTrue()) + if (_isinstance.Invoke(dfIndex, _multiIndex).GetAndDispose()) { - foreach (PyObject level in dfIndex.GetAttr("levels").GetIterator()) + using var levels = dfIndex.GetAttr("levels"); + using var levelsIterator = levels.GetIterator(); + foreach (PyObject level in levelsIterator) { level.SetAttr("__has_symbols__", _pyTrue); + level.Dispose(); } } } From 257474617d11680a6050c9545dd3dbaf5841f2e4 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 3 Oct 2024 13:34:32 -0400 Subject: [PATCH 31/38] Add PandasColumn python class to handle proper indexing This allows propery hash and equality between Symbols, C# strings and Python strings --- ...iverseSelectionModelRegressionAlgorithm.py | 6 +- Algorithm/QCAlgorithm.Python.cs | 5 +- Common/PandasMapper.py | 94 ++++++++++++------- Common/Python/PandasConverter.cs | 6 +- Common/Python/PandasData.cs | 37 +------- Common/Symbol.cs | 19 ++-- Tests/Python/PandasConverterTests.cs | 3 +- Tests/Python/PandasIndexingTests.cs | 2 +- 8 files changed, 90 insertions(+), 82 deletions(-) diff --git a/Algorithm.Python/OptionChainedUniverseSelectionModelRegressionAlgorithm.py b/Algorithm.Python/OptionChainedUniverseSelectionModelRegressionAlgorithm.py index 0a03690afe09..e800f1fa47e6 100644 --- a/Algorithm.Python/OptionChainedUniverseSelectionModelRegressionAlgorithm.py +++ b/Algorithm.Python/OptionChainedUniverseSelectionModelRegressionAlgorithm.py @@ -23,7 +23,7 @@ def initialize(self): self.set_start_date(2014, 6, 6) self.set_end_date(2014, 6, 6) self.set_cash(100000) - + universe = self.add_universe("my-minute-universe-name", lambda time: [ "AAPL", "TWX" ]) self.add_universe_selection( OptionChainedUniverseSelectionModel( @@ -34,9 +34,9 @@ def initialize(self): .expiration(0, 180)) ) ) - + def on_data(self, slice): - if self.portfolio.invested or not (self.is_market_open("AAPL") and self.is_market_open("AAPL")): return + if self.portfolio.invested or not (self.is_market_open("AAPL") and self.is_market_open("TWX")): return values = list(map(lambda x: x.value, filter(lambda x: x.key == "?AAPL" or x.key == "?TWX", slice.option_chains))) for chain in values: # we sort the contracts to find at the money (ATM) contract with farthest expiration diff --git a/Algorithm/QCAlgorithm.Python.cs b/Algorithm/QCAlgorithm.Python.cs index 607f42ad8f0f..4e59351ea91d 100644 --- a/Algorithm/QCAlgorithm.Python.cs +++ b/Algorithm/QCAlgorithm.Python.cs @@ -1767,8 +1767,9 @@ private PyObject TryCleanupCollectionDataFrame(Type dataType, PyObject history) { if (!dynamic.empty) { - using PyObject columns = dynamic.columns; - if (columns.As().Contains("data")) + using var columns = new PySequence(dynamic.columns); + using var dataKey = "data".ToPython(); + if (columns.Contains(dataKey)) { history = dynamic["data"]; } diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index 186eb13e9d09..5a0c6ade99b9 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -25,45 +25,74 @@ AddReference("QuantConnect.Common") from QuantConnect import * -def mapper(key, force_symbol_conversion = False): +class PandasColumn(): + ''' + PandasColumn is a wrapper class for a pandas column that allows for the column to be used as a key + and properly compared to strings, regardless of whether it's a C# or Python string + (since the hash of a C# string and the same Python string are different). + ''' + + def __init__(self, key): + self.key = key + + def __str__(self): + return self.key + + def __repr__(self): + return self.key + + def __eq__(self, other): + # We need this since Lean created data frames might contain Symbol objects in the indexes + if isinstance(other, Symbol): + return False + + if isinstance(other, PandasColumn): + return self.key == other.key + + if isinstance(other, str): + return self.key == other + + return False + + def __hash__(self): + return hash(self.key) + + def __lt__(self, other): + if isinstance(other, str): + return self.key < other + return self.key < other.key + + def __le__(self, other): + if isinstance(other, str): + return self.key <= other + return self.key <= other.key + + def __gt__(self, other): + if isinstance(other, str): + return self.key > other + return self.key > other.key + + def __ge__(self, other): + if isinstance(other, str): + return self.key >= other + return self.key >= other.key + +def mapper(key): '''Maps a Symbol object or a Symbol Ticker (string) to the string representation of Symbol SecurityIdentifier.If cannot map, returns the object ''' keyType = type(key) if keyType is str: - if force_symbol_conversion: - kvp = SymbolCache.try_get_symbol(key, None) - if kvp[0]: - return kvp[1] - else: - reserved = ['high', 'low', 'open', 'close'] - if key in reserved: - return key - + kvp = SymbolCache.try_get_symbol(key, None) + if kvp[0]: + return kvp[1] + return key if keyType is list: - return [mapper(x, force_symbol_conversion) for x in key] + return [mapper(x) for x in key] if keyType is tuple: - # If 'self' (the first arg) is an index and it contains symbols, we try to convert string keys into symbols - if not force_symbol_conversion: - self_value = key[0] - if type(self_value) is pd.Index: - # We add the __has_symbols__ attribute to the index to avoid checking for symbols in the future - if hasattr(self_value, '__has_symbols__'): - if getattr(self_value, '__has_symbols__'): - force_symbol_conversion = True - # Check whether the index contains symbols, if it does we add the __has_symbols__ attribute and force conversion - else: - for index_value in self_value: - if type(index_value) is Symbol: - force_symbol_conversion = True - break - - setattr(self_value, '__has_symbols__', force_symbol_conversion) - - - return tuple([mapper(x, force_symbol_conversion) for x in key]) + return tuple([mapper(x) for x in key]) if keyType is dict: - return { k: mapper(v, force_symbol_conversion) for k, v in key.items()} + return { k: mapper(v) for k, v in key.items()} return key def wrap_keyerror_function(f): @@ -84,13 +113,14 @@ def wrapped_function(*args, **kwargs): return f(*newargs, **newkwargs) except KeyError as e: - mKey = [str(arg) for arg in newargs if isinstance(arg, str) or isinstance(arg, Symbol)] + pass # Execute original # Allows for df, Series, etc indexing for keys like 'SPY' if they exist try: return f(*args, **kwargs) except KeyError as e: + mKey = [str(arg) for arg in newargs if isinstance(arg, str) or isinstance(arg, Symbol)] oKey = [str(arg) for arg in args if isinstance(arg, str) or isinstance(arg, Symbol)] raise KeyError(f"No key found for either mapped or original key. Mapped Key: {mKey}; Original Key: {oKey}") diff --git a/Common/Python/PandasConverter.cs b/Common/Python/PandasConverter.cs index e9b4c840f438..a32011e92ab6 100644 --- a/Common/Python/PandasConverter.cs +++ b/Common/Python/PandasConverter.cs @@ -245,8 +245,6 @@ public static PyObject ConcatDataFrames(IEnumerable dataFrames, IEnume result.GetAttr("dropna").Invoke(Array.Empty(), dropnaKwargs); } - PandasData.SetUpIndex(result); - pyKeys?.Dispose(); pyNames?.Dispose(); @@ -281,9 +279,7 @@ private void AddSeriesToPyDict(string key, List points, PyDi /// containing a pandas.DataFrame private PyObject MakeIndicatorDataFrame(PyDict pyDict) { - var result = _pandas.DataFrame(pyDict, columns: pyDict.Keys().Select(x => x.As().ToLowerInvariant()).OrderBy(x => x)); - PandasData.SetUpIndex(result); - return result; + return _pandas.DataFrame(pyDict, columns: pyDict.Keys().Select(x => x.As().ToLowerInvariant()).OrderBy(x => x)); } /// diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 7c70cbb92076..1446c49c2de4 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -90,13 +90,11 @@ public class PandasData // we keep these so we don't need to ask for them each time private static PyString _empty; private static PyObject _pandas; + private static PyObject _pandasColumn; private static PyObject _seriesFactory; private static PyObject _dataFrameFactory; private static PyObject _multiIndexFactory; private static PyObject _multiIndex; - private static PyObject _isinstance; - private static PyObject _pyTrue; - private static PyObject _pyFalse; private static PyList _defaultNames; private static PyList _level1Names; @@ -138,17 +136,13 @@ static PandasData() { // Use our PandasMapper class that modifies pandas indexing to support tickers, symbols and SIDs _pandas = Py.Import("PandasMapper"); + _pandasColumn = _pandas.GetAttr("PandasColumn"); _seriesFactory = _pandas.GetAttr("Series"); _dataFrameFactory = _pandas.GetAttr("DataFrame"); _multiIndex = _pandas.GetAttr("MultiIndex"); _multiIndexFactory = _multiIndex.GetAttr("from_tuples"); _empty = new PyString(string.Empty); - using var builtins = Py.Import("builtins"); - _isinstance = builtins.GetAttr("isinstance"); - _pyTrue = builtins.GetAttr("True"); - _pyFalse = builtins.GetAttr("False"); - var time = new PyString("time"); var symbol = new PyString("symbol"); var expiry = new PyString("expiry"); @@ -504,7 +498,9 @@ public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns pyvalues.Append(pyObject); } using var series = _seriesFactory.Invoke(pyvalues, index); - pyDict.SetItem(kvp.Key, series); + using var pyStrKey = kvp.Key.ToPython(); + using var pyKey = _pandasColumn.Invoke(pyStrKey); + pyDict.SetItem(pyKey, series); } _series.Clear(); foreach (var kvp in indexCache) @@ -519,7 +515,6 @@ public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns // Create the DataFrame var result = _dataFrameFactory.Invoke(pyDict); - SetUpIndex(result); foreach (var item in pyDict) { @@ -529,28 +524,6 @@ public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns return result; } - /// - /// Adds an internal "__has_symbols__" attribute to the columns and index of the DataFrame - /// to be used as metadata to determine if the DataFrame has symbols in the columns or index, - /// so that we map string keys to symbols when needed for data frame indexing. - /// - /// - internal static void SetUpIndex(PyObject result) - { - using var dfIndex = result.GetAttr("index"); - dfIndex.SetAttr("__has_symbols__", _pyTrue); - if (_isinstance.Invoke(dfIndex, _multiIndex).GetAndDispose()) - { - using var levels = dfIndex.GetAttr("levels"); - using var levelsIterator = levels.GetIterator(); - foreach (PyObject level in levelsIterator) - { - level.SetAttr("__has_symbols__", _pyTrue); - level.Dispose(); - } - } - } - /// /// Only dipose of the PyObject if it was set to something different than empty /// diff --git a/Common/Symbol.cs b/Common/Symbol.cs index 317a037c64fe..8b60cd44ac7b 100644 --- a/Common/Symbol.cs +++ b/Common/Symbol.cs @@ -770,15 +770,22 @@ public bool Equals(Symbol other) return true; } - var leftSymbol = left as Symbol; - // Use the implicit conversion for strings - if (leftSymbol == null && left is string leftStr) + if (left is null) { - return leftStr == (string)right; + return false; } - // We already have an implementation for (Symbol left, object left), we can reuse it by inverting the operands - return leftSymbol == right; + if (left is Symbol leftSymbol) + { + return leftSymbol.Equals(right); + } + + if (left is string leftStr) + { + return leftStr.Equals(right?.ToString(), StringComparison.InvariantCulture); + } + + return false; } /// diff --git a/Tests/Python/PandasConverterTests.cs b/Tests/Python/PandasConverterTests.cs index ca8808f5749b..3a4e30b6e4bb 100644 --- a/Tests/Python/PandasConverterTests.cs +++ b/Tests/Python/PandasConverterTests.cs @@ -387,7 +387,6 @@ def Test2(dataFrame): def Test3(dataFrame): # Should not throw, access all LOW ticker data data = dataFrame.loc['low'] - def Test4(dataFrame): # Should not throw, access data column low for all tickers data = dataFrame.low @@ -1297,6 +1296,8 @@ def Test(dataFrame, symbol): [TestCase("['SPY','AAPL']", true)] [TestCase("symbols")] [TestCase("[str(symbols[0].ID), str(symbols[1].ID)]")] + [TestCase("('SPY','AAPL')", true)] + [TestCase("(str(symbols[0].ID), str(symbols[1].ID))")] public void BackwardsCompatibilityDataFrame_loc_list(string index, bool cache = false) { if (cache) diff --git a/Tests/Python/PandasIndexingTests.cs b/Tests/Python/PandasIndexingTests.cs index 6bc3690c2d02..c57999fbc8b2 100644 --- a/Tests/Python/PandasIndexingTests.cs +++ b/Tests/Python/PandasIndexingTests.cs @@ -82,7 +82,7 @@ public void ExpectedException() PyObject result = _pandasDataFrameTests.test_expected_exception(); var exception = result.As(); - Assert.IsTrue(exception.Contains("No key found for either mapped or original key. Mapped Key: ['aapl']; Original Key: ['aapl']", StringComparison.InvariantCulture), exception); + Assert.IsTrue(exception.Contains("No key found for either mapped or original key.", StringComparison.InvariantCulture), exception); } } } From 39917a5b7f61e5efe6945b345dbe807424ff327f Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 3 Oct 2024 16:08:25 -0400 Subject: [PATCH 32/38] Minor fixes --- Common/PandasMapper.py | 44 ++++------------------- Tests/Algorithm/AlgorithmHistoryTests.cs | 45 ++++++++++++++++-------- 2 files changed, 36 insertions(+), 53 deletions(-) diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index 5a0c6ade99b9..a3137b93129c 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -25,57 +25,25 @@ AddReference("QuantConnect.Common") from QuantConnect import * -class PandasColumn(): +class PandasColumn(str): ''' PandasColumn is a wrapper class for a pandas column that allows for the column to be used as a key and properly compared to strings, regardless of whether it's a C# or Python string (since the hash of a C# string and the same Python string are different). ''' - def __init__(self, key): - self.key = key - - def __str__(self): - return self.key - - def __repr__(self): - return self.key + def __new__(cls, key): + return super().__new__(cls, key) def __eq__(self, other): # We need this since Lean created data frames might contain Symbol objects in the indexes - if isinstance(other, Symbol): + if type(other) is Symbol: return False - if isinstance(other, PandasColumn): - return self.key == other.key - - if isinstance(other, str): - return self.key == other - - return False + return super().__eq__(other) def __hash__(self): - return hash(self.key) - - def __lt__(self, other): - if isinstance(other, str): - return self.key < other - return self.key < other.key - - def __le__(self, other): - if isinstance(other, str): - return self.key <= other - return self.key <= other.key - - def __gt__(self, other): - if isinstance(other, str): - return self.key > other - return self.key > other.key - - def __ge__(self, other): - if isinstance(other, str): - return self.key >= other - return self.key >= other.key + return super().__hash__() def mapper(key): '''Maps a Symbol object or a Symbol Ticker (string) to the string representation of diff --git a/Tests/Algorithm/AlgorithmHistoryTests.cs b/Tests/Algorithm/AlgorithmHistoryTests.cs index d8db3b042e38..73f809be91fa 100644 --- a/Tests/Algorithm/AlgorithmHistoryTests.cs +++ b/Tests/Algorithm/AlgorithmHistoryTests.cs @@ -214,15 +214,17 @@ def getTradesAndQuotesHistory(algorithm, symbol, start): def getTradesOnlyHistory(algorithm, symbol, start): return algorithm.History(Tick, symbol, start + timedelta(hours=9.8), start + timedelta(hours=10), Resolution.Tick).loc[symbol].to_dict() "); - var getTradesAndQuotesHistory = pythonModule.GetAttr("getTradesAndQuotesHistory"); - var getTradesOnlyHistory = pythonModule.GetAttr("getTradesOnlyHistory"); + using var getTradesAndQuotesHistory = pythonModule.GetAttr("getTradesAndQuotesHistory"); + using var getTradesOnlyHistory = pythonModule.GetAttr("getTradesOnlyHistory"); _algorithm.SetPandasConverter(); - var pySymbol = Symbols.SPY.ToPython(); - var pyAlgorithm = _algorithm.ToPython(); - var pyStart = start.ToPython(); + using var pySymbol = Symbols.SPY.ToPython(); + using var pyAlgorithm = _algorithm.ToPython(); + using var pyStart = start.ToPython(); - var result = getTradesAndQuotesHistory.Invoke(pyAlgorithm, pySymbol, pyStart).ConvertToDictionary(); - var result2 = getTradesOnlyHistory.Invoke(pyAlgorithm, pySymbol, pyStart).ConvertToDictionary(); + using var dict = getTradesAndQuotesHistory.Invoke(pyAlgorithm, pySymbol, pyStart); + var result = GetDataFrameDictionary(dict); + using var dict2 = getTradesOnlyHistory.Invoke(pyAlgorithm, pySymbol, pyStart); + var result2 = GetDataFrameDictionary(dict2); Assert.IsNotEmpty(result); Assert.IsNotEmpty(result2); @@ -820,10 +822,10 @@ def getOpenInterestHistory(algorithm, symbol, start, end, resolution): _algorithm.SetPandasConverter(); using var symbols = new PyList(new [] {optionSymbol.ToPython()}); - var openInterestsDataFrameDict = getOpenInterestHistory + using var dict = getOpenInterestHistory .Invoke(_algorithm.ToPython(), symbols, start.ToPython(), end.ToPython(), - historyResolution.ToPython()) - .ConvertToDictionary(); + historyResolution.ToPython()); + var openInterestsDataFrameDict = GetDataFrameDictionary(dict); Assert.That(openInterestsDataFrameDict, Does.ContainKey("openinterest")); Assert.That(openInterestsDataFrameDict, Does.ContainKey("time")); @@ -880,11 +882,12 @@ def getOpenInterestHistory(algorithm, symbol, start, end, resolution): _algorithm.SetPandasConverter(); using var symbols = new PyList(new [] {optionSymbol.ToPython()}); - var openInterests = getOpenInterestHistory.Invoke(_algorithm.ToPython(), symbols, start.ToPython(), end.ToPython(), + using var openInterests = getOpenInterestHistory.Invoke(_algorithm.ToPython(), symbols, start.ToPython(), end.ToPython(), historyResolution.ToPython()); Assert.AreEqual(780, openInterests.GetAttr("shape")[0].As()); - var dataFrameDict = openInterests.GetAttr("to_dict").Invoke().ConvertToDictionary(); + using var dict = openInterests.GetAttr("to_dict").Invoke(); + var dataFrameDict = GetDataFrameDictionary(dict); Assert.That(dataFrameDict, Does.Not.ContainKey("openinterest")); } } @@ -934,10 +937,9 @@ def getOpenInterestHistory(algorithm, symbol, start, end, resolution): .Invoke(_algorithm.ToPython(), symbols, start.ToPython(), end.ToPython(), historyResolution.ToPython()); Assert.AreEqual(1170, result.GetAttr("shape")[0].As()); - var dataFrameDict = result + var dataFrameDict = GetDataFrameDictionary(result .GetAttr("reset_index").Invoke() - .GetAttr("to_dict").Invoke() - .ConvertToDictionary(); + .GetAttr("to_dict").Invoke()); var dataFrameSymbols = dataFrameDict["symbol"].ConvertToDictionary().Values.ToHashSet(); CollectionAssert.AreEquivalent(dataFrameSymbols, new[] { optionSymbol.ID.ToString(), optionSymbol2.ID.ToString() }); @@ -3481,6 +3483,19 @@ private static List GetTimesFromHistoryDataFrameIndex(List i return index.Select(x => x[builtins.len(x) > 2 ? 2 : 1].As()).ToList(); } + private static Dictionary GetDataFrameDictionary(PyObject dict) + { + // Using PyObject because our data frames use our PandasColunm class to wrap strings + return dict.ConvertToDictionary().ToDictionary( + kvp => + { + var strKey = kvp.Key.ToString(); + kvp.Key.Dispose(); + return strKey; + }, + kvp => kvp.Value); + } + #region Fill-forwarded history assertions /// From 03d203f2ee3565f6642bb3d667d0b0950ed907af Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Thu, 3 Oct 2024 20:14:24 -0300 Subject: [PATCH 33/38] Symbol cache improvements --- Common/SymbolCache.cs | 147 ++++++++++++++++++++++-------------------- 1 file changed, 77 insertions(+), 70 deletions(-) diff --git a/Common/SymbolCache.cs b/Common/SymbolCache.cs index b00e4927ab78..6cf08d2ec823 100644 --- a/Common/SymbolCache.cs +++ b/Common/SymbolCache.cs @@ -15,8 +15,9 @@ */ using System; -using System.Collections.Concurrent; using System.Linq; +using System.Collections.Generic; +using System.Runtime.CompilerServices; namespace QuantConnect { @@ -28,7 +29,8 @@ namespace QuantConnect public static class SymbolCache { // we aggregate the two maps into a class so we can assign a new one as an atomic operation - private static Cache _cache = new Cache(); + private static readonly Dictionary Symbols = new(StringComparer.OrdinalIgnoreCase); + private static readonly Dictionary Tickers = new(); /// /// Adds a mapping for the specified ticker @@ -37,8 +39,11 @@ public static class SymbolCache /// The symbol object that maps to the string ticker symbol public static void Set(string ticker, Symbol symbol) { - _cache.Symbols[ticker] = symbol; - _cache.Tickers[symbol] = ticker; + lock (Symbols) + { + Symbols[ticker] = symbol; + Tickers[symbol] = ticker; + } } /// @@ -49,11 +54,10 @@ public static void Set(string ticker, Symbol symbol) public static Symbol GetSymbol(string ticker) { var result = TryGetSymbol(ticker); - if (result.Item3 != null) + if (!result.Item1) { - throw result.Item3; + throw result.Item3 ?? throw new InvalidOperationException(Messages.SymbolCache.UnableToLocateTicker(ticker)); } - return result.Item2; } @@ -66,14 +70,7 @@ public static Symbol GetSymbol(string ticker) public static bool TryGetSymbol(string ticker, out Symbol symbol) { var result = TryGetSymbol(ticker); - // ignore errors - if (result.Item1) - { - symbol = result.Item2; - return true; - } - - symbol = null; + symbol = result.Item2; return result.Item1; } @@ -84,8 +81,10 @@ public static bool TryGetSymbol(string ticker, out Symbol symbol) /// The string ticker symbol that maps to the specified symbol object public static string GetTicker(Symbol symbol) { - string ticker; - return _cache.Tickers.TryGetValue(symbol, out ticker) ? ticker : symbol.ID.ToString(); + lock (Symbols) + { + return Tickers.TryGetValue(symbol, out var ticker) ? ticker : symbol.ID.ToString(); + } } /// @@ -96,7 +95,10 @@ public static string GetTicker(Symbol symbol) /// The string ticker symbol that maps to the specified symbol object public static bool TryGetTicker(Symbol symbol, out string ticker) { - return _cache.Tickers.TryGetValue(symbol, out ticker); + lock (Symbols) + { + return Tickers.TryGetValue(symbol, out ticker); + } } /// @@ -106,8 +108,10 @@ public static bool TryGetTicker(Symbol symbol, out string ticker) /// True if the symbol mapping were removed from the cache public static bool TryRemove(Symbol symbol) { - string ticker; - return _cache.Tickers.TryRemove(symbol, out ticker) && _cache.Symbols.TryRemove(ticker, out symbol); + lock (Symbols) + { + return Tickers.Remove(symbol, out var ticker) && Symbols.Remove(ticker, out symbol); + } } /// @@ -117,76 +121,79 @@ public static bool TryRemove(Symbol symbol) /// True if the symbol mapping were removed from the cache public static bool TryRemove(string ticker) { - Symbol symbol; - return _cache.Symbols.TryRemove(ticker, out symbol) && _cache.Tickers.TryRemove(symbol, out ticker); + lock (Symbols) + { + return Symbols.Remove(ticker, out var symbol) && Tickers.Remove(symbol, out ticker); + } } /// /// Clears the current caches /// + /// Just used for testing public static void Clear() { - _cache = new Cache(); + lock (Symbols) + { + Symbols.Clear(); + Tickers.Clear(); + } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Tuple TryGetSymbol(string ticker) { - Symbol symbol; - InvalidOperationException error = null; - if (!_cache.TryGetSymbol(ticker, out symbol)) + lock (Symbols) { - // fall-back full-text search as a back-shim for custom data symbols. - // permitting a user to use BTC to resolve to BTC.Bitcoin - var search = $"{ticker.ToUpperInvariant()}."; - var match = _cache.Symbols.Where(kvp => kvp.Key.StartsWith(search)).ToList(); - - if (match.Count == 0) - { - // no matches - error = new InvalidOperationException(Messages.SymbolCache.UnableToLocateTicker(ticker)); - } - else if (match.Count == 1) - { - // exactly one match - symbol = match.Single().Value; - } - else if (match.Count > 1) + if (!TryGetSymbolCached(ticker, out var symbol)) { - // too many matches - error = new InvalidOperationException( - Messages.SymbolCache.MultipleMatchingTickersLocated(match.Select(kvp => kvp.Key))); + // fall-back full-text search as a back-shim for custom data symbols. + // permitting a user to use BTC to resolve to BTC.Bitcoin + var search = $"{ticker}."; + var match = Symbols.Where(kvp => kvp.Key.StartsWith(search, StringComparison.InvariantCultureIgnoreCase) && kvp.Value is not null).ToList(); + + if (match.Count == 0) + { + // no matches, cache the miss! else it will get expensive + Symbols[ticker] = null; + return new(false, null, null); + } + else if (match.Count == 1) + { + // exactly one match + return new(true, match.Single().Value, null); + } + else if (match.Count > 1) + { + // too many matches + return new(false, null, new InvalidOperationException( + Messages.SymbolCache.MultipleMatchingTickersLocated(match.Select(kvp => kvp.Key)))); + } } + return new(symbol is not null, symbol, null); } - - return Tuple.Create(symbol != null, symbol, error); } - class Cache + /// + /// Attempts to resolve the ticker to a Symbol via the cache. If not found in the + /// cache then + /// + /// The ticker to resolver to a symbol + /// The resolves symbol + /// True if we successfully resolved a symbol, false otherwise + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryGetSymbolCached(string ticker, out Symbol symbol) { - public readonly ConcurrentDictionary Symbols = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); - public readonly ConcurrentDictionary Tickers = new ConcurrentDictionary(); - - /// - /// Attempts to resolve the ticker to a Symbol via the cache. If not found in the - /// cache then - /// - /// The ticker to resolver to a symbol - /// The resolves symbol - /// True if we successfully resolved a symbol, false otherwise - public bool TryGetSymbol(string ticker, out Symbol symbol) + if (Symbols.TryGetValue(ticker, out symbol)) { - if (Symbols.TryGetValue(ticker, out symbol)) - { - return true; - } - SecurityIdentifier sid; - if (SecurityIdentifier.TryParse(ticker, out sid)) - { - symbol = new Symbol(sid, sid.Symbol); - return true; - } - return false; + return true; + } + if (SecurityIdentifier.TryParse(ticker, out var sid)) + { + symbol = new Symbol(sid, sid.Symbol); + return true; } + return false; } } } From aa49a1be20fc062b2e3a607a14b8b52e1c960bd9 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 4 Oct 2024 10:13:19 -0300 Subject: [PATCH 34/38] Minor fix for cache miss --- Common/SymbolCache.cs | 15 ++++++++++++++- Tests/Common/SymbolCacheTests.cs | 26 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Common/SymbolCache.cs b/Common/SymbolCache.cs index 6cf08d2ec823..1f174f2bd3e0 100644 --- a/Common/SymbolCache.cs +++ b/Common/SymbolCache.cs @@ -43,6 +43,16 @@ public static void Set(string ticker, Symbol symbol) { Symbols[ticker] = symbol; Tickers[symbol] = ticker; + + var index = ticker.IndexOf('.'); + if (index != -1) + { + var related = ticker.Substring(0, index); + if (Symbols.TryGetValue(related, out symbol) && symbol is null) + { + Symbols.Remove(related); + } + } } } @@ -106,6 +116,7 @@ public static bool TryGetTicker(Symbol symbol, out string ticker) /// /// The symbol whose mappings are to be removed /// True if the symbol mapping were removed from the cache + /// Just used for testing public static bool TryRemove(Symbol symbol) { lock (Symbols) @@ -119,6 +130,7 @@ public static bool TryRemove(Symbol symbol) /// /// The ticker whose mappings are to be removed /// True if the symbol mapping were removed from the cache + /// Just used for testing public static bool TryRemove(string ticker) { lock (Symbols) @@ -161,7 +173,8 @@ private static Tuple TryGetSymbol(strin else if (match.Count == 1) { // exactly one match - return new(true, match.Single().Value, null); + Symbols[ticker] = match[0].Value; + return new(true, match[0].Value, null); } else if (match.Count > 1) { diff --git a/Tests/Common/SymbolCacheTests.cs b/Tests/Common/SymbolCacheTests.cs index eb6a628743ba..6e9be9d1ed9a 100644 --- a/Tests/Common/SymbolCacheTests.cs +++ b/Tests/Common/SymbolCacheTests.cs @@ -163,6 +163,32 @@ public void TryGetSymbol_FromTicker_WithConflictingSymbolWithCustomDataSuffix() Assert.AreEqual(customSymbol, fetchedCustomSymbol); } + [Test] + public void TryGetSymbol_FromTicker_CacheMissDoesNotStick() + { + var ticker = "My-Ticker"; + Assert.IsFalse(SymbolCache.TryGetSymbol(ticker, out var _)); + + var customTicker = $"{ticker}.CustomDataTypeName"; + var customSymbol = Symbol.Create(customTicker, SecurityType.Base, Market.USA, baseDataType: typeof(Bitcoin)); + SymbolCache.Set(customTicker, customSymbol); + + Assert.IsTrue(SymbolCache.TryGetSymbol(ticker, out var fetchedSymbol)); + Assert.AreEqual(customSymbol, fetchedSymbol); + + Assert.IsTrue(SymbolCache.TryGetSymbol(customTicker, out var fetchedCustomSymbol)); + Assert.AreEqual(customSymbol, fetchedCustomSymbol); + + // now we set symbol + var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA); + SymbolCache.Set(ticker, symbol); + Assert.IsTrue(SymbolCache.TryGetSymbol(ticker, out var equitySymbol)); + Assert.AreEqual(symbol, equitySymbol); + + Assert.IsTrue(SymbolCache.TryGetSymbol(customTicker, out fetchedCustomSymbol)); + Assert.AreEqual(customSymbol, fetchedCustomSymbol); + } + [Test] public void TryGetSymbol_FromTicker_WithoutCustomDataSuffix() { From 6222d60b4a435cb022a2e643084e21a61616615e Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 4 Oct 2024 09:43:31 -0400 Subject: [PATCH 35/38] Revert PandasMapper reserved names and improvements --- Common/PandasMapper.py | 14 ++++++++++++++ Common/Symbol.cs | 11 +++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index a3137b93129c..1be4a38367af 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -45,12 +45,26 @@ def __eq__(self, other): def __hash__(self): return super().__hash__() +# Reserved names list as PandasColumn objects for performace reasons: +# Example case: +# for index, row in df.loc["SPY"].iterrows(): +# high = row["high"] +# ... +# 1. Multiple accesses to "high" column in the loop: this allows skipping symbol lookup for each access +# 2. We wrap the name in a PandasColumn object to allow for proper comparison with strings and Symbol. +# a. Avoids creating a new PandasColumn object for each access +reserved_names = [PandasColumn(name) for name in ["open", "high", "low", "close"]] + def mapper(key): '''Maps a Symbol object or a Symbol Ticker (string) to the string representation of Symbol SecurityIdentifier.If cannot map, returns the object ''' keyType = type(key) if keyType is str: + for reserved in reserved_names: + if key == reserved: + return reserved + kvp = SymbolCache.try_get_symbol(key, None) if kvp[0]: return kvp[1] diff --git a/Common/Symbol.cs b/Common/Symbol.cs index 8b60cd44ac7b..71c1a69f9da8 100644 --- a/Common/Symbol.cs +++ b/Common/Symbol.cs @@ -728,10 +728,12 @@ public bool Equals(Symbol other) // this is a performance shortcut return true; } - if (ReferenceEquals(left, null) || left.Equals(Empty)) + + if (left is null) { - return ReferenceEquals(right, null) || right.Equals(Empty); + return false; } + return left.Equals(right); } @@ -751,6 +753,11 @@ public bool Equals(Symbol other) return true; } + if (left is null) + { + return false; + } + return left.Equals(right); } From 97c27939bd0407811e6f762db9d2147b914f40c4 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 4 Oct 2024 09:47:53 -0400 Subject: [PATCH 36/38] Minor fix --- Tests/Python/PandasConverterTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Python/PandasConverterTests.cs b/Tests/Python/PandasConverterTests.cs index 3a4e30b6e4bb..ce07f8501f07 100644 --- a/Tests/Python/PandasConverterTests.cs +++ b/Tests/Python/PandasConverterTests.cs @@ -399,7 +399,7 @@ def Test4(dataFrame): Assert.DoesNotThrow(() => test1(dataFrame)); Assert.Throws(() => test2(dataFrame)); - Assert.DoesNotThrow(() => test3(dataFrame)); + Assert.Throws(() => test3(dataFrame)); Assert.DoesNotThrow(() => test4(dataFrame)); } } From d0f709c2cc29cff7a98a8c35e6f39166c9ab00fe Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 4 Oct 2024 10:52:18 -0400 Subject: [PATCH 37/38] Revert reserved names --- Common/PandasMapper.py | 25 ++++--------------------- Tests/Python/PandasConverterTests.cs | 2 +- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/Common/PandasMapper.py b/Common/PandasMapper.py index 1be4a38367af..c18bd1812a83 100644 --- a/Common/PandasMapper.py +++ b/Common/PandasMapper.py @@ -37,44 +37,27 @@ def __new__(cls, key): def __eq__(self, other): # We need this since Lean created data frames might contain Symbol objects in the indexes - if type(other) is Symbol: - return False - - return super().__eq__(other) + return super().__eq__(other) and type(other) is not Symbol def __hash__(self): return super().__hash__() -# Reserved names list as PandasColumn objects for performace reasons: -# Example case: -# for index, row in df.loc["SPY"].iterrows(): -# high = row["high"] -# ... -# 1. Multiple accesses to "high" column in the loop: this allows skipping symbol lookup for each access -# 2. We wrap the name in a PandasColumn object to allow for proper comparison with strings and Symbol. -# a. Avoids creating a new PandasColumn object for each access -reserved_names = [PandasColumn(name) for name in ["open", "high", "low", "close"]] - def mapper(key): '''Maps a Symbol object or a Symbol Ticker (string) to the string representation of Symbol SecurityIdentifier.If cannot map, returns the object ''' keyType = type(key) + if keyType is tuple: + return tuple(mapper(x) for x in key) if keyType is str: - for reserved in reserved_names: - if key == reserved: - return reserved - kvp = SymbolCache.try_get_symbol(key, None) if kvp[0]: return kvp[1] return key if keyType is list: return [mapper(x) for x in key] - if keyType is tuple: - return tuple([mapper(x) for x in key]) if keyType is dict: - return { k: mapper(v) for k, v in key.items()} + return {k: mapper(v) for k, v in key.items()} return key def wrap_keyerror_function(f): diff --git a/Tests/Python/PandasConverterTests.cs b/Tests/Python/PandasConverterTests.cs index ce07f8501f07..3a4e30b6e4bb 100644 --- a/Tests/Python/PandasConverterTests.cs +++ b/Tests/Python/PandasConverterTests.cs @@ -399,7 +399,7 @@ def Test4(dataFrame): Assert.DoesNotThrow(() => test1(dataFrame)); Assert.Throws(() => test2(dataFrame)); - Assert.Throws(() => test3(dataFrame)); + Assert.DoesNotThrow(() => test3(dataFrame)); Assert.DoesNotThrow(() => test4(dataFrame)); } } From 3fd171a334bbe2f046a9bddb5f238b345efa9a57 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 4 Oct 2024 11:55:36 -0400 Subject: [PATCH 38/38] Minor fix for Symbol equality operators --- Common/Symbol.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Common/Symbol.cs b/Common/Symbol.cs index 71c1a69f9da8..933e25566d84 100644 --- a/Common/Symbol.cs +++ b/Common/Symbol.cs @@ -731,7 +731,8 @@ public bool Equals(Symbol other) if (left is null) { - return false; + // Rely on the Equals method if possible + return right is null || right.Equals(left); } return left.Equals(right); @@ -755,7 +756,8 @@ public bool Equals(Symbol other) if (left is null) { - return false; + // Rely on the Equals method if possible + return right is null || right.Equals(left); } return left.Equals(right); @@ -779,7 +781,7 @@ public bool Equals(Symbol other) if (left is null) { - return false; + return right is null; } if (left is Symbol leftSymbol)