From a31e94d0837c82f46a405ff5e131c54b2dc5fa17 Mon Sep 17 00:00:00 2001 From: Chip Kent <5250374+chipkent@users.noreply.github.com> Date: Tue, 26 Jul 2022 14:52:37 -0600 Subject: [PATCH] Support the crypto in the ibapi 10.16.01 (#66) * Updated code to work with crypto data and added code for code review. * Updated git ignore * Updates to remove uneeded files * Removed example code * updated Dockerfile * Fixed .gitignore * Added crypto to the read only example * Updated install with extra pip installs Co-authored-by: joshuahilgartner <32405513+Firelement@users.noreply.github.com> --- .github/workflows/Dockerfile | 3 +- .gitignore | 3 +- README.md | 1 + docker/dev/Dockerfile | 2 +- docker/release/Dockerfile | 1 + examples/example_all_functionality.py | 36 +++++++++++++++++++++ examples/example_read_only_functionality.py | 36 +++++++++++++++++++++ src/deephaven_ib/__init__.py | 5 +-- src/deephaven_ib/_internal/tablewriter.py | 6 ++++ src/deephaven_ib/_tws/ib_type_logger.py | 9 +++--- src/deephaven_ib/_tws/tws_client.py | 23 +++++++------ 11 files changed, 105 insertions(+), 20 deletions(-) diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile index fb71cfddc..a007bff29 100644 --- a/.github/workflows/Dockerfile +++ b/.github/workflows/Dockerfile @@ -4,7 +4,8 @@ FROM ubuntu:22.04 ARG IB_VERSION RUN apt update && \ - apt install -y openjdk-11-jdk python3-pip curl zip + apt install -y openjdk-11-jdk python3-pip curl zip && \ + pip3 install --upgrade pip setuptools wheel ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/ diff --git a/.gitignore b/.gitignore index 72df4f165..73b047755 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ dist src/deephaven_ib.egg-info docker/data docker/*/data -docker/*/build \ No newline at end of file +docker/*/build + diff --git a/README.md b/README.md index afd7a8aea..e9f30478d 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,7 @@ new feature and has not been well tested. To do this: ``` 4) Install [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib): ```bash + pip3 install --upgrade pip setuptools wheel pip3 install deephaven-ib ``` 5) Install Java 11 and set the appropriate `JAVA_HOME` environment variable. diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 68cd4eeab..5e8474510 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -12,7 +12,7 @@ RUN apt update && \ apt install -y openjdk-11-jdk && \ ln -s /usr/lib/jvm/java-11-openjdk-*/ /usr/lib/jvm/java-11-openjdk && \ apt install --yes git python3-venv python3-pip && \ - pip3 install --upgrade setuptools wheel build twine + pip3 install --upgrade pip setuptools wheel build twine ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk diff --git a/docker/release/Dockerfile b/docker/release/Dockerfile index 0eeaf8764..5b9595002 100644 --- a/docker/release/Dockerfile +++ b/docker/release/Dockerfile @@ -7,6 +7,7 @@ FROM ubuntu:22.04 RUN apt update && \ apt install -y openjdk-11-jdk python3-pip && \ ln -s /usr/lib/jvm/java-11-openjdk-*/ /usr/lib/jvm/java-11-openjdk && \ + pip3 install --upgrade pip setuptools wheel ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk/ diff --git a/examples/example_all_functionality.py b/examples/example_all_functionality.py index 35a17f4ce..199ca82b5 100644 --- a/examples/example_all_functionality.py +++ b/examples/example_all_functionality.py @@ -331,6 +331,42 @@ def get_contracts() -> Dict[str, Contract]: client.request_bars_historical(rc, duration=dhib.Duration.days(22), bar_size=dhib.BarSize.DAY_1, bar_type=dhib.BarDataType.YIELD_LAST, keep_up_to_date=False) +print("==============================================================================================================") +print("==== Request bars (Crypto).") +print("==============================================================================================================") + +contract = Contract() +contract.symbol = "BTC" +contract.secType = "CRYPTO" +contract.exchange = "PAXOS" +contract.currency = "USD" + +rc = client.get_registered_contract(c2) +client.request_bars_historical(rc2, duration=dhib.Duration.days(253), + bar_size=dhib.BarSize.DAY_1, + bar_type=dhib.BarDataType.AGGTRADES, + keep_up_to_date = False +) +client.request_bars_realtime(rc2, bar_type=dhib.BarDataType.TRADES) + +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.MIDPOINT) +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.BID) +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.ASK) +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.BID_ASK, keep_up_to_date=False) +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.AGGTRADES) +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.ADJUSTED_LAST, keep_up_to_date=False) + +client.request_bars_realtime(rc, bar_type=dhib.BarDataType.MIDPOINT) +client.request_bars_realtime(rc, bar_type=dhib.BarDataType.BID) +client.request_bars_realtime(rc, bar_type=dhib.BarDataType.ASK) +client.request_bars_realtime(rc, bar_type=dhib.BarDataType.TRADES) + print("==============================================================================================================") print("==== Request tick data.") print("==============================================================================================================") diff --git a/examples/example_read_only_functionality.py b/examples/example_read_only_functionality.py index 41efc2cef..c9863a006 100644 --- a/examples/example_read_only_functionality.py +++ b/examples/example_read_only_functionality.py @@ -327,6 +327,42 @@ def get_contracts() -> Dict[str, Contract]: client.request_bars_historical(rc, duration=dhib.Duration.days(22), bar_size=dhib.BarSize.DAY_1, bar_type=dhib.BarDataType.YIELD_LAST, keep_up_to_date=False) +print("==============================================================================================================") +print("==== Request bars (Crypto).") +print("==============================================================================================================") + +contract = Contract() +contract.symbol = "BTC" +contract.secType = "CRYPTO" +contract.exchange = "PAXOS" +contract.currency = "USD" + +rc = client.get_registered_contract(c2) +client.request_bars_historical(rc2, duration=dhib.Duration.days(253), + bar_size=dhib.BarSize.DAY_1, + bar_type=dhib.BarDataType.AGGTRADES, + keep_up_to_date = False +) +client.request_bars_realtime(rc2, bar_type=dhib.BarDataType.TRADES) + +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.MIDPOINT) +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.BID) +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.ASK) +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.BID_ASK, keep_up_to_date=False) +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.AGGTRADES) +client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5, + bar_type=dhib.BarDataType.ADJUSTED_LAST, keep_up_to_date=False) + +client.request_bars_realtime(rc, bar_type=dhib.BarDataType.MIDPOINT) +client.request_bars_realtime(rc, bar_type=dhib.BarDataType.BID) +client.request_bars_realtime(rc, bar_type=dhib.BarDataType.ASK) +client.request_bars_realtime(rc, bar_type=dhib.BarDataType.TRADES) + print("==============================================================================================================") print("==== Request tick data.") print("==============================================================================================================") diff --git a/src/deephaven_ib/__init__.py b/src/deephaven_ib/__init__.py index b307d9928..86cb9b0a9 100644 --- a/src/deephaven_ib/__init__.py +++ b/src/deephaven_ib/__init__.py @@ -150,7 +150,8 @@ class BarDataType(Enum): """Bid/Ask yield.""" YIELD_LAST = 14 """Last yield.""" - + AGGTRADES = 15 + """Aggregate trade prices.""" class BarSize(Enum): """Bar data sizes.""" @@ -966,7 +967,7 @@ def request_bars_realtime(self, contract: RegisteredContract, bar_type: BarDataT self._assert_connected() requests = [] - if bar_type not in [BarDataType.TRADES, BarDataType.MIDPOINT, BarDataType.BID, BarDataType.ASK]: + if bar_type not in [BarDataType.TRADES, BarDataType.AGGTRADES, BarDataType.MIDPOINT, BarDataType.BID, BarDataType.ASK]: raise Exception(f"Unsupported bar type: {bar_type}") for cd in contract.contract_details: diff --git a/src/deephaven_ib/_internal/tablewriter.py b/src/deephaven_ib/_internal/tablewriter.py index 91ccb01fb..e80ea0c26 100644 --- a/src/deephaven_ib/_internal/tablewriter.py +++ b/src/deephaven_ib/_internal/tablewriter.py @@ -10,6 +10,8 @@ from deephaven.table import Table from deephaven import dtypes from deephaven.dtypes import DType +import decimal +from decimal import Decimal from .trace import trace_str @@ -67,6 +69,10 @@ def write_row(self, values: List) -> None: if self._receive_time: values.insert(0, now()) + for i in range(len(values)): + if isinstance(values[i], decimal.Decimal): + values[i] = float(values[i]) + self._check_logged_value_types(values) for i in self._string_indices: diff --git a/src/deephaven_ib/_tws/ib_type_logger.py b/src/deephaven_ib/_tws/ib_type_logger.py index 5f74c02a0..40d248921 100644 --- a/src/deephaven_ib/_tws/ib_type_logger.py +++ b/src/deephaven_ib/_tws/ib_type_logger.py @@ -141,7 +141,6 @@ def map_sec_id_list(value): ("LiquidHours", dtypes.StringSet, lambda cd: to_string_set(cd.liquidHours.split(";"))), ("EvRule", dtypes.string, lambda cd: cd.evRule), ("EvMultiplier", dtypes.int64, lambda cd: cd.evMultiplier), - ("MdSizeMultiplier", dtypes.int64, lambda cd: cd.mdSizeMultiplier), ("AggGroup", dtypes.int64, lambda cd: map_null_int(cd.aggGroup)), ("UnderSymbol", dtypes.string, lambda cd: cd.underSymbol), ("UnderSecType", dtypes.string, lambda cd: cd.underSecType), @@ -218,9 +217,9 @@ def parse_timestamp(bd): ("High", dtypes.float64, lambda bd: bd.high), ("Low", dtypes.float64, lambda bd: bd.low), ("Close", dtypes.float64, lambda bd: bd.close), - ("Volume", dtypes.int64, lambda bd: map_null(bd.volume)), + ("Volume", dtypes.float64, lambda bd: map_null(bd.volume)), ("BarCount", dtypes.int64, lambda bd: map_null(bd.barCount)), - ("Average", dtypes.float64, lambda bd: map_null(bd.average)), + ("WAP", dtypes.float64, lambda bd: map_null(bd.wap)), ] @@ -245,7 +244,7 @@ def map_null(val): ("High", dtypes.float64, lambda bd: bd.high), ("Low", dtypes.float64, lambda bd: bd.low), ("Close", dtypes.float64, lambda bd: bd.close), - ("Volume", dtypes.int64, lambda bd: map_null(bd.volume)), + ("Volume", dtypes.float64, lambda bd: map_null(bd.volume)), ("WAP", dtypes.float64, lambda bd: map_null(bd.wap)), ("Count", dtypes.int64, lambda bd: map_null(bd.count)), ] @@ -330,7 +329,7 @@ def map_special_conditions(special_conditions: str) -> Any: return [ ("Timestamp", dtypes.DateTime, lambda t: unix_sec_to_dh_datetime(t.time)), ("Price", dtypes.float64, lambda t: t.price), - ("Size", dtypes.int64, lambda t: t.size), + ("Size", dtypes.float64, lambda t: t.size), *_include_details(_details_tick_attrib_last(), lambda t: t.tickAttribLast), ("Exchange", dtypes.string, lambda t: t.exchange), ("SpecialConditions", dtypes.StringSet, lambda t: map_special_conditions(t.specialConditions)) diff --git a/src/deephaven_ib/_tws/tws_client.py b/src/deephaven_ib/_tws/tws_client.py index 562bf5298..3d439230e 100644 --- a/src/deephaven_ib/_tws/tws_client.py +++ b/src/deephaven_ib/_tws/tws_client.py @@ -11,6 +11,9 @@ from threading import Thread from typing import Set +import decimal +from decimal import Decimal + from deephaven.table import Table from deephaven import dtypes @@ -208,7 +211,7 @@ def _build_table_writers() -> Dict[str, TableWriter]: table_writers["ticks_size"] = TableWriter( ["RequestId", "TickType", "Size"], - [dtypes.int64, dtypes.string, dtypes.int64]) + [dtypes.int64, dtypes.string, dtypes.float64]) table_writers["ticks_string"] = TableWriter( ["RequestId", "TickType", "Value"], @@ -404,8 +407,8 @@ def log_request(self, req_id: int, request_type: str, contract: Union[Contract, # Always present #### - def error(self, reqId: TickerId, errorCode: int, errorString: str): - EWrapper.error(self, reqId, errorCode, errorString) + def error(self, reqId: TickerId, errorCode: int, errorString: str, advancedOrderRejectJson: str = ""): + EWrapper.error(self, reqId, errorCode, errorString, advancedOrderRejectJson) if reqId == 2147483647: reqId = None @@ -701,7 +704,7 @@ def accountSummary(self, reqId: int, account: str, tag: str, value: str, currenc # reqPositionsMulti #### - def positionMulti(self, reqId: int, account: str, modelCode: str, contract: Contract, pos: float, avgCost: float): + def positionMulti(self, reqId: int, account: str, modelCode: str, contract: Contract, pos: decimal.Decimal, avgCost: float): EWrapper.positionMulti(self, reqId, account, modelCode, contract, pos, avgCost) # The returned contract seems to be inconsistent with IB's API to request contract details. @@ -804,7 +807,7 @@ def tickPrice(self, reqId: TickerId, tickType: TickType, price: float, attrib: T self._table_writers["ticks_price"].write_row([reqId, TickTypeEnum.to_str(tickType), price, *logger_tick_attrib.vals(attrib)]) - def tickSize(self, reqId: TickerId, tickType: TickType, size: int): + def tickSize(self, reqId: TickerId, tickType: TickType, size: decimal.Decimal): EWrapper.tickSize(self, reqId, tickType, size) self._table_writers["ticks_size"].write_row([reqId, TickTypeEnum.to_str(tickType), size]) @@ -846,7 +849,7 @@ def tickSnapshotEnd(self, reqId: int): #### def tickByTickAllLast(self, reqId: int, tickType: int, timestamp: int, price: float, - size: int, tickAttribLast: TickAttribLast, exchange: str, + size: decimal.Decimal, tickAttribLast: TickAttribLast, exchange: str, specialConditions: str): EWrapper.tickByTickAllLast(self, reqId, tickType, timestamp, price, size, tickAttribLast, exchange, specialConditions) @@ -869,7 +872,7 @@ def historicalTicksLast(self, reqId: int, ticks: ListOfHistoricalTickLast, done: self._table_writers["ticks_trade"].write_row([reqId, *logger_hist_tick_last.vals(t)]) def tickByTickBidAsk(self, reqId: int, timestamp: int, bidPrice: float, askPrice: float, - bidSize: int, askSize: int, tickAttribBidAsk: TickAttribBidAsk): + bidSize: decimal.Decimal, askSize: decimal.Decimal, tickAttribBidAsk: TickAttribBidAsk): EWrapper.tickByTickBidAsk(self, reqId, timestamp, bidPrice, askPrice, bidSize, askSize, tickAttribBidAsk) t = HistoricalTickBidAsk() @@ -920,7 +923,7 @@ def reqRealTimeBars(self, reqId: TickerId, contract: Contract, barSize: int, EClient.reqRealTimeBars(self, reqId, contract, barSize, whatToShow, useRTH, realTimeBarsOptions) def realtimeBar(self, reqId: TickerId, timestamp: int, open_: float, high: float, low: float, close: float, - volume: int, wap: float, count: int): + volume: decimal.Decimal, wap: decimal.Decimal, count: int): EWrapper.realtimeBar(self, reqId, timestamp, open_, high, low, close, volume, wap, count) bar_size = self._realtime_bar_sizes[reqId] bar = RealTimeBar(time=timestamp, endTime=timestamp + bar_size, open_=open_, high=high, low=low, close=close, @@ -967,8 +970,8 @@ def openOrder(self, orderId: OrderId, contract: Contract, order: Order, orderSta [*logger_contract.vals(contract), *logger_order.vals(order), *logger_order_state.vals(orderState)]) self.contract_registry.request_contract_details_nonblocking(contract) - def orderStatus(self, orderId: OrderId, status: str, filled: float, - remaining: float, avgFillPrice: float, permId: int, + def orderStatus(self, orderId: OrderId, status: str, filled: decimal.Decimal, + remaining: decimal.Decimal, avgFillPrice: float, permId: int, parentId: int, lastFillPrice: float, clientId: int, whyHeld: str, mktCapPrice: float): EWrapper.orderStatus(self, orderId, status, filled, remaining, avgFillPrice, permId, parentId, lastFillPrice,