Support the crypto in the ibapi 10.16.01 (#66)

* Updated code to work with crypto data and added code for code review.

* Updated git ignore

* Updates to remove uneeded files

* Removed example code

* updated Dockerfile

* Fixed .gitignore

* Added crypto to the read only example

* Updated install with extra pip installs

Co-authored-by: joshuahilgartner <32405513+Firelement@users.noreply.github.com>

.github/workflows/Dockerfile

@@ -4,7 +4,8 @@ FROM ubuntu:22.04
 ARG IB_VERSION
 
 RUN apt update && \
-    apt install -y openjdk-11-jdk python3-pip curl zip
+    apt install -y openjdk-11-jdk python3-pip curl zip && \
+    pip3 install --upgrade pip setuptools wheel
 
 ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
 

.gitignore

@@ -4,4 +4,5 @@ dist
 src/deephaven_ib.egg-info
 docker/data
 docker/*/data
-docker/*/build
+docker/*/build
+

README.md

@@ -234,6 +234,7 @@ new feature and has not been well tested.  To do this:
     ```
 4) Install [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib):
     ```bash
+    pip3 install --upgrade pip setuptools wheel
     pip3 install deephaven-ib
     ```
 5) Install Java 11 and set the appropriate `JAVA_HOME` environment variable.    

docker/dev/Dockerfile

@@ -12,7 +12,7 @@ RUN apt update && \
     apt install -y openjdk-11-jdk && \
     ln -s /usr/lib/jvm/java-11-openjdk-*/ /usr/lib/jvm/java-11-openjdk && \
     apt install --yes git python3-venv python3-pip && \
-    pip3 install --upgrade setuptools wheel build twine
+    pip3 install --upgrade pip setuptools wheel build twine
 
 ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk
 

docker/release/Dockerfile

@@ -7,6 +7,7 @@ FROM ubuntu:22.04
 RUN apt update && \
     apt install -y openjdk-11-jdk python3-pip && \
     ln -s /usr/lib/jvm/java-11-openjdk-*/ /usr/lib/jvm/java-11-openjdk && \
+    pip3 install --upgrade pip setuptools wheel
 
 ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk/
 

examples/example_all_functionality.py

@@ -331,6 +331,42 @@ def get_contracts() -> Dict[str, Contract]:
 client.request_bars_historical(rc, duration=dhib.Duration.days(22), bar_size=dhib.BarSize.DAY_1,
                                bar_type=dhib.BarDataType.YIELD_LAST, keep_up_to_date=False)
 
+print("==============================================================================================================")
+print("==== Request bars (Crypto).")
+print("==============================================================================================================")
+
+contract = Contract()
+contract.symbol = "BTC"
+contract.secType = "CRYPTO"
+contract.exchange = "PAXOS"
+contract.currency = "USD"
+
+rc = client.get_registered_contract(c2)
+client.request_bars_historical(rc2, duration=dhib.Duration.days(253), 
+  bar_size=dhib.BarSize.DAY_1, 
+  bar_type=dhib.BarDataType.AGGTRADES, 
+  keep_up_to_date = False
+)
+client.request_bars_realtime(rc2, bar_type=dhib.BarDataType.TRADES)
+
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.MIDPOINT)
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.BID)
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.ASK)
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.BID_ASK, keep_up_to_date=False)
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.AGGTRADES)
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.ADJUSTED_LAST, keep_up_to_date=False)
+
+client.request_bars_realtime(rc, bar_type=dhib.BarDataType.MIDPOINT)
+client.request_bars_realtime(rc, bar_type=dhib.BarDataType.BID)
+client.request_bars_realtime(rc, bar_type=dhib.BarDataType.ASK)
+client.request_bars_realtime(rc, bar_type=dhib.BarDataType.TRADES)
+
 print("==============================================================================================================")
 print("==== Request tick data.")
 print("==============================================================================================================")

examples/example_read_only_functionality.py

@@ -327,6 +327,42 @@ def get_contracts() -> Dict[str, Contract]:
 client.request_bars_historical(rc, duration=dhib.Duration.days(22), bar_size=dhib.BarSize.DAY_1,
                                bar_type=dhib.BarDataType.YIELD_LAST, keep_up_to_date=False)
 
+print("==============================================================================================================")
+print("==== Request bars (Crypto).")
+print("==============================================================================================================")
+
+contract = Contract()
+contract.symbol = "BTC"
+contract.secType = "CRYPTO"
+contract.exchange = "PAXOS"
+contract.currency = "USD"
+
+rc = client.get_registered_contract(c2)
+client.request_bars_historical(rc2, duration=dhib.Duration.days(253),
+  bar_size=dhib.BarSize.DAY_1,
+  bar_type=dhib.BarDataType.AGGTRADES,
+  keep_up_to_date = False
+)
+client.request_bars_realtime(rc2, bar_type=dhib.BarDataType.TRADES)
+
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.MIDPOINT)
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.BID)
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.ASK)
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.BID_ASK, keep_up_to_date=False)
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.AGGTRADES)
+client.request_bars_historical(rc, duration=dhib.Duration.days(10), bar_size=dhib.BarSize.MIN_5,
+                               bar_type=dhib.BarDataType.ADJUSTED_LAST, keep_up_to_date=False)
+
+client.request_bars_realtime(rc, bar_type=dhib.BarDataType.MIDPOINT)
+client.request_bars_realtime(rc, bar_type=dhib.BarDataType.BID)
+client.request_bars_realtime(rc, bar_type=dhib.BarDataType.ASK)
+client.request_bars_realtime(rc, bar_type=dhib.BarDataType.TRADES)
+
 print("==============================================================================================================")
 print("==== Request tick data.")
 print("==============================================================================================================")

src/deephaven_ib/__init__.py

@@ -150,7 +150,8 @@ class BarDataType(Enum):
     """Bid/Ask yield."""
     YIELD_LAST = 14
     """Last yield."""
-
+    AGGTRADES = 15
+    """Aggregate trade prices."""
 
 class BarSize(Enum):
     """Bar data sizes."""
@@ -966,7 +967,7 @@ def request_bars_realtime(self, contract: RegisteredContract, bar_type: BarDataT
         self._assert_connected()
         requests = []
 
-        if bar_type not in [BarDataType.TRADES, BarDataType.MIDPOINT, BarDataType.BID, BarDataType.ASK]:
+        if bar_type not in [BarDataType.TRADES, BarDataType.AGGTRADES, BarDataType.MIDPOINT, BarDataType.BID, BarDataType.ASK]:
             raise Exception(f"Unsupported bar type: {bar_type}")
 
         for cd in contract.contract_details:

src/deephaven_ib/_internal/tablewriter.py

@@ -10,6 +10,8 @@
 from deephaven.table import Table
 from deephaven import dtypes
 from deephaven.dtypes import DType
+import decimal
+from decimal import Decimal
 
 from .trace import trace_str
 
@@ -67,6 +69,10 @@ def write_row(self, values: List) -> None:
         if self._receive_time:
             values.insert(0, now())
 
+        for i in range(len(values)):
+            if  isinstance(values[i], decimal.Decimal): 
+                values[i] = float(values[i])
+
         self._check_logged_value_types(values)
 
         for i in self._string_indices:

src/deephaven_ib/_tws/ib_type_logger.py

@@ -141,7 +141,6 @@ def map_sec_id_list(value):
         ("LiquidHours", dtypes.StringSet, lambda cd: to_string_set(cd.liquidHours.split(";"))),
         ("EvRule", dtypes.string, lambda cd: cd.evRule),
         ("EvMultiplier", dtypes.int64, lambda cd: cd.evMultiplier),
-        ("MdSizeMultiplier", dtypes.int64, lambda cd: cd.mdSizeMultiplier),
         ("AggGroup", dtypes.int64, lambda cd: map_null_int(cd.aggGroup)),
         ("UnderSymbol", dtypes.string, lambda cd: cd.underSymbol),
         ("UnderSecType", dtypes.string, lambda cd: cd.underSecType),
@@ -218,9 +217,9 @@ def parse_timestamp(bd):
         ("High", dtypes.float64, lambda bd: bd.high),
         ("Low", dtypes.float64, lambda bd: bd.low),
         ("Close", dtypes.float64, lambda bd: bd.close),
-        ("Volume", dtypes.int64, lambda bd: map_null(bd.volume)),
+        ("Volume", dtypes.float64, lambda bd: map_null(bd.volume)),
         ("BarCount", dtypes.int64, lambda bd: map_null(bd.barCount)),
-        ("Average", dtypes.float64, lambda bd: map_null(bd.average)),
+        ("WAP", dtypes.float64, lambda bd: map_null(bd.wap)),
     ]
 
 
@@ -245,7 +244,7 @@ def map_null(val):
         ("High", dtypes.float64, lambda bd: bd.high),
         ("Low", dtypes.float64, lambda bd: bd.low),
         ("Close", dtypes.float64, lambda bd: bd.close),
-        ("Volume", dtypes.int64, lambda bd: map_null(bd.volume)),
+        ("Volume", dtypes.float64, lambda bd: map_null(bd.volume)),
         ("WAP", dtypes.float64, lambda bd: map_null(bd.wap)),
         ("Count", dtypes.int64, lambda bd: map_null(bd.count)),
     ]
@@ -330,7 +329,7 @@ def map_special_conditions(special_conditions: str) -> Any:
     return [
         ("Timestamp", dtypes.DateTime, lambda t: unix_sec_to_dh_datetime(t.time)),
         ("Price", dtypes.float64, lambda t: t.price),
-        ("Size", dtypes.int64, lambda t: t.size),
+        ("Size", dtypes.float64, lambda t: t.size),
         *_include_details(_details_tick_attrib_last(), lambda t: t.tickAttribLast),
         ("Exchange", dtypes.string, lambda t: t.exchange),
         ("SpecialConditions", dtypes.StringSet, lambda t: map_special_conditions(t.specialConditions))

src/deephaven_ib/_tws/tws_client.py

@@ -11,6 +11,9 @@
 from threading import Thread
 from typing import Set
 
+import decimal
+from decimal import Decimal
+
 from deephaven.table import Table
 from deephaven import dtypes
 
@@ -208,7 +211,7 @@ def _build_table_writers() -> Dict[str, TableWriter]:
 
         table_writers["ticks_size"] = TableWriter(
             ["RequestId", "TickType", "Size"],
-            [dtypes.int64, dtypes.string, dtypes.int64])
+            [dtypes.int64, dtypes.string, dtypes.float64])
 
         table_writers["ticks_string"] = TableWriter(
             ["RequestId", "TickType", "Value"],
@@ -404,8 +407,8 @@ def log_request(self, req_id: int, request_type: str, contract: Union[Contract,
     # Always present
     ####
 
-    def error(self, reqId: TickerId, errorCode: int, errorString: str):
-        EWrapper.error(self, reqId, errorCode, errorString)
+    def error(self, reqId: TickerId, errorCode: int, errorString: str, advancedOrderRejectJson: str = ""):
+        EWrapper.error(self, reqId, errorCode, errorString, advancedOrderRejectJson)
 
         if reqId == 2147483647:
             reqId = None
@@ -701,7 +704,7 @@ def accountSummary(self, reqId: int, account: str, tag: str, value: str, currenc
     # reqPositionsMulti
     ####
 
-    def positionMulti(self, reqId: int, account: str, modelCode: str, contract: Contract, pos: float, avgCost: float):
+    def positionMulti(self, reqId: int, account: str, modelCode: str, contract: Contract, pos: decimal.Decimal, avgCost: float):
         EWrapper.positionMulti(self, reqId, account, modelCode, contract, pos, avgCost)
 
         # The returned contract seems to be inconsistent with IB's API to request contract details.
@@ -804,7 +807,7 @@ def tickPrice(self, reqId: TickerId, tickType: TickType, price: float, attrib: T
         self._table_writers["ticks_price"].write_row([reqId, TickTypeEnum.to_str(tickType), price,
                                                       *logger_tick_attrib.vals(attrib)])
 
-    def tickSize(self, reqId: TickerId, tickType: TickType, size: int):
+    def tickSize(self, reqId: TickerId, tickType: TickType, size: decimal.Decimal):
         EWrapper.tickSize(self, reqId, tickType, size)
         self._table_writers["ticks_size"].write_row([reqId, TickTypeEnum.to_str(tickType), size])
 
@@ -846,7 +849,7 @@ def tickSnapshotEnd(self, reqId: int):
     ####
 
     def tickByTickAllLast(self, reqId: int, tickType: int, timestamp: int, price: float,
-                          size: int, tickAttribLast: TickAttribLast, exchange: str,
+                          size: decimal.Decimal, tickAttribLast: TickAttribLast, exchange: str,
                           specialConditions: str):
         EWrapper.tickByTickAllLast(self, reqId, tickType, timestamp, price, size, tickAttribLast, exchange,
                                    specialConditions)
@@ -869,7 +872,7 @@ def historicalTicksLast(self, reqId: int, ticks: ListOfHistoricalTickLast, done:
             self._table_writers["ticks_trade"].write_row([reqId, *logger_hist_tick_last.vals(t)])
 
     def tickByTickBidAsk(self, reqId: int, timestamp: int, bidPrice: float, askPrice: float,
-                         bidSize: int, askSize: int, tickAttribBidAsk: TickAttribBidAsk):
+                         bidSize: decimal.Decimal, askSize: decimal.Decimal, tickAttribBidAsk: TickAttribBidAsk):
         EWrapper.tickByTickBidAsk(self, reqId, timestamp, bidPrice, askPrice, bidSize, askSize, tickAttribBidAsk)
 
         t = HistoricalTickBidAsk()
@@ -920,7 +923,7 @@ def reqRealTimeBars(self, reqId: TickerId, contract: Contract, barSize: int,
         EClient.reqRealTimeBars(self, reqId, contract, barSize, whatToShow, useRTH, realTimeBarsOptions)
 
     def realtimeBar(self, reqId: TickerId, timestamp: int, open_: float, high: float, low: float, close: float,
-                    volume: int, wap: float, count: int):
+                    volume: decimal.Decimal, wap: decimal.Decimal, count: int):
         EWrapper.realtimeBar(self, reqId, timestamp, open_, high, low, close, volume, wap, count)
         bar_size = self._realtime_bar_sizes[reqId]
         bar = RealTimeBar(time=timestamp, endTime=timestamp + bar_size, open_=open_, high=high, low=low, close=close,
@@ -967,8 +970,8 @@ def openOrder(self, orderId: OrderId, contract: Contract, order: Order, orderSta
             [*logger_contract.vals(contract), *logger_order.vals(order), *logger_order_state.vals(orderState)])
         self.contract_registry.request_contract_details_nonblocking(contract)
 
-    def orderStatus(self, orderId: OrderId, status: str, filled: float,
-                    remaining: float, avgFillPrice: float, permId: int,
+    def orderStatus(self, orderId: OrderId, status: str, filled: decimal.Decimal,
+                    remaining: decimal.Decimal, avgFillPrice: float, permId: int,
                     parentId: int, lastFillPrice: float, clientId: int,
                     whyHeld: str, mktCapPrice: float):
         EWrapper.orderStatus(self, orderId, status, filled, remaining, avgFillPrice, permId, parentId, lastFillPrice,