Better import of split frequency CSV price files

.github/workflows/lint.yml

@@ -6,7 +6,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
           version: "23.11.0"

.github/workflows/quick-test.yml

@@ -9,17 +9,17 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: [ 3.10.13 ]
 
     steps:
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/slow-test-develop.yml

@@ -9,19 +9,19 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: [ 3.10.13 ]
 
     steps:
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: "develop"
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 

requirements.txt

@@ -1,6 +1,6 @@
 pandas==2.1.3
 matplotlib>=3.0.0
-pyyaml==5.3.1
+pyyaml==6.0.1
 numpy>=1.24.0
 scipy>=1.0.0
 pymongo==3.11.3
@@ -12,4 +12,4 @@ Werkzeug>=2.0.1
 statsmodels==0.14.0
 PyPDF2>=2.5.0
 pyarrow>=14.0.1
-scikit-learn>1.3.0
+scikit-learn>1.3.0

sysbrokers/IB/client/ib_orders_client.py

@@ -91,7 +91,7 @@ def broker_submit_order(
         account_id: str = arg_not_supplied,
         order_type: brokerOrderType = market_order_type,
         limit_price: float = None,
-        what_if: bool = False
+        what_if: bool = False,
     ) -> tradeWithContract:
         """
 

sysbrokers/IB/ib_broker_commissions.py

@@ -1,7 +1,9 @@
 from sysbrokers.IB.ib_connection import connectionIB
 from sysbrokers.IB.ib_orders import ibExecutionStackData
 from sysbrokers.IB.ib_translate_broker_order_objects import tradeWithContract
-from sysbrokers.broker_contract_commission_data import brokerFuturesContractCommissionData
+from sysbrokers.broker_contract_commission_data import (
+    brokerFuturesContractCommissionData,
+)
 from sysdata.data_blob import dataBlob
 from sysexecution.orders.broker_orders import brokerOrder
 from sysobjects.contracts import futuresContract
@@ -40,17 +42,20 @@ def ibconnection(self) -> connectionIB:
     def execution_stack(self) -> ibExecutionStackData:
         return self.data.broker_execution_stack
 
-    def get_commission_for_contract(self, futures_contract: futuresContract) -> currencyValue:
+    def get_commission_for_contract(
+        self, futures_contract: futuresContract
+    ) -> currencyValue:
         instrument_code = futures_contract.instrument_code
         contract_date = futures_contract.contract_date.list_of_date_str[0]
 
-        broker_order =brokerOrder(test_commission_strategy, instrument_code, contract_date,
-                    size_of_test_trade)
+        broker_order = brokerOrder(
+            test_commission_strategy, instrument_code, contract_date, size_of_test_trade
+        )
 
         order = self.execution_stack.what_if_order(broker_order)
 
         timer = quickTimer(5)
-        comm_currency_value = currencyValue(currency='', value=0)
+        comm_currency_value = currencyValue(currency="", value=0)
         while timer.unfinished:
             try:
                 comm_currency_value = get_commission_and_currency_from_ib_order(order)
@@ -59,8 +64,15 @@ def get_commission_for_contract(self, futures_contract: futuresContract) -> curr
 
         return comm_currency_value
 
-def get_commission_and_currency_from_ib_order(ib_order: tradeWithContract) -> currencyValue:
-    return currencyValue(value=ib_order.trade.commission / size_of_test_trade, currency=ib_order.trade.commissionCurrency)
 
-test_commission_strategy = "testCommmission" ## whatever not put on stack
+def get_commission_and_currency_from_ib_order(
+    ib_order: tradeWithContract,
+) -> currencyValue:
+    return currencyValue(
+        value=ib_order.trade.commission / size_of_test_trade,
+        currency=ib_order.trade.commissionCurrency,
+    )
+
+
+test_commission_strategy = "testCommmission"  ## whatever not put on stack
 size_of_test_trade = 10  ## arbitrary

sysbrokers/IB/ib_orders.py

@@ -266,10 +266,16 @@ def put_order_on_stack(self, broker_order: brokerOrder) -> ibOrderWithControls:
         :param broker_order: key properties are instrument_code, contract_id, quantity
         :return: ibOrderWithControls or missing_order
         """
-        trade_with_contract_from_ib = self._send_broker_order_to_IB(broker_order, what_if=False)
+        trade_with_contract_from_ib = self._send_broker_order_to_IB(
+            broker_order, what_if=False
+        )
 
-        placed_broker_order_with_controls = self._return_place_order_given_ib_trade_with_contract(trade_with_contract_from_ib=trade_with_contract_from_ib,
-                                                                                                  broker_order=broker_order)
+        placed_broker_order_with_controls = (
+            self._return_place_order_given_ib_trade_with_contract(
+                trade_with_contract_from_ib=trade_with_contract_from_ib,
+                broker_order=broker_order,
+            )
+        )
 
         return placed_broker_order_with_controls
 
@@ -279,13 +285,15 @@ def what_if_order(self, broker_order: brokerOrder) -> tradeWithContract:
         :param broker_order: key properties are instrument_code, contract_id, quantity
         :return: ibOrderWithControls or missing_order
         """
-        trade_with_contract_from_ib = self._send_broker_order_to_IB(broker_order, what_if=True)
+        trade_with_contract_from_ib = self._send_broker_order_to_IB(
+            broker_order, what_if=True
+        )
 
         return trade_with_contract_from_ib
 
-
-
-    def _return_place_order_given_ib_trade_with_contract(self, trade_with_contract_from_ib: tradeWithContract, broker_order: brokerOrder) -> ibOrderWithControls:
+    def _return_place_order_given_ib_trade_with_contract(
+        self, trade_with_contract_from_ib: tradeWithContract, broker_order: brokerOrder
+    ) -> ibOrderWithControls:
         if trade_with_contract_from_ib is missing_order:
             return missing_order
 
@@ -307,8 +315,9 @@ def _return_place_order_given_ib_trade_with_contract(self, trade_with_contract_f
 
         return placed_broker_order_with_controls
 
-
-    def _send_broker_order_to_IB(self, broker_order: brokerOrder, what_if: bool = False) -> tradeWithContract:
+    def _send_broker_order_to_IB(
+        self, broker_order: brokerOrder, what_if: bool = False
+    ) -> tradeWithContract:
         """
 
         :param broker_order: key properties are instrument_code, contract_id, quantity
@@ -337,7 +346,7 @@ def _send_broker_order_to_IB(self, broker_order: brokerOrder, what_if: bool = Fa
             account_id=account_id,
             order_type=order_type,
             limit_price=limit_price,
-            what_if=what_if
+            what_if=what_if,
         )
         if placed_broker_trade_object is missing_order:
             self.log.warning("Couldn't submit order", **log_attrs)
@@ -347,7 +356,6 @@ def _send_broker_order_to_IB(self, broker_order: brokerOrder, what_if: bool = Fa
 
         return placed_broker_trade_object
 
-
     def match_db_broker_order_to_order_from_brokers(
         self, broker_order_to_match: brokerOrder
     ) -> brokerOrder:

sysbrokers/broker_contract_commission_data.py

@@ -1,5 +1,3 @@
-
-
 from sysdata.futures.contracts import futuresContractData
 from sysdata.data_blob import dataBlob
 from sysobjects.contracts import futuresContract
@@ -9,7 +7,9 @@
 
 
 class brokerFuturesContractCommissionData(futuresContractData):
-    def __init__(self, data: dataBlob, log=get_logger("brokerFuturesContractCommissionData")):
+    def __init__(
+        self, data: dataBlob, log=get_logger("brokerFuturesContractCommissionData")
+    ):
         super().__init__(log=log)
         self._data = data
 

sysbrokers/broker_factory.py

@@ -45,5 +45,5 @@ def get_ib_class_list():
         ibCapitalData,
         ibFuturesInstrumentData,
         ibFxHandlingData,
-        ibFuturesContractCommissionData
+        ibFuturesContractCommissionData,
     ]

sysdata/csv/csv_spread_costs.py

@@ -64,8 +64,6 @@ def get_spread_costs_as_series(self) -> pd.Series:
 
         return spread_cost_series
 
-
-
     def write_all_instrument_spreads(self, spread_cost_as_series: pd.Series):
         spread_cost_as_df = pd.DataFrame(spread_cost_as_series)
         spread_cost_as_df.columns = [SPREAD_COST_COLUMN_NAME]

sysdata/futures/spread_costs.py

@@ -24,7 +24,6 @@ def get_spread_cost(self, instrument_code: str) -> float:
     def get_spread_costs_as_series(self) -> pd.Series:
         raise NotImplementedError
 
-
     def _get_spread_cost_if_series_provided(self, instrument_code: str) -> float:
         all_data = self.get_spread_costs_as_series()
         return all_data[instrument_code]

sysinit/futures/contract_prices_from_split_freq_csv_to_db.py

@@ -0,0 +1,127 @@
+from syscore.constants import arg_not_supplied
+from syscore.dateutils import MIXED_FREQ, HOURLY_FREQ, DAILY_PRICE_FREQ
+from syscore.pandas.frequency import merge_data_with_different_freq
+from sysdata.csv.csv_futures_contract_prices import csvFuturesContractPriceData
+from sysproduction.data.prices import diagPrices
+from sysobjects.contracts import futuresContract
+from sysobjects.futures_per_contract_prices import futuresContractPrices
+
+diag_prices = diagPrices()
+db_prices = diag_prices.db_futures_contract_price_data
+
+
+def init_db_with_split_freq_csv_prices(
+    datapath: str,
+    csv_config=arg_not_supplied,
+):
+    csv_prices = csvFuturesContractPriceData(datapath)
+    input(
+        "WARNING THIS WILL ERASE ANY EXISTING DATABASE PRICES WITH DATA FROM %s ARE YOU SURE?! (CTRL-C TO STOP)"
+        % csv_prices.datapath
+    )
+
+    instrument_codes = csv_prices.get_list_of_instrument_codes_with_merged_price_data()
+    instrument_codes.sort()
+    for instrument_code in instrument_codes:
+        init_db_with_split_freq_csv_prices_for_code(
+            instrument_code, datapath, csv_config=csv_config
+        )
+
+
+def init_db_with_split_freq_csv_prices_for_code(
+    instrument_code: str,
+    datapath: str,
+    csv_config=arg_not_supplied,
+):
+    same_length = []
+    too_short = []
+    print(f"Importing split freq csv prices for {instrument_code}")
+    csv_prices = csvFuturesContractPriceData(datapath, config=csv_config)
+
+    print(f"Getting split freq .csv prices may take some time")
+    hourly_dict = csv_prices.get_prices_at_frequency_for_instrument(
+        instrument_code,
+        frequency=HOURLY_FREQ,
+    )
+    daily_dict = csv_prices.get_prices_at_frequency_for_instrument(
+        instrument_code,
+        frequency=DAILY_PRICE_FREQ,
+    )
+
+    hourly_and_daily = sorted(hourly_dict.keys() & daily_dict.keys())
+    daily_only = sorted(set(daily_dict.keys()) - set(hourly_dict.keys()))
+    hourly_only = sorted(set(hourly_dict.keys()) - set(daily_dict.keys()))
+
+    print(f"hourly_and_daily: {sorted(hourly_and_daily)}")
+    print(f"daily_only: {sorted(daily_only)}")
+    print(f"hourly_only: {sorted(hourly_only)}")
+
+    print(f"Have hourly and daily .csv prices for: {str(hourly_and_daily)}")
+    for contract_date_str in hourly_and_daily:
+        print(f"Processing {contract_date_str}")
+
+        contract = futuresContract(instrument_code, contract_date_str)
+        print(f"Contract object is {str(contract)}")
+
+        hourly = hourly_dict[contract_date_str]
+        write_prices_for_contract_at_frequency(contract, hourly, HOURLY_FREQ)
+
+        daily = daily_dict[contract_date_str]
+        write_prices_for_contract_at_frequency(contract, daily, DAILY_PRICE_FREQ)
+
+        merged = futuresContractPrices(merge_data_with_different_freq([hourly, daily]))
+        write_prices_for_contract_at_frequency(contract, merged, MIXED_FREQ)
+
+        if len(hourly) == len(daily):
+            same_length.append(contract_date_str)
+
+    print(f"Have daily only .csv prices for: {str(daily_only)}")
+    for contract_date_str in daily_only:
+        print(f"Processing {contract_date_str}")
+
+        contract = futuresContract(instrument_code, contract_date_str)
+        print(f"Contract object is {str(contract)}")
+
+        daily = daily_dict[contract_date_str]
+        write_prices_for_contract_at_frequency(contract, daily, DAILY_PRICE_FREQ)
+        write_prices_for_contract_at_frequency(contract, daily, MIXED_FREQ)
+
+        if len(daily) < 65:
+            too_short.append(contract_date_str)
+
+    print(f"Have hourly only .csv prices for: {str(hourly_only)}")
+    for contract_date_str in hourly_only:
+        print(f"Processing {contract_date_str}")
+
+        contract = futuresContract(instrument_code, contract_date_str)
+        print(f"Contract object is {str(contract)}")
+
+        hourly = hourly_dict[contract_date_str]
+        write_prices_for_contract_at_frequency(contract, hourly, HOURLY_FREQ)
+        write_prices_for_contract_at_frequency(contract, hourly, MIXED_FREQ)
+
+    print(f"These contracts have the same length for daily and hourly: {same_length}")
+    print(f"These daily contracts are short: {too_short}")
+
+
+def write_prices_for_contract_at_frequency(contract, prices, frequency):
+    print(f"{frequency} .csv prices are \n{str(prices)}")
+    print("Writing to db")
+    db_prices.write_prices_at_frequency_for_contract_object(
+        contract,
+        prices,
+        ignore_duplication=True,
+        frequency=frequency,
+    )
+    print("Reading back prices from db to check")
+    written_prices = db_prices.get_prices_at_frequency_for_contract_object(
+        contract, frequency=frequency
+    )
+    print(f"Read back prices ({frequency}) are \n{str(written_prices)}")
+
+
+if __name__ == "__main__":
+    input("Will overwrite existing prices are you sure?! CTL-C to abort")
+    # modify flags as required
+    datapath = "*** NEED TO DEFINE A DATAPATH***"
+    init_db_with_split_freq_csv_prices(datapath)

sysproduction/data/broker.py

@@ -1,6 +1,8 @@
 from copy import copy
 
-from sysbrokers.broker_contract_commission_data import brokerFuturesContractCommissionData
+from sysbrokers.broker_contract_commission_data import (
+    brokerFuturesContractCommissionData,
+)
 from sysbrokers.broker_factory import get_broker_class_list
 from sysbrokers.broker_fx_handling import brokerFxHandlingData
 from sysbrokers.broker_static_data import brokerStaticData
@@ -103,9 +105,12 @@ def diag_controls(self) -> diagControlProcess:
         return self._diag_controls
 
     ## Methods
-    def get_commission_for_contract_in_currency_value(self, contract: futuresContract) -> currencyValue:
-
-        return self.broker_futures_contract_commission.get_commission_for_contract(contract)
+    def get_commission_for_contract_in_currency_value(
+        self, contract: futuresContract
+    ) -> currencyValue:
+        return self.broker_futures_contract_commission.get_commission_for_contract(
+            contract
+        )
 
     def get_list_of_contract_dates_for_instrument_code(
         self, instrument_code: str, allow_expired: bool = False