From 84c1f15abbe81581a339ea0e005ca910afbcf5a9 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Thu, 7 Mar 2024 17:06:17 +0000 Subject: [PATCH 1/5] pyyaml bumped to 6.0.1 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bc06d8a2ff..bf277a1bb0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pandas==2.1.3 matplotlib>=3.0.0 -pyyaml==5.3.1 +pyyaml==6.0.1 numpy>=1.24.0 scipy>=1.0.0 pymongo==3.11.3 @@ -12,4 +12,4 @@ Werkzeug>=2.0.1 statsmodels==0.14.0 PyPDF2>=2.5.0 pyarrow>=14.0.1 -scikit-learn>1.3.0 \ No newline at end of file +scikit-learn>1.3.0 From 574ff76c9a8f24ad945f3ba7802364e51b72ec61 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Thu, 7 Mar 2024 17:18:44 +0000 Subject: [PATCH 2/5] update action versions --- .github/workflows/lint.yml | 2 +- .github/workflows/quick-test.yml | 6 +++--- .github/workflows/slow-test-develop.yml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cff5da7040..7c914a7f0b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,7 +6,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: version: "23.11.0" diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml index 3fc83381d2..d254693ff6 100644 --- a/.github/workflows/quick-test.yml +++ b/.github/workflows/quick-test.yml @@ -9,17 +9,17 @@ on: jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: python-version: [ 3.10.13 ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/slow-test-develop.yml b/.github/workflows/slow-test-develop.yml index 55b45fa9d9..0ec92ec7ac 100644 --- a/.github/workflows/slow-test-develop.yml +++ b/.github/workflows/slow-test-develop.yml @@ -9,19 +9,19 @@ on: jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: python-version: [ 3.10.13 ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: "develop" - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From b77f65a7512c2153e362e904d39e76f49c64de77 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Wed, 3 Apr 2024 12:55:04 +0100 Subject: [PATCH 3/5] black --- sysbrokers/IB/client/ib_orders_client.py | 2 +- sysbrokers/IB/ib_broker_commissions.py | 28 ++- sysbrokers/IB/ib_orders.py | 30 +-- sysbrokers/broker_contract_commission_data.py | 6 +- sysbrokers/broker_factory.py | 2 +- sysdata/csv/csv_spread_costs.py | 2 - sysdata/futures/spread_costs.py | 1 - sysproduction/data/broker.py | 13 +- sysproduction/data/instruments.py | 16 +- sysproduction/interactive_diagnostics.py | 3 +- sysproduction/reporting/api.py | 9 +- sysproduction/reporting/commissions_report.py | 3 +- sysproduction/reporting/data/commissions.py | 183 ++++++++++++------ sysproduction/reporting/report_configs.py | 2 +- 14 files changed, 202 insertions(+), 98 deletions(-) diff --git a/sysbrokers/IB/client/ib_orders_client.py b/sysbrokers/IB/client/ib_orders_client.py index 4f65bd521c..e58568b35d 100644 --- a/sysbrokers/IB/client/ib_orders_client.py +++ b/sysbrokers/IB/client/ib_orders_client.py @@ -91,7 +91,7 @@ def broker_submit_order( account_id: str = arg_not_supplied, order_type: brokerOrderType = market_order_type, limit_price: float = None, - what_if: bool = False + what_if: bool = False, ) -> tradeWithContract: """ diff --git a/sysbrokers/IB/ib_broker_commissions.py b/sysbrokers/IB/ib_broker_commissions.py index 67058db76c..736e56dcad 100644 --- a/sysbrokers/IB/ib_broker_commissions.py +++ b/sysbrokers/IB/ib_broker_commissions.py @@ -1,7 +1,9 @@ from sysbrokers.IB.ib_connection import connectionIB from sysbrokers.IB.ib_orders import ibExecutionStackData from sysbrokers.IB.ib_translate_broker_order_objects import tradeWithContract -from sysbrokers.broker_contract_commission_data import brokerFuturesContractCommissionData +from sysbrokers.broker_contract_commission_data import ( + brokerFuturesContractCommissionData, +) from sysdata.data_blob import dataBlob from sysexecution.orders.broker_orders import brokerOrder from sysobjects.contracts import futuresContract @@ -40,17 +42,20 @@ def ibconnection(self) -> connectionIB: def execution_stack(self) -> ibExecutionStackData: return self.data.broker_execution_stack - def get_commission_for_contract(self, futures_contract: futuresContract) -> currencyValue: + def get_commission_for_contract( + self, futures_contract: futuresContract + ) -> currencyValue: instrument_code = futures_contract.instrument_code contract_date = futures_contract.contract_date.list_of_date_str[0] - broker_order =brokerOrder(test_commission_strategy, instrument_code, contract_date, - size_of_test_trade) + broker_order = brokerOrder( + test_commission_strategy, instrument_code, contract_date, size_of_test_trade + ) order = self.execution_stack.what_if_order(broker_order) timer = quickTimer(5) - comm_currency_value = currencyValue(currency='', value=0) + comm_currency_value = currencyValue(currency="", value=0) while timer.unfinished: try: comm_currency_value = get_commission_and_currency_from_ib_order(order) @@ -59,8 +64,15 @@ def get_commission_for_contract(self, futures_contract: futuresContract) -> curr return comm_currency_value -def get_commission_and_currency_from_ib_order(ib_order: tradeWithContract) -> currencyValue: - return currencyValue(value=ib_order.trade.commission / size_of_test_trade, currency=ib_order.trade.commissionCurrency) -test_commission_strategy = "testCommmission" ## whatever not put on stack +def get_commission_and_currency_from_ib_order( + ib_order: tradeWithContract, +) -> currencyValue: + return currencyValue( + value=ib_order.trade.commission / size_of_test_trade, + currency=ib_order.trade.commissionCurrency, + ) + + +test_commission_strategy = "testCommmission" ## whatever not put on stack size_of_test_trade = 10 ## arbitrary diff --git a/sysbrokers/IB/ib_orders.py b/sysbrokers/IB/ib_orders.py index 287fbebd90..6516579b06 100644 --- a/sysbrokers/IB/ib_orders.py +++ b/sysbrokers/IB/ib_orders.py @@ -266,10 +266,16 @@ def put_order_on_stack(self, broker_order: brokerOrder) -> ibOrderWithControls: :param broker_order: key properties are instrument_code, contract_id, quantity :return: ibOrderWithControls or missing_order """ - trade_with_contract_from_ib = self._send_broker_order_to_IB(broker_order, what_if=False) + trade_with_contract_from_ib = self._send_broker_order_to_IB( + broker_order, what_if=False + ) - placed_broker_order_with_controls = self._return_place_order_given_ib_trade_with_contract(trade_with_contract_from_ib=trade_with_contract_from_ib, - broker_order=broker_order) + placed_broker_order_with_controls = ( + self._return_place_order_given_ib_trade_with_contract( + trade_with_contract_from_ib=trade_with_contract_from_ib, + broker_order=broker_order, + ) + ) return placed_broker_order_with_controls @@ -279,13 +285,15 @@ def what_if_order(self, broker_order: brokerOrder) -> tradeWithContract: :param broker_order: key properties are instrument_code, contract_id, quantity :return: ibOrderWithControls or missing_order """ - trade_with_contract_from_ib = self._send_broker_order_to_IB(broker_order, what_if=True) + trade_with_contract_from_ib = self._send_broker_order_to_IB( + broker_order, what_if=True + ) return trade_with_contract_from_ib - - - def _return_place_order_given_ib_trade_with_contract(self, trade_with_contract_from_ib: tradeWithContract, broker_order: brokerOrder) -> ibOrderWithControls: + def _return_place_order_given_ib_trade_with_contract( + self, trade_with_contract_from_ib: tradeWithContract, broker_order: brokerOrder + ) -> ibOrderWithControls: if trade_with_contract_from_ib is missing_order: return missing_order @@ -307,8 +315,9 @@ def _return_place_order_given_ib_trade_with_contract(self, trade_with_contract_f return placed_broker_order_with_controls - - def _send_broker_order_to_IB(self, broker_order: brokerOrder, what_if: bool = False) -> tradeWithContract: + def _send_broker_order_to_IB( + self, broker_order: brokerOrder, what_if: bool = False + ) -> tradeWithContract: """ :param broker_order: key properties are instrument_code, contract_id, quantity @@ -337,7 +346,7 @@ def _send_broker_order_to_IB(self, broker_order: brokerOrder, what_if: bool = Fa account_id=account_id, order_type=order_type, limit_price=limit_price, - what_if=what_if + what_if=what_if, ) if placed_broker_trade_object is missing_order: self.log.warning("Couldn't submit order", **log_attrs) @@ -347,7 +356,6 @@ def _send_broker_order_to_IB(self, broker_order: brokerOrder, what_if: bool = Fa return placed_broker_trade_object - def match_db_broker_order_to_order_from_brokers( self, broker_order_to_match: brokerOrder ) -> brokerOrder: diff --git a/sysbrokers/broker_contract_commission_data.py b/sysbrokers/broker_contract_commission_data.py index 044004d4c8..5655a0502f 100644 --- a/sysbrokers/broker_contract_commission_data.py +++ b/sysbrokers/broker_contract_commission_data.py @@ -1,5 +1,3 @@ - - from sysdata.futures.contracts import futuresContractData from sysdata.data_blob import dataBlob from sysobjects.contracts import futuresContract @@ -9,7 +7,9 @@ class brokerFuturesContractCommissionData(futuresContractData): - def __init__(self, data: dataBlob, log=get_logger("brokerFuturesContractCommissionData")): + def __init__( + self, data: dataBlob, log=get_logger("brokerFuturesContractCommissionData") + ): super().__init__(log=log) self._data = data diff --git a/sysbrokers/broker_factory.py b/sysbrokers/broker_factory.py index fc7dbd1ac9..f5ef32653c 100644 --- a/sysbrokers/broker_factory.py +++ b/sysbrokers/broker_factory.py @@ -45,5 +45,5 @@ def get_ib_class_list(): ibCapitalData, ibFuturesInstrumentData, ibFxHandlingData, - ibFuturesContractCommissionData + ibFuturesContractCommissionData, ] diff --git a/sysdata/csv/csv_spread_costs.py b/sysdata/csv/csv_spread_costs.py index 14c1fd3755..de2d5f0ef0 100644 --- a/sysdata/csv/csv_spread_costs.py +++ b/sysdata/csv/csv_spread_costs.py @@ -64,8 +64,6 @@ def get_spread_costs_as_series(self) -> pd.Series: return spread_cost_series - - def write_all_instrument_spreads(self, spread_cost_as_series: pd.Series): spread_cost_as_df = pd.DataFrame(spread_cost_as_series) spread_cost_as_df.columns = [SPREAD_COST_COLUMN_NAME] diff --git a/sysdata/futures/spread_costs.py b/sysdata/futures/spread_costs.py index 1edb4fafbf..98d01f10e8 100644 --- a/sysdata/futures/spread_costs.py +++ b/sysdata/futures/spread_costs.py @@ -24,7 +24,6 @@ def get_spread_cost(self, instrument_code: str) -> float: def get_spread_costs_as_series(self) -> pd.Series: raise NotImplementedError - def _get_spread_cost_if_series_provided(self, instrument_code: str) -> float: all_data = self.get_spread_costs_as_series() return all_data[instrument_code] diff --git a/sysproduction/data/broker.py b/sysproduction/data/broker.py index 34f1849db1..bad8b49e8d 100644 --- a/sysproduction/data/broker.py +++ b/sysproduction/data/broker.py @@ -1,6 +1,8 @@ from copy import copy -from sysbrokers.broker_contract_commission_data import brokerFuturesContractCommissionData +from sysbrokers.broker_contract_commission_data import ( + brokerFuturesContractCommissionData, +) from sysbrokers.broker_factory import get_broker_class_list from sysbrokers.broker_fx_handling import brokerFxHandlingData from sysbrokers.broker_static_data import brokerStaticData @@ -103,9 +105,12 @@ def diag_controls(self) -> diagControlProcess: return self._diag_controls ## Methods - def get_commission_for_contract_in_currency_value(self, contract: futuresContract) -> currencyValue: - - return self.broker_futures_contract_commission.get_commission_for_contract(contract) + def get_commission_for_contract_in_currency_value( + self, contract: futuresContract + ) -> currencyValue: + return self.broker_futures_contract_commission.get_commission_for_contract( + contract + ) def get_list_of_contract_dates_for_instrument_code( self, instrument_code: str, allow_expired: bool = False diff --git a/sysproduction/data/instruments.py b/sysproduction/data/instruments.py index efe2e77f9f..6a7b8f4ac7 100644 --- a/sysproduction/data/instruments.py +++ b/sysproduction/data/instruments.py @@ -51,14 +51,22 @@ def _add_required_classes_to_data(self, data: dataBlob) -> dataBlob: def get_spread_costs_as_series(self): return self.db_spread_cost_data.get_spread_costs_as_series() - def get_block_commission_for_instrument_as_currency_value(self, instrument_code: str) -> currencyValue: + def get_block_commission_for_instrument_as_currency_value( + self, instrument_code: str + ) -> currencyValue: currency = self.get_currency(instrument_code) - block_commission = self.get_block_commission_for_instrument_as_in_instrument_currency(instrument_code) + block_commission = ( + self.get_block_commission_for_instrument_as_in_instrument_currency( + instrument_code + ) + ) ccy_value = currencyValue(currency=currency, value=block_commission) return ccy_value - def get_block_commission_for_instrument_as_in_instrument_currency(self, instrument_code: str) -> float: + def get_block_commission_for_instrument_as_in_instrument_currency( + self, instrument_code: str + ) -> float: costs = self.get_cost_object(instrument_code) block_commission = costs.value_of_block_commission @@ -66,7 +74,7 @@ def get_block_commission_for_instrument_as_in_instrument_currency(self, instrume def has_percentage_commission(self, instrument_code: str) -> float: costs = self.get_cost_object(instrument_code) - return costs.percentage_cost>0 + return costs.percentage_cost > 0 def get_cost_object(self, instrument_code: str) -> instrumentCosts: meta_data = self.get_meta_data(instrument_code) diff --git a/sysproduction/interactive_diagnostics.py b/sysproduction/interactive_diagnostics.py index 4591a1489f..7f0d1bf95e 100644 --- a/sysproduction/interactive_diagnostics.py +++ b/sysproduction/interactive_diagnostics.py @@ -64,7 +64,7 @@ remove_markets_report_config, market_monitor_report_config, account_curve_report_config, -commission_report_config + commission_report_config, ) @@ -233,6 +233,7 @@ def slippage_report(data): report_config.modify_kwargs(start_date=start_date, end_date=end_date) run_report(report_config, data=data) + def commission_report(data): report_config = email_or_print_or_file(commission_report_config) run_report(report_config, data=data) diff --git a/sysproduction/reporting/api.py b/sysproduction/reporting/api.py index ade3a007bc..a351d32477 100644 --- a/sysproduction/reporting/api.py +++ b/sysproduction/reporting/api.py @@ -88,7 +88,9 @@ get_position_limits_as_df, ) from sysproduction.reporting.data.volume import get_liquidity_data_df -from sysproduction.reporting.data.commissions import df_of_configure_and_broker_block_cost_sorted_by_diff +from sysproduction.reporting.data.commissions import ( + df_of_configure_and_broker_block_cost_sorted_by_diff, +) REPORT_DATETIME_FORMAT = "%d/%m/%Y %H:%M" @@ -953,11 +955,12 @@ def df_commissions(self): return self.cache.get(self._df_commissions) def _df_commissions(self): - combined_df_costs = df_of_configure_and_broker_block_cost_sorted_by_diff(self.data) + combined_df_costs = df_of_configure_and_broker_block_cost_sorted_by_diff( + self.data + ) return combined_df_costs - ##### TRADES ###### def table_of_orders_overview(self): broker_orders = self.broker_orders diff --git a/sysproduction/reporting/commissions_report.py b/sysproduction/reporting/commissions_report.py index e59e5cf3cc..6103dfb00e 100644 --- a/sysproduction/reporting/commissions_report.py +++ b/sysproduction/reporting/commissions_report.py @@ -18,8 +18,7 @@ def commissions_report( data = dataBlob() reporting_api = reportingApi( - data, - start_date=datetime.datetime.now() ## not required for this report + data, start_date=datetime.datetime.now() ## not required for this report ) formatted_output = [] diff --git a/sysproduction/reporting/data/commissions.py b/sysproduction/reporting/data/commissions.py index ecd04d7b86..bd156430fd 100644 --- a/sysproduction/reporting/data/commissions.py +++ b/sysproduction/reporting/data/commissions.py @@ -9,46 +9,59 @@ from sysobjects.contracts import futuresContract from sysdata.data_blob import dataBlob -error_getting_costs = currencyValue(currency='Error', value=0) -missing_contract = 'price not collected' -prices_not_collected = currencyValue(currency='No prices', value=0) -percentage_commissions = currencyValue(currency='% commission ignore', value=0) +error_getting_costs = currencyValue(currency="Error", value=0) +missing_contract = "price not collected" +prices_not_collected = currencyValue(currency="No prices", value=0) +percentage_commissions = currencyValue(currency="% commission ignore", value=0) -def df_of_configure_and_broker_block_cost_sorted_by_index(data: dataBlob) -> pd.DataFrame: + +def df_of_configure_and_broker_block_cost_sorted_by_index( + data: dataBlob, +) -> pd.DataFrame: both = df_of_configure_and_broker_block_cost_sorted_by_diff(data) both = both.sort_index() return both -def df_of_configure_and_broker_block_cost_sorted_by_diff(data: dataBlob) -> pd.DataFrame: + +def df_of_configure_and_broker_block_cost_sorted_by_diff( + data: dataBlob, +) -> pd.DataFrame: list_of_instrument_codes = get_instrument_list(data) - configured_costs = get_current_configured_block_costs(data=data, list_of_instrument_codes=list_of_instrument_codes) - broker_costs = get_broker_block_costs(data=data, list_of_instrument_codes=list_of_instrument_codes) + configured_costs = get_current_configured_block_costs( + data=data, list_of_instrument_codes=list_of_instrument_codes + ) + broker_costs = get_broker_block_costs( + data=data, list_of_instrument_codes=list_of_instrument_codes + ) valid_costs = {} missing_values = {} for instrument_code in list_of_instrument_codes: - update_valid_and_missing_costs_for_instrument_code(instrument_code=instrument_code, - missing_values=missing_values, - valid_costs=valid_costs, - broker_costs=broker_costs, - configured_costs=configured_costs) + update_valid_and_missing_costs_for_instrument_code( + instrument_code=instrument_code, + missing_values=missing_values, + valid_costs=valid_costs, + broker_costs=broker_costs, + configured_costs=configured_costs, + ) valid_costs_df = create_df_in_commission_report(valid_costs) missing_values_df = create_df_in_commission_report(missing_values) both = pd.concat([valid_costs_df, missing_values_df], axis=0) - return both - -def update_valid_and_missing_costs_for_instrument_code(instrument_code: str, - configured_costs: Dict[str, currencyValue], - broker_costs: dict[str, currencyValue], - valid_costs: dict, missing_values: dict): +def update_valid_and_missing_costs_for_instrument_code( + instrument_code: str, + configured_costs: Dict[str, currencyValue], + broker_costs: dict[str, currencyValue], + valid_costs: dict, + missing_values: dict, +): configured_cost = configured_costs.get(instrument_code, error_getting_costs) broker_cost = broker_costs.get(instrument_code, error_getting_costs) @@ -57,80 +70,132 @@ def update_valid_and_missing_costs_for_instrument_code(instrument_code: str, return if configured_cost is error_getting_costs or broker_cost is error_getting_costs: - update_missing_values_with_currencies_and_errors(instrument_code=instrument_code, - missing_values=missing_values, - configured_cost=configured_cost, - broker_cost=broker_cost, - error_str="One or both missing") + update_missing_values_with_currencies_and_errors( + instrument_code=instrument_code, + missing_values=missing_values, + configured_cost=configured_cost, + broker_cost=broker_cost, + error_str="One or both missing", + ) return currency_mismatch = not configured_cost.currency == broker_cost.currency if currency_mismatch: - update_missing_values_with_currencies_and_errors(instrument_code=instrument_code, - missing_values=missing_values, - configured_cost=configured_cost, - broker_cost=broker_cost, - error_str="Currency doesn't match") + update_missing_values_with_currencies_and_errors( + instrument_code=instrument_code, + missing_values=missing_values, + configured_cost=configured_cost, + broker_cost=broker_cost, + error_str="Currency doesn't match", + ) return - update_valid_costs(valid_costs=valid_costs, - instrument_code=instrument_code, - configured_cost=configured_cost, - broker_cost=broker_cost) - - -def update_missing_values_with_currencies_and_errors(missing_values: dict, configured_cost: currencyValue, broker_cost: currencyValue, instrument_code: str, error_str:str): - missing_values[instrument_code] = [configured_cost.currency, broker_cost.currency, error_str] - -def update_valid_costs(valid_costs: dict, configured_cost: currencyValue, broker_cost: currencyValue, instrument_code: str): + update_valid_costs( + valid_costs=valid_costs, + instrument_code=instrument_code, + configured_cost=configured_cost, + broker_cost=broker_cost, + ) + + +def update_missing_values_with_currencies_and_errors( + missing_values: dict, + configured_cost: currencyValue, + broker_cost: currencyValue, + instrument_code: str, + error_str: str, +): + missing_values[instrument_code] = [ + configured_cost.currency, + broker_cost.currency, + error_str, + ] + + +def update_valid_costs( + valid_costs: dict, + configured_cost: currencyValue, + broker_cost: currencyValue, + instrument_code: str, +): configured_cost_instrument = configured_cost.value broker_cost_instrument = broker_cost.value diff = broker_cost_instrument - configured_cost_instrument - valid_costs[instrument_code] = [configured_cost_instrument, broker_cost_instrument, diff] + valid_costs[instrument_code] = [ + configured_cost_instrument, + broker_cost_instrument, + diff, + ] def create_df_in_commission_report(some_dict: dict): some_df = pd.DataFrame(some_dict) some_df = some_df.transpose() - some_df.columns = [CONFIGURED_COLUMN, BROKER_COLUMN, DIFF_COLUMN] + some_df.columns = [CONFIGURED_COLUMN, BROKER_COLUMN, DIFF_COLUMN] some_df = some_df.sort_values(DIFF_COLUMN, ascending=False) return some_df -CONFIGURED_COLUMN = 'configured' -BROKER_COLUMN = 'broker' -DIFF_COLUMN = 'diff' -def get_instrument_list(data: dataBlob)-> list: +CONFIGURED_COLUMN = "configured" +BROKER_COLUMN = "broker" +DIFF_COLUMN = "diff" + + +def get_instrument_list(data: dataBlob) -> list: db = dataBroker(data) - list_of_instruments = db.broker_futures_contract_price_data.get_list_of_instrument_codes_with_merged_price_data() + list_of_instruments = ( + db.broker_futures_contract_price_data.get_list_of_instrument_codes_with_merged_price_data() + ) return list_of_instruments -def get_current_configured_block_costs(data: dataBlob, list_of_instrument_codes: List[str]) -> Dict[str, currencyValue]: + +def get_current_configured_block_costs( + data: dataBlob, list_of_instrument_codes: List[str] +) -> Dict[str, currencyValue]: block_costs_from_config = {} for instrument_code in list_of_instrument_codes: - block_costs_from_config[instrument_code] = get_configured_block_cost_for_instrument(data=data, instrument_code=instrument_code) + block_costs_from_config[ + instrument_code + ] = get_configured_block_cost_for_instrument( + data=data, instrument_code=instrument_code + ) return block_costs_from_config -def get_configured_block_cost_for_instrument(data: dataBlob, instrument_code: str)-> currencyValue: + +def get_configured_block_cost_for_instrument( + data: dataBlob, instrument_code: str +) -> currencyValue: diag_instruments = diagInstruments(data) if diag_instruments.has_percentage_commission(instrument_code): return percentage_commissions try: - costs = diag_instruments.get_block_commission_for_instrument_as_currency_value(instrument_code) + costs = diag_instruments.get_block_commission_for_instrument_as_currency_value( + instrument_code + ) except: costs = error_getting_costs return costs -def get_broker_block_costs(data: dataBlob, list_of_instrument_codes: List[str]) -> Dict[str, currencyValue]: - priced_contracts = get_series_of_priced_contracts(data=data, list_of_instrument_codes=list_of_instrument_codes) - block_costs_from_broker = get_costs_given_priced_contracts(data=data, priced_contracts=priced_contracts) + +def get_broker_block_costs( + data: dataBlob, list_of_instrument_codes: List[str] +) -> Dict[str, currencyValue]: + priced_contracts = get_series_of_priced_contracts( + data=data, list_of_instrument_codes=list_of_instrument_codes + ) + block_costs_from_broker = get_costs_given_priced_contracts( + data=data, priced_contracts=priced_contracts + ) return block_costs_from_broker -def get_series_of_priced_contracts(data: dataBlob, list_of_instrument_codes: List[str]) -> pd.Series: + +def get_series_of_priced_contracts( + data: dataBlob, list_of_instrument_codes: List[str] +) -> pd.Series: dc = dataContracts(data) list_of_priced_contracts = {} for instrument_code in list_of_instrument_codes: @@ -144,13 +209,19 @@ def get_series_of_priced_contracts(data: dataBlob, list_of_instrument_codes: Lis return pd.Series(list_of_priced_contracts) -def get_costs_given_priced_contracts(data: dataBlob, priced_contracts: pd.Series) -> Dict[str, currencyValue]: + +def get_costs_given_priced_contracts( + data: dataBlob, priced_contracts: pd.Series +) -> Dict[str, currencyValue]: block_costs_from_broker = {} for instrument_code, contract in priced_contracts.items(): - block_costs_from_broker[instrument_code] = get_block_cost_for_single_contract(data=data, contract=contract) + block_costs_from_broker[instrument_code] = get_block_cost_for_single_contract( + data=data, contract=contract + ) return block_costs_from_broker + def get_block_cost_for_single_contract(data: dataBlob, contract: futuresContract): db = dataBroker(data) if contract is missing_contract: diff --git a/sysproduction/reporting/report_configs.py b/sysproduction/reporting/report_configs.py index c9a844df13..2db2c6a5db 100644 --- a/sysproduction/reporting/report_configs.py +++ b/sysproduction/reporting/report_configs.py @@ -163,5 +163,5 @@ def modify_kwargs(self, **kwargs): remove_markets_report=remove_markets_report_config, market_monitor_report=market_monitor_report_config, account_curve_report=account_curve_report_config, - commission_report_config=commission_report_config + commission_report_config=commission_report_config, ) From 31fcb98e035cbab00afc01c11dfaa60b143cea7f Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Tue, 14 May 2024 12:01:55 +0100 Subject: [PATCH 4/5] black --- sysquant/returns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sysquant/returns.py b/sysquant/returns.py index da852524a8..f992644621 100644 --- a/sysquant/returns.py +++ b/sysquant/returns.py @@ -164,7 +164,7 @@ def _adjust_df_column_for_SR_costs( daily_gross_return_std = daily_gross_returns_for_column.std() daily_SR_cost = dict_of_SR_costs[column_name] / ROOT_BDAYS_INYEAR - daily_returns_cost = - daily_SR_cost * daily_gross_return_std + daily_returns_cost = -daily_SR_cost * daily_gross_return_std daily_returns_cost_as_list = [daily_returns_cost] * len(gross_returns.index) daily_returns_cost_as_ts = pd.Series( From d7eb7000023aa5a8f3f165527947bdc7559b1a77 Mon Sep 17 00:00:00 2001 From: Andy Geach Date: Thu, 22 Aug 2024 12:52:22 +0100 Subject: [PATCH 5/5] better import of split frequency prices --- ...ntract_prices_from_split_freq_csv_to_db.py | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 sysinit/futures/contract_prices_from_split_freq_csv_to_db.py diff --git a/sysinit/futures/contract_prices_from_split_freq_csv_to_db.py b/sysinit/futures/contract_prices_from_split_freq_csv_to_db.py new file mode 100644 index 0000000000..ae4b5de708 --- /dev/null +++ b/sysinit/futures/contract_prices_from_split_freq_csv_to_db.py @@ -0,0 +1,127 @@ +from syscore.constants import arg_not_supplied +from syscore.dateutils import MIXED_FREQ, HOURLY_FREQ, DAILY_PRICE_FREQ +from syscore.pandas.frequency import merge_data_with_different_freq +from sysdata.csv.csv_futures_contract_prices import csvFuturesContractPriceData +from sysproduction.data.prices import diagPrices +from sysobjects.contracts import futuresContract +from sysobjects.futures_per_contract_prices import futuresContractPrices + +diag_prices = diagPrices() +db_prices = diag_prices.db_futures_contract_price_data + + +def init_db_with_split_freq_csv_prices( + datapath: str, + csv_config=arg_not_supplied, +): + csv_prices = csvFuturesContractPriceData(datapath) + input( + "WARNING THIS WILL ERASE ANY EXISTING DATABASE PRICES WITH DATA FROM %s ARE YOU SURE?! (CTRL-C TO STOP)" + % csv_prices.datapath + ) + + instrument_codes = csv_prices.get_list_of_instrument_codes_with_merged_price_data() + instrument_codes.sort() + for instrument_code in instrument_codes: + init_db_with_split_freq_csv_prices_for_code( + instrument_code, datapath, csv_config=csv_config + ) + + +def init_db_with_split_freq_csv_prices_for_code( + instrument_code: str, + datapath: str, + csv_config=arg_not_supplied, +): + same_length = [] + too_short = [] + print(f"Importing split freq csv prices for {instrument_code}") + csv_prices = csvFuturesContractPriceData(datapath, config=csv_config) + + print(f"Getting split freq .csv prices may take some time") + hourly_dict = csv_prices.get_prices_at_frequency_for_instrument( + instrument_code, + frequency=HOURLY_FREQ, + ) + daily_dict = csv_prices.get_prices_at_frequency_for_instrument( + instrument_code, + frequency=DAILY_PRICE_FREQ, + ) + + hourly_and_daily = sorted(hourly_dict.keys() & daily_dict.keys()) + daily_only = sorted(set(daily_dict.keys()) - set(hourly_dict.keys())) + hourly_only = sorted(set(hourly_dict.keys()) - set(daily_dict.keys())) + + print(f"hourly_and_daily: {sorted(hourly_and_daily)}") + print(f"daily_only: {sorted(daily_only)}") + print(f"hourly_only: {sorted(hourly_only)}") + + print(f"Have hourly and daily .csv prices for: {str(hourly_and_daily)}") + for contract_date_str in hourly_and_daily: + print(f"Processing {contract_date_str}") + + contract = futuresContract(instrument_code, contract_date_str) + print(f"Contract object is {str(contract)}") + + hourly = hourly_dict[contract_date_str] + write_prices_for_contract_at_frequency(contract, hourly, HOURLY_FREQ) + + daily = daily_dict[contract_date_str] + write_prices_for_contract_at_frequency(contract, daily, DAILY_PRICE_FREQ) + + merged = futuresContractPrices(merge_data_with_different_freq([hourly, daily])) + write_prices_for_contract_at_frequency(contract, merged, MIXED_FREQ) + + if len(hourly) == len(daily): + same_length.append(contract_date_str) + + print(f"Have daily only .csv prices for: {str(daily_only)}") + for contract_date_str in daily_only: + print(f"Processing {contract_date_str}") + + contract = futuresContract(instrument_code, contract_date_str) + print(f"Contract object is {str(contract)}") + + daily = daily_dict[contract_date_str] + write_prices_for_contract_at_frequency(contract, daily, DAILY_PRICE_FREQ) + write_prices_for_contract_at_frequency(contract, daily, MIXED_FREQ) + + if len(daily) < 65: + too_short.append(contract_date_str) + + print(f"Have hourly only .csv prices for: {str(hourly_only)}") + for contract_date_str in hourly_only: + print(f"Processing {contract_date_str}") + + contract = futuresContract(instrument_code, contract_date_str) + print(f"Contract object is {str(contract)}") + + hourly = hourly_dict[contract_date_str] + write_prices_for_contract_at_frequency(contract, hourly, HOURLY_FREQ) + write_prices_for_contract_at_frequency(contract, hourly, MIXED_FREQ) + + print(f"These contracts have the same length for daily and hourly: {same_length}") + print(f"These daily contracts are short: {too_short}") + + +def write_prices_for_contract_at_frequency(contract, prices, frequency): + print(f"{frequency} .csv prices are \n{str(prices)}") + print("Writing to db") + db_prices.write_prices_at_frequency_for_contract_object( + contract, + prices, + ignore_duplication=True, + frequency=frequency, + ) + print("Reading back prices from db to check") + written_prices = db_prices.get_prices_at_frequency_for_contract_object( + contract, frequency=frequency + ) + print(f"Read back prices ({frequency}) are \n{str(written_prices)}") + + +if __name__ == "__main__": + input("Will overwrite existing prices are you sure?! CTL-C to abort") + # modify flags as required + datapath = "*** NEED TO DEFINE A DATAPATH***" + init_db_with_split_freq_csv_prices(datapath)