Fix #1560: Reduce and improve db queries

pdr_backend/pdr_dashboard/callbacks/callbacks_feeds.py

@@ -17,8 +17,8 @@ def get_callbacks_feeds(app):
         [
             Input("base_token", "value"),
             Input("quote_token", "value"),
-            Input("exchange", "value"),
-            Input("time", "value"),
+            Input("source", "value"),
+            Input("timeframe", "value"),
             Input("sales_button", "n_clicks"),
             Input("revenue_button", "n_clicks"),
             Input("accuracy_button", "n_clicks"),
@@ -39,8 +39,8 @@ def get_callbacks_feeds(app):
     def filter_table(
         base_token,
         quote_token,
-        exchange,
-        time,
+        source,
+        timeframe,
         _n_clicks_sales,
         _n_clicks_revenue,
         _n_clicks_accuracy,
@@ -63,8 +63,8 @@ def filter_table(
         conditions = [
             ("filter", "base_token", base_token),
             ("filter", "quote_token", quote_token),
-            ("filter", "exchange", exchange),
-            ("filter", "time", time),
+            ("filter", "source", source),
+            ("filter", "timeframe", timeframe),
             ("range", "sales_raw", sales_min, sales_max),
             ("range", "sales_revenue_(OCEAN)", revenue_min, revenue_max),
             ("range", "avg_accuracy", accuracy_min, accuracy_max),
@@ -145,8 +145,8 @@ def filter_table_by_volume_range(
     @app.callback(
         Output("base_token", "value"),
         Output("quote_token", "value"),
-        Output("exchange", "value"),
-        Output("time", "value"),
+        Output("source", "value"),
+        Output("timeframe", "value"),
         Output("sales_min", "value"),
         Output("sales_max", "value"),
         Output("revenue_min", "value"),

pdr_backend/pdr_dashboard/callbacks/callbacks_home.py

@@ -9,6 +9,7 @@
     get_date_period_text,
     get_start_date_from_period,
     select_or_clear_all_by_table,
+    get_predictoors_home_page_table_data,
 )
 from pdr_backend.pdr_dashboard.util.format import format_value
 
@@ -112,13 +113,16 @@ def update_predictoors_table_on_search(
         predictoors_table,
         show_favourite_addresses,
     ):
+        formatted_predictoors_data = get_predictoors_home_page_table_data(
+            app.predictoors_data
+        )
         selected_predictoors = [predictoors_table[i] for i in selected_rows]
-        filtered_data = app.predictoors_data
+        filtered_data = formatted_predictoors_data
 
         if "show-favourite-addresses.value" in dash.callback_context.triggered_prop_ids:
             custom_predictoors = [
                 predictoor
-                for predictoor in app.predictoors_data
+                for predictoor in formatted_predictoors_data
                 if predictoor["user"] in app.favourite_addresses
             ]
 

pdr_backend/pdr_dashboard/dash_components/modal.py

@@ -42,7 +42,7 @@ def get_header(self) -> html.Div:
             elif self.modal_id == "feeds_modal":
                 sr = self.selected_row
                 modal_header_title = f"""{sr["base_token"]}-{sr["quote_token"]}
-                {sr["time"]} {sr["exchange"]}
+                {sr["timeframe"]} {sr["source"]}
                 """
 
         header = html.Div(
@@ -85,10 +85,10 @@ def create_figures(self):
 
         if self.modal_id == "feeds_modal":
             feed = ArgFeed(
-                exchange=selected_row["exchange"].lower(),
+                exchange=selected_row["source"].lower(),
                 signal=None,
                 pair=f'{selected_row["base_token"]}-{selected_row["quote_token"]}',
-                timeframe=selected_row["time"],
+                timeframe=selected_row["timeframe"],
                 contract=selected_row["full_addr"],
             )
 

pdr_backend/pdr_dashboard/pages/feeds.py

@@ -1,23 +1,17 @@
-from typing import List, Dict, Any, Tuple, Union
-
 from dash import html, dcc, dash_table
-from pdr_backend.pdr_dashboard.util.data import (
-    get_feed_column_ids,
-    find_with_key_value,
-)
 from pdr_backend.pdr_dashboard.dash_components.view_elements import (
     get_metric,
     get_search_bar,
 )
-from pdr_backend.pdr_dashboard.util.format import format_table
+from pdr_backend.pdr_dashboard.util.data import get_feeds_data_for_feeds_table
 from pdr_backend.pdr_dashboard.pages.common import TabularPage, Filter, add_to_filter
 from pdr_backend.pdr_dashboard.dash_components.modal import get_modal
 
 filters = [
     {"name": "base_token", "placeholder": "Base Token", "options": []},
     {"name": "quote_token", "placeholder": "Quote Token", "options": []},
-    {"name": "exchange", "placeholder": "Exchange", "options": []},
-    {"name": "time", "placeholder": "Time", "options": []},
+    {"name": "source", "placeholder": "Source", "options": []},
+    {"name": "timeframe", "placeholder": "Timeframe", "options": []},
 ]
 
 filters_objects = [Filter(**item) for item in filters]
@@ -120,8 +114,8 @@ def get_main_container(self):
             self.app.network_name
         )
 
-        feed_cols, feed_data, raw_feed_data = self.get_feeds_data_for_feeds_table(
-            feed_stats, feed_subscriptions
+        feed_cols, feed_data, raw_feed_data = get_feeds_data_for_feeds_table(
+            self.app.feeds_data, feed_stats, feed_subscriptions
         )
 
         self.app.feeds_table_data = raw_feed_data
@@ -153,96 +147,3 @@ def get_feeds_table_area(
             ],
             style={"width": "100%", "overflow": "scroll"},
         )
-
-    def get_feeds_stat_with_contract(
-        self, contract: str, feed_stats: List[Dict[str, Any]]
-    ) -> Dict[str, Union[float, int]]:
-        result = find_with_key_value(feed_stats, "contract", contract)
-
-        if result:
-            return {
-                "avg_accuracy": float(result["avg_accuracy"]),
-                "avg_stake_per_epoch_(OCEAN)": float(result["avg_stake"]),
-                "volume_(OCEAN)": float(result["volume"]),
-            }
-
-        return {
-            "avg_accuracy": 0,
-            "avg_stake_per_epoch_(OCEAN)": 0,
-            "volume_(OCEAN)": 0,
-        }
-
-    def get_feeds_subscription_stat_with_contract(
-        self, contract: str, feed_subcription_stats: List[Dict[str, Any]]
-    ) -> Dict[str, Union[float, int, str]]:
-        result = find_with_key_value(feed_subcription_stats, "contract", contract)
-
-        if result:
-            sales_str = f"{result['sales']}"
-
-            df_buy_count_str = (
-                f"{result['df_buy_count']}-DF" if result["df_buy_count"] > 0 else ""
-            )
-            ws_buy_count_str = (
-                f"{result['ws_buy_count']}-WS" if result["ws_buy_count"] > 0 else ""
-            )
-
-            if df_buy_count_str or ws_buy_count_str:
-                counts_str = "_".join(
-                    filter(None, [df_buy_count_str, ws_buy_count_str])
-                )
-
-                if counts_str:
-                    sales_str += f"_{counts_str}"
-
-            return {
-                "price_(OCEAN)": result["price"],
-                "sales": sales_str,
-                "sales_raw": result["sales"],
-                "sales_revenue_(OCEAN)": result["sales_revenue"],
-            }
-
-        return {
-            "price_(OCEAN)": 0,
-            "sales": 0,
-            "sales_raw": 0,
-            "sales_revenue_(OCEAN)": 0,
-        }
-
-    def get_feeds_data_for_feeds_table(
-        self,
-        feed_stats: List[Dict[str, Any]],
-        feed_subcription_stats: List[Dict[str, Any]],
-    ) -> Tuple[List[Dict[str, str]], List[Dict[str, Any]], List[Dict[str, Any]]]:
-
-        temp_data = self.app.feeds_data
-
-        new_feed_data = []
-        ## split the pair column into two columns
-        for feed in temp_data:
-            split_pair = feed["pair"].split("/")
-            feed_item = {}
-            feed_item["addr"] = feed["contract"]
-            feed_item["base_token"] = split_pair[0]
-            feed_item["quote_token"] = split_pair[1]
-            feed_item["exchange"] = feed["source"].capitalize()
-            feed_item["time"] = feed["timeframe"]
-            feed_item["full_addr"] = feed["contract"]
-
-            result = self.get_feeds_stat_with_contract(feed["contract"], feed_stats)
-
-            feed_item.update(result)
-
-            subscription_result = self.get_feeds_subscription_stat_with_contract(
-                feed["contract"], feed_subcription_stats
-            )
-
-            feed_item.update(subscription_result)
-
-            new_feed_data.append(feed_item)
-
-        columns = get_feed_column_ids(new_feed_data[0])
-
-        formatted_data = format_table(new_feed_data, columns)
-
-        return columns, formatted_data, new_feed_data

pdr_backend/pdr_dashboard/pages/home.py

@@ -3,6 +3,7 @@
 
 from pdr_backend.pdr_dashboard.util.data import (
     col_to_human,
+    get_predictoors_home_page_table_data,
 )
 from pdr_backend.pdr_dashboard.dash_components.view_elements import (
     get_date_period_selection_component,
@@ -79,7 +80,8 @@ def get_feeds_cols_data(self):
         return (columns, hidden_columns), data
 
     def get_predictoors_cols_data(self):
-        data = self.app.predictoors_data
+        predictoor_data = self.app.predictoors_data
+        data = get_predictoors_home_page_table_data(predictoor_data)
 
         columns = [{"name": col_to_human(col), "id": col} for col in data[0].keys()]
         hidden_columns = ["user"]

pdr_backend/pdr_dashboard/pages/predictoors.py

@@ -87,10 +87,8 @@ def get_search_bar_row(self):
         )
 
     def get_main_container(self):
-        predictoor_stats = self.app.db_getter.predictoor_payouts_stats()
-
         predictoor_cols, predictoor_data, raw_predictoor_data = (
-            self.get_data_for_predictoors_table(predictoor_stats)
+            self.get_data_for_predictoors_table()
         )
 
         self.app.predictoor_table_data = raw_predictoor_data
@@ -125,10 +123,9 @@ def get_predictoor_table_area(
 
     def get_data_for_predictoors_table(
         self,
-        predictoor_stats: List[Dict[str, Any]],
     ) -> Tuple[List[Dict[str, str]], List[Dict[str, Any]], List[Dict[str, Any]]]:
 
-        temp_data = predictoor_stats
+        temp_data = self.app.predictoors_data
 
         new_predictoor_data = []
 

pdr_backend/pdr_dashboard/predictoor_dash.py

@@ -11,7 +11,6 @@
     get_callbacks_predictoors,
 )
 from pdr_backend.pdr_dashboard.dash_components.view_elements import get_layout
-from pdr_backend.pdr_dashboard.util.data import get_predictoors_data_from_payouts
 from pdr_backend.pdr_dashboard.util.db import DBGetter
 from pdr_backend.pdr_dashboard.util.prices import (
     calculate_tx_gas_fee_cost_in_OCEAN,
@@ -47,10 +46,11 @@ def setup_app(app, ppss: PPSS):
     app.db_getter = DBGetter(ppss.lake_ss.lake_dir)
     app.network_name = ppss.web3_pp.network
 
+    app.feed_stats = app.db_getter.feed_payouts_stats()
+    app.feed_subscriptions = app.db_getter.feed_subscription_stats(app.network_name)
     app.feeds_data = app.db_getter.feeds_data()
-    app.predictoors_data = get_predictoors_data_from_payouts(
-        app.db_getter.payouts_stats()
-    )
+
+    app.predictoors_data = app.db_getter.predictoor_payouts_stats()
 
     valid_addresses = [p["user"].lower() for p in app.predictoors_data]
     app.favourite_addresses = [