feat: [candles] Improved history access, download all data, renamed P…

…eriod into Timeframe

src/cryptocom/exchange/__init__.py

@@ -16,8 +16,8 @@
     OrderStatus,
     OrderType,
     Pair,
-    Period,
     PrivateTrade,
+    TimeDelta,
     Timeframe,
     Withdrawal,
     WithdrawalStatus,
@@ -35,11 +35,11 @@
     "Pair",
     "instruments",
     "Instrument",
-    "Period",
+    "Timeframe",
     "Candle",
     "MarketTrade",
     "PrivateTrade",
-    "Timeframe",
+    "TimeDelta",
     "Deposit",
     "Withdrawal",
     "DepositStatus",

src/cryptocom/exchange/api.py

@@ -57,6 +57,8 @@ def params_to_str(obj, level):
 
 
 class ApiListenAsyncIterable:
+    """Listen websocket iterator."""
+
     def __init__(self, api, ws, channels, sign):
         self.api = api
         self.ws = ws
@@ -309,12 +311,15 @@ def __init__(
 
         if self.capture:
             self.cache_file.parent.mkdir(exist_ok=True, parents=True)
-            if self.cache_file.exists():
-                self.cache_file.unlink()
-                self.cache_file.touch()
-            self.records = {}
-        else:
+            # TODO: implement correct overwrite
+            # if self.cache_file.exists():
+            #     self.cache_file.unlink()
+            #     self.cache_file.touch()
+
+        if self.cache_file.exists():
             self.records = json.loads(self.cache_file.read_text())
+        else:
+            self.records = {}
 
         kwargs = {"from_env": capture}
         if not self.capture:

src/cryptocom/exchange/market.py

@@ -11,7 +11,7 @@
     OrderInBook,
     OrderSide,
     Pair,
-    Period,
+    Timeframe,
 )
 
 
@@ -63,31 +63,65 @@ async def get_orderbook(self, pair: Pair, depth: int = 150) -> OrderBook:
     async def get_candles(
         self,
         pair: Pair,
-        period: Period,
+        timeframe: Timeframe,
         start_ts: int = None,
         end_ts: int = None,
-        count: int = 300,
+        count: int = 1500,
+        include_all: bool = False,
+        include_last: bool = False,
     ) -> List[Candle]:
         data = []
-        while not data:
-            data = await self.api.get(
-                "public/get-candlestick",
-                {
-                    "instrument_name": pair.exchange_name,
-                    "timeframe": period.value,
-                    "start_ts": start_ts,
-                    "end_ts": end_ts,
-                    "count": count,
-                },
-            )
-            data = [Candle.from_api(pair, candle) for candle in data]
-            # print(datetime.datetime.fromtimestamp(next_data[0].time), datetime.datetime.fromtimestamp(next_data[-1].time))
-            # if not data or data[-1].time != next_data[-1].time:
-            #     data += next_data
-            #     start_ts = next_data[-1].time
-            #     end_ts = next_data[-1].time + (next_data[-1].time - next_data[-2].time) * count
-            #     next_data = []
-        return data
+        chunk_size = 300
+        result = []
+        chunk_start_ts = start_ts
+        chunk_end_ts = end_ts
+        prev_timestamps = set()
+        max_count = count if include_last else count + 1
+
+        while True:
+            params = {
+                "instrument_name": pair.exchange_name,
+                "timeframe": timeframe.value,
+                "count": chunk_size,
+            }
+            if chunk_start_ts and chunk_end_ts:
+                params.update(
+                    {
+                        "start_ts": int(chunk_start_ts * 1000),
+                        "end_ts": int(chunk_end_ts * 1000),
+                    }
+                )
+            data = await self.api.get("public/get-candlestick", params)
+
+            candles = (Candle.from_api(pair, candle) for candle in data)
+            candles = [
+                candle
+                for candle in candles
+                if candle.time not in prev_timestamps
+            ]
+            prev_timestamps = set(candle.time for candle in candles)
+            result = candles + result
+
+            if (
+                not data
+                or len(data) < chunk_size
+                or (len(result) >= max_count and not include_all)
+            ):
+                break
+
+            # NOTE: [start1, end1], [start0, end0]
+            size = candles[1].time - candles[0].time
+            if not end_ts:
+                chunk_end_ts = candles[0].time
+            chunk_start_ts = candles[0].time - size * chunk_size
+
+        if not include_last:
+            del result[-1]
+
+        if not include_all:
+            result = result[:count]
+
+        return result
 
     async def get_trades(self, pair: Pair) -> List[MarketTrade]:
         """Get last 200 trades in a specified market."""
@@ -119,13 +153,14 @@ async def get_price(self, pair: Pair) -> float:
         return (await self.get_ticker(pair)).trade_price
 
     async def listen_candles(
-        self, period: Period, *pairs: List[Pair]
+        self, timeframe: Timeframe, *pairs: List[Pair]
     ) -> AsyncGenerator[Candle, None]:
-        if not isinstance(period, Period):
-            raise ValueError(f"Provide Period enum not {period}")
+        if not isinstance(timeframe, Timeframe):
+            raise ValueError(f"Provide Timeframe enum not {timeframe}")
 
         channels = [
-            f"candlestick.{period.value}.{pair.exchange_name}" for pair in pairs
+            f"candlestick.{timeframe.value}.{pair.exchange_name}"
+            for pair in pairs
         ]
 
         async for data in self.api.listen("market", *channels):

src/cryptocom/exchange/structs.py

@@ -118,19 +118,19 @@ def from_api(cls, pair: Pair, data: Dict):
         )
 
 
-class Period(str, Enum):
-    MINS = "1m"
-    MINS_5 = "5m"
-    MINS_15 = "15m"
-    MINS_30 = "30m"
-    HOURS = "1h"
-    HOURS_2 = "2h"
-    HOURS_4 = "4h"
-    HOURS_12 = "12h"
+class Timeframe(str, Enum):
+    MIN = "1m"
+    MIN_5 = "5m"
+    MIN_15 = "15m"
+    MIN_30 = "30m"
+    HOUR = "1h"
+    HOUR_2 = "2h"
+    HOUR_4 = "4h"
+    HOUR_12 = "12h"
     DAY = "1D"
     WEEK = "7D"
     WEEK_2 = "14D"
-    MONTH_1 = "1M"
+    MONTH = "1M"
 
 
 @dataclass
@@ -531,11 +531,11 @@ def create_from_api(cls, data: Dict) -> "Withdrawal":
         return cls(**params)
 
 
-class Timeframe(IntEnum):
+class TimeDelta(IntEnum):
     NOW = 0
     MINUTES = 60
-    HOURS = 60 * MINUTES
-    DAYS = 24 * HOURS
+    HOUR = 60 * MINUTES
+    DAYS = 24 * HOUR
     WEEKS = 7 * DAYS
     MONTHS = 30 * DAYS