From 78d053f0a6a1aac1c812c40c8d817134369dc0dc Mon Sep 17 00:00:00 2001 From: eschechter Date: Mon, 1 May 2023 09:13:21 -0700 Subject: [PATCH] remove all hard-coding of the ipfs host --- ipldstore/__init__.py | 16 +++++++++++----- ipldstore/hamt_wrapper.py | 17 +++++++++++------ ipldstore/ipldstore.py | 15 +++++++++++---- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/ipldstore/__init__.py b/ipldstore/__init__.py index d203a01..1c3f4e6 100644 --- a/ipldstore/__init__.py +++ b/ipldstore/__init__.py @@ -6,11 +6,17 @@ from .contentstore import ContentAddressableStore, MappingCAStore, IPFSStore -def get_ipfs_mapper(host: str = "http://127.0.0.1:5001", - max_nodes_per_level: int = 10000, - chunker: str = "size-262144", - should_async_get: bool = True) -> IPLDStore: +def get_ipfs_mapper( + host: str = "http://127.0.0.1:5001", + max_nodes_per_level: int = 10000, + chunker: str = "size-262144", + should_async_get: bool = True, +) -> IPLDStore: """ Get an IPLDStore for IPFS running on the given host. """ - return IPLDStore(IPFSStore(host, chunker=chunker, max_nodes_per_level=max_nodes_per_level), should_async_get=should_async_get) + return IPLDStore( + host, + IPFSStore(host, chunker=chunker, max_nodes_per_level=max_nodes_per_level), + should_async_get=should_async_get, + ) diff --git a/ipldstore/hamt_wrapper.py b/ipldstore/hamt_wrapper.py index 5e4bade..474da4f 100644 --- a/ipldstore/hamt_wrapper.py +++ b/ipldstore/hamt_wrapper.py @@ -74,7 +74,8 @@ class HamtMemoryStore: wish the HAMT to persist. """ - def __init__(self): + def __init__(self, host): + self._host = host self.mapping: typing.Dict[CID, bytes] = {} self.num_bytes_in_mapping = 0 @@ -136,7 +137,7 @@ def load(self, cid: typing.Union[cbor2.CBORTag, CID]): except KeyError: try: res = requests.post( - "http://localhost:5001/api/v0/block/get", + f"{self._host}/api/v0/block/get", params={"arg": str(cid)}, timeout=30, ) @@ -167,11 +168,13 @@ class HamtWrapper: def __init__( self, + host: str, starting_id: typing.Optional[CID] = None, others_dict: dict = None, ) -> None: """ Args: + host (str): The location of the IPFS HTTP API to use for IPFS operations starting_id (typing.Optional[CID], optional): CID with which to initialize HAMT. In this implementation, the HAMT will only include keys corresponding to zarr chunks, represented by dag-pb CIDs. Defaults to None. @@ -183,7 +186,8 @@ def __init__( Hamt.register_hasher( 0x12, 32, lambda x: hashlib.sha256(x).digest() ) - store = HamtMemoryStore() + self._host = host + store = HamtMemoryStore(self._host) if starting_id: # Load HAMT from store using given id @@ -292,18 +296,19 @@ def to_dict(self) -> dict: id = CID.decode(id.value[1:]).set(base="base32") obj_cbor = self.hamt.store.mapping[id] res = requests.post( - "http://localhost:5001/api/v0/block/put?cid-codec=dag-cbor", + f"{self._host}/api/v0/block/put?cid-codec=dag-cbor", files={"dummy": obj_cbor}, ) res.raise_for_status() return {**self.others_dict, "hamt": self.hamt.id} @staticmethod - def from_dict(d: dict) -> "HamtWrapper": + def from_dict(d: dict, host: str) -> "HamtWrapper": """Takes a dict generated by `to_dict` and turns it back into a HAMT. Args: d (dict): generated by `to_dict` + host (str): The location of the IPFS HTTP API to use for IPFS operations Returns: HamtWrapper: corresponds to this dict `d` @@ -311,7 +316,7 @@ def from_dict(d: dict) -> "HamtWrapper": others_dict = d hamt_id = others_dict["hamt"] del others_dict["hamt"] - return HamtWrapper(starting_id=hamt_id, others_dict=others_dict) + return HamtWrapper(host, starting_id=hamt_id, others_dict=others_dict) def set_recursive(obj, path, value) -> None: diff --git a/ipldstore/ipldstore.py b/ipldstore/ipldstore.py index 3322193..769ac08 100644 --- a/ipldstore/ipldstore.py +++ b/ipldstore/ipldstore.py @@ -26,9 +26,16 @@ class IPLDStore(MutableMappingSB): - def __init__(self, castore: Optional[ContentAddressableStore] = None, sep: str = "/", should_async_get: bool = True): + def __init__( + self, + host: str, + castore: Optional[ContentAddressableStore] = None, + sep: str = "/", + should_async_get: bool = True, + ): + self._host = host # In this iteration of IPLDStore, we use a HAMT to store zarr chunks instead of a dict - self._mapping = HamtWrapper() + self._mapping = HamtWrapper(self._host) self._store = castore or MappingCAStore() if isinstance(self._store, IPFSStore) and should_async_get: # Monkey patch zarr to use the async get of multiple chunks @@ -119,7 +126,7 @@ def freeze(self) -> CID: def clear(self) -> None: self.root_cid = None - self._mapping = HamtWrapper() + self._mapping = HamtWrapper(self._host) @overload def to_car(self, stream: BufferedIOBase) -> int: @@ -150,7 +157,7 @@ def set_root(self, cid: CID) -> None: assert cid in self._store self.root_cid = cid whole_mapping = self._store.get(cid) - self._mapping = HamtWrapper.from_dict(whole_mapping) + self._mapping = HamtWrapper.from_dict(whole_mapping, self._host) _T = TypeVar("_T")