diff --git a/.isort.cfg b/.isort.cfg index 291c8862f28e..b68fc92d5581 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -109,7 +109,6 @@ extend_skip= chia/util/profiler.py chia/util/service_groups.py chia/util/ssl_check.py - chia/util/streamable.py chia/util/ws_message.py chia/wallet/cat_wallet/cat_info.py chia/wallet/cat_wallet/cat_utils.py @@ -192,7 +191,6 @@ extend_skip= tests/core/util/test_files.py tests/core/util/test_keychain.py tests/core/util/test_keyring_wrapper.py - tests/core/util/test_streamable.py tests/generator/test_compression.py tests/generator/test_generator_types.py tests/generator/test_list_to_batches.py diff --git a/benchmarks/streamable.py b/benchmarks/streamable.py index 47e77d224a06..d940a421a659 100644 --- a/benchmarks/streamable.py +++ b/benchmarks/streamable.py @@ -17,14 +17,14 @@ _version = 1 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BenchmarkInner(Streamable): a: str -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BenchmarkMiddle(Streamable): a: uint64 b: List[bytes32] @@ -33,8 +33,8 @@ class BenchmarkMiddle(Streamable): e: BenchmarkInner -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BenchmarkClass(Streamable): a: Optional[BenchmarkMiddle] b: Optional[BenchmarkMiddle] diff --git a/benchmarks/utils.py b/benchmarks/utils.py index bda6216385c4..7969a146ed0d 100644 --- a/benchmarks/utils.py +++ b/benchmarks/utils.py @@ -3,7 +3,7 @@ from chia.consensus.coinbase import create_farmer_coin, create_pool_coin from chia.types.blockchain_format.classgroup import ClassgroupElement from chia.types.blockchain_format.coin import Coin -from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.types.blockchain_format.sized_bytes import bytes32, bytes100 from chia.types.blockchain_format.vdf import VDFInfo, VDFProof from chia.types.blockchain_format.foliage import Foliage, FoliageBlockData, FoliageTransactionBlock, TransactionsInfo from chia.types.blockchain_format.pool_target import PoolTarget @@ -56,9 +56,7 @@ def rand_bytes(num) -> bytes: def rand_hash() -> bytes32: - # TODO: address hint errors and remove ignores - # error: Incompatible return value type (got "bytes", expected "bytes32") [return-value] - return rand_bytes(32) # type: ignore[return-value] + return bytes32(rand_bytes(32)) def rand_g1() -> G1Element: @@ -72,9 +70,7 @@ def rand_g2() -> G2Element: def rand_class_group_element() -> ClassgroupElement: - # TODO: address hint errors and remove ignores - # error: Argument 1 to "ClassgroupElement" has incompatible type "bytes"; expected "bytes100" [arg-type] - return ClassgroupElement(rand_bytes(100)) # type: ignore[arg-type] + return ClassgroupElement(bytes100(rand_bytes(100))) def rand_vdf() -> VDFInfo: @@ -84,7 +80,7 @@ def rand_vdf() -> VDFInfo: def rand_vdf_proof() -> VDFProof: return VDFProof( uint8(1), # witness_type - rand_hash(), # witness + bytes(rand_hash()), # witness bool(random.randint(0, 1)), # normalized_to_identity ) diff --git a/chia/consensus/block_record.py b/chia/consensus/block_record.py index 520a9497b1e9..a6ccfc4f7cf3 100644 --- a/chia/consensus/block_record.py +++ b/chia/consensus/block_record.py @@ -11,8 +11,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BlockRecord(Streamable): """ This class is not included or hashed into the blockchain, but it is kept in memory as a more diff --git a/chia/consensus/cost_calculator.py b/chia/consensus/cost_calculator.py index d090116d56e1..e2f3a8cea2b4 100644 --- a/chia/consensus/cost_calculator.py +++ b/chia/consensus/cost_calculator.py @@ -6,8 +6,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NPCResult(Streamable): error: Optional[uint16] npc_list: List[NPC] diff --git a/chia/consensus/multiprocess_validation.py b/chia/consensus/multiprocess_validation.py index ba19fd1e4c7c..8ae42e1de774 100644 --- a/chia/consensus/multiprocess_validation.py +++ b/chia/consensus/multiprocess_validation.py @@ -34,8 +34,8 @@ log = logging.getLogger(__name__) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PreValidationResult(Streamable): error: Optional[uint16] required_iters: Optional[uint64] # Iff error is None diff --git a/chia/full_node/block_height_map.py b/chia/full_node/block_height_map.py index 216e4d4a32d2..a07910ee7d2e 100644 --- a/chia/full_node/block_height_map.py +++ b/chia/full_node/block_height_map.py @@ -13,8 +13,8 @@ log = logging.getLogger(__name__) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SesCache(Streamable): content: List[Tuple[uint32, bytes]] diff --git a/chia/full_node/full_node_store.py b/chia/full_node/full_node_store.py index 728266998790..0626e8e1a1e5 100644 --- a/chia/full_node/full_node_store.py +++ b/chia/full_node/full_node_store.py @@ -29,8 +29,8 @@ log = logging.getLogger(__name__) -@dataclasses.dataclass(frozen=True) @streamable +@dataclasses.dataclass(frozen=True) class FullNodeStorePeakResult(Streamable): added_eos: Optional[EndOfSubSlotBundle] new_signage_points: List[Tuple[uint8, SignagePoint]] diff --git a/chia/full_node/signage_point.py b/chia/full_node/signage_point.py index be79026fd5c6..9230d6f3f3a1 100644 --- a/chia/full_node/signage_point.py +++ b/chia/full_node/signage_point.py @@ -5,8 +5,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SignagePoint(Streamable): cc_vdf: Optional[VDFInfo] cc_proof: Optional[VDFProof] diff --git a/chia/plotting/manager.py b/chia/plotting/manager.py index aed8921c61dc..b6992212cb2a 100644 --- a/chia/plotting/manager.py +++ b/chia/plotting/manager.py @@ -32,16 +32,16 @@ CURRENT_VERSION: uint16 = uint16(0) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CacheEntry(Streamable): pool_public_key: Optional[G1Element] pool_contract_puzzle_hash: Optional[bytes32] plot_public_key: G1Element -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class DiskCache(Streamable): version: uint16 data: List[Tuple[bytes32, CacheEntry]] diff --git a/chia/pools/pool_config.py b/chia/pools/pool_config.py index 07d2bd05f065..61abbe58026a 100644 --- a/chia/pools/pool_config.py +++ b/chia/pools/pool_config.py @@ -25,8 +25,8 @@ log = logging.getLogger(__name__) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PoolWalletConfig(Streamable): launcher_id: bytes32 pool_url: str diff --git a/chia/pools/pool_wallet_info.py b/chia/pools/pool_wallet_info.py index 42c5e4aebc9c..3fef250952e2 100644 --- a/chia/pools/pool_wallet_info.py +++ b/chia/pools/pool_wallet_info.py @@ -38,8 +38,8 @@ class PoolSingletonState(IntEnum): FARMING_TO_POOL = PoolSingletonState.FARMING_TO_POOL -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PoolState(Streamable): """ `PoolState` is a type that is serialized to the blockchain to track the state of the user's pool singleton @@ -97,8 +97,8 @@ def create_pool_state( return ps -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PoolWalletInfo(Streamable): """ Internal Pool Wallet state, not destined for the blockchain. This can be completely derived with diff --git a/chia/protocols/farmer_protocol.py b/chia/protocols/farmer_protocol.py index 1d2c4f062e78..e60a421d16d3 100644 --- a/chia/protocols/farmer_protocol.py +++ b/chia/protocols/farmer_protocol.py @@ -15,8 +15,8 @@ """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewSignagePoint(Streamable): challenge_hash: bytes32 challenge_chain_sp: bytes32 @@ -26,8 +26,8 @@ class NewSignagePoint(Streamable): signage_point_index: uint8 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class DeclareProofOfSpace(Streamable): challenge_hash: bytes32 challenge_chain_sp: bytes32 @@ -41,16 +41,16 @@ class DeclareProofOfSpace(Streamable): pool_signature: Optional[G2Element] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestSignedValues(Streamable): quality_string: bytes32 foliage_block_data_hash: bytes32 foliage_transaction_block_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class FarmingInfo(Streamable): challenge_hash: bytes32 sp_hash: bytes32 @@ -60,8 +60,8 @@ class FarmingInfo(Streamable): total_plots: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SignedValues(Streamable): quality_string: bytes32 foliage_block_data_signature: G2Element diff --git a/chia/protocols/full_node_protocol.py b/chia/protocols/full_node_protocol.py index cbafdad00cc8..793dabbe9415 100644 --- a/chia/protocols/full_node_protocol.py +++ b/chia/protocols/full_node_protocol.py @@ -18,8 +18,8 @@ """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewPeak(Streamable): header_hash: bytes32 height: uint32 @@ -28,102 +28,102 @@ class NewPeak(Streamable): unfinished_reward_block_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewTransaction(Streamable): transaction_id: bytes32 cost: uint64 fees: uint64 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestTransaction(Streamable): transaction_id: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondTransaction(Streamable): transaction: SpendBundle -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestProofOfWeight(Streamable): total_number_of_blocks: uint32 tip: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondProofOfWeight(Streamable): wp: WeightProof tip: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestBlock(Streamable): height: uint32 include_transaction_block: bool -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectBlock(Streamable): height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestBlocks(Streamable): start_height: uint32 end_height: uint32 include_transaction_block: bool -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondBlocks(Streamable): start_height: uint32 end_height: uint32 blocks: List[FullBlock] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectBlocks(Streamable): start_height: uint32 end_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondBlock(Streamable): block: FullBlock -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewUnfinishedBlock(Streamable): unfinished_reward_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestUnfinishedBlock(Streamable): unfinished_reward_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondUnfinishedBlock(Streamable): unfinished_block: UnfinishedBlock -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewSignagePointOrEndOfSubSlot(Streamable): prev_challenge_hash: Optional[bytes32] challenge_hash: bytes32 @@ -131,16 +131,16 @@ class NewSignagePointOrEndOfSubSlot(Streamable): last_rc_infusion: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestSignagePointOrEndOfSubSlot(Streamable): challenge_hash: bytes32 index_from_challenge: uint8 last_rc_infusion: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondSignagePoint(Streamable): index_from_challenge: uint8 challenge_chain_vdf: VDFInfo @@ -149,20 +149,20 @@ class RespondSignagePoint(Streamable): reward_chain_proof: VDFProof -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondEndOfSubSlot(Streamable): end_of_slot_bundle: EndOfSubSlotBundle -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestMempoolTransactions(Streamable): filter: bytes -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewCompactVDF(Streamable): height: uint32 header_hash: bytes32 @@ -170,8 +170,8 @@ class NewCompactVDF(Streamable): vdf_info: VDFInfo -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestCompactVDF(Streamable): height: uint32 header_hash: bytes32 @@ -179,8 +179,8 @@ class RequestCompactVDF(Streamable): vdf_info: VDFInfo -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondCompactVDF(Streamable): height: uint32 header_hash: bytes32 @@ -189,15 +189,15 @@ class RespondCompactVDF(Streamable): vdf_proof: VDFProof -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestPeers(Streamable): """ Return full list of peers """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondPeers(Streamable): peer_list: List[TimestampedPeerInfo] diff --git a/chia/protocols/harvester_protocol.py b/chia/protocols/harvester_protocol.py index b48165773fc9..3a918933f223 100644 --- a/chia/protocols/harvester_protocol.py +++ b/chia/protocols/harvester_protocol.py @@ -14,23 +14,23 @@ """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PoolDifficulty(Streamable): difficulty: uint64 sub_slot_iters: uint64 pool_contract_puzzle_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class HarvesterHandshake(Streamable): farmer_public_keys: List[G1Element] pool_public_keys: List[G1Element] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewSignagePointHarvester(Streamable): challenge_hash: bytes32 difficulty: uint64 @@ -40,8 +40,8 @@ class NewSignagePointHarvester(Streamable): pool_difficulties: List[PoolDifficulty] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewProofOfSpace(Streamable): challenge_hash: bytes32 sp_hash: bytes32 @@ -50,8 +50,8 @@ class NewProofOfSpace(Streamable): signage_point_index: uint8 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestSignatures(Streamable): plot_identifier: str challenge_hash: bytes32 @@ -59,8 +59,8 @@ class RequestSignatures(Streamable): messages: List[bytes32] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondSignatures(Streamable): plot_identifier: str challenge_hash: bytes32 @@ -70,8 +70,8 @@ class RespondSignatures(Streamable): message_signatures: List[Tuple[bytes32, G2Element]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Plot(Streamable): filename: str size: uint8 @@ -83,14 +83,14 @@ class Plot(Streamable): time_modified: uint64 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestPlots(Streamable): pass -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondPlots(Streamable): plots: List[Plot] failed_to_open_filenames: List[str] diff --git a/chia/protocols/introducer_protocol.py b/chia/protocols/introducer_protocol.py index 7eadacb21462..e9eeab997833 100644 --- a/chia/protocols/introducer_protocol.py +++ b/chia/protocols/introducer_protocol.py @@ -10,15 +10,15 @@ """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestPeersIntroducer(Streamable): """ Return full list of peers """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondPeersIntroducer(Streamable): peer_list: List[TimestampedPeerInfo] diff --git a/chia/protocols/pool_protocol.py b/chia/protocols/pool_protocol.py index e6a1f7f8ff15..3b8678ec90a1 100644 --- a/chia/protocols/pool_protocol.py +++ b/chia/protocols/pool_protocol.py @@ -33,8 +33,8 @@ class PoolErrorCode(Enum): # Used to verify GET /farmer and GET /login -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class AuthenticationPayload(Streamable): method_name: str launcher_id: bytes32 @@ -43,8 +43,8 @@ class AuthenticationPayload(Streamable): # GET /pool_info -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class GetPoolInfoResponse(Streamable): name: str logo_url: str @@ -60,8 +60,8 @@ class GetPoolInfoResponse(Streamable): # POST /partial -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostPartialPayload(Streamable): launcher_id: bytes32 authentication_token: uint64 @@ -71,16 +71,16 @@ class PostPartialPayload(Streamable): harvester_id: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostPartialRequest(Streamable): payload: PostPartialPayload aggregate_signature: G2Element # Response in success case -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostPartialResponse(Streamable): new_difficulty: uint64 @@ -89,8 +89,8 @@ class PostPartialResponse(Streamable): # Response in success case -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class GetFarmerResponse(Streamable): authentication_public_key: G1Element payout_instructions: str @@ -101,8 +101,8 @@ class GetFarmerResponse(Streamable): # POST /farmer -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostFarmerPayload(Streamable): launcher_id: bytes32 authentication_token: uint64 @@ -111,16 +111,16 @@ class PostFarmerPayload(Streamable): suggested_difficulty: Optional[uint64] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostFarmerRequest(Streamable): payload: PostFarmerPayload signature: G2Element # Response in success case -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostFarmerResponse(Streamable): welcome_message: str @@ -128,8 +128,8 @@ class PostFarmerResponse(Streamable): # PUT /farmer -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PutFarmerPayload(Streamable): launcher_id: bytes32 authentication_token: uint64 @@ -138,16 +138,16 @@ class PutFarmerPayload(Streamable): suggested_difficulty: Optional[uint64] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PutFarmerRequest(Streamable): payload: PutFarmerPayload signature: G2Element # Response in success case -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PutFarmerResponse(Streamable): authentication_public_key: Optional[bool] payout_instructions: Optional[bool] @@ -158,8 +158,8 @@ class PutFarmerResponse(Streamable): # Response in error case for all endpoints of the pool protocol -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ErrorResponse(Streamable): error_code: uint16 error_message: Optional[str] diff --git a/chia/protocols/shared_protocol.py b/chia/protocols/shared_protocol.py index d7c0e6cd94a8..8f469e8a79d5 100644 --- a/chia/protocols/shared_protocol.py +++ b/chia/protocols/shared_protocol.py @@ -19,8 +19,8 @@ class Capability(IntEnum): BASE = 1 # Base capability just means it supports the chia protocol at mainnet -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Handshake(Streamable): network_id: str protocol_version: str diff --git a/chia/protocols/timelord_protocol.py b/chia/protocols/timelord_protocol.py index 282bb09bfbd6..6db1c32e8ddc 100644 --- a/chia/protocols/timelord_protocol.py +++ b/chia/protocols/timelord_protocol.py @@ -16,8 +16,8 @@ """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewPeakTimelord(Streamable): reward_chain_block: RewardChainBlock difficulty: uint64 @@ -31,8 +31,8 @@ class NewPeakTimelord(Streamable): passes_ses_height_but_not_yet_included: bool -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewUnfinishedBlockTimelord(Streamable): reward_chain_block: RewardChainBlockUnfinished # Reward chain trunk data difficulty: uint64 @@ -44,8 +44,8 @@ class NewUnfinishedBlockTimelord(Streamable): rc_prev: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewInfusionPointVDF(Streamable): unfinished_reward_hash: bytes32 challenge_chain_ip_vdf: VDFInfo @@ -56,8 +56,8 @@ class NewInfusionPointVDF(Streamable): infused_challenge_chain_ip_proof: Optional[VDFProof] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewSignagePointVDF(Streamable): index_from_challenge: uint8 challenge_chain_sp_vdf: VDFInfo @@ -66,14 +66,14 @@ class NewSignagePointVDF(Streamable): reward_chain_sp_proof: VDFProof -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewEndOfSubSlotVDF(Streamable): end_of_sub_slot_bundle: EndOfSubSlotBundle -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestCompactProofOfTime(Streamable): new_proof_of_time: VDFInfo header_hash: bytes32 @@ -81,8 +81,8 @@ class RequestCompactProofOfTime(Streamable): field_vdf: uint8 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondCompactProofOfTime(Streamable): vdf_info: VDFInfo vdf_proof: VDFProof diff --git a/chia/protocols/wallet_protocol.py b/chia/protocols/wallet_protocol.py index efb0650aa774..a96edca4be47 100644 --- a/chia/protocols/wallet_protocol.py +++ b/chia/protocols/wallet_protocol.py @@ -15,15 +15,15 @@ """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestPuzzleSolution(Streamable): coin_name: bytes32 height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PuzzleSolutionResponse(Streamable): coin_name: bytes32 height: uint32 @@ -31,35 +31,35 @@ class PuzzleSolutionResponse(Streamable): solution: Program -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondPuzzleSolution(Streamable): response: PuzzleSolutionResponse -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectPuzzleSolution(Streamable): coin_name: bytes32 height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SendTransaction(Streamable): transaction: SpendBundle -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class TransactionAck(Streamable): txid: bytes32 status: uint8 # MempoolInclusionStatus error: Optional[str] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewPeakWallet(Streamable): header_hash: bytes32 height: uint32 @@ -67,34 +67,34 @@ class NewPeakWallet(Streamable): fork_point_with_previous_peak: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestBlockHeader(Streamable): height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondBlockHeader(Streamable): header_block: HeaderBlock -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectHeaderRequest(Streamable): height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestRemovals(Streamable): height: uint32 header_hash: bytes32 coin_names: Optional[List[bytes32]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondRemovals(Streamable): height: uint32 header_hash: bytes32 @@ -102,23 +102,23 @@ class RespondRemovals(Streamable): proofs: Optional[List[Tuple[bytes32, bytes]]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectRemovalsRequest(Streamable): height: uint32 header_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestAdditions(Streamable): height: uint32 header_hash: Optional[bytes32] puzzle_hashes: Optional[List[bytes32]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondAdditions(Streamable): height: uint32 header_hash: bytes32 @@ -126,75 +126,75 @@ class RespondAdditions(Streamable): proofs: Optional[List[Tuple[bytes32, bytes, Optional[bytes]]]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectAdditionsRequest(Streamable): height: uint32 header_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestHeaderBlocks(Streamable): start_height: uint32 end_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectHeaderBlocks(Streamable): start_height: uint32 end_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondHeaderBlocks(Streamable): start_height: uint32 end_height: uint32 header_blocks: List[HeaderBlock] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CoinState(Streamable): coin: Coin spent_height: Optional[uint32] created_height: Optional[uint32] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RegisterForPhUpdates(Streamable): puzzle_hashes: List[bytes32] min_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondToPhUpdates(Streamable): puzzle_hashes: List[bytes32] min_height: uint32 coin_states: List[CoinState] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RegisterForCoinUpdates(Streamable): coin_ids: List[bytes32] min_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondToCoinUpdates(Streamable): coin_ids: List[bytes32] min_height: uint32 coin_states: List[CoinState] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CoinStateUpdate(Streamable): height: uint32 fork_height: uint32 @@ -202,27 +202,27 @@ class CoinStateUpdate(Streamable): items: List[CoinState] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestChildren(Streamable): coin_name: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondChildren(Streamable): coin_states: List[CoinState] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestSESInfo(Streamable): start_height: uint32 end_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondSESInfo(Streamable): reward_chain_hash: List[bytes32] heights: List[List[uint32]] diff --git a/chia/seeder/peer_record.py b/chia/seeder/peer_record.py index d307c4065a88..2a6f8dde481c 100644 --- a/chia/seeder/peer_record.py +++ b/chia/seeder/peer_record.py @@ -6,8 +6,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PeerRecord(Streamable): peer_id: str ip_address: str diff --git a/chia/server/address_manager_store.py b/chia/server/address_manager_store.py index 3f725cb3dd04..25069133c31b 100644 --- a/chia/server/address_manager_store.py +++ b/chia/server/address_manager_store.py @@ -21,8 +21,8 @@ log = logging.getLogger(__name__) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PeerDataSerialization(Streamable): """ Serializable property bag for the peer data that was previously stored in sqlite. diff --git a/chia/server/outbound_message.py b/chia/server/outbound_message.py index 66861b413afa..ba55ca7b8b29 100644 --- a/chia/server/outbound_message.py +++ b/chia/server/outbound_message.py @@ -31,8 +31,8 @@ class Delivery(IntEnum): SPECIFIC = 6 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Message(Streamable): type: uint8 # one of ProtocolMessageTypes # message id diff --git a/chia/simulator/simulator_protocol.py b/chia/simulator/simulator_protocol.py index f512ab04c3a2..29d8492d17a5 100644 --- a/chia/simulator/simulator_protocol.py +++ b/chia/simulator/simulator_protocol.py @@ -5,14 +5,14 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class FarmNewBlockProtocol(Streamable): puzzle_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ReorgProtocol(Streamable): old_index: uint32 new_index: uint32 diff --git a/chia/timelord/timelord.py b/chia/timelord/timelord.py index e3fd5ae58a1b..456b6641d15a 100644 --- a/chia/timelord/timelord.py +++ b/chia/timelord/timelord.py @@ -41,8 +41,8 @@ log = logging.getLogger(__name__) -@dataclasses.dataclass(frozen=True) @streamable +@dataclasses.dataclass(frozen=True) class BlueboxProcessData(Streamable): challenge: bytes32 size_bits: uint16 diff --git a/chia/types/blockchain_format/classgroup.py b/chia/types/blockchain_format/classgroup.py index a3bf22ad9301..10a83e93e934 100644 --- a/chia/types/blockchain_format/classgroup.py +++ b/chia/types/blockchain_format/classgroup.py @@ -5,8 +5,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ClassgroupElement(Streamable): """ Represents a classgroup element (a,b,c) where a, b, and c are 512 bit signed integers. However this is using diff --git a/chia/types/blockchain_format/coin.py b/chia/types/blockchain_format/coin.py index 1ba4e0a04b48..0709f62c621b 100644 --- a/chia/types/blockchain_format/coin.py +++ b/chia/types/blockchain_format/coin.py @@ -9,8 +9,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Coin(Streamable): """ This structure is used in the body for the reward and fees genesis coins. diff --git a/chia/types/blockchain_format/foliage.py b/chia/types/blockchain_format/foliage.py index 043f361b35d4..412e40ba3956 100644 --- a/chia/types/blockchain_format/foliage.py +++ b/chia/types/blockchain_format/foliage.py @@ -10,8 +10,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class TransactionsInfo(Streamable): # Information that goes along with each transaction block generator_root: bytes32 # sha256 of the block generator in this block @@ -22,8 +22,8 @@ class TransactionsInfo(Streamable): reward_claims_incorporated: List[Coin] # These can be in any order -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class FoliageTransactionBlock(Streamable): # Information that goes along with each transaction block that is relevant for light clients prev_transaction_block_hash: bytes32 @@ -34,8 +34,8 @@ class FoliageTransactionBlock(Streamable): transactions_info_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class FoliageBlockData(Streamable): # Part of the block that is signed by the plot key unfinished_reward_block_hash: bytes32 @@ -45,8 +45,8 @@ class FoliageBlockData(Streamable): extension_data: bytes32 # Used for future updates. Can be any 32 byte value initially -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Foliage(Streamable): # The entire foliage block, containing signature and the unsigned back pointer # The hash of this is the "header hash". Note that for unfinished blocks, the prev_block_hash diff --git a/chia/types/blockchain_format/pool_target.py b/chia/types/blockchain_format/pool_target.py index 57659c9f634e..6d9b126d76dd 100644 --- a/chia/types/blockchain_format/pool_target.py +++ b/chia/types/blockchain_format/pool_target.py @@ -5,8 +5,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PoolTarget(Streamable): puzzle_hash: bytes32 max_height: uint32 # A max height of 0 means it is valid forever diff --git a/chia/types/blockchain_format/proof_of_space.py b/chia/types/blockchain_format/proof_of_space.py index 1b4617c8a33e..1a33d4ea9ca8 100644 --- a/chia/types/blockchain_format/proof_of_space.py +++ b/chia/types/blockchain_format/proof_of_space.py @@ -15,8 +15,8 @@ log = logging.getLogger(__name__) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ProofOfSpace(Streamable): challenge: bytes32 pool_public_key: Optional[G1Element] # Only one of these two should be present diff --git a/chia/types/blockchain_format/reward_chain_block.py b/chia/types/blockchain_format/reward_chain_block.py index 9bcb4bf59707..515d032c09aa 100644 --- a/chia/types/blockchain_format/reward_chain_block.py +++ b/chia/types/blockchain_format/reward_chain_block.py @@ -10,8 +10,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RewardChainBlockUnfinished(Streamable): total_iters: uint128 signage_point_index: uint8 @@ -23,8 +23,8 @@ class RewardChainBlockUnfinished(Streamable): reward_chain_sp_signature: G2Element -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RewardChainBlock(Streamable): weight: uint128 height: uint32 diff --git a/chia/types/blockchain_format/slots.py b/chia/types/blockchain_format/slots.py index 0f55073f5d47..a230dcf0432e 100644 --- a/chia/types/blockchain_format/slots.py +++ b/chia/types/blockchain_format/slots.py @@ -10,8 +10,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ChallengeBlockInfo(Streamable): # The hash of this is used as the challenge_hash for the ICC VDF proof_of_space: ProofOfSpace challenge_chain_sp_vdf: Optional[VDFInfo] # Only present if not the first sp @@ -19,8 +19,8 @@ class ChallengeBlockInfo(Streamable): # The hash of this is used as the challen challenge_chain_ip_vdf: VDFInfo -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ChallengeChainSubSlot(Streamable): challenge_chain_end_of_slot_vdf: VDFInfo infused_challenge_chain_sub_slot_hash: Optional[bytes32] # Only at the end of a slot @@ -29,14 +29,14 @@ class ChallengeChainSubSlot(Streamable): new_difficulty: Optional[uint64] # Only at the end of epoch, sub-epoch, and slot -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class InfusedChallengeChainSubSlot(Streamable): infused_challenge_chain_end_of_slot_vdf: VDFInfo -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RewardChainSubSlot(Streamable): end_of_slot_vdf: VDFInfo challenge_chain_sub_slot_hash: bytes32 @@ -44,8 +44,8 @@ class RewardChainSubSlot(Streamable): deficit: uint8 # 16 or less. usually zero -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SubSlotProofs(Streamable): challenge_chain_slot_proof: VDFProof infused_challenge_chain_slot_proof: Optional[VDFProof] diff --git a/chia/types/blockchain_format/sub_epoch_summary.py b/chia/types/blockchain_format/sub_epoch_summary.py index 419182374349..6a1fc89fa697 100644 --- a/chia/types/blockchain_format/sub_epoch_summary.py +++ b/chia/types/blockchain_format/sub_epoch_summary.py @@ -6,8 +6,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SubEpochSummary(Streamable): prev_subepoch_summary_hash: bytes32 reward_chain_hash: bytes32 # hash of reward chain at end of last segment diff --git a/chia/types/blockchain_format/vdf.py b/chia/types/blockchain_format/vdf.py index 4d417502cc70..55b47595b978 100644 --- a/chia/types/blockchain_format/vdf.py +++ b/chia/types/blockchain_format/vdf.py @@ -44,16 +44,16 @@ def verify_vdf( ) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class VDFInfo(Streamable): challenge: bytes32 # Used to generate the discriminant (VDF group) number_of_iterations: uint64 output: ClassgroupElement -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class VDFProof(Streamable): witness_type: uint8 witness: bytes diff --git a/chia/types/coin_record.py b/chia/types/coin_record.py index 85579e687d26..b195ad3de085 100644 --- a/chia/types/coin_record.py +++ b/chia/types/coin_record.py @@ -8,8 +8,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CoinRecord(Streamable): """ These are values that correspond to a CoinName that are used diff --git a/chia/types/coin_spend.py b/chia/types/coin_spend.py index b37a61ac0889..895fd761c3dc 100644 --- a/chia/types/coin_spend.py +++ b/chia/types/coin_spend.py @@ -7,8 +7,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CoinSpend(Streamable): """ This is a rather disparate data structure that validates coin transfers. It's generally populated diff --git a/chia/types/condition_with_args.py b/chia/types/condition_with_args.py index 2222baa3e3d5..5f0dbce11fd9 100644 --- a/chia/types/condition_with_args.py +++ b/chia/types/condition_with_args.py @@ -5,8 +5,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ConditionWithArgs(Streamable): """ This structure is used to store parsed CLVM conditions diff --git a/chia/types/end_of_slot_bundle.py b/chia/types/end_of_slot_bundle.py index 0e7292b61b67..72d708f5e7a4 100644 --- a/chia/types/end_of_slot_bundle.py +++ b/chia/types/end_of_slot_bundle.py @@ -10,8 +10,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class EndOfSubSlotBundle(Streamable): challenge_chain: ChallengeChainSubSlot infused_challenge_chain: Optional[InfusedChallengeChainSubSlot] diff --git a/chia/types/full_block.py b/chia/types/full_block.py index 0f4156cd68fe..0449f481df56 100644 --- a/chia/types/full_block.py +++ b/chia/types/full_block.py @@ -10,8 +10,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class FullBlock(Streamable): # All the information required to validate a block finished_sub_slots: List[EndOfSubSlotBundle] # If first sb diff --git a/chia/types/generator_types.py b/chia/types/generator_types.py index 060f3d85d987..7b86520bf3f3 100644 --- a/chia/types/generator_types.py +++ b/chia/types/generator_types.py @@ -21,8 +21,8 @@ class CompressorArg: end: int -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BlockGenerator(Streamable): program: SerializedProgram generator_refs: List[SerializedProgram] diff --git a/chia/types/header_block.py b/chia/types/header_block.py index afc5c5a50450..e0b44abd2442 100644 --- a/chia/types/header_block.py +++ b/chia/types/header_block.py @@ -8,8 +8,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class HeaderBlock(Streamable): # Same as a FullBlock but without TransactionInfo and Generator (but with filter), used by light clients finished_sub_slots: List[EndOfSubSlotBundle] # If first sb diff --git a/chia/types/mempool_item.py b/chia/types/mempool_item.py index 9e89b86cc2e2..dfd588a7ed69 100644 --- a/chia/types/mempool_item.py +++ b/chia/types/mempool_item.py @@ -10,8 +10,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class MempoolItem(Streamable): spend_bundle: SpendBundle fee: uint64 diff --git a/chia/types/name_puzzle_condition.py b/chia/types/name_puzzle_condition.py index e3ff30f903ac..143a9bde8e46 100644 --- a/chia/types/name_puzzle_condition.py +++ b/chia/types/name_puzzle_condition.py @@ -7,8 +7,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NPC(Streamable): coin_name: bytes32 puzzle_hash: bytes32 diff --git a/chia/types/peer_info.py b/chia/types/peer_info.py index 4404b1fc8561..7435cb180951 100644 --- a/chia/types/peer_info.py +++ b/chia/types/peer_info.py @@ -6,8 +6,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PeerInfo(Streamable): host: str port: uint16 @@ -59,8 +59,8 @@ def get_group(self): return group -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class TimestampedPeerInfo(Streamable): host: str port: uint16 diff --git a/chia/types/spend_bundle.py b/chia/types/spend_bundle.py index f8e9977cce2f..5c003838a0d0 100644 --- a/chia/types/spend_bundle.py +++ b/chia/types/spend_bundle.py @@ -15,8 +15,8 @@ from .coin_spend import CoinSpend -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SpendBundle(Streamable): """ This is a list of coins being spent along with their solution programs, and a single diff --git a/chia/types/unfinished_block.py b/chia/types/unfinished_block.py index 491ab364f2c9..da110a66c116 100644 --- a/chia/types/unfinished_block.py +++ b/chia/types/unfinished_block.py @@ -10,8 +10,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class UnfinishedBlock(Streamable): # Full block, without the final VDFs finished_sub_slots: List[EndOfSubSlotBundle] # If first sb diff --git a/chia/types/unfinished_header_block.py b/chia/types/unfinished_header_block.py index 39db047c1eb4..da6f3b436adc 100644 --- a/chia/types/unfinished_header_block.py +++ b/chia/types/unfinished_header_block.py @@ -8,8 +8,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class UnfinishedHeaderBlock(Streamable): # Same as a FullBlock but without TransactionInfo and Generator, used by light clients finished_sub_slots: List[EndOfSubSlotBundle] # If first sb diff --git a/chia/types/weight_proof.py b/chia/types/weight_proof.py index bb958040eaaa..47f5b9118777 100644 --- a/chia/types/weight_proof.py +++ b/chia/types/weight_proof.py @@ -11,8 +11,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SubEpochData(Streamable): reward_chain_hash: bytes32 num_blocks_overflow: uint8 @@ -31,8 +31,8 @@ class SubEpochData(Streamable): # total number of challenge blocks == total number of reward chain blocks -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SubSlotData(Streamable): # if infused proof_of_space: Optional[ProofOfSpace] @@ -65,37 +65,37 @@ def is_end_of_slot(self) -> bool: return False -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SubEpochChallengeSegment(Streamable): sub_epoch_n: uint32 sub_slots: List[SubSlotData] rc_slot_end_info: Optional[VDFInfo] # in first segment of each sub_epoch -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) # this is used only for serialization to database class SubEpochSegments(Streamable): challenge_segments: List[SubEpochChallengeSegment] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) # this is used only for serialization to database class RecentChainData(Streamable): recent_chain_data: List[HeaderBlock] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ProofBlockHeader(Streamable): finished_sub_slots: List[EndOfSubSlotBundle] reward_chain_block: RewardChainBlock -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class WeightProof(Streamable): sub_epochs: List[SubEpochData] sub_epoch_segments: List[SubEpochChallengeSegment] # sampled sub epoch diff --git a/chia/util/streamable.py b/chia/util/streamable.py index 73584bac7ed1..84fc5ab57082 100644 --- a/chia/util/streamable.py +++ b/chia/util/streamable.py @@ -5,7 +5,7 @@ import pprint import sys from enum import Enum -from typing import Any, BinaryIO, Dict, get_type_hints, List, Tuple, Type, TypeVar, Callable, Optional, Iterator +from typing import Any, BinaryIO, Callable, Dict, Iterator, List, Optional, Tuple, Type, TypeVar, Union, get_type_hints from blspy import G1Element, G2Element, PrivateKey from typing_extensions import Literal @@ -14,16 +14,18 @@ from chia.util.byte_types import hexstr_to_bytes from chia.util.hash import std_hash from chia.util.ints import int64, int512, uint32, uint64, uint128 -from chia.util.type_checking import is_type_List, is_type_SpecificOptional, is_type_Tuple, strictdataclass if sys.version_info < (3, 8): def get_args(t: Type[Any]) -> Tuple[Any, ...]: return getattr(t, "__args__", ()) + def get_origin(t: Type[Any]) -> Optional[Type[Any]]: + return getattr(t, "__origin__", None) + else: - from typing import get_args + from typing import get_args, get_origin pp = pprint.PrettyPrinter(indent=1, width=120, compact=True) @@ -48,7 +50,29 @@ def get_args(t: Type[Any]) -> Tuple[Any, ...]: _T_Streamable = TypeVar("_T_Streamable", bound="Streamable") -def dataclass_from_dict(klass, d): +# Caches to store the fields and (de)serialization methods for all available streamable classes. +FIELDS_FOR_STREAMABLE_CLASS: Dict[Type[Any], Dict[str, Type[Any]]] = {} +VALIDATE_FUNCTIONS_FOR_STREAMABLE_CLASS: Dict[Type[Any], List[Callable[[Any], None]]] = {} +STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS: Dict[Type[Any], List[Callable[[Any, BinaryIO], Any]]] = {} +PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS: Dict[Type[Any], List[Callable[[Any], Any]]] = {} + + +def is_type_List(f_type: Type[Any]) -> bool: + return get_origin(f_type) == list or f_type == list + + +def is_type_SpecificOptional(f_type: Type[Any]) -> bool: + """ + Returns true for types such as Optional[T], but not Optional, or T. + """ + return get_origin(f_type) == Union and get_args(f_type)[1]() is None + + +def is_type_Tuple(f_type: Type[Any]) -> bool: + return get_origin(f_type) == tuple or f_type == tuple + + +def dataclass_from_dict(klass: Type[Any], d: Any) -> Any: """ Converts a dictionary based on a dataclass, into an instance of that dataclass. Recursively goes through lists, optionals, and dictionaries. @@ -68,7 +92,8 @@ def dataclass_from_dict(klass, d): return tuple(klass_properties) elif dataclasses.is_dataclass(klass): # Type is a dataclass, data is a dictionary - fieldtypes = {f.name: f.type for f in dataclasses.fields(klass)} + hints = get_type_hints(klass) + fieldtypes = {f.name: hints.get(f.name, f.type) for f in dataclasses.fields(klass)} return klass(**{f: dataclass_from_dict(fieldtypes[f], d[f]) for f in d}) elif is_type_List(klass): # Type is a list, data is a list @@ -84,7 +109,7 @@ def dataclass_from_dict(klass, d): return klass(d) -def recurse_jsonify(d): +def recurse_jsonify(d: Any) -> Any: """ Makes bytes objects and unhashable types into strings with 0x, and makes large ints into strings. @@ -124,81 +149,6 @@ def recurse_jsonify(d): return d -STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS = {} -PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS = {} -FIELDS_FOR_STREAMABLE_CLASS = {} - - -def streamable(cls: Any): - """ - This is a decorator for class definitions. It applies the strictdataclass decorator, - which checks all types at construction. It also defines a simple serialization format, - and adds parse, from bytes, stream, and __bytes__ methods. - - The primitives are: - * Sized ints serialized in big endian format, e.g. uint64 - * Sized bytes serialized in big endian format, e.g. bytes32 - * BLS public keys serialized in bls format (48 bytes) - * BLS signatures serialized in bls format (96 bytes) - * bool serialized into 1 byte (0x01 or 0x00) - * bytes serialized as a 4 byte size prefix and then the bytes. - * ConditionOpcode is serialized as a 1 byte value. - * str serialized as a 4 byte size prefix and then the utf-8 representation in bytes. - - An item is one of: - * primitive - * Tuple[item1, .. itemx] - * List[item1, .. itemx] - * Optional[item] - * Custom item - - A streamable must be a Tuple at the root level (although a dataclass is used here instead). - Iters are serialized in the following way: - - 1. A tuple of x items is serialized by appending the serialization of each item. - 2. A List is serialized into a 4 byte size prefix (number of items) and the serialization of each item. - 3. An Optional is serialized into a 1 byte prefix of 0x00 or 0x01, and if it's one, it's followed by the - serialization of the item. - 4. A Custom item is serialized by calling the .parse method, passing in the stream of bytes into it. An example is - a CLVM program. - - All of the constituents must have parse/from_bytes, and stream/__bytes__ and therefore - be of fixed size. For example, int cannot be a constituent since it is not a fixed size, - whereas uint32 can be. - - Furthermore, a get_hash() member is added, which performs a serialization and a sha256. - - This class is used for deterministic serialization and hashing, for consensus critical - objects such as the block header. - - Make sure to use the Streamable class as a parent class when using the streamable decorator, - as it will allow linters to recognize the methods that are added by the decorator. Also, - use the @dataclass(frozen=True) decorator as well, for linters to recognize constructor - arguments. - """ - - cls1 = strictdataclass(cls) - t = type(cls.__name__, (cls1, Streamable), {}) - - stream_functions = [] - parse_functions = [] - try: - hints = get_type_hints(t) - fields = {field.name: hints.get(field.name, field.type) for field in dataclasses.fields(t)} - except Exception: - fields = {} - - FIELDS_FOR_STREAMABLE_CLASS[t] = fields - - for _, f_type in fields.items(): - stream_functions.append(cls.function_to_stream_one_item(f_type)) - parse_functions.append(cls.function_to_parse_one_item(f_type)) - - STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS[t] = stream_functions - PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS[t] = parse_functions - return t - - def parse_bool(f: BinaryIO) -> bool: bool_byte = f.read(1) assert bool_byte is not None and len(bool_byte) == 1 # Checks for EOF @@ -216,7 +166,7 @@ def parse_uint32(f: BinaryIO, byteorder: Literal["little", "big"] = "big") -> ui return uint32(int.from_bytes(size_bytes, byteorder)) -def write_uint32(f: BinaryIO, value: uint32, byteorder: Literal["little", "big"] = "big"): +def write_uint32(f: BinaryIO, value: uint32, byteorder: Literal["little", "big"] = "big") -> None: f.write(value.to_bytes(4, byteorder)) @@ -239,7 +189,7 @@ def parse_bytes(f: BinaryIO) -> bytes: def parse_list(f: BinaryIO, parse_inner_type_f: Callable[[BinaryIO], Any]) -> List[Any]: - full_list: List = [] + full_list: List[Any] = [] # wjb assert inner_type != get_args(List)[0] list_size = parse_uint32(f) for list_index in range(list_size): @@ -248,13 +198,13 @@ def parse_list(f: BinaryIO, parse_inner_type_f: Callable[[BinaryIO], Any]) -> Li def parse_tuple(f: BinaryIO, list_parse_inner_type_f: List[Callable[[BinaryIO], Any]]) -> Tuple[Any, ...]: - full_list = [] + full_list: List[Any] = [] for parse_f in list_parse_inner_type_f: full_list.append(parse_f(f)) return tuple(full_list) -def parse_size_hints(f: BinaryIO, f_type: Type, bytes_to_read: int) -> Any: +def parse_size_hints(f: BinaryIO, f_type: Type[Any], bytes_to_read: int) -> Any: bytes_read = f.read(bytes_to_read) assert bytes_read is not None and len(bytes_read) == bytes_to_read return f_type.from_bytes(bytes_read) @@ -298,14 +248,177 @@ def stream_str(item: Any, f: BinaryIO) -> None: f.write(str_bytes) +def validate_dataclass(validate_funcs: Dict[str, Callable[[Any], Any]], item: Dict[str, Any]) -> None: + for name, validate_func in validate_funcs.items(): + validate_func(item[name]) + + +def validate_optional(validate_inner: Callable[[Any], Any], item: Any) -> None: + if item is None: + return + validate_inner(item) + + +def validate_tuple(validate_funcs: List[Callable[[Any], Any]], items: Tuple[Any, ...]) -> None: + assert len(validate_funcs) == len(items) + if type(items) != tuple: + raise TypeError() + for i in range(len(items)): + validate_funcs[i](items[i]) + + +def validate_list(validate_func: Callable[[Any], Any], items: List[Any]) -> None: + if type(items) != list: + raise TypeError() + for i in range(len(items)): + validate_func(items[i]) + + +def validate_primitive(f_type: Type[Any], item: Any) -> None: + if type(item) != f_type: + raise TypeError(f"Expected {f_type}, got {type(item)}") + + +def function_to_validate_one_item(f_type: Type[Any]) -> Any: + if is_type_SpecificOptional(f_type): + inner_type = get_args(f_type)[0] + validate_inner_func = function_to_validate_one_item(inner_type) + return lambda item: validate_optional(validate_inner_func, item) + if is_type_List(f_type): + inner_type = get_args(f_type)[0] + validate_inner_func = function_to_validate_one_item(inner_type) + return lambda items: validate_list(validate_inner_func, items) + if is_type_Tuple(f_type): + validate_inner_funcs = [] + for inner_type in get_args(f_type): + validate_inner_funcs.append(function_to_validate_one_item(inner_type)) + return lambda items: validate_tuple(validate_inner_funcs, items) + + return lambda item: validate_primitive(f_type, item) + + +def streamable(cls: Type[_T_Streamable]) -> Type[_T_Streamable]: + """ + This decorator forces correct streamable protocol syntax/usage and populates the caches for types hints and + (de)serialization methods for all members of the class. The correct usage is: + + @streamable + @dataclass(frozen=True) + class Example(Streamable): + ... + + The order how the decorator are applied and the inheritance from Stremable are forced. The explicit inheritance is + required because mypy doesn't analyse the type returned by decorators, so we can't just inherit from inside the + decorator. The dataclass decorator is required to fetch type hints, let mypy validate constructor calls and restrict + direct modification of objects by `frozen=True`. + """ + + correct_usage_string: str = ( + "Correct usage is:\n\n@streamable\n@dataclass(frozen=True)\nclass Example(Streamable):\n\t..." + ) + + if not dataclasses.is_dataclass(cls): + raise SyntaxError(f"@dataclass(frozen=True) required first. {correct_usage_string}") + + try: + # Ignore mypy here because we especially want to access a not available member to test if + # the dataclass is frozen. + object.__new__(cls)._streamable_test_if_dataclass_frozen_ = None # type: ignore[attr-defined] + except dataclasses.FrozenInstanceError: + pass + else: + raise SyntaxError(f"dataclass needs to be frozen. {correct_usage_string}") + + if not issubclass(cls, Streamable): + raise SyntaxError(f"Streamable inheritance required. {correct_usage_string}") + + validate_functions = [] + stream_functions = [] + parse_functions = [] + try: + hints = get_type_hints(cls) + fields = {field.name: hints.get(field.name, field.type) for field in dataclasses.fields(cls)} + except Exception: + fields = {} + + FIELDS_FOR_STREAMABLE_CLASS[cls] = fields + + for _, f_type in fields.items(): + validate_functions.append(function_to_validate_one_item(f_type)) + stream_functions.append(cls.function_to_stream_one_item(f_type)) + parse_functions.append(cls.function_to_parse_one_item(f_type)) + + VALIDATE_FUNCTIONS_FOR_STREAMABLE_CLASS[cls] = validate_functions + STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS[cls] = stream_functions + PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS[cls] = parse_functions + return cls + + class Streamable: + """ + This class defines a simple serialization format, and adds methods to parse from/to bytes and json. It also + validates and parses all fields at construction in ยด__post_init__` to make sure all fields have the correct type + and can be streamed/parsed properly. + + The available primitives are: + * Sized ints serialized in big endian format, e.g. uint64 + * Sized bytes serialized in big endian format, e.g. bytes32 + * BLS public keys serialized in bls format (48 bytes) + * BLS signatures serialized in bls format (96 bytes) + * bool serialized into 1 byte (0x01 or 0x00) + * bytes serialized as a 4 byte size prefix and then the bytes. + * ConditionOpcode is serialized as a 1 byte value. + * str serialized as a 4 byte size prefix and then the utf-8 representation in bytes. + + An item is one of: + * primitive + * Tuple[item1, .. itemx] + * List[item1, .. itemx] + * Optional[item] + * Custom item + + A streamable must be a Tuple at the root level (although a dataclass is used here instead). + Iters are serialized in the following way: + + 1. A tuple of x items is serialized by appending the serialization of each item. + 2. A List is serialized into a 4 byte size prefix (number of items) and the serialization of each item. + 3. An Optional is serialized into a 1 byte prefix of 0x00 or 0x01, and if it's one, it's followed by the + serialization of the item. + 4. A Custom item is serialized by calling the .parse method, passing in the stream of bytes into it. An example is + a CLVM program. + + All of the constituents must have parse/from_bytes, and stream/__bytes__ and therefore + be of fixed size. For example, int cannot be a constituent since it is not a fixed size, + whereas uint32 can be. + + Furthermore, a get_hash() member is added, which performs a serialization and a sha256. + + This class is used for deterministic serialization and hashing, for consensus critical + objects such as the block header. + + Make sure to use the streamable decorator when inheriting from the Streamable class to prepare the streaming caches. + """ + + def __post_init__(self) -> None: + try: + fields = FIELDS_FOR_STREAMABLE_CLASS[type(self)] + validate_funcs = VALIDATE_FUNCTIONS_FOR_STREAMABLE_CLASS[type(self)] + except Exception: + fields = {} + validate_funcs = [] + data = self.__dict__ + for field, validate_func in zip(fields, validate_funcs): + if field not in data: + raise ValueError(f"Field {field} not present") + validate_func(data[field]) + @classmethod - def function_to_parse_one_item(cls, f_type: Type) -> Callable[[BinaryIO], Any]: + def function_to_parse_one_item(cls, f_type: Type[Any]) -> Callable[[BinaryIO], Any]: """ This function returns a function taking one argument `f: BinaryIO` that parses and returns a value of the given type. """ - inner_type: Type + inner_type: Type[Any] if f_type is bool: return parse_bool if is_type_SpecificOptional(f_type): @@ -313,7 +426,7 @@ def function_to_parse_one_item(cls, f_type: Type) -> Callable[[BinaryIO], Any]: parse_inner_type_f = cls.function_to_parse_one_item(inner_type) return lambda f: parse_optional(f, parse_inner_type_f) if hasattr(f_type, "parse"): - return f_type.parse + return lambda f: f_type.parse(f) if f_type == bytes: return parse_bytes if is_type_List(f_type): @@ -336,7 +449,7 @@ def parse(cls: Type[_T_Streamable], f: BinaryIO) -> _T_Streamable: # Create the object without calling __init__() to avoid unnecessary post-init checks in strictdataclass obj: _T_Streamable = object.__new__(cls) fields: Iterator[str] = iter(FIELDS_FOR_STREAMABLE_CLASS.get(cls, {})) - values: Iterator = (parse_f(f) for parse_f in PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS[cls]) + values: Iterator[Callable[[Any], Any]] = (parse_f(f) for parse_f in PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS[cls]) for field, value in zip(fields, values): object.__setattr__(obj, field, value) @@ -348,8 +461,8 @@ def parse(cls: Type[_T_Streamable], f: BinaryIO) -> _T_Streamable: return obj @classmethod - def function_to_stream_one_item(cls, f_type: Type) -> Callable[[Any, BinaryIO], Any]: - inner_type: Type + def function_to_stream_one_item(cls, f_type: Type[Any]) -> Callable[[Any, BinaryIO], Any]: + inner_type: Type[Any] if is_type_SpecificOptional(f_type): inner_type = get_args(f_type)[0] stream_inner_type_func = cls.function_to_stream_one_item(inner_type) @@ -410,9 +523,10 @@ def __str__(self: Any) -> str: def __repr__(self: Any) -> str: return pp.pformat(recurse_jsonify(dataclasses.asdict(self))) - def to_json_dict(self) -> Dict: - return recurse_jsonify(dataclasses.asdict(self)) + def to_json_dict(self) -> Dict[str, Any]: + # TODO, Remove the ignore, maybe use a wrapper with the correct type for recurse_jsonify? + return recurse_jsonify(dataclasses.asdict(self)) # type:ignore[no-any-return] @classmethod - def from_json_dict(cls: Any, json_dict: Dict) -> Any: + def from_json_dict(cls: Any, json_dict: Dict[str, Any]) -> Any: return dataclass_from_dict(cls, json_dict) diff --git a/chia/util/type_checking.py b/chia/util/type_checking.py deleted file mode 100644 index 8f9adf38e50d..000000000000 --- a/chia/util/type_checking.py +++ /dev/null @@ -1,103 +0,0 @@ -import dataclasses -import sys -from typing import Any, List, Optional, Tuple, Type, Union - -if sys.version_info < (3, 8): - - def get_args(t: Type[Any]) -> Tuple[Any, ...]: - return getattr(t, "__args__", ()) - - def get_origin(t: Type[Any]) -> Optional[Type[Any]]: - return getattr(t, "__origin__", None) - -else: - - from typing import get_args, get_origin - - -def is_type_List(f_type: Type) -> bool: - return get_origin(f_type) == list or f_type == list - - -def is_type_SpecificOptional(f_type) -> bool: - """ - Returns true for types such as Optional[T], but not Optional, or T. - """ - return get_origin(f_type) == Union and get_args(f_type)[1]() is None - - -def is_type_Tuple(f_type: Type) -> bool: - return get_origin(f_type) == tuple or f_type == tuple - - -def strictdataclass(cls: Any): - class _Local: - """ - Dataclass where all fields must be type annotated, and type checking is performed - at initialization, even recursively through Lists. Non-annotated fields are ignored. - Also, for any fields which have a type with .from_bytes(bytes) or constructor(bytes), - bytes can be passed in and the type can be constructed. - """ - - def parse_item(self, item: Any, f_name: str, f_type: Type) -> Any: - if is_type_List(f_type): - collected_list: List = [] - inner_type: Type = get_args(f_type)[0] - # wjb assert inner_type != get_args(List)[0] # type: ignore - if not is_type_List(type(item)): - raise ValueError(f"Wrong type for {f_name}, need a list.") - for el in item: - collected_list.append(self.parse_item(el, f_name, inner_type)) - return collected_list - if is_type_SpecificOptional(f_type): - if item is None: - return None - else: - inner_type: Type = get_args(f_type)[0] # type: ignore - return self.parse_item(item, f_name, inner_type) - if is_type_Tuple(f_type): - collected_list = [] - if not is_type_Tuple(type(item)) and not is_type_List(type(item)): - raise ValueError(f"Wrong type for {f_name}, need a tuple.") - if len(item) != len(get_args(f_type)): - raise ValueError(f"Wrong number of elements in tuple {f_name}.") - for i in range(len(item)): - inner_type = get_args(f_type)[i] - tuple_item = item[i] - collected_list.append(self.parse_item(tuple_item, f_name, inner_type)) - return tuple(collected_list) - if not isinstance(item, f_type): - try: - item = f_type(item) - except (TypeError, AttributeError, ValueError): - try: - item = f_type.from_bytes(item) - except Exception: - item = f_type.from_bytes(bytes(item)) - if not isinstance(item, f_type): - raise ValueError(f"Wrong type for {f_name}") - return item - - def __post_init__(self): - try: - fields = self.__annotations__ # pylint: disable=no-member - except Exception: - fields = {} - data = self.__dict__ - for (f_name, f_type) in fields.items(): - if f_name not in data: - raise ValueError(f"Field {f_name} not present") - try: - if not isinstance(data[f_name], f_type): - object.__setattr__(self, f_name, self.parse_item(data[f_name], f_name, f_type)) - except TypeError: - # Throws a TypeError because we cannot call isinstance for subscripted generics like Optional[int] - object.__setattr__(self, f_name, self.parse_item(data[f_name], f_name, f_type)) - - class NoTypeChecking: - __no_type_check__ = True - - cls1 = dataclasses.dataclass(cls, init=False, frozen=True) # type: ignore - if dataclasses.fields(cls1) == (): - return type(cls.__name__, (cls1, _Local, NoTypeChecking), {}) - return type(cls.__name__, (cls1, _Local), {}) diff --git a/chia/wallet/block_record.py b/chia/wallet/block_record.py index 81cc7c57c00f..8324592523db 100644 --- a/chia/wallet/block_record.py +++ b/chia/wallet/block_record.py @@ -6,8 +6,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class HeaderBlockRecord(Streamable): """ These are values that are stored in the wallet database, corresponding to information diff --git a/chia/wallet/cat_wallet/cat_info.py b/chia/wallet/cat_wallet/cat_info.py index 9e2d6eadf5cf..78c9fec5ed1a 100644 --- a/chia/wallet/cat_wallet/cat_info.py +++ b/chia/wallet/cat_wallet/cat_info.py @@ -7,8 +7,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CATInfo(Streamable): limitations_program_hash: bytes32 my_tail: Optional[Program] # this is the program @@ -16,8 +16,8 @@ class CATInfo(Streamable): # We used to store all of the lineage proofs here but it was very slow to serialize for a lot of transactions # so we moved it to CATLineageStore. We keep this around for migration purposes. -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class LegacyCATInfo(Streamable): limitations_program_hash: bytes32 my_tail: Optional[Program] # this is the program diff --git a/chia/wallet/did_wallet/did_info.py b/chia/wallet/did_wallet/did_info.py index 5b57402afa6b..d4a83a77e42c 100644 --- a/chia/wallet/did_wallet/did_info.py +++ b/chia/wallet/did_wallet/did_info.py @@ -9,8 +9,8 @@ from chia.types.blockchain_format.coin import Coin -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class DIDInfo(Streamable): origin_coin: Optional[Coin] # Coin ID of this coin is our DID backup_ids: List[bytes] diff --git a/chia/wallet/lineage_proof.py b/chia/wallet/lineage_proof.py index 177f2d127434..ec5d3119d378 100644 --- a/chia/wallet/lineage_proof.py +++ b/chia/wallet/lineage_proof.py @@ -7,8 +7,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class LineageProof(Streamable): parent_name: Optional[bytes32] = None inner_puzzle_hash: Optional[bytes32] = None diff --git a/chia/wallet/rl_wallet/rl_wallet.py b/chia/wallet/rl_wallet/rl_wallet.py index abfd7126f9b1..83489a6a56b6 100644 --- a/chia/wallet/rl_wallet/rl_wallet.py +++ b/chia/wallet/rl_wallet/rl_wallet.py @@ -34,8 +34,8 @@ from chia.wallet.wallet_info import WalletInfo -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RLInfo(Streamable): type: str admin_pubkey: Optional[bytes] diff --git a/chia/wallet/settings/settings_objects.py b/chia/wallet/settings/settings_objects.py index 9878a2c4eee1..27f6f92c6070 100644 --- a/chia/wallet/settings/settings_objects.py +++ b/chia/wallet/settings/settings_objects.py @@ -3,8 +3,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BackupInitialized(Streamable): """ Stores user decision regarding import of backup info diff --git a/chia/wallet/trade_record.py b/chia/wallet/trade_record.py index 5e0dd9656528..712d9fe9dd46 100644 --- a/chia/wallet/trade_record.py +++ b/chia/wallet/trade_record.py @@ -9,8 +9,8 @@ from chia.wallet.trading.trade_status import TradeStatus -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class TradeRecord(Streamable): """ Used for storing transaction data and status in wallets. diff --git a/chia/wallet/transaction_record.py b/chia/wallet/transaction_record.py index f43907f1ba25..b13b6d366010 100644 --- a/chia/wallet/transaction_record.py +++ b/chia/wallet/transaction_record.py @@ -12,8 +12,8 @@ from chia.wallet.util.transaction_type import TransactionType -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class TransactionRecord(Streamable): """ Used for storing transaction data and status in wallets. diff --git a/chia/wallet/wallet_info.py b/chia/wallet/wallet_info.py index 4567540f6df2..1d02bb6e7276 100644 --- a/chia/wallet/wallet_info.py +++ b/chia/wallet/wallet_info.py @@ -5,8 +5,8 @@ from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class WalletInfo(Streamable): """ This object represents the wallet data as it is stored in DB. @@ -24,8 +24,8 @@ class WalletInfo(Streamable): data: str -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class WalletInfoBackup(Streamable): """ Used for transforming list of WalletInfo objects into bytes. diff --git a/mypy.ini b/mypy.ini index 8dd59a99a268..eb0b115ca7ec 100644 --- a/mypy.ini +++ b/mypy.ini @@ -17,7 +17,7 @@ no_implicit_reexport = True strict_equality = True # list created by: venv/bin/mypy | sed -n 's/.py:.*//p' | sort | uniq | tr '/' '.' | tr '\n' ',' -[mypy-benchmarks.block_ref,benchmarks.block_store,benchmarks.coin_store,benchmarks.utils,build_scripts.installer-version,chia.clvm.spend_sim,chia.cmds.configure,chia.cmds.db,chia.cmds.db_upgrade_func,chia.cmds.farm_funcs,chia.cmds.init,chia.cmds.init_funcs,chia.cmds.keys,chia.cmds.keys_funcs,chia.cmds.passphrase,chia.cmds.passphrase_funcs,chia.cmds.plotnft,chia.cmds.plotnft_funcs,chia.cmds.plots,chia.cmds.plotters,chia.cmds.show,chia.cmds.start_funcs,chia.cmds.wallet,chia.cmds.wallet_funcs,chia.consensus.block_body_validation,chia.consensus.blockchain,chia.consensus.blockchain_interface,chia.consensus.block_creation,chia.consensus.block_header_validation,chia.consensus.block_record,chia.consensus.block_root_validation,chia.consensus.coinbase,chia.consensus.constants,chia.consensus.difficulty_adjustment,chia.consensus.get_block_challenge,chia.consensus.multiprocess_validation,chia.consensus.pos_quality,chia.consensus.vdf_info_computation,chia.daemon.client,chia.daemon.keychain_proxy,chia.daemon.keychain_server,chia.daemon.server,chia.farmer.farmer,chia.farmer.farmer_api,chia.full_node.block_height_map,chia.full_node.block_store,chia.full_node.bundle_tools,chia.full_node.coin_store,chia.full_node.full_node,chia.full_node.full_node_api,chia.full_node.full_node_store,chia.full_node.generator,chia.full_node.hint_store,chia.full_node.lock_queue,chia.full_node.mempool,chia.full_node.mempool_check_conditions,chia.full_node.mempool_manager,chia.full_node.pending_tx_cache,chia.full_node.sync_store,chia.full_node.weight_proof,chia.harvester.harvester,chia.harvester.harvester_api,chia.introducer.introducer,chia.introducer.introducer_api,chia.plotters.bladebit,chia.plotters.chiapos,chia.plotters.install_plotter,chia.plotters.madmax,chia.plotters.plotters,chia.plotters.plotters_util,chia.plotting.check_plots,chia.plotting.create_plots,chia.plotting.manager,chia.plotting.util,chia.pools.pool_config,chia.pools.pool_puzzles,chia.pools.pool_wallet,chia.pools.pool_wallet_info,chia.protocols.pool_protocol,chia.rpc.crawler_rpc_api,chia.rpc.farmer_rpc_api,chia.rpc.farmer_rpc_client,chia.rpc.full_node_rpc_api,chia.rpc.full_node_rpc_client,chia.rpc.harvester_rpc_api,chia.rpc.harvester_rpc_client,chia.rpc.rpc_client,chia.rpc.rpc_server,chia.rpc.timelord_rpc_api,chia.rpc.util,chia.rpc.wallet_rpc_api,chia.rpc.wallet_rpc_client,chia.seeder.crawler,chia.seeder.crawler_api,chia.seeder.crawl_store,chia.seeder.dns_server,chia.seeder.peer_record,chia.seeder.start_crawler,chia.server.address_manager,chia.server.address_manager_store,chia.server.connection_utils,chia.server.introducer_peers,chia.server.node_discovery,chia.server.peer_store_resolver,chia.server.rate_limits,chia.server.reconnect_task,chia.server.server,chia.server.ssl_context,chia.server.start_farmer,chia.server.start_full_node,chia.server.start_harvester,chia.server.start_introducer,chia.server.start_service,chia.server.start_timelord,chia.server.start_wallet,chia.server.upnp,chia.server.ws_connection,chia.simulator.full_node_simulator,chia.simulator.start_simulator,chia.ssl.create_ssl,chia.timelord.iters_from_block,chia.timelord.timelord,chia.timelord.timelord_api,chia.timelord.timelord_launcher,chia.timelord.timelord_state,chia.types.announcement,chia.types.blockchain_format.classgroup,chia.types.blockchain_format.coin,chia.types.blockchain_format.program,chia.types.blockchain_format.proof_of_space,chia.types.blockchain_format.tree_hash,chia.types.blockchain_format.vdf,chia.types.full_block,chia.types.header_block,chia.types.mempool_item,chia.types.name_puzzle_condition,chia.types.peer_info,chia.types.spend_bundle,chia.types.transaction_queue_entry,chia.types.unfinished_block,chia.types.unfinished_header_block,chia.util.api_decorators,chia.util.block_cache,chia.util.byte_types,chia.util.cached_bls,chia.util.check_fork_next_block,chia.util.chia_logging,chia.util.config,chia.util.db_wrapper,chia.util.dump_keyring,chia.util.file_keyring,chia.util.files,chia.util.hash,chia.util.ints,chia.util.json_util,chia.util.keychain,chia.util.keyring_wrapper,chia.util.log_exceptions,chia.util.lru_cache,chia.util.make_test_constants,chia.util.merkle_set,chia.util.network,chia.util.partial_func,chia.util.pip_import,chia.util.profiler,chia.util.safe_cancel_task,chia.util.service_groups,chia.util.ssl_check,chia.util.streamable,chia.util.struct_stream,chia.util.type_checking,chia.util.validate_alert,chia.wallet.block_record,chia.wallet.cat_wallet.cat_utils,chia.wallet.cat_wallet.cat_wallet,chia.wallet.cat_wallet.lineage_store,chia.wallet.chialisp,chia.wallet.did_wallet.did_wallet,chia.wallet.did_wallet.did_wallet_puzzles,chia.wallet.key_val_store,chia.wallet.lineage_proof,chia.wallet.payment,chia.wallet.puzzles.genesis_checkers,chia.wallet.puzzles.load_clvm,chia.wallet.puzzles.p2_conditions,chia.wallet.puzzles.p2_delegated_conditions,chia.wallet.puzzles.p2_delegated_puzzle,chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle,chia.wallet.puzzles.p2_m_of_n_delegate_direct,chia.wallet.puzzles.p2_puzzle_hash,chia.wallet.puzzles.prefarm.spend_prefarm,chia.wallet.puzzles.puzzle_utils,chia.wallet.puzzles.rom_bootstrap_generator,chia.wallet.puzzles.singleton_top_layer,chia.wallet.puzzles.tails,chia.wallet.rl_wallet.rl_wallet,chia.wallet.rl_wallet.rl_wallet_puzzles,chia.wallet.secret_key_store,chia.wallet.settings.user_settings,chia.wallet.trade_manager,chia.wallet.trade_record,chia.wallet.trading.offer,chia.wallet.trading.trade_store,chia.wallet.transaction_record,chia.wallet.util.debug_spend_bundle,chia.wallet.util.new_peak_queue,chia.wallet.util.peer_request_cache,chia.wallet.util.wallet_sync_utils,chia.wallet.wallet,chia.wallet.wallet_action_store,chia.wallet.wallet_blockchain,chia.wallet.wallet_coin_store,chia.wallet.wallet_interested_store,chia.wallet.wallet_node,chia.wallet.wallet_node_api,chia.wallet.wallet_pool_store,chia.wallet.wallet_puzzle_store,chia.wallet.wallet_state_manager,chia.wallet.wallet_sync_store,chia.wallet.wallet_transaction_store,chia.wallet.wallet_user_store,chia.wallet.wallet_weight_proof_handler,installhelper,tests.blockchain.blockchain_test_utils,tests.blockchain.test_blockchain,tests.blockchain.test_blockchain_transactions,tests.block_tools,tests.build-init-files,tests.build-workflows,tests.clvm.coin_store,tests.clvm.test_chialisp_deserialization,tests.clvm.test_clvm_compilation,tests.clvm.test_program,tests.clvm.test_puzzle_compression,tests.clvm.test_puzzles,tests.clvm.test_serialized_program,tests.clvm.test_singletons,tests.clvm.test_spend_sim,tests.conftest,tests.connection_utils,tests.core.cmds.test_keys,tests.core.consensus.test_pot_iterations,tests.core.custom_types.test_coin,tests.core.custom_types.test_proof_of_space,tests.core.custom_types.test_spend_bundle,tests.core.daemon.test_daemon,tests.core.full_node.full_sync.test_full_sync,tests.core.full_node.stores.test_block_store,tests.core.full_node.stores.test_coin_store,tests.core.full_node.stores.test_full_node_store,tests.core.full_node.stores.test_hint_store,tests.core.full_node.stores.test_sync_store,tests.core.full_node.test_address_manager,tests.core.full_node.test_block_height_map,tests.core.full_node.test_conditions,tests.core.full_node.test_full_node,tests.core.full_node.test_mempool,tests.core.full_node.test_mempool_performance,tests.core.full_node.test_node_load,tests.core.full_node.test_peer_store_resolver,tests.core.full_node.test_performance,tests.core.full_node.test_transactions,tests.core.make_block_generator,tests.core.node_height,tests.core.server.test_dos,tests.core.server.test_rate_limits,tests.core.ssl.test_ssl,tests.core.test_cost_calculation,tests.core.test_crawler_rpc,tests.core.test_daemon_rpc,tests.core.test_db_conversion,tests.core.test_farmer_harvester_rpc,tests.core.test_filter,tests.core.test_full_node_rpc,tests.core.test_merkle_set,tests.core.test_setproctitle,tests.core.util.test_cached_bls,tests.core.util.test_config,tests.core.util.test_file_keyring_synchronization,tests.core.util.test_files,tests.core.util.test_keychain,tests.core.util.test_keyring_wrapper,tests.core.util.test_lru_cache,tests.core.util.test_significant_bits,tests.core.util.test_streamable,tests.core.util.test_type_checking,tests.farmer_harvester.test_farmer_harvester,tests.generator.test_compression,tests.generator.test_generator_types,tests.generator.test_list_to_batches,tests.generator.test_rom,tests.generator.test_scan,tests.plotting.test_plot_manager,tests.pools.test_pool_cmdline,tests.pools.test_pool_config,tests.pools.test_pool_puzzles_lifecycle,tests.pools.test_pool_rpc,tests.pools.test_wallet_pool_store,tests.setup_nodes,tests.simulation.test_simulation,tests.time_out_assert,tests.tools.test_full_sync,tests.tools.test_run_block,tests.util.alert_server,tests.util.benchmark_cost,tests.util.blockchain,tests.util.build_network_protocol_files,tests.util.db_connection,tests.util.generator_tools_testing,tests.util.keyring,tests.util.key_tool,tests.util.misc,tests.util.network,tests.util.rpc,tests.util.test_full_block_utils,tests.util.test_lock_queue,tests.util.test_network_protocol_files,tests.util.test_struct_stream,tests.wallet.cat_wallet.test_cat_lifecycle,tests.wallet.cat_wallet.test_cat_wallet,tests.wallet.cat_wallet.test_offer_lifecycle,tests.wallet.cat_wallet.test_trades,tests.wallet.did_wallet.test_did,tests.wallet.did_wallet.test_did_rpc,tests.wallet.rl_wallet.test_rl_rpc,tests.wallet.rl_wallet.test_rl_wallet,tests.wallet.rpc.test_wallet_rpc,tests.wallet.simple_sync.test_simple_sync_protocol,tests.wallet.sync.test_wallet_sync,tests.wallet.test_bech32m,tests.wallet.test_chialisp,tests.wallet.test_puzzle_store,tests.wallet.test_singleton,tests.wallet.test_singleton_lifecycle,tests.wallet.test_singleton_lifecycle_fast,tests.wallet.test_taproot,tests.wallet.test_wallet,tests.wallet.test_wallet_blockchain,tests.wallet.test_wallet_interested_store,tests.wallet.test_wallet_key_val_store,tests.wallet.test_wallet_user_store,tests.wallet_tools,tests.weight_proof.test_weight_proof,tools.analyze-chain,tools.run_block,tools.test_full_sync] +[mypy-benchmarks.block_ref,benchmarks.block_store,benchmarks.coin_store,benchmarks.utils,build_scripts.installer-version,chia.clvm.spend_sim,chia.cmds.configure,chia.cmds.db,chia.cmds.db_upgrade_func,chia.cmds.farm_funcs,chia.cmds.init,chia.cmds.init_funcs,chia.cmds.keys,chia.cmds.keys_funcs,chia.cmds.passphrase,chia.cmds.passphrase_funcs,chia.cmds.plotnft,chia.cmds.plotnft_funcs,chia.cmds.plots,chia.cmds.plotters,chia.cmds.show,chia.cmds.start_funcs,chia.cmds.wallet,chia.cmds.wallet_funcs,chia.consensus.block_body_validation,chia.consensus.blockchain,chia.consensus.blockchain_interface,chia.consensus.block_creation,chia.consensus.block_header_validation,chia.consensus.block_record,chia.consensus.block_root_validation,chia.consensus.coinbase,chia.consensus.constants,chia.consensus.difficulty_adjustment,chia.consensus.get_block_challenge,chia.consensus.multiprocess_validation,chia.consensus.pos_quality,chia.consensus.vdf_info_computation,chia.daemon.client,chia.daemon.keychain_proxy,chia.daemon.keychain_server,chia.daemon.server,chia.farmer.farmer,chia.farmer.farmer_api,chia.full_node.block_height_map,chia.full_node.block_store,chia.full_node.bundle_tools,chia.full_node.coin_store,chia.full_node.full_node,chia.full_node.full_node_api,chia.full_node.full_node_store,chia.full_node.generator,chia.full_node.hint_store,chia.full_node.lock_queue,chia.full_node.mempool,chia.full_node.mempool_check_conditions,chia.full_node.mempool_manager,chia.full_node.pending_tx_cache,chia.full_node.sync_store,chia.full_node.weight_proof,chia.harvester.harvester,chia.harvester.harvester_api,chia.introducer.introducer,chia.introducer.introducer_api,chia.plotters.bladebit,chia.plotters.chiapos,chia.plotters.install_plotter,chia.plotters.madmax,chia.plotters.plotters,chia.plotters.plotters_util,chia.plotting.check_plots,chia.plotting.create_plots,chia.plotting.manager,chia.plotting.util,chia.pools.pool_config,chia.pools.pool_puzzles,chia.pools.pool_wallet,chia.pools.pool_wallet_info,chia.protocols.pool_protocol,chia.rpc.crawler_rpc_api,chia.rpc.farmer_rpc_api,chia.rpc.farmer_rpc_client,chia.rpc.full_node_rpc_api,chia.rpc.full_node_rpc_client,chia.rpc.harvester_rpc_api,chia.rpc.harvester_rpc_client,chia.rpc.rpc_client,chia.rpc.rpc_server,chia.rpc.timelord_rpc_api,chia.rpc.util,chia.rpc.wallet_rpc_api,chia.rpc.wallet_rpc_client,chia.seeder.crawler,chia.seeder.crawler_api,chia.seeder.crawl_store,chia.seeder.dns_server,chia.seeder.peer_record,chia.seeder.start_crawler,chia.server.address_manager,chia.server.address_manager_store,chia.server.connection_utils,chia.server.introducer_peers,chia.server.node_discovery,chia.server.peer_store_resolver,chia.server.rate_limits,chia.server.reconnect_task,chia.server.server,chia.server.ssl_context,chia.server.start_farmer,chia.server.start_full_node,chia.server.start_harvester,chia.server.start_introducer,chia.server.start_service,chia.server.start_timelord,chia.server.start_wallet,chia.server.upnp,chia.server.ws_connection,chia.simulator.full_node_simulator,chia.simulator.start_simulator,chia.ssl.create_ssl,chia.timelord.iters_from_block,chia.timelord.timelord,chia.timelord.timelord_api,chia.timelord.timelord_launcher,chia.timelord.timelord_state,chia.types.announcement,chia.types.blockchain_format.classgroup,chia.types.blockchain_format.coin,chia.types.blockchain_format.program,chia.types.blockchain_format.proof_of_space,chia.types.blockchain_format.tree_hash,chia.types.blockchain_format.vdf,chia.types.full_block,chia.types.header_block,chia.types.mempool_item,chia.types.name_puzzle_condition,chia.types.peer_info,chia.types.spend_bundle,chia.types.transaction_queue_entry,chia.types.unfinished_block,chia.types.unfinished_header_block,chia.util.api_decorators,chia.util.block_cache,chia.util.byte_types,chia.util.cached_bls,chia.util.check_fork_next_block,chia.util.chia_logging,chia.util.config,chia.util.db_wrapper,chia.util.dump_keyring,chia.util.file_keyring,chia.util.files,chia.util.hash,chia.util.ints,chia.util.json_util,chia.util.keychain,chia.util.keyring_wrapper,chia.util.log_exceptions,chia.util.lru_cache,chia.util.make_test_constants,chia.util.merkle_set,chia.util.network,chia.util.partial_func,chia.util.pip_import,chia.util.profiler,chia.util.safe_cancel_task,chia.util.service_groups,chia.util.ssl_check,chia.util.struct_stream,chia.util.validate_alert,chia.wallet.block_record,chia.wallet.cat_wallet.cat_utils,chia.wallet.cat_wallet.cat_wallet,chia.wallet.cat_wallet.lineage_store,chia.wallet.chialisp,chia.wallet.did_wallet.did_wallet,chia.wallet.did_wallet.did_wallet_puzzles,chia.wallet.key_val_store,chia.wallet.lineage_proof,chia.wallet.payment,chia.wallet.puzzles.genesis_checkers,chia.wallet.puzzles.load_clvm,chia.wallet.puzzles.p2_conditions,chia.wallet.puzzles.p2_delegated_conditions,chia.wallet.puzzles.p2_delegated_puzzle,chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle,chia.wallet.puzzles.p2_m_of_n_delegate_direct,chia.wallet.puzzles.p2_puzzle_hash,chia.wallet.puzzles.prefarm.spend_prefarm,chia.wallet.puzzles.puzzle_utils,chia.wallet.puzzles.rom_bootstrap_generator,chia.wallet.puzzles.singleton_top_layer,chia.wallet.puzzles.tails,chia.wallet.rl_wallet.rl_wallet,chia.wallet.rl_wallet.rl_wallet_puzzles,chia.wallet.secret_key_store,chia.wallet.settings.user_settings,chia.wallet.trade_manager,chia.wallet.trade_record,chia.wallet.trading.offer,chia.wallet.trading.trade_store,chia.wallet.transaction_record,chia.wallet.util.debug_spend_bundle,chia.wallet.util.new_peak_queue,chia.wallet.util.peer_request_cache,chia.wallet.util.wallet_sync_utils,chia.wallet.wallet,chia.wallet.wallet_action_store,chia.wallet.wallet_blockchain,chia.wallet.wallet_coin_store,chia.wallet.wallet_interested_store,chia.wallet.wallet_node,chia.wallet.wallet_node_api,chia.wallet.wallet_pool_store,chia.wallet.wallet_puzzle_store,chia.wallet.wallet_state_manager,chia.wallet.wallet_sync_store,chia.wallet.wallet_transaction_store,chia.wallet.wallet_user_store,chia.wallet.wallet_weight_proof_handler,installhelper,tests.blockchain.blockchain_test_utils,tests.blockchain.test_blockchain,tests.blockchain.test_blockchain_transactions,tests.block_tools,tests.build-init-files,tests.build-workflows,tests.clvm.coin_store,tests.clvm.test_chialisp_deserialization,tests.clvm.test_clvm_compilation,tests.clvm.test_program,tests.clvm.test_puzzle_compression,tests.clvm.test_puzzles,tests.clvm.test_serialized_program,tests.clvm.test_singletons,tests.clvm.test_spend_sim,tests.conftest,tests.connection_utils,tests.core.cmds.test_keys,tests.core.consensus.test_pot_iterations,tests.core.custom_types.test_coin,tests.core.custom_types.test_proof_of_space,tests.core.custom_types.test_spend_bundle,tests.core.daemon.test_daemon,tests.core.full_node.full_sync.test_full_sync,tests.core.full_node.stores.test_block_store,tests.core.full_node.stores.test_coin_store,tests.core.full_node.stores.test_full_node_store,tests.core.full_node.stores.test_hint_store,tests.core.full_node.stores.test_sync_store,tests.core.full_node.test_address_manager,tests.core.full_node.test_block_height_map,tests.core.full_node.test_conditions,tests.core.full_node.test_full_node,tests.core.full_node.test_mempool,tests.core.full_node.test_mempool_performance,tests.core.full_node.test_node_load,tests.core.full_node.test_peer_store_resolver,tests.core.full_node.test_performance,tests.core.full_node.test_transactions,tests.core.make_block_generator,tests.core.node_height,tests.core.server.test_dos,tests.core.server.test_rate_limits,tests.core.ssl.test_ssl,tests.core.test_cost_calculation,tests.core.test_crawler_rpc,tests.core.test_daemon_rpc,tests.core.test_db_conversion,tests.core.test_farmer_harvester_rpc,tests.core.test_filter,tests.core.test_full_node_rpc,tests.core.test_merkle_set,tests.core.test_setproctitle,tests.core.util.test_cached_bls,tests.core.util.test_config,tests.core.util.test_file_keyring_synchronization,tests.core.util.test_files,tests.core.util.test_keychain,tests.core.util.test_keyring_wrapper,tests.core.util.test_lru_cache,tests.core.util.test_significant_bits,tests.farmer_harvester.test_farmer_harvester,tests.generator.test_compression,tests.generator.test_generator_types,tests.generator.test_list_to_batches,tests.generator.test_rom,tests.generator.test_scan,tests.plotting.test_plot_manager,tests.pools.test_pool_cmdline,tests.pools.test_pool_config,tests.pools.test_pool_puzzles_lifecycle,tests.pools.test_pool_rpc,tests.pools.test_wallet_pool_store,tests.setup_nodes,tests.simulation.test_simulation,tests.time_out_assert,tests.tools.test_full_sync,tests.tools.test_run_block,tests.util.alert_server,tests.util.benchmark_cost,tests.util.blockchain,tests.util.build_network_protocol_files,tests.util.db_connection,tests.util.generator_tools_testing,tests.util.keyring,tests.util.key_tool,tests.util.misc,tests.util.network,tests.util.rpc,tests.util.test_full_block_utils,tests.util.test_lock_queue,tests.util.test_network_protocol_files,tests.util.test_struct_stream,tests.wallet.cat_wallet.test_cat_lifecycle,tests.wallet.cat_wallet.test_cat_wallet,tests.wallet.cat_wallet.test_offer_lifecycle,tests.wallet.cat_wallet.test_trades,tests.wallet.did_wallet.test_did,tests.wallet.did_wallet.test_did_rpc,tests.wallet.rl_wallet.test_rl_rpc,tests.wallet.rl_wallet.test_rl_wallet,tests.wallet.rpc.test_wallet_rpc,tests.wallet.simple_sync.test_simple_sync_protocol,tests.wallet.sync.test_wallet_sync,tests.wallet.test_bech32m,tests.wallet.test_chialisp,tests.wallet.test_puzzle_store,tests.wallet.test_singleton,tests.wallet.test_singleton_lifecycle,tests.wallet.test_singleton_lifecycle_fast,tests.wallet.test_taproot,tests.wallet.test_wallet,tests.wallet.test_wallet_blockchain,tests.wallet.test_wallet_interested_store,tests.wallet.test_wallet_key_val_store,tests.wallet.test_wallet_user_store,tests.wallet_tools,tests.weight_proof.test_weight_proof,tools.analyze-chain,tools.run_block,tools.test_full_sync] disallow_any_generics = False disallow_subclassing_any = False disallow_untyped_calls = False diff --git a/tests/core/util/test_streamable.py b/tests/core/util/test_streamable.py index 0d029eace28b..1ca2ed748ecc 100644 --- a/tests/core/util/test_streamable.py +++ b/tests/core/util/test_streamable.py @@ -1,9 +1,12 @@ -from dataclasses import dataclass -from typing import List, Optional, Tuple +from __future__ import annotations + import io +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple, Union from clvm_tools import binutils from pytest import raises +from typing_extensions import Literal from chia.protocols.wallet_protocol import RespondRemovals from chia.types.blockchain_format.coin import Coin @@ -11,26 +14,171 @@ from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.full_block import FullBlock from chia.types.weight_proof import SubEpochChallengeSegment -from chia.util.ints import uint8, uint32 +from chia.util.ints import uint8, uint32, uint64 from chia.util.streamable import ( Streamable, - streamable, + is_type_List, + is_type_SpecificOptional, parse_bool, - parse_uint32, - write_uint32, - parse_optional, parse_bytes, parse_list, - parse_tuple, + parse_optional, parse_size_hints, parse_str, + parse_tuple, + parse_uint32, + streamable, + write_uint32, ) from tests.setup_nodes import bt, test_constants -def test_basic(): +def test_int_not_supported() -> None: + with raises(NotImplementedError): + + @streamable + @dataclass(frozen=True) + class TestClassInt(Streamable): + a: int + + +def test_float_not_supported() -> None: + with raises(NotImplementedError): + + @streamable + @dataclass(frozen=True) + class TestClassFloat(Streamable): + a: float + + +def test_dict_not_suppported() -> None: + with raises(NotImplementedError): + + @streamable + @dataclass(frozen=True) + class TestClassDict(Streamable): + a: Dict[str, str] + + +@dataclass(frozen=True) +class DataclassOnly: + a: uint8 + + +def test_pure_dataclass_not_supported() -> None: + + with raises(NotImplementedError): + + @streamable + @dataclass(frozen=True) + class TestClassDataclass(Streamable): + a: DataclassOnly + + +class PlainClass: + a: uint8 + + +def test_plain_class_not_supported() -> None: + + with raises(NotImplementedError): + + @streamable + @dataclass(frozen=True) + class TestClassPlain(Streamable): + a: PlainClass + + +def test_basic_list() -> None: + a = [1, 2, 3] + assert is_type_List(type(a)) + assert is_type_List(List) + assert is_type_List(List[int]) + assert is_type_List(List[uint8]) + assert is_type_List(list) + assert not is_type_List(type(Tuple)) + assert not is_type_List(tuple) + assert not is_type_List(dict) + + +def test_not_lists() -> None: + assert not is_type_List(Dict) + + +def test_basic_optional() -> None: + assert is_type_SpecificOptional(type(Optional[int])) + assert is_type_SpecificOptional(type(Optional[Optional[int]])) + assert not is_type_SpecificOptional(type(List[int])) + + +def test_StrictDataClass() -> None: + @streamable + @dataclass(frozen=True) + class TestClass1(Streamable): + a: uint8 + b: str + + # we want to test invalid here, hence the ignore. + good: TestClass1 = TestClass1(24, "!@12") # type: ignore[arg-type] + assert TestClass1.__name__ == "TestClass1" + assert good + assert good.a == 24 + assert good.b == "!@12" + # we want to test invalid here, hence the ignore. + good2 = TestClass1(52, bytes([1, 2, 3])) # type: ignore[arg-type] + assert good2.b == str(bytes([1, 2, 3])) + + +def test_StrictDataClassBad() -> None: + @streamable + @dataclass(frozen=True) + class TestClass2(Streamable): + a: uint8 + b = 0 + + # we want to test invalid here, hence the ignore. + assert TestClass2(25) # type: ignore[arg-type] + + # we want to test invalid here, hence the ignore. + with raises(TypeError): + TestClass2(1, 2) # type: ignore[call-arg,arg-type] + + +def test_StrictDataClassLists() -> None: + @streamable + @dataclass(frozen=True) + class TestClass(Streamable): + a: List[uint8] + b: List[List[uint8]] + + # we want to test invalid here, hence the ignore. + assert TestClass([1, 2, 3], [[uint8(200), uint8(25)], [uint8(25)]]) # type: ignore[list-item] + + # we want to test invalid here, hence the ignore. + with raises(ValueError): + TestClass({"1": 1}, [[uint8(200), uint8(25)], [uint8(25)]]) # type: ignore[arg-type] + + # we want to test invalid here, hence the ignore. + with raises(ValueError): + TestClass([1, 2, 3], [uint8(200), uint8(25)]) # type: ignore[list-item] + + +def test_StrictDataClassOptional() -> None: + @streamable @dataclass(frozen=True) + class TestClass(Streamable): + a: Optional[uint8] + b: Optional[uint8] + c: Optional[Optional[uint8]] + d: Optional[Optional[uint8]] + + good = TestClass(12, None, 13, None) # type: ignore[arg-type] # we want to test invalid here, hence the ignore. + assert good + + +def test_basic() -> None: @streamable + @dataclass(frozen=True) class TestClass(Streamable): a: uint32 b: uint32 @@ -40,15 +188,16 @@ class TestClass(Streamable): f: Optional[uint32] g: Tuple[uint32, str, bytes] - a = TestClass(24, 352, [1, 2, 4], [[1, 2, 3], [3, 4]], 728, None, (383, "hello", b"goodbye")) + # we want to test invalid here, hence the ignore. + a = TestClass(24, 352, [1, 2, 4], [[1, 2, 3], [3, 4]], 728, None, (383, "hello", b"goodbye")) # type: ignore[arg-type,list-item] # noqa: E501 b: bytes = bytes(a) assert a == TestClass.from_bytes(b) -def test_variable_size(): - @dataclass(frozen=True) +def test_variable_size() -> None: @streamable + @dataclass(frozen=True) class TestClass2(Streamable): a: uint32 b: uint32 @@ -59,53 +208,55 @@ class TestClass2(Streamable): with raises(NotImplementedError): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClass3(Streamable): a: int -def test_json(): - block = bt.create_genesis_block(test_constants, bytes([0] * 32), b"0") +def test_json() -> None: + block = bt.create_genesis_block(test_constants, bytes32([0] * 32), uint64(0)) dict_block = block.to_json_dict() assert FullBlock.from_json_dict(dict_block) == block -def test_recursive_json(): - @dataclass(frozen=True) - @streamable - class TestClass1(Streamable): - a: List[uint32] +@streamable +@dataclass(frozen=True) +class TestClassRecursive1(Streamable): + a: List[uint32] - @dataclass(frozen=True) - @streamable - class TestClass2(Streamable): - a: uint32 - b: List[Optional[List[TestClass1]]] - c: bytes32 - tc1_a = TestClass1([uint32(1), uint32(2)]) - tc1_b = TestClass1([uint32(4), uint32(5)]) - tc1_c = TestClass1([uint32(7), uint32(8)]) +@streamable +@dataclass(frozen=True) +class TestClassRecursive2(Streamable): + a: uint32 + b: List[Optional[List[TestClassRecursive1]]] + c: bytes32 + - tc2 = TestClass2(uint32(5), [[tc1_a], [tc1_b, tc1_c], None], bytes32(bytes([1] * 32))) - assert TestClass2.from_json_dict(tc2.to_json_dict()) == tc2 +def test_recursive_json() -> None: + tc1_a = TestClassRecursive1([uint32(1), uint32(2)]) + tc1_b = TestClassRecursive1([uint32(4), uint32(5)]) + tc1_c = TestClassRecursive1([uint32(7), uint32(8)]) + tc2 = TestClassRecursive2(uint32(5), [[tc1_a], [tc1_b, tc1_c], None], bytes32(bytes([1] * 32))) + assert TestClassRecursive2.from_json_dict(tc2.to_json_dict()) == tc2 -def test_recursive_types(): + +def test_recursive_types() -> None: coin: Optional[Coin] = None l1 = [(bytes32([2] * 32), coin)] rr = RespondRemovals(uint32(1), bytes32([1] * 32), l1, None) RespondRemovals(rr.height, rr.header_hash, rr.coins, rr.proofs) -def test_ambiguous_deserialization_optionals(): +def test_ambiguous_deserialization_optionals() -> None: with raises(AssertionError): SubEpochChallengeSegment.from_bytes(b"\x00\x00\x00\x03\xff\xff\xff\xff") - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClassOptional(Streamable): a: Optional[uint8] @@ -117,9 +268,9 @@ class TestClassOptional(Streamable): TestClassOptional.from_bytes(bytes([1, 2])) -def test_ambiguous_deserialization_int(): - @dataclass(frozen=True) +def test_ambiguous_deserialization_int() -> None: @streamable + @dataclass(frozen=True) class TestClassUint(Streamable): a: uint32 @@ -128,9 +279,9 @@ class TestClassUint(Streamable): TestClassUint.from_bytes(b"\x00\x00") -def test_ambiguous_deserialization_list(): - @dataclass(frozen=True) +def test_ambiguous_deserialization_list() -> None: @streamable + @dataclass(frozen=True) class TestClassList(Streamable): a: List[uint8] @@ -139,9 +290,9 @@ class TestClassList(Streamable): TestClassList.from_bytes(bytes([0, 0, 100, 24])) -def test_ambiguous_deserialization_tuple(): - @dataclass(frozen=True) +def test_ambiguous_deserialization_tuple() -> None: @streamable + @dataclass(frozen=True) class TestClassTuple(Streamable): a: Tuple[uint8, str] @@ -150,9 +301,9 @@ class TestClassTuple(Streamable): TestClassTuple.from_bytes(bytes([0, 0, 100, 24])) -def test_ambiguous_deserialization_str(): - @dataclass(frozen=True) +def test_ambiguous_deserialization_str() -> None: @streamable + @dataclass(frozen=True) class TestClassStr(Streamable): a: str @@ -161,9 +312,9 @@ class TestClassStr(Streamable): TestClassStr.from_bytes(bytes([0, 0, 100, 24, 52])) -def test_ambiguous_deserialization_bytes(): - @dataclass(frozen=True) +def test_ambiguous_deserialization_bytes() -> None: @streamable + @dataclass(frozen=True) class TestClassBytes(Streamable): a: bytes @@ -178,9 +329,9 @@ class TestClassBytes(Streamable): TestClassBytes.from_bytes(bytes([0, 0, 0, 2, 52, 21])) -def test_ambiguous_deserialization_bool(): - @dataclass(frozen=True) +def test_ambiguous_deserialization_bool() -> None: @streamable + @dataclass(frozen=True) class TestClassBool(Streamable): a: bool @@ -192,9 +343,9 @@ class TestClassBool(Streamable): TestClassBool.from_bytes(bytes([1])) -def test_ambiguous_deserialization_program(): - @dataclass(frozen=True) +def test_ambiguous_deserialization_program() -> None: @streamable + @dataclass(frozen=True) class TestClassProgram(Streamable): a: Program @@ -206,16 +357,16 @@ class TestClassProgram(Streamable): TestClassProgram.from_bytes(bytes(program) + b"9") -def test_streamable_empty(): - @dataclass(frozen=True) +def test_streamable_empty() -> None: @streamable + @dataclass(frozen=True) class A(Streamable): pass assert A.from_bytes(bytes(A())) == A() -def test_parse_bool(): +def test_parse_bool() -> None: assert not parse_bool(io.BytesIO(b"\x00")) assert parse_bool(io.BytesIO(b"\x01")) @@ -230,7 +381,7 @@ def test_parse_bool(): parse_bool(io.BytesIO(b"\x02")) -def test_uint32(): +def test_uint32() -> None: assert parse_uint32(io.BytesIO(b"\x00\x00\x00\x00")) == 0 assert parse_uint32(io.BytesIO(b"\x00\x00\x00\x01")) == 1 assert parse_uint32(io.BytesIO(b"\x00\x00\x00\x01"), "little") == 16777216 @@ -238,7 +389,7 @@ def test_uint32(): assert parse_uint32(io.BytesIO(b"\x01\x00\x00\x00"), "little") == 1 assert parse_uint32(io.BytesIO(b"\xff\xff\xff\xff"), "little") == 4294967295 - def test_write(value, byteorder): + def test_write(value: int, byteorder: Union[Literal["little"], Literal["big"]]) -> None: f = io.BytesIO() write_uint32(f, uint32(value), byteorder) f.seek(0) @@ -259,7 +410,7 @@ def test_write(value, byteorder): parse_uint32(io.BytesIO(b"\x00\x00\x00")) -def test_parse_optional(): +def test_parse_optional() -> None: assert parse_optional(io.BytesIO(b"\x00"), parse_bool) is None assert parse_optional(io.BytesIO(b"\x01\x01"), parse_bool) assert not parse_optional(io.BytesIO(b"\x01\x00"), parse_bool) @@ -276,7 +427,7 @@ def test_parse_optional(): parse_optional(io.BytesIO(b"\xff\x00"), parse_bool) -def test_parse_bytes(): +def test_parse_bytes() -> None: assert parse_bytes(io.BytesIO(b"\x00\x00\x00\x00")) == b"" assert parse_bytes(io.BytesIO(b"\x00\x00\x00\x01\xff")) == b"\xff" @@ -302,7 +453,7 @@ def test_parse_bytes(): parse_bytes(io.BytesIO(b"\x00\x00\x02\x01" + b"a" * 512)) -def test_parse_list(): +def test_parse_list() -> None: assert parse_list(io.BytesIO(b"\x00\x00\x00\x00"), parse_bool) == [] assert parse_list(io.BytesIO(b"\x00\x00\x00\x01\x01"), parse_bool) == [True] @@ -323,7 +474,7 @@ def test_parse_list(): parse_list(io.BytesIO(b"\x00\x00\x00\x01\x02"), parse_bool) -def test_parse_tuple(): +def test_parse_tuple() -> None: assert parse_tuple(io.BytesIO(b""), []) == () assert parse_tuple(io.BytesIO(b"\x00\x00"), [parse_bool, parse_bool]) == (False, False) @@ -338,12 +489,12 @@ def test_parse_tuple(): parse_tuple(io.BytesIO(b"\x00"), [parse_bool, parse_bool]) -def test_parse_size_hints(): +def test_parse_size_hints() -> None: class TestFromBytes: b: bytes @classmethod - def from_bytes(cls, b): + def from_bytes(cls, b: bytes) -> TestFromBytes: ret = TestFromBytes() ret.b = b return ret @@ -356,7 +507,7 @@ def from_bytes(cls, b): class FailFromBytes: @classmethod - def from_bytes(cls, b): + def from_bytes(cls, b: bytes) -> FailFromBytes: raise ValueError() # error in underlying type @@ -364,7 +515,7 @@ def from_bytes(cls, b): parse_size_hints(io.BytesIO(b"1337"), FailFromBytes, 4) -def test_parse_str(): +def test_parse_str() -> None: assert parse_str(io.BytesIO(b"\x00\x00\x00\x00")) == "" assert parse_str(io.BytesIO(b"\x00\x00\x00\x01a")) == "a" @@ -388,3 +539,42 @@ def test_parse_str(): # EOF off by one with raises(AssertionError): parse_str(io.BytesIO(b"\x00\x00\x02\x01" + b"a" * 512)) + + +def test_wrong_decorator_order() -> None: + + with raises(SyntaxError): + + @dataclass(frozen=True) + @streamable + class WrongDecoratorOrder(Streamable): + pass + + +def test_dataclass_not_frozen() -> None: + + with raises(SyntaxError): + + @streamable + @dataclass(frozen=False) + class DataclassNotFrozen(Streamable): + pass + + +def test_dataclass_missing() -> None: + + with raises(SyntaxError): + + @streamable + class DataclassMissing(Streamable): + pass + + +def test_streamable_inheritance_missing() -> None: + + with raises(SyntaxError): + + @streamable + @dataclass(frozen=True) + class StreamableInheritanceMissing: # type: ignore[type-var] # we want to test invalid here, hence the ignore. + pass diff --git a/tests/core/util/test_type_checking.py b/tests/core/util/test_type_checking.py deleted file mode 100644 index 775affbe66b4..000000000000 --- a/tests/core/util/test_type_checking.py +++ /dev/null @@ -1,91 +0,0 @@ -import unittest -from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple - -from pytest import raises - -from chia.util.ints import uint8 -from chia.util.type_checking import is_type_List, is_type_SpecificOptional, strictdataclass - - -class TestIsTypeList(unittest.TestCase): - def test_basic_list(self): - a = [1, 2, 3] - assert is_type_List(type(a)) - assert is_type_List(List) - assert is_type_List(List[int]) - assert is_type_List(List[uint8]) - assert is_type_List(list) - assert not is_type_List(Tuple) - assert not is_type_List(tuple) - assert not is_type_List(dict) - - def test_not_lists(self): - assert not is_type_List(Dict) - - -class TestIsTypeSpecificOptional(unittest.TestCase): - def test_basic_optional(self): - assert is_type_SpecificOptional(Optional[int]) - assert is_type_SpecificOptional(Optional[Optional[int]]) - assert not is_type_SpecificOptional(List[int]) - - -class TestStrictClass(unittest.TestCase): - def test_StrictDataClass(self): - @dataclass(frozen=True) - @strictdataclass - class TestClass1: - a: int - b: str - - good: TestClass1 = TestClass1(24, "!@12") - assert TestClass1.__name__ == "TestClass1" - assert good - assert good.a == 24 - assert good.b == "!@12" - good2 = TestClass1(52, bytes([1, 2, 3])) - assert good2.b == str(bytes([1, 2, 3])) - - def test_StrictDataClassBad(self): - @dataclass(frozen=True) - @strictdataclass - class TestClass2: - a: int - b = 0 - - assert TestClass2(25) - - with raises(TypeError): - TestClass2(1, 2) - - def test_StrictDataClassLists(self): - @dataclass(frozen=True) - @strictdataclass - class TestClass: - a: List[int] - b: List[List[uint8]] - - assert TestClass([1, 2, 3], [[uint8(200), uint8(25)], [uint8(25)]]) - - with raises(ValueError): - TestClass({"1": 1}, [[uint8(200), uint8(25)], [uint8(25)]]) - - with raises(ValueError): - TestClass([1, 2, 3], [uint8(200), uint8(25)]) - - def test_StrictDataClassOptional(self): - @dataclass(frozen=True) - @strictdataclass - class TestClass: - a: Optional[int] - b: Optional[int] - c: Optional[Optional[int]] - d: Optional[Optional[int]] - - good = TestClass(12, None, 13, None) - assert good - - -if __name__ == "__main__": - unittest.main()