Merge #1042

1042: use a special 'ByronHybrid' index with range going from 0 to 2^32  r=KtorZ a=KtorZ

# Issue Number

<!-- Put here a reference to the issue this PR relates to and which requirements it tackles -->

#1041 

# Overview

<!-- Detail in a few bullet points the work accomplished in this PR -->

- [x] I have used a special 'ByronHybrid' index with range going from 0 to 2^32 for the address path.

# Comments

<!-- Additional comments or screenshots to attach if any -->

:question: Should we also worry about the account path :question: 

<!-- 
Don't forget to:

 ✓ Self-review your changes to make sure nothing unexpected slipped through
 ✓ Assign yourself to the PR
 ✓ Assign one or several reviewer(s)
 ✓ Once created, link this PR to its corresponding ticket
 ✓ Acknowledge any changes required to the Wiki
-->


Co-authored-by: KtorZ <[email protected]>

lib/core/src/Cardano/Byron/Codec/Cbor.hs

@@ -175,8 +175,8 @@ decodeAddressPayload = do
 decodeAddressDerivationPath
     :: Passphrase "addr-derivation-payload"
     -> CBOR.Decoder s (Maybe
-        ( Index 'Hardened 'AccountK
-        , Index 'Hardened 'AddressK
+        ( Index 'WholeDomain 'AccountK
+        , Index 'WholeDomain 'AddressK
         ))
 decodeAddressDerivationPath pwd = do
     _ <- CBOR.decodeListLenCanonicalOf 3
@@ -209,8 +209,8 @@ decodeDerivationPathAttr
     :: Passphrase "addr-derivation-payload"
     -> [(Word8, ByteString)]
     -> CBOR.Decoder s (Maybe
-        ( Index 'Hardened 'AccountK
-        , Index 'Hardened 'AddressK
+        ( Index 'WholeDomain 'AccountK
+        , Index 'WholeDomain 'AddressK
         ))
 decodeDerivationPathAttr pwd attrs = do
     case lookup derPathTag attrs of
@@ -223,8 +223,8 @@ decodeDerivationPathAttr pwd attrs = do
   where
     derPathTag = 1
     decoder :: CBOR.Decoder s (Maybe
-        ( Index 'Hardened 'AccountK
-        , Index 'Hardened 'AddressK
+        ( Index 'WholeDomain 'AccountK
+        , Index 'WholeDomain 'AddressK
         ))
     decoder = do
         bytes <- CBOR.decodeBytes
@@ -237,30 +237,20 @@ decodeDerivationPathAttr pwd attrs = do
 -- Opposite of 'encodeDerivationPath'.
 decodeDerivationPath
     :: CBOR.Decoder s
-        ( Index 'Hardened 'AccountK
-        , Index 'Hardened 'AddressK
+        ( Index 'WholeDomain 'AccountK
+        , Index 'WholeDomain 'AddressK
         )
 decodeDerivationPath = do
     ixs <- decodeListIndef CBOR.decodeWord32
     case ixs of
-        [acctIx, addrIx] | isValidIx acctIx && isValidIx addrIx ->
+        [acctIx, addrIx] ->
             pure (toEnum $ fromIntegral acctIx, toEnum $ fromIntegral addrIx)
-        [_, _] ->
-            fail $ mconcat
-                [ "decodeDerivationPath: indexes out of ranges: "
-                , show ixs
-                ]
         _ ->
             fail $ mconcat
                 [ "decodeDerivationPath: invalid derivation path payload: "
                 , "expected two indexes but got: "
                 , show ixs
                 ]
-  where
-    isValidIx i =
-        i >= getIndex (minBound @(Index 'Hardened _))
-        &&
-        i <= getIndex (maxBound @(Index 'Hardened _))
 
 decodeBlockHeader :: CBOR.Decoder s BlockHeader
 decodeBlockHeader = do
@@ -622,8 +612,8 @@ encodeProtocolMagicAttr pm = mempty
 -- NOTE: The caller must ensure that the passphrase length is 32 bytes.
 encodeDerivationPathAttr
     :: Passphrase "addr-derivation-payload"
-    -> Index 'Hardened 'AccountK
-    -> Index 'Hardened 'AddressK
+    -> Index 'WholeDomain 'AccountK
+    -> Index 'WholeDomain 'AddressK
     -> CBOR.Encoding
 encodeDerivationPathAttr pwd acctIx addrIx = mempty
     <> CBOR.encodeWord8 1 -- Tag for 'DerivationPath' attribute
@@ -632,8 +622,8 @@ encodeDerivationPathAttr pwd acctIx addrIx = mempty
     path = encodeDerivationPath acctIx addrIx
 
 encodeDerivationPath
-    :: Index 'Hardened 'AccountK
-    -> Index 'Hardened 'AddressK
+    :: Index 'WholeDomain 'AccountK
+    -> Index 'WholeDomain 'AddressK
     -> CBOR.Encoding
 encodeDerivationPath (Index acctIx) (Index addrIx) = mempty
     <> CBOR.encodeListLenIndef

lib/core/src/Cardano/Wallet/DB/Sqlite.hs

@@ -985,7 +985,7 @@ selectSeqStatePendingIxs wid =
 -- | Type alias for the index -> address map so that lines do not exceed 80
 -- characters in width.
 type RndStateAddresses = Map
-    (W.Index 'W.Hardened 'W.AccountK, W.Index 'W.Hardened 'W.AddressK)
+    (W.Index 'W.WholeDomain 'W.AccountK, W.Index 'W.WholeDomain 'W.AddressK)
     W.Address
 
 -- Persisting 'RndState' requires that the wallet root key has already been

lib/core/src/Cardano/Wallet/Primitive/AddressDerivation.hs

@@ -216,6 +216,10 @@ instance Bounded (Index 'Soft level) where
     minBound = Index minBound
     maxBound = let (Index ix) = minBound @(Index 'Hardened _) in Index (ix - 1)
 
+instance Bounded (Index 'WholeDomain level) where
+    minBound = Index minBound
+    maxBound = Index maxBound
+
 instance Enum (Index 'Hardened level) where
     fromEnum (Index ix) = fromIntegral ix
     toEnum ix
@@ -232,11 +236,25 @@ instance Enum (Index 'Soft level) where
         | otherwise =
             Index (fromIntegral ix)
 
+instance Enum (Index 'WholeDomain level) where
+    fromEnum (Index ix) = fromIntegral ix
+    toEnum ix
+        | Index (fromIntegral ix) > maxBound @(Index 'WholeDomain _) =
+            error "[email protected]: bad argument"
+        | otherwise =
+            Index (fromIntegral ix)
+
 instance Buildable (Index derivationType level) where
     build (Index ix) = fromString (show ix)
 
 -- | Type of derivation that should be used with the given indexes.
-data DerivationType = Hardened | Soft
+--
+-- In theory, we should only consider two derivation types: soft and hard.
+--
+-- However, historically, addresses in Cardano used to be generated across the
+-- both soft and hard domain. We therefore introduce a 'WholeDomain' derivation
+-- type that is the exact union of `Hardened` and `Soft`.
+data DerivationType = Hardened | Soft | WholeDomain
 
 -- | An interface for doing hard derivations from the root private key
 class HardDerivation (key :: Depth -> * -> *) where

lib/core/src/Cardano/Wallet/Primitive/AddressDerivation/Byron.hs

@@ -126,10 +126,10 @@ type family DerivationPath (depth :: Depth) :: * where
         ()
     -- The account key is generated from the root key and account index.
     DerivationPath 'AccountK =
-        Index 'Hardened 'AccountK
+        Index 'WholeDomain 'AccountK
     -- The address key is generated from the account key and address index.
     DerivationPath 'AddressK =
-        (Index 'Hardened 'AccountK, Index 'Hardened 'AddressK)
+        (Index 'WholeDomain 'AccountK, Index 'WholeDomain 'AddressK)
 
 instance WalletKey ByronKey where
     changePassphrase = changePassphraseRnd
@@ -314,7 +314,7 @@ changePassphraseRnd (Passphrase oldPwd) (Passphrase newPwd) key = ByronKey
 deriveAccountPrivateKey
     :: Passphrase "encryption"
     -> ByronKey 'RootK XPrv
-    -> Index 'Hardened 'AccountK
+    -> Index 'WholeDomain 'AccountK
     -> ByronKey 'AccountK XPrv
 deriveAccountPrivateKey (Passphrase pwd) masterKey idx@(Index accIx) = ByronKey
     { getKey = deriveXPrv DerivationScheme1 pwd (getKey masterKey) accIx
@@ -332,7 +332,7 @@ deriveAccountPrivateKey (Passphrase pwd) masterKey idx@(Index accIx) = ByronKey
 deriveAddressPrivateKey
     :: Passphrase "encryption"
     -> ByronKey 'AccountK XPrv
-    -> Index 'Hardened 'AddressK
+    -> Index 'WholeDomain 'AddressK
     -> ByronKey 'AddressK XPrv
 deriveAddressPrivateKey (Passphrase pwd) accountKey idx@(Index addrIx) = ByronKey
     { getKey = deriveXPrv DerivationScheme1 pwd (getKey accountKey) addrIx

lib/core/src/Cardano/Wallet/Primitive/AddressDiscovery/Random.hs

@@ -76,7 +76,7 @@ import qualified Data.Set as Set
 data RndState (network :: NetworkDiscriminant) = RndState
     { hdPassphrase :: Passphrase "addr-derivation-payload"
     -- ^ The HD derivation passphrase
-    , accountIndex :: Index 'Hardened 'AccountK
+    , accountIndex :: Index 'WholeDomain 'AccountK
     -- ^ The account index used for address _generation_ in this wallet. Note
     -- that addresses will be _discovered_ from any and all account indices,
     -- regardless of this value.
@@ -109,7 +109,7 @@ instance Buildable (RndState network) where
         <> indentF 4 ("Change addresses: " <> blockMapF' tupleF build pending)
 
 -- | Shortcut type alias for HD random address derivation path.
-type DerivationPath = (Index 'Hardened 'AccountK, Index 'Hardened 'AddressK)
+type DerivationPath = (Index 'WholeDomain 'AccountK, Index 'WholeDomain 'AddressK)
 
 -- An address is considered to belong to the 'RndState' wallet if it can be decoded
 -- as a Byron HD random address, and where the wallet key can be used to decrypt
@@ -177,7 +177,7 @@ unavailablePaths st = Map.keysSet $ addresses st <> pendingAddresses st
 -- account's address space is used up. We may have to improve it later.
 findUnusedPath
     :: StdGen
-    -> Index 'Hardened 'AccountK
+    -> Index 'WholeDomain 'AccountK
     -> Set DerivationPath
     -> (DerivationPath, StdGen)
 findUnusedPath g accIx used
@@ -188,7 +188,7 @@ findUnusedPath g accIx used
     (addrIx, gen') = randomIndex g
 
 randomIndex
-    :: forall ix g. (RandomGen g, ix ~ Index 'Hardened 'AddressK)
+    :: forall ix g. (RandomGen g, ix ~ Index 'WholeDomain 'AddressK)
     => g
     -> (ix, g)
 randomIndex g = (Index ix, g')

lib/core/test/unit/Cardano/Byron/Codec/CborSpec.hs

@@ -174,8 +174,8 @@ decodeDerivationPathTest DecodeDerivationPath{..} =
 prop_derivationPathRoundTrip
     :: Passphrase "addr-derivation-payload"
     -> Passphrase "addr-derivation-payload"
-    -> Index 'Hardened 'AccountK
-    -> Index 'Hardened 'AddressK
+    -> Index 'WholeDomain 'AccountK
+    -> Index 'WholeDomain 'AddressK
     -> Property
 prop_derivationPathRoundTrip pwd pwd' acctIx addrIx =
     let
@@ -199,11 +199,11 @@ instance {-# OVERLAPS #-} Arbitrary (Passphrase "addr-derivation-payload") where
         bytes <- BS.pack <$> vectorOf 32 arbitrary
         return $ Passphrase $ BA.convert bytes
 
-instance Arbitrary (Index 'Hardened 'AddressK) where
+instance Arbitrary (Index 'WholeDomain 'AddressK) where
     shrink _ = []
     arbitrary = arbitraryBoundedEnum
 
-instance Arbitrary (Index 'Hardened 'AccountK) where
+instance Arbitrary (Index 'WholeDomain 'AccountK) where
     shrink _ = []
     arbitrary = arbitraryBoundedEnum
 

lib/core/test/unit/Cardano/Wallet/DB/Arbitrary.hs

@@ -383,11 +383,11 @@ instance Arbitrary (Index 'Soft 'AddressK) where
     shrink _ = []
     arbitrary = arbitraryBoundedEnum
 
-instance Arbitrary (Index 'Hardened 'AccountK) where
+instance Arbitrary (Index 'WholeDomain 'AccountK) where
     shrink _ = []
     arbitrary = arbitraryBoundedEnum
 
-instance Arbitrary (Index 'Hardened 'AddressK) where
+instance Arbitrary (Index 'WholeDomain 'AddressK) where
     shrink _ = []
     arbitrary = arbitraryBoundedEnum
 

lib/core/test/unit/Cardano/Wallet/Primitive/AddressDerivation/ByronSpec.hs

@@ -64,8 +64,8 @@ spec = do
 prop_keyDerivation
     :: Passphrase "seed"
     -> Passphrase "encryption"
-    -> Index 'Hardened 'AccountK
-    -> Index 'Hardened 'AddressK
+    -> Index 'WholeDomain 'AccountK
+    -> Index 'WholeDomain 'AddressK
     -> Property
 prop_keyDerivation seed encPwd accIx addrIx =
     rndKey `seq` property () -- NOTE Making sure this doesn't throw

lib/core/test/unit/Cardano/Wallet/Primitive/AddressDerivationSpec.hs

@@ -207,7 +207,7 @@ prop_predMinBoundSoftIx :: Property
 prop_predMinBoundSoftIx = expectFailure $
     property $ pred (minBound @(Index 'Soft _)) `seq` ()
 
-prop_roundtripEnumIndexHard :: Index 'Hardened 'AccountK -> Property
+prop_roundtripEnumIndexHard :: Index 'WholeDomain 'AccountK -> Property
 prop_roundtripEnumIndexHard ix =
     (toEnum . fromEnum) ix === ix .&&. (toEnum . fromEnum . getIndex) ix === ix
 
@@ -259,11 +259,15 @@ instance Arbitrary (Index 'Soft 'AddressK) where
     shrink _ = []
     arbitrary = arbitraryBoundedEnum
 
-instance Arbitrary (Index 'Hardened 'AddressK) where
+instance Arbitrary (Index 'Hardened 'AccountK) where
     shrink _ = []
     arbitrary = arbitraryBoundedEnum
 
-instance Arbitrary (Index 'Hardened 'AccountK) where
+instance Arbitrary (Index 'WholeDomain 'AddressK) where
+    shrink _ = []
+    arbitrary = arbitraryBoundedEnum
+
+instance Arbitrary (Index 'WholeDomain 'AccountK) where
     shrink _ = []
     arbitrary = arbitraryBoundedEnum
 

lib/core/test/unit/Cardano/Wallet/Primitive/AddressDiscovery/RandomSpec.hs

@@ -241,7 +241,7 @@ data Rnd = Rnd
 prop_derivedKeysAreOurs
     :: Rnd
     -> Rnd
-    -> Index 'Hardened 'AddressK
+    -> Index 'WholeDomain 'AddressK
     -> Property
 prop_derivedKeysAreOurs
     (Rnd st@(RndState _ accIx _ _ _) rk pwd) (Rnd st' _ _) addrIx =
@@ -254,7 +254,7 @@ prop_derivedKeysAreOurs
 prop_derivedKeysAreOwned
     :: Rnd
     -> Rnd
-    -> Index 'Hardened 'AddressK
+    -> Index 'WholeDomain 'AddressK
     -> Property
 prop_derivedKeysAreOwned
     (Rnd st@(RndState _ accIx _ _ _) rk pwd)
@@ -281,7 +281,7 @@ prop_changeAddressesBelongToUs
 
 prop_forbiddenAddreses
     :: Rnd
-    -> Index 'Hardened 'AddressK
+    -> Index 'WholeDomain 'AddressK
     -> Property
 prop_forbiddenAddreses (Rnd st@(RndState _ accIx _ _ _) rk pwd) addrIx = conjoin
     [ (Set.notMember addr (forbidden st))

lib/core/test/unit/Cardano/Wallet/Primitive/AddressDiscoverySpec.hs

@@ -15,7 +15,7 @@ import Prelude
 
 import Cardano.Wallet.Primitive.AddressDerivation
     ( Depth (AccountK, AddressK, RootK)
-    , DerivationType (Hardened)
+    , DerivationType (..)
     , Index
     , NetworkDiscriminant (..)
     , Passphrase (..)
@@ -76,8 +76,8 @@ prop_derivedKeysAreOurs
     :: forall (n :: NetworkDiscriminant). (PaymentAddress n ByronKey)
     => Passphrase "seed"
     -> Passphrase "encryption"
-    -> Index 'Hardened 'AccountK
-    -> Index 'Hardened 'AddressK
+    -> Index 'WholeDomain 'AccountK
+    -> Index 'WholeDomain 'AddressK
     -> ByronKey 'RootK XPrv
     -> Property
 prop_derivedKeysAreOurs seed encPwd accIx addrIx rk' =
@@ -94,11 +94,11 @@ prop_derivedKeysAreOurs seed encPwd accIx addrIx rk' =
                              Arbitrary Instances
 -------------------------------------------------------------------------------}
 
-instance Arbitrary (Index 'Hardened 'AccountK) where
+instance Arbitrary (Index 'WholeDomain 'AccountK) where
     shrink _ = []
     arbitrary = arbitraryBoundedEnum
 
-instance Arbitrary (Index 'Hardened 'AddressK) where
+instance Arbitrary (Index 'WholeDomain 'AddressK) where
     shrink _ = []
     arbitrary = arbitraryBoundedEnum