From 8c49df7d8771277af9ee0bc6a2a649e579dca95c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Apr 2021 19:15:40 +0200 Subject: [PATCH 01/10] miner: Remote storage on market node --- api/api_storage.go | 24 ++ api/cbor_gen.go | 379 +++++++++++++++++ api/proxy_gen.go | 10 + build/openrpc/full.json.gz | Bin 22483 -> 22473 bytes build/openrpc/miner.json.gz | Bin 7848 -> 8012 bytes build/openrpc/worker.json.gz | Bin 2577 -> 2579 bytes cmd/lotus-storage-miner/init.go | 16 +- documentation/en/api-v0-methods-miner.md | 25 ++ .../sector-storage/ffiwrapper/partialfile.go | 24 +- .../sector-storage/ffiwrapper/sealer_cgo.go | 12 +- extern/sector-storage/manager.go | 91 +---- extern/sector-storage/piece_provider.go | 109 +++++ extern/sector-storage/stores/http_handler.go | 125 +++++- extern/sector-storage/stores/local.go | 4 +- extern/sector-storage/stores/remote.go | 144 +++++++ extern/sector-storage/storiface/ffi.go | 8 + extern/storage-sealing/cbor_gen.go | 386 +----------------- extern/storage-sealing/gen/main.go | 2 - extern/storage-sealing/input.go | 19 +- .../storage-sealing/precommit_policy_test.go | 17 +- extern/storage-sealing/sealing.go | 2 +- extern/storage-sealing/types.go | 23 +- gen/main.go | 2 + markets/retrievaladapter/provider.go | 57 ++- markets/storageadapter/provider.go | 12 +- node/builder.go | 5 +- node/impl/storminer.go | 65 +-- node/modules/storageminer.go | 40 +- storage/sealing.go | 80 +++- storage/sectorblocks/blocks.go | 22 +- 30 files changed, 1030 insertions(+), 673 deletions(-) create mode 100644 extern/sector-storage/piece_provider.go diff --git a/api/api_storage.go b/api/api_storage.go index 1131f45a0fe..99f0696832b 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -54,6 +54,8 @@ type StorageMiner interface { // Get the status of a given sector by ID SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (SectorInfo, error) //perm:read + SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error //perm:admin + // List all staged sectors SectorsList(context.Context) ([]abi.SectorNumber, error) //perm:read @@ -289,3 +291,25 @@ type PendingDealInfo struct { PublishPeriodStart time.Time PublishPeriod time.Duration } + +type SectorOffset struct { + Sector abi.SectorNumber + Offset abi.PaddedPieceSize +} + +// DealInfo is a tuple of deal identity and its schedule +type PieceDealInfo struct { + PublishCid *cid.Cid + DealID abi.DealID + DealProposal *market.DealProposal + DealSchedule DealSchedule + KeepUnsealed bool +} + +// DealSchedule communicates the time interval of a storage deal. The deal must +// appear in a sealed (proven) sector no later than StartEpoch, otherwise it +// is invalid. +type DealSchedule struct { + StartEpoch abi.ChainEpoch + EndEpoch abi.ChainEpoch +} diff --git a/api/cbor_gen.go b/api/cbor_gen.go index 808e516ad62..4434b45ede9 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -8,6 +8,7 @@ import ( "sort" abi "github.com/filecoin-project/go-state-types/abi" + market "github.com/filecoin-project/specs-actors/actors/builtin/market" paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -738,3 +739,381 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { return nil } +func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{165}); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.PublishCid (cid.Cid) (struct) + if len("PublishCid") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PublishCid\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("PublishCid")); err != nil { + return err + } + + if t.PublishCid == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil { + return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err) + } + } + + // t.DealID (abi.DealID) (uint64) + if len("DealID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealID\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("DealID")); err != nil { + return err + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { + return err + } + + // t.DealProposal (market.DealProposal) (struct) + if len("DealProposal") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealProposal\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("DealProposal")); err != nil { + return err + } + + if err := t.DealProposal.MarshalCBOR(w); err != nil { + return err + } + + // t.DealSchedule (api.DealSchedule) (struct) + if len("DealSchedule") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealSchedule\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("DealSchedule")); err != nil { + return err + } + + if err := t.DealSchedule.MarshalCBOR(w); err != nil { + return err + } + + // t.KeepUnsealed (bool) (bool) + if len("KeepUnsealed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.KeepUnsealed); err != nil { + return err + } + return nil +} + +func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error { + *t = PieceDealInfo{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("PieceDealInfo: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.PublishCid (cid.Cid) (struct) + case "PublishCid": + + { + + b, err := br.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := br.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err) + } + + t.PublishCid = &c + } + + } + // t.DealID (abi.DealID) (uint64) + case "DealID": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = abi.DealID(extra) + + } + // t.DealProposal (market.DealProposal) (struct) + case "DealProposal": + + { + + b, err := br.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := br.UnreadByte(); err != nil { + return err + } + t.DealProposal = new(market.DealProposal) + if err := t.DealProposal.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err) + } + } + + } + // t.DealSchedule (api.DealSchedule) (struct) + case "DealSchedule": + + { + + if err := t.DealSchedule.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err) + } + + } + // t.KeepUnsealed (bool) (bool) + case "KeepUnsealed": + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.KeepUnsealed = false + case 21: + t.KeepUnsealed = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *DealSchedule) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{162}); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.StartEpoch (abi.ChainEpoch) (int64) + if len("StartEpoch") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"StartEpoch\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("StartEpoch")); err != nil { + return err + } + + if t.StartEpoch >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil { + return err + } + } + + // t.EndEpoch (abi.ChainEpoch) (int64) + if len("EndEpoch") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"EndEpoch\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("EndEpoch")); err != nil { + return err + } + + if t.EndEpoch >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil { + return err + } + } + return nil +} + +func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error { + *t = DealSchedule{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("DealSchedule: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.StartEpoch (abi.ChainEpoch) (int64) + case "StartEpoch": + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.StartEpoch = abi.ChainEpoch(extraI) + } + // t.EndEpoch (abi.ChainEpoch) (int64) + case "EndEpoch": + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.EndEpoch = abi.ChainEpoch(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/api/proxy_gen.go b/api/proxy_gen.go index bfaaade94cd..ba284651e2e 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -679,6 +679,8 @@ type StorageMinerStruct struct { SectorsSummary func(p0 context.Context) (map[SectorState]int, error) `perm:"read"` + SectorsUnsealPiece func(p0 context.Context, p1 storage.SectorRef, p2 storiface.UnpaddedByteIndex, p3 abi.UnpaddedPieceSize, p4 abi.SealRandomness, p5 *cid.Cid) error `perm:"admin"` + SectorsUpdate func(p0 context.Context, p1 abi.SectorNumber, p2 SectorState) error `perm:"admin"` StorageAddLocal func(p0 context.Context, p1 string) error `perm:"admin"` @@ -3187,6 +3189,14 @@ func (s *StorageMinerStub) SectorsSummary(p0 context.Context) (map[SectorState]i return *new(map[SectorState]int), xerrors.New("method not supported") } +func (s *StorageMinerStruct) SectorsUnsealPiece(p0 context.Context, p1 storage.SectorRef, p2 storiface.UnpaddedByteIndex, p3 abi.UnpaddedPieceSize, p4 abi.SealRandomness, p5 *cid.Cid) error { + return s.Internal.SectorsUnsealPiece(p0, p1, p2, p3, p4, p5) +} + +func (s *StorageMinerStub) SectorsUnsealPiece(p0 context.Context, p1 storage.SectorRef, p2 storiface.UnpaddedByteIndex, p3 abi.UnpaddedPieceSize, p4 abi.SealRandomness, p5 *cid.Cid) error { + return xerrors.New("method not supported") +} + func (s *StorageMinerStruct) SectorsUpdate(p0 context.Context, p1 abi.SectorNumber, p2 SectorState) error { return s.Internal.SectorsUpdate(p0, p1, p2) } diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index df2b87f1330c334cb1a830105c124265d0b69486..6044b4b31e5407feae2fa2eb19874b27ce6fd6c7 100644 GIT binary patch literal 22473 zcmb4~V{;~5+qUD0?VPcbiEZ1qZQHhOdy-6S+qP}n?_Bru3*PRk)m8m@^{V~Yw!QJA zAc6jO{aki=DB-lVvwx~6`~IYp9MM}7LaDTD8$tzhk?a9UTyqe<#BS1TV6Q926%V_Y z9(VjmO5H>)*?>YYyPSi$NyRN{=K8#KhzSX+pFX*I;aFu^adr73BoSU+eQ%+tgO#^z zaFr14?|xnaUFPIGzIWjavX=@!t*)wT`CS!dZS#F?gGDerdDy*-?C^k)*IyvP$BQv& zY>3Sk9b!fx2>;cn-FZNI!GsTic>K8Oi91EZ5*-AkV|RB!1@ALEFM{bKlIsL}yQb@R z*;OE#tybrn{dpE>1>Oa!#f>H;!xub{(F??=9sDi430u?t3~U%(pSF8L?MRA=j702T zn}ZmD8PKO`K>+o7g!Xn4`T6~E)8!i{mIP(oux(4XgKDl02xkBReS#)u@;?Nu^p4#|j<&R4VLI}c12K{G0uh1;ihhj>`RWDS>LdD7!h_I5mb<|b5!nMJ zUaJ}Vb8`v+dU3)**&%1q&YrvojqMB57!4h@AR1N(kJM?pJ#sdh?TZ?P-kPRpf&m(wmHw^D9eqU;G(@^i-nwFT^4A?|ue=loGH*Z2R#QbpR{a(mMN6zDb z$>OJY5#!#|qYsw_VqKe$To$Aj4+6*7ejX3blrIA;EPa0-vU!s+DK476AcT(R*MsnD6yT z2~N}-5f|Por0uYU@E0MSn+bqKc*8B$ciUvk)N5Rn2a~|L@|~j0wjRv?b!vg%j%H4XjJJ3nSUP?l6Oqi zXmrHN1NPDFM$Kg^v|*}GZ&t|OYW(YN+8W08D!dQ9M>Odc!SllmbLy1j(}*6caLwMz zE4-y}O%}D_YAVad!w#a34^0r0ghT&7LBc zPO!Xj1-HV++S2UxwN3h5lYsrT@ivE5d)j!D9!hE3Crl=c+b~E%#9tjZEM%Ha^u6yhZ=<0(uydb?7?vV~8H?18 z5!%j1G{uoND`o+Mz<AhIkm%x0bkMbb-2fRNHek$H)OnGU|JujKp*CIKK1@V!Mt_EqgH6N z?4wyP6bJwbp8-hZJzRdjfaTUFIy`qBGnA01(@%r~s&SWlFq)jc4~C^VwXRG(T7bVS z9EJe3+cU#ES`2E0*Ko9l$np3T-6MgU-x9n>+jkW_4PTD7w+tZ-zXuQ)6{pvoix)_j zC-(#Q%;#T=fe+s|m*!;uxp#km_`vY`xVLY3^Ev>|{s#NpJG2%SlvI=i`q%IMXqOG@ z?iS|t^GNY|BB1M&h|t&zKiik*)4seW6q~1svab&B6 zVWCG5UW&#gjlDA-IV$HgMbH|(Rk|ZP!PYp}g0M74qNeY@h%bNoG^w-6U`jD{qTD4( zuMXM4hnDWS82(f?DQc7<(xi0y*WzOfD|Oa$a~TUbpPq>RI!I`Kt$X;P+ngK85&|F_ zKz9F(c1J=&Tmv&6&&0{6(?;(GxUiRdUx#=j8T9p1V#2Ce(|B+{vm3Y^4GnFOAKp%8 z^2$57-A{ueF2x(%ToN5eN|{Ri+U^@OSUcy$4Cck&`b*P!r4U0uj%|~;jKC;F+{S4Kv)17lrN_u8&7D$!1q@dI_mzbU# z9PlGjT|z_3Vt+Aqnn;nu5zUIZ&rWw4koJeVO$Zt~kEN_ffb1C`KKkLjJidv(>&UFJoB?yF6(fuj<3& zD9kzX!f%u{=x4zoj2%=bQ0yTv^=KWY$~3T6xYQ~YsH*X)FRl@3L>*rR?@3LhX3rr? z341Vvs82fygly-)59LX7Y9$%Zx=+o`GiLOPnNShVAz&r0Jg>)vS}p<5-ywV>k3OWf zlLQ~HwQ_3#+Z~`mcVzlQ`cezfh}=0jH~f8W`t0oR@xj3w%|tN_qwma6GjO2s^=)Uq z{A_s(nG z>W)&(%X2N{>&{B8UXSP3yPu5MLoGWWTYS#~#qE@t|Bm13Sr6ie7u^TH&mc@)OWvE; z&C$$GTIdbThG__GYB1>s>5^&d&`4myRrN(nQFS=|;m?z+yHvB7U+3ETeaq={>T8m9 zXcl{C7j6TNop+Gu&u1r2gM7PrmhBvRksc*_d7)-oi22iZ5x2gG2 z!mWiwTn9HXN#Gbq&v_wRs(!A0i3DDf?1W-ih{7h@?`6x#pCeSGtNnwifji0|5Hnf_ zZywv{8$_IlsXND%ZwC%6S=Wi)f4j+$BFX%cy_Hjdh*RtgZ7d`T1zXHft>xwfNDu_k z7+6%gr;ErT-o?6H2T#`1*gJY5n5(#bUfiW>jOZQdep*7sW#`S822>K1dNCa+dw~1Q zwHJptCBhI=KT{JaUu#mS1qB1_!8%*;kb%1Tj|V%x!)O9yL&6s#N!;hT9UE2^FVU%6 z=PTs<1oS@h-%MMsKwnwOd!o|K+2Q+;=i8;1%^`c;*+J~mLLR`}%+Z1R5bVg@1&*0PDUG&p-bce4>p@fr z8A@Qh!ISTsjBj?GHi-muV5m6~&1c+U##R^hg55o^-|=66PTe^DLt>|xWlw1_zh6bHR(-MfV|^7Y%lsFTm#gOaG|6NPaY#N%v5~n#sdxZWSDo-@GL&)FLAw^Noc`{o*HKUh9W|91lSy{d0Hb>E)hIcChw6 z8YVNBXRhC^7b_B$`VzI=LKDK^7a?5R*fbCys(TaD@v;Gup%bw=bpCqrMQM%dNVc&* zf^|4T)6AF5-Eu0m=^UOV;L|+~32xK7zw>8T5(hqf<#OiKt=HC>5vWj3T(u%o1LbXf zRXvQD9&9vT>AoIM$OV_sH)a2-9R)OuRM&Ul>e_@U!)`@e5Sb@ay#x37_hQ4GUj0sGj6^`h<3z zTHu;Wfe0NzL$E{i?-i0RzqJSC`%VMX=ivRKqYxlqrpiv6rrDLy?+_hYtTO9nA#?YP zs7XEZskCpM?CLz=PKZ`1-F^=q!y3>H6NJFM@U5~n8!fc@27P`cwBXA(xTb!4fzS3+ zsjgvS&mtf~J?@@so3xfxJ><+mj3XS~Kzx1{oY=ySeDA6aCT^p~gCN2(Hsp=h zDpxx?>A_}l;%ggFGNq(u^I%SV?JyGlY{9sVuf#LDOklk9Nh9G8%E z+05+F6FhOwl5`8b>iiblhE~?sHn0%q0hTJcnU9apLDT!r(ffPN`;MPyrC_EZ^1cyf z?QI_>av;bTB{>)jEO^Ss{FqST4wo2rvG9$x7@{Ms*3aofd@GCze28c5jQ*fXU!Tut zS_cj}cLS^YMusrP7}s{6y}Ja=1NIK8vh=O2dX8l`bDH}8`Gnk zoD%L8cCaqYTccp`nD#2``|@G;R)tHsyIr-XbK>FsXk{+l)M)ci(U##GqgmG$vy)kS zWUZ{6`_hKAPZ9Fz-qfhu|eoBGBvrzw(e*7IaDrc~g-6>T+P2W$Mp*nnE3V_`8^Dp8h;p1c}Vh?1R+3;H4y`kRrgk zf@=QCiQyWFO_C6j`x)ur-{Z*CJ0eDK!-JoN16KX9w;_rl*-)lW43!^sqARzoKP)bs z88X$hs26u_wr&{hZvR2l=V_l$pOITuSD!Dw&^bfM!Q&jASDF-2eOpVRq3f%d zHhVuXYfeZwxI~m&f|Pj`YnEF-wr&d*DH8$;B8d2lBB8~pJbeR0&R6}p3jM3?T-EJ@ z@}zMYp}NZ{vdyfx$+!O6SF7nsOQ|?U|1Y9O!k5*U1hroN?^cgSp(d$Q(X~4_2PBUn{_-UCM=LK6QgAjoLk1dwd>GdXJu>JBYO4fH;z^()W&fZgp>D9d z;$h(gk2`wcJVzrd<;$;aJ9F1Gzz=v#`pW19-`0nUmZLhZA{yETBDJ;K2(-NAM2w}< zWhPX`)H1ic2N}~F^tvfE#Y`{jm2rVtp{{>MGT0EmAcC|pol8&}j)J;65my;33f?_X z$We}fA1N|bvXW_gF!Y2>`K&*Y7MT(~>OCGv^S*gCvn`cOd%j?)qr$_v1Z9DLWOJny zrau3%(4wOUdg39}nb(-HTg;V22?=ih!nFLAqQY;9Q|AhbYdW-{B2yk5p9<9N&4U$O zkLgw~+=dJ?`kmj5x&LhiZFvZ>QIe&u*`DXuCftq;tW7d8kxlLaZAWgR(O+l1?UJV3 zNKy_N6GpW7BJVehX>E?&4kcDjlk9u)~rPnOPMoahori&!>XTT#QhrOMg()&*0TF zgzh_jIwPi3imN;1fyKjQ(mw&jw|T=b5BOHY-&`WAh7Ios|~cV z`zZXTw~li3+#*ge*uR=gZl?nFKp5+S_ZiXRudCaopH`bayCb)OpaaMXcKPLdAHCG9 zhLMD{F zmg5VAfDRf*xrspxb;vTtHE89KC;h&HgtHgRDOBPM4Ou-qaQ>aHq-gZ&h24sFViE># z?9AzRhZ~UuYa(4jz+-1Dd4F{mvM*%trV$tvx{aBLakCNe z;KEjw(BO#n-CQ{9{jJYP<|45y@NBy^4QP`kBO+ih!A9jWu;3{q>i>pfV}QtU9qAz1 zoQJ>y(-Pfazak~h+ME8SPY&YIDT8)nJKC~tgx8@k=<*5}EB4#s@k(iAS28;#b-$x> z5UsBRHW5pAYf5Y;Q2uTn9}rk_Ry+r9wU&~*JNR9ZQz`WT={Vt=23eL7p>{nB-VhEl z@DO2WgmDat4JJ>J3W-BTnA&2(x( zV99fV8*a!|TcGp_XKHVaumqkw=+O)_SnhitZ~-p}=vTCA@wS30_uHP;u48n~=s&EV zQl@BAN?BT3lqoV8fG`(~>~X0s9L`->*R~&-?c4V0dFS`x^{dB+=cDJVD+=Mqt=a7j ziNG-ki!_qFCjfRpz#m9V@P0R2Hh%njfvpcpUK@!ca~v{x+L2Qf%$*+cFud(%P>?_T|V6GWZyF%-x` z^(E01>yTKKd~GL>@-8~|Gpn5;6OK3Z^B+}`R}qdQWc$eO*?g>gY(TnlC@Cq zvNC;W>cm~JV#s-GSG#w`YNu1K;pd`-PWiBf+n)K~-;eWll{5`{vh$yPbG7hi}*1{7KW^Mfy#WwMPbTXE$;= zjI)!!&Bu2mBa~J`$pVwn*{CnKd_$pc#o`fu^@OeB9U4p4>iBQ-ht37YQ8*!lHvBck zVGLsB0#$yb{0oP)(qCkpNA8)}#yO_`M<)RVI27?0VAo(?oDUH-U+YlBFoGc>_Q5_o z?0{P}Kg*qGvclhHaIUP!4y0!CFpJBV6v_}eJC4b^qo2Ee_WNjbn%xn zB>i2Wc;UY<=QRg5nN*PX6E{EJ)2N%fM&E>5kGn>_Ui8dbrB0Fagptgpaa1DtMJSln zlBk<;bb0jm$loqvHPU2s#Utv60}!MiB4OQ_!uHjLOz0!BB}tu}=ij5bQY1@llTvgO()1@+;*zi&>--MxYzU93FP4{~3wNv{|8Vcw2W8QO#% z;pe!7do;s+MKoTgu+?hisMyAHZ*B&F#%b&(x-ftnB_uRTM8~0x==xx!#O@+BCQlZl zat^&)W5a<0EeGN)LGsB@@B63KG+6Fe(Kx7yC)AK`Id|60t~Eb;*Lc)WjqH|MK`j5c zL=ST8hi~QIJJbuUVL>h(%~gXIrkvc(n18`$U$ViFOh z1Eae|KK#n6;M2*zYRsPKw%=g!WqLQxEdmmVZ6B&k6=*a_y8o?EG~<25Kb6aPMC*Ts z&p?-~UDNmb4f*Nt)Z$m1raQ==UKdB-hx_a0^s$MNriU#V0q){(rS!1s_@pcJos(Bj z>Y|p&1fNnhj})^D5&-!gw+hsWOpJy`(VrXjXdj3&TqM`cFTw;DQ~2g^&g;U!k3)Qb zFiHOTAeJ+(S*>e(MM`e zofJ&rp3jaK+$2zJBpa^W1uJ}kIAoKz0|#gvqYGKz6Q)t)#7;ZA=OcZvq&ciEL{mTt zR(*~KJ71LTlJ)_BIcKuupNvyw)y@K`t*&ny%v#=;TA;alZ8`%!d+z(UG^=Dt)=!7^ zz9Bk45{{-E;3Abg*SSr+ix%b{QC|mAv zlR%gf+tAkXc98I5LhkZE%zZJ*#9HK=I?}Rf<$+njZUG*UVeiefzr0f!>3;2hpABuB z56+GzGpwlFHQ3pk9d)Cs3n%&78E<@r*IR|mfx1LE%R0ANf5aCJa;WtmiKsq-c=&XKFNeTbLqy+c~hBjvL5P;^Vu($N-L$B|8tcN>iB(kQL~7=B86hLtpTefz@WjA!sWqXIFE zx~Y~iF>|hb zhazv6IHFTAU4svNYfS}>NP~6$LITs`nlnnBr z=)BVy9Jy+1*M8y5y&jP(y6ovC4)iAG>;$9p{cL>dr;c4f^i8&chPbCg6!rcy2wGzb z#OTpSW=`bwn{GnwX+<-ES1Gf9a{+(4_X>7%x_4{2H>YnE>y1At_&VD*u#)QfLwjbt zl=>U2i&hFR5%I9KNnnEdf#N zkXPc)2gyK87MG)XjAH@q7w>_$(UNm(XostG>j1VKm=!yjcSuIhi%VTBN6iT}((nC3 zDcLkj_LmZ^F?jPTV~KZlD=p_sqvXc}WwcMLBTT+Zy zx?=W{ICtjjD_d(@liZy73J>SlejrbCKiA+{4%GDV6DzH+ZLBCr&V1(Ev*brtx8V`YI^Iz1@4Y3Y)Ga-a5wm7`3@OwfUfxlGWBs!@y| zt|e2NV(UvjOIzM5n#4w$#5K2U{Opl;ONNX13{dF>6H>{3FRw(I^lM?;E3&edi_{OEp zb?Gf{%QztVD+IIuqECj5HK0JmBytuZ`Mw7xjTr045t2UKHYYiQX76avjUCg_Asa!n z*A>kjHr2^TVa$m?yR9AQksZ}GSv!(R>X<|BoUaDVrxjb+?1kmJ3FrZzIEm5VHaRcO zD@cjaozBD=KC}yr2@E5GE=}yK_VJ!=ORBn*PDkFlW zSAm=#nFEglk?A6iF+tNS*v5}DIU>s6tb1r~)s?9f>5gUNR1K}KYHysM|D0}i0o0CK zD@UePco^JjbI49kqJ|3dX|W6Q9gi28@kYsf>0n%Ap9w9X|3UVc6($Bb{#iw9)VO~7 z-b^-Y9CQ;SJ?A<(+UK(xqdaZzle6Kh7MIvRfVgLGKRQ^fuCraL#EWk8tp|1>>dhMW zuhRXsSSY9FWO$q}}<5P%*@Lx;>H{2;i@3}YB5Gf_o zq~suw?lH{Qxbp=0N|aa@$!#z-g-=g3a!|*BxcwUyIfb3hB>}@bE=oNKbyF(+ezCXX zF>Ax4`>_HGdl5$(^(T8UnI_N7qdd70@#utD{RIqVS<-I&QkY-f(nY<$KuN%PLkYyX zLm!-j7INk9UYFo_h9I~oMBu(I)%&kU;Da$a1HO=Nlz2=3%Fej1Ej zoo~2~T{!Pp#wIIQGeuKHU;a%$W@wn$JV~^l(Ff{)k$v`5u0wSI!mhjj(h0EZ@`ZI> z=6;Ta;Cu=c{T9X%$oNciN5Z!=ugLn2_=`+S`mhrRp~q;99ClQ70|Qi#P^4XHJmFhx zAy;Jgd;^V=qZkgZjW=v#-SjkNML$ug|Ff&VFgz{NKq~WJ#j+j#&&=f4S|tXiQscT# zj&$HYf<5`A!?gVqMsY(%UEpR*)Fc_11gEb&N+>T_!);;$lh{~x&U!@LHS>BZt3gi> zDc*isk3>m=R3Q~hmYQ{96Z|;YaQV_B$D$U>O}GEqwZ;gH%J+QEj|PBrEMuB4mb}fF zI$u^U!T$X;TsbDjXj~i0Jtv7`CJC9ic$=h&(Cqyl+8lA8KAg6xV(NTB2Bvbm31v`) zZXk%T`ygWaocM_E$)Oy&7U<88kHi)U;(m=OVp5_-!4G&sWv64)%7v57Y3T zuy}hlw`zjba%^@8#TLJ}r~^dqD=AxPEI`Hpk-%WAh18a;;v~Yf74?a9Ns~3^347WzX`KRpk6VnO zuG#2J{FBn*<<<=M5B6N(xb1(QtJfbj7*jWt#_l=5&IrQJxCUX|Fd%!eA^x&Q%=eKY z;Bw$YvtK`L5w5V;$$iV1nNpV1mz^#-_m%i2EzvEIri?zNA2|N(H?kE>OJ{anqeM5# z>!Sot2F4>I%B{d2Hi@YhI|AoyC=@m~anK6FIA%c3iLyCvrwbkP{q!d7(*c_RSxVWPEbsDAWN-_%LmUexF>kYh^G;x|t|vE!ge@X)VY>T3o|jhJAV396jz;Vt zqK8E?){jp1iWu;N|Cy3fS8K&n89N`Bxn@Rn z@L1R#@rWK|TcknpYJLH67C*yG>o*}+s*6Gr_dIO1WG_Hc$dHMXm1RNjDbg}XR3Wd_ z!!rai1Q8she@Z*39|boC`8Nh0e8LESUK)w@0)QJ-h}%V+6lcC>Gjkw%NP_Gcuyovw z4#fn_LS%s_4He^qEr6N{nBVrQ2lEA60Bdb?H~w*8gTO12Ih_=T!sub0$Aq<$8no3$ z-#Idz%P2N^(M`4O+!G;(a*Z68YHKn7gvt1*hq-n3vBSh;#ZcWk%D$Bsck z51Po59Zn}4`dD6F@gWvCe$e^fnn4ePvZNDpAKjR@T!^DXY_|gvPx9})BWJ)g@FMB4 zx`gP<*&u)iLNBwvCBUh~ITt)GuFYd*SHnJ*>=EM3I4y@g`?vqnD3WS=f=}vS>yhM? zgh8$A893NR63hCe^1zg}Xamh7C?13mhpCA43?x+PU6y&>C?j!E$WgEOJs8Cr;aO@T zEHt>jV9SX<=Z{7#%=8Gt9=q!hi5h2-$&|zv=4o6}ECS_IO$>M8%1;dPOAvXfQIxP4W3;JfKR_v> z=;N}fd7fS>f~7y5^~CiRrOR=3+u=kVXl=Yv9Q?Q;I&Ew1)|)lHqeW6``K$$ zB*iE&h>~$wx%`;0e8xCj`cL=-di!<2C;{emusCkc(P(_6Y?yH9 zd=^?-Hkcd&ljJ9vYu7JcVli_bT@Mge0-DZh|7BD4#EGs5`tL6?^`6IFuEBl5c}*7d zh`{`v54GSz>m57h`{D2$;zQ!(fOST6g15qE0id@Tiw!M0l?~msr}I^|o<=A3QpIv- zQZuKEbqZ}dHRa`wfMs1307nm-c zHzT9ZHCiEWrmb@*jK!+z0&)hv!+8@7hTNjkt65}PV+_cS)ze6iW;qVd zAO-LmIH`N2?h1l)t`zKC{W(r%v3(x=5M|J7DQCl#3n^$K;9N4P1xk}#^%nJJLrZE{ z{vF!xpmuvwx;f4++ZLY6Lp-Wa`79QH_=?W3vL(7(Hyors=%@gnuNI4J7=-u5&icEW zQoq_NCNYo!D?Nq{G)`Y_#11cGsVc0i&!1OeT7A2lwjp)g0vaC$@CE)k3%@taoWIKq zPvz3J2@2lVinTiiQJf{aF63^TO@3{12xy6Q>QK&`#LH9AE``8Pmw3+FAj%8l;)c5HpvU%S=(@J>;`MI5t0$Ll)%8nJD+D6< zmG5~dIqV}c6-OaoHOq|T1g&N^$&C28S~Tr++cBPC7MRc2>DiUd_QY6TsZe~>Q~+SQ zfejw5f-?9^61Qn4Zj#MRF$mw4?YxVct1ao)DPbd9xJOK~qCYuQx45YzX^yPBu-;Z6 zo6ace2HTLNYL2cWjN7_0MwRYo)DzWs(@6IwCuvieo61aOS?3uiOb2>t60($N*p~v8 zXt<@8`Wg9fFtFqm_zJTfk*r;@RZ@>)$AS9GEp7E#?+6A8=r36{9Y)SpF@<>u6}cB;1$8m2qKSnp+f; zQOdJcjBwLJ&%MEz(+|WTuEHM7X(MTZGNqm#^&sXbN@kMes2Om;jlEFUs2T7e-pXN1 zggJ5D+g$#|0Fk<5}M_UJlQ&{)tzR!i<_=UD)- zJbHmkNYMcEyy!v@QJQD$-KHRh!9yYOE(f^#_q&IbK*O-6$b)tL8tptVuRa2x;KY?v z*5oYMNh^j>5c!!X%ydJBVX%MYEQEYFxcJXV5MLDbL-F4TEk~-C)7t3C+8x2?FYsG5 z=HK#RHRHQTJ*`?OVZ0`;M0K5lHbY-#`SLzW2WVyTGr)gtks1a6)|=sK`jpVEkq>eJ z3a}gB&x1GRXQq#6xC+4qQ)qyrV1Je0?F!3@j$tx{4nc2>Ls2M866?$mCl|tEC~{*7 zF$#KLMQSIYGQti3W!2v^s0%E~#n|}m5d=$*vuL|&f~FZci|m)CSDh1LM^%XDt)Dsm&RiI(8SV9dh2?(H+>Z+1A8eulCnG?%;A}xiAYt`{y)VNxV!TROIHa;WRIcmj+@dmP&aZdm3y^Rdl~fvBNy9Ly+h z9nxpxTGF{f# zBgTIh`qg~&C9f4p)Z3L+aGCz7MU{-zDd5(~bSm1|3H0MMSK;NF9cdzD+56adsotg9 z7z|3Ae$<~p@vSR}WcKjEV{sXGG8RVEqVE90 zIan-HB#5fjxB~VNx{&yq#$|x(m?Tj}o9bkZI}ZPJ8Abyw#+4b3C$Qq%8-)Ual9WPP zboi$$8m-FP^!AMr_7J7%!S`GSMlr;I=&2E5+;TQc`dcv{0HM$#KK}faWFTd$tqXaU zuvEltP5)7kII!ea(!5wVW%abG>#l2ST2NYI`y@S=;Uj7lws99tXhQ%k_kCP$x}yme zG+b=}Z0UL}X_7GLYmHE%wt{k*lIZ5TcA?wX`dMD~ZDh96$uKfJ6aqm@i()8XLf~#` zNa>%i216q(1M7AauY{l8VsLHe8@9f;o}bfl=&3rx@Bnu2qh6hv?-p1v0rb z-wICU4N2m6eR`!q^@Q7Qv$Rd&&q`prj3TFAZT25Mj?5$uEUKmqBnSKveM63*uNM{? z3qB@?nsA-n&KLmN+86Nm7b8cBKuP|wSO%v$(IZ6?W%(X^Wfceoih}e<28t0iS~}Pu zr%!<#BlsIXhjU4cAFa}ue&6@b`#)zIzcW+Kn!Zo4+C@~`M;YSV-IFVXFHgph=XoL` z-Dc+DNZgAlfWKSNALBp=MHQOE>564A@oI@CDz7IyaA-3OdKn3Hi`_|}ETDuR&an#; zvMyueADT>P&osAFsk+@7hBd;7x&fkRslLu?P9lsLLJZ+U!UvL=p)2Y!^+ge1u1iNT z|0|X;;$8w33>g_H$lr4XFe(-4Eul0&vm}H&SXePaF?V0_sSIV{dqOP1k#2qYFb zL7fapHs&4tg-4?A&%B3$YR{bjD?Cija1{b|duJcMNRC8OnyeFqL-XgZ={=>5$g;B% ze4fs{&di(cC}N6k7A+#Y*Rvy0E`?3oH#oP_FWx*3 zk7-J@38ir?Hl`zjmFWYTTrsXIwf*JgA(;t|H4bvF6XTQYOTa~);-4_5dM8=e>9Og5 z?~OYhM^`!4Sk&jNG78H~&HPr9J&q77wz~Rw4gU=S`TVdDAOC(k6RYhc3zs3QXFB~D z(d07oa-9b>|WJd%;cYlUI{h z(e4O<(;pxyZzzFlW7?*~=z#z~B}!PE1f@+K+Wh~Du^yF!M-xk9=*@@lXboC9Fw4y6 zCN~xNWlA|TVlJvKWl}Jj3!_y1IOGLgx6M*WxJh;mEJhm{9Nt7Er#?!enL>vO_*mV4 zh{q%d8zrY>-`VwPd3&Os?M-t}`OHI;_Iu=p*31>+b(FOIoMn8zm zowt7_LP=T|{|B(fdDO7F52V8*Q7hzF)7>LbjgA!eSQ0`vw|8(&v_LexU^AR?QyR(~ zv6vM#M?Se=GF9bz>#07=1Vp!M*}ze8i50H2?L-9kx1&tDe3n?#DVY9<02DSk$@K^K zP=;U5Flu~QQA_d7(_T*^4{%j24VrBufYw}3N-%0I>#S$` zOFZJMDacl>p+Do@$ev@aVqpybWo`$VAUSzua$L3Xuuhymf>^04LzQHP6{FM7iXKB8 z@)<)78cqg?#kx~UzlVVWBR)}&yybvR^hZxghM9eO!Qq777%?(agI#`8=h(Qa(5xkv zT6?`D{%Y<5o+HKX5@X&fck_VtJzsJzt?8A2mig6w54Hf6>}(_#gUzZDgvQVdM$?aR z@%_?=K~Ai3JJf!~-R(}Kbp}+xaM%k&ka|K>td#tB!t8t`HCfY=_dA51DnsiL?-8or zio}rAhFm%Yv<|a9|OKuN-01$hi`8Td9)WzzC;u*pQ*Y$sl`? z**I(iV4D@Rk!h!MR~<7fnx}oxQZi$UVI>1wTHQ^Ccc;N+-{rSpEHl^P8LmI8U_A-+ zHw}l0<6-`rhTe*M6R^jyz+O!x8k&4&jso5VXXfd4BdA&jc3!nsMkjCZvI=Rzxsie1i8BMTrnsuSYUSnjhB)rLnTs5b$je#*j=9G2SjCYuN&7_NrDqk#9IXr8Oop_XydN-5mj(U5b zY8T^*eSvqWT&mT&MLoBbp%X+#FCIMMndVD7tQjy~WtYDu=4xct`14rBVOELqlyQiK z*@#s&ZLJD92rIMon8Olu=!6jjp18SpJ7nG1Yk6Ey+eFGAPZ;lX;007?s+~N*vWJsf zBJ~5Ymem#G7WUy}=Riv`TN?IlMcTnM0B75JM)pvx*-dG zF04hq&N=t2E3!e}-_m7s51~+_;+5OWCSL2RCeRFK#*lOomsgw2w~`t9973@&^$%va zZfY+%Td%e$_5k?IFga{>&o=+$^y<=;$ZZOo`Hml}UR@k~W z2A9qa8%OgiNfW&jm0=17h`h{bPm=IXJD}xHd&@RMsRTX=vvaB#!L|ejnK}5s%Z1GM z!Zr|FB;>scm3G~5uLaEn3Wc-`VL=bw3OuBhav5#4@D^5N#J`#r6~_RA_$_cU`sJ1m)!1r8Ua{OpyETUFPJ$d1ZUx(@RL zRu(EB_7)N8MCHi*gL^%f=iU_yl2f8;FGWFcnz*WK{AI0hAYCMQKYzh2dOheRHs}FU+7~ds>6=Nrj z<`S-AHuQdE+XO zf?A^zqA9$%vFj)UhUz*g|Fr{prbZfwq^Tqj4EBGNd{)(ql5ZD-fgQs>e7gj2E;_tXA$ z(em^80patBkA!z~>G1s2^U`(Gg8vdn%gT3;@c4S4>ciRfcI7wtcjcyH6Zzh~%*bGW z;p|^yB$BX8nfi;iPMJ#UVflGE>&32R+vcis!o^>@oG0qankr-qm=u77=P=~u4XIyl zNo|f1;ED&rl5gF5bU(ko3`K2`5>@$jiEZldTDhYToWns@Ju!VMw7NcTC#xKiC-@Ax z`p08o%zTZHCsGxci&kTZ$ICVYifx7M7<<7NZ;o#5MVk5hl0T7QfkkF|CL)6Ly_cD_ls(K{Ei7x4A>%PwY z|EuIIyP{y*HB2epT{3icBOo($w}g~*iAYH^0@6KnNY~IcGL#@gcXzim2#OMWo@eiO zt^M`=a{YpPt$SU^b)H8LFXwgVQqPKMlDdoGElQO+oZ@0-CFB@}nAcgGuFaEl;I_xH z1Um4s+eU>bgs$+XQVzU7H-z{N4`;E1*1?J(w{Zqt#C*W|^$jQS# zuRWhn=-hL6ik9UrJpu6L6TY`gt!IpyL>rbJ>=Tps8 zG|5PuY=})aJpS3;Fri3{dwKG>_A+wjAxm!mE6&|tBMfvN`+!G=qnTEaZ)7k7<5PIF zAIq!u(tg0gH#Lrl5FOmXQtYo~xNt?|4mbbcXom9)X;IBzw`NZT3W>f^Wb3j;v5yBIrh?Skab@(fYeq68Hbrs#FQn5T`i`1ybWy{h2#{ETYU9bsUP z)JK|LfShfrtZw#Jn9AgqfZwgTWLPC|I#c4=pY{)=5F-`w!H;U2F6E8c8Rh@-mArfe z;}u%PUBhbNvOj-WzT;j)#lis8$&KIDf0tsRO}-DbsCZdLYBgh!PgHeLexlptfm6M7 zbQ`q8Y`fOvWqlg3ul@7-S}9O?fmRT<%|k|?gXULnT)RrJ5#*I>O7jpn&)r(#3CpU>BktdMTA zkZTQkJitxJ{P37%-Ll=kmL)tDaG?HkQYCBJ10%vuB4ln?C5tz1u0N5sy({myz34nb ztF4~H-cX04`g;bbsfx-}u!Hv?^C(SU_-KWjx;Il9!2%H;-h7=9&J{+)UDI0Cb>KdG z|05vl+MKT|BgCmVOsY@3|F&F)>SklPROfL`4@ipMFdsR3vi!vq6y)6aw;ldS9?QOa zniTWP@N<&I4&a+_#JY!@MZg(MNccU_Oj}3D5}n;NlVMv9kUoXxF6TaJbapGK6^Vzc z(E+$helmN9x#wvW*|Bi0qU4=hKO%A>gx{{w%Qtf&f5D~sf*?ORQ_ttYr0W;~wM8*- z=R(F0Et!97yyT`}Cj9!TYEWcUJ>cq-AR5t{(vxNoiy(GB(z+lw`C1x8{+ci3trI7A zj3PXRp`Jnof>*?7Z)?$2e)E8uo$Koy{8tiKk)DU zl<2@se@B?fy78ua78mwF@KTed=s+~PpKvj;ObT!)&X04^ z%ghd#RaTFy5)d6;e!Zl=Kig*UB`a9{Ln%Bpdp)O|-T~GyWgDr-NYYfRVtU)MyGj;u zTmQ`m92A|UJ&1oZKKHC%Va#5DyMt#1t?Y5j303ePvL_49RVLL?LctaIEUl~P{ALDF zu z-9Z*(jftORaA8CiJ-^Gs%=sHWexbERCi+hY40K6jy}!UO5cHEe^JHXnvqg)}6WvL! zuT=mOUZ1)}%qF11Ch|!b&Wa_3a!0L?7%3IuAd))@iQLKXERV&|{Wzr2dj;xAw%8`;yQ3#1QZ$nYp9`Y^ znU>`NL*Fnh4r7Tt7k@Y^f9n1c_*W|Y3&Q@A!=u51#;ZDIH-V9%Zm+a)&o#toO`8=r zvq#?Ug}~D^mtYvk4%t+ey-dcSh#eW;4N`4#DnS;acRe)q`!0qeHVn+2#u8u12gk4* zR}6003?}|R9=qSeuE-kDPW!N)1$^Ea0sXvM8nW@Q5quu%?wWBrNVdMdDQOOzaXTEenO;2kPu%>w63p*zc|-*S{iR zV6imfYYwLBn=kr5n3@znZFWLACD#bt|ELarhdy2`#>OL@9D`WfBfUm}4Dyw}JG;5p zNL4JrX%=tH$xQ$`Ge+MA{0mcu{r-WeQ5n^amwT0V2Em3!Sw)-~Z)#)f-#Jh_d)u@# zio=9E8gh!pil}Ut?3u~0T&wtp3ML55@o8WtIXo6_FZJTSzv~wi>E)7`ND50ZBw{&r zHb1b0G|aDAXD^+)m(Eu7`Ke)N?A->p@OctLdbDsYZ$5Ft)swA`^bEa6>n&=UA=F~f z-j--1=qay?Z?Dhtm^Al-|6=&Di}^}%=vUYL2R-4ecYp1EPxa{+OWK|!T?!>PBsafV z-!6|wHrT$rM zYespOvTV%&S?}FiIAi%_9#_YwbA>X@+7vS-mwPTHQwtvkW9Ed z{RQ(&>gztQII0+4iH7leZPVHy zE@MTZAGh2d?jBeODiRx&p-i&TnEUOwPn0!)a|iI398n#)y&iDe1NjsVVlkSaugK@V zKxCukrErA?6JO)=w5ycaW&K&BU!S5+^fqo@;-N{2Qm@``=rf0%RM+>=aL`))Jkq_5 zMgwyf$>|qd#0$?#$p1UWghFkuM1GyoVJ(QyxBn$7W2k?Sd%HX*IcP(pn;a^AJv=cobi*eJMj8@{2=PuMmqMwBKK+fl57c4TU~ZcJ9v@rjr_60WwhCNy3)t!jLXvS|EATM zWN_;*bkUi|(zw+?>tsY@F3UL4+S{_NKSGnAd10m&H)7a|-^hP}o?@xOJhjM^I~ryb zC7|0C!!u=E*R|E=SqryCUCwRC&@KvS*KI)GbidNeBxyeF6(#bi!dzD)Rhe}Yqt1E5 zy3PPrg|uFUJDe7o-oBOnR7P{-l{~ihzu)H*2+Sygh7OrDI_6d+3?kM#|ni`5VL^wpmt`w01tY8c}436g?74=jm^tQGKt#D<0 zFFjJdH_3Tj*WAQVtS%tn4ruv)dUc`z`7&$fCra5 z*GYw0nxkO@r$FgxemVg)!&=P&_E!q&zes*qdQaCEQEi(;gu^6nU9odX1KhtvdvB&^ z?m+1Bo4-3wUaF4tgr4W%{oG)r$P@O@9QXF+A6sj{X3{{6-+IjWmh^sA*?ueKt+mx) zPm)5r@F3d_VmCYhFri_>4fc}+1H*jT7%*96sTlK;L5zszf(YdyqQ6nTSid>Qc0@DI zd-wjj-^sp~-iTB;Z=k8<9_~nIll-{WDAz3Ua zOzk!S*Y>LW&2wRWww%A3+OOVt&&N|=O3fc81iUu; zm9g1ZwcCmR@dY~c+mAh(6$?c3qyCCR{M8%Y$hH;mM5`Rc=WDCpTfJ{8a3e7s(zIsu zAq!Q!>95)WZ>g;OEKD&s_q)BTh+7E#GES)4+#(UoIzN2^WXTR&x=9U6)efO`9M9sT zLfMKLMHJ=OP8@u(mnu5tHqL{}PiPtfQAU~43WXfDmOiVzcN0G(*fQ~VjK6YKHS&~mDK=!ld0{DVrq_+>UaXy$yIv^+ocT%3bT<_7uYm zo?hKWvutU?Mf|)26yuWC6r}MC&70Rpf}z1EU1dyIdg4#3Pg+b~woc3@K1F+-E!xtJ z1(BC1p)INb7ybdx>NBjHX&)FP7PA;=n8nK0nb6;67L(MMcS4#1P?Ea?EVyFem@6z; z*3S$}Dx}q5eldDg_Vl?WX5!4X)2UwS=)v^Fins47_1RqYq^fCX)a+6|7vZ8@=}xz7 z9DPy?BThFlVtjwsRZDyS`}u4i`FPZ?u~o2oT20wZ%|uvzqR{kRFn@T6&3W^^T`+G7vWCy1JT)1mA~i4ym9O|B(1P!c%I~Y%G(cL zPuJ~6?zuK;-NmF#`)e!PL^noa3qlS?FESna3vzK!1FdJwl8 zeq?VK7ggc#I|6CNE}WRh_vq)3$%};R0)9->F0R3(lbns>)1AbfQN+!g0(XXaXe-_K zj{84&?HT}$S7UoiV~yu4Yo>@1Z|DmM))kh<=SvE8v$^GRm9f3#H>*Ot*pxhl(%!o* zKz<(z-Zw!^*B2@{e+u5Y)9TGzSPc2V4g%xk``l}zex?mr&;UY_==(QffRdyL3rfJ*?FDE|p)GNlfik3x}0|#L9 zF0-V6`~$L?uS;zNl*u&n?^kv=5Ll}$K0 zuRg&EFe@m!r>`=zD|dWZ4q!ftw{8Fe11?q0-pZ`=P+<38Ff;#ZV3TKO^4)a{)NrJB zP?3zQNTLKU$uX4b){5xk21;F1r+AM`eTYA!%R|<>*^FCV6^s`~Wzgv|&mV0z&r>F8X)DI8SE0xPjudPD43Eyeq zzi8pd$@6&!d3E=X=SrpMX4s}lXR;+$#H0X!NoYsZL0u~cY8Jlq2B3TZ*p|Zimdk!u zxgOdd7vQgd!e)}Pp00YN7Fom-i-g{k$~kBqrW9Nq3k9j{ppauVg7ROY1hBJ&IJ|H% ziJ?PrjfvS&#vaBEX_&J15uR&j`a1aE=bS>9WDn)ONP|6_eDOO zIckKx6^ax;c6;BTJF;6lu_^T{fD>XvUG(U_$kHTAfGJ5gON?p9jMd~2Vn=;lo~`z= zsy4jQ-gIX&w9gEPtMkLyr&GrKW_1s^e>90Bld@Pb8-UvH?3TwpXFGA3tQ>SIh|`0# z&$^IW+3t6WxIL`Tz>({h`FgVwdJ6PzuIv6S%roF7m+P1QbaS%@fm+0P`fsNWZzwv^ zh19{xGN!;AZ3RX4Y3yl#=^ZrPBlA=*HBh#}v#Z%}E9FsD{&F_Hq>(^bO2U%4ndf5j i^X_V|4h&&DURBVE$NPh)XDBF7PlMPg<4{u!l>Y&R)SyNH literal 22483 zcmbTdV{j)=_^%y%W81ckZ*1GPZQFLTv2EMh*tYFtbN2Uto>S-bIn!1BVydQUW~#gD z?$34I1kq4H|8xCZ^>`}dbaZllsjB$>WRM;++7Lmjw(S@}L%T~10w--b18(p-J{xKZ z=$@OG0?e&?KkJ&GPV}nnvJC{4%j6u|o+N6H*VFsFX~D2V_3Yc)8d^JA7cXC-@EUh+ zcKL)vfO@ufpi!8_&C4JyZlBs$Z*_%-p^GZVMn^Y3qzP)hf1bQi5RFeBcWz$$yiw&% zR!NWp#-MaJA+MM|@P~P*3oYw*9|OISphF>^K5zTt*U0cjmxSjyy&O>?2PjU4IwFkZ zdBsO=<_2Aln5g&Es`1YJKPk5aJp@GDo6FrX3%{O5fVS{i4SJnE0DGnE&p501& zQ4u3LiXiFpl!6dP1-C2+BHukqf6S46f4}Z}f8)nfpy0}Scl8)47aKZ~js`$npn*Ia zjT1jRh|~ra7HA@YImKC@d3Z=3Yj3;7a23k@rh;Mx`b`;=LEydjiUx%M3EW8}6aWGI zSgRld4y~~9odn-+jIsx%Z;ZSr1eD&qdK=0)h|-hdNZMw~=!?ZHw{(!b8fpv6A;qJj zvpEPDC)mLx=k&L*;KdSD0eO_<6AK}USs-_arZ5C@g2B1`Hv~TTIvjhpZ=o{?{{2Lx zFRWy!ubtWmqlADVC|=s5WFmOZuSa3_14qImk?FFOctK0KR9P#229D)EZ*$sGKYMrY zrd^(4kXJYFpVEO3|Jm0u{O?<)ugku_RTQTozWQ`*@Ny*R0Mdx7y3(!N&<=5b97g|F z@;#L2u>b{J*_mH@w6+?KQ&4(9IhM_xfBnG7IQM6cz`*8|G4>z7eLh|lY)zIf$C));vodHSI# zCKFekK4_$uA0C*^-6fba@O&L7X`4jn+*M&H!T!?$ zf|^9R25-ED`-na_!LL*7hNb;fOC7URp=X4QR{@+kxh``VN;=z{iRk*kW^w*`? z2wDCk%hU>ialXHU;3P*}1qk`0jxzTJL%SbwgZ8Dj%>>UR>L}5mKK;OQg5xD%Y=7aw zMpA{*W}P97o>i6d`G(8`KMn}b*^Lh4kN|v%=Spymks*UT91ipG_Ha1Ad= zV)!T3jHkz}Md6;@Z`ECA!WL%+^w&VUU5CE!XI)_JuOkM~`+_9jAiKUnQP4%_1bYn%-yNY*?mn=&&k$U)F%YzC$(8&W z4pV-YpcMJHQyqMbovpk6ZBoQrn58rQxi~qJ(XiAl>AqKbAo)1Ejj!!bu`}T4inHt9 z>$%IE29>D3#PSLT;G9Nh!;f?GGR`E5vbRT^gn5l8H}A_HzuVyV0?gs!k&b?lSxjYu z#+YvzAb+sgd#KrdP-K{1-aCjpWTKn=oZDD2q?mdj=x0Fg7Gd}A zcjrY81P9dpijSNvCJ(47eSj<-c`i|H9Ln4ugSLset%7$!W>kws_IPt_XwUN@xQdLX ztdiSIA)xI}_21iW$#sOVxYY>SsEPwaM=2XM8?@vT?a}$}WL5NS;XcG zzj85vrU{hTw)0`OJtgyXx8lZ1uh(}P65y4;+7ePS1RE02VXw6%qdJ6W$wrICsk!8m zAa&{`3gIe?M{_W#x@vS6q(35|*F?vJ*9k#FNdBA=!NuH7nA*pHL32g$6CYs2fjEH& zrie5M=E&|Pm6BN5vJwiwgN8czM3(sNLqX;gBsj`J08^5AVDTcu#on{&DCNVYnn|1N zGRLfm_(S9Y9SHhHAfm66OqN4hpKW56=WSqy6DjQU6~{nmIp8Nnlrhe;u(hT&*hgR% z1#AOhNKpDb^L}H-E6Vvq^!F7zojAbi(IO;h4a{NOy3LP-A38tXZ;yn}g`a&5oib2_ zpFNf(`xkw?pH01Y3A;SK!}a*PJ34-y^L@PCeEHmdcR9O!L0=ai3;Z%S{lcu*xiS`1T*N~~@AOOkGo#pEx==12VoyjO5$l>IJ z+92u8f5Ueo4g|@0we%b!6G4{!7yoe)ik$R@aUytYY4yIsQzYABm@0>#PeIt-9#Ay z2M3>w$<6eouz3ni6 z`16yQKcCj-PCZIW>ni1O=PNmt1yQNpf!F`$>Z~4nw8%j!`ZFw>Dkqena7Qsxi+AcM6drA%7t^B4-dLHHzD)e)<$B%pQ|H=bf zrix4e4_-Kar5R>Vjw{HPaYu!Hmc7-y-Jkba^nvNq(7qpw35axvjJaPJU96hZfMw`|YI~Sor1Qmr28!EeOly z)giHB`;#|(3!CH*2D&f_6h`)lX3Fx+yz4(!cAVPHSuT?ERw=xgjg&(TXydAoJu}4W zE=*x?VfRK*P#9)Fk!_qtsCqCjNs52Od_v18MB(xYnbDXiG=4=&eOiZuutkC%@R0sQ zB>Ei7RtUbkUf-kvb$5{g?7~`({)PsZy!n>Wj~3dQ#bdPf?ViD4Xa^X9GI z{y;49!*H1ATjKrz*ZVV$Hvbx`4oe-9sDE_m#4O9uDa0577kBuT_=Xm4$SC?yPAyLq zM;U<7J6uD8DCOxRNhUSR+r?;V1gA(~i9ziu;I@31j(YwBCjBGw<1LvAgH6=XMnUjn z|DbVoTQTbWnU4B>JC)9$-|h3>S6cM0o|B6;zN>-zcEa-KcK7)yKk~ch)VG-bAaiYV z?z`vpYwT8f=#^#DER;SigzS@S*{pqJjJR-9TfI(Pn<=*c=kob3%`X10M@!4GeZw`~ zJ;gR0hopx$Ph4%!J=@dYcSjDh0-JgE?Ls=SK4w}4(H0o{+cdQjhzCuoeH!=;Y9#^z z-*t_A;~?IlC36&*90F0BLgQ%e#cXN=+p~BKS>b^d_;oN8FL>nl2`rsGMS@UmmQ3~! z$}O>7f4EYRydv`lUG{FelFCd6K*v$JQ-&Fg8n}p457TN&RF!kS{5+AQa>|Pq{^6~I z79>Ey|Llf#cie1Mhol2x-78^2$%&v|S4iT>w6#euMW$vdWi@KN*r@=vjz^)JK3hTS zURk z6C;}24hPmW7wZHo>wOj}oKNay_9nPZ<)ODE3wKGq{Y8AFeL~JXvg$~jR}JQ~RBArW zZxmhCrG~{~d``&u+``P+mC1p3RS@O~XT)$zN`?T)Ax@~$h~Pb{)J5FtVpr=kSdQ5R ze6`5w9O65Ela;cx3I9XaWH3+q`ZJkE*;6i3ZS8R7hi}ij@GEoCz9wK=7n@Q?Dff>4 zAO|7dxVwN6EqV4R2hO~Li@y2+f!YBJi&!N|@F>7K<ZLz+I)fE~uO+#J) zb2avL?joDNQR_UYSh#I-oz*{aQ$Pq3ro9slp(0RY>~3}UMOWA^d2Q$)Y}S;@S=!aM=+|JBCsGk`*;#JK;+u98%kv^#x8qVxnP!U)TV4a`v@x8 zkIyRUWLb@^;c?t#%(e=L;9y5+^E1BM{Oq&)`M4JN=?z2pAsKw-@#g(_*!|pT+eG*j zKg}k1<9Yde!2aRk^u75LVd(B_le~F5*Wkd-w>B|);+6ZC%UyI%Pz0>j(E+Nxo7*-) z1y~6$EhK;$6)^QU)Oofo#ZhY<4a{KcOhS zSp&&V?r+d=Y=Hr`8}{y4rTR2p&jJX>?)o@SnFluGmG#8FZC}}}1#zp5Z5Cu|q;nUY zsI*Wyqd--6L&tk7?N_?*M>0wQyEUv`*;>^@|N5cY`Zj!Bt8gX64R1&m>&)RI6f!!= zNC~_xzge?pG)sCoqd$z7ccVn5;+(TcdEI1>i?EWUJC1)4hNpkO{|KV2075w8x?#I{o-lboaU75jr%NwxwM?y2fAI8_z zFK@`%zDl)Ksyw*>n9zC(W^JsvtyU%1k&65L#E|uQg#k?jPwNFb?jJy>}oM{(XMct#!$=ODeGhd z5O$(OsXEAn*Eoy86KbGHLvLwCp@nKVw-|Y(g<*7*;wC}}Agd}x#>T=pAyrv{#O2)I zSA;@H-C=E^;QbkeVG^e`!G)Br!EvFlXXJ0NWQx+t*(m7J}f${8s2V|M2Ruvs`!)_GMT>IU^?Us+v4I{keA6p-Na#0wM zoaF+F7tPNOyuur1txB~qY0ht|u4`ueZV5qh?_pr)dC&H6{VbZ{3eqb#!mz=S-HerWb$@`k{!#Whd(dGNli3JYWXgT6ez6 zA?)vsWo40W@LoxRJdH@kR;X=}z3lB~u2#6&ZH;QYT#^ow=JI0+*Sgz_MVm(NmZm+& z%wT5iK~)oq-ZQ)UZaEmYdn3(`nR)Ywxmj-KQm^_c*4Ghd+oCP*$+_*wYfio5X=HA_ zBH1)U!LIAz<7LjPcm7?m*XfPR)+F%*g!QJ*81JC{B7Y;@Xwrd0m$dbG?XM%^$4AcD z@2*WmUSPk`nD(iLttmx~^$zoDXcC6KAVm{_P5eSE$D-@CNR>Zs8s6CUBI)dJ#r-60 z^M;zl(bSze$_(bXEofPjoZrk?{x&OTxt-+$-A*G3IiV_|*kZYjl;kQ`HUi>vPrc1b zs3i$>s-m5~M5BBsVpJiCV_(Nyf6nGQS?ss2XFZ#%Sjj6&U-Hn}tGraLE<>t@o`D}x zPF!C+hF|;x#6HL!XiroI@uD&6YMjXlESn{9T+lf znodgdWkxJDteV9<+O3*L`rluO27H|h8FTV0oK3}+t6d8O{OoDntpQb?B@`ji-49JP z+Im6CxpPmS@jfIZW9vkDl}K54@izIbbZbt(qG!XPAce3EDHd97%5!!x=>_ zDJ5-v$@@XMcHyJ z)4ZjkNfRBB$S*WZx>))W|mpEa&`0l~mg=wKFkmBTFK zG0umY<___lDp9T=mwAp%P0LXL0FAQAlo?PM&qu6J6|pvTRMuPoM`Kov$WoV=f(BP< zR@EH@0&T=BR2y$fAeSHCktXl5wvi_%J4eiO3$sd(GLi(8NsHb;Dk<`stTeT*^cZ!c z?ra^Z<9W^&d*d}_k}>Y)nDXWqnKQH-AmIl}R(jX^U*FU4xUq1w%f>|&d&YGfnTQOC zulILKnav|fJ7!K9(-Vk%+%l(kIPp4`SgUUbQa73XqPs3!X7S2Bi+h+O<;s?g|3KPJ z%b!uYfn^4!zavedV_)G&fyf+R&&WT$JEd4DXjKLom6FuTs8Ed`eARk61GMrmLz^vE zqh_qbYh(&Xb^SC)&ZreBZ)3@ket(i(rP>2= zI8((IBTa&NJlE0H{dh@#{{>Rk*9zCm#i+9jQW7YhX`9Y(b+(EH@S+A_MbL@O1T)aqDM#QN$P8n)X0!%)G zG~a5hOhHmbG!zC!VVUSD2S(Z{!<^itlSiuY_2C`KS)r&{NhmsE^W?_C{@DXd?ocn{ zTe1_EH27d>!I(VJh9>k7?G+A^v}nz5)Kl6KVcT}bzf=&LIhybEwZCZ0gq=%(TMP&%}bulx1f4i;Z3+ z&uy}+cw;G=L{MjV2kpT);g5tJEo(|JpI#Y^JBK`bs~dKW@Tru>t)2n%wZ8kDUNN2A zGL~0dUXMg>qK%E6+?$#&(5drH|Ls zB{PKmg$)+9h82Ogy5m|N&A4l=G4z#G3Mw$0!e1TXqY46C#S>8MGxKWV=eWj$2?Tv% z4CD6Vf2lsXe94B4vW%=aTXY5q!BB{z=T%{blBWoY^9UK=yU+dc*U!rFwWJm)iqS5HDF`F*JF-S=+|)3$+Zh; zR2}-E@Vk2a7Y)7zPCt#~|Gr}Web)c^7kN{|BEdlvAQW9%MB8}B48>x~zn?8%FWbJI zCfM!3_4K)suVx=YcOi3t{=C5K+C8p>W<9I7nI<~zZWmN(3gECa_WiDDxi!1C9c9Y4 zEt^{Td~>%4;a3#55u}VOy)7T}iIg}dcUoJY9hp99_gV=gSw5`;C7Ghby5Uiz+qI=$ zYm=^DNT1ne$ykJ`p{2Sh(FXwSqZ33&*xBp$;MYr1g!owhiaI zuxyd;Ute=}zKpK^`8*wyU;J}yDK!HqWuRM2e&&H$(^ADR6hq4|Dx#{@35b?3Xx<~4 z+zQ^OtzHk?`aLrXgyw6u7M5YXuh=+;!Z!iPH6dQ1KW`8emH~+yXS6LQfC}3~@7ADY zBA<7}k9%~;8KZW_kBiz=z7LW1#_Zm-LVmn{#@^IT>?MOTWpc9n&5Ds$4)dr~8F+O4 z@{j@+%5{te?u3ccY43GP`7?tFEBbdR!Xw}<58dCum}Z)2r_CwNa60epeFIr&@NKQG|uC(BRGxr`+?y|5}TBHRa9ytS2|Ng zk*HkZ7ng`d5%~d4xJ~Tz1Ad$PQDw*ipwh?CF9-B_PWa9Wb-9VJrb59`sb*cAjDiI#sUR1#nCao7x05{SfCj+$R3JPJ3!7=jrQQ??8^O zH^F|EmfSsJhsZ1ZOdWQIpySxK<5d&LR!K>%BJp8x&G~71 zL9yEi{gLZ=UCxPLM_hz(pxID@O=u0p#?$b$sv6t#EGj28@wf`gBiH^K#l5EI^eT@g zs-ewF2bgWe3*i8cQ)E?vU(St^2XcTLP@78JXO-bav7=V?&RwF*yUl>a05lsAE3y>@=@U zXpT?Y5kJMMHdc>n%oR${WEHPA#XcwWX@eAJoz?Utwy7_P(UtgHDaoK*=PpI#?92eV zB89GsL;hEAQ3ixeE@yCU^3}2&N6SK!YsRMG%~({2`$aW$2*6}8g>1npzRMhFsS~)O z4TBLHl$P-EMloLCVP5A;2fs{@t%X)RXE4V$_N(b!P0o>&VQwMrq_KMMdk6f!*5x6p z&Ro0<$#{r*vsMbOQ2+1$k40MX=FVFX){Ip&|9T`1j;?hs?vB7QcaYw0X8UqslDW7{ z&W2)%n}*q#Z!#6J2q2)S)(B5w6PrFP z{vtyI(%7P1ck!qGeR*xASnuu0ZwF1FetMMVZ9gRH3uH4XqP}#{aI6J#cSI`J#lmn~ z9w9MW<3r*ShBdJr^)+WNC}wQPd&ZwZ5H>MSr*eHmRtBpQ@C%q16hWfg{?e)clr5bb;5$0q^I4mxGcw|AUAXN{*|eVg5=KtK{{!HR7P2vg|f` zGlfB=3{jhc$O;k4l#LrFM{@6Pn5H@uEB~qGla;nJY`URE2}aJ+_3QhDH=)(*j0#v= ze_bhURCX(R;`xtzp+w-I>mC#222*IHSQ==!fG2L%=7>qO%}c(uwYKT?`1U-TS)ay| z<~Za|=vPiRB89bPFN39SUb*vbJ`swp1Sm*gdXez{rKh|8YhL>^af^t4Di%=?4-BZIKGnmZwPpcM zo&)5JiF|44$5kKJH4^z$vW7S3?&f=L5O(H!cGi2dhF3X0za;_gavY+nXs^Dt=BA5i z{vmqlWC#;+pY@f)PqMl(N`X&HrtX&F%D+-Nid(4SvkIG4MJn*tXG)nV;2SDeX^G;_ z=SW8cqM>-4F2*foXESR#@1Rd{s!E60wpxq_@m4%YHHMh>X{Nt=YNB0f9r*OiBaG-f zk6bFk9-CZf-hpjXX)Ot?ZKLCSV;24XodO`GWm<0T`i6ese*{dvZc`ZLpce(h^=no| zYo$J%5P|6JbJ5OKbuY`W0~!qU4oZ0wxKLHtg!8`q;{d-OtjL#`^9cl7b+Ii|b`4ym zk0QtMvWBfLldv@N=VQ6;sXGd8g>V_UNss?vRyk+6)p1sJ5JO*NRJf`67~oqC4ajDf zh`!_*9yK^bt=)BSH<1_R^LIJ<2{T){hGajOnG>jH^K|_g7Nc^)MnXqo+J5|=t~tv2 z0*_Mu5EMJyxfwlCyIF_sVv3(7gsX@ATqFr#p@mAGTW$U5S)?p==-1e5!r9o|)|CI5 z%BY`PMa(QjzCbaVOS8)4R`6GUk3r{OT%j_kE%7~(w#{gdQmx9kn?~2*B0_aLn<$C! zYneB>@<#x$Z0I(`UD)PZu(nphal`t+@5^XiEES^Kffru5C$X{8bk7BTApWD1@=hbF zqw_iOjSjp6 zUu{`3l-u75**Xf=F{HLKq;7d-ljl$7o!M?;3*aiBV3p!C1|Zqhh_CA1lZry(;CN^* z(2BZ%rTpn^8;;CWbhpcvyR(+Z(`)*nHrJyWnt|ppILHB1LwZk~l7Ymv);bNm9d&bh zQ-8Q!XGz$YIT0?TNzge9#^vV^`%ytfsR+Lr)vs026r{L-zr@TvJ7hh@yC!9D*pNhA z)z5yK-tNLdK(R)9xcdsk?GXmR7Elh7WPcx!iYg`saf|`SI~JtpFq~bTd2th(y5wVM z4tipFqh`9lQJ8ZR&hP4m`sBuSOxKT9le^}TyBBJ37t)I@ZTG|VT!i$2Po2f-@LF7! z78Iq$>2jr3DoilUt+TeU=7c8_HCR^d@p}szx@}bT%(8e$KXfaMh37 zE5m11`Iz17a>-9mA4dv*&*BvRhB;ZH#UCg4V?c0=dm*xo;)Kc@xy;OnT z_^v!ykArq@U0I)%sIy+gCu;xQ5Gw<|U&Ry?8aj9Fc6<0Z>*W|rzz=L_xitO_1Za#!k2s-IT*6A~Xa zv$#$=d3-3mxR->kC9*uE*gk)8UjFuF24@7v9VDx-z?FLBmqCwv&lC+x@JkTJ2gv}w zH|)_PcoARm{%!eJ-jtax{JZuso8R+qv#urp7FG`a%DG`pM{b+ziK%Qq0^{@V=^ssI zp3YaiNABFWEEh9X8(9+RB5z{(?EHphT`O@Qsb?Uf;~f3^0N0 z>C7lfJeXpvnyep{KSdaM?!i*8S`=bq6{U0bBOw_6YmR;329LrY?~*0s%~_QsV@rAuv?nL;tf4h|?QWe$5QmFVIP2 z$vh*&0UtYpSxg$`Y7hAIlZ9$uMH*C!P05S5q8bD0mh?G;T8@==k^oriYhxsL5)w}B zm2V%|kPR?rO8U$)WNRYLww5y-Q`DFb4v3_K?^hJl)B1E_S4BRxz7{tpoG@dGTZs$=$kCNCxyo zmq;K^h0|aV|D{QpXD!Ep{`MkDrS>m?CiG>%~2pNgWYxY3&~hukOQ>= z>K?J&qtFU7ooy07b?#3l8X2G}YO@umn{hYzOvk3_k1vzV%l8;8^bPSMzxF|s5%6q> zMRhHlx!m>4VpIo-(=S$xq~FJ_Y)_g8&69bYAgnRL(=)@j(h|zLw!zG(jNCzINe7Hz zxCKm+MdwRLH7mI65{(Tu0fa^^JB2b0Hj9WQf&bYNykgIj%kSlaQ_|cRb+s75uHHu3$ zaUqbOo)^9=q4LJzRF^u{yaUS2hL{Ut$*8yfGeOVe$mt1Gl}KV^CibE^_<+ZvCWdU^ zA+XB%C#ifwt`J7;geA-}mEwgLDme=UnuP>2YmerSUbL}{c)$_g_&qq5I`JA=YD`p^ z0ifmBpwo8+?$`8i|82InVG@n5l5@!sovc%M1C%1UOARb@;iMC1TtK|$-YIi9_!R`P zuQ67rbQCCgAR-ezoW0sn#pHe!?`AlD#{qLRSLFv#`9(HFe%d1rE`5?N?dlIcbu4X4 zZZ+rgOJ#`8kEf})iLz`Bfl&{Vs2h!w@9YVXE2`VE*=3_+*9Q>-y>6tj2kuZ4;?QI3 zfU8u}0l1Q_$+kaz*J57FxrPVnC3o3_#OPwR%Q@0mH|NO+69Yn~r{h(CF(kSnq+T6| zN1G&dYs98V?T_vnD*6dX=Mv$>^HdD%7esgg?{l^#QoL%p@WJ>e3GwZ}cZL*P!7FeH zaUueAC?0BiM3wD$FhFJG+(VaMOPWFlku~_l<&il#vGi173xZUaI>o1=s~85Tn0Zka zE>-HqVeg{)J|v|eh%-HVj$JAS82St8;l?nPAfkS5>SnAHZKoza*6PhKv@NkW-7j$X zdUx@mP;f$zs61BwJ8e)JCGXfh>~q(%f4cUcB}h1Ra3GEL?SfsV9wY4`pbCxQVyV#*@ z#O+c|c3Q<_#CQ0OL-{t1(iMWR+{hnObs0)<&gEz}W(TcLbuQ_rwrMQ|SQjR|a7f-_8qn!8`us8fmEiuakv?r!k4 zZX3+wjAPN5%XoEtP~PBIi~xplgkVWj(SL4j$T_N~4YMMJwt2p^HBK#`n#e=F$5J>_ z;l{A!l(Wv1Kz90d`$-B`p6O6p;ENynX%`gi``TKAx@>FGE8&q=F7QiPuA>V~0lX$o z+CG<;8erayl2c$X*V#O7phFy@3}!v`e6(^gRaF#RHmv|3Y8)Hyzc1T05qDd5EXJNeqSxWv!yxW*O*Z-S@A**$0?1zQ zfafO34mTlkhr#Zu__tP5$OSGTZSihhs)geOdCg*)ULB%DdeuN@TZnf}Vm>M2>MtAzz6 zO~*NKMaF=+9G1;JCg1oA_wW=`VC;=gVze%*;vE&zv~-krx+5TCpOoKwk9JBdnMN7# zLeg&}RFeiW->c)J70M~u`h&~otr*~0+FGAVO_Lc7D{Vpf*$acc51A?6yKr56FrL;i zb|J>TWjtto!!6brn0WO%J4TBvj0>jQ`0JS4-DSdliilfnrJ*q{)i-MnbJzx2CeH@T=t~qDNbTkW3HUFLymYfQ?8u$#a)kLPWxN* zIh%RWsNH2VBIfg7f!=TJMjRD@Qr&bXDv0=kzb;18{?M{OCyAJDpy98&Q^EfA#Ca+g!Y# z#;R)Q{G)B{BL&XQ+9d}!HyvHBW9tfD=i>8A5nQ;W(nDHWt-Gb4W#NP3LjW!zhl0)W zVv4{;=-$7^HwG|KKIlz$+95oBf9?T74MSQ@hZ_d9fP7T%eL}&M`E&=INm;Rz7EI&7 zjMEXMP9h^2N*K&8Y=c{vz^^E*FLH@-AzFksG_|W)9rP5Pu8@mY_-#6izY5{ClY5EX z?b;~e{HAWi4c!3Sk#CCM3cjj`Xk`j>z_oWs&4Btwb385I0)}<*J>2-;5H-D=rK{6T z&Yn~87J&+;(x@Y0pi3SOg%m|5F8!yq zLyUx_)jl9#1UJ+pjBx3q7=)J5Go2*h*>){%c zdJND##n>+!3ksf&={7x0%^78`$s?UsO{%L-Qqy&u5d&w8yT*P1a*-#u-AaQ0eA;Qf zJTXS1d(wFiv#dqsM@7}j)!@S+J*wsN{~M>&z6SnUO_1(&8?Q6#xoIHk9o&{DsfW2K zHH&_ItTwBW{(1B1-Flmj>q~Aap6_w#xAtL-2NLmDFk zakXLz5AuunLGBTPGIv1j^6IO1xMRpO-tNCtW)adN29|x^V{e~OVj;Z^bS;T&++Zz1 zm?g1qDy7!P=*AHFz2wTw_Ny4t1>C~kgoCTShCnen0{Iv|W7V}&Nw6~KMp358C#8u& zJ|b6gc~owaZdG)xwdn}0&X(h7!!2ypO}UvF|B?yM{D$)}F2~dFTghrk3?mhpe(HL8 zX{yu_0Kr4&vkYl=L@yjvBCWd2H0IT!kp+UYyuU$8v;2Zy+-=x(u`jAt`&>`8y4q@v zX1|UuHfYky;GOqSgr{0O@_CrhAUrB9JSw(e=NV1Ff$VQHuf)SPucTF1i+bJbw0>oA zPZyA(T%?vs+UV7Sg6p@6b1z)Ag^I{iG(26-z7pw^9@L^918n&HW;V@f@d(Ue_OCt0 z5%l*p$Uzv^P?%LPd>q0%SdZa3d?m!9&pabaY=SP}_gtM;s z^GV8DiJ&W2vjKT**`u_1rDewEX~pQjHifdXO8YnIrEEW-ZV_7#xx|im&}zS@m6m(D zVSv%vBFMI$_q-MQPU7pAw`A(}6+yZYBbClXGf$p9ic*==2m zv=>n(@S!K7^i|CHlNlURaU28^%Cs>sF`)3J7f>5QA{JaBvPH>7;xypFs|ef2YYt|9 zVY45q#T%W_TLo^t=QlQ{%rH$ez=)G#4(0Cw!3K1czbp zkPTT|AYpXi&b%{wRT}%Pp%`N<($@Y{?6lTj*w9EL`;>$Hsyg+ADjfgvG(2n}}7#2=D^7%Zh;I=sq*MOdGS#aeu*1ylhUyYCW8@K>>}Qz2_6 zbSBw+qC*dP_7K&UGs1VIu*YS1b`D245lCkQ%C7WDR2gMfjnDc-6Iy>c%h8f}{CR2N zb=ETfkv<`};pVNbI(9qiHla9a;3HtFD`feXCEQpO;3$&PDMg!VdnIl>QEef?5Kbui zIt4F38Ytotg(4ycZ6{z=OXJY`nZ{Xi*+Uh! zSigQfZ=xrKqP1NxowYRR>{?1db;scy$*XeGky$(A2%=&nJ)BUpS32t#=O1fa=Ll5x z_mr--e4!xqdRT{TT)06LN)GAGMl5<(tre36kM3+EyT^W#yp*SgD=Paf zP}>Gn2A8@?V}nzML~Ip)t>ik^UJ$?gq1_};Yui2^;@OfXdkv7C14rLB_N388_n<{e zx(A;#u?_tsQ#ZlHrJtpJ?}#9pp1x*#j&dDGu+-CX@Jgv%HYcp;wv?u1yw`bJ8e?M{ zZ)2DmcuR5|S(M?Ou6cKoJewE&O6+WPx0-X3bDbNV<=kc`W*hKnM!YyFcuq}DbFTi~ z7apYvkI7Be4Myupqw=zGd*P7HREHP3zG1s3;x-Tp02kq0GiQ3{@j7M`vSCilHQI&z z`Da_69x>z!3$A+BHpfmFnhj`Ujh#%-Ef}qUl^6`rFkA^mkd(xw}Amf@3KgS9z3y-z|UWS`BvfEzUC!|0Cd))bp;pMVXLv-S8Xqg|vn7TN?(E{3YJ+HcLuI<)r8bp0 zaj`1>mj}RRsmb?0P=AdHiS5*KfTQ+6Dqic@ivp5%qD^~z=h-tTIjTi*6&E@Q4M_J< zMqke{YJJ&JO9?E}-^!v6@zksgTkWFcJ?HtUkBLSLn^7R}ExRX33ea$5=gIyFVWPzH zO@CkRNq7BSCL;APE#)XBS6T%Lliqk!$uVn*vamA){&eS8z+Se8+jKoHM60)Gw3_cP z@Q$mYp|FDcjq@bGjk=A2Lx*fUwU-Wv(^e+O(HslU|57V}o2oQYMQKnmIrXmSKESP* zK6tP0;DlnXJHa6yg(`~tLOu4GLyZ2zXp{`L_=JQ(3wJyPu+YL@X{dH-eOGNUkj-z=F;0|^vhh}5g` z`h(@;$ejePU%fed7){Zb3Xh0w9M=tm;<*EG1EuQnsr0)#=VTdU$uzh+rcHS|7$HrXtX!Z9B4!;6gaFjrdU~ZP+zJ`VP;d zMTkglLMx*Q(uUQ-{H{(|zS={_tokzo_2^jcu*7pCdL_oxO(EkrILfXPJA!O>y2V*x zIS${(wZ#tE#BRI~+hE0{oompz)=^C~241tcNEQ*lUIX)9k?UfI?!=|NT=`+d1|!>v0bOgo zL~mx1S1usao1b?`vBCrRG#EufJ)?~1Tbx?@9_wbw>r5O@b&FD0{IVP zsxW;)pBzY^P3h)kiUquL{+#8LnI%r&8b{krY_z?AQ_{(6L#@-Vi!5er*Fm++5^T8f zLv##Tq6MbnPqyZ3xsGoPjuWtEudC(1!#C)3TxQk!A)71U*<@`cppVu2m{<2UxPaGq z9oOyy-em|FR+mgXSB6he#e+OZhnuapFPU?|J<4A|E~fj{wp4oMg~)bkORUIgrn!0a ztX(~8xRL4$45j0$)EIT}Ssdg6hee_hJhG$WzPQu6wI<8jdqcPxrN5Qm|Gw4FW_Km) zXyUGGS*KsF5SpXo^_FyZECtB-;JTlz-C2Fc9q{6n+U=ih6T1*#i)Jv`H>^Z|c~16a=5D2(Rpo%R&f;He`Q7nv=`FF^MAVla_T7ni)H(jepb z702PNBW9NQN_<8n&V&NCi7K*_l2q`PT*Ykagp#uwp+HIAu5BP=MEi!cr0Z%i7lD=W z7G=}MIyqy((EF9hX@~ez5iRq5LWPkexRAMcILtuN1xvt4!XSZtt@6W(Uqkp!r9+#A zs@p_E6_i-KOq*oS7h#S2tjnNiId}h33d|LVp)`0mE?_%R*yv}U<=yt8yN}dilkCgO z*f{71ZZ+zKtNV@L&{K6n=4O(VwjZYg737d{8d$pT86zWRd-x~|~VolOrs zpWXt|%fB4xx@h@6LD>(&o|nYvS7eDlollwz-v%+~P*Gy9$^mC~vo!%X8-n%}x$8=g>00gd}Rb1-8l66Nx3=j)g6;g1V9xipiHHd?4w< zTo&}+Ot!P0Tqbz>F<*s$?OshcJ{6Lhi-+A4OU<0YjQ#kydx+j5GRe=(@z2xB;Aw>e zq3Tr3wI1ny4$_lfi&uNLbmJv0)ZPKhwK}kG*Vlcn+D6vpc9yHNxlL#+p$tU+ziF%w ztKDBm=xt|R6(ty6iK9;7BydXqS0!f^76rF9P`ZSnW9XKWl#Y+?hG9S&q`Nx=1f;t| z7-~R<1_4E+8R=$VC~1&X5F`$M&p9{$-G8$$_r<<@-@VrQm+@ZolwFUgko^IC-zSAQ zlQJ2OQsNtz{pCKNAbd9@v%Uqiuqx{?iWnXaQ0~Z(0 z-jmi*6d!Jr+EV+qW5?aEC`&9@mTpOY+e%l9Lq2)Al4iQt9llWD@A(1Co$dma8{vAj#grd_!eBov=Q}l39)1sTqfqR zFBLa<4mT5gEI3&uOsLvaJG^M?WxbuZvB$hwzX;w27-9N;=Ro7KM-uNWgAO!j9ERuO zOvm4b0hnyh0BA!@e}sqRP<*qb^$IqljT<-l!>+W%qoH!md19EiggIyn=Sk)SjRByd0j5CIg*#(w8Mn`rEjq0i#&buu^R11PBGPz?F32()rsMM8^SF&p zwL0;j-v$pgCgPY2$G(4hMv0qGOeC7Fq!XXt9t5%U6dWbz8GBJAq>M+dn(~0H@!IiK z7Dkz?xYd7{^TNIfknqBe9-3Ar%=Vb-9haV=RXty)8yuH)j@~*zVbYfoJk{|CDu61#&Phhy~q{@mK(3|Xn2iJN}oPuHo*~X zDvkrfV$yr%1rD3stht%L_TK9@3vKv4qm1n*4@>@`6T*M^rA?#kuDddd9o43uFdxw- z-4t`X^!j?y=boEFXJmzm{4^1d>GP0W_&NUPoz+iQt8RAoYd|k)rO{9VIv4A3RWwvR z?PtBjY;u;JE6>H~R3dSHMJ-`JLJ8|trn2$Lo%rH=$=N-%3ZL?&Spapa-Mt;cTk0Bfka z^jW%9eP9gN2eXI18GR#>rVYk9z_mB_%xX?t81uS_NzWK{*xPt8W@?*w7FKK+NSQXH zPZ@~f25=SGEs9DSyc&eKq#U&c#a(`CV%!u2k)+%=JA83YQ86*DaSIUTI!sQUkT{$l zAfh?aT7U@k_P}iCaK$T-ls~_$2>nWqp_BVY=u$SSSvXlSp(cOcNNb)j(mZaE?L&9r zyvdSVyx83H{%k_|pZvKBe&OppLXD(4eqlJmXjx-#uFIk%KU~ABvNNk-t#Fvmp}lGP zb(}sE$t+Cu)m7W}Dp~B+-}FT2yZqDx;me6R%tpm=dw%XuJS*j@_x&wsHofBXatziJ z-*L-W6fwHKN`aJ|O=q7bV30Un{O-XM56dP^&+Hlw;m%VpXE?-r8*aH=*+{>VX`82K z2VqKtrcx=bQIR5^y!;s4c!A4l0^E%EFJqtq)tt_;dN#<`-w{&$1@iRFq``qZRqf_Oqt6#k;TkqxEMjc5+Zsh<0E%ztOs z&3dY}*cT;nmYRhOM;LTvOz`$gWac^)H5W9LO&U0!E;*f z&Het#=CJ}0W(W0dY19qi<7K}{J%4(65@Tjw?#ye1eE(M&iIwE(qhw3ij zKc#n{zm|_lxC=|YA?`&5wB%M3YpO~K5Xx|IEw3Q=ueAvT_wNg_jvmF=8A5)389r-x zyyY9+HW~yF{y*q*pFynfZHSPL+eEi1^GocmwbNoweIw~N*Le|oCS~gnkZ@Xl7)t0I zRbbGcG(bg;1A~tk+S1}>hK`J-!q1dv<y(RFVZ&?wim}5s0=A3&V?Q4!I zi>~sgbt(_!qK;&ItiZc-rweDrKc~!Sv7N@-O|svmn-js4AeZANqR%whew#sidptG& z-ny`D^!rXb3meC(J*|S!hp0<@@qJ@Vxf$4k--9slX(?Tnj|6~~lV258CwZE|GzH)!DPqCe0Q2 z?P{4%ICHt(GKIlhO1MB_2ms_G1=6T>@4N_KRlS?eHz-(zPok=<8JA~kQ)IV>U_%lv zn1|7utBO=Xg0V7q9n+CbvTV>D$lvL93ci|As|4v zr!AhzZo24?Q5?zMGO;Zo6Ld^Ndi)P7miqOwnnnAJ#BG__l{YYCzMLJ2Pr7JLR&K>YNqJ9Ks^;t5TiCi6 zQ6o*}*YQVbA;FOeJem(>&8W(y^me6rsGe%%jCcv&7H*0~H9o;ZXQ7H@^?iVlnOc5p)6kJEwGI=#g zye5M(6~uIxn28g?DXZF2>cr4MLxc%VhCy4P>bL?0T%>Tz!FNobm$eS!Y82y*4Rpnu zj$BsFK{Is<&SCn}B5km#QH?#==h=pMH}Tp^HwtG&MyE|CN5kD`J?m(%(+<4;O>k0w zWun$q@-t&559)y=E#|Dd;<9fWJkSX%%=>^zq`d%b^(RFSwjQtp8D&956rja}V2{fy zHsjW}?jnz}(66@2=C-l8NfP$8znu}oAu7u+^~{jgied`lbEsd?K*rlex6w}jMqPQHa?}=Jhd-qabuMkXHs&>ngbH>B&hcsS4qpgwA8pM)J@I<{HAk5 zwel)+Bl9`0GA2XQ(-7WI?UQ!29XI3bA=%G)YRLB`Al7mYsnSv2Y131`QSzOcfVn8o z14$#vuweyz+K$4l5(gK87rhcyvL^*WQq~rx(hA}3^?}2!EYu{2m#rU08l2eN-CDW4 zQ+2xb3vByJ*hzEDzS#qK7U8k?Tnit#v!E`PDKI614x{-+dYZw&d9pX==2}gX^CRe8 z=x{v~wC2KFYya9GIBGpUa^&efO|wd3wOcjt?^Rrvun*?>i&y16&Rs{fJrjmuKd$XH zXlQ=O-hPyFqX8+#c{iK0{d(~6q|KdP}=sfv1d1A)$u1VzD}P46v9u%tD-bfh{aQxu268RNEf`e1tQ)Z9Maa= zPu6Cqs6EZ-oo@+*{1u}b;FCyxVinfks*VqP;k0_`_fpntpa(zqjpI$AXFO~(?M#lA z?SSKzAoKE+o8Ir^4-!9hdYTo$EunP5IpBKqwAZ4gBKONq-gg=!zvN^i@oqRyjE^CB zGf_~rY!vq~!zZ>zlFLw zwK9pS$kKYFQAj+W{tth$lJlnEw@*b+TBI}sK9#>6JZV^;9%|t54@GW&&#{UTq9W!k zKLFb9TJc6QIz+aq9?2vMn_Bf>yubW?9yBsF#=iMA^NIs_K$*fkWC+t5e!i}fb4_w+ z={M6@PPL_#DHJV!HBBN`y}uSzRfFC^CN492uEw2&MOWHN%k#T^RQ8d8bXpt|cVHHG z-MmUy`0x0}Cz`Ax5}L?LUuCdIM(@JT)aivhWnuQ!_e-(iQq@(pOM%3jp-=97@10X& znF4Z>$&cYV2POgD5ze7+D{u_C@UF*wyWYGA`w-YenA>F|tPjo#dP0*<#(p`3Txy3# znDr6ldiJxedg6+j^(BX7WV`pEoaDpzfeq5ug2q8t5-rFimQVW&Gk&+`eW5(hO!wqN9 zLYZxiEF2(BdSE6pQ(!%vB<~XqOo$~eWMnStpAU1?{bx{anQ~fcjA_vj11AbQoKN&n zv(Ml#Y(USU%k6f8ZN_S8LEl~hU-3l#*0s_fZi98M@B5p441si0=`JVJC(+# z;9DrqMkOO@jXs<2JUA;#c_4`$!t_#ULv?BL3{3@cg%Dn0maXje5S*z z80=FgSL*r19Vi}a+^!KA*xes3Wl|P67`e$@q+=H9V<5(Aw-L;Q1}{;_G!Q&gTZiO* z@qJqSN){OKDX1R+;o9!6#u-n;$rtZ6ur+N=)5~$0n;YU|kIEx}!0zoy+0-A5xnQH` ziw~&um|tvRAx$tc+E8~TiR4nrjjan~`58HA2!2yE8^q8!yf4Yc><1FozgOPpl%%XG z4fjuA`@At&`nCohr7z-C_U)YzdrtMvLL@=0P=e9_DV=F@-amUA@MNYc@OX~*n5E%* zzIk4j5(64OEK2&T&8Ui0i|N*yXV#+C@MCsmysG$vI@z;GHiGf5H=_oAChGzLG2n5p zX`lb2k=gug%^uYOpjpUa?Z@7glb}!7Mf@G8IFnrb%JIr!Y)-)xi!c3)C!zI?hT{9d z3L%nduPxx$1Cbk`t6hJ&|3*Skh)@X=yq*P;_+O>Q{em3h8{hQT&!ciANxN+KD!`QX7&cF`v8k~qZf-5hiOWksqB7`g<-)e#ZxpyBbi z+{aJ>hsaIMOQm7!LSGhfdrr#M1A4<2`R8u$Y+gA;)Jj>Q zXQxE}7W0x_h~Vh!KE1mru2)llynHg{=-H+S(iGq$n?XDTKEc^4RII z=~iAf=;`bqoudZWb;?++kg4Rr+z&v$cV`nCobc@KXZfcEE)Dy%=-r&*x=WNy=y=Y$%H0xFQNoXxlBJw}Upme{ z|0rANOOG7OEAO7w7Vi?F%N?)yCQ3IZY*G$aU*Qt>oZZyA7*ab51~-! zdF9-nQUG#2+Gd>TKs4fW&otk4S-#89qSG3!5#GGwBggSP%bs==D*z3Rf(}i|dKNZ~ zt~%4?f~-r!7pv!Rdz2g~KaXs2fMv@NJ4R=pRmDTK_veC~B;KY*IGj$oT0v@gwU(RE zO`8DN#)ntz=a%`;&7ah-hN|Vi`Vy? zYW0G|vBmLSqAYypAjB8EqZFVb^xM&OoUHY842f`3j%FUV~r9cH=r-vJ<~1 z#hE?wN3q+SVCNC($BBE3p>5;Xnso+|QMcrNIRLibfZPpE-`%pM$)eN$hyX-^$ z0&A<40C)<>;m6axRsst?xnR8Xbrh1GS*jTeL4EG*Ud%oJdh-0MZO;|e;67H=MkOx4 z;H3q1$HVMwr$psXy0z9Wo2FfUdBxZt-xzi!JHV4r?|;~|G`RQP`|gLnD;+YemSB9- zfHwgPi@E%zXz0sa7`gBrNExqwTHb0vJyXGz=J0@&L5CfpCZGF}VBHu>r5WkaEE${j gbKVz$mz$^RYi0g?FMXs$L3w;!?eiwFP!00000|LlEjbKADE|5w5A|KcVcS<#biSu^K_V<)Lwr`5Hb=AI^= z79t@D$0Wc*LXO>xzx#gxkQ6BrBKROhQFuR+wL4$=)rhy84h9#YWc?K z(^rE)hgW@zZDdkR+UF*t-nnIY$oGwwVIkl2u*)QwzyJPwPCt{`l9=Fy4}O|3>cKgB zj|uW7%v+l5C5RJv;dX#Y2}6&peIft*61;%&WjiR;0?6~I_Y!>lOg=LYv86|V-@YOY zc?8<+HG(fcljqOm@4x>xT86)L9Oyk6Bg1GJF7%)yd$IKRn5IuhF7>hKd+?f%`L1Pn z$Y0uwf4X4!Yh6kWJpDe=v~Mnu135kP2p!i(yd7iK>Gt{q`@na4D%8* zX#1#TuqPMs;&l2SWHRIKnfLF%OY9-b7%^{&cxPM>nUL`(qSkZ9vzCD#)HYWE*%UF80F)|j6x&G+YtGOuBHmUP!7BIw->rvjXS2ky^S5uo# zUpdfc$a@7{{L1y{wE*8kVAA9tH17Ru2Nl;MA%@nt9&>ZvrA!|%0Z0u1wYXP?h zKHd3}Y%s+)h(GTGj=nF!3kSNNR`V~X@Xlu*Cb^1yy=%NHyh%=eyE2zO+^sQgd3Ifm zqL*qlV`8+>gBuZdtx$)3%28rTqu}Z?SL-D!2wu$mgn=>)em}Q@uc%Y@IYYOK8N3xw z3lNB5V{DG3f-XEo{nQk38PS>(PCeFv(6T9y*c-$$)yvIgby2WJ)yh6-q>j=N+iM4l4J9wVRv_D^@yCAJwR z!Q@_%85O9`=bHoH4I~zIh%}Z~rDG}}2g8${2qAo{@SQcNts~zAi9&Ry2WGrDvMwZE zhQ-Cq5C*O+L>Qjoc-4^odDWE|emgKr>zg#ol>6vqlxa$PmKEG#-R#RPg!Z-_}P&R-*A3Vn1kh%HAq0dHA8+EXk(K2cz?haf|* zp?SY_%VBYhfC`J{cYY}WRf(rbLh9vbydW^e4<8U}AuyHCD6x{n5iNjIu0eYBQ_P|FBFbaI?AmBo_XaPtp(bi|e8{^Mno&2&u z1enmad5ei*Vqf$C6VXS`$fE9NQj=mSVDkiJ!2`6H$cTwaK%+by{kF923{VsgV=Iyf z5N-QA7s?Zb5Bd^8+rOa1#}@K_pUrGc&<$c9Mi0;yFH4DPKG|#^HMhz0tRXGhTX9tf zD~)fl{Mc8%I#u{1vnDBElzO=;PPAU0TBF=>)NVLxsPNJyr-*C?s4}S5#Z%d&8>ZS3 zQz`rd8FN%KN~LJl%1{kPeG_&%Q!H^3GxUKxAWvA^^|Dkp-I~}+NxWgLov_wvN5@eA zV=Ii6MYSfjQjl(#YBx;P-DIU-IWG0sC1kKX3$19V95W`i+85XoesL`)O`7gb{N6!7 z^S3E8{me;P7uW(GX9yn=<^u?PWY3OdnJFf7`Jo64Ln@F3a73*Qt{A{Rm{HF`7MMN( z2>y`4i!A`b1+|a~UVK?#a{(~1uqm8V5I9SlVUxc-a-n&Ttf&>v(=UZv2!i*#)B^g8 zc5)Z>tMN*>FK17^=|9j!u>Zxk7AH@A*gs*@zJHG#_doRUzwYvhI_`g_&I~=x`hywn zFP~19OEl;`42fX*qU{eO4YccX z>}dWPON0K_1GK+g4BroD&<{2_n9{sZkBezD-TFCrje5hshr_?en|xVuPN;Q@Fo$rR z9Gp$QVd2AJ;gd~1l@u$~kH9*6lGWI^!QaDW-c9Sq7$elZ{h3R;MkZD^&@l4hF!HJ9 za<1OZ)*Qk$GO>bu!@`Hd!o!@kT#atLENbclQGshrlVh_8Z=FpW|J}Cyca3OX_fF3g z%T&2tcmaut>`Tbttp|xe%itOv$y5sBSorEv1ZB7gd2R7`Uq{<)XgdFh zxvDZS#UswYysIGWYjh?C%<%&vtLImjf{`g(AWw{-wqq)aPqn?g3tKt99!#oU&_uU) z+5eT&kMI6@{O8v{|NVe|{y*lvJ305g55K%NKR*8a`t79mv;T&^yM6TDUqAf%e{5Dd zxMEL_bpL2LHS8#S5mH3oA$IQHSHLDoAbZR2ry#Y0BTBX>cupc~_1wSD+X!51U@O~y zIbBSBnEdiOC;fRz5$=Ia-ieN}+N6&^dyh#hyrJ9a_gcn#_?Uhs(?f&_ZCb`Hc974Y zMuXmS%iv>x^2>cSWbTJv%Mjs$Bjex1?)*7*mJDWlieXK{y2xs475Emh!WywJr05`uDr{?#pPDGD^Bj~O22>{8o36!04Q0QyNkSEzbLOv+)4Rfu;|Qd7Zya`QycDy9q4?QBUaD0-?+AD&TiyZYkXvfeI< zE4(`&n1?39rOZR!kzS)YG@r&EWz?h!OLdc} zIfhR*h2?JQ_y$o6)y%l7;0D?vYaIbTCAScYtT?U_nKMmY`TK0Nlw8?X-9ag4)KCWPNw~VtEnaiD&ViXCszmuxnU4|S zQ(x5xiId4TKVl{1HB~yM#v0kJ7#V9FOMQg&d!4Ym09fUfzQKZSj66&&v4}NBMyJ#1 z9*KW$JDrjE=TBp;tT8(2ME@%obXN)5zf%ud;{a+LK>OeT(y}y?g0A)PGjQaqF{7RX znLIg$ju>{(2J_@V!ctXtbp&c7{7&`gwluX&6{iu2noVjeYVC)MFI3&MA;0_$^0~a! zbnrEU+RjeCX1A?-^fjY4fQ%u>Wj&QUr@Nos^yAo|F~JAnTN=E`YDm-+B_I=0xnFpNMS`;<|UHx@?g;5W+gGza9n9<1AwNP`$`x~C| z&P0kd-=y&(vy2vM$PJb{1|4j?2<*A0ril9vFsPs@QGVBFz43Beg#}|e%kNa(AX`5& zq`;ZbLqTwb|2x9PRg6$243EeZL-DAZHjc*|YLNU#MshXhs!D0C17k>352VL}K6V+b z(Zt~$>_+#Q>gq3vlGoU07ue!VUv7sxO{U@{d6@*83-S&m!S=pNiRx;x6*tPiD$0ob zt2POV$TjlO$j4UYLv?l8*hbbIN!?veHev*_e!aS3FdC6(`gyuxx%x8M&arhbgucOe{}KC3?$W+W;cSn z&MmnQT8g=oBvi{?zWBg1XrlH7wCy0WOKiigR=jQ>{FE&BMhe2l&9zkf`9WT*Mm|#S z9(j^PZ82HvW7U*~Npx04As6yMn9(_v|3Aw3SNMOUpb*I=M6|5Z*x}BXHU~alNrK&z zmf>I$`?bE1nUp!Fvt7xro~(9Lv`T{Ybr~dW>!Wv=Z1W4{kPl##J<;!VxTW!CS;cbXHk)DC!r|g{3uL*wjwa zH8-lQQEhputpV8vWCM^@-QgJ^7beig|5RC7XA#;F?5?-j;BSM!4gNOxyHEJ5x`0+f z*n9dA-UMF(d}CC0gS^QG8`y1Nw}IUTcJ~Q(RhJ7({JKG~vb#N^Q3!IUl-kwY?D2N< z{p=0;Ht4$}^xe%-?8dp=D1dwg(C?^jZ4N=yM$kv{v=~Z9o)l372}--MztfF2Hh7~2 z_CgCZ5Zpj;2*Dfo_a+G17u0buO9PEQ{Klvhg5KcLk#6KDBV~^5VUD~GCXq!gc?4!03 zT;3>}M$sImqUmki)}4qPM5vzS&5BS-b%W|$h@`Wtt+ z6Hr}swX-5rN&`tGy1OZBtjc1CUO~oNHD2sgTKqHY zB#}y^XB$1Mp=Sph0||nBvlU~>VpMh$K#**(2_9&4=w9g126h|R4PjRiJiyHhOy=iP z>ZLO*>mY8NfG*hPVCovlt>=JjptOO~1}7v=94mHw1DTwd3uIklIIn^g5fK$$$UD(q zU=9deF26qKWsu6CtcH#C!oXTOt`BfP4EXXr;1g^uZR7!;L56C}rt|GC*@}J9RIifn zGyqTyS+Yyz?3q24ixb5Ll?1|fh`o7qk;#z7nd@@t!GIyk2+0PD*}2vjc2GMkk}oQo z>8oUBLoF~z%on!=qTnsyT@snvAn-+1z^D+c$|>6tgH7L=wf1&9Hz37sXD)r4)1ETA z>e}##0mQ^38$MOz?ubmCEFCAB!^ct474r(7WedpotfMdX(~;>RK0g5qea{ zCi#K@5(^NzBFLZF_#y0p558dA1}0qk2&^n);35KTZW*_0TS7E)_+7}M>W*JIb(7jx z4GN0NTY%#xgqE198_1JeMNQyuY3Z=>`rq%xbZUg?y5CjZmKyO@9V?Q_<`qeKafwsa zkdf80#NUTyNh3vvB1NhjMFY;`EE4!dt4L9dOs(eO8sS_)iWTP;9+)CvLfb|b5GqGn zYnXsHj?10^^}t7nnYfHe`g;A(gi*jeJf9;katI^n*-woP_hSi;>W0*{WkLn1iD=X+ zGI6ST@)DI@t(6i>t8mc-UhriNnRBheD1fV#5G7y=nYjQn?D_0vi`OQf30yMjK!&;S zJw=N)%o2;WMxuT&iQ4F$hz4XgvEYBjOxbRx@@+JjEEtwL90d^c@VaSlpO*>-w zG3i0Tkx?(w0TJpaAyCK)F1(2Zt^JwEs52jMW&n7$st$$2!$*Ww_CEilx<=}!ANh=bA z`*SF9PgzxUUto+cuE<0zkgKw6NJq8i`E(a8UzU~sc+D%k_7Vl*-FEK?kCAqcbrY3jjS6q)0DM< z>{jlz&F)z0BPfgMgmj+`JDpA}xcll-M0KyOMDKP6!{gJFPAj^)PW*lr{O3Z(KgT#> zHMM^-N?z~2t!vmz+*qf)u>G)Z743Ibcc=yY%>jE^N$?~ET}NM(DKk8TgLa%bV+YW`E5O1#oPy29b&tYt_m zWMqsz5#?gEj0@^Gm&S-29rDLb_(P_l8X5m4cIVH@>xD^vS)l*@*T0OGA$Cu~a<|yL zm)P}(I4%}Ee&bSe!BMl@=?c`6=aV;;i82(t_vi60B_7-$<_Xjs^jgL(4TJWA?|Th> z_QvzpkFd3jYZ_6I$q>ZghArdVWcWeuYs!waj5m)i_L7hIh(97cUoc~2bc~kq2lDU? zBP;ddieQGJE$_Jwp8Za4v4d9hp=UpZ!!C9hDBfEg_z$MH=!x=x5z15xCaY-G_;<`v zohdyPuw&haq}C5Y1=FVU+2b6fQ+|zUihh8I>r)3Q1I90BmHdi{l}S^5aFfGWm2%WZI4F)J&fC&PO)}bC?`*!6pi!lPVmdTBwayqaf$24Cw?9lLd&Q?#wQ036f zebsHAp#>e7QdR7INkYBWj(LLaFk!m3eMDAm)Ca!+CNs*-`nnvo8NJS2g{RBtn5Ll> zI_ma1Ig9EVr0UQZT6H~Qz&2IWtE9@+RBW!?rSxb-{(iEpAQ z5>k;XlMD#M;8t+)uv8j>z1ahdsj}Fh%55Unl7LLXt6TF~bUTAie>kYYbCJ%$l_^_W z97(T^VGI=hqc{+J*Eva#oA8<%!)u!Gy6b*dbysJo1YN6IxN1P5kPkbtm#d|!!!~x3 zOzk^&N*CQtj*RR^&dSSKPjw|?AUWrjbxlo}{w7+ImZYJ!viYRtNmPo~4l-(aawkE9 z7ZX1Dh9Rpkq-ygrU;xITSu&tHPGQrM5BXtha--7iftkN#*lxcS`;nTg8n&|;{KE!= z-7gD0*0xKT9!r6>nsItE>z)mV(<3zNca8?uDLR_=k$H3mPkN`b!4w|Dq0urv++0T+ zW}!cB&}$j*;bYLS{CxpMuI`W_`A^uumqPWk;~z13M4%2CIh zqbZ8(F8!6S@|!FK6AY}IJ`=jo#Oz6Iu%16aY{Mz<`B}?&J4yb&K|b;xcwf%M#}xnl z;EM-EHv(6Lwe|s{PU@6S^jj;FCm)MW!POVg;vZkHWRnK4f+|s6{1!<1Yvi+Y+oom} zZ4wd5Pa7&Bp7-MR$wfk~tW5U6N7#>*A49gd6>H5GpYF2sQjU{7-NA2gH$rmI+w;_H zWl55po~?DhKj$^c)2YE3>Jv7l; zPn15As0ZWPX|WaiRK+aWot-tSQA2!3`Z~R>^6lbUe>mG(b<K-!2(%`mHp zuJPt~YCAWu+Q8~gVYR`u>wZslt5X06Z!xi|W7>*J{@!Ek&P0W=*iX9RcjsZ*C&N=M+*Ped~UX*ru2Mz4x%B_8|)U&Wb!_(Sq5g&{<_j^Vh<%M1HgO|@Pw5q=; zQA8ldcaes?M5e6?HXG9n)sihDvv-P=5vsB}BCRFON>f{8m{_w#N4>4&l-ODP{z0dS z=qYal3mCg<+@C7wFC}&o5q0G)pt=Krivj>QNm_v@L0u6uURjXAS1x1=BS(BxXNwPn zMV0q>MZ9odkwfq6@xoMdGzb0InTxq`ulykHmG5U5jRX3+-&0=ZCSl?RxzuAnBWSZq z5zuiudK+uOFIh!a*F(0orgm};W}lYjxgfLlRdrJzG05a9x1I>ymXVQKs35qj(zJwq zZ9;%6h5$ErgV%zwo1L3MczkJW<@I%(n~8XQ&81I3`-k^yuAgOSPC6VC%pJ(mRe5Wo zl;vCRsp{%AAhGQQ?29R-y3Wv5mzuR~vzBevvMZexLzch2bZo8Cm*!Bsh5kJkUIgt|51m(>#YHODS6qwVEXhZRnK9}J zCyK|6kQQbB-lM^84f0~U+cjZtNGa*qc82?iS0#4+@PO3cRvRgG4oerMm+~|wzeQcM+`eC zD@Cysk`j584{S-}8I^QPrtW;e0hj zj9mgvC1eY~3SD|;AdJWvFP?mCLfaViP6xx&)3eUW*dsl5`2_sIF+ zBkMgTLiL0{;iLHZ&Rt5EP9PT-;${+&en?@0jkLUgaW2xVAi1$FMHlBc%??EDy35^^z10#!v6IxT)9SID=Gi8m z79t@DYZBm+kYhLF@BSYEBt?pZ2wtSf3U@oLMFIx^anA2tZ~%NWsEdf_n#RcJwtMYk zqh(++qsHi)!7#ItF*2?wTl(O7d=AdWmqyFDN1l%<5jDEqcIU`K_r_Dpa1dKi%Qr@! zzZnENyy`n_Ba>p%J~J8h&MeDAzHhV)3;CvpT_(x={rBH<`i0Dv!~`#V@Y94*56;mC zOprHW-qK_*L7cz~w*yQ{71fMG<`gBsgFh9gV%h_w=Kg% z{?car(*?s{>r!gq>Gz4IeRF{v$myX6=(slG?HIF8x7QySw=G^>bOYLBkGjZXn3tGA z+ea;fJ-Ub&r_=u+lNq;9ynp{)Vh>ryh%|(&6Nu5`-fFX`tkMe%KvN?0Tn%Z>w z%7H#Z-Ye+hSFT4N9{-vn(w@`*chx)Y^^B*d)@HUkr(}lbm6=PD$!4)cks=Hh5#0)# zZT6?KWa>N@A@d%~kDS7}s4?gnW*}eZ3mYY&)BS>N; z*T<4oH2qy`>u-7%Y9jq~U|7=cY`~HU692~kpz=VnqLG0jE#P*+8t!xDHO1{!JuPTS zG{ebO#+K-txZV9%RhjxW@fFi!Sbwb;LTLDqr87ldip+1~F~B~6ASx0Yc7M=+eR|R# z_K%0RtHA{&pSkP|vt{Refy}$HNAEchmXe$V1*P0far+w0G4CI;#&p8Q9;LGuaC_kM ztuM(2Q+$p1^FH9{`x3lxp!<0>|8fd%edb}3tH{^8#=F9sg-s`5y+LDf$aM z1TA1w^UmNMbRpwXKTa@5`=^*S7c_WqX3^kS0qxXrmkfo-Qv%sz1XRHO`F6U*Hp3*C z+)FZ}0@e9^bKtv%#G($7#?q>EOa!8#p2fk1r~bg*?C(2>32CIoeo&e>T7e z-v&1e^o7XCG6*Kr2W$bEs3Jb+k;8|`CDB_Fg2^0w#caXfe<4@D-+Y|>_2%sS*N@}B z-keR|{xv?k`57Q`k3C8pj&k?V!`!k5K4O4T;8OBbJrEO<`qIekdNFqSA z?Po5O$BG#A1%kGJPKl2#)c@ED zV`WjTiLDf*8>ZR~Q*}34>6ea6J$3;ZEYCtK8mhpIiLLfIwnSW9D@c>3yOX$g(9is3 z%FH-(lGZu4fX5laM}+wR0w3A4BUxsO$y|OY!orjaWC0veYlBM$un%U`bC3n5j{t%{ zWb$GQKyXeiWP%r87uZ|?Oe}1Q;1mST(q`D?Z;xDP-XSY$h4b`F5f*~r1246J{-T{+ zMg3;H65-3)V{iHoG!g87^{vJ6V;}a9*|hK9A;efY1te58*1pQ$rLkF)+@hWpFM zXa266>u?&4u3L$`za8|hGb9<%uP{W#4KG%dnj}`b}fn<$6Y@#vpqMF584Q(^Q!WoI5 zS&1MM1{o^IhF8#MA6<(BT#+%#l((W^#)kevyL@_thrdjB(%TdgkphRUH#Rlz>`Ew~ zBBg@-$U@Y>`z)ec0{@jkxQ_g}rlQUQ{Q;SgDI@E^B&NS9>N7!XOUr0^lvN&=%pJ}5 zu{7v!Jwf~X#^4j9>zK*up&~*M5b5&(v zibtG(c~?Q$*Jw`+nB#jyR?jal1S3folzHW%)0si>VKj zUtVXYKQAfLJ+R3;(J@ww^zmmOFo~r%bUXcC%lH5v($D1a5aEF~E#n3|$Y;=Td5LbP z+cMrhxR`e;_&GuU{f!!-lSl5F(% zuCUMI0hw|O9ST~i#Lsx4^93XXzb^?zpzHUu^xn<=>!_J#$-aSGAL1>1%qlt1xV# z@&c%}d|(>o(NjSbV0hEEolWsPu0hVXO!HozBse2cS~aV=h!Na zoM-nC+i+^5chrlbui0+EO2MS4$XCy78K5P71x!g=I|Q%$h((D zY_Z;Ka?(B^1}(_o7_vonf#_KIOwFpS*O)4esj?rYO2~<&KybXA+Sp%|s(X!Q1-U=W zf!sRpT*I#-&$2)bN?Gx4h0b>b#R`WyvAA9AQ1+s>Ud(pY<62ZS(hvmwF7u%}>&lP( zcCsRNajx@Sx)!lhfA1f_&Q+ayys{W;*dM4Orm}L^7m-6q=9L70>f5`!IZsrh%Nd5R zPMz(Vz@ab}y-w$Q2xH4QUwU5Q)%n0YG!ZUk9_o(t3eBPUIQA%`CRJFfn@r6ye6%Sn zcT>lA$Xcjo#$5(C&=&D^1o)I(LMY;KTp==Nn!57$*~~|*NTf!u7dq<`Wv3w92&|HX z5(t-KB@5YN;00&w8h$P5*;ZXaDQ47A2JK1c-RBmsIbG+#OcB*V`BIsW5#du`)d-1` z$u>V?CFC_#I;O@N*{v8EYaL5{g!Frzw7URU<(0mHLpMeqrj~GGjgismbh=04pPNo+ zB>wr+SSxFcjyuu+3MSoEg7#+`L2Cj)O#o;g0zg`pMpDqVK7Iy{d^2X$b0CvD$IubO z4%%Rz97tNK>aLDJZG_*c9^ICvmZ{=2GEuWhZAGp9aQTI*n>OT^zeYZnmzp-dW>DMN z&e!a=b+^7|)CQ0-)-Z}!19`(T@Wusgvm)s-$%UME%e!A?~BUv8J7}liZ!MLK$T3GSvm}HTqjWM5);kd+0l28-%#(ovQB7`hW>0^EZ6_ErZ2&ao9m=S5Od^y_DY=L=?x^26(4|=a(0m%Og7MwoW8xe>Z19S5#rtL+zka9vfyfvUM%g?D77Nd%QD|BF#5x zyvQu0g&J~!rH(-d3oinDuBa*Uz5@&@XiAh{^;vJc+)`n|oX+wqRoBSY&kQNBC-guN zTy?Edgo~>jp-LDYktv4aQ8jHGk2lmH`Hzg`YR*-a(((giNK_A`$C5sFnXJ*|;T`Nk z_o?dYFNu;@*k|Y1;!Iy|g*#2A;w5>R1e*);4&;FCeU%c`)nY4dlz&x}5&2hb5)zSX zawwotT~dpyPRyq2xQ9zoZ@z99OQ~dl3KP{+#tsV(8ZiD07E{*WL_xg zBuiN0z06!wi@nsbErT7*-A<>|GQxd@rv;6|)i&r{^Gkp0;-MKxsw>T|1$UiUavii3 zb07g1vg%OOFQf}gYre3logCNPsJ2G6 z<*Bv?WE+qTKvs2!XMkLoKpX#4#k0;Lv?JJEZ?nPQ27ep;ZSZ%W@K<#Kt%R@-^gg@^ zz5@8hsO$!LlMObo+rVxEyAAB_6YQ!k7nb;SjbP<)dqkrUEo2}SQaJf+cS_)v}a_cp+5%iJVEr!yWCq>jig3@m6?{uS$4c=&hz0d*;1UC>Y zAh_S@Y+T=)AZVXc$H6QOH2Uxxqf!WZgG)!+LEYb%teRDx>_=mlH;BA9h@58iYU#8^ z4&g{0xluM>wXymem7}3@R9E`1gLGO-%BY-PDU~z!D4o@%ZF-HiX|&BgY8%1jjiPB3 z&0#8<-o|C!i8zA@)#Ki*2$fVfs7@y*(VIr|G@54*H4kcBokjy4x(4cRT;WbYbyaU? zMW~bpl1OxpEx$(f(>Okj`q@kMv*LD5E{PT)YdhX$QY*cJjJImM*siqrXV}R>Dvh3P z^sJ7a-Iz!aoHtuBmn=qQHwgsE2Akx8Mu+Z&4sBq!f!zRh2a4nYZeCzAKbul7eZsO1 z;>HQ+f^80_u94jO8ITQ>Hc;B&gv5#0iWT2LCMV_sSr-`2t6)V$M1>dfPP7-80|J-J zZ%=tANM%q~!^V1HU@aZj2RI-GeEA;m3AUCt@_^4EL$#fz^ZhQ_iha>kuafUH08kBC zvPtgA_fFa5V$p(tqXRR^ppmta!UsN{JSINwV zT40WtFK!7$!CSz)Bu;9Bz!y~kqe8GMAK8`|Z2HcuwYS^30V#GnbLrch_LR|8*M>(7 zASNE!@Ua?qM`Y?`={V6GK8}j6m{;&DTR_HV9euSQk4z8o`3YF~3;7B$69-cQhk6KD z2--fFq8a~z84y=!`l0@4IDOAKT@i_rEb;+*ckih8(VctHDi@5(BBU~TR%cDh8Oe2# zGD&aURa)=n*7)wUxnQmWYpd==mQ#0)9C}}cr6qw{SvexP$!uR06yNV2N!$($jG0HB zXpt0LGGI~UlmB6WAyscU8BgMtoI=M>5&$k(3vgI8_Z9S-BBxvm~L}s1&fHMQYvsHB{0vKBo*i6~3V=wKZ#R(syEc0ta;oTfeKiq)<+rxDl{| zK#BVKQY2?YPm)5C+2X@r0%+UZQJP0V2)|Zl|1H}r%Pf-5^q;%EeyiIX-0sh@wI z)qR06zPKb4;UHIK*^rLP=lOgax$+EEW#(98YF9`wb}TmFLXTl{X+tlH3Do4LB*s>9 zOYOS#QIZoIN!(LO+!%H6uhFcM1oHaXO2dKfXUSBzoK2yPtZOvWl(m5DR_?XU?pW$0 zD2wT&be|48olY#d`|?6$b+4{O?{)^m*C)rFR&;fp`28gK&xMSCj&Z_jYX4-Eyxx6V z*RYqku}*nm`(fQG+V86FPz(5*1NO3#;7JO)fxhUB^OxX7Fva3tQ7vJNn%cY{?LA^L z)F+B_zXyUjzDHr4oV(SR7e7#pZVyi7&ceUd{HHpVc%{E}g~P>Z%aB&c$QXMf%f)CJ z=hSg7j1f0FCaDFZqa%_!GkO1v5rQ$7mUUAP>(lvQjTD31%4D z@}BG9+3(~AJ7`58diG;D>|&dN;=R>@|6qEH?kEo!p-i=4vWixXf5#lvnbK1MJJ!8P zYJDSAFl{=YJ|=m&_nKDCiDVEl4c$*-7LnKacKH#rsNT-A;sOV7NE&}j?S zV1U8}m>|Gt9qIwLZ&wbp7$ZuO!Gy<4h@f|Z1q$ORSwPESKa0rTF`+h zRmI+yB-FlkEE05!3DdRhBeH6vKKKPNnNe=m*A=ME=yj$So-U(fnub>BsN3u0IMp>s z)uA!8>UzY0Z9ckIwN#aWLLvV(5y#a?Tv@fTlY>;AxkFksk{k(G*(K~Ih2^T>Ro#ad zNY0sMT~QOJzloNlC26P+)!>tsCs8R{J2*|llTic>UQGDo4MSF8NY#2lzyORvvt%+= zoWiCPUXUNQCWDl256t`}!*=_fh(&6$YS_+Z@DCddB3~AIs4YU79*d^wB2IfbIi7V- zhr{U+n)N$J1M37GP5a0^I)%r*li6SjU&Eo%GCp2kMZvPr79aFl#s~NiUSKSKpF?p@ zXULHJC-0v8UGxy9kXDv+u%evxHt;1=K7XvYI-Zdnb<8nJQCxTDSANRYOu0?kHNn8T zM`uA7nwUL`;Of~u#5SDro}adica!ArYvd#Ep7-Tcd`$7*kG^YlMc(%>o!K@ut7wymNdA#w65@F;ZXR7E z)XK_apWuIxI0boCehk^-Mx=4R_uZ_Op?Zp9@@PS4gl-yif} zpPuxG{o`Q`!$==mn`S9Zqc+$^Ri{0h#mN~-IVDY(Dp{dFo>6gFnMOU;9bds5Tp-hi z9tsZps{r{4LOFg!a2{_m2|R)FDXlOF@+5S8jP0iKiEI>?J5>Y<62KT&!+f*y=( zABm|r2(y?ayR$s=8a2c-($|AM%4zxGY-`m$S0ONY)U5$&BPun+tR}j~o8PJJ+`wuB zt2>3&2Gg$kJ=M)g0UW%;#Hx;ID=PVGkFoob6vkpd9YP50$5_xHSXnC?I!JVNr>12C zEjqUfJr8Kvh>*`R!m>|>r&`E2L{&11KDXG)*u%2FS#_E~N;N&Wc?mX?xe91NUO(5K ze+gcc4tWO+?Bu$QeX-QDut6iz+AI+tOf>g7BaZS?ANj${pDeJdziE?bAjY?mhP*(g ztqC?8j~J+Br)XyH6x&5rWpzYaOPZCYw%A>|W{Zw`TiH*#qw9ALe2ZvLc|%UX*j4lX zR7rm+v6G0XD{leS9SB^M0Jurg3Srrdpsm=*P}N%uRUZ2MMn{pJ6ls=&OEDd1aA=iEHFikNu3G%_?O;$LZ)TtOdVh z6CcWHxGIU!;#<2ng!CkdQJM3$d0$ec#xVaa! z7L45-+zis=OJggquM^x%K? zZ3p(nlv3Sb=&DPNFWdOCjW4^>SwV)t_edMqSA=u+dq>B^>Gjm&W{?YdzUwH9vb}^rZtlv9;)k8}!f3+t0y8%>2!mQO|*#tu4sV5yKA3I-XDpNr^KQ z56q?Pt8PA&*f?R3RZ%yhzK*_G2W3%FPv3-X>_FjU_A*D|sOr?GaK0HL$1Z`U60${H zg)Tia5Ju#T7mq$Rp>2$MCxhY1$!X{KbTDY~SCg;MHAdaua5y;abce%XM?79dW_sPh z$za$&9t>N?`S_!6jE)B_tIArq*G!Vdt|_`J&Ed=J^FPQI`r#%wLhwidoSa z>-(UjeoA42jkKbGaW2xVAi1$FMHlBc%??EDy35s+z10#!Hh}+0RR7la1k!z GeE|S6C0=#_ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 04d82b39a9604730726072bbfec48f749f9947ce..dcb410ce0056baaef6a154a896b1246547580e6a 100644 GIT binary patch literal 2579 zcmV+u3hebCiwFP!00000|Li?&bJ{xAe?_C`OLNEJJ!#9gK4fn<;ef){5K%$G`0DTyqV+-Gdkd(6!qTOz^jvctia|?T*rreRZfAxeJ@ige4)CHSU z*7)E!Lkc!v3p*ffL49RV+}_?!_%#~`%tmh{`r|tncOWj62yAtP7GQ7u7P2{0@TeLC z^RI>cHj;`9GJ(b=*oun_7+c8ipsO1)qK$9NCyoOK!6p3(7V^6&@0;Ed*BMdd2))rC z>6qBi7!cPzKeG@B!Nn1Jx@Ol(fC>advT@E72u9pSj8sjUCPWXYQ6F8i`8B(}y~P%m zf#(r1!+pFOz|2_>Bd-~@uulZ>%n}4bFXL!->~l%OC5ygJEVJKEt!rv!5@YKR_7ps# z_Zc3E=euAqo}cNa&y<0{{dTL>vhbI{8|hE)=N29c&c}bve8By8^BW5j#|gRY+!_C~n z0{#xD00;M#2taR!FTf^Bw}g|MTdUpbq;@NZV+-vW3tacc!c;-T6sZ{`$1-b$s6+&< zP4a31bEEfm(ME(Iv+}d-rbaF~$13xPtxLPr>D^?^tmHI$xuT^NMKP($mlRDvA&PEv zCmFa(KrUr>*usHuaUV~W z@@2o-oP=E(Husw2$jl%6f@`{(u67xkBbSew9+3(}llZjh3;sC!&jeUw!vF76xBI@M z=ahC9{=vw|1*fF15J6?Q3Piu&8aGR&-rJZq({Mo8 z%VvOHtCh_Fe}b~7CAU9>Dd#7y)T3$F<7+n{bQD)#4vCsZ&AnLj2RmH$&)k6u?&5;n zV!mSS6uv^IFpoWV=P_-;T}pd>X+bK)byxy%DVyLJQBsMpWjj7xN{oxqikY9>BN@|4|n6*wK(`Y&G>f2|7(CT!w}JS8uuF(M zF2j3^)3~`6eyK^y{?-U?%CGyxZQ_QsXMOt}=ww(Pco;yfqy=FP_mIYo_!vEeq#>b~1eE#%87>8xYJIyOAC*zhFheBuIJAQHYY38Z`>A=~iJIuEnu6HcmA*KY1!XlyLdyx=Zuz$)w`OU&FM*rBSrIkHVvl9i;JPndG~VW% z30wkk2i)_hN^{Ti@)3h9=kFP&B<@9+n`9v6@|tZZGsIT(9#TDe(Iwq5W;7Kq{>j=9 z1M0{M{$p|~>!~3?#8&QPgVNH9x1iQqrWpE9xTzH;8P7VSpTq{zeUQ%dbk-%^ zQ#IK>)qZeb2bq+r(E=Rjvjvx1aH4CUi%ZJGJj}PWD&+T-tmWb!t21t1ESDO_eNN>}!Xj3QH&?*bLe7;rt{!zJXs83vy%MB-Fz1=P_VO%xZI^1hbYOPrq?J?j zQte*Tb}lq+y2q4zh-IGqgUe}znVE6u0Sn=l*r59zG$VfE)*qFjN@Z`ctPV8l> z7puG&5reGyeqGip5zqZhSLR{(39_lv7*MA|#ymNP`o2J&IxqAV-b;MH;b4D~>%a;W z(gpgFdfU7+nNvS%2#^vSM3>ka=v?Uq7<1u) zgGMt%(7(+OLRyHR0e8SgZyu)9o+8Q|s;8*O@B&v+TYr1(6Z;OFL7#Q6 zDUs7gl5w_k8KC~rj{2afK4>z&;Au`~GLq$zeqaQZ9oHdN;hl7QP1|DQ6)>yWMCVJ8+NZ7WP0*xg&A^+Y@HQ)1ZG)7i>;h zfw)p4u+> zJdcPO?&H+}X3lyTdCjneeIkfwmLL#%8Ar2YpGz7pS@d;cnf-QZT~jNQ7+Zg^r{EF2 z&+te*-vxv5;#@aI6`kmoV%MW_zFR`!7gE$v!qp%keX7|xLGRo-o~_Uuh1#XW6#}rOj~f5(q3O$kP2}fmOxy}COAfvR3dEIjt`d-<6?Aq{2nS3Rl9Mu z8+T}K+}pDpD^3}4=?{qZqNF)b0GG}GEZQ##n^dyS>!UBsbZwsZLWK6gJg;$CwOmb! zrTsh}OHjquHcYHkT$qZPpW3q3mhHeSTd#GRZP_l^n3uF?8YDY4v3?Gqdv3B%5w@M= zDYEE!MsO3IiU{w&xm3%cwCjxSi5qDBc?psS!j4!Lyy0{vsZCtm?{zHQaP>g^<%dg< zAnrB8=!U`Yi!}dxAO25k+IdOQUQzb6;Oxk`pRW;D0;wUah~&MGdM2$wL%PXJE^M*zHX9++R@!Bi@Fbv?#szT)jnwL zgC3gE>Es-e4T+k{lIA9a$@W4`Gz+p3nF8H)ljT^T+ld>RMNm!*p6ayQKzu;h6+|AF z;l0Ia+}sMk)FfqpYXmpt*L~nNaYNd(zWoYxGA$JNZ!w(TS8?d(92pJ;w?WD)kkT+$ zDY-}hXA4LWP^azH zdhRehB^^5~gzUAtXS=C|d^shZb!=G2hKCj#_HxcAF2MyN;R};M$`?|uLI3Kr_w?{} zPI9lLXdiNt_2GMM+-l=?(8lc~=d|q-!V1i?HcbQNnKJFgnytM!KQiVi-#SHzG+?$t zigJ;ti9V|-h^<}ei&I%pR&yk@oY3u-e@k*}mZtj@xape}QDZFjSVj%5`@%)zH0Mm< z3Wz)4o<~)hd!Cn%7-Ttr&oCu%FT>m<11Xo+Y(tqLwxait>d}iX>4q_*sd({E)`l2R zM^^A3lT%qw4e4=eKXNY&);{==Y4$z9JGZ%l_@d{UPDZV%l${3!;Yfq%OfOoLP-8NM zE*xNrj%l2eS4Vn(Lx>Ptxswe_ODo=jT5FkN=s)46R+wZw>x_O78%Xy-I@8lxmvm3n zs4MTl^S|@Qzx-gvJ^%a28^dhe?Tu+Sn4Jaz^g8!F<6>_UUrAND8obERMK9Iv zHEriY)25qDALt#dU7@njC{Sp5#fe2VHL-h3nqwU*!xuN}J4>(T&eBOMw{%5Sd-in& znCEbfICnQxbtRBIQ`xobhCCwfBvMmY@H3GJ%y-QNUpLSvu8^7%{TG!;f>M-Cu9-(> zNRc~01dZVVNw7I{BvKq{B}yhlf&QR~yhEB%oxt5<9oesWbG9zj`p`XT=bdR85A2=e zgjavdvV0OcKhO}AO*V0U3Rgb+Z!K5|lNrSx0gb;9w9Q91Umq*xWVQb!^6uOytdeXP zgUn??Ag>-)h=n_|)vwtobvn*~v-9=shv^wgP7w zz^yXo5@pSouYxbTIlOvV_YH$UNeIOK<#*456Rqo5Hj>_H5={^gzegL95jT zWL_5e>F{d<;i*;H+`mXlX2I*4a~AC0e+n!9e4_NC3=Au)IO*lYUZ#4n z%8L;($g1zxWxW#d+|P7n9)_PFn>vjFbt+`clXIx=3)HFeLT}-{#P=Hx_9wXxtUw`M zpdYEX%{!A>MUccbpbipxcnM7DsYr0g$EMFnN85IUgE)==DZxQ>iLHSylwN=_7allh zG(!aa*Zd%)g$NpO2W<4_VM^^OqRgRsih2w$a22)nx5qxQ@4#7(qC)4JFb5IzQTLh> zIc+2vXFHbx>L2Z>51Q(OCgTg9C!nB8GSCt(lZBqm%)~*nIcM1eja+WuNfaA&6znT9(00960SA~2>PkI0Vz-#yv diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index a02520116ce..d59401b6e08 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -316,10 +316,10 @@ func migratePreSealMeta(ctx context.Context, api v1api.FullNode, metadata string Size: abi.PaddedPieceSize(meta.SectorSize), PieceCID: commD, }, - DealInfo: &sealing.DealInfo{ + DealInfo: &lapi.PieceDealInfo{ DealID: dealID, DealProposal: §or.Deal, - DealSchedule: sealing.DealSchedule{ + DealSchedule: lapi.DealSchedule{ StartEpoch: sector.Deal.StartEpoch, EndEpoch: sector.Deal.EndEpoch, }, @@ -453,14 +453,22 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode wsts := statestore.New(namespace.Wrap(mds, modules.WorkerCallsPrefix)) smsts := statestore.New(namespace.Wrap(mds, modules.ManagerWorkPrefix)) - smgr, err := sectorstorage.New(ctx, lr, stores.NewIndex(), sectorstorage.SealerConfig{ + si := stores.NewIndex() + + lstor, err := stores.NewLocal(ctx, lr, si, nil) + if err != nil { + return err + } + stor := stores.NewRemote(lstor, si, nil, 10) + + smgr, err := sectorstorage.New(ctx, lstor, stor, lr, si, sectorstorage.SealerConfig{ ParallelFetchLimit: 10, AllowAddPiece: true, AllowPreCommit1: true, AllowPreCommit2: true, AllowCommit: true, AllowUnseal: true, - }, nil, sa, wsts, smsts) + }, sa, wsts, smsts) if err != nil { return err } diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index ea5ca75f851..128be234aa0 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -114,6 +114,7 @@ * [SectorsRefs](#SectorsRefs) * [SectorsStatus](#SectorsStatus) * [SectorsSummary](#SectorsSummary) + * [SectorsUnsealPiece](#SectorsUnsealPiece) * [SectorsUpdate](#SectorsUpdate) * [Storage](#Storage) * [StorageAddLocal](#StorageAddLocal) @@ -1814,6 +1815,30 @@ Response: } ``` +### SectorsUnsealPiece + + +Perms: admin + +Inputs: +```json +[ + { + "ID": { + "Miner": 1000, + "Number": 9 + }, + "ProofType": 8 + }, + 1040384, + 1024, + null, + null +] +``` + +Response: `{}` + ### SectorsUpdate diff --git a/extern/sector-storage/ffiwrapper/partialfile.go b/extern/sector-storage/ffiwrapper/partialfile.go index e19930ac1ca..0e8827dd3f2 100644 --- a/extern/sector-storage/ffiwrapper/partialfile.go +++ b/extern/sector-storage/ffiwrapper/partialfile.go @@ -25,7 +25,7 @@ const veryLargeRle = 1 << 20 // unsealed sector files internally have this structure // [unpadded (raw) data][rle+][4B LE length fo the rle+ field] -type partialFile struct { +type PartialFile struct { maxPiece abi.PaddedPieceSize path string @@ -57,7 +57,7 @@ func writeTrailer(maxPieceSize int64, w *os.File, r rlepluslazy.RunIterator) err return w.Truncate(maxPieceSize + int64(rb) + 4) } -func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialFile, error) { +func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) // nolint if err != nil { return nil, xerrors.Errorf("openning partial file '%s': %w", path, err) @@ -89,10 +89,10 @@ func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialF return nil, xerrors.Errorf("close empty partial file: %w", err) } - return openPartialFile(maxPieceSize, path) + return OpenPartialFile(maxPieceSize, path) } -func openPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialFile, error) { +func OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { f, err := os.OpenFile(path, os.O_RDWR, 0644) // nolint if err != nil { return nil, xerrors.Errorf("openning partial file '%s': %w", path, err) @@ -165,7 +165,7 @@ func openPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialFil return nil, err } - return &partialFile{ + return &PartialFile{ maxPiece: maxPieceSize, path: path, allocated: rle, @@ -173,11 +173,11 @@ func openPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialFil }, nil } -func (pf *partialFile) Close() error { +func (pf *PartialFile) Close() error { return pf.file.Close() } -func (pf *partialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (io.Writer, error) { +func (pf *PartialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (io.Writer, error) { if _, err := pf.file.Seek(int64(offset), io.SeekStart); err != nil { return nil, xerrors.Errorf("seek piece start: %w", err) } @@ -206,7 +206,7 @@ func (pf *partialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedP return pf.file, nil } -func (pf *partialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { +func (pf *PartialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { have, err := pf.allocated.RunIterator() if err != nil { return err @@ -224,7 +224,7 @@ func (pf *partialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi. return nil } -func (pf *partialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { +func (pf *PartialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { have, err := pf.allocated.RunIterator() if err != nil { return err @@ -246,7 +246,7 @@ func (pf *partialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPie return nil } -func (pf *partialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (*os.File, error) { +func (pf *PartialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (*os.File, error) { if _, err := pf.file.Seek(int64(offset), io.SeekStart); err != nil { return nil, xerrors.Errorf("seek piece start: %w", err) } @@ -275,11 +275,11 @@ func (pf *partialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedP return pf.file, nil } -func (pf *partialFile) Allocated() (rlepluslazy.RunIterator, error) { +func (pf *PartialFile) Allocated() (rlepluslazy.RunIterator, error) { return pf.allocated.RunIterator() } -func (pf *partialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { +func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { have, err := pf.Allocated() if err != nil { return false, err diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index dca8b44b531..e8d656cac26 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -66,7 +66,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi } var done func() - var stagedFile *partialFile + var stagedFile *PartialFile defer func() { if done != nil { @@ -97,7 +97,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) } - stagedFile, err = openPartialFile(maxPieceSize, stagedPath.Unsealed) + stagedFile, err = OpenPartialFile(maxPieceSize, stagedPath.Unsealed) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("opening unsealed sector file: %w", err) } @@ -244,7 +244,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off // try finding existing unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTNone, storiface.PathStorage) - var pf *partialFile + var pf *PartialFile switch { case xerrors.Is(err, storiface.ErrSectorNotFound): @@ -262,7 +262,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off case err == nil: defer done() - pf, err = openPartialFile(maxPieceSize, unsealedPath.Unsealed) + pf, err = OpenPartialFile(maxPieceSize, unsealedPath.Unsealed) if err != nil { return xerrors.Errorf("opening partial file: %w", err) } @@ -414,7 +414,7 @@ func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector storag } maxPieceSize := abi.PaddedPieceSize(ssize) - pf, err := openPartialFile(maxPieceSize, path.Unsealed) + pf, err := OpenPartialFile(maxPieceSize, path.Unsealed) if err != nil { if xerrors.Is(err, os.ErrNotExist) { return false, nil @@ -598,7 +598,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef, } defer done() - pf, err := openPartialFile(maxPieceSize, paths.Unsealed) + pf, err := OpenPartialFile(maxPieceSize, paths.Unsealed) if err == nil { var at uint64 for sr.HasNext() { diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index d3fef8533f6..d8f421d988b 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -29,8 +29,6 @@ var log = logging.Logger("advmgr") var ErrNoWorkers = errors.New("no suitable workers found") -type URLs []string - type Worker interface { storiface.WorkerCalls @@ -47,8 +45,6 @@ type Worker interface { } type SectorManager interface { - ReadPiece(context.Context, io.Writer, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error - ffiwrapper.StorageSealer storage.Prover storiface.WorkerReturn @@ -105,19 +101,12 @@ type StorageAuth http.Header type WorkerStateStore *statestore.StateStore type ManagerStateStore *statestore.StateStore -func New(ctx context.Context, ls stores.LocalStorage, si stores.SectorIndex, sc SealerConfig, urls URLs, sa StorageAuth, wss WorkerStateStore, mss ManagerStateStore) (*Manager, error) { - lstor, err := stores.NewLocal(ctx, ls, si, urls) - if err != nil { - return nil, err - } - +func New(ctx context.Context, lstor *stores.Local, stor *stores.Remote, ls stores.LocalStorage, si stores.SectorIndex, sc SealerConfig, sa StorageAuth, wss WorkerStateStore, mss ManagerStateStore) (*Manager, error) { prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor, index: si}) if err != nil { return nil, xerrors.Errorf("creating prover instance: %w", err) } - stor := stores.NewRemote(lstor, si, http.Header(sa), sc.ParallelFetchLimit) - m := &Manager{ ls: ls, storage: stor, @@ -221,56 +210,7 @@ func (m *Manager) readPiece(sink io.Writer, sector storage.SectorRef, offset sto } } -func (m *Manager) tryReadUnsealedPiece(ctx context.Context, sink io.Writer, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (foundUnsealed bool, readOk bool, selector WorkerSelector, returnErr error) { - - // acquire a lock purely for reading unsealed sectors - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - log.Debugf("acquire read sector lock for sector %d", sector.ID) - if err := m.index.StorageLock(ctx, sector.ID, storiface.FTUnsealed, storiface.FTNone); err != nil { - returnErr = xerrors.Errorf("acquiring read sector lock: %w", err) - return - } - - log.Debugf("find unsealed sector %d", sector.ID) - // passing 0 spt because we only need it when allowFetch is true - best, err := m.index.StorageFindSector(ctx, sector.ID, storiface.FTUnsealed, 0, false) - if err != nil { - returnErr = xerrors.Errorf("read piece: checking for already existing unsealed sector: %w", err) - return - } - - foundUnsealed = len(best) > 0 - if foundUnsealed { // append to existing - // There is unsealed sector, see if we can read from it - log.Debugf("found unsealed sector %d", sector.ID) - - selector = newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, false) - - log.Debugf("scheduling read of unsealed sector %d", sector.ID) - err = m.sched.Schedule(ctx, sector, sealtasks.TTReadUnsealed, selector, m.schedFetch(sector, storiface.FTUnsealed, storiface.PathSealing, storiface.AcquireMove), - m.readPiece(sink, sector, offset, size, &readOk)) - if err != nil { - returnErr = xerrors.Errorf("reading piece from sealed sector: %w", err) - } - } else { - log.Debugf("did not find unsealed sector %d", sector.ID) - selector = newAllocSelector(m.index, storiface.FTUnsealed, storiface.PathSealing) - } - return -} - -func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) error { - log.Debugf("fetch and read piece in sector %d, offset %d, size %d", sector.ID, offset, size) - foundUnsealed, readOk, selector, err := m.tryReadUnsealedPiece(ctx, sink, sector, offset, size) - if err != nil { - return err - } - if readOk { - log.Debugf("completed read of unsealed piece in sector %d, offset %d, size %d", sector.ID, offset, size) - return nil - } +func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed *cid.Cid) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -285,16 +225,10 @@ func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector storage. return xerrors.Errorf("copy sealed/cache sector data: %w", err) } - if foundUnsealed { - log.Debugf("copy unsealed sector data for sector %d", sector.ID) - if _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTUnsealed, storiface.PathSealing, storiface.AcquireMove)); err != nil { - return xerrors.Errorf("copy unsealed sector data: %w", err) - } - } return nil } - if unsealed == cid.Undef { + if unsealed == nil { return xerrors.Errorf("cannot unseal piece (sector: %d, offset: %d size: %d) - unsealed cid is undefined", sector, offset, size) } @@ -303,6 +237,8 @@ func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector storage. return xerrors.Errorf("getting sector size: %w", err) } + selector := newExistingSelector(m.index, sector.ID, storiface.FTSealed|storiface.FTCache, false) + log.Debugf("schedule unseal for sector %d", sector.ID) err = m.sched.Schedule(ctx, sector, sealtasks.TTUnseal, selector, unsealFetch, func(ctx context.Context, w Worker) error { // TODO: make restartable @@ -311,7 +247,7 @@ func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector storage. // unseal the sector partially. Requesting the whole sector here can // save us some work in case another piece is requested from here log.Debugf("unseal sector %d", sector.ID) - _, err := m.waitSimpleCall(ctx)(w.UnsealPiece(ctx, sector, 0, abi.PaddedPieceSize(ssize).Unpadded(), ticket, unsealed)) + _, err := m.waitSimpleCall(ctx)(w.UnsealPiece(ctx, sector, 0, abi.PaddedPieceSize(ssize).Unpadded(), ticket, *unsealed)) log.Debugf("completed unseal sector %d", sector.ID) return err }) @@ -319,20 +255,6 @@ func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector storage. return err } - selector = newExistingSelector(m.index, sector.ID, storiface.FTUnsealed, false) - - log.Debugf("schedule read piece for sector %d, offset %d, size %d", sector.ID, offset, size) - err = m.sched.Schedule(ctx, sector, sealtasks.TTReadUnsealed, selector, m.schedFetch(sector, storiface.FTUnsealed, storiface.PathSealing, storiface.AcquireMove), - m.readPiece(sink, sector, offset, size, &readOk)) - if err != nil { - return xerrors.Errorf("reading piece from sealed sector: %w", err) - } - - if !readOk { - return xerrors.Errorf("failed to read unsealed piece") - } - - log.Debugf("completed read of piece in sector %d, offset %d, size %d", sector.ID, offset, size) return nil } @@ -768,3 +690,4 @@ func (m *Manager) Close(ctx context.Context) error { } var _ SectorManager = &Manager{} +var _ Unsealer = &Manager{} \ No newline at end of file diff --git a/extern/sector-storage/piece_provider.go b/extern/sector-storage/piece_provider.go new file mode 100644 index 00000000000..3c5b80aa371 --- /dev/null +++ b/extern/sector-storage/piece_provider.go @@ -0,0 +1,109 @@ +package sectorstorage + +import ( + "context" + "io" + + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/lotus/extern/sector-storage/fr32" + "github.com/filecoin-project/lotus/extern/sector-storage/stores" + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" +) + +type Unsealer interface { + SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error +} + +type PieceProvider struct { + storage *stores.Remote + index stores.SectorIndex + uns Unsealer +} + +func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unsealer) *PieceProvider { + return &PieceProvider{ + storage: storage, + index: index, + uns: uns, + } +} + +func (p *PieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (io.ReadCloser, context.CancelFunc, error) { + // acquire a lock purely for reading unsealed sectors + ctx, cancel := context.WithCancel(ctx) + if err := p.index.StorageLock(ctx, sector.ID, storiface.FTUnsealed, storiface.FTNone); err != nil { + cancel() + return nil, nil, xerrors.Errorf("acquiring read sector lock: %w", err) + } + + r, err := p.storage.Reader(ctx, sector, abi.PaddedPieceSize(offset.Padded()), size.Padded(), storiface.FTUnsealed) + if err != nil { + cancel() + return nil, nil, err + } + if r == nil { + cancel() + } + + return r, cancel, nil +} + +func (p *PieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) { + if err := offset.Valid(); err != nil { + return nil, false, xerrors.Errorf("offset is not valid: %w", err) + } + if err := size.Validate(); err != nil { + return nil, false, xerrors.Errorf("size is not a valid piece size: %w", err) + } + + r, unlock, err := p.tryReadUnsealedPiece(ctx, sector, offset, size) + if err != nil { + return nil, false, err + } + + var uns bool + if r == nil { + uns = true + commd := &unsealed + if unsealed == cid.Undef { + commd = nil + } + if err := p.uns.SectorsUnsealPiece(ctx, sector, offset, size, ticket, commd); err != nil { + return nil, false, xerrors.Errorf("unsealing piece: %w", err) + } + + r, unlock, err = p.tryReadUnsealedPiece(ctx, sector, offset, size) + if err != nil { + return nil, true, xerrors.Errorf("read after unsealing: %w", err) + } + if r == nil { + return nil, true, xerrors.Errorf("got no reader after unsealing piece") + } + } + + upr, err := fr32.NewUnpadReader(r, size.Padded()) + if err != nil { + return nil, uns, xerrors.Errorf("creating unpadded reader: %w", err) + } + + return &funcCloser{ + Reader: upr, + close: func() error { + err = r.Close() + unlock() + return err + }, + }, uns, nil +} + +type funcCloser struct { + io.Reader + close func() error +} + +func (fc *funcCloser) Close() error { return fc.close() } diff --git a/extern/sector-storage/stores/http_handler.go b/extern/sector-storage/stores/http_handler.go index 3e34684709c..b65b5272b66 100644 --- a/extern/sector-storage/stores/http_handler.go +++ b/extern/sector-storage/stores/http_handler.go @@ -5,15 +5,19 @@ import ( "io" "net/http" "os" + "strconv" "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" - "github.com/filecoin-project/specs-storage/storage" ) var log = logging.Logger("stores") @@ -26,6 +30,7 @@ func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { mux := mux.NewRouter() mux.HandleFunc("/remote/stat/{id}", handler.remoteStatFs).Methods("GET") + mux.HandleFunc("/remote/{type}/{id}/{spt}/allocated/{offset}/{size}", handler.remoteGetAllocated).Methods("GET") mux.HandleFunc("/remote/{type}/{id}", handler.remoteGetSector).Methods("GET") mux.HandleFunc("/remote/{type}/{id}", handler.remoteDeleteSector).Methods("DELETE") @@ -103,32 +108,128 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ return } - var rd io.Reader if stat.IsDir() { - rd, err = tarutil.TarDirectory(path) + if _, has := r.Header["Range"]; has { + log.Error("Range not supported on directories") + w.WriteHeader(500) + return + } + + rd, err := tarutil.TarDirectory(path) + if err != nil { + log.Errorf("%+v", err) + w.WriteHeader(500) + return + } + w.Header().Set("Content-Type", "application/x-tar") + w.WriteHeader(200) + if _, err := io.CopyBuffer(w, rd, make([]byte, CopyBuf)); err != nil { + log.Errorf("%+v", err) + return + } } else { - rd, err = os.OpenFile(path, os.O_RDONLY, 0644) // nolint w.Header().Set("Content-Type", "application/octet-stream") + http.ServeFile(w, r, path) } +} + +func (handler *FetchHandler) remoteGetAllocated(w http.ResponseWriter, r *http.Request) { + log.Infof("SERVE Alloc check %s", r.URL) + vars := mux.Vars(r) + + id, err := storiface.ParseSectorID(vars["id"]) if err != nil { log.Errorf("%+v", err) w.WriteHeader(500) return } - if !stat.IsDir() { - defer func() { - if err := rd.(*os.File).Close(); err != nil { - log.Errorf("closing source file: %+v", err) - } - }() + + ft, err := ftFromString(vars["type"]) + if err != nil { + log.Errorf("%+v", err) + w.WriteHeader(500) + return + } + if ft != storiface.FTUnsealed { + log.Errorf("/allocated only supports unsealed sector files") + w.WriteHeader(500) + return } - w.WriteHeader(200) - if _, err := io.CopyBuffer(w, rd, make([]byte, CopyBuf)); err != nil { + spti, err := strconv.ParseInt(vars["spt"], 10, 64) + if err != nil { + log.Errorf("parsing spt: %+v", err) + w.WriteHeader(500) + return + } + spt := abi.RegisteredSealProof(spti) + ssize, err := spt.SectorSize() + if err != nil { log.Errorf("%+v", err) + w.WriteHeader(500) + return + } + + offi, err := strconv.ParseInt(vars["offset"], 10, 64) + if err != nil { + log.Errorf("parsing offset: %+v", err) + w.WriteHeader(500) + return + } + szi, err := strconv.ParseInt(vars["size"], 10, 64) + if err != nil { + log.Errorf("parsing spt: %+v", err) + w.WriteHeader(500) + return + } + + // The caller has a lock on this sector already, no need to get one here + + // passing 0 spt because we don't allocate anything + si := storage.SectorRef{ + ID: id, + ProofType: 0, + } + + paths, _, err := handler.Local.AcquireSector(r.Context(), si, ft, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) + if err != nil { + log.Errorf("%+v", err) + w.WriteHeader(500) + return + } + + path := storiface.PathByType(paths, ft) + if path == "" { + log.Error("acquired path was empty") + w.WriteHeader(500) + return + } + + pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path) + if err != nil { + log.Error("opening partial file: ", err) + w.WriteHeader(500) + return + } + defer func() { + if err := pf.Close(); err != nil { + log.Error("close partial file: ", err) + } + }() + + has, err := pf.HasAllocated(storiface.UnpaddedByteIndex(offi), abi.UnpaddedPieceSize(szi)) + if err != nil { + log.Error("has allocated: ", err) + w.WriteHeader(500) + return + } + + if has { + w.WriteHeader(http.StatusOK) return } + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) } func (handler *FetchHandler) remoteDeleteSector(w http.ResponseWriter, r *http.Request) { diff --git a/extern/sector-storage/stores/local.go b/extern/sector-storage/stores/local.go index 5a10b21b906..ab0a8eaebc8 100644 --- a/extern/sector-storage/stores/local.go +++ b/extern/sector-storage/stores/local.go @@ -158,7 +158,9 @@ func (p *path) sectorPath(sid abi.SectorID, fileType storiface.SectorFileType) s return filepath.Join(p.local, fileType.String(), storiface.SectorName(sid)) } -func NewLocal(ctx context.Context, ls LocalStorage, index SectorIndex, urls []string) (*Local, error) { +type URLs []string + +func NewLocal(ctx context.Context, ls LocalStorage, index SectorIndex, urls URLs) (*Local, error) { l := &Local{ localStorage: ls, index: index, diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index 4388a2ffbee..c990d583db8 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -3,6 +3,7 @@ package stores import ( "context" "encoding/json" + "fmt" "io" "io/ioutil" "math/bits" @@ -15,6 +16,7 @@ import ( "sort" "sync" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" @@ -293,6 +295,148 @@ func (r *Remote) fetch(ctx context.Context, url, outname string) error { } } +func (r *Remote) checkAllocated(ctx context.Context, url string, spt abi.RegisteredSealProof, offset, size abi.PaddedPieceSize) (bool, error) { + url = fmt.Sprintf("%s/%d/allocated/%d/%d", url, spt, offset.Unpadded(), size.Unpadded()) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return false, xerrors.Errorf("request: %w", err) + } + req.Header = r.auth.Clone() + req = req.WithContext(ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, xerrors.Errorf("do request: %w", err) + } + defer resp.Body.Close() // nolint + + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusRequestedRangeNotSatisfiable: + return false, nil + default: + return false, xerrors.Errorf("unexpected http response: %d", resp.StatusCode) + } +} + +func (r *Remote) readRemote(ctx context.Context, url string, spt abi.RegisteredSealProof, offset, size abi.PaddedPieceSize) (io.ReadCloser, error) { + if len(r.limit) >= cap(r.limit) { + log.Infof("Throttling remote read, %d already running", len(r.limit)) + } + + // TODO: Smarter throttling + // * Priority (just going sequentially is still pretty good) + // * Per interface + // * Aware of remote load + select { + case r.limit <- struct{}{}: + defer func() { <-r.limit }() + case <-ctx.Done(): + return nil, xerrors.Errorf("context error while waiting for fetch limiter: %w", ctx.Err()) + } + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, xerrors.Errorf("request: %w", err) + } + req.Header = r.auth.Clone() + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+size)) + req = req.WithContext(ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, xerrors.Errorf("do request: %w", err) + } + + if resp.StatusCode != 200 { + resp.Body.Close() // nolint + return nil, xerrors.Errorf("non-200 code: %d", resp.StatusCode) + } + + return resp.Body, nil +} + +// Reated gets a reader for unsealed file range. Can return nil in case the requested range isn't allocated in the file +func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size abi.PaddedPieceSize, ft storiface.SectorFileType) (io.ReadCloser, error) { + if ft != storiface.FTUnsealed { + return nil, xerrors.Errorf("reader only supports unsealed files") + } + + paths, _, err := r.local.AcquireSector(ctx, s, ft, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) + if err != nil { + return nil, xerrors.Errorf("acquire local: %w", err) + } + + path := storiface.PathByType(paths, ft) + var rd io.ReadCloser + if path == "" { + si, err := r.index.StorageFindSector(ctx, s.ID, ft, 0, false) + if err != nil { + return nil, err + } + + if len(si) == 0 { + return nil, xerrors.Errorf("failed to read sector %v from remote(%d): %w", s, ft, storiface.ErrSectorNotFound) + } + + sort.Slice(si, func(i, j int) bool { + return si[i].Weight < si[j].Weight + }) + + iloop: + for _, info := range si { + for _, url := range info.URLs { + ok, err := r.checkAllocated(ctx, url, s.ProofType, offset, size) + if err != nil { + log.Warnw("check if remote has piece", "url", url, "error", err) + continue + } + if !ok { + continue + } + + rd, err = r.readRemote(ctx, url, s.ProofType, offset, size) + if err != nil { + log.Warnw("reading from remote", "url", url, "error", err) + continue + } + log.Infof("Read remote %s (+%d,%d)", url, offset, size) + break iloop + } + } + } else { + log.Infof("Read local %s (+%d,%d)", path, offset, size) + ssize, err := s.ProofType.SectorSize() + if err != nil { + return nil, err + } + + pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path) + if err != nil { + return nil, xerrors.Errorf("opening partial file: %w", err) + } + + has, err := pf.HasAllocated(storiface.UnpaddedByteIndex(offset.Unpadded()), size.Unpadded()) + if err != nil { + return nil, xerrors.Errorf("has allocated: %w", err) + } + + if !has { + if err := pf.Close(); err != nil { + return nil, xerrors.Errorf("close partial file: %w", err) + } + + return nil, nil + } + + return pf.Reader(storiface.PaddedByteIndex(offset), size) + } + + // note: rd can be nil + return rd, nil +} + func (r *Remote) MoveStorage(ctx context.Context, s storage.SectorRef, types storiface.SectorFileType) error { // Make sure we have the data local _, _, err := r.AcquireSector(ctx, s, types, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove) diff --git a/extern/sector-storage/storiface/ffi.go b/extern/sector-storage/storiface/ffi.go index f6b2cbdd31d..5109307eb5f 100644 --- a/extern/sector-storage/storiface/ffi.go +++ b/extern/sector-storage/storiface/ffi.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" ) @@ -16,6 +17,13 @@ type UnpaddedByteIndex uint64 func (i UnpaddedByteIndex) Padded() PaddedByteIndex { return PaddedByteIndex(abi.UnpaddedPieceSize(i).Padded()) } +func (i UnpaddedByteIndex) Valid() error { + if i%127 != 0 { + return xerrors.Errorf("unpadded byte index must be a multiple of 127") + } + + return nil +} type PaddedByteIndex uint64 diff --git a/extern/storage-sealing/cbor_gen.go b/extern/storage-sealing/cbor_gen.go index 9e12b8649e9..b71c2863cff 100644 --- a/extern/storage-sealing/cbor_gen.go +++ b/extern/storage-sealing/cbor_gen.go @@ -8,7 +8,7 @@ import ( "sort" abi "github.com/filecoin-project/go-state-types/abi" - market "github.com/filecoin-project/specs-actors/actors/builtin/market" + api "github.com/filecoin-project/lotus/api" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -46,7 +46,7 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { return err } - // t.DealInfo (sealing.DealInfo) (struct) + // t.DealInfo (api.PieceDealInfo) (struct) if len("DealInfo") > cbg.MaxLength { return xerrors.Errorf("Value in field \"DealInfo\" was too long") } @@ -107,7 +107,7 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { } } - // t.DealInfo (sealing.DealInfo) (struct) + // t.DealInfo (api.PieceDealInfo) (struct) case "DealInfo": { @@ -120,7 +120,7 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { if err := br.UnreadByte(); err != nil { return err } - t.DealInfo = new(DealInfo) + t.DealInfo = new(api.PieceDealInfo) if err := t.DealInfo.UnmarshalCBOR(br); err != nil { return xerrors.Errorf("unmarshaling t.DealInfo pointer: %w", err) } @@ -136,384 +136,6 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *DealInfo) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{165}); err != nil { - return err - } - - scratch := make([]byte, 9) - - // t.PublishCid (cid.Cid) (struct) - if len("PublishCid") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"PublishCid\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("PublishCid")); err != nil { - return err - } - - if t.PublishCid == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err) - } - } - - // t.DealID (abi.DealID) (uint64) - if len("DealID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"DealID\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("DealID")); err != nil { - return err - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { - return err - } - - // t.DealProposal (market.DealProposal) (struct) - if len("DealProposal") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"DealProposal\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("DealProposal")); err != nil { - return err - } - - if err := t.DealProposal.MarshalCBOR(w); err != nil { - return err - } - - // t.DealSchedule (sealing.DealSchedule) (struct) - if len("DealSchedule") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"DealSchedule\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("DealSchedule")); err != nil { - return err - } - - if err := t.DealSchedule.MarshalCBOR(w); err != nil { - return err - } - - // t.KeepUnsealed (bool) (bool) - if len("KeepUnsealed") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil { - return err - } - - if err := cbg.WriteBool(w, t.KeepUnsealed); err != nil { - return err - } - return nil -} - -func (t *DealInfo) UnmarshalCBOR(r io.Reader) error { - *t = DealInfo{} - - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) - - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("DealInfo: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadStringBuf(br, scratch) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.PublishCid (cid.Cid) (struct) - case "PublishCid": - - { - - b, err := br.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err) - } - - t.PublishCid = &c - } - - } - // t.DealID (abi.DealID) (uint64) - case "DealID": - - { - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = abi.DealID(extra) - - } - // t.DealProposal (market.DealProposal) (struct) - case "DealProposal": - - { - - b, err := br.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { - return err - } - t.DealProposal = new(market.DealProposal) - if err := t.DealProposal.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err) - } - } - - } - // t.DealSchedule (sealing.DealSchedule) (struct) - case "DealSchedule": - - { - - if err := t.DealSchedule.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err) - } - - } - // t.KeepUnsealed (bool) (bool) - case "KeepUnsealed": - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.KeepUnsealed = false - case 21: - t.KeepUnsealed = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} -func (t *DealSchedule) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{162}); err != nil { - return err - } - - scratch := make([]byte, 9) - - // t.StartEpoch (abi.ChainEpoch) (int64) - if len("StartEpoch") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"StartEpoch\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("StartEpoch")); err != nil { - return err - } - - if t.StartEpoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil { - return err - } - } - - // t.EndEpoch (abi.ChainEpoch) (int64) - if len("EndEpoch") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"EndEpoch\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("EndEpoch")); err != nil { - return err - } - - if t.EndEpoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil { - return err - } - } - return nil -} - -func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error { - *t = DealSchedule{} - - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) - - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("DealSchedule: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadStringBuf(br, scratch) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.StartEpoch (abi.ChainEpoch) (int64) - case "StartEpoch": - { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.StartEpoch = abi.ChainEpoch(extraI) - } - // t.EndEpoch (abi.ChainEpoch) (int64) - case "EndEpoch": - { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.EndEpoch = abi.ChainEpoch(extraI) - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} func (t *SectorInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) diff --git a/extern/storage-sealing/gen/main.go b/extern/storage-sealing/gen/main.go index 97c2bacd5bd..825ce8d284b 100644 --- a/extern/storage-sealing/gen/main.go +++ b/extern/storage-sealing/gen/main.go @@ -12,8 +12,6 @@ import ( func main() { err := gen.WriteMapEncodersToFile("./cbor_gen.go", "sealing", sealing.Piece{}, - sealing.DealInfo{}, - sealing.DealSchedule{}, sealing.SectorInfo{}, sealing.Log{}, ) diff --git a/extern/storage-sealing/input.go b/extern/storage-sealing/input.go index 44d2e8275b4..e3c626be292 100644 --- a/extern/storage-sealing/input.go +++ b/extern/storage-sealing/input.go @@ -2,6 +2,7 @@ package sealing import ( "context" + "github.com/filecoin-project/lotus/api" "sort" "time" @@ -224,34 +225,34 @@ func (m *Sealing) handleAddPieceFailed(ctx statemachine.Context, sector SectorIn return nil } -func (m *Sealing) AddPieceToAnySector(ctx context.Context, size abi.UnpaddedPieceSize, data storage.Data, deal DealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) { +func (m *Sealing) SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, data storage.Data, deal api.PieceDealInfo) (api.SectorOffset, error) { log.Infof("Adding piece for deal %d (publish msg: %s)", deal.DealID, deal.PublishCid) if (padreader.PaddedSize(uint64(size))) != size { - return 0, 0, xerrors.Errorf("cannot allocate unpadded piece") + return api.SectorOffset{}, xerrors.Errorf("cannot allocate unpadded piece") } sp, err := m.currentSealProof(ctx) if err != nil { - return 0, 0, xerrors.Errorf("getting current seal proof type: %w", err) + return api.SectorOffset{}, xerrors.Errorf("getting current seal proof type: %w", err) } ssize, err := sp.SectorSize() if err != nil { - return 0, 0, err + return api.SectorOffset{}, err } if size > abi.PaddedPieceSize(ssize).Unpadded() { - return 0, 0, xerrors.Errorf("piece cannot fit into a sector") + return api.SectorOffset{}, xerrors.Errorf("piece cannot fit into a sector") } if _, err := deal.DealProposal.Cid(); err != nil { - return 0, 0, xerrors.Errorf("getting proposal CID: %w", err) + return api.SectorOffset{}, xerrors.Errorf("getting proposal CID: %w", err) } m.inputLk.Lock() if _, exist := m.pendingPieces[proposalCID(deal)]; exist { m.inputLk.Unlock() - return 0, 0, xerrors.Errorf("piece for deal %s already pending", proposalCID(deal)) + return api.SectorOffset{}, xerrors.Errorf("piece for deal %s already pending", proposalCID(deal)) } resCh := make(chan struct { @@ -283,7 +284,7 @@ func (m *Sealing) AddPieceToAnySector(ctx context.Context, size abi.UnpaddedPiec res := <-resCh - return res.sn, res.offset.Padded(), res.err + return api.SectorOffset{Sector: res.sn, Offset: res.offset.Padded()}, res.err } // called with m.inputLk @@ -425,7 +426,7 @@ func (m *Sealing) StartPacking(sid abi.SectorNumber) error { return m.sectors.Send(uint64(sid), SectorStartPacking{}) } -func proposalCID(deal DealInfo) cid.Cid { +func proposalCID(deal api.PieceDealInfo) cid.Cid { pc, err := deal.DealProposal.Cid() if err != nil { log.Errorf("DealProposal.Cid error: %+v", err) diff --git a/extern/storage-sealing/precommit_policy_test.go b/extern/storage-sealing/precommit_policy_test.go index 52814167a57..96abaea558f 100644 --- a/extern/storage-sealing/precommit_policy_test.go +++ b/extern/storage-sealing/precommit_policy_test.go @@ -2,6 +2,7 @@ package sealing_test import ( "context" + "github.com/filecoin-project/lotus/api" "testing" "github.com/filecoin-project/go-state-types/network" @@ -58,9 +59,9 @@ func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { Size: abi.PaddedPieceSize(1024), PieceCID: fakePieceCid(t), }, - DealInfo: &sealing.DealInfo{ + DealInfo: &api.PieceDealInfo{ DealID: abi.DealID(42), - DealSchedule: sealing.DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: abi.ChainEpoch(70), EndEpoch: abi.ChainEpoch(75), }, @@ -71,9 +72,9 @@ func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { Size: abi.PaddedPieceSize(1024), PieceCID: fakePieceCid(t), }, - DealInfo: &sealing.DealInfo{ + DealInfo: &api.PieceDealInfo{ DealID: abi.DealID(43), - DealSchedule: sealing.DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: abi.ChainEpoch(80), EndEpoch: abi.ChainEpoch(100), }, @@ -98,9 +99,9 @@ func TestBasicPolicyIgnoresExistingScheduleIfExpired(t *testing.T) { Size: abi.PaddedPieceSize(1024), PieceCID: fakePieceCid(t), }, - DealInfo: &sealing.DealInfo{ + DealInfo: &api.PieceDealInfo{ DealID: abi.DealID(44), - DealSchedule: sealing.DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: abi.ChainEpoch(1), EndEpoch: abi.ChainEpoch(10), }, @@ -125,9 +126,9 @@ func TestMissingDealIsIgnored(t *testing.T) { Size: abi.PaddedPieceSize(1024), PieceCID: fakePieceCid(t), }, - DealInfo: &sealing.DealInfo{ + DealInfo: &api.PieceDealInfo{ DealID: abi.DealID(44), - DealSchedule: sealing.DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: abi.ChainEpoch(1), EndEpoch: abi.ChainEpoch(10), }, diff --git a/extern/storage-sealing/sealing.go b/extern/storage-sealing/sealing.go index 8feca3b7b11..8556a4902b9 100644 --- a/extern/storage-sealing/sealing.go +++ b/extern/storage-sealing/sealing.go @@ -122,7 +122,7 @@ type openSector struct { type pendingPiece struct { size abi.UnpaddedPieceSize - deal DealInfo + deal api.PieceDealInfo data storage.Data diff --git a/extern/storage-sealing/types.go b/extern/storage-sealing/types.go index 58c35cf36ff..c5aed505a65 100644 --- a/extern/storage-sealing/types.go +++ b/extern/storage-sealing/types.go @@ -11,39 +11,22 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" - "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" ) // Piece is a tuple of piece and deal info type PieceWithDealInfo struct { Piece abi.PieceInfo - DealInfo DealInfo + DealInfo api.PieceDealInfo } // Piece is a tuple of piece info and optional deal type Piece struct { Piece abi.PieceInfo - DealInfo *DealInfo // nil for pieces which do not appear in deals (e.g. filler pieces) -} - -// DealInfo is a tuple of deal identity and its schedule -type DealInfo struct { - PublishCid *cid.Cid - DealID abi.DealID - DealProposal *market.DealProposal - DealSchedule DealSchedule - KeepUnsealed bool -} - -// DealSchedule communicates the time interval of a storage deal. The deal must -// appear in a sealed (proven) sector no later than StartEpoch, otherwise it -// is invalid. -type DealSchedule struct { - StartEpoch abi.ChainEpoch - EndEpoch abi.ChainEpoch + DealInfo *api.PieceDealInfo // nil for pieces which do not appear in deals (e.g. filler pieces) } type Log struct { diff --git a/gen/main.go b/gen/main.go index 9548344fd2a..0018b241d62 100644 --- a/gen/main.go +++ b/gen/main.go @@ -53,6 +53,8 @@ func main() { api.SealedRefs{}, api.SealTicket{}, api.SealSeed{}, + api.PieceDealInfo{}, + api.DealSchedule{}, ) if err != nil { fmt.Println(err) diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index e58257c8abc..2a1c0312d00 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -9,31 +9,32 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - "github.com/filecoin-project/lotus/chain/actors/builtin/paych" - "github.com/filecoin-project/lotus/chain/types" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" - "github.com/filecoin-project/lotus/extern/sector-storage/storiface" - "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" + "github.com/filecoin-project/lotus/chain/types" + sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/storage/sectorblocks" specstorage "github.com/filecoin-project/specs-storage/storage" ) var log = logging.Logger("retrievaladapter") type retrievalProviderNode struct { - miner *storage.Miner - sealer sectorstorage.SectorManager + maddr address.Address + secb sectorblocks.SectorBuilder + pp *sectorstorage.PieceProvider full v1api.FullNode } // NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the // Lotus Node -func NewRetrievalProviderNode(miner *storage.Miner, sealer sectorstorage.SectorManager, full v1api.FullNode) retrievalmarket.RetrievalProviderNode { - return &retrievalProviderNode{miner, sealer, full} +func NewRetrievalProviderNode(maddr dtypes.MinerAddress, secb sectorblocks.SectorBuilder, pp *sectorstorage.PieceProvider, full v1api.FullNode) retrievalmarket.RetrievalProviderNode { + return &retrievalProviderNode{address.Address(maddr), secb, pp, full} } func (rpn *retrievalProviderNode) GetMinerWorkerAddress(ctx context.Context, miner address.Address, tok shared.TipSetToken) (address.Address, error) { @@ -49,12 +50,12 @@ func (rpn *retrievalProviderNode) GetMinerWorkerAddress(ctx context.Context, min func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (io.ReadCloser, error) { log.Debugf("get sector %d, offset %d, length %d", sectorID, offset, length) - si, err := rpn.miner.GetSectorInfo(sectorID) + si, err := rpn.secb.SectorsStatus(ctx, sectorID, false) if err != nil { return nil, err } - mid, err := address.IDFromAddress(rpn.miner.Address()) + mid, err := address.IDFromAddress(rpn.maddr) if err != nil { return nil, err } @@ -64,27 +65,21 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID abi Miner: abi.ActorID(mid), Number: sectorID, }, - ProofType: si.SectorType, + ProofType: si.SealProof, + } + + var commD cid.Cid + if si.CommD != nil { + commD = *si.CommD } - // Set up a pipe so that data can be written from the unsealing process - // into the reader returned by this function - r, w := io.Pipe() - go func() { - var commD cid.Cid - if si.CommD != nil { - commD = *si.CommD - } - - // Read the piece into the pipe's writer, unsealing the piece if necessary - log.Debugf("read piece in sector %d, offset %d, length %d from miner %d", sectorID, offset, length, mid) - err := rpn.sealer.ReadPiece(ctx, w, ref, storiface.UnpaddedByteIndex(offset), length, si.TicketValue, commD) - if err != nil { - log.Errorf("failed to unseal piece from sector %d: %s", sectorID, err) - } - // Close the reader with any error that was returned while reading the piece - _ = w.CloseWithError(err) - }() + // Read the piece into the pipe's writer, unsealing the piece if necessary + log.Debugf("read piece in sector %d, offset %d, length %d from miner %d", sectorID, offset, length, mid) + r, unsealed, err := rpn.pp.ReadPiece(ctx, ref, storiface.UnpaddedByteIndex(offset), length, si.Ticket.Value, commD) + if err != nil { + log.Errorf("failed to unseal piece from sector %d: %s", sectorID, err) + } + _ = unsealed // todo: use return r, nil } diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index fbeaf3b3dca..b899c081074 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -95,11 +95,11 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema return nil, xerrors.Errorf("deal.PublishCid can't be nil") } - sdInfo := sealing.DealInfo{ + sdInfo := api.PieceDealInfo{ DealID: deal.DealID, DealProposal: &deal.Proposal, PublishCid: deal.PublishCid, - DealSchedule: sealing.DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: deal.ClientDealProposal.Proposal.StartEpoch, EndEpoch: deal.ClientDealProposal.Proposal.EndEpoch, }, @@ -240,19 +240,19 @@ func (n *ProviderNodeAdapter) LocatePieceForDealWithinSector(ctx context.Context // TODO: better strategy (e.g. look for already unsealed) var best api.SealedRef - var bestSi sealing.SectorInfo + var bestSi api.SectorInfo for _, r := range refs { - si, err := n.secb.Miner.GetSectorInfo(r.SectorID) + si, err := n.secb.SectorBuilder.SectorsStatus(ctx, r.SectorID, false) if err != nil { return 0, 0, 0, xerrors.Errorf("getting sector info: %w", err) } - if si.State == sealing.Proving { + if si.State == api.SectorState(sealing.Proving) { best = r bestSi = si break } } - if bestSi.State == sealing.UndefinedSectorState { + if bestSi.State == api.SectorState(sealing.UndefinedSectorState) { return 0, 0, 0, xerrors.New("no sealed sector found") } return best.SectorID, best.Offset, best.Size.Padded(), nil diff --git a/node/builder.go b/node/builder.go index c884b169b8b..acdecd86701 100644 --- a/node/builder.go +++ b/node/builder.go @@ -400,6 +400,7 @@ var MinerNode = Options( Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync), Override(new(dtypes.ProviderPieceStore), modules.NewProviderPieceStore), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), + Override(new(sectorblocks.SectorBuilder), From(new(*storage.Miner))), // Markets (retrieval) Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), @@ -488,10 +489,10 @@ func ConfigCommon(cfg *config.Common) Option { Override(SetApiEndpointKey, func(lr repo.LockedRepo, e dtypes.APIEndpoint) error { return lr.SetAPIEndpoint(e) }), - Override(new(sectorstorage.URLs), func(e dtypes.APIEndpoint) (sectorstorage.URLs, error) { + Override(new(stores.URLs), func(e dtypes.APIEndpoint) (stores.URLs, error) { ip := cfg.API.RemoteListenAddress - var urls sectorstorage.URLs + var urls stores.URLs urls = append(urls, "http://"+ip+"/remote") // TODO: This makes no assumptions, and probably could... return urls, nil }), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 27ab1af5f8a..786f68991b5 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -25,8 +25,6 @@ import ( storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" "github.com/filecoin-project/lotus/extern/sector-storage/stores" @@ -135,12 +133,12 @@ func (sm *StorageMinerAPI) PledgeSector(ctx context.Context) (abi.SectorID, erro // wait for the sector to enter the Packing state // TODO: instead of polling implement some pubsub-type thing in storagefsm for { - info, err := sm.Miner.GetSectorInfo(sr.ID.Number) + info, err := sm.Miner.SectorsStatus(ctx, sr.ID.Number, false) if err != nil { return abi.SectorID{}, xerrors.Errorf("getting pledged sector info: %w", err) } - if info.State != sealing.UndefinedSectorState { + if info.State != api.SectorState(sealing.UndefinedSectorState) { return sr.ID, nil } @@ -153,62 +151,11 @@ func (sm *StorageMinerAPI) PledgeSector(ctx context.Context) (abi.SectorID, erro } func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) { - info, err := sm.Miner.GetSectorInfo(sid) + sInfo, err := sm.Miner.SectorsStatus(ctx, sid, false) if err != nil { return api.SectorInfo{}, err } - deals := make([]abi.DealID, len(info.Pieces)) - for i, piece := range info.Pieces { - if piece.DealInfo == nil { - continue - } - deals[i] = piece.DealInfo.DealID - } - - log := make([]api.SectorLog, len(info.Log)) - for i, l := range info.Log { - log[i] = api.SectorLog{ - Kind: l.Kind, - Timestamp: l.Timestamp, - Trace: l.Trace, - Message: l.Message, - } - } - - sInfo := api.SectorInfo{ - SectorID: sid, - State: api.SectorState(info.State), - CommD: info.CommD, - CommR: info.CommR, - Proof: info.Proof, - Deals: deals, - Ticket: api.SealTicket{ - Value: info.TicketValue, - Epoch: info.TicketEpoch, - }, - Seed: api.SealSeed{ - Value: info.SeedValue, - Epoch: info.SeedEpoch, - }, - PreCommitMsg: info.PreCommitMessage, - CommitMsg: info.CommitMessage, - Retries: info.InvalidProofs, - ToUpgrade: sm.Miner.IsMarkedForUpgrade(sid), - - LastErr: info.LastErr, - Log: log, - // on chain info - SealProof: 0, - Activation: 0, - Expiration: 0, - DealWeight: big.Zero(), - VerifiedDealWeight: big.Zero(), - InitialPledge: big.Zero(), - OnTime: 0, - Early: 0, - } - if !showOnChainInfo { return sInfo, nil } @@ -237,6 +184,10 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb return sInfo, nil } +func (sm *StorageMinerAPI) SectorsUnsealPiece(ctx context.Context, sector sto.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error { + return sm.StorageMgr.SectorsUnsealPiece(ctx, sector, offset, size, randomness, commd) +} + // List all staged sectors func (sm *StorageMinerAPI) SectorsList(context.Context) ([]abi.SectorNumber, error) { sectors, err := sm.Miner.ListSectors() @@ -666,7 +617,7 @@ func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredP var rg storiface.RGetter if expensive { rg = func(ctx context.Context, id abi.SectorID) (cid.Cid, error) { - si, err := sm.Miner.GetSectorInfo(id.Number) + si, err := sm.Miner.SectorsStatus(ctx, id.Number, false) if err != nil { return cid.Undef, err } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index be949255f7e..dda48b78724 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -67,7 +67,6 @@ import ( "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/markets" marketevents "github.com/filecoin-project/lotus/markets/loggers" - "github.com/filecoin-project/lotus/markets/retrievaladapter" lotusminer "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -632,42 +631,45 @@ func RetrievalDealFilter(userFilter dtypes.RetrievalDealFilter) func(onlineOk dt } } +func RetrievalNetwork(h host.Host) rmnet.RetrievalMarketNetwork { + return rmnet.NewFromLibp2pHost(h) +} + // RetrievalProvider creates a new retrieval provider attached to the provider blockstore -func RetrievalProvider(h host.Host, - miner *storage.Miner, - sealer sectorstorage.SectorManager, - full v1api.FullNode, +func RetrievalProvider( + maddr dtypes.MinerAddress, + adapter retrievalmarket.RetrievalProviderNode, + netwk rmnet.RetrievalMarketNetwork, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, mds dtypes.StagingMultiDstore, dt dtypes.ProviderDataTransfer, - onlineOk dtypes.ConsiderOnlineRetrievalDealsConfigFunc, - offlineOk dtypes.ConsiderOfflineRetrievalDealsConfigFunc, userFilter dtypes.RetrievalDealFilter, ) (retrievalmarket.RetrievalProvider, error) { - adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full) - - maddr, err := minerAddrFromDS(ds) - if err != nil { - return nil, err - } - - netwk := rmnet.NewFromLibp2pHost(h) opt := retrievalimpl.DealDeciderOpt(retrievalimpl.DealDecider(userFilter)) - - return retrievalimpl.NewProvider(maddr, adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt) + return retrievalimpl.NewProvider(address.Address(maddr), adapter, netwk, pieceStore, mds, dt, namespace.Wrap(ds, datastore.NewKey("/retrievals/provider")), opt) } var WorkerCallsPrefix = datastore.NewKey("/worker/calls") var ManagerWorkPrefix = datastore.NewKey("/stmgr/calls") -func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, sc sectorstorage.SealerConfig, urls sectorstorage.URLs, sa sectorstorage.StorageAuth, ds dtypes.MetadataDS) (*sectorstorage.Manager, error) { + +func LocalStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, urls stores.URLs) (*stores.Local, error) { + ctx := helpers.LifecycleCtx(mctx, lc) + return stores.NewLocal(ctx, ls, si, urls) +} + +func RemoteStorage(lstor *stores.Local, si stores.SectorIndex, sa sectorstorage.StorageAuth, sc sectorstorage.SealerConfig) *stores.Remote { + return stores.NewRemote(lstor, si, http.Header(sa), sc.ParallelFetchLimit) +} + +func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, lstor *stores.Local, stor *stores.Remote, ls stores.LocalStorage, si stores.SectorIndex, sc sectorstorage.SealerConfig, sa sectorstorage.StorageAuth, ds dtypes.MetadataDS) (*sectorstorage.Manager, error) { ctx := helpers.LifecycleCtx(mctx, lc) wsts := statestore.New(namespace.Wrap(ds, WorkerCallsPrefix)) smsts := statestore.New(namespace.Wrap(ds, ManagerWorkPrefix)) - sst, err := sectorstorage.New(ctx, ls, si, sc, urls, sa, wsts, smsts) + sst, err := sectorstorage.New(ctx, lstor, stor, ls, si, sc, sa, wsts, smsts) if err != nil { return nil, err } diff --git a/storage/sealing.go b/storage/sealing.go index 8981c373866..d300a359d3f 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -2,9 +2,11 @@ package storage import ( "context" - "io" - + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/storage/sectorblocks" "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -19,8 +21,72 @@ func (m *Miner) Address() address.Address { return m.sealing.Address() } -func (m *Miner) AddPieceToAnySector(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d sealing.DealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) { - return m.sealing.AddPieceToAnySector(ctx, size, r, d) +func (m *Miner) SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d api.PieceDealInfo) (api.SectorOffset, error) { + return m.sealing.SectorAddPieceToAny(ctx, size, r, d) +} + +func (m *Miner) SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) { + if showOnChainInfo { + return api.SectorInfo{}, xerrors.Errorf("on-chain info not supported") + } + + info, err := m.sealing.GetSectorInfo(sid) + if err != nil { + return api.SectorInfo{}, err + } + + deals := make([]abi.DealID, len(info.Pieces)) + for i, piece := range info.Pieces { + if piece.DealInfo == nil { + continue + } + deals[i] = piece.DealInfo.DealID + } + + log := make([]api.SectorLog, len(info.Log)) + for i, l := range info.Log { + log[i] = api.SectorLog{ + Kind: l.Kind, + Timestamp: l.Timestamp, + Trace: l.Trace, + Message: l.Message, + } + } + + sInfo := api.SectorInfo{ + SectorID: sid, + State: api.SectorState(info.State), + CommD: info.CommD, + CommR: info.CommR, + Proof: info.Proof, + Deals: deals, + Ticket: api.SealTicket{ + Value: info.TicketValue, + Epoch: info.TicketEpoch, + }, + Seed: api.SealSeed{ + Value: info.SeedValue, + Epoch: info.SeedEpoch, + }, + PreCommitMsg: info.PreCommitMessage, + CommitMsg: info.CommitMessage, + Retries: info.InvalidProofs, + ToUpgrade: m.IsMarkedForUpgrade(sid), + + LastErr: info.LastErr, + Log: log, + // on chain info + SealProof: 0, + Activation: 0, + Expiration: 0, + DealWeight: big.Zero(), + VerifiedDealWeight: big.Zero(), + InitialPledge: big.Zero(), + OnTime: 0, + Early: 0, + } + + return sInfo, nil } func (m *Miner) StartPackingSector(sectorNum abi.SectorNumber) error { @@ -31,10 +97,6 @@ func (m *Miner) ListSectors() ([]sealing.SectorInfo, error) { return m.sealing.ListSectors() } -func (m *Miner) GetSectorInfo(sid abi.SectorNumber) (sealing.SectorInfo, error) { - return m.sealing.GetSectorInfo(sid) -} - func (m *Miner) PledgeSector(ctx context.Context) (storage.SectorRef, error) { return m.sealing.PledgeSector(ctx) } @@ -66,3 +128,5 @@ func (m *Miner) MarkForUpgrade(id abi.SectorNumber) error { func (m *Miner) IsMarkedForUpgrade(id abi.SectorNumber) bool { return m.sealing.IsMarkedForUpgrade(id) } + +var _ sectorblocks.SectorBuilder = &Miner{} diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index bc8456a1f28..efc5c5d8da6 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -16,11 +16,10 @@ import ( cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/abi" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/specs-storage/storage" ) type SealSerialization uint8 @@ -49,15 +48,20 @@ func DsKeyToDealID(key datastore.Key) (uint64, error) { } type SectorBlocks struct { - *storage.Miner + SectorBuilder keys datastore.Batching keyLk sync.Mutex } -func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS) *SectorBlocks { +type SectorBuilder interface { + SectorAddPieceToAny(ctx context.Context, size abi.UnpaddedPieceSize, r storage.Data, d api.PieceDealInfo) (api.SectorOffset, error) + SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnChainInfo bool) (api.SectorInfo, error) +} + +func NewSectorBlocks(sb SectorBuilder, ds dtypes.MetadataDS) *SectorBlocks { sbc := &SectorBlocks{ - Miner: miner, + SectorBuilder: sb, keys: namespace.Wrap(ds, dsPrefix), } @@ -96,19 +100,19 @@ func (st *SectorBlocks) writeRef(dealID abi.DealID, sectorID abi.SectorNumber, o return st.keys.Put(DealIDToDsKey(dealID), newRef) // TODO: batch somehow } -func (st *SectorBlocks) AddPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d sealing.DealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) { - sn, offset, err := st.Miner.AddPieceToAnySector(ctx, size, r, d) +func (st *SectorBlocks) AddPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d api.PieceDealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) { + so, err := st.SectorBuilder.SectorAddPieceToAny(ctx, size, r, d) if err != nil { return 0, 0, err } // TODO: DealID has very low finality here - err = st.writeRef(d.DealID, sn, offset, size) + err = st.writeRef(d.DealID, so.Sector, so.Offset, size) if err != nil { return 0, 0, xerrors.Errorf("writeRef: %w", err) } - return sn, offset, nil + return so.Sector, so.Offset, nil } func (st *SectorBlocks) List() (map[uint64][]api.SealedRef, error) { From ddcd0c8654e385ee7d61c031f429c749f7300533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Apr 2021 17:54:12 +0200 Subject: [PATCH 02/10] fix most tests --- extern/sector-storage/fr32/readers.go | 4 ++-- extern/sector-storage/mock/mock.go | 10 +++++++--- extern/sector-storage/piece_provider.go | 17 +++++++++++------ extern/sector-storage/stores/remote.go | 4 ++-- extern/storage-sealing/types_test.go | 5 +++-- markets/retrievaladapter/provider.go | 7 ++++--- node/builder.go | 8 ++++++++ node/test/builder.go | 18 ++++++++++++++---- 8 files changed, 51 insertions(+), 22 deletions(-) diff --git a/extern/sector-storage/fr32/readers.go b/extern/sector-storage/fr32/readers.go index 20f3e9b3185..256821c80f0 100644 --- a/extern/sector-storage/fr32/readers.go +++ b/extern/sector-storage/fr32/readers.go @@ -51,13 +51,13 @@ func (r *unpadReader) Read(out []byte) (int, error) { r.left -= uint64(todo) - n, err := r.src.Read(r.work[:todo]) + n, err := io.ReadAtLeast(r.src, r.work[:todo], int(todo)) if err != nil && err != io.EOF { return n, err } if n != int(todo) { - return 0, xerrors.Errorf("didn't read enough: %w", err) + return 0, xerrors.Errorf("didn't read enough: %d / %d, left %d, out %d", n, todo, r.left, len(out)) } Unpad(r.work[:todo], out[:todo.Unpadded()]) diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index ae7d54985fe..d3e76e88109 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "fmt" "io" + "io/ioutil" "math/rand" "sync" @@ -372,13 +373,12 @@ func generateFakePoSt(sectorInfo []proof2.SectorInfo, rpt func(abi.RegisteredSea } } -func (mgr *SectorMgr) ReadPiece(ctx context.Context, w io.Writer, sectorID storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error { +func (mgr *SectorMgr) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) { if offset != 0 { panic("implme") } - _, err := io.CopyN(w, bytes.NewReader(mgr.pieces[mgr.sectors[sectorID.ID].pieces[0]]), int64(size)) - return err + return ioutil.NopCloser(bytes.NewReader(mgr.pieces[mgr.sectors[sector.ID].pieces[0]][:size])), false, nil } func (mgr *SectorMgr) StageFakeData(mid abi.ActorID, spt abi.RegisteredSealProof) (storage.SectorRef, []abi.PieceInfo, error) { @@ -489,6 +489,10 @@ func (mgr *SectorMgr) ReturnFetch(ctx context.Context, callID storiface.CallID, panic("not supported") } +func (mgr *SectorMgr) SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error { + return nil +} + func (m mockVerif) VerifySeal(svi proof2.SealVerifyInfo) (bool, error) { plen, err := svi.SealProof.ProofSize() if err != nil { diff --git a/extern/sector-storage/piece_provider.go b/extern/sector-storage/piece_provider.go index 3c5b80aa371..008c06e63a0 100644 --- a/extern/sector-storage/piece_provider.go +++ b/extern/sector-storage/piece_provider.go @@ -1,6 +1,7 @@ package sectorstorage import ( + "bufio" "context" "io" @@ -19,21 +20,25 @@ type Unsealer interface { SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error } -type PieceProvider struct { +type PieceProvider interface { + ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) +} + +type pieceProvider struct { storage *stores.Remote index stores.SectorIndex uns Unsealer } -func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unsealer) *PieceProvider { - return &PieceProvider{ +func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unsealer) PieceProvider { + return &pieceProvider{ storage: storage, index: index, uns: uns, } } -func (p *PieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (io.ReadCloser, context.CancelFunc, error) { +func (p *pieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (io.ReadCloser, context.CancelFunc, error) { // acquire a lock purely for reading unsealed sectors ctx, cancel := context.WithCancel(ctx) if err := p.index.StorageLock(ctx, sector.ID, storiface.FTUnsealed, storiface.FTNone); err != nil { @@ -53,7 +58,7 @@ func (p *PieceProvider) tryReadUnsealedPiece(ctx context.Context, sector storage return r, cancel, nil } -func (p *PieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) { +func (p *pieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (io.ReadCloser, bool, error) { if err := offset.Valid(); err != nil { return nil, false, xerrors.Errorf("offset is not valid: %w", err) } @@ -92,7 +97,7 @@ func (p *PieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, } return &funcCloser{ - Reader: upr, + Reader: bufio.NewReaderSize(upr, 127), close: func() error { err = r.Close() unlock() diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index c990d583db8..280b3760b53 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -341,7 +341,7 @@ func (r *Remote) readRemote(ctx context.Context, url string, spt abi.RegisteredS return nil, xerrors.Errorf("request: %w", err) } req.Header = r.auth.Clone() - req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+size)) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)) req = req.WithContext(ctx) resp, err := http.DefaultClient.Do(req) @@ -349,7 +349,7 @@ func (r *Remote) readRemote(ctx context.Context, url string, spt abi.RegisteredS return nil, xerrors.Errorf("do request: %w", err) } - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { resp.Body.Close() // nolint return nil, xerrors.Errorf("non-200 code: %d", resp.StatusCode) } diff --git a/extern/storage-sealing/types_test.go b/extern/storage-sealing/types_test.go index aa314c37a68..d9c88e5bec2 100644 --- a/extern/storage-sealing/types_test.go +++ b/extern/storage-sealing/types_test.go @@ -10,6 +10,7 @@ import ( cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" tutils "github.com/filecoin-project/specs-actors/v2/support/testing" ) @@ -22,9 +23,9 @@ func TestSectorInfoSerialization(t *testing.T) { t.Fatal(err) } - dealInfo := DealInfo{ + dealInfo := api.PieceDealInfo{ DealID: d, - DealSchedule: DealSchedule{ + DealSchedule: api.DealSchedule{ StartEpoch: 0, EndEpoch: 100, }, diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index 2a1c0312d00..7c858d088c8 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -8,6 +8,7 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -27,13 +28,13 @@ var log = logging.Logger("retrievaladapter") type retrievalProviderNode struct { maddr address.Address secb sectorblocks.SectorBuilder - pp *sectorstorage.PieceProvider + pp sectorstorage.PieceProvider full v1api.FullNode } // NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the // Lotus Node -func NewRetrievalProviderNode(maddr dtypes.MinerAddress, secb sectorblocks.SectorBuilder, pp *sectorstorage.PieceProvider, full v1api.FullNode) retrievalmarket.RetrievalProviderNode { +func NewRetrievalProviderNode(maddr dtypes.MinerAddress, secb sectorblocks.SectorBuilder, pp sectorstorage.PieceProvider, full v1api.FullNode) retrievalmarket.RetrievalProviderNode { return &retrievalProviderNode{address.Address(maddr), secb, pp, full} } @@ -77,7 +78,7 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID abi log.Debugf("read piece in sector %d, offset %d, length %d from miner %d", sectorID, offset, length, mid) r, unsealed, err := rpn.pp.ReadPiece(ctx, ref, storiface.UnpaddedByteIndex(offset), length, si.Ticket.Value, commD) if err != nil { - log.Errorf("failed to unseal piece from sector %d: %s", sectorID, err) + return nil, xerrors.Errorf("failed to unseal piece from sector %d: %w", sectorID, err) } _ = unsealed // todo: use diff --git a/node/builder.go b/node/builder.go index acdecd86701..6a205d9a7c5 100644 --- a/node/builder.go +++ b/node/builder.go @@ -36,6 +36,7 @@ import ( "github.com/filecoin-project/go-fil-markets/discovery" discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl" "github.com/filecoin-project/go-fil-markets/retrievalmarket" + rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" @@ -63,6 +64,7 @@ import ( _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/markets/dealfilter" + "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" @@ -373,9 +375,12 @@ var MinerNode = Options( Override(new(*stores.Index), stores.NewIndex), Override(new(stores.SectorIndex), From(new(*stores.Index))), Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), + Override(new(*stores.Local), modules.LocalStorage), + Override(new(*stores.Remote), modules.RemoteStorage), Override(new(*sectorstorage.Manager), modules.SectorStorage), Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))), Override(new(storiface.WorkerReturn), From(new(sectorstorage.SectorManager))), + Override(new(sectorstorage.Unsealer), From(new(*sectorstorage.Manager))), // Sector storage: Proofs Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier), @@ -403,6 +408,9 @@ var MinerNode = Options( Override(new(sectorblocks.SectorBuilder), From(new(*storage.Miner))), // Markets (retrieval) + Override(new(sectorstorage.PieceProvider), sectorstorage.NewPieceProvider), + Override(new(retrievalmarket.RetrievalProviderNode), retrievaladapter.NewRetrievalProviderNode), + Override(new(rmnet.RetrievalMarketNetwork), modules.RetrievalNetwork), Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), Override(new(dtypes.RetrievalDealFilter), modules.RetrievalDealFilter(nil)), Override(HandleRetrievalKey, modules.HandleRetrieval), diff --git a/node/test/builder.go b/node/test/builder.go index 7e26966a807..f4088a5bab2 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -485,11 +485,16 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes } fulls[i].Stb = storageBuilder(fulls[i], mn, node.Options( - node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) { + node.Override(new(*mock.SectorMgr), func() (sectorstorage.SectorManager, error) { return mock.NewMockSectorMgr(nil), nil }), - node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), + node.Override(new(sectorstorage.SectorManager), node.From(new(*mock.SectorMgr))), + node.Override(new(sectorstorage.Unsealer), node.From(new(*mock.SectorMgr))), + node.Override(new(sectorstorage.PieceProvider), node.From(new(*mock.SectorMgr))), + node.Unset(new(*sectorstorage.Manager)), + + node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), )) } @@ -523,11 +528,16 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes opts = node.Options() } storers[i] = CreateTestStorageNode(ctx, t, genms[i].Worker, maddrs[i], pidKeys[i], f, mn, node.Options( - node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) { + node.Override(new(*mock.SectorMgr), func() (*mock.SectorMgr, error) { return mock.NewMockSectorMgr(sectors), nil }), - node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), + node.Override(new(sectorstorage.SectorManager), node.From(new(*mock.SectorMgr))), + node.Override(new(sectorstorage.Unsealer), node.From(new(*mock.SectorMgr))), + node.Override(new(sectorstorage.PieceProvider), node.From(new(*mock.SectorMgr))), + node.Unset(new(*sectorstorage.Manager)), + + node.Override(new(ffiwrapper.Verifier), mock.MockVerifier), opts, )) From 7c08207762a1829bae94d141334c5975c428baab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Apr 2021 17:54:23 +0200 Subject: [PATCH 03/10] gofmt --- extern/sector-storage/manager.go | 2 +- extern/sector-storage/stores/http_handler.go | 1 - markets/retrievaladapter/provider.go | 8 ++++---- node/modules/storageminer.go | 1 - storage/sectorblocks/blocks.go | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index d8f421d988b..3bf0ac19e63 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -690,4 +690,4 @@ func (m *Manager) Close(ctx context.Context) error { } var _ SectorManager = &Manager{} -var _ Unsealer = &Manager{} \ No newline at end of file +var _ Unsealer = &Manager{} diff --git a/extern/sector-storage/stores/http_handler.go b/extern/sector-storage/stores/http_handler.go index b65b5272b66..1f9ed30dded 100644 --- a/extern/sector-storage/stores/http_handler.go +++ b/extern/sector-storage/stores/http_handler.go @@ -17,7 +17,6 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" - ) var log = logging.Logger("stores") diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index 7c858d088c8..84066be2f85 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -26,10 +26,10 @@ import ( var log = logging.Logger("retrievaladapter") type retrievalProviderNode struct { - maddr address.Address - secb sectorblocks.SectorBuilder - pp sectorstorage.PieceProvider - full v1api.FullNode + maddr address.Address + secb sectorblocks.SectorBuilder + pp sectorstorage.PieceProvider + full v1api.FullNode } // NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index dda48b78724..6271d4f4c10 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -653,7 +653,6 @@ func RetrievalProvider( var WorkerCallsPrefix = datastore.NewKey("/worker/calls") var ManagerWorkPrefix = datastore.NewKey("/stmgr/calls") - func LocalStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, urls stores.URLs) (*stores.Local, error) { ctx := helpers.LifecycleCtx(mctx, lc) return stores.NewLocal(ctx, ls, si, urls) diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index efc5c5d8da6..cb5469e202b 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -62,7 +62,7 @@ type SectorBuilder interface { func NewSectorBlocks(sb SectorBuilder, ds dtypes.MetadataDS) *SectorBlocks { sbc := &SectorBlocks{ SectorBuilder: sb, - keys: namespace.Wrap(ds, dsPrefix), + keys: namespace.Wrap(ds, dsPrefix), } return sbc From ee591d42254e900a4d9aca995802ead50d1281bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Apr 2021 18:15:55 +0200 Subject: [PATCH 04/10] Fix api dep on ffi --- .../sector-storage/ffiwrapper/sealer_cgo.go | 19 ++++++++++--------- .../ffiwrapper/unseal_ranges.go | 3 ++- .../partialfile.go | 19 +++++++++++-------- extern/sector-storage/stores/http_handler.go | 4 ++-- extern/sector-storage/stores/remote.go | 4 ++-- 5 files changed, 27 insertions(+), 22 deletions(-) rename extern/sector-storage/{ffiwrapper => partialfile}/partialfile.go (93%) diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index e8d656cac26..21d14a4d9e6 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -23,6 +23,7 @@ import ( commpffi "github.com/filecoin-project/go-commp-utils/ffiwrapper" "github.com/filecoin-project/go-commp-utils/zerocomm" "github.com/filecoin-project/lotus/extern/sector-storage/fr32" + "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) @@ -66,7 +67,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi } var done func() - var stagedFile *PartialFile + var stagedFile *partialfile.PartialFile defer func() { if done != nil { @@ -87,7 +88,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) } - stagedFile, err = createPartialFile(maxPieceSize, stagedPath.Unsealed) + stagedFile, err = partialfile.CreatePartialFile(maxPieceSize, stagedPath.Unsealed) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("creating unsealed sector file: %w", err) } @@ -97,7 +98,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) } - stagedFile, err = OpenPartialFile(maxPieceSize, stagedPath.Unsealed) + stagedFile, err = partialfile.OpenPartialFile(maxPieceSize, stagedPath.Unsealed) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("opening unsealed sector file: %w", err) } @@ -244,7 +245,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off // try finding existing unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTNone, storiface.PathStorage) - var pf *PartialFile + var pf *partialfile.PartialFile switch { case xerrors.Is(err, storiface.ErrSectorNotFound): @@ -254,7 +255,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off } defer done() - pf, err = createPartialFile(maxPieceSize, unsealedPath.Unsealed) + pf, err = partialfile.CreatePartialFile(maxPieceSize, unsealedPath.Unsealed) if err != nil { return xerrors.Errorf("create unsealed file: %w", err) } @@ -262,7 +263,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off case err == nil: defer done() - pf, err = OpenPartialFile(maxPieceSize, unsealedPath.Unsealed) + pf, err = partialfile.OpenPartialFile(maxPieceSize, unsealedPath.Unsealed) if err != nil { return xerrors.Errorf("opening partial file: %w", err) } @@ -414,7 +415,7 @@ func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector storag } maxPieceSize := abi.PaddedPieceSize(ssize) - pf, err := OpenPartialFile(maxPieceSize, path.Unsealed) + pf, err := partialfile.OpenPartialFile(maxPieceSize, path.Unsealed) if err != nil { if xerrors.Is(err, os.ErrNotExist) { return false, nil @@ -576,7 +577,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef, if len(keepUnsealed) > 0 { - sr := pieceRun(0, maxPieceSize) + sr := partialfile.PieceRun(0, maxPieceSize) for _, s := range keepUnsealed { si := &rlepluslazy.RunSliceIterator{} @@ -598,7 +599,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector storage.SectorRef, } defer done() - pf, err := OpenPartialFile(maxPieceSize, paths.Unsealed) + pf, err := partialfile.OpenPartialFile(maxPieceSize, paths.Unsealed) if err == nil { var at uint64 for sr.HasNext() { diff --git a/extern/sector-storage/ffiwrapper/unseal_ranges.go b/extern/sector-storage/ffiwrapper/unseal_ranges.go index 4519fc21e6a..3a13c73a74a 100644 --- a/extern/sector-storage/ffiwrapper/unseal_ranges.go +++ b/extern/sector-storage/ffiwrapper/unseal_ranges.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) @@ -17,7 +18,7 @@ const mergeGaps = 32 << 20 // TODO const expandRuns = 16 << 20 // unseal more than requested for future requests func computeUnsealRanges(unsealed rlepluslazy.RunIterator, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (rlepluslazy.RunIterator, error) { - todo := pieceRun(offset.Padded(), size.Padded()) + todo := partialfile.PieceRun(offset.Padded(), size.Padded()) todo, err := rlepluslazy.Subtract(todo, unsealed) if err != nil { return nil, xerrors.Errorf("compute todo-unsealed: %w", err) diff --git a/extern/sector-storage/ffiwrapper/partialfile.go b/extern/sector-storage/partialfile/partialfile.go similarity index 93% rename from extern/sector-storage/ffiwrapper/partialfile.go rename to extern/sector-storage/partialfile/partialfile.go index 0e8827dd3f2..2ef68de738c 100644 --- a/extern/sector-storage/ffiwrapper/partialfile.go +++ b/extern/sector-storage/partialfile/partialfile.go @@ -1,4 +1,4 @@ -package ffiwrapper +package partialfile import ( "encoding/binary" @@ -7,6 +7,7 @@ import ( "syscall" "github.com/detailyang/go-fallocate" + logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" rlepluslazy "github.com/filecoin-project/go-bitfield/rle" @@ -16,6 +17,8 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/storiface" ) +var log = logging.Logger("partialfile") + const veryLargeRle = 1 << 20 // Sectors can be partially unsealed. We support this by appending a small @@ -57,7 +60,7 @@ func writeTrailer(maxPieceSize int64, w *os.File, r rlepluslazy.RunIterator) err return w.Truncate(maxPieceSize + int64(rb) + 4) } -func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { +func CreatePartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) // nolint if err != nil { return nil, xerrors.Errorf("openning partial file '%s': %w", path, err) @@ -188,7 +191,7 @@ func (pf *PartialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedP return nil, err } - and, err := rlepluslazy.And(have, pieceRun(offset, size)) + and, err := rlepluslazy.And(have, PieceRun(offset, size)) if err != nil { return nil, err } @@ -212,7 +215,7 @@ func (pf *PartialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi. return err } - ored, err := rlepluslazy.Or(have, pieceRun(offset, size)) + ored, err := rlepluslazy.Or(have, PieceRun(offset, size)) if err != nil { return err } @@ -234,7 +237,7 @@ func (pf *PartialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPie return xerrors.Errorf("deallocating: %w", err) } - s, err := rlepluslazy.Subtract(have, pieceRun(offset, size)) + s, err := rlepluslazy.Subtract(have, PieceRun(offset, size)) if err != nil { return err } @@ -257,7 +260,7 @@ func (pf *PartialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedP return nil, err } - and, err := rlepluslazy.And(have, pieceRun(offset, size)) + and, err := rlepluslazy.And(have, PieceRun(offset, size)) if err != nil { return nil, err } @@ -285,7 +288,7 @@ func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi return false, err } - u, err := rlepluslazy.And(have, pieceRun(offset.Padded(), size.Padded())) + u, err := rlepluslazy.And(have, PieceRun(offset.Padded(), size.Padded())) if err != nil { return false, err } @@ -298,7 +301,7 @@ func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi return abi.PaddedPieceSize(uc) == size.Padded(), nil } -func pieceRun(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) rlepluslazy.RunIterator { +func PieceRun(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) rlepluslazy.RunIterator { var runs []rlepluslazy.Run if offset > 0 { runs = append(runs, rlepluslazy.Run{ diff --git a/extern/sector-storage/stores/http_handler.go b/extern/sector-storage/stores/http_handler.go index 1f9ed30dded..239b571cf27 100644 --- a/extern/sector-storage/stores/http_handler.go +++ b/extern/sector-storage/stores/http_handler.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" ) @@ -205,7 +205,7 @@ func (handler *FetchHandler) remoteGetAllocated(w http.ResponseWriter, r *http.R return } - pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path) + pf, err := partialfile.OpenPartialFile(abi.PaddedPieceSize(ssize), path) if err != nil { log.Error("opening partial file: ", err) w.WriteHeader(500) diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index 280b3760b53..4142dac091b 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -16,8 +16,8 @@ import ( "sort" "sync" - "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" + "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/extern/sector-storage/tarutil" @@ -412,7 +412,7 @@ func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size a return nil, err } - pf, err := ffiwrapper.OpenPartialFile(abi.PaddedPieceSize(ssize), path) + pf, err := partialfile.OpenPartialFile(abi.PaddedPieceSize(ssize), path) if err != nil { return nil, xerrors.Errorf("opening partial file: %w", err) } From 17c7ae6ba84a152912e7c7f6e94fcb0a02b0054b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Apr 2021 18:22:02 +0200 Subject: [PATCH 05/10] Bump worker api version --- api/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/version.go b/api/version.go index 743170f049f..f56397095ef 100644 --- a/api/version.go +++ b/api/version.go @@ -58,7 +58,7 @@ var ( FullAPIVersion1 = newVer(2, 1, 0) MinerAPIVersion0 = newVer(1, 0, 1) - WorkerAPIVersion0 = newVer(1, 0, 0) + WorkerAPIVersion0 = newVer(1, 1, 0) ) //nolint:varcheck,deadcode From bb874ccd737da45afebbb3c2f11b79832f8fe70e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Apr 2021 18:22:15 +0200 Subject: [PATCH 06/10] miner: Check worker API version on connect --- node/impl/storminer.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 786f68991b5..96a6ea7d194 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -334,6 +334,15 @@ func (sm *StorageMinerAPI) WorkerConnect(ctx context.Context, url string) error return xerrors.Errorf("connecting remote storage failed: %w", err) } + vw, err := w.Version(ctx) + if err != nil { + return xerrors.Errorf("getting worker version: %w", err) + } + + if !vw.EqMajorMinor(api.WorkerAPIVersion0) { + return xerrors.Errorf("remote API version didn't match (expected %s, remote %s)", api.WorkerAPIVersion0, vw) + } + log.Infof("Connected to a remote worker at %s", url) return sm.StorageMgr.AddWorker(ctx, w) From aabeca5839b9f70857f8e3ae36ef6440ef3312dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Apr 2021 18:23:24 +0200 Subject: [PATCH 07/10] appease the linter --- extern/storage-sealing/input.go | 2 +- extern/storage-sealing/precommit_policy_test.go | 2 +- storage/sealing.go | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/extern/storage-sealing/input.go b/extern/storage-sealing/input.go index e3c626be292..8f80b257f1d 100644 --- a/extern/storage-sealing/input.go +++ b/extern/storage-sealing/input.go @@ -2,7 +2,6 @@ package sealing import ( "context" - "github.com/filecoin-project/lotus/api" "sort" "time" @@ -15,6 +14,7 @@ import ( "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/lotus/api" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" diff --git a/extern/storage-sealing/precommit_policy_test.go b/extern/storage-sealing/precommit_policy_test.go index 96abaea558f..9dbc5d92d91 100644 --- a/extern/storage-sealing/precommit_policy_test.go +++ b/extern/storage-sealing/precommit_policy_test.go @@ -2,7 +2,6 @@ package sealing_test import ( "context" - "github.com/filecoin-project/lotus/api" "testing" "github.com/filecoin-project/go-state-types/network" @@ -15,6 +14,7 @@ import ( commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" ) diff --git a/storage/sealing.go b/storage/sealing.go index d300a359d3f..4b17ac46876 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -2,17 +2,18 @@ package storage import ( "context" - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/lotus/api" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + "github.com/filecoin-project/lotus/storage/sectorblocks" ) // TODO: refactor this to be direct somehow From ac39417f4445eb2352f5a14ff298c6c1be8427e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Apr 2021 18:35:13 +0200 Subject: [PATCH 08/10] Fix deadline toggling test --- node/test/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/test/builder.go b/node/test/builder.go index f4088a5bab2..b247847d623 100644 --- a/node/test/builder.go +++ b/node/test/builder.go @@ -485,7 +485,7 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []tes } fulls[i].Stb = storageBuilder(fulls[i], mn, node.Options( - node.Override(new(*mock.SectorMgr), func() (sectorstorage.SectorManager, error) { + node.Override(new(*mock.SectorMgr), func() (*mock.SectorMgr, error) { return mock.NewMockSectorMgr(nil), nil }), node.Override(new(sectorstorage.SectorManager), node.From(new(*mock.SectorMgr))), From 911b558da807db07f9b17f6e8c69870e65c845a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Apr 2021 20:08:50 +0200 Subject: [PATCH 09/10] Fix unsealed sector acquire logic --- extern/sector-storage/ffiwrapper/sealer_cgo.go | 6 +++--- extern/sector-storage/manager.go | 2 +- extern/sector-storage/piece_provider.go | 3 +++ extern/sector-storage/stores/index.go | 3 +++ extern/sector-storage/stores/local.go | 4 ++++ extern/sector-storage/stores/remote.go | 2 +- extern/sector-storage/storiface/storage.go | 2 ++ 7 files changed, 17 insertions(+), 5 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index 21d14a4d9e6..a17a82eefd5 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -244,12 +244,12 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off maxPieceSize := abi.PaddedPieceSize(ssize) // try finding existing - unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTNone, storiface.PathStorage) + unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTNone, storiface.PathStoragePrefer) var pf *partialfile.PartialFile switch { case xerrors.Is(err, storiface.ErrSectorNotFound): - unsealedPath, done, err = sb.sectors.AcquireSector(ctx, sector, storiface.FTNone, storiface.FTUnsealed, storiface.PathStorage) + unsealedPath, done, err = sb.sectors.AcquireSector(ctx, sector, storiface.FTNone, storiface.FTUnsealed, storiface.PathStoragePrefer) if err != nil { return xerrors.Errorf("acquire unsealed sector path (allocate): %w", err) } @@ -286,7 +286,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off return nil } - srcPaths, srcDone, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTCache|storiface.FTSealed, storiface.FTNone, storiface.PathStorage) + srcPaths, srcDone, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTCache|storiface.FTSealed, storiface.FTNone, storiface.PathStoragePrefer) if err != nil { return xerrors.Errorf("acquire sealed sector paths: %w", err) } diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index 3bf0ac19e63..ef9c4b867da 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -237,7 +237,7 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storage.SectorR return xerrors.Errorf("getting sector size: %w", err) } - selector := newExistingSelector(m.index, sector.ID, storiface.FTSealed|storiface.FTCache, false) + selector := newExistingSelector(m.index, sector.ID, storiface.FTSealed|storiface.FTCache, true) log.Debugf("schedule unseal for sector %d", sector.ID) err = m.sched.Schedule(ctx, sector, sealtasks.TTUnseal, selector, unsealFetch, func(ctx context.Context, w Worker) error { diff --git a/extern/sector-storage/piece_provider.go b/extern/sector-storage/piece_provider.go index 008c06e63a0..30510395708 100644 --- a/extern/sector-storage/piece_provider.go +++ b/extern/sector-storage/piece_provider.go @@ -67,6 +67,9 @@ func (p *pieceProvider) ReadPiece(ctx context.Context, sector storage.SectorRef, } r, unlock, err := p.tryReadUnsealedPiece(ctx, sector, offset, size) + if xerrors.Is(err, storiface.ErrSectorNotFound) { + err = nil + } if err != nil { return nil, false, err } diff --git a/extern/sector-storage/stores/index.go b/extern/sector-storage/stores/index.go index 4acc2ecdb6c..a9f7ef7f6ce 100644 --- a/extern/sector-storage/stores/index.go +++ b/extern/sector-storage/stores/index.go @@ -395,6 +395,9 @@ func (i *Index) StorageBestAlloc(ctx context.Context, allocate storiface.SectorF if (pathType == storiface.PathStorage) && !p.info.CanStore { continue } + if (pathType == storiface.PathStoragePrefer) && !p.info.CanStore && !p.info.CanSeal { + continue + } if spaceReq > uint64(p.fsi.Available) { log.Debugf("not allocating on %s, out of space (available: %d, need: %d)", p.info.ID, p.fsi.Available, spaceReq) diff --git a/extern/sector-storage/stores/local.go b/extern/sector-storage/stores/local.go index ab0a8eaebc8..bd03d34ed4f 100644 --- a/extern/sector-storage/stores/local.go +++ b/extern/sector-storage/stores/local.go @@ -493,6 +493,10 @@ func (st *Local) AcquireSector(ctx context.Context, sid storage.SectorRef, exist continue } + if (pathType == storiface.PathStoragePrefer) && !si.CanStore && !si.CanSeal { + continue + } + // TODO: Check free space best = p.sectorPath(sid.ID, fileType) diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index 4142dac091b..5c89fd24193 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -117,7 +117,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s storage.SectorRef, existin } odt := storiface.FSOverheadSeal - if pathType == storiface.PathStorage { + if pathType == storiface.PathStorage || pathType == storiface.PathStoragePrefer { odt = storiface.FsOverheadFinalized } diff --git a/extern/sector-storage/storiface/storage.go b/extern/sector-storage/storiface/storage.go index e836002d5de..98d128b1cf3 100644 --- a/extern/sector-storage/storiface/storage.go +++ b/extern/sector-storage/storiface/storage.go @@ -5,6 +5,8 @@ type PathType string const ( PathStorage PathType = "storage" PathSealing PathType = "sealing" + + PathStoragePrefer PathType = "storage-prefer" ) type AcquireMode string From 900fea74f3daf698ec479312501bc4b3575c09d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Apr 2021 18:53:42 +0200 Subject: [PATCH 10/10] Address review --- extern/sector-storage/fr32/readers.go | 2 +- extern/sector-storage/stores/remote.go | 2 +- markets/retrievaladapter/provider.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extern/sector-storage/fr32/readers.go b/extern/sector-storage/fr32/readers.go index 256821c80f0..1fb7ebb7ebe 100644 --- a/extern/sector-storage/fr32/readers.go +++ b/extern/sector-storage/fr32/readers.go @@ -56,7 +56,7 @@ func (r *unpadReader) Read(out []byte) (int, error) { return n, err } - if n != int(todo) { + if n < int(todo) { return 0, xerrors.Errorf("didn't read enough: %d / %d, left %d, out %d", n, todo, r.left, len(out)) } diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go index 5c89fd24193..cc2bedd1f3c 100644 --- a/extern/sector-storage/stores/remote.go +++ b/extern/sector-storage/stores/remote.go @@ -357,7 +357,7 @@ func (r *Remote) readRemote(ctx context.Context, url string, spt abi.RegisteredS return resp.Body, nil } -// Reated gets a reader for unsealed file range. Can return nil in case the requested range isn't allocated in the file +// Reader gets a reader for unsealed file range. Can return nil in case the requested range isn't allocated in the file func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size abi.PaddedPieceSize, ft storiface.SectorFileType) (io.ReadCloser, error) { if ft != storiface.FTUnsealed { return nil, xerrors.Errorf("reader only supports unsealed files") diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index 84066be2f85..648d7b54295 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -74,7 +74,7 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID abi commD = *si.CommD } - // Read the piece into the pipe's writer, unsealing the piece if necessary + // Get a reader for the piece, unsealing the piece if necessary log.Debugf("read piece in sector %d, offset %d, length %d from miner %d", sectorID, offset, length, mid) r, unsealed, err := rpn.pp.ReadPiece(ctx, ref, storiface.UnpaddedByteIndex(offset), length, si.Ticket.Value, commD) if err != nil {