Skip to content

Commit

Permalink
Merge pull request #4873 from ipfs/fix/4871
Browse files Browse the repository at this point in the history
make the tar writer handle sharded ipfs directories
  • Loading branch information
whyrusleeping authored Mar 25, 2018
2 parents d4a4355 + 71a0a5c commit a98a4f4
Show file tree
Hide file tree
Showing 4 changed files with 20 additions and 13 deletions.
4 changes: 4 additions & 0 deletions core/commands/unixfs/ls.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,10 @@ possible, please use 'ipfs ls' instead.
switch t {
case unixfspb.Data_File:
break
case unixfspb.Data_HAMTShard:
// We need a streaming ls API for this.
res.SetError(fmt.Errorf("cannot list large directories yet"), cmdkit.ErrNormal)
return
case unixfspb.Data_Directory:
links := make([]LsLink, len(merkleNode.Links()))
output.Objects[hash].Links = links
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# MIT Licensed; see the LICENSE file in this repository.
#

test_description="Test global enable sharding flag"
test_description="Test directory sharding"

. lib/test-lib.sh

Expand All @@ -23,6 +23,10 @@ test_add_large_dir() {
echo "$exphash" > sharddir_exp &&
test_cmp sharddir_exp sharddir_out
'
test_expect_success "ipfs get on very large directory succeeds" '
ipfs get -o testdata-out "$exphash" &&
test_cmp testdata testdata-out
'
}

test_init_ipfs
Expand Down
21 changes: 10 additions & 11 deletions unixfs/archive/tar/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,23 +39,22 @@ func NewWriter(ctx context.Context, dag ipld.DAGService, archive bool, compressi
}

func (w *Writer) writeDir(nd *mdag.ProtoNode, fpath string) error {
dir, err := uio.NewDirectoryFromNode(w.Dag, nd)
if err != nil {
return err
}
if err := writeDirHeader(w.TarW, fpath); err != nil {
return err
}

for i, ng := range ipld.GetDAG(w.ctx, w.Dag, nd) {
child, err := ng.Get(w.ctx)
return dir.ForEachLink(w.ctx, func(l *ipld.Link) error {
child, err := w.Dag.Get(w.ctx, l.Cid)
if err != nil {
return err
}

npath := path.Join(fpath, nd.Links()[i].Name)
if err := w.WriteNode(child, npath); err != nil {
return err
}
}

return nil
npath := path.Join(fpath, l.Name)
return w.WriteNode(child, npath)
})
}

func (w *Writer) writeFile(nd *mdag.ProtoNode, pb *upb.Data, fpath string) error {
Expand Down Expand Up @@ -83,7 +82,7 @@ func (w *Writer) WriteNode(nd ipld.Node, fpath string) error {
switch pb.GetType() {
case upb.Data_Metadata:
fallthrough
case upb.Data_Directory:
case upb.Data_Directory, upb.Data_HAMTShard:
return w.writeDir(nd, fpath)
case upb.Data_Raw:
fallthrough
Expand Down
2 changes: 1 addition & 1 deletion unixfs/io/pbdagreader.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func (dr *PBDagReader) precalcNextBuf(ctx context.Context) error {
}

switch pb.GetType() {
case ftpb.Data_Directory:
case ftpb.Data_Directory, ftpb.Data_HAMTShard:
// A directory should not exist within a file
return ft.ErrInvalidDirLocation
case ftpb.Data_File:
Expand Down

0 comments on commit a98a4f4

Please sign in to comment.