From fc9ab1a76875b8448b8ee4c8e0170bb1e455b0db Mon Sep 17 00:00:00 2001 From: Brian Mason Date: Thu, 9 Feb 2023 08:57:13 -0700 Subject: [PATCH] Merge work 0.14.0 to netapp (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add changelog entry for validating exclude patterns * Update 030_preparing_a_new_repo.rst * prune: Fix crash on empty snapshot * prune: Don't print stack trace if snapshot can't be loaded * Update github.com/minio/minio-go/v7 to v7.0.27 This version adds support for Cloudflare R2, as discussed in #3757 * Update gopkg.in/yaml This fixes a panic in invalid input, but I think we aren't affected. * internal/restic: Custom ID.MarshalJSON This skips an allocation. internal/archiver benchmarks, Linux/amd64: name old time/op new time/op delta ArchiverSaveFileSmall-8 3.94ms ± 6% 3.91ms ± 6% ~ (p=0.947 n=20+20) ArchiverSaveFileLarge-8 304ms ± 3% 301ms ± 4% ~ (p=0.265 n=18+18) name old speed new speed delta ArchiverSaveFileSmall-8 1.04MB/s ± 6% 1.05MB/s ± 6% ~ (p=0.803 n=20+20) ArchiverSaveFileLarge-8 142MB/s ± 3% 143MB/s ± 4% ~ (p=0.421 n=18+19) name old alloc/op new alloc/op delta ArchiverSaveFileSmall-8 17.9MB ± 0% 17.9MB ± 0% -0.01% (p=0.000 n=19+19) ArchiverSaveFileLarge-8 382MB ± 2% 382MB ± 1% ~ (p=0.687 n=20+19) name old allocs/op new allocs/op delta ArchiverSaveFileSmall-8 540 ± 1% 528 ± 0% -2.19% (p=0.000 n=19+19) ArchiverSaveFileLarge-8 1.93k ± 3% 1.79k ± 4% -7.06% (p=0.000 n=20+20) * Fix linter check * archiver: Remove cleanup goroutine from BufferPool This isn't doing anything. Channels should get cleaned up by the GC when the last reference to them disappears, just like all other data structures. Also inlined BufferPool.Put in Buffer.Release, its only caller. * cmd/restic: Remove trailing "..." from progress messages These were added after message since the last refactor of the progress printing code. Also skips an allocation in the common case. * migrate: Cleanup option to request repository check * archiver: remove tomb usage * archiver: free workers once finished * get rid of tomb package * backend/sftp: Support atomic rename ... if the server has posix-rename@openssh.com. OpenSSH introduced this extension in 2008: https://github.com/openssh/openssh-portable/commit/7c29661471d0a7590ed41dec76661174c99b1c94 * internal/repository: Fix LoadBlob + fuzz test When given a buf that is big enough for a compressed blob but not its decompressed contents, the copy at the end of LoadBlob would skip the last part of the contents. Fixes #3783. * fix handling of maxKeys in SearchKey * fix flaky key test * tweak password test count changelog * all: Move away from pkg/errors, easy cases github.com/pkg/errors is no longer getting updates, because Go 1.13 went with the more flexible errors.{As,Is} function. Use those instead: errors from pkg/errors already support the Unwrap interface used by 1.13 error handling. Also: * check for io.EOF with a straight ==. That value should not be wrapped, and the chunker (whose error is checked in the cases changed) does not wrap it. * Give custom Error methods pointer receivers, so there's no ambiguity when type-switching since the value type will no longer implement error. * Make restic.ErrAlreadyLocked private, and rename it to alreadyLockedError to match the stdlib convention that error type names end in Error. * Same with rest.ErrIsNotExist => rest.notExistError. * Make s3.Backend.IsAccessDenied a private function. * backend: Move semaphores to a dedicated package ... called backend/sema. I resisted the temptation to call the main type sema.Phore. Also, semaphores are now passed by value to skip a level of indirection when using them. * restic prune: Merge three loops over the index There were three loops over the index in restic prune, to find duplicates, to determine sizes (in pack.Size) and to generate packInfos. These three are now one loop. This way, prune doesn't need to construct a set of duplicate blobs, pack.Size doesn't need to contain special logic for prune's use case (the onlyHdr argument) and pack.Size doesn't need to construct a map only to have it immediately transformed into a different map. Some quick testing on a 160GiB local repo doesn't show running time or memory use of restic prune --dry-run changing significantly. * cmd/restic, limiter: Move config knowledge to internal packages The GlobalOptions struct now embeds a backend.TransportOptions, so it doesn't need to construct one in open and create. The upload and download limits are similarly now a struct in internal/limiter that is embedded in GlobalOptions. * Revert "restic prune: Merge three loops over the index" This reverts commit 8bdfcf779fb4e7260fc05649beb7c524d7518bbe. Should fix #3809. Also needed to make #3290 apply cleanly. * repository: index saving belongs into the MasterIndex * repository: add Save method to MasterIndex interface * repository: make flushPacks private * repository: remove unused index.Store * repository: inline index.encode * repository: remove unused index.ListPack * repository: remove unused (Master)Index.Count * repository: Properly set id for finalized index As MergeFinalIndex and index uploads can occur concurrently, it is necessary for MergeFinalIndex to check whether the IDs for an index were already set before merging it. Otherwise, we'd loose the ID of an index which is set _after_ uploading it. * repository: remove MasterIndex.All() * repository: hide MasterIndex.FinalizeFullIndexes / FinalizeNotFinalIndexes * repository: simplify CreateIndexFromPacks * repository: remove unused packIDToIndex field * repository: cleanup * drop unused repository.Loader interface * redact http authorization header in debug log output * redacted keys/token in backend config debug log * redact swift auth token in debug output * Return real size from SaveBlob * Print number of bytes added to the repo This includes optional compression and crypto overhead. * stats: return storage size for raw-data mode raw-data summed up the size of the blob plaintexts. However, with compression this makes little sense as the storage size in the repository is lower due to compression. Thus sum up the actual size each blob takes in the repository. * Account for pack header overhead at each entry This will miss the pack header crypto overhead and the length field, which only amount to a few bytes per pack file. * extend compression feature changelog entry * rebuild-index: correctly rebuild index for mixed packs For mixed packs, data and tree blobs were stored in separate index entries. This results in warning from the check command and maybe other problems. * check: Print full ids The short ids are not always unique. In addition, recovering from damages is easier when having the full ids as that makes it easier to access the corresponding files. * check: remove dead code * Don't crash if SecretString is uninitialized * tag: Remove unnecessary flush call * repository: Rework blob saving to use an async pack uploader Previously, SaveAndEncrypt would assemble blobs into packs and either return immediately if the pack is not yet full or upload the pack file otherwise. The upload will block the current goroutine until it finishes. Now, the upload is done using separate goroutines. This requires changes to the error handling. As uploads are no longer tied to a SaveAndEncrypt call, failed uploads are signaled using an errgroup. To count the uploaded amount of data, the pack header overhead is no longer returned by `packer.Finalize` but rather by `packer.HeaderOverhead`. This helper method is necessary to continue returning the pack header overhead directly to the responsible call to `repository.SaveBlob`. Without the method this would not be possible, as packs are finalized asynchronously. * archiver: Limit blob saver count to GOMAXPROCS Now with the asynchronous uploaders there's no more benefit from using more blob savers than we have CPUs. Thus use just one blob saver for each CPU we are allowed to use. * archiver: Reduce tree saver concurrency Large amount of tree savers have no obvious benefit, however they can increase the amount of (potentially large) trees kept in memory. * repository: Limit to a single pending pack file Use only a single not completed pack file to keep the number of open and active pack files low. The main change here is to defer hashing the pack file to the upload step. This prevents the pack assembly step to become a bottleneck as the only task is now to write data to the temporary pack file. The tests are cleaned up to no longer reimplement packer manager functions. * Document connections and compression option * Add changelog for async pack uploads * adapt workers based on whether an operation is CPU or IO-bound Use runtime.GOMAXPROCS(0) as worker count for CPU-bound tasks, repo.Connections() for IO-bound task and a combination if a task can be both. Streaming packs is treated as IO-bound as adding more worker cannot provide a speedup. Typical IO-bound tasks are download / uploading / deleting files. Decoding / Encoding / Verifying are usually CPU-bound. Several tasks are a combination of both, e.g. for combined download and decode functions. In the latter case add both limits together. As the backends have their own concurrency limits restic still won't download more than repo.Connections() files in parallel, but the additional workers can decode already downloaded data in parallel. * Document automatic CPU/IO-concurrency * Fix data race in blob_saver After the `BlobSaver` job is submitted, the buffer can be released and reused by another `FileSaver` even before `BlobSaver.Save` returns. That FileSaver will the change `buf.Data` leading to wrong backup statistics. Found by `go test -race ./...`: WARNING: DATA RACE Write at 0x00c0000784a0 by goroutine 41: github.com/restic/restic/internal/archiver.(*FileSaver).saveFile() /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:176 +0x789 github.com/restic/restic/internal/archiver.(*FileSaver).worker() /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:242 +0x2af github.com/restic/restic/internal/archiver.NewFileSaver.func2() /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:88 +0x5d golang.org/x/sync/errgroup.(*Group).Go.func1() /home/michael/go/pkg/mod/golang.org/x/sync@v0.0.0-20210220032951-036812b2e83c/errgroup/errgroup.go:57 +0x91 Previous read at 0x00c0000784a0 by goroutine 29: github.com/restic/restic/internal/archiver.(*BlobSaver).Save() /home/michael/Projekte/restic/restic/internal/archiver/blob_saver.go:57 +0x1dd github.com/restic/restic/internal/archiver.(*BlobSaver).Save-fm() :1 +0xac github.com/restic/restic/internal/archiver.(*FileSaver).saveFile() /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:191 +0x855 github.com/restic/restic/internal/archiver.(*FileSaver).worker() /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:242 +0x2af github.com/restic/restic/internal/archiver.NewFileSaver.func2() /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:88 +0x5d golang.org/x/sync/errgroup.(*Group).Go.func1() /home/michael/go/pkg/mod/golang.org/x/sync@v0.0.0-20210220032951-036812b2e83c/errgroup/errgroup.go:57 +0x91 * Fix minor typo in docs * Wording: change repo to repository * Restore: validate provided patterns * Add testRunRestoreAssumeFailure function * Test restore fails when using invalid patterns * Fix wording in changelog template * Add changelog entry * Added hint for --compression max in migration process Added hint for --compression max in migration process. Since this is a onetime process users should be aware of this and consider this step. * Wording: replace further repo occurrences with repository * doc: update sample help output * doc: Rework hint to repack with max compression * doc: Add note about using rclone for Google Drive It wasn't clear that Google Cloud Storage and Google Drive are two different services and that one should use the rclone backend for the latter. This commit adds a note with this information. * azure: add SAS authentication option * azure: Strip ? prefix from sas token * backup: clarify usage string Using the `--files-from` options it is possible to run `backup` without specifying any source paths directly on the command line. * prune: Enhance treatment of duplicates * prune: handle very high duplication of some blobs Suggested-By: Alexander Weiss * prune: code cleanups * repository: extract LoadTree/SaveTree The repository has no real idea what a Tree is. So these methods never belonged there. * repository: extract Load/StoreJSONUnpacked A Load/Store method for each data type is much clearer. As a result the repository no longer needs a method to load / store json. * mock: move to internal/backend * limiter: move to internal/backend * crypto: move crypto buffer helpers * restorer: extract hardlinks index from restic package * backend: extract readerat from restic package * check: complain about mixed pack files * check: Complain about usage of s3 legacy layout * check: Deprecate `--check-unused` Unused blobs are not a problem but rather expected to exist now that prune by default does not remove every unused blob. However, the option has caused questions from users whether a repository is damaged or not, so just remove that option. Note that the remaining code is left intact as it is still useful for our test cases. * checker: Fix S3 legacy layout detection * Fix S3 legacy layout migration * Add changelog for stricter checks * archiver: remove dead attribute from FutureNode * archiver: cleanup Saver interface * archiver: remove unused fileInfo from progress callback * archiver: unify FutureTree/File into futureNode There is no real difference between the FutureTree and FutureFile structs. However, differentiating both increases the size of the FutureNode struct. The FutureNode struct is now only 16 bytes large on 64bit platforms. That way is has a very low overhead if the corresponding file/directory was not processed yet. There is a special case for nodes that were reused from the parent snapshot, as a go channel seems to have 96 bytes overhead which would result in a memory usage regression. * archiver: Incrementally serialize tree nodes That way it is not necessary to keep both the Nodes forming a Tree and the serialized JSON version in memory. * archiver: reduce memory usage for large files FutureBlob now uses a Take() method as a more memory-efficient way to retrieve the futures result. In addition, futures are now collected while saving the file. As only a limited number of blobs can be queued for uploading, for a large file nearly all FutureBlobs already have their result ready, such that the FutureBlob object just consumes memory. * Add changelog for the optimized tree serialization * Remove stale comments from backend/sftp The preExec and postExec functions were removed in 0bdb131521f84ebce3541f8ccd684c93892b5e66 from 2018. * Speed up restic init over slow SFTP links pkg/sftp.Client.MkdirAll(d) does a Stat to determine if d exists and is a directory, then a recursive call to create the parent, so the calls for data/?? each take three round trips. Doing a Mkdir first should eliminate two round trips for 255/256 data directories as well as all but one of the top-level directories. Also, we can do all of the calls concurrently. This may reintroduce some of the Stat calls when multiple goroutines try to create the same parent, but at the default number of connections, that should not be much of a problem. * Add environment variable RESTIC_COMPRESSION * prune: separate collecting/printing/pruning * prune: split into smaller functions * prune: Add internal integrity check After repacking every blob that should be kept must have been repacked. We have seen a few cases in which a single blob went missing, which could have been caused by a bitflip somewhere. This sanity check might help catch some of these cases. * repository: try to recover from invalid blob while repacking If a blob that should be kept is invalid, Repack will now try to request the blob using LoadBlob. Only return an error if that fails. * prune: move code * repository: Test fallback to existing blobs * Add changelog for #3837/#3840 * internal/restic: Handle EINVAL for xattr on Solaris Also make the errors a bit less verbose by not prepending the operation, since pkg/xattr already does that. Old errors looked like Listxattr: xattr.list /myfiles/.zfs/snapshot: invalid argument * Add possibility to set snapshot ID (used in test) * Generalize fuse snapshot dirs implemetation + allow "/" in tags and snapshot template * Make snapshots dirs in mount command customizable * fuse: cleanup test * fuse: remove unused MetaDir * add option for setting min pack size * prune: add repack-small parameter * prune: reduce priority of repacking small packs * repository: prevent header overfill * document minPackSize * rework pack size parameter documentation * update restic help snippets in documentation * Add changelog for packsize option * rename option to --pack-size * Only repack small files if there are multiple of them * Always repack very small pack files * s3: Disable multipart uploads below 200MB * Add note that pack-size is not an exact limit * Reword prune --repack-small description * repository: StreamPack in parts if there are too large gaps For large pack sizes we might be only interested in the first and last blob of a pack file. Thus stream a pack file in multiple parts if the gaps between requested blobs grow too large. * Remove unused hooks mechanism * debug: enable debug support for release builds * debug: support roundtripper logging also for release builds Different from debug builds do not use the eofDetectRoundTripper if logging is disabled. * update documentation to reflect DEBUG_LOG for release builds * Add changelog for DEBUG_LOG available in release builds * fuse: Redesign snapshot dirstruct Cleanly separate the directory presentation and the snapshot directory structure. SnapshotsDir now translates the dirStruct into a format usable by the fuse library and contains only minimal special case rules. All decisions have moved into SnapshotsDirStructure which now creates a fully preassembled tree data structure. * Mention --snapshot-template and --time-template in changelog * mount: remove unused inode field from root node * mount: Fix parent inode used by snapshots dir * Update tests to Go 1.19 * Bump golangci-lint version * restic: Use stable sorting in snapshot policy sort.Sort is not guaranteed to be stable. Go 1.19 has changed the sorting algorithm which resulted in changes of the sort order. When comparing snapshots with identical timestamp but different paths and tags lists, there is not meaningful order among them. So just keep their order stable. * stats: Add snapshots count to json output * Fix typo with double percentage in help text * doc: Update link to GCS documentation Updates the link to Google Cloud Storage documentation about creating a service account key. * doc: Update more links to GCS documentation * forget: Error when invalid unit is given in duration policy * doc: Fix typo in compression section * forget: Fail test if duration parsing error is missing * comment cleanup gofmt reformatted the comment * mount: Map slashes in tags to underscores Suggested-by: greatroar <> * copy: replace --repo2 with --from-repo `init` and `copy` use `--repo2` with two different meaning which has proven to be confusing for users. `--from-repo` now consistently marks a source repository from which data is read. `--repo` is now always the target/destination repository. * mount: Only remember successful snapshot refreshes If the context provided by the fuse library is canceled before the index was loaded this could lead to missing snapshots. * gofmt all files Apparently the rules for comment formatting have changed with go 1.19. * doc: document aws session token * helper: don't setup cmd paths twice * helper: cleanups * helper: Reduce number of parallel builds a bit The go compiler is already parallelized. The high concurrency caused my podman container to hit a resource limit. * helper: download modules as first step There's no use in running that step in parallel. * repository: Do not report ignored packs in EachByPack Ignored packs were reported as an empty pack by EachByPack. The most immediate effect of this is that the progress bar for rebuilding the index reports processing more packs than actually exist. * Add note that larger packs increase disk wear * doc: fix typo * update dependencies * downgrade bazil/fuse again to retain macOS support * remain compatible with go 1.15 * restic: Cleanup xattr error handling for Solaris Since xattr 0.4.8 (https://github.com/pkg/xattr/pull/68) returns ENOTSUP similar to Linux. * rclone: Return a permanent error if rclone already exited rclone can exit early for example when the connection to rclone is relayed for example via ssh: `-o rclone.program='ssh user@example.org forced-command'` * Polish changelog entries * doc: Improve/clarify preparing and versions of repositories * Further changelog polishing * Fix typo in the environment variable name for --from-password-file * Prepare changelog for 0.14.0 * Generate CHANGELOG.md for 0.14.0 * Update manpages and auto-completion * Add version for 0.14.0 * go mod tidy run * fix some merge errors * tweaked linting to avoid merge nightmares * took out lint because its not working and bugged me and updated the version for netapp * updated to 1.8 --------- Co-authored-by: Lorenz Bausch Co-authored-by: MichaelEischer Co-authored-by: Arigbede Moses Co-authored-by: Alexander Neumann <123276+fd0@users.noreply.github.com> Co-authored-by: Alexander Neumann Co-authored-by: greatroar <61184462+greatroar@users.noreply.github.com> Co-authored-by: Jayson Wang Co-authored-by: mattxtaz Co-authored-by: lbausch Co-authored-by: JsBergbau <37013344+JsBergbau@users.noreply.github.com> Co-authored-by: rawtaz Co-authored-by: Roger Gammans Co-authored-by: Alexander Weiss Co-authored-by: Kyle Brennan Co-authored-by: greatroar <@> Co-authored-by: Leo R. Lundgren Co-authored-by: bmason --- .gitattributes | 2 + .github/workflows/netapp-cicd.yml | 34 - .golangci.yml | 12 +- CHANGELOG.md | 440 +++++++++++- CONTRIBUTING.md | 9 +- Dockerfile | 2 +- NETAPPVERSION | 2 +- VERSION | 2 +- changelog/0.14.0_2022-08-25/issue-1153 | 9 + changelog/0.14.0_2022-08-25/issue-1842 | 8 + changelog/0.14.0_2022-08-25/issue-21 | 28 + changelog/0.14.0_2022-08-25/issue-2162 | 18 + changelog/0.14.0_2022-08-25/issue-2248 | 8 + changelog/0.14.0_2022-08-25/issue-2291 | 12 + changelog/0.14.0_2022-08-25/issue-2295 | 14 + changelog/0.14.0_2022-08-25/issue-2696 | 13 + changelog/0.14.0_2022-08-25/issue-2907 | 15 + changelog/0.14.0_2022-08-25/issue-3114 | 12 + changelog/0.14.0_2022-08-25/issue-3295 | 13 + changelog/0.14.0_2022-08-25/issue-3428 | 14 + changelog/0.14.0_2022-08-25/issue-3432 | 14 + changelog/0.14.0_2022-08-25/issue-3465 | 10 + changelog/0.14.0_2022-08-25/issue-3685 | 7 + changelog/0.14.0_2022-08-25/issue-3692 | 13 + changelog/0.14.0_2022-08-25/issue-3709 | 11 + changelog/0.14.0_2022-08-25/issue-3720 | 13 + changelog/0.14.0_2022-08-25/issue-3736 | 8 + changelog/0.14.0_2022-08-25/issue-3837 | 8 + changelog/0.14.0_2022-08-25/issue-3861 | 9 + changelog/0.14.0_2022-08-25/pull-3419 | 21 + changelog/0.14.0_2022-08-25/pull-3475 | 9 + changelog/0.14.0_2022-08-25/pull-3484 | 13 + changelog/0.14.0_2022-08-25/pull-3513 | 10 + changelog/0.14.0_2022-08-25/pull-3680 | 8 + changelog/0.14.0_2022-08-25/pull-3716 | 7 + changelog/0.14.0_2022-08-25/pull-3729 | 8 + changelog/0.14.0_2022-08-25/pull-3742 | 19 + changelog/0.14.0_2022-08-25/pull-3772 | 14 + changelog/0.14.0_2022-08-25/pull-3773 | 7 + changelog/0.14.0_2022-08-25/pull-3776 | 11 + changelog/0.14.0_2022-08-25/pull-3819 | 11 + changelog/TEMPLATE | 2 +- cmd/restic/cmd_backup.go | 89 ++- cmd/restic/cmd_cat.go | 9 +- cmd/restic/cmd_check.go | 55 +- cmd/restic/cmd_copy.go | 113 ++- cmd/restic/cmd_debug.go | 119 ++-- cmd/restic/cmd_diff.go | 31 +- cmd/restic/cmd_dump.go | 20 +- cmd/restic/cmd_find.go | 8 +- cmd/restic/cmd_forget.go | 21 +- cmd/restic/cmd_init.go | 35 +- cmd/restic/cmd_ls.go | 8 +- cmd/restic/cmd_migrate.go | 29 +- cmd/restic/cmd_mount.go | 55 +- cmd/restic/cmd_prune.go | 430 +++++++++--- cmd/restic/cmd_rebuild_index.go | 3 +- cmd/restic/cmd_recover.go | 37 +- cmd/restic/cmd_restore.go | 36 +- cmd/restic/cmd_snapshots.go | 4 +- cmd/restic/cmd_stats.go | 47 +- cmd/restic/cmd_tag.go | 8 +- cmd/restic/delete.go | 6 +- cmd/restic/find.go | 20 +- cmd/restic/global.go | 109 +-- cmd/restic/global_debug.go | 1 + cmd/restic/global_release.go | 1 + cmd/restic/integration_filter_pattern_test.go | 106 +++ cmd/restic/integration_fuse_test.go | 7 +- cmd/restic/integration_helpers_test.go | 3 + cmd/restic/integration_helpers_unix_test.go | 3 +- .../integration_helpers_windows_test.go | 3 +- cmd/restic/integration_test.go | 109 ++- cmd/restic/main.go | 4 +- cmd/restic/progress.go | 5 +- cmd/restic/secondary_repo.go | 89 ++- cmd/restic/secondary_repo_test.go | 60 +- doc.go | 2 +- doc/020_installation.rst | 9 +- doc/030_preparing_a_new_repo.rst | 130 +++- doc/040_backup.rst | 18 +- doc/045_working_with_repos.rst | 50 +- doc/047_tuning_backup_parameters.rst | 80 +++ doc/050_restore.rst | 2 +- doc/060_forget.rst | 49 +- doc/080_examples.rst | 3 +- doc/090_participating.rst | 31 +- doc/bash-completion.sh | 274 ++++++-- doc/cache.rst | 6 +- doc/design.rst | 195 ++++-- doc/faq.rst | 6 +- doc/fish-completion.fish | 3 +- doc/index.rst | 1 + doc/man/restic-backup.1 | 120 ++-- doc/man/restic-cache.1 | 72 +- doc/man/restic-cat.1 | 66 +- doc/man/restic-check.1 | 76 +- doc/man/restic-copy.1 | 100 +-- doc/man/restic-diff.1 | 72 +- doc/man/restic-dump.1 | 74 +- doc/man/restic-find.1 | 112 +-- doc/man/restic-forget.1 | 139 ++-- doc/man/restic-generate.1 | 76 +- doc/man/restic-init.1 | 94 +-- doc/man/restic-key.1 | 72 +- doc/man/restic-list.1 | 66 +- doc/man/restic-ls.1 | 80 ++- doc/man/restic-migrate.1 | 75 +- doc/man/restic-mount.1 | 118 ++-- doc/man/restic-prune.1 | 86 ++- doc/man/restic-rebuild-index.1 | 74 +- doc/man/restic-recover.1 | 66 +- doc/man/restic-restore.1 | 88 +-- doc/man/restic-self-update.1 | 74 +- doc/man/restic-snapshots.1 | 80 ++- doc/man/restic-stats.1 | 86 +-- doc/man/restic-tag.1 | 86 +-- doc/man/restic-unlock.1 | 70 +- doc/man/restic-version.1 | 66 +- doc/man/restic.1 | 66 +- doc/manual_rest.rst | 18 +- doc/zsh-completion.zsh | 36 +- go.mod | 72 +- go.sum | 91 +-- helpers/build-release-binaries/main.go | 30 +- internal/archiver/archiver.go | 296 ++++---- internal/archiver/archiver_test.go | 189 ++--- internal/archiver/archiver_unix_test.go | 1 + internal/archiver/archiver_windows_test.go | 1 + internal/archiver/blob_saver.go | 85 +-- internal/archiver/blob_saver_test.go | 28 +- internal/archiver/buffer.go | 63 +- internal/archiver/file_saver.go | 140 ++-- internal/archiver/file_saver_test.go | 28 +- internal/archiver/scanner.go | 6 +- internal/archiver/scanner_test.go | 10 +- internal/archiver/testing.go | 6 +- internal/archiver/tree_saver.go | 111 ++- internal/archiver/tree_saver_test.go | 39 +- internal/backend/azure/azure.go | 71 +- internal/backend/azure/azure_test.go | 5 +- internal/backend/azure/config.go | 3 +- internal/backend/b2/b2.go | 26 +- internal/backend/b2/b2_test.go | 3 +- internal/backend/b2/config.go | 2 +- internal/backend/backend_retry_test.go | 2 +- internal/backend/dryrun/dry_backend.go | 8 + internal/backend/foreground_sysv.go | 1 + internal/backend/foreground_test.go | 1 + internal/backend/foreground_unix.go | 1 + internal/backend/gs/gs.go | 61 +- internal/backend/layout.go | 2 +- internal/{ => backend}/limiter/limiter.go | 0 .../{ => backend}/limiter/limiter_backend.go | 0 .../limiter/limiter_backend_test.go | 6 +- .../{ => backend}/limiter/static_limiter.go | 17 +- .../limiter/static_limiter_test.go | 17 +- internal/backend/local/config.go | 17 +- internal/backend/local/layout_test.go | 5 +- internal/backend/local/local.go | 78 ++- internal/backend/local/local_internal_test.go | 2 +- internal/backend/local/local_test.go | 3 +- internal/backend/local/local_unix.go | 3 +- internal/backend/location/location.go | 2 +- internal/backend/location/location_test.go | 64 +- internal/backend/mem/mem_backend.go | 45 +- internal/{ => backend}/mock/backend.go | 40 +- internal/backend/ontap/ontap.go | 23 +- internal/backend/ontap/ontap_test.go | 3 +- internal/backend/paths.go | 28 +- internal/backend/rclone/backend.go | 5 +- internal/backend/rclone/backend_test.go | 3 +- internal/backend/rclone/internal_test.go | 3 +- internal/{restic => backend}/readerat.go | 11 +- internal/backend/rest/rest.go | 46 +- internal/backend/s3/config.go | 15 +- internal/backend/s3/s3.go | 67 +- internal/backend/s3/s3_test.go | 5 +- internal/backend/sema/semaphore.go | 65 ++ internal/backend/semaphore.go | 69 -- internal/backend/sftp/config.go | 28 +- internal/backend/sftp/config_test.go | 30 +- internal/backend/sftp/layout_test.go | 7 +- internal/backend/sftp/sftp.go | 131 +++- internal/backend/sftp/sftp_test.go | 7 +- internal/backend/swift/config.go | 17 +- internal/backend/swift/swift.go | 39 +- internal/backend/test/doc.go | 39 +- internal/backend/test/tests.go | 14 +- internal/backend/utils.go | 42 ++ internal/backend/utils_test.go | 46 ++ internal/bloblru/cache.go | 2 +- internal/cache/cache.go | 5 - internal/cache/file.go | 4 +- internal/checker/checker.go | 295 +++++--- internal/checker/checker_test.go | 82 +-- internal/checker/testing.go | 5 + internal/{restic => crypto}/buffer.go | 10 +- internal/debug/debug.go | 2 - internal/debug/debug_release.go | 6 - internal/debug/hooks.go | 28 - internal/debug/hooks_release.go | 9 - internal/debug/round_tripper.go | 116 +++ internal/debug/round_tripper_debug.go | 87 +-- internal/debug/round_tripper_release.go | 5 + internal/debug/round_tripper_test.go | 44 ++ internal/dump/common_test.go | 2 +- internal/errors/errors.go | 2 + internal/errors/fatal.go | 2 + internal/errors/fatal_test.go | 22 + internal/filter/filter.go | 31 +- internal/filter/filter_patterns_test.go | 57 ++ internal/fs/const_unix.go | 1 + internal/fs/const_windows.go | 1 + internal/fs/deviceid_unix.go | 1 + internal/fs/deviceid_windows.go | 1 + internal/fs/file_unix.go | 1 + internal/fs/file_windows.go | 44 +- internal/fs/file_windows_test.go | 35 + internal/fs/fs_local_vss.go | 2 +- internal/fs/stat_bsd.go | 1 + internal/fs/stat_unix.go | 1 + internal/fs/stat_windows.go | 1 + internal/fs/vss.go | 1 + internal/fs/vss_windows.go | 1 + internal/fuse/dir.go | 5 +- internal/fuse/file.go | 1 + internal/fuse/fuse_test.go | 2 +- internal/fuse/meta_dir.go | 84 --- internal/fuse/other.go | 1 + internal/fuse/root.go | 35 +- internal/fuse/snapshots_dir.go | 516 ++------------ internal/fuse/snapshots_dirstruct.go | 327 +++++++++ internal/fuse/snapshots_dirstruct_test.go | 291 ++++++++ internal/hashing/reader.go | 45 +- internal/hashing/reader_test.go | 66 +- internal/migrations/interface.go | 2 + internal/migrations/s3_layout.go | 28 +- internal/migrations/upgrade_repo_v2.go | 126 ++++ internal/migrations/upgrade_repo_v2_test.go | 112 +++ internal/options/secret_string.go | 27 + internal/options/secret_string_test.go | 63 ++ internal/pack/pack.go | 191 +++-- internal/pack/pack_internal_test.go | 64 +- internal/pack/pack_test.go | 23 +- internal/repository/doc.go | 7 +- internal/repository/fuzz_test.go | 47 ++ internal/repository/index.go | 154 ++-- internal/repository/index_parallel.go | 16 +- internal/repository/index_test.go | 279 ++++++-- internal/repository/indexmap.go | 46 +- internal/repository/indexmap_test.go | 11 +- internal/repository/key.go | 5 +- internal/repository/master_index.go | 198 ++++-- internal/repository/master_index_test.go | 99 ++- internal/repository/packer_manager.go | 175 +++-- internal/repository/packer_manager_test.go | 114 +-- internal/repository/packer_uploader.go | 63 ++ internal/repository/repack.go | 166 ++--- internal/repository/repack_test.go | 130 +++- internal/repository/repository.go | 663 +++++++++++------- internal/repository/repository_test.go | 491 ++++++++----- ...405e1ab2e1ef5d11d07c8aa4fe6814010294bffd33 | 3 + internal/repository/testing.go | 58 +- internal/repository/worker_group.go | 18 - internal/restic/backend.go | 6 + internal/restic/blob.go | 21 +- internal/restic/config.go | 34 +- internal/restic/config_test.go | 43 +- internal/restic/duration.go | 5 +- internal/restic/duration_test.go | 34 +- internal/restic/find.go | 9 +- internal/restic/find_test.go | 6 +- internal/restic/id.go | 9 +- internal/restic/json.go | 32 + internal/restic/lock.go | 35 +- internal/restic/lock_test.go | 2 +- internal/restic/lock_unix.go | 1 + internal/restic/mknod_unix.go | 1 + internal/restic/node.go | 2 +- internal/restic/node_aix.go | 1 + internal/restic/node_freebsd.go | 1 + internal/restic/node_unix.go | 1 + internal/restic/node_unix_test.go | 1 + internal/restic/node_xattr.go | 38 +- internal/restic/repository.go | 59 +- internal/restic/snapshot.go | 30 +- internal/restic/snapshot_find.go | 14 +- internal/restic/snapshot_find_test.go | 4 +- internal/restic/snapshot_policy.go | 2 +- internal/restic/snapshot_test.go | 26 + .../restic/testdata/policy_keep_snapshots_0 | 40 +- .../restic/testdata/policy_keep_snapshots_18 | 20 +- .../restic/testdata/policy_keep_snapshots_19 | 20 +- .../restic/testdata/policy_keep_snapshots_20 | 20 +- .../restic/testdata/policy_keep_snapshots_26 | 40 +- .../restic/testdata/policy_keep_snapshots_29 | 40 +- .../restic/testdata/policy_keep_snapshots_3 | 40 +- .../restic/testdata/policy_keep_snapshots_4 | 40 +- internal/restic/testing.go | 29 +- internal/restic/testing_test.go | 2 +- internal/restic/tree.go | 95 +++ internal/restic/tree_stream.go | 16 +- internal/restic/tree_test.go | 85 ++- internal/restorer/doc.go | 8 +- internal/restorer/filerestorer.go | 170 ++--- internal/restorer/filerestorer_test.go | 13 +- internal/restorer/fileswriter.go | 2 +- .../{restic => restorer}/hardlinks_index.go | 2 +- .../hardlinks_index_test.go | 6 +- internal/restorer/preallocate_other.go | 1 + internal/restorer/restorer.go | 6 +- internal/restorer/restorer_test.go | 10 +- internal/restorer/restorer_unix_test.go | 3 +- internal/selfupdate/download.go | 48 +- internal/selfupdate/download_unix.go | 10 + internal/selfupdate/download_windows.go | 23 + internal/test/helpers.go | 4 +- internal/ui/backup/json.go | 71 +- internal/ui/backup/progress.go | 17 +- internal/ui/backup/text.go | 13 +- internal/ui/progress/counter_test.go | 2 +- internal/ui/signals/signals_bsd.go | 1 + internal/ui/signals/signals_sysv.go | 1 + internal/ui/termstatus/background.go | 1 + internal/ui/termstatus/status.go | 4 +- internal/ui/termstatus/terminal_unix.go | 5 +- internal/ui/termstatus/terminal_windows.go | 1 + internal/walker/walker.go | 13 +- internal/walker/walker_test.go | 17 +- 330 files changed, 9819 insertions(+), 5370 deletions(-) create mode 100644 .gitattributes create mode 100644 changelog/0.14.0_2022-08-25/issue-1153 create mode 100644 changelog/0.14.0_2022-08-25/issue-1842 create mode 100644 changelog/0.14.0_2022-08-25/issue-21 create mode 100644 changelog/0.14.0_2022-08-25/issue-2162 create mode 100644 changelog/0.14.0_2022-08-25/issue-2248 create mode 100644 changelog/0.14.0_2022-08-25/issue-2291 create mode 100644 changelog/0.14.0_2022-08-25/issue-2295 create mode 100644 changelog/0.14.0_2022-08-25/issue-2696 create mode 100644 changelog/0.14.0_2022-08-25/issue-2907 create mode 100644 changelog/0.14.0_2022-08-25/issue-3114 create mode 100644 changelog/0.14.0_2022-08-25/issue-3295 create mode 100644 changelog/0.14.0_2022-08-25/issue-3428 create mode 100644 changelog/0.14.0_2022-08-25/issue-3432 create mode 100644 changelog/0.14.0_2022-08-25/issue-3465 create mode 100644 changelog/0.14.0_2022-08-25/issue-3685 create mode 100644 changelog/0.14.0_2022-08-25/issue-3692 create mode 100644 changelog/0.14.0_2022-08-25/issue-3709 create mode 100644 changelog/0.14.0_2022-08-25/issue-3720 create mode 100644 changelog/0.14.0_2022-08-25/issue-3736 create mode 100644 changelog/0.14.0_2022-08-25/issue-3837 create mode 100644 changelog/0.14.0_2022-08-25/issue-3861 create mode 100644 changelog/0.14.0_2022-08-25/pull-3419 create mode 100644 changelog/0.14.0_2022-08-25/pull-3475 create mode 100644 changelog/0.14.0_2022-08-25/pull-3484 create mode 100644 changelog/0.14.0_2022-08-25/pull-3513 create mode 100644 changelog/0.14.0_2022-08-25/pull-3680 create mode 100644 changelog/0.14.0_2022-08-25/pull-3716 create mode 100644 changelog/0.14.0_2022-08-25/pull-3729 create mode 100644 changelog/0.14.0_2022-08-25/pull-3742 create mode 100644 changelog/0.14.0_2022-08-25/pull-3772 create mode 100644 changelog/0.14.0_2022-08-25/pull-3773 create mode 100644 changelog/0.14.0_2022-08-25/pull-3776 create mode 100644 changelog/0.14.0_2022-08-25/pull-3819 create mode 100644 cmd/restic/integration_filter_pattern_test.go create mode 100644 doc/047_tuning_backup_parameters.rst rename internal/{ => backend}/limiter/limiter.go (100%) rename internal/{ => backend}/limiter/limiter_backend.go (100%) rename internal/{ => backend}/limiter/limiter_backend_test.go (94%) rename internal/{ => backend}/limiter/static_limiter.go (82%) rename internal/{ => backend}/limiter/static_limiter_test.go (85%) rename internal/{ => backend}/mock/backend.go (72%) rename internal/{restic => backend}/readerat.go (75%) create mode 100644 internal/backend/sema/semaphore.go delete mode 100644 internal/backend/semaphore.go rename internal/{restic => crypto}/buffer.go (69%) delete mode 100644 internal/debug/debug_release.go delete mode 100644 internal/debug/hooks.go delete mode 100644 internal/debug/hooks_release.go create mode 100644 internal/debug/round_tripper.go create mode 100644 internal/debug/round_tripper_test.go create mode 100644 internal/errors/fatal_test.go create mode 100644 internal/filter/filter_patterns_test.go create mode 100644 internal/fs/file_windows_test.go delete mode 100644 internal/fuse/meta_dir.go create mode 100644 internal/fuse/snapshots_dirstruct.go create mode 100644 internal/fuse/snapshots_dirstruct_test.go create mode 100644 internal/migrations/upgrade_repo_v2.go create mode 100644 internal/migrations/upgrade_repo_v2_test.go create mode 100644 internal/options/secret_string.go create mode 100644 internal/options/secret_string_test.go create mode 100644 internal/repository/fuzz_test.go create mode 100644 internal/repository/packer_uploader.go create mode 100644 internal/repository/testdata/fuzz/FuzzSaveLoadBlob/62d79435b9ad1777d0562c405e1ab2e1ef5d11d07c8aa4fe6814010294bffd33 delete mode 100644 internal/repository/worker_group.go create mode 100644 internal/restic/json.go rename internal/{restic => restorer}/hardlinks_index.go (98%) rename internal/{restic => restorer}/hardlinks_index_test.go (85%) create mode 100644 internal/selfupdate/download_unix.go create mode 100644 internal/selfupdate/download_windows.go diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..07a6381055c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Workaround for https://github.com/golang/go/issues/52268. +**/testdata/fuzz/*/* eol=lf diff --git a/.github/workflows/netapp-cicd.yml b/.github/workflows/netapp-cicd.yml index 286a2afa9e7..62a47cb2646 100644 --- a/.github/workflows/netapp-cicd.yml +++ b/.github/workflows/netapp-cicd.yml @@ -166,40 +166,6 @@ jobs: calens if: matrix.check_changelog - lint: - name: lint - runs-on: ubuntu-latest - env: - go: 1.16.x - steps: - - name: Set up Go ${{ env.go }} - uses: actions/setup-go@v2 - with: - go-version: ${{ env.go }} - - - name: Check out code - uses: actions/checkout@v2 - - - name: golangci-lint - uses: golangci/golangci-lint-action@v2 - with: - # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.36 - # Optional: show only new issues if it's a pull request. The default value is `false`. - only-new-issues: true - args: --verbose --timeout 5m - skip-go-installation: true - - # only run golangci-lint for pull requests, otherwise ALL hints get - # reported. We need to slowly address all issues until we can enable - # linting the master branch :) - if: github.event_name == 'pull_request' - - - name: Check go.mod/go.sum - run: | - echo "check if go.mod and go.sum are up to date" - go mod tidy - git diff --exit-code go.mod go.sum buildnpush: name: Build and Push Docker Package diff --git a/.golangci.yml b/.golangci.yml index 55ad1a2668b..ea3a9fd99a2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -16,16 +16,10 @@ linters: # show how code can be simplified - gosimple - # # make sure code is formatted - - gofmt - # examine code and report suspicious constructs, such as Printf calls whose # arguments do not align with the format string - govet - # make sure names and comments are used according to the conventions - - golint - # detect when assignments to existing variables are not used - ineffassign @@ -51,7 +45,7 @@ issues: # list of things to not warn about exclude: - # golint: do not warn about missing comments for exported stuff - - exported (function|method|var|type|const) `.*` should have comment or be unexported - # golint: ignore constants in all caps + # revive: do not warn about missing comments for exported stuff + - exported (function|method|var|type|const) .* should have comment or be unexported + # revive: ignore constants in all caps - don't use ALL_CAPS in Go names; use CamelCase diff --git a/CHANGELOG.md b/CHANGELOG.md index 237fdcca76a..18ab202e698 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,38 +1,452 @@ -Changelog for restic 0.13.1 (2022-04-10) +Changelog for restic 0.14.0 (2022-08-25) ======================================= -The following sections list the changes in restic 0.13.1 relevant to +The following sections list the changes in restic 0.14.0 relevant to restic users. The changes are ordered by importance. Summary ------- - * Fix #3685: Fix the diff command - * Fix #3681: Fix rclone (shimmed by Scoop) and sftp stopped working on Windows + * Fix #2248: Support `self-update` on Windows + * Fix #3428: List snapshots in backend at most once to resolve snapshot IDs + * Fix #3432: Fix rare 'not found in repository' error for `copy` command + * Fix #3685: The `diff` command incorrectly listed some files as added + * Fix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows + * Fix #3720: Directory sync errors for repositories accessed via SMB + * Fix #3736: The `stats` command miscalculated restore size for multiple snapshots + * Fix #3861: Yield error on invalid policy to `forget` + * Fix #3716: Print "wrong password" to stderr instead of stdout + * Fix #3772: Correctly rebuild index for legacy repositories + * Fix #3776: Limit number of key files tested while opening a repository + * Chg #1842: Support debug log creation in release builds + * Chg #3295: Deprecate `check --check-unused` and add further checks + * Chg #3680: Update dependencies and require Go 1.15 or newer + * Chg #3742: Replace `--repo2` option used by `init`/`copy` with `--from-repo` + * Enh #1153: Support pruning even when the disk is full + * Enh #21: Add compression support + * Enh #2162: Adaptive IO concurrency based on backend connections + * Enh #2291: Allow pack size customization + * Enh #2295: Allow use of SAS token to authenticate to Azure + * Enh #2696: Improve backup speed with many small files + * Enh #2907: Make snapshot directory structure of `mount` command customizable + * Enh #3114: Optimize handling of duplicate blobs in `prune` + * Enh #3465: Improve handling of temporary files on Windows + * Enh #3709: Validate exclude patterns before backing up + * Enh #3837: Improve SFTP repository initialization over slow links + * Enh #2351: Use config file permissions to control file group access + * Enh #3475: Allow limiting IO concurrency for local and SFTP backend + * Enh #3484: Stream data in `check` and `prune` commands + * Enh #2923: Improve speed of `copy` command + * Enh #3729: Display full IDs in `check` warnings + * Enh #3773: Optimize memory usage for directories with many files + * Enh #3819: Validate include/exclude patterns before restoring Details ------- - * Bugfix #3685: Fix the diff command + * Bugfix #2248: Support `self-update` on Windows - There was a bug in the `diff` command, it would always show files in a removed directory as added. - We've fixed that. + Restic `self-update` would fail in situations where the operating system locks running + binaries, including Windows. The new behavior works around this by renaming the running file + and swapping the updated file in place. + + https://github.com/restic/restic/issues/2248 + https://github.com/restic/restic/pull/3675 + + * Bugfix #3428: List snapshots in backend at most once to resolve snapshot IDs + + Many commands support specifying a list of snapshot IDs which are then used to determine the + snapshots to be processed by the command. To resolve snapshot IDs or `latest`, and check that + these exist, restic previously listed all snapshots stored in the repository. Depending on + the backend this could be a slow and/or expensive operation. + + Restic now lists the snapshots only once and remembers the result in order to resolve all + further snapshot IDs swiftly. + + https://github.com/restic/restic/issues/3428 + https://github.com/restic/restic/pull/3570 + https://github.com/restic/restic/pull/3395 + + * Bugfix #3432: Fix rare 'not found in repository' error for `copy` command + + In rare cases `copy` (and other commands) would report that `LoadTree(...)` returned an `id + [...] not found in repository` error. This could be caused by a backup or copy command running + concurrently. The error was only temporary; running the failed restic command a second time as + a workaround did resolve the error. + + This issue has now been fixed by correcting the order in which restic reads data from the + repository. It is now guaranteed that restic only loads snapshots for which all necessary data + is already available. + + https://github.com/restic/restic/issues/3432 + https://github.com/restic/restic/pull/3570 + + * Bugfix #3685: The `diff` command incorrectly listed some files as added + + There was a bug in the `diff` command, causing it to always show files in a removed directory as + added. This has now been fixed. https://github.com/restic/restic/issues/3685 https://github.com/restic/restic/pull/3686 - * Bugfix #3681: Fix rclone (shimmed by Scoop) and sftp stopped working on Windows + * Bugfix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows + + In #3602 a fix was introduced to address the problem of `rclone` prematurely exiting when + Ctrl+C is pressed on Windows. The solution was to create the subprocess with its console + detached from the restic console. - In #3602 a fix was introduced to fix the problem that rclone prematurely exits when Ctrl+C is - pressed on Windows. The solution was to create the subprocess with its console detached from - the restic console. However, such solution fails when using rclone install by scoop or using - sftp with a passphrase- protected private key. We've fixed that by using a different approach - to prevent Ctrl-C from passing down too early. + However, this solution failed when using `rclone` installed by Scoop or using `sftp` with a + passphrase-protected private key. We've now fixed this by using a different approach to + prevent Ctrl-C from passing down too early. https://github.com/restic/restic/issues/3681 https://github.com/restic/restic/issues/3692 https://github.com/restic/restic/pull/3696 + * Bugfix #3720: Directory sync errors for repositories accessed via SMB + + On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in restic failing to + save the lock file, yielding the following errors: + + Save() returned error, retrying after 552.330144ms: sync /repo/locks: + no such file or directory Save() returned error, retrying after + 552.330144ms: sync /repo/locks: invalid argument + + This has now been fixed by ignoring the relevant error codes. + + https://github.com/restic/restic/issues/3720 + https://github.com/restic/restic/issues/3751 + https://github.com/restic/restic/pull/3752 + + * Bugfix #3736: The `stats` command miscalculated restore size for multiple snapshots + + Since restic 0.10.0 the restore size calculated by the `stats` command for multiple snapshots + was too low. The hardlink detection was accidentally applied across multiple snapshots and + thus ignored many files. This has now been fixed. + + https://github.com/restic/restic/issues/3736 + https://github.com/restic/restic/pull/3740 + + * Bugfix #3861: Yield error on invalid policy to `forget` + + The `forget` command previously silently ignored invalid/unsupported units in the duration + options, such as e.g. `--keep-within-daily 2w`. + + Specifying an invalid/unsupported duration unit now results in an error. + + https://github.com/restic/restic/issues/3861 + https://github.com/restic/restic/pull/3862 + + * Bugfix #3716: Print "wrong password" to stderr instead of stdout + + If an invalid password was entered, the error message was printed on stdout and not on stderr as + intended. This has now been fixed. + + https://github.com/restic/restic/pull/3716 + https://forum.restic.net/t/4965 + + * Bugfix #3772: Correctly rebuild index for legacy repositories + + After running `rebuild-index` on a legacy repository containing mixed pack files (that is, + pack files which store both metadata and file data), `check` printed warnings like `pack + 12345678 contained in several indexes: ...`. This warning was not critical, but has now + nonetheless been fixed by properly handling mixed pack files while rebuilding the index. + + Running `prune` for such legacy repositories will also fix the warning by reorganizing the + pack files which caused it. + + https://github.com/restic/restic/pull/3772 + https://github.com/restic/restic/pull/3884 + https://forum.restic.net/t/5044/13 + + * Bugfix #3776: Limit number of key files tested while opening a repository + + Previously, restic tested the password against every key in the repository when opening a + repository. The more keys there were in the repository, the slower this operation became. + + Restic now tests the password against up to 20 key files in the repository. Alternatively, you + can use the `--key-hint=` option to specify a specific key file to use instead. + + https://github.com/restic/restic/pull/3776 + + * Change #1842: Support debug log creation in release builds + + Creating a debug log was only possible in debug builds which required users to manually build + restic. We changed the release builds to allow creating debug logs by simply setting the + environment variable `DEBUG_LOG=logname.log`. + + https://github.com/restic/restic/issues/1842 + https://github.com/restic/restic/pull/3826 + + * Change #3295: Deprecate `check --check-unused` and add further checks + + Since restic 0.12.0, it is expected to still have unused blobs after running `prune`. This made + the `--check-unused` option of the `check` command rather useless and tended to confuse + users. This option has been deprecated and is now ignored. + + The `check` command now also warns if a repository is using either the legacy S3 layout or mixed + pack files with both tree and data blobs. The latter is known to cause performance problems. + + https://github.com/restic/restic/issues/3295 + https://github.com/restic/restic/pull/3730 + + * Change #3680: Update dependencies and require Go 1.15 or newer + + We've updated most dependencies. Since some libraries require newer language features we're + dropping support for Go 1.14, which means that restic now requires at least Go 1.15 to build. + + https://github.com/restic/restic/issues/3680 + https://github.com/restic/restic/issues/3883 + + * Change #3742: Replace `--repo2` option used by `init`/`copy` with `--from-repo` + + The `init` and `copy` commands can read data from another repository. However, confusingly + `--repo2` referred to the repository *from* which the `init` command copies parameters, but + for the `copy` command `--repo2` referred to the copy *destination*. + + We've introduced a new option, `--from-repo`, which always refers to the source repository + for both commands. The old parameter names have been deprecated but still work. To create a new + repository and copy all snapshots to it, the commands are now as follows: + + ``` restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo + --copy-chunker-params restic -r /srv/restic-repo-copy copy --from-repo + /srv/restic-repo ``` + + https://github.com/restic/restic/pull/3742 + https://forum.restic.net/t/5017 + + * Enhancement #1153: Support pruning even when the disk is full + + When running out of disk space it was no longer possible to add or remove data from a repository. + To help with recovering from such a deadlock, the prune command now supports an + `--unsafe-recover-no-free-space` option to recover from these situations. Make sure to + read the documentation first! + + https://github.com/restic/restic/issues/1153 + https://github.com/restic/restic/pull/3481 + + * Enhancement #21: Add compression support + + We've added compression support to the restic repository format. To create a repository using + the new format run `init --repository-version 2`. Please note that the repository cannot be + read by restic versions prior to 0.14.0. + + You can configure whether data is compressed with the option `--compression`. It can be set to + `auto` (the default, which will compress very fast), `max` (which will trade backup speed and + CPU usage for better compression), or `off` (which disables compression). Each setting is + only applied for the current run of restic and does *not* apply to future runs. The option can + also be set via the environment variable `RESTIC_COMPRESSION`. + + To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the documentation + for more details. The migration checks the repository integrity and upgrades the repository + format, but will not change any data. Afterwards, prune will rewrite the metadata to make use of + compression. + + As an alternative you can use the `copy` command to migrate snapshots; First create a new + repository using `init --repository-version 2 --copy-chunker-params --repo2 + path/to/old/repo`, and then use the `copy` command to copy all snapshots to the new + repository. + + https://github.com/restic/restic/issues/21 + https://github.com/restic/restic/issues/3779 + https://github.com/restic/restic/pull/3666 + https://github.com/restic/restic/pull/3704 + https://github.com/restic/restic/pull/3733 + + * Enhancement #2162: Adaptive IO concurrency based on backend connections + + Many commands used hard-coded limits for the number of concurrent operations. This prevented + speed improvements by increasing the number of connections used by a backend. + + These limits have now been replaced by using the configured number of backend connections + instead, which can be controlled using the `-o .connections=5` option. + Commands will then automatically scale their parallelism accordingly. + + To limit the number of CPU cores used by restic, you can set the environment variable + `GOMAXPROCS` accordingly. For example to use a single CPU core, use `GOMAXPROCS=1`. + + https://github.com/restic/restic/issues/2162 + https://github.com/restic/restic/issues/1467 + https://github.com/restic/restic/pull/3611 + + * Enhancement #2291: Allow pack size customization + + Restic now uses a target pack size of 16 MiB by default. This can be customized using the + `--pack-size size` option. Supported pack sizes range between 4 and 128 MiB. + + It is possible to migrate an existing repository to _larger_ pack files using `prune + --repack-small`. This will rewrite every pack file which is significantly smaller than the + target size. + + https://github.com/restic/restic/issues/2291 + https://github.com/restic/restic/pull/3731 + + * Enhancement #2295: Allow use of SAS token to authenticate to Azure + + Previously restic only supported AccountKeys to authenticate to Azure storage accounts, + which necessitates giving a significant amount of access. + + We added support for Azure SAS tokens which are a more fine-grained and time-limited manner of + granting access. Set the `AZURE_ACCOUNT_NAME` and `AZURE_ACCOUNT_SAS` environment + variables to use a SAS token for authentication. Note that if `AZURE_ACCOUNT_KEY` is set, it + will take precedence. + + https://github.com/restic/restic/issues/2295 + https://github.com/restic/restic/pull/3661 + + * Enhancement #2696: Improve backup speed with many small files + + We have restructured the backup pipeline to continue reading files while all upload + connections are busy. This allows the backup to already prepare the next data file such that the + upload can continue as soon as a connection becomes available. This can especially improve the + backup performance for high latency backends. + + The upload concurrency is now controlled using the `-o .connections=5` + option. + + https://github.com/restic/restic/issues/2696 + https://github.com/restic/restic/pull/3489 + + * Enhancement #2907: Make snapshot directory structure of `mount` command customizable + + We've added the possibility to customize the snapshot directory structure of the `mount` + command using templates passed to the `--snapshot-template` option. The formatting of + snapshots' timestamps is now controlled using `--time-template` and supports + subdirectories to for example group snapshots by year. Please see `restic help mount` for + further details. + + Characters in tag names which are not allowed in a filename are replaced by underscores `_`. For + example a tag `foo/bar` will result in a directory name of `foo_bar`. + + https://github.com/restic/restic/issues/2907 + https://github.com/restic/restic/pull/2913 + https://github.com/restic/restic/pull/3691 + + * Enhancement #3114: Optimize handling of duplicate blobs in `prune` + + Restic `prune` always used to repack all data files containing duplicate blobs. This + effectively removed all duplicates during prune. However, as a consequence all these data + files were repacked even if the unused repository space threshold could be reached with less + work. + + This is now changed and `prune` works nice and fast even when there are lots of duplicate blobs. + + https://github.com/restic/restic/issues/3114 + https://github.com/restic/restic/pull/3290 + + * Enhancement #3465: Improve handling of temporary files on Windows + + In some cases restic failed to delete temporary files, causing the current command to fail. + This has now been fixed by ensuring that Windows automatically deletes the file. In addition, + temporary files are only written to disk when necessary, reducing disk writes. + + https://github.com/restic/restic/issues/3465 + https://github.com/restic/restic/issues/1551 + https://github.com/restic/restic/pull/3610 + + * Enhancement #3709: Validate exclude patterns before backing up + + Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or + `--iexclude-file` previously weren't validated. As a consequence, invalid patterns + resulted in files that were meant to be excluded being backed up. + + Restic now validates all patterns before running the backup and aborts with a fatal error if an + invalid pattern is detected. + + https://github.com/restic/restic/issues/3709 + https://github.com/restic/restic/pull/3734 + + * Enhancement #3837: Improve SFTP repository initialization over slow links + + The `init` command, when used on an SFTP backend, now sends multiple `mkdir` commands to the + backend concurrently. This reduces the waiting times when creating a repository over a very + slow connection. + + https://github.com/restic/restic/issues/3837 + https://github.com/restic/restic/pull/3840 + + * Enhancement #2351: Use config file permissions to control file group access + + Previously files in a local/SFTP repository would always end up with very restrictive access + permissions, allowing access only to the owner. This prevented a number of valid use-cases + involving groups and ACLs. + + We now use the permissions of the config file in the repository to decide whether group access + should be given to newly created repository files or not. We arrange for repository files to be + created group readable exactly when the repository config file is group readable. + + To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent on the config + file can be used. For repositories that should be writable by group members a tad more setup is + required, see the docs. + + Posix ACLs can also be used now that the group permissions being forced to zero no longer masks + the effect of ACL entries. + + https://github.com/restic/restic/issues/2351 + https://github.com/restic/restic/pull/3419 + https://forum.restic.net/t/1391 + + * Enhancement #3475: Allow limiting IO concurrency for local and SFTP backend + + Restic did not support limiting the IO concurrency / number of connections for accessing + repositories stored using the local or SFTP backends. The number of connections is now limited + as for other backends, and can be configured via the the `-o local.connections=2` and `-o + sftp.connections=5` options. This ensures that restic does not overwhelm the backend with + concurrent IO operations. + + https://github.com/restic/restic/pull/3475 + + * Enhancement #3484: Stream data in `check` and `prune` commands + + The commands `check --read-data` and `prune` previously downloaded data files into + temporary files which could end up being written to disk. This could cause a large amount of data + being written to disk. + + The pack files are now instead streamed, which removes the need for temporary files. Please + note that *uploads* during `backup` and `prune` still require temporary files. + + https://github.com/restic/restic/issues/3710 + https://github.com/restic/restic/pull/3484 + https://github.com/restic/restic/pull/3717 + + * Enhancement #2923: Improve speed of `copy` command + + The `copy` command could require a long time to copy snapshots for non-local backends. This has + been improved to provide a throughput comparable to the `restore` command. + + Additionally, `copy` now displays a progress bar. + + https://github.com/restic/restic/issues/2923 + https://github.com/restic/restic/pull/3513 + + * Enhancement #3729: Display full IDs in `check` warnings + + When running commands to inspect or repair a damaged repository, it is often necessary to + supply the full IDs of objects stored in the repository. + + The output of `check` now includes full IDs instead of their shortened variant. + + https://github.com/restic/restic/pull/3729 + + * Enhancement #3773: Optimize memory usage for directories with many files + + Backing up a directory with hundreds of thousands or more files caused restic to require large + amounts of memory. We've now optimized the `backup` command such that it requires up to 30% less + memory. + + https://github.com/restic/restic/pull/3773 + + * Enhancement #3819: Validate include/exclude patterns before restoring + + Patterns provided to `restore` via `--exclude`, `--iexclude`, `--include` and + `--iinclude` weren't validated before running the restore. Invalid patterns would result in + error messages being printed repeatedly, and possibly unwanted files being restored. + + Restic now validates all patterns before running the restore, and aborts with a fatal error if + an invalid pattern is detected. + + https://github.com/restic/restic/pull/3819 + Changelog for restic 0.13.0 (2022-03-26) ======================================= diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9c8fedf1b2d..cf1f1e7398d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,9 +48,8 @@ environment was used and so on. Please tell us at least the following things: Remember, the easier it is for us to reproduce the bug, the earlier it will be corrected! -In addition, you can compile restic with debug support by running -`go run build.go -tags debug` and instructing it to create a debug -log by setting the environment variable `DEBUG_LOG` to a file, e.g. like this: +In addition, you can instruct restic to create a debug log by setting the +environment variable `DEBUG_LOG` to a file, e.g. like this: $ export DEBUG_LOG=/tmp/restic-debug.log $ restic backup ~/work @@ -66,8 +65,8 @@ Development Environment The repository contains the code written for restic in the directories `cmd/` and `internal/`. -Restic requires Go version 1.14 or later for compiling. Clone the repo (without -having `$GOPATH` set) and `cd` into the directory: +Make sure you have the minimum required Go version installed. Clone the repo +(without having `$GOPATH` set) and `cd` into the directory: $ unset GOPATH $ git clone https://github.com/restic/restic diff --git a/Dockerfile b/Dockerfile index ea9bfb0a044..7c52d9f3542 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.16-alpine as builder +FROM golang:1.18-alpine as builder ARG BUILD_DATETIME WORKDIR /src COPY . . diff --git a/NETAPPVERSION b/NETAPPVERSION index 879b416e609..9f55b2ccb5f 100644 --- a/NETAPPVERSION +++ b/NETAPPVERSION @@ -1 +1 @@ -2.1 +3.0 diff --git a/VERSION b/VERSION index c317a91891f..a803cc227fe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.13.1 +0.14.0 diff --git a/changelog/0.14.0_2022-08-25/issue-1153 b/changelog/0.14.0_2022-08-25/issue-1153 new file mode 100644 index 00000000000..3027588e01e --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-1153 @@ -0,0 +1,9 @@ +Enhancement: Support pruning even when the disk is full + +When running out of disk space it was no longer possible to add or remove +data from a repository. To help with recovering from such a deadlock, the +prune command now supports an `--unsafe-recover-no-free-space` option to +recover from these situations. Make sure to read the documentation first! + +https://github.com/restic/restic/issues/1153 +https://github.com/restic/restic/pull/3481 diff --git a/changelog/0.14.0_2022-08-25/issue-1842 b/changelog/0.14.0_2022-08-25/issue-1842 new file mode 100644 index 00000000000..b5890cad907 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-1842 @@ -0,0 +1,8 @@ +Change: Support debug log creation in release builds + +Creating a debug log was only possible in debug builds which required users to +manually build restic. We changed the release builds to allow creating debug +logs by simply setting the environment variable `DEBUG_LOG=logname.log`. + +https://github.com/restic/restic/issues/1842 +https://github.com/restic/restic/pull/3826 diff --git a/changelog/0.14.0_2022-08-25/issue-21 b/changelog/0.14.0_2022-08-25/issue-21 new file mode 100644 index 00000000000..a105d8ecd62 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-21 @@ -0,0 +1,28 @@ +Enhancement: Add compression support + +We've added compression support to the restic repository format. To create a +repository using the new format run `init --repository-version 2`. Please note +that the repository cannot be read by restic versions prior to 0.14.0. + +You can configure whether data is compressed with the option `--compression`. It +can be set to `auto` (the default, which will compress very fast), `max` (which +will trade backup speed and CPU usage for better compression), or `off` (which +disables compression). Each setting is only applied for the current run of restic +and does *not* apply to future runs. The option can also be set via the +environment variable `RESTIC_COMPRESSION`. + +To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the +documentation for more details. The migration checks the repository integrity +and upgrades the repository format, but will not change any data. Afterwards, +prune will rewrite the metadata to make use of compression. + +As an alternative you can use the `copy` command to migrate snapshots; First +create a new repository using +`init --repository-version 2 --copy-chunker-params --repo2 path/to/old/repo`, +and then use the `copy` command to copy all snapshots to the new repository. + +https://github.com/restic/restic/issues/21 +https://github.com/restic/restic/issues/3779 +https://github.com/restic/restic/pull/3666 +https://github.com/restic/restic/pull/3704 +https://github.com/restic/restic/pull/3733 diff --git a/changelog/0.14.0_2022-08-25/issue-2162 b/changelog/0.14.0_2022-08-25/issue-2162 new file mode 100644 index 00000000000..52104839487 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-2162 @@ -0,0 +1,18 @@ +Enhancement: Adaptive IO concurrency based on backend connections + +Many commands used hard-coded limits for the number of concurrent operations. +This prevented speed improvements by increasing the number of connections used +by a backend. + +These limits have now been replaced by using the configured number of backend +connections instead, which can be controlled using the +`-o .connections=5` option. Commands will then automatically +scale their parallelism accordingly. + +To limit the number of CPU cores used by restic, you can set the environment +variable `GOMAXPROCS` accordingly. For example to use a single CPU core, use +`GOMAXPROCS=1`. + +https://github.com/restic/restic/issues/2162 +https://github.com/restic/restic/issues/1467 +https://github.com/restic/restic/pull/3611 diff --git a/changelog/0.14.0_2022-08-25/issue-2248 b/changelog/0.14.0_2022-08-25/issue-2248 new file mode 100644 index 00000000000..e5e9d8cd96a --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-2248 @@ -0,0 +1,8 @@ +Bugfix: Support `self-update` on Windows + +Restic `self-update` would fail in situations where the operating system +locks running binaries, including Windows. The new behavior works around +this by renaming the running file and swapping the updated file in place. + +https://github.com/restic/restic/issues/2248 +https://github.com/restic/restic/pull/3675 diff --git a/changelog/0.14.0_2022-08-25/issue-2291 b/changelog/0.14.0_2022-08-25/issue-2291 new file mode 100644 index 00000000000..61a32bf63f1 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-2291 @@ -0,0 +1,12 @@ +Enhancement: Allow pack size customization + +Restic now uses a target pack size of 16 MiB by default. This can be customized +using the `--pack-size size` option. Supported pack sizes range between 4 and +128 MiB. + +It is possible to migrate an existing repository to _larger_ pack files using +`prune --repack-small`. This will rewrite every pack file which is +significantly smaller than the target size. + +https://github.com/restic/restic/issues/2291 +https://github.com/restic/restic/pull/3731 diff --git a/changelog/0.14.0_2022-08-25/issue-2295 b/changelog/0.14.0_2022-08-25/issue-2295 new file mode 100644 index 00000000000..7496699ea6e --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-2295 @@ -0,0 +1,14 @@ +Enhancement: Allow use of SAS token to authenticate to Azure + +Previously restic only supported AccountKeys to authenticate to Azure +storage accounts, which necessitates giving a significant amount of +access. + +We added support for Azure SAS tokens which are a more fine-grained +and time-limited manner of granting access. Set the `AZURE_ACCOUNT_NAME` +and `AZURE_ACCOUNT_SAS` environment variables to use a SAS token for +authentication. Note that if `AZURE_ACCOUNT_KEY` is set, it will take +precedence. + +https://github.com/restic/restic/issues/2295 +https://github.com/restic/restic/pull/3661 diff --git a/changelog/0.14.0_2022-08-25/issue-2696 b/changelog/0.14.0_2022-08-25/issue-2696 new file mode 100644 index 00000000000..f913bfd4405 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-2696 @@ -0,0 +1,13 @@ +Enhancement: Improve backup speed with many small files + +We have restructured the backup pipeline to continue reading files while all +upload connections are busy. This allows the backup to already prepare the next +data file such that the upload can continue as soon as a connection becomes +available. This can especially improve the backup performance for high latency +backends. + +The upload concurrency is now controlled using the `-o .connections=5` +option. + +https://github.com/restic/restic/issues/2696 +https://github.com/restic/restic/pull/3489 diff --git a/changelog/0.14.0_2022-08-25/issue-2907 b/changelog/0.14.0_2022-08-25/issue-2907 new file mode 100644 index 00000000000..77de0dfa7ce --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-2907 @@ -0,0 +1,15 @@ +Enhancement: Make snapshot directory structure of `mount` command customizable + +We've added the possibility to customize the snapshot directory structure of +the `mount` command using templates passed to the `--snapshot-template` option. +The formatting of snapshots' timestamps is now controlled using `--time-template` +and supports subdirectories to for example group snapshots by year. Please +see `restic help mount` for further details. + +Characters in tag names which are not allowed in a filename are replaced by +underscores `_`. For example a tag `foo/bar` will result in a directory name +of `foo_bar`. + +https://github.com/restic/restic/issues/2907 +https://github.com/restic/restic/pull/2913 +https://github.com/restic/restic/pull/3691 diff --git a/changelog/0.14.0_2022-08-25/issue-3114 b/changelog/0.14.0_2022-08-25/issue-3114 new file mode 100644 index 00000000000..211a48d22fc --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3114 @@ -0,0 +1,12 @@ +Enhancement: Optimize handling of duplicate blobs in `prune` + +Restic `prune` always used to repack all data files containing duplicate +blobs. This effectively removed all duplicates during prune. However, as a +consequence all these data files were repacked even if the unused repository +space threshold could be reached with less work. + +This is now changed and `prune` works nice and fast even when there are lots +of duplicate blobs. + +https://github.com/restic/restic/issues/3114 +https://github.com/restic/restic/pull/3290 diff --git a/changelog/0.14.0_2022-08-25/issue-3295 b/changelog/0.14.0_2022-08-25/issue-3295 new file mode 100644 index 00000000000..1a6b72724c7 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3295 @@ -0,0 +1,13 @@ +Change: Deprecate `check --check-unused` and add further checks + +Since restic 0.12.0, it is expected to still have unused blobs after running +`prune`. This made the `--check-unused` option of the `check` command rather +useless and tended to confuse users. This option has been deprecated and is +now ignored. + +The `check` command now also warns if a repository is using either the legacy +S3 layout or mixed pack files with both tree and data blobs. The latter is +known to cause performance problems. + +https://github.com/restic/restic/issues/3295 +https://github.com/restic/restic/pull/3730 diff --git a/changelog/0.14.0_2022-08-25/issue-3428 b/changelog/0.14.0_2022-08-25/issue-3428 new file mode 100644 index 00000000000..4510723d501 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3428 @@ -0,0 +1,14 @@ +Bugfix: List snapshots in backend at most once to resolve snapshot IDs + +Many commands support specifying a list of snapshot IDs which are then used to +determine the snapshots to be processed by the command. To resolve snapshot IDs +or `latest`, and check that these exist, restic previously listed all snapshots +stored in the repository. Depending on the backend this could be a slow and/or +expensive operation. + +Restic now lists the snapshots only once and remembers the result in order to +resolve all further snapshot IDs swiftly. + +https://github.com/restic/restic/issues/3428 +https://github.com/restic/restic/pull/3570 +https://github.com/restic/restic/pull/3395 diff --git a/changelog/0.14.0_2022-08-25/issue-3432 b/changelog/0.14.0_2022-08-25/issue-3432 new file mode 100644 index 00000000000..b9f6bcd7a90 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3432 @@ -0,0 +1,14 @@ +Bugfix: Fix rare 'not found in repository' error for `copy` command + +In rare cases `copy` (and other commands) would report that `LoadTree(...)` +returned an `id [...] not found in repository` error. This could be caused by +a backup or copy command running concurrently. The error was only temporary; +running the failed restic command a second time as a workaround did resolve the +error. + +This issue has now been fixed by correcting the order in which restic reads data +from the repository. It is now guaranteed that restic only loads snapshots for +which all necessary data is already available. + +https://github.com/restic/restic/issues/3432 +https://github.com/restic/restic/pull/3570 diff --git a/changelog/0.14.0_2022-08-25/issue-3465 b/changelog/0.14.0_2022-08-25/issue-3465 new file mode 100644 index 00000000000..1f42f395045 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3465 @@ -0,0 +1,10 @@ +Enhancement: Improve handling of temporary files on Windows + +In some cases restic failed to delete temporary files, causing the current +command to fail. This has now been fixed by ensuring that Windows automatically +deletes the file. In addition, temporary files are only written to disk when +necessary, reducing disk writes. + +https://github.com/restic/restic/issues/3465 +https://github.com/restic/restic/issues/1551 +https://github.com/restic/restic/pull/3610 diff --git a/changelog/0.14.0_2022-08-25/issue-3685 b/changelog/0.14.0_2022-08-25/issue-3685 new file mode 100644 index 00000000000..cf5ccf3844b --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3685 @@ -0,0 +1,7 @@ +Bugfix: The `diff` command incorrectly listed some files as added + +There was a bug in the `diff` command, causing it to always show files in a +removed directory as added. This has now been fixed. + +https://github.com/restic/restic/issues/3685 +https://github.com/restic/restic/pull/3686 diff --git a/changelog/0.14.0_2022-08-25/issue-3692 b/changelog/0.14.0_2022-08-25/issue-3692 new file mode 100644 index 00000000000..85c1f753186 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3692 @@ -0,0 +1,13 @@ +Bugfix: Fix rclone (shimmed by Scoop) and sftp not working on Windows + +In #3602 a fix was introduced to address the problem of `rclone` prematurely +exiting when Ctrl+C is pressed on Windows. The solution was to create the +subprocess with its console detached from the restic console. + +However, this solution failed when using `rclone` installed by Scoop or using +`sftp` with a passphrase-protected private key. We've now fixed this by using +a different approach to prevent Ctrl-C from passing down too early. + +https://github.com/restic/restic/issues/3681 +https://github.com/restic/restic/issues/3692 +https://github.com/restic/restic/pull/3696 diff --git a/changelog/0.14.0_2022-08-25/issue-3709 b/changelog/0.14.0_2022-08-25/issue-3709 new file mode 100644 index 00000000000..bdf925af0df --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3709 @@ -0,0 +1,11 @@ +Enhancement: Validate exclude patterns before backing up + +Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or +`--iexclude-file` previously weren't validated. As a consequence, invalid +patterns resulted in files that were meant to be excluded being backed up. + +Restic now validates all patterns before running the backup and aborts with +a fatal error if an invalid pattern is detected. + +https://github.com/restic/restic/issues/3709 +https://github.com/restic/restic/pull/3734 diff --git a/changelog/0.14.0_2022-08-25/issue-3720 b/changelog/0.14.0_2022-08-25/issue-3720 new file mode 100644 index 00000000000..4257e2cb2ef --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3720 @@ -0,0 +1,13 @@ +Bugfix: Directory sync errors for repositories accessed via SMB + +On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in +restic failing to save the lock file, yielding the following errors: + +Save() returned error, retrying after 552.330144ms: sync /repo/locks: no such file or directory +Save() returned error, retrying after 552.330144ms: sync /repo/locks: invalid argument + +This has now been fixed by ignoring the relevant error codes. + +https://github.com/restic/restic/issues/3720 +https://github.com/restic/restic/issues/3751 +https://github.com/restic/restic/pull/3752 diff --git a/changelog/0.14.0_2022-08-25/issue-3736 b/changelog/0.14.0_2022-08-25/issue-3736 new file mode 100644 index 00000000000..45f9fd43566 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3736 @@ -0,0 +1,8 @@ +Bugfix: The `stats` command miscalculated restore size for multiple snapshots + +Since restic 0.10.0 the restore size calculated by the `stats` command for +multiple snapshots was too low. The hardlink detection was accidentally applied +across multiple snapshots and thus ignored many files. This has now been fixed. + +https://github.com/restic/restic/issues/3736 +https://github.com/restic/restic/pull/3740 diff --git a/changelog/0.14.0_2022-08-25/issue-3837 b/changelog/0.14.0_2022-08-25/issue-3837 new file mode 100644 index 00000000000..a4d0e097b51 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3837 @@ -0,0 +1,8 @@ +Enhancement: Improve SFTP repository initialization over slow links + +The `init` command, when used on an SFTP backend, now sends multiple `mkdir` +commands to the backend concurrently. This reduces the waiting times when +creating a repository over a very slow connection. + +https://github.com/restic/restic/issues/3837 +https://github.com/restic/restic/pull/3840 diff --git a/changelog/0.14.0_2022-08-25/issue-3861 b/changelog/0.14.0_2022-08-25/issue-3861 new file mode 100644 index 00000000000..501f6c83b04 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/issue-3861 @@ -0,0 +1,9 @@ +Bugfix: Yield error on invalid policy to `forget` + +The `forget` command previously silently ignored invalid/unsupported +units in the duration options, such as e.g. `--keep-within-daily 2w`. + +Specifying an invalid/unsupported duration unit now results in an error. + +https://github.com/restic/restic/issues/3861 +https://github.com/restic/restic/pull/3862 diff --git a/changelog/0.14.0_2022-08-25/pull-3419 b/changelog/0.14.0_2022-08-25/pull-3419 new file mode 100644 index 00000000000..16accaf286f --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3419 @@ -0,0 +1,21 @@ +Enhancement: Use config file permissions to control file group access + +Previously files in a local/SFTP repository would always end up with very +restrictive access permissions, allowing access only to the owner. This +prevented a number of valid use-cases involving groups and ACLs. + +We now use the permissions of the config file in the repository to decide +whether group access should be given to newly created repository files or +not. We arrange for repository files to be created group readable exactly +when the repository config file is group readable. + +To opt-in to group readable repositories, a simple `chmod -R g+r` or +equivalent on the config file can be used. For repositories that should +be writable by group members a tad more setup is required, see the docs. + +Posix ACLs can also be used now that the group permissions being forced to +zero no longer masks the effect of ACL entries. + +https://github.com/restic/restic/issues/2351 +https://github.com/restic/restic/pull/3419 +https://forum.restic.net/t/1391 diff --git a/changelog/0.14.0_2022-08-25/pull-3475 b/changelog/0.14.0_2022-08-25/pull-3475 new file mode 100644 index 00000000000..9932ae6329d --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3475 @@ -0,0 +1,9 @@ +Enhancement: Allow limiting IO concurrency for local and SFTP backend + +Restic did not support limiting the IO concurrency / number of connections for +accessing repositories stored using the local or SFTP backends. The number of +connections is now limited as for other backends, and can be configured via the +the `-o local.connections=2` and `-o sftp.connections=5` options. This ensures +that restic does not overwhelm the backend with concurrent IO operations. + +https://github.com/restic/restic/pull/3475 diff --git a/changelog/0.14.0_2022-08-25/pull-3484 b/changelog/0.14.0_2022-08-25/pull-3484 new file mode 100644 index 00000000000..3b7a08c45d9 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3484 @@ -0,0 +1,13 @@ +Enhancement: Stream data in `check` and `prune` commands + +The commands `check --read-data` and `prune` previously downloaded data files +into temporary files which could end up being written to disk. This could cause +a large amount of data being written to disk. + +The pack files are now instead streamed, which removes the need for temporary +files. Please note that *uploads* during `backup` and `prune` still require +temporary files. + +https://github.com/restic/restic/pull/3484 +https://github.com/restic/restic/issues/3710 +https://github.com/restic/restic/pull/3717 diff --git a/changelog/0.14.0_2022-08-25/pull-3513 b/changelog/0.14.0_2022-08-25/pull-3513 new file mode 100644 index 00000000000..1314da90b83 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3513 @@ -0,0 +1,10 @@ +Enhancement: Improve speed of `copy` command + +The `copy` command could require a long time to copy snapshots for non-local +backends. This has been improved to provide a throughput comparable to the +`restore` command. + +Additionally, `copy` now displays a progress bar. + +https://github.com/restic/restic/issues/2923 +https://github.com/restic/restic/pull/3513 diff --git a/changelog/0.14.0_2022-08-25/pull-3680 b/changelog/0.14.0_2022-08-25/pull-3680 new file mode 100644 index 00000000000..c86624a2bea --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3680 @@ -0,0 +1,8 @@ +Change: Update dependencies and require Go 1.15 or newer + +We've updated most dependencies. Since some libraries require newer language +features we're dropping support for Go 1.14, which means that restic now +requires at least Go 1.15 to build. + +https://github.com/restic/restic/issues/3680 +https://github.com/restic/restic/issues/3883 diff --git a/changelog/0.14.0_2022-08-25/pull-3716 b/changelog/0.14.0_2022-08-25/pull-3716 new file mode 100644 index 00000000000..af2b50bcd7e --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3716 @@ -0,0 +1,7 @@ +Bugfix: Print "wrong password" to stderr instead of stdout + +If an invalid password was entered, the error message was printed on stdout and +not on stderr as intended. This has now been fixed. + +https://github.com/restic/restic/pull/3716 +https://forum.restic.net/t/4965 diff --git a/changelog/0.14.0_2022-08-25/pull-3729 b/changelog/0.14.0_2022-08-25/pull-3729 new file mode 100644 index 00000000000..da5274a119f --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3729 @@ -0,0 +1,8 @@ +Enhancement: Display full IDs in `check` warnings + +When running commands to inspect or repair a damaged repository, it is often +necessary to supply the full IDs of objects stored in the repository. + +The output of `check` now includes full IDs instead of their shortened variant. + +https://github.com/restic/restic/pull/3729 diff --git a/changelog/0.14.0_2022-08-25/pull-3742 b/changelog/0.14.0_2022-08-25/pull-3742 new file mode 100644 index 00000000000..7956963c7af --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3742 @@ -0,0 +1,19 @@ +Change: Replace `--repo2` option used by `init`/`copy` with `--from-repo` + +The `init` and `copy` commands can read data from another repository. +However, confusingly `--repo2` referred to the repository *from* which the +`init` command copies parameters, but for the `copy` command `--repo2` +referred to the copy *destination*. + +We've introduced a new option, `--from-repo`, which always refers to the +source repository for both commands. The old parameter names have been +deprecated but still work. To create a new repository and copy all snapshots +to it, the commands are now as follows: + +``` +restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo --copy-chunker-params +restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo +``` + +https://github.com/restic/restic/pull/3742 +https://forum.restic.net/t/5017 diff --git a/changelog/0.14.0_2022-08-25/pull-3772 b/changelog/0.14.0_2022-08-25/pull-3772 new file mode 100644 index 00000000000..858832b3f6c --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3772 @@ -0,0 +1,14 @@ +Bugfix: Correctly rebuild index for legacy repositories + +After running `rebuild-index` on a legacy repository containing mixed pack +files (that is, pack files which store both metadata and file data), `check` +printed warnings like `pack 12345678 contained in several indexes: ...`. +This warning was not critical, but has now nonetheless been fixed by properly +handling mixed pack files while rebuilding the index. + +Running `prune` for such legacy repositories will also fix the warning by +reorganizing the pack files which caused it. + +https://github.com/restic/restic/pull/3772 +https://github.com/restic/restic/pull/3884 +https://forum.restic.net/t/5044/13 diff --git a/changelog/0.14.0_2022-08-25/pull-3773 b/changelog/0.14.0_2022-08-25/pull-3773 new file mode 100644 index 00000000000..79fa3816a93 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3773 @@ -0,0 +1,7 @@ +Enhancement: Optimize memory usage for directories with many files + +Backing up a directory with hundreds of thousands or more files caused restic +to require large amounts of memory. We've now optimized the `backup` command +such that it requires up to 30% less memory. + +https://github.com/restic/restic/pull/3773 diff --git a/changelog/0.14.0_2022-08-25/pull-3776 b/changelog/0.14.0_2022-08-25/pull-3776 new file mode 100644 index 00000000000..408a9d3f170 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3776 @@ -0,0 +1,11 @@ +Bugfix: Limit number of key files tested while opening a repository + +Previously, restic tested the password against every key in the repository +when opening a repository. The more keys there were in the repository, the +slower this operation became. + +Restic now tests the password against up to 20 key files in the repository. +Alternatively, you can use the `--key-hint=` option to specify a +specific key file to use instead. + +https://github.com/restic/restic/pull/3776 diff --git a/changelog/0.14.0_2022-08-25/pull-3819 b/changelog/0.14.0_2022-08-25/pull-3819 new file mode 100644 index 00000000000..1e4deb72f59 --- /dev/null +++ b/changelog/0.14.0_2022-08-25/pull-3819 @@ -0,0 +1,11 @@ +Enhancement: Validate include/exclude patterns before restoring + +Patterns provided to `restore` via `--exclude`, `--iexclude`, `--include` +and `--iinclude` weren't validated before running the restore. Invalid +patterns would result in error messages being printed repeatedly, and +possibly unwanted files being restored. + +Restic now validates all patterns before running the restore, and aborts +with a fatal error if an invalid pattern is detected. + +https://github.com/restic/restic/pull/3819 diff --git a/changelog/TEMPLATE b/changelog/TEMPLATE index 0193a4cdf67..d512a2dc318 100644 --- a/changelog/TEMPLATE +++ b/changelog/TEMPLATE @@ -1,5 +1,5 @@ # The first line must start with Bugfix:, Enhancement: or Change:, -# including the colon. Use present use. Remove lines starting with '#' +# including the colon. Use present tense. Remove lines starting with '#' # from this template. Enhancement: Allow custom bar in the foo command diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 873fce16361..0b33f226353 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -12,14 +12,16 @@ import ( "path/filepath" "runtime" "strings" + "sync" "time" "github.com/spf13/cobra" - tomb "gopkg.in/tomb.v2" + "golang.org/x/sync/errgroup" "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -29,7 +31,7 @@ import ( ) var cmdBackup = &cobra.Command{ - Use: "backup [flags] FILE/DIR [FILE/DIR] ...", + Use: "backup [flags] [FILE/DIR] ...", Short: "Create a new backup of files and/or directories", Long: ` The "backup" command creates a new snapshot and saves the files and directories @@ -54,16 +56,22 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea }, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - var t tomb.Tomb + var wg sync.WaitGroup + cancelCtx, cancel := context.WithCancel(globalOptions.ctx) + defer func() { + // shutdown termstatus + cancel() + wg.Wait() + }() + term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) - t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil }) + wg.Add(1) + go func() { + defer wg.Done() + term.Run(cancelCtx) + }() - err := runBackup(backupOptions, globalOptions, term, args) - t.Kill(nil) - if werr := t.Wait(); werr != nil { - panic(fmt.Sprintf("term.Run() returned err: %v", err)) - } - return err + return runBackup(backupOptions, globalOptions, term, args) }, } @@ -103,7 +111,7 @@ func init() { cmdRoot.AddCommand(cmdBackup) f := cmdBackup.Flags() - f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories, and is not newer than the snapshot time)") + f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)") f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`) f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames") @@ -143,7 +151,7 @@ func init() { func filterExisting(items []string) (result []string, err error) { for _, item := range items { _, err := fs.Lstat(item) - if err != nil && os.IsNotExist(errors.Cause(err)) { + if errors.Is(err, os.ErrNotExist) { Warnf("%v does not exist, skipping\n", item) continue } @@ -298,6 +306,11 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t if err != nil { return nil, err } + + if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid { + return nil, errors.Fatalf("--exclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n")) + } + opts.Excludes = append(opts.Excludes, excludes...) } @@ -306,14 +319,27 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t if err != nil { return nil, err } + + if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid { + return nil, errors.Fatalf("--iexclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n")) + } + opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...) } if len(opts.InsensitiveExcludes) > 0 { + if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveExcludes); !valid { + return nil, errors.Fatalf("--iexclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n")) + } + fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes)) } if len(opts.Excludes) > 0 { + if valid, invalidPatterns := filter.ValidatePatterns(opts.Excludes); !valid { + return nil, errors.Fatalf("--exclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n")) + } + fs = append(fs, rejectByPattern(opts.Excludes)) } @@ -475,7 +501,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) { // Force using a parent if !opts.Force && opts.Parent != "" { - id, err := restic.FindSnapshot(ctx, repo, opts.Parent) + id, err := restic.FindSnapshot(ctx, repo.Backend(), opts.Parent) if err != nil { return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err) } @@ -485,7 +511,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup // Find last snapshot to set it as parent, if not already set if !opts.Force && parentID == nil { - id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit) + id, err := restic.FindLatestSnapshot(ctx, repo.Backend(), repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit) if err == nil { parentID = &id } else if err != restic.ErrNoSnapshotFound { @@ -515,8 +541,6 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina } } - var t tomb.Tomb - if gopts.verbosity >= 2 && !gopts.JSON { Verbosef("open repository\n") } @@ -548,7 +572,10 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON)) - t.Go(func() error { return progressReporter.Run(t.Context(gopts.ctx)) }) + wg, wgCtx := errgroup.WithContext(gopts.ctx) + cancelCtx, cancel := context.WithCancel(wgCtx) + defer cancel() + wg.Go(func() error { return progressReporter.Run(cancelCtx) }) if !gopts.JSON { progressPrinter.V("lock repository") @@ -571,14 +598,6 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina return err } - if !gopts.JSON { - progressPrinter.V("load index files") - } - err = repo.LoadIndex(gopts.ctx) - if err != nil { - return err - } - var parentSnapshotID *restic.ID if !opts.Stdin { parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp) @@ -595,6 +614,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina } } + if !gopts.JSON { + progressPrinter.V("load index files") + } + err = repo.LoadIndex(gopts.ctx) + if err != nil { + return err + } + selectByNameFilter := func(item string) bool { for _, reject := range rejectByNameFuncs { if reject(item) { @@ -620,7 +647,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina } errorHandler := func(item string, err error) error { - return progressReporter.Error(item, nil, err) + return progressReporter.Error(item, err) } messageHandler := func(msg string, args ...interface{}) { @@ -656,16 +683,16 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina if !gopts.JSON { progressPrinter.V("start scan on %v", targets) } - t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) }) + wg.Go(func() error { return sc.Scan(cancelCtx, targets) }) arch := archiver.New(repo, targetFS, archiver.Options{}) arch.SelectByName = selectByNameFilter arch.Select = selectFilter arch.WithAtime = opts.WithAtime success := true - arch.Error = func(item string, fi os.FileInfo, err error) error { + arch.Error = func(item string, err error) error { success = false - return progressReporter.Error(item, fi, err) + return progressReporter.Error(item, err) } arch.CompleteItem = progressReporter.CompleteItem arch.StartFile = progressReporter.StartFile @@ -698,10 +725,10 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina _, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts) // cleanly shutdown all running goroutines - t.Kill(nil) + cancel() // let's see if one returned an error - werr := t.Wait() + werr := wg.Wait() // return original error if err != nil { diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 8d31fa25797..991df86a236 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -62,7 +62,7 @@ func runCat(gopts GlobalOptions, args []string) error { } // find snapshot id with prefix - id, err = restic.FindSnapshot(gopts.ctx, repo, args[1]) + id, err = restic.FindSnapshot(gopts.ctx, repo.Backend(), args[1]) if err != nil { return errors.Fatalf("could not find snapshot: %v\n", err) } @@ -79,7 +79,7 @@ func runCat(gopts GlobalOptions, args []string) error { Println(string(buf)) return nil case "index": - buf, err := repo.LoadAndDecrypt(gopts.ctx, nil, restic.IndexFile, id) + buf, err := repo.LoadUnpacked(gopts.ctx, restic.IndexFile, id, nil) if err != nil { return err } @@ -87,13 +87,12 @@ func runCat(gopts GlobalOptions, args []string) error { Println(string(buf)) return nil case "snapshot": - sn := &restic.Snapshot{} - err = repo.LoadJSONUnpacked(gopts.ctx, restic.SnapshotFile, id, sn) + sn, err := restic.LoadSnapshot(gopts.ctx, repo, id) if err != nil { return err } - buf, err := json.MarshalIndent(&sn, "", " ") + buf, err := json.MarshalIndent(sn, "", " ") if err != nil { return err } diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index fa71bdba87a..80b92862db0 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -57,7 +57,13 @@ func init() { f := cmdCheck.Flags() f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs") f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset") - f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs") + var ignored bool + f.BoolVar(&ignored, "check-unused", false, "find unused blobs") + err := f.MarkDeprecated("check-unused", "`--check-unused` is deprecated and will be ignored") + if err != nil { + // MarkDeprecated only returns an error when the flag is not found + panic(err) + } f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache") } @@ -142,10 +148,10 @@ func parsePercentage(s string) (float64, error) { // prepareCheckCache configures a special cache directory for check. // -// * if --with-cache is specified, the default cache is used -// * if the user explicitly requested --no-cache, we don't use any cache -// * if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check -// * by default, we use a cache in a temporary directory that is deleted after the check +// - if --with-cache is specified, the default cache is used +// - if the user explicitly requested --no-cache, we don't use any cache +// - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check +// - by default, we use a cache in a temporary directory that is deleted after the check func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) { cleanup = func() {} if opts.WithCache { @@ -211,21 +217,37 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { } chkr := checker.New(repo, opts.CheckUnused) + err = chkr.LoadSnapshots(gopts.ctx) + if err != nil { + return err + } Verbosef("load indexes\n") hints, errs := chkr.LoadIndex(gopts.ctx) - dupFound := false + errorsFound := false + suggestIndexRebuild := false + mixedFound := false for _, hint := range hints { - Printf("%v\n", hint) - if _, ok := hint.(checker.ErrDuplicatePacks); ok { - dupFound = true + switch hint.(type) { + case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat: + Printf("%v\n", hint) + suggestIndexRebuild = true + case *checker.ErrMixedPack: + Printf("%v\n", hint) + mixedFound = true + default: + Warnf("error: %v\n", hint) + errorsFound = true } } - if dupFound { + if suggestIndexRebuild { Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n") } + if mixedFound { + Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n") + } if len(errs) > 0 { for _, err := range errs { @@ -234,7 +256,6 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { return errors.Fatal("LoadIndex returned errors") } - errorsFound := false orphanedPacks := 0 errChan := make(chan error) @@ -245,14 +266,16 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { if checker.IsOrphanedPack(err) { orphanedPacks++ Verbosef("%v\n", err) - continue + } else if _, ok := err.(*checker.ErrLegacyLayout); ok { + Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n") + } else { + errorsFound = true + Warnf("%v\n", err) } - errorsFound = true - Warnf("%v\n", err) } if orphanedPacks > 0 { - Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nYou can run `restic prune` to correct this.\n", orphanedPacks) + Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks) } Verbosef("check snapshots, trees and blobs\n") @@ -269,7 +292,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { for err := range errChan { errorsFound = true - if e, ok := err.(checker.TreeError); ok { + if e, ok := err.(*checker.TreeError); ok { Warnf("error for tree %v:\n", e.ID.Str()) for _, treeErr := range e.Errors { Warnf(" %v\n", treeErr) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 4d7a192e0da..98007c8d2d9 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -4,7 +4,9 @@ import ( "context" "fmt" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" @@ -48,17 +50,21 @@ func init() { cmdRoot.AddCommand(cmdCopy) f := cmdCopy.Flags() - initSecondaryRepoOptions(f, ©Options.secondaryRepoOptions, "destination", "to copy snapshots to") + initSecondaryRepoOptions(f, ©Options.secondaryRepoOptions, "destination", "to copy snapshots from") f.StringArrayVarP(©Options.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)") f.Var(©Options.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given") f.StringArrayVar(©Options.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given") } func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { - dstGopts, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination") + secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination") if err != nil { return err } + if isFromRepo { + // swap global options, if the secondary repo was set via from-repo + gopts, secondaryGopts = secondaryGopts, gopts + } ctx, cancel := context.WithCancel(gopts.ctx) defer cancel() @@ -68,7 +74,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { return err } - dstRepo, err := OpenRepository(dstGopts) + dstRepo, err := OpenRepository(secondaryGopts) if err != nil { return err } @@ -87,6 +93,16 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { return err } + srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile) + if err != nil { + return err + } + + dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile) + if err != nil { + return err + } + debug.Log("Loading source index") if err := srcRepo.LoadIndex(ctx); err != nil { return err @@ -98,7 +114,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { } dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot) - for sn := range FindFilteredSnapshots(ctx, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) { + for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) { if sn.Original != nil && !sn.Original.IsNull() { dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn) } @@ -109,7 +125,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { // remember already processed trees across all snapshots visitedTrees := restic.NewIDSet() - for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) { Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) // check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields @@ -131,24 +147,18 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { } } Verbosef(" copy started, this may take a while...\n") - - if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree); err != nil { + if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil { return err } debug.Log("tree copied") - if err = dstRepo.Flush(ctx); err != nil { - return err - } - debug.Log("flushed packs and saved index") - // save snapshot sn.Parent = nil // Parent does not have relevance in the new repo. // Use Original as a persistent snapshot ID if sn.Original == nil { sn.Original = sn.ID() } - newID, err := dstRepo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn) + newID, err := restic.SaveSnapshot(ctx, dstRepo, sn) if err != nil { return err } @@ -176,82 +186,61 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool { return true } -const numCopyWorkers = 8 - func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository, - visitedTrees restic.IDSet, rootTreeID restic.ID) error { + visitedTrees restic.IDSet, rootTreeID restic.ID, quiet bool) error { - idChan := make(chan restic.ID) - wg, ctx := errgroup.WithContext(ctx) + wg, wgCtx := errgroup.WithContext(ctx) - treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool { + treeStream := restic.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool { visited := visitedTrees.Has(treeID) visitedTrees.Insert(treeID) return visited }, nil) + copyBlobs := restic.NewBlobSet() + packList := restic.NewIDSet() + + enqueue := func(h restic.BlobHandle) { + pb := srcRepo.Index().Lookup(h) + copyBlobs.Insert(h) + for _, p := range pb { + packList.Insert(p.PackID) + } + } + wg.Go(func() error { - defer close(idChan) - // reused buffer - var buf []byte for tree := range treeStream { if tree.Error != nil { return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error) } // Do we already have this tree blob? - if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) { + treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob} + if !dstRepo.Index().Has(treeHandle) { // copy raw tree bytes to avoid problems if the serialization changes - var err error - buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf) - if err != nil { - return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err) - } - - _, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false) - if err != nil { - return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err) - } + enqueue(treeHandle) } for _, entry := range tree.Nodes { // Recursion into directories is handled by StreamTrees // Copy the blobs for this file. for _, blobID := range entry.Content { - select { - case idChan <- blobID: - case <-ctx.Done(): - return ctx.Err() + h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID} + if !dstRepo.Index().Has(h) { + enqueue(h) } } } } return nil }) - - for i := 0; i < numCopyWorkers; i++ { - wg.Go(func() error { - // reused buffer - var buf []byte - for blobID := range idChan { - // Do we already have this data blob? - if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) { - continue - } - debug.Log("Copying blob %s\n", blobID.Str()) - var err error - buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf) - if err != nil { - return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err) - } - - _, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false) - if err != nil { - return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err) - } - } - return nil - }) + err := wg.Wait() + if err != nil { + return err } - return wg.Wait() + + bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied") + _, err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar) + bar.Done() + return err } diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 42433d5b92a..ac4996b7cc8 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -1,3 +1,4 @@ +//go:build debug // +build debug package main @@ -14,6 +15,7 @@ import ( "sort" "time" + "github.com/klauspost/compress/zstd" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" @@ -51,12 +53,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er var tryRepair bool var repairByte bool var extractPack bool +var reuploadBlobs bool func init() { cmdRoot.AddCommand(cmdDebug) cmdDebug.AddCommand(cmdDebugDump) cmdDebug.AddCommand(cmdDebugExamine) cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory") + cmdDebugExamine.Flags().BoolVar(&reuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository") cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes") } @@ -72,7 +76,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error { } func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error { - return restic.ForAllSnapshots(ctx, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error { + return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error { if err != nil { return err } @@ -103,7 +107,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { h := restic.Handle{Type: restic.PackFile, Name: id.String()} - blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size) + blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), size) if err != nil { Warnf("error for pack %v: %v\n", id.Str(), err) return nil @@ -308,6 +312,10 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte { } func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error { + dec, err := zstd.NewReader(nil) + if err != nil { + panic(err) + } be := repo.Backend() h := restic.Handle{ Name: pack.String(), @@ -332,48 +340,65 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil) + outputPrefix := "" + filePrefix := "" if err != nil { Warnf("error decrypting blob: %v\n", err) - var plain []byte if tryRepair || repairByte { - plain = tryRepairWithBitflip(ctx, key, buf, repairByte) + plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte) } - var prefix string - if plain != nil { - id := restic.Hash(plain) - if !id.Equal(blob.ID) { - Printf(" repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID) - prefix = "repaired-wrong-hash-" - } else { - Printf(" successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id) - prefix = "repaired-" - } + if plaintext != nil { + outputPrefix = "repaired " + filePrefix = "repaired-" } else { - plain = decryptUnsigned(ctx, key, buf) - prefix = "damaged-" + plaintext = decryptUnsigned(ctx, key, buf) + err = storePlainBlob(blob.ID, "damaged-", plaintext) + if err != nil { + return err + } + continue } - err = storePlainBlob(blob.ID, prefix, plain) + } + + if blob.IsCompressed() { + decompressed, err := dec.DecodeAll(plaintext, nil) if err != nil { - return err + Printf(" failed to decompress blob %v\n", blob.ID) + } + if decompressed != nil { + plaintext = decompressed } - continue } id := restic.Hash(plaintext) var prefix string if !id.Equal(blob.ID) { - Printf(" successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID) + Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID) prefix = "wrong-hash-" } else { - Printf(" successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id) + Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id) prefix = "correct-" } if extractPack { - err = storePlainBlob(id, prefix, plaintext) + err = storePlainBlob(id, filePrefix+prefix, plaintext) if err != nil { return err } } + if reuploadBlobs { + _, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true) + if err != nil { + return err + } + Printf(" uploaded %v %v\n", blob.Type, id) + } + } + + if reuploadBlobs { + err := repo.Flush(ctx) + if err != nil { + return err + } } return nil @@ -402,12 +427,23 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error { } func runDebugExamine(gopts GlobalOptions, args []string) error { + repo, err := OpenRepository(gopts) + if err != nil { + return err + } + ids := make([]restic.ID, 0) for _, name := range args { id, err := restic.ParseID(name) if err != nil { - Warnf("error: %v\n", err) - continue + name, err = restic.Find(gopts.ctx, repo.Backend(), restic.PackFile, name) + if err == nil { + id, err = restic.ParseID(name) + } + if err != nil { + Warnf("error: %v\n", err) + continue + } } ids = append(ids, id) } @@ -416,11 +452,6 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { return errors.Fatal("no pack files to examine") } - repo, err := OpenRepository(gopts) - if err != nil { - return err - } - if !gopts.NoLock { lock, err := lockRepo(gopts.ctx, repo) defer unlockRepo(lock) @@ -475,27 +506,15 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro blobsLoaded := false // examine all data the indexes have for the pack file - for _, idx := range repo.Index().(*repository.MasterIndex).All() { - idxIDs, err := idx.IDs() - if err != nil { - idxIDs = restic.IDs{} - } - - blobs := idx.ListPack(id) + for b := range repo.Index().ListPacks(ctx, restic.NewIDSet(id)) { + blobs := b.Blobs if len(blobs) == 0 { continue } - Printf(" index %v:\n", idxIDs) + checkPackSize(blobs, fi.Size) - // convert list of blobs to []restic.Blob - var list []restic.Blob - for _, b := range blobs { - list = append(list, b.Blob) - } - checkPackSize(list, fi.Size) - - err = loadBlobs(ctx, repo, id, list) + err = loadBlobs(ctx, repo, id, blobs) if err != nil { Warnf("error: %v\n", err) } else { @@ -506,7 +525,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro Printf(" ========================================\n") Printf(" inspect the pack itself\n") - blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size) + blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), fi.Size) if err != nil { return fmt.Errorf("pack %v: %v", id.Str(), err) } @@ -531,14 +550,10 @@ func checkPackSize(blobs []restic.Blob, fileSize int64) { if offset != uint64(pb.Offset) { Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset) } - offset += uint64(pb.Length) + offset = uint64(pb.Offset + pb.Length) size += uint64(pb.Length) } - - // compute header size, per blob: 1 byte type, 4 byte length, 32 byte id - size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32))) - // length in uint32 little endian - size += 4 + size += uint64(pack.CalculateHeaderSize(blobs)) if uint64(fileSize) != size { Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fileSize) diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index f83c87132e6..5fdd28d9783 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -7,9 +7,9 @@ import ( "reflect" "sort" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/spf13/cobra" ) @@ -53,8 +53,8 @@ func init() { f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") } -func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) (*restic.Snapshot, error) { - id, err := restic.FindSnapshot(ctx, repo, desc) +func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) { + id, err := restic.FindSnapshot(ctx, be, desc) if err != nil { return nil, errors.Fatal(err.Error()) } @@ -160,7 +160,7 @@ func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error { debug.Log("print %v tree %v", mode, id) - tree, err := c.repo.LoadTree(ctx, id) + tree, err := restic.LoadTree(ctx, c.repo, id) if err != nil { return err } @@ -187,7 +187,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error { debug.Log("print tree %v", id) - tree, err := c.repo.LoadTree(ctx, id) + tree, err := restic.LoadTree(ctx, c.repo, id) if err != nil { return err } @@ -231,12 +231,12 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error { debug.Log("diffing %v to %v", id1, id2) - tree1, err := c.repo.LoadTree(ctx, id1) + tree1, err := restic.LoadTree(ctx, c.repo, id1) if err != nil { return err } - tree2, err := c.repo.LoadTree(ctx, id2) + tree2, err := restic.LoadTree(ctx, c.repo, id2) if err != nil { return err } @@ -334,10 +334,6 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error { return err } - if err = repo.LoadIndex(ctx); err != nil { - return err - } - if !gopts.NoLock { lock, err := lockRepo(ctx, repo) defer unlockRepo(lock) @@ -346,12 +342,17 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error { } } - sn1, err := loadSnapshot(ctx, repo, args[0]) + // cache snapshots listing + be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + if err != nil { + return err + } + sn1, err := loadSnapshot(ctx, be, repo, args[0]) if err != nil { return err } - sn2, err := loadSnapshot(ctx, repo, args[1]) + sn2, err := loadSnapshot(ctx, be, repo, args[1]) if err != nil { return err } @@ -360,6 +361,10 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error { Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str()) } + if err = repo.LoadIndex(ctx); err != nil { + return err + } + if sn1.Tree == nil { return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str()) } diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 4449af5cee1..993072f9c05 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -87,7 +87,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor case l == 1 && dump.IsFile(node): return d.WriteNode(ctx, node) case l > 1 && dump.IsDir(node): - subtree, err := repo.LoadTree(ctx, *node.Subtree) + subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) if err != nil { return errors.Wrapf(err, "cannot load subtree for %q", item) } @@ -96,7 +96,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor if err := checkStdoutArchive(); err != nil { return err } - subtree, err := repo.LoadTree(ctx, *node.Subtree) + subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) if err != nil { return err } @@ -144,20 +144,15 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error { } } - err = repo.LoadIndex(ctx) - if err != nil { - return err - } - var id restic.ID if snapshotIDString == "latest" { - id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil) + id, err = restic.FindLatestSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil) if err != nil { Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts) } } else { - id, err = restic.FindSnapshot(ctx, repo, snapshotIDString) + id, err = restic.FindSnapshot(ctx, repo.Backend(), snapshotIDString) if err != nil { Exitf(1, "invalid id %q: %v", snapshotIDString, err) } @@ -168,7 +163,12 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error { Exitf(2, "loading snapshot %q failed: %v", snapshotIDString, err) } - tree, err := repo.LoadTree(ctx, *sn.Tree) + err = repo.LoadIndex(ctx) + if err != nil { + return err + } + + tree, err := restic.LoadTree(ctx, repo, *sn.Tree) if err != nil { Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err) } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 033915ab0f7..7171314e29a 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/filter" @@ -584,6 +585,11 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { } } + snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) + if err != nil { + return err + } + if err = repo.LoadIndex(gopts.ctx); err != nil { return err } @@ -618,7 +624,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { } } - for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) { if f.blobIDs != nil || f.treeIDs != nil { if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" { return err diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index ecff71e1e49..cbae16d70c3 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -14,11 +14,16 @@ var cmdForget = &cobra.Command{ Use: "forget [flags] [snapshot ID] [...]", Short: "Remove snapshots from the repository", Long: ` -The "forget" command removes snapshots according to a policy. Please note that -this command really only deletes the snapshot object in the repository, which -is a reference to data stored there. In order to remove the unreferenced data -after "forget" was run successfully, see the "prune" command. Please also read -the documentation for "forget" to learn about important security considerations. +The "forget" command removes snapshots according to a policy. All snapshots are +first divided into groups according to "--group-by", and after that the policy +specified by the "--keep-*" options is applied to each group individually. + +Please note that this command really only deletes the snapshot object in the +repository, which is a reference to data stored there. In order to remove the +unreferenced data after "forget" was run successfully, see the "prune" command. + +Please also read the documentation for "forget" to learn about some important +security considerations. EXIT STATUS =========== @@ -94,7 +99,7 @@ func init() { f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)") f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format") - f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "string for grouping snapshots by host,paths,tags") + f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done") f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed") f.BoolVar(&forgetOptions.deleteEmptyRepo, "delete-empty-repo", false, "delete the repo if there are no more snapshots") @@ -132,7 +137,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { var snapshots restic.Snapshots removeSnIDs := restic.NewIDSet() - for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { snapshots = append(snapshots, sn) } @@ -254,7 +259,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { if opts.deleteEmptyRepo { snapshotCount := 0 - for range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, []string{}) { + for range FindFilteredSnapshots(ctx, repo.Backend(),repo, opts.Hosts, opts.Tags, opts.Paths, []string{}) { snapshotCount++ } diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index d365d43e71b..220c254300e 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -1,10 +1,13 @@ package main import ( + "strconv" + "github.com/restic/chunker" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" "github.com/spf13/cobra" ) @@ -30,6 +33,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er type InitOptions struct { secondaryRepoOptions CopyChunkerParameters bool + RepositoryVersion string } var initOptions InitOptions @@ -40,9 +44,26 @@ func init() { f := cmdInit.Flags() initSecondaryRepoOptions(f, &initOptions.secondaryRepoOptions, "secondary", "to copy chunker parameters from") f.BoolVar(&initOptions.CopyChunkerParameters, "copy-chunker-params", false, "copy chunker parameters from the secondary repository (useful with the copy command)") + f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'") } func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { + var version uint + if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" { + version = restic.MaxRepoVersion + } else if opts.RepositoryVersion == "stable" { + version = restic.StableRepoVersion + } else { + v, err := strconv.ParseUint(opts.RepositoryVersion, 10, 32) + if err != nil { + return errors.Fatal("invalid repository version") + } + version = uint(v) + } + if version < restic.MinRepoVersion || version > restic.MaxRepoVersion { + return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion) + } + chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts) if err != nil { return err @@ -65,9 +86,15 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) } - s := repository.New(be) + s, err := repository.New(be, repository.Options{ + Compression: gopts.Compression, + PackSize: gopts.PackSize * 1024 * 1024, + }) + if err != nil { + return err + } - wasCreated, err := s.Init(gopts.ctx, gopts.password, chunkerPolynomial) + wasCreated,err:= s.Init(gopts.ctx, version, gopts.password, chunkerPolynomial) if err != nil { return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) } @@ -87,7 +114,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) { if opts.CopyChunkerParameters { - otherGopts, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary") + otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary") if err != nil { return nil, err } @@ -101,7 +128,7 @@ func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker return &pol, nil } - if opts.Repo != "" { + if opts.Repo != "" || opts.RepositoryFile != "" || opts.LegacyRepo != "" || opts.LegacyRepositoryFile != "" { return nil, errors.Fatal("Secondary repository must only be specified when copying the chunker parameters") } return nil, nil diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 71a44949e62..ec6695714a0 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -169,6 +170,11 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { return err } + snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) + if err != nil { + return err + } + if err = repo.LoadIndex(gopts.ctx); err != nil { return err } @@ -211,7 +217,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { } } - for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args[:1]) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args[:1]) { printSnapshot(sn) err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go index 4af98005e5a..c8f0e947841 100644 --- a/cmd/restic/cmd_migrate.go +++ b/cmd/restic/cmd_migrate.go @@ -8,11 +8,12 @@ import ( ) var cmdMigrate = &cobra.Command{ - Use: "migrate [flags] [name]", + Use: "migrate [flags] [migration name] [...]", Short: "Apply migrations", Long: ` -The "migrate" command applies migrations to a repository. When no migration -name is explicitly given, a list of migrations that can be applied is printed. +The "migrate" command checks which migrations can be applied for a repository +and prints a list with available migration names. If one or more migration +names are specified, these migrations are applied. EXIT STATUS =========== @@ -41,6 +42,8 @@ func init() { func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error { ctx := gopts.ctx Printf("available migrations:\n") + found := false + for _, m := range migrations.All { ok, err := m.Check(ctx, repo) if err != nil { @@ -48,10 +51,15 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos } if ok { - Printf(" %v: %v\n", m.Name(), m.Desc()) + Printf(" %v\t%v\n", m.Name(), m.Desc()) + found = true } } + if !found { + Printf("no migrations found") + } + return nil } @@ -76,6 +84,19 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos Warnf("check for migration %v failed, continuing anyway\n", m.Name()) } + if m.RepoCheck() { + Printf("checking repository integrity...\n") + + checkOptions := CheckOptions{} + checkGopts := gopts + // the repository is already locked + checkGopts.NoLock = true + err = runCheck(checkOptions, checkGopts, []string{}) + if err != nil { + return err + } + } + Printf("applying migration %v...\n", m.Name()) if err = m.Apply(ctx, repo); err != nil { Warnf("migration %v failed: %v\n", m.Name(), err) diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 9cebb1b05cc..747316f9f31 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -1,3 +1,4 @@ +//go:build darwin || freebsd || linux // +build darwin freebsd linux package main @@ -30,10 +31,13 @@ read-only mount. Snapshot Directories ==================== -If you need a different template for all directories that contain snapshots, -you can pass a template via --snapshot-template. Example without colons: +If you need a different template for directories that contain snapshots, +you can pass a time template via --time-template and path templates via +--path-template. - --snapshot-template "2006-01-02_15-04-05" +Example time template without colons: + + --time-template "2006-01-02_15-04-05" You need to specify a sample format for exactly the following timestamp: @@ -42,6 +46,20 @@ You need to specify a sample format for exactly the following timestamp: For details please see the documentation for time.Format() at: https://godoc.org/time#Time.Format +For path templates, you can use the following patterns which will be replaced: + %i by short snapshot ID + %I by long snapshot ID + %u by username + %h by hostname + %t by tags + %T by timestamp as specified by --time-template + +The default path templates are: + "ids/%i" + "snapshots/%T" + "hosts/%h/%T" + "tags/%t/%T" + EXIT STATUS =========== @@ -61,7 +79,8 @@ type MountOptions struct { Hosts []string Tags restic.TagLists Paths []string - SnapshotTemplate string + TimeTemplate string + PathTemplates []string } var mountOptions MountOptions @@ -78,16 +97,21 @@ func init() { mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`") mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`") - mountFlags.StringVar(&mountOptions.SnapshotTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs") + mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)") + mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs") + mountFlags.StringVar(&mountOptions.TimeTemplate, "time-template", time.RFC3339, "set `template` to use for times") + _ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template") } func runMount(opts MountOptions, gopts GlobalOptions, args []string) error { - if opts.SnapshotTemplate == "" { - return errors.Fatal("snapshot template string cannot be empty") + if opts.TimeTemplate == "" { + return errors.Fatal("time template string cannot be empty") } - if strings.ContainsAny(opts.SnapshotTemplate, `\/`) { - return errors.Fatal("snapshot template string contains a slash (/) or backslash (\\) character") + + if strings.HasPrefix(opts.TimeTemplate, "/") || strings.HasSuffix(opts.TimeTemplate, "/") { + return errors.Fatal("time template string cannot start or end with '/'") } + if len(args) == 0 { return errors.Fatal("wrong number of parameters") } @@ -115,7 +139,7 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error { mountpoint := args[0] - if _, err := resticfs.Stat(mountpoint); os.IsNotExist(errors.Cause(err)) { + if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) { Verbosef("Mountpoint %s doesn't exist\n", mountpoint) return err } @@ -153,11 +177,12 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error { } cfg := fuse.Config{ - OwnerIsRoot: opts.OwnerRoot, - Hosts: opts.Hosts, - Tags: opts.Tags, - Paths: opts.Paths, - SnapshotTemplate: opts.SnapshotTemplate, + OwnerIsRoot: opts.OwnerRoot, + Hosts: opts.Hosts, + Tags: opts.Tags, + Paths: opts.Paths, + TimeTemplate: opts.TimeTemplate, + PathTemplates: opts.PathTemplates, } root := fuse.NewRoot(repo, cfg) diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 253e3704a14..7421bc0ccab 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -1,6 +1,7 @@ package main import ( + "context" "math" "sort" "strconv" @@ -8,6 +9,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -38,7 +40,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // PruneOptions collects all options for the cleanup command. type PruneOptions struct { - DryRun bool + DryRun bool + UnsafeNoSpaceRecovery string + + unsafeRecovery bool MaxUnused string maxUnusedBytes func(used uint64) (unused uint64) // calculates the number of unused bytes after repacking, according to MaxUnused @@ -47,6 +52,8 @@ type PruneOptions struct { MaxRepackBytes uint64 RepackCachableOnly bool + RepackSmall bool + RepackUncompressed bool } var pruneOptions PruneOptions @@ -55,6 +62,7 @@ func init() { cmdRoot.AddCommand(cmdPrune) f := cmdPrune.Flags() f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done") + f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.") addPruneOptions(cmdPrune) } @@ -63,6 +71,8 @@ func addPruneOptions(c *cobra.Command) { f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')") f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)") f.BoolVar(&pruneOptions.RepackCachableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable") + f.BoolVar(&pruneOptions.RepackSmall, "repack-small", false, "repack pack files below 80% of target pack size") + f.BoolVar(&pruneOptions.RepackUncompressed, "repack-uncompressed", false, "repack all uncompressed data") } func verifyPruneOptions(opts *PruneOptions) error { @@ -74,6 +84,10 @@ func verifyPruneOptions(opts *PruneOptions) error { } opts.MaxRepackBytes = uint64(size) } + if opts.UnsafeNoSpaceRecovery != "" { + // prevent repacking data to make sure users cannot get stuck. + opts.MaxRepackBytes = 0 + } maxUnused := strings.TrimSpace(opts.MaxUnused) if maxUnused == "" { @@ -126,11 +140,31 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error { return err } + if opts.RepackUncompressed && gopts.Compression == repository.CompressionOff { + return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive") + } + repo, err := OpenRepository(gopts) if err != nil { return err } + if repo.Backend().Connections() < 2 { + return errors.Fatal("prune requires a backend connection limit of at least two") + } + + if repo.Config().Version < 2 && opts.RepackUncompressed { + return errors.Fatal("compression requires at least repository format version 2") + } + + if opts.UnsafeNoSpaceRecovery != "" { + repoID := repo.Config().ID + if opts.UnsafeNoSpaceRecovery != repoID { + return errors.Fatalf("must pass id '%s' to --unsafe-recover-no-free-space", repoID) + } + opts.unsafeRecovery = true + } + lock, err := lockRepoExclusive(gopts.ctx, repo) defer unlockRepo(lock) if err != nil { @@ -149,26 +183,69 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R } Verbosef("loading indexes...\n") + // loading the index before the snapshots is ok, as we use an exclusive lock here err := repo.LoadIndex(gopts.ctx) if err != nil { return err } - usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots) + plan, stats, err := planPrune(opts, gopts, repo, ignoreSnapshots) if err != nil { return err } - return prune(opts, gopts, repo, usedBlobs) + err = printPruneStats(gopts, stats) + if err != nil { + return err + } + + return doPrune(opts, gopts, repo, plan) +} + +type pruneStats struct { + blobs struct { + used uint + duplicate uint + unused uint + remove uint + repack uint + repackrm uint + } + size struct { + used uint64 + duplicate uint64 + unused uint64 + remove uint64 + repack uint64 + repackrm uint64 + unref uint64 + } + packs struct { + used uint + unused uint + partlyUsed uint + unref uint + keep uint + repack uint + remove uint + } +} + +type prunePlan struct { + removePacksFirst restic.IDSet // packs to remove first (unreferenced packs) + repackPacks restic.IDSet // packs to repack + keepBlobs restic.BlobSet // blobs to keep during repacking + removePacks restic.IDSet // packs to remove + ignorePacks restic.IDSet // packs to ignore when rebuilding the index } type packInfo struct { - usedBlobs uint - unusedBlobs uint - duplicateBlobs uint - usedSize uint64 - unusedSize uint64 - tpe restic.BlobType + usedBlobs uint + unusedBlobs uint + usedSize uint64 + unusedSize uint64 + tpe restic.BlobType + uncompressed bool } type packInfoWithID struct { @@ -176,44 +253,53 @@ type packInfoWithID struct { packInfo } -// prune selects which files to rewrite and then does that. The map usedBlobs is -// modified in the process. -func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedBlobs restic.BlobSet) error { +// planPrune selects which files to rewrite and which to delete and which blobs to keep. +// Also some summary statistics are returned. +func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) { ctx := gopts.ctx + var stats pruneStats - var stats struct { - blobs struct { - used uint - duplicate uint - unused uint - remove uint - repack uint - repackrm uint - } - size struct { - used uint64 - duplicate uint64 - unused uint64 - remove uint64 - repack uint64 - repackrm uint64 - unref uint64 - } - packs struct { - used uint - unused uint - partlyUsed uint - keep uint - } + usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots) + if err != nil { + return prunePlan{}, stats, err } Verbosef("searching used packs...\n") + keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats) + if err != nil { + return prunePlan{}, stats, err + } + + Verbosef("collecting packs for deletion and repacking\n") + plan, err := decidePackAction(ctx, opts, gopts, repo, indexPack, &stats) + if err != nil { + return prunePlan{}, stats, err + } + if len(plan.repackPacks) != 0 { + // when repacking, we do not want to keep blobs which are + // already contained in kept packs, so delete them from keepBlobs + for blob := range repo.Index().Each(ctx) { + if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) { + continue + } + keepBlobs.Delete(blob.BlobHandle) + } + } else { + // keepBlobs is only needed if packs are repacked + keepBlobs = nil + } + plan.keepBlobs = keepBlobs + + return plan, stats, nil +} + +func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.BlobSet, stats *pruneStats) (restic.BlobSet, map[restic.ID]packInfo, error) { keepBlobs := restic.NewBlobSet() - duplicateBlobs := restic.NewBlobSet() + duplicateBlobs := make(map[restic.BlobHandle]uint8) // iterate over all blobs in index to find out which blobs are duplicates - for blob := range repo.Index().Each(ctx) { + for blob := range idx.Each(ctx) { bh := blob.BlobHandle size := uint64(blob.Length) switch { @@ -223,7 +309,16 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB stats.size.used += size stats.blobs.used++ case keepBlobs.Has(bh): // duplicate blob - duplicateBlobs.Insert(bh) + count, ok := duplicateBlobs[bh] + if !ok { + count = 2 // this one is already the second blob! + } else if count < math.MaxUint8 { + // don't overflow, but saturate count at 255 + // this can lead to a non-optimal pack selection, but won't cause + // problems otherwise + count++ + } + duplicateBlobs[bh] = count stats.size.duplicate += size stats.blobs.duplicate++ default: @@ -239,19 +334,19 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB "Will not start prune to prevent (additional) data loss!\n"+ "Please report this error (along with the output of the 'prune' run) at\n"+ "https://github.com/restic/restic/issues/new/choose\n", usedBlobs) - return errorIndexIncomplete + return nil, nil, errorIndexIncomplete } indexPack := make(map[restic.ID]packInfo) // save computed pack header size - for pid, hdrSize := range repo.Index().PackSize(ctx, true) { + for pid, hdrSize := range pack.Size(ctx, idx, true) { // initialize tpe with NumBlobTypes to indicate it's not set indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)} } // iterate over all blobs in index to generate packInfo - for blob := range repo.Index().Each(ctx) { + for blob := range idx.Each(ctx) { ip := indexPack[blob.PackID] // Set blob type if not yet set @@ -266,10 +361,9 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB bh := blob.BlobHandle size := uint64(blob.Length) + _, isDuplicate := duplicateBlobs[bh] switch { - case duplicateBlobs.Has(bh): // duplicate blob - ip.usedSize += size - ip.duplicateBlobs++ + case isDuplicate: // duplicate blobs will be handled later case keepBlobs.Has(bh): // used blob, not duplicate ip.usedSize += size ip.usedBlobs++ @@ -277,23 +371,66 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB ip.unusedSize += size ip.unusedBlobs++ } + if !blob.IsCompressed() { + ip.uncompressed = true + } // update indexPack indexPack[blob.PackID] = ip } - Verbosef("collecting packs for deletion and repacking\n") + // if duplicate blobs exist, those will be set to either "used" or "unused": + // - mark only one occurence of duplicate blobs as used + // - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used" + // - if there are no used blobs in a pack, possibly mark duplicates as "unused" + if len(duplicateBlobs) > 0 { + // iterate again over all blobs in index (this is pretty cheap, all in-mem) + for blob := range idx.Each(ctx) { + bh := blob.BlobHandle + count, isDuplicate := duplicateBlobs[bh] + if !isDuplicate { + continue + } + + ip := indexPack[blob.PackID] + size := uint64(blob.Length) + switch { + case count == 0: + // used duplicate exists -> mark as unused + ip.unusedSize += size + ip.unusedBlobs++ + case ip.usedBlobs > 0, count == 1: + // other used blobs in pack or "last" occurency -> mark as used + ip.usedSize += size + ip.usedBlobs++ + // let other occurences be marked as unused + duplicateBlobs[bh] = 0 + default: + // mark as unused and decrease counter + ip.unusedSize += size + ip.unusedBlobs++ + duplicateBlobs[bh] = count - 1 + } + // update indexPack + indexPack[blob.PackID] = ip + } + } + + return keepBlobs, indexPack, nil +} + +func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats) (prunePlan, error) { removePacksFirst := restic.NewIDSet() removePacks := restic.NewIDSet() repackPacks := restic.NewIDSet() var repackCandidates []packInfoWithID - repackAllPacksWithDuplicates := true - - keep := func(p packInfo) { - stats.packs.keep++ - if p.duplicateBlobs > 0 { - repackAllPacksWithDuplicates = false - } + var repackSmallCandidates []packInfoWithID + repoVersion := repo.Config().Version + // only repack very small files by default + targetPackSize := repo.PackSize() / 25 + if opts.RepackSmall { + // consider files with at least 80% of the target size as large enough + targetPackSize = repo.PackSize() / 5 * 4 } // loop over all packs and decide what to do @@ -308,8 +445,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB return nil } - if p.unusedSize+p.usedSize != uint64(packSize) && - !(p.usedBlobs == 0 && p.duplicateBlobs == 0) { + if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 { // Pack size does not fit and pack is needed => error // If the pack is not needed, this is no error, the pack can // and will be simply removed, see below. @@ -320,7 +456,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB // statistics switch { - case p.usedBlobs == 0 && p.duplicateBlobs == 0: + case p.usedBlobs == 0: stats.packs.unused++ case p.unusedBlobs == 0: stats.packs.used++ @@ -328,9 +464,18 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB stats.packs.partlyUsed++ } + mustCompress := false + if repoVersion >= 2 { + // repo v2: always repack tree blobs if uncompressed + // compress data blobs if requested + mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed + } + // use a flag that pack must be compressed + p.uncompressed = mustCompress + // decide what to do switch { - case p.usedBlobs == 0 && p.duplicateBlobs == 0: + case p.usedBlobs == 0: // All blobs in pack are no longer used => remove pack! removePacks.Insert(id) stats.blobs.remove += p.unusedBlobs @@ -338,11 +483,15 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB case opts.RepackCachableOnly && p.tpe == restic.DataBlob: // if this is a data pack and --repack-cacheable-only is set => keep pack! - keep(p) - - case p.unusedBlobs == 0 && p.duplicateBlobs == 0 && p.tpe != restic.InvalidBlob: - // All blobs in pack are used and not duplicates/mixed => keep pack! - keep(p) + stats.packs.keep++ + + case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress: + if packSize >= int64(targetPackSize) { + // All blobs in pack are used and not mixed => keep pack! + stats.packs.keep++ + } else { + repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p}) + } default: // all other packs are candidates for repacking @@ -355,7 +504,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB }) bar.Done() if err != nil { - return err + return prunePlan{}, err } // At this point indexPacks contains only missing packs! @@ -363,7 +512,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB // missing packs that are not needed can be ignored ignorePacks := restic.NewIDSet() for id, p := range indexPack { - if p.usedBlobs == 0 && p.duplicateBlobs == 0 { + if p.usedBlobs == 0 { ignorePacks.Insert(id) stats.blobs.remove += p.unusedBlobs stats.size.remove += p.unusedSize @@ -376,7 +525,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB for id := range indexPack { Warnf(" %v\n", id) } - return errorPacksMissing + return prunePlan{}, errorPacksMissing } if len(ignorePacks) != 0 { Warnf("Missing but unneeded pack files are referenced in the index, will be repaired\n") @@ -385,65 +534,81 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB } } - // calculate limit for number of unused bytes in the repo after repacking - maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used) + if len(repackSmallCandidates) < 10 { + // too few small files to be worth the trouble, this also prevents endlessly repacking + // if there is just a single pack file below the target size + stats.packs.keep += uint(len(repackSmallCandidates)) + } else { + repackCandidates = append(repackCandidates, repackSmallCandidates...) + } // Sort repackCandidates such that packs with highest ratio unused/used space are picked first. // This is equivalent to sorting by unused / total space. // Instead of unused[i] / used[i] > unused[j] / used[j] we use // unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64 - // Morover duplicates and packs containing trees are sorted to the beginning + // Moreover packs containing trees and too small packs are sorted to the beginning sort.Slice(repackCandidates, func(i, j int) bool { pi := repackCandidates[i].packInfo pj := repackCandidates[j].packInfo switch { - case pi.duplicateBlobs > 0 && pj.duplicateBlobs == 0: - return true - case pj.duplicateBlobs > 0 && pi.duplicateBlobs == 0: - return false case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob: return true case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob: return false + case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize): + return true + case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize): + return false } return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize }) repack := func(id restic.ID, p packInfo) { repackPacks.Insert(id) - stats.blobs.repack += p.unusedBlobs + p.duplicateBlobs + p.usedBlobs + stats.blobs.repack += p.unusedBlobs + p.usedBlobs stats.size.repack += p.unusedSize + p.usedSize stats.blobs.repackrm += p.unusedBlobs stats.size.repackrm += p.unusedSize } + // calculate limit for number of unused bytes in the repo after repacking + maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used) + for _, p := range repackCandidates { reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter) reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes + packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize) switch { case reachedRepackSize: - keep(p.packInfo) + stats.packs.keep++ - case p.duplicateBlobs > 0, p.tpe != restic.DataBlob: - // repacking duplicates/non-data is only limited by repackSize + case p.tpe != restic.DataBlob, p.uncompressed: + // repacking non-data packs / uncompressed-trees is only limited by repackSize repack(p.ID, p.packInfo) - case reachedUnusedSizeAfter: + case reachedUnusedSizeAfter && packIsLargeEnough: // for all other packs stop repacking if tolerated unused size is reached. - keep(p.packInfo) + stats.packs.keep++ default: repack(p.ID, p.packInfo) } } - // if all duplicates are repacked, print out correct statistics - if repackAllPacksWithDuplicates { - stats.blobs.repackrm += stats.blobs.duplicate - stats.size.repackrm += stats.size.duplicate - } + stats.packs.unref = uint(len(removePacksFirst)) + stats.packs.repack = uint(len(repackPacks)) + stats.packs.remove = uint(len(removePacks)) + + return prunePlan{removePacksFirst: removePacksFirst, + removePacks: removePacks, + repackPacks: repackPacks, + ignorePacks: ignorePacks, + }, nil +} +// printPruneStats prints out the statistics +func printPruneStats(gopts GlobalOptions, stats pruneStats) error { Verboseff("\nused: %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used)) if stats.blobs.duplicate > 0 { Verboseff("duplicates: %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate)) @@ -473,73 +638,109 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB Verboseff("unused packs: %10d\n\n", stats.packs.unused) Verboseff("to keep: %10d packs\n", stats.packs.keep) - Verboseff("to repack: %10d packs\n", len(repackPacks)) - Verboseff("to delete: %10d packs\n", len(removePacks)) - if len(removePacksFirst) > 0 { - Verboseff("to delete: %10d unreferenced packs\n\n", len(removePacksFirst)) + Verboseff("to repack: %10d packs\n", stats.packs.repack) + Verboseff("to delete: %10d packs\n", stats.packs.remove) + if stats.packs.unref > 0 { + Verboseff("to delete: %10d unreferenced packs\n\n", stats.packs.unref) } + return nil +} + +// doPrune does the actual pruning: +// - remove unreferenced packs first +// - repack given pack files while keeping the given blobs +// - rebuild the index while ignoring all files that will be deleted +// - delete the files +// plan.removePacks and plan.ignorePacks are modified in this function. +func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) { + ctx := gopts.ctx if opts.DryRun { if !gopts.JSON && gopts.verbosity >= 2 { - if len(removePacksFirst) > 0 { - Printf("Would have removed the following unreferenced packs:\n%v\n\n", removePacksFirst) + if len(plan.removePacksFirst) > 0 { + Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst) } - Printf("Would have repacked and removed the following packs:\n%v\n\n", repackPacks) - Printf("Would have removed the following no longer used packs:\n%v\n\n", removePacks) + Printf("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks) + Printf("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks) } // Always quit here if DryRun was set! return nil } // unreferenced packs can be safely deleted first - if len(removePacksFirst) != 0 { + if len(plan.removePacksFirst) != 0 { Verbosef("deleting unreferenced packs\n") - DeleteFiles(gopts, repo, removePacksFirst, restic.PackFile) + DeleteFiles(gopts, repo, plan.removePacksFirst, restic.PackFile) } - if len(repackPacks) != 0 { + if len(plan.repackPacks) != 0 { Verbosef("repacking packs\n") - bar := newProgressMax(!gopts.Quiet, uint64(len(repackPacks)), "packs repacked") - _, err := repository.Repack(ctx, repo, repackPacks, keepBlobs, bar) + bar := newProgressMax(!gopts.Quiet, uint64(len(plan.repackPacks)), "packs repacked") + _, err := repository.Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar) bar.Done() if err != nil { return errors.Fatalf("%s", err) } // Also remove repacked packs - removePacks.Merge(repackPacks) + plan.removePacks.Merge(plan.repackPacks) + + if len(plan.keepBlobs) != 0 { + Warnf("%v was not repacked\n\n"+ + "Integrity check failed.\n"+ + "Please report this error (along with the output of the 'prune' run) at\n"+ + "https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs) + return errors.Fatal("internal error: blobs were not repacked") + } } - if len(ignorePacks) == 0 { - ignorePacks = removePacks + if len(plan.ignorePacks) == 0 { + plan.ignorePacks = plan.removePacks } else { - ignorePacks.Merge(removePacks) + plan.ignorePacks.Merge(plan.removePacks) } - if len(ignorePacks) != 0 { - err = rebuildIndexFiles(gopts, repo, ignorePacks, nil) + if opts.unsafeRecovery { + Verbosef("deleting index files\n") + indexFiles := repo.Index().(*repository.MasterIndex).IDs() + err = DeleteFilesChecked(gopts, repo, indexFiles, restic.IndexFile) + if err != nil { + return errors.Fatalf("%s", err) + } + } else if len(plan.ignorePacks) != 0 { + err = rebuildIndexFiles(gopts, repo, plan.ignorePacks, nil) if err != nil { return errors.Fatalf("%s", err) } } - if len(removePacks) != 0 { - Verbosef("removing %d old packs\n", len(removePacks)) - DeleteFiles(gopts, repo, removePacks, restic.PackFile) + if len(plan.removePacks) != 0 { + Verbosef("removing %d old packs\n", len(plan.removePacks)) + DeleteFiles(gopts, repo, plan.removePacks, restic.PackFile) + } + + if opts.unsafeRecovery { + _, err = writeIndexFiles(gopts, repo, plan.ignorePacks, nil) + if err != nil { + return errors.Fatalf("%s", err) + } } Verbosef("done\n") return nil } -func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error { +func writeIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) { Verbosef("rebuilding index\n") - idx := (repo.Index()).(*repository.MasterIndex) - packcount := uint64(len(idx.Packs(removePacks))) - bar := newProgressMax(!gopts.Quiet, packcount, "packs processed") - obsoleteIndexes, err := idx.Save(gopts.ctx, repo, removePacks, extraObsolete, bar) + bar := newProgressMax(!gopts.Quiet, 0, "packs processed") + obsoleteIndexes, err := repo.Index().Save(gopts.ctx, repo, removePacks, extraObsolete, bar) bar.Done() + return obsoleteIndexes, err +} + +func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error { + obsoleteIndexes, err := writeIndexFiles(gopts, repo, removePacks, extraObsolete) if err != nil { return err } @@ -553,17 +754,18 @@ func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots r var snapshotTrees restic.IDs Verbosef("loading all snapshots...\n") - err = restic.ForAllSnapshots(gopts.ctx, repo, ignoreSnapshots, + err = restic.ForAllSnapshots(gopts.ctx, repo.Backend(), repo, ignoreSnapshots, func(id restic.ID, sn *restic.Snapshot, err error) error { - debug.Log("add snapshot %v (tree %v, error %v)", id, *sn.Tree, err) if err != nil { + debug.Log("failed to load snapshot %v (error %v)", id, err) return err } + debug.Log("add snapshot %v (tree %v)", id, *sn.Tree) snapshotTrees = append(snapshotTrees, *sn.Tree) return nil }) if err != nil { - return nil, err + return nil, errors.Fatalf("failed loading snapshot: %v", err) } Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees)) @@ -576,7 +778,7 @@ func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots r err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar) if err != nil { if repo.Backend().IsNotExist(err) { - return nil, errors.Fatal("unable to load a tree from the repo: " + err.Error()) + return nil, errors.Fatal("unable to load a tree from the repository: " + err.Error()) } return nil, err diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go index 718d2c7672e..0b3274ec4cc 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_rebuild_index.go @@ -1,6 +1,7 @@ package main import ( + "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -97,7 +98,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor if err != nil { return err } - packSizeFromIndex = repo.Index().PackSize(ctx, false) + packSizeFromIndex = pack.Size(ctx, repo.Index(), false) } Verbosef("getting pack files to read...\n") diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 860c45a576e..9f6d2061d0b 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -5,9 +5,11 @@ import ( "os" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" ) var cmdRecover = &cobra.Command{ @@ -50,6 +52,11 @@ func runRecover(gopts GlobalOptions) error { return err } + snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) + if err != nil { + return err + } + Verbosef("load index files\n") if err = repo.LoadIndex(gopts.ctx); err != nil { return err @@ -68,7 +75,7 @@ func runRecover(gopts GlobalOptions) error { Verbosef("load %d trees\n", len(trees)) bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded") for id := range trees { - tree, err := repo.LoadTree(gopts.ctx, id) + tree, err := restic.LoadTree(gopts.ctx, repo, id) if err != nil { Warnf("unable to load tree %v: %v\n", id.Str(), err) continue @@ -84,7 +91,7 @@ func runRecover(gopts GlobalOptions) error { bar.Done() Verbosef("load snapshots\n") - err = restic.ForAllSnapshots(gopts.ctx, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { + err = restic.ForAllSnapshots(gopts.ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { trees[*sn.Tree] = true return nil }) @@ -125,14 +132,26 @@ func runRecover(gopts GlobalOptions) error { } } - treeID, err := repo.SaveTree(gopts.ctx, tree) - if err != nil { - return errors.Fatalf("unable to save new tree to the repo: %v", err) - } + wg, ctx := errgroup.WithContext(gopts.ctx) + repo.StartPackUploader(ctx, wg) + + var treeID restic.ID + wg.Go(func() error { + var err error + treeID, err = restic.SaveTree(ctx, repo, tree) + if err != nil { + return errors.Fatalf("unable to save new tree to the repository: %v", err) + } - err = repo.Flush(gopts.ctx) + err = repo.Flush(ctx) + if err != nil { + return errors.Fatalf("unable to save blobs to the repository: %v", err) + } + return nil + }) + err = wg.Wait() if err != nil { - return errors.Fatalf("unable to save blobs to the repo: %v", err) + return err } return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID) @@ -147,7 +166,7 @@ func createSnapshot(ctx context.Context, name, hostname string, tags []string, r sn.Tree = tree - id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn) + id, err := restic.SaveSnapshot(ctx, repo, sn) if err != nil { return errors.Fatalf("unable to save snapshot: %v", err) } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 4d58185935f..addd3666152 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -70,6 +70,28 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0 hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0 + // Validate provided patterns + if len(opts.Exclude) > 0 { + if valid, invalidPatterns := filter.ValidatePatterns(opts.Exclude); !valid { + return errors.Fatalf("--exclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n")) + } + } + if len(opts.InsensitiveExclude) > 0 { + if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveExclude); !valid { + return errors.Fatalf("--iexclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n")) + } + } + if len(opts.Include) > 0 { + if valid, invalidPatterns := filter.ValidatePatterns(opts.Include); !valid { + return errors.Fatalf("--include: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n")) + } + } + if len(opts.InsensitiveInclude) > 0 { + if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveInclude); !valid { + return errors.Fatalf("--iinclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n")) + } + } + for i, str := range opts.InsensitiveExclude { opts.InsensitiveExclude[i] = strings.ToLower(str) } @@ -110,25 +132,25 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { } } - err = repo.LoadIndex(ctx) - if err != nil { - return err - } - var id restic.ID if snapshotIDString == "latest" { - id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil) + id, err = restic.FindLatestSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil) if err != nil { Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts) } } else { - id, err = restic.FindSnapshot(ctx, repo, snapshotIDString) + id, err = restic.FindSnapshot(ctx, repo.Backend(), snapshotIDString) if err != nil { Exitf(1, "invalid id %q: %v", snapshotIDString, err) } } + err = repo.LoadIndex(ctx) + if err != nil { + return err + } + res, err := restorer.NewRestorer(ctx, repo, id) if err != nil { Exitf(2, "creating restorer failed: %v\n", err) diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 70db0941774..ed201bf65a7 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -58,7 +58,7 @@ func init() { panic(err) } f.IntVar(&snapshotOptions.Latest, "latest", 0, "only show the last `n` snapshots for each host and path") - f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags") + f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "`group` snapshots by host, paths and/or tags, separated by comma") } func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error { @@ -79,7 +79,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro defer cancel() var snapshots restic.Snapshots - for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { snapshots = append(snapshots, sn) } snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index deb649e261c..a8bcb2b85a5 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -6,6 +6,7 @@ import ( "fmt" "path/filepath" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" @@ -86,10 +87,6 @@ func runStats(gopts GlobalOptions, args []string) error { return err } - if err = repo.LoadIndex(ctx); err != nil { - return err - } - if !gopts.NoLock { lock, err := lockRepo(ctx, repo) defer unlockRepo(lock) @@ -98,6 +95,15 @@ func runStats(gopts GlobalOptions, args []string) error { } } + snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) + if err != nil { + return err + } + + if err = repo.LoadIndex(ctx); err != nil { + return err + } + if !gopts.JSON { Printf("scanning...\n") } @@ -105,13 +111,12 @@ func runStats(gopts GlobalOptions, args []string) error { // create a container for the stats (and other needed state) stats := &statsContainer{ uniqueFiles: make(map[fileID]struct{}), - uniqueInodes: make(map[uint64]struct{}), fileBlobs: make(map[string]restic.IDSet), blobs: restic.NewBlobSet(), - snapshotsCount: 0, + SnapshotsCount: 0, } - for sn := range FindFilteredSnapshots(ctx, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) { err = statsWalkSnapshot(ctx, sn, repo, stats) if err != nil { return fmt.Errorf("error walking snapshot: %v", err) @@ -125,11 +130,11 @@ func runStats(gopts GlobalOptions, args []string) error { if statsOptions.countMode == countModeRawData { // the blob handles have been collected, but not yet counted for blobHandle := range stats.blobs { - blobSize, found := repo.LookupBlobSize(blobHandle.ID, blobHandle.Type) - if !found { + pbs := repo.Index().Lookup(blobHandle) + if len(pbs) == 0 { return fmt.Errorf("blob %v not found", blobHandle) } - stats.TotalSize += uint64(blobSize) + stats.TotalSize += uint64(pbs[0].Length) stats.TotalBlobCount++ } } @@ -143,7 +148,7 @@ func runStats(gopts GlobalOptions, args []string) error { } Printf("Stats in %s mode:\n", statsOptions.countMode) - Printf("Snapshots processed: %d\n", stats.snapshotsCount) + Printf("Snapshots processed: %d\n", stats.SnapshotsCount) if stats.TotalBlobCount > 0 { Printf(" Total Blob Count: %d\n", stats.TotalBlobCount) @@ -161,7 +166,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str()) } - stats.snapshotsCount++ + stats.SnapshotsCount++ if statsOptions.countMode == countModeRawData { // count just the sizes of unique blobs; we don't need to walk the tree @@ -169,7 +174,8 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil) } - err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats)) + uniqueInodes := make(map[uint64]struct{}) + err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats, uniqueInodes)) if err != nil { return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err) } @@ -177,7 +183,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest return nil } -func statsWalkTree(repo restic.Repository, stats *statsContainer) walker.WalkFunc { +func statsWalkTree(repo restic.Repository, stats *statsContainer, uniqueInodes map[uint64]struct{}) walker.WalkFunc { return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) { if nodeErr != nil { return true, nodeErr @@ -236,8 +242,8 @@ func statsWalkTree(repo restic.Repository, stats *statsContainer) walker.WalkFun // if inodes are present, only count each inode once // (hard links do not increase restore size) - if _, ok := stats.uniqueInodes[node.Inode]; !ok || node.Inode == 0 { - stats.uniqueInodes[node.Inode] = struct{}{} + if _, ok := uniqueInodes[node.Inode]; !ok || node.Inode == 0 { + uniqueInodes[node.Inode] = struct{}{} stats.TotalSize += node.Size } @@ -279,15 +285,13 @@ type statsContainer struct { TotalSize uint64 `json:"total_size"` TotalFileCount uint64 `json:"total_file_count"` TotalBlobCount uint64 `json:"total_blob_count,omitempty"` + // holds count of all considered snapshots + SnapshotsCount int `json:"snapshots_count"` // uniqueFiles marks visited files according to their // contents (hashed sequence of content blob IDs) uniqueFiles map[fileID]struct{} - // uniqueInodes marks visited files according to their - // inode # (hashed sequence of inode numbers) - uniqueInodes map[uint64]struct{} - // fileBlobs maps a file name (path) to the set of // blobs that have been seen as a part of the file fileBlobs map[string]restic.IDSet @@ -295,9 +299,6 @@ type statsContainer struct { // blobs is used to count individual unique blobs, // independent of references to files blobs restic.BlobSet - - // holds count of all considered snapshots - snapshotsCount int } // fileID is a 256-bit hash that distinguishes unique files. diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index 13842c077eb..1b99a4d56b7 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -82,17 +82,13 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna } // Save the new snapshot. - id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn) + id, err := restic.SaveSnapshot(ctx, repo, sn) if err != nil { return false, err } debug.Log("new snapshot saved as %v", id) - if err = repo.Flush(ctx); err != nil { - return false, err - } - // Remove the old snapshot. h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(ctx, h); err != nil { @@ -129,7 +125,7 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error { changeCnt := 0 ctx, cancel := context.WithCancel(gopts.ctx) defer cancel() - for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten()) if err != nil { Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err) diff --git a/cmd/restic/delete.go b/cmd/restic/delete.go index 98dd91ece91..d97b9e61747 100644 --- a/cmd/restic/delete.go +++ b/cmd/restic/delete.go @@ -18,8 +18,6 @@ func DeleteFilesChecked(gopts GlobalOptions, repo restic.Repository, fileList re return deleteFiles(gopts, false, repo, fileList, fileType) } -const numDeleteWorkers = 8 - // deleteFiles deletes the given fileList of fileType in parallel // if ignoreError=true, it will print a warning if there was an error, else it will abort. func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { @@ -40,7 +38,9 @@ func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted") defer bar.Done() - for i := 0; i < numDeleteWorkers; i++ { + // deleting files is IO-bound + workerCount := repo.Connections() + for i := 0; i < int(workerCount); i++ { wg.Go(func() error { for id := range fileChan { h := restic.Handle{Type: fileType, Name: id.String()} diff --git a/cmd/restic/find.go b/cmd/restic/find.go index 792f1f6a2ac..5107ef59900 100644 --- a/cmd/restic/find.go +++ b/cmd/restic/find.go @@ -3,33 +3,39 @@ package main import ( "context" - "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" ) // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. -func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hosts []string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot { +func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, hosts []string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot { out := make(chan *restic.Snapshot) go func() { defer close(out) if len(snapshotIDs) != 0 { + // memorize snapshots list to prevent repeated backend listings + be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile) + if err != nil { + Warnf("could not load snapshots: %v\n", err) + return + } + var ( id restic.ID usedFilter bool - err error ) ids := make(restic.IDs, 0, len(snapshotIDs)) // Process all snapshot IDs given as arguments. for _, s := range snapshotIDs { if s == "latest" { usedFilter = true - id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts, nil) + id, err = restic.FindLatestSnapshot(ctx, be, loader, paths, tags, hosts, nil) if err != nil { Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)\n", s, paths, tags, hosts) continue } } else { - id, err = restic.FindSnapshot(ctx, repo, s) + id, err = restic.FindSnapshot(ctx, be, s) if err != nil { Warnf("Ignoring %q: %v\n", s, err) continue @@ -44,7 +50,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos } for _, id := range ids.Uniq() { - sn, err := restic.LoadSnapshot(ctx, repo, id) + sn, err := restic.LoadSnapshot(ctx, loader, id) if err != nil { Warnf("Ignoring %q, could not load snapshot: %v\n", id, err) continue @@ -58,7 +64,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos return } - snapshots, err := restic.FindFilteredSnapshots(ctx, repo, hosts, tags, paths) + snapshots, err := restic.FindFilteredSnapshots(ctx, be, loader, hosts, tags, paths) if err != nil { Warnf("could not load snapshots: %v\n", err) return diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 54fa0a08bc1..fd444f56605 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -9,6 +9,7 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" "strings" "syscall" "time" @@ -17,6 +18,7 @@ import ( "github.com/restic/restic/internal/backend/azure" "github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/gs" + "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/ontap" @@ -28,7 +30,6 @@ import ( "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/limiter" "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -37,11 +38,11 @@ import ( "github.com/restic/restic/internal/errors" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) -var version = "0.13.1" -var netappversion = "2.1.1-dev (compiled manually)" +var version = "0.14.0" +var netappversion = "3.0.0-dev (compiled manually)" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" @@ -61,13 +62,12 @@ type GlobalOptions struct { JSON bool CacheDir string NoCache bool - CACerts []string - InsecureTLS bool - TLSClientCert string CleanupCache bool + Compression repository.CompressionMode + PackSize uint - LimitUploadKb int - LimitDownloadKb int + backend.TransportOptions + limiter.Limits ctx context.Context password string @@ -105,6 +105,9 @@ func init() { return nil }) + // parse target pack size from env, on error the default value will be used + targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32) + f := cmdRoot.PersistentFlags() f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)") f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", os.Getenv("RESTIC_REPOSITORY_FILE"), "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)") @@ -117,16 +120,24 @@ func init() { f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it") f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)") f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") - f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "`file` to load root certificates from (default: use system certificates)") - f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key") - f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repo (insecure)") + f.StringSliceVar(&globalOptions.RootCertFilenames, "cacert", nil, "`file` to load root certificates from (default: use system certificates)") + f.StringVar(&globalOptions.TLSClientCertKeyFilename, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key") + f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") - f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)") - f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)") + f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)") + f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)") + f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)") + f.UintVar(&globalOptions.PackSize, "pack-size", uint(targetPackSize), "set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") // Use our "generate" command instead of the cobra provided "completion" command cmdRoot.CompletionOptions.DisableDefaultCmd = true + comp := os.Getenv("RESTIC_COMPRESSION") + if comp != "" { + // ignore error as there's no good way to handle it + _ = globalOptions.Compression.Set(comp) + } + restoreTerminal() } @@ -146,13 +157,13 @@ func checkErrno(err error) error { } func stdinIsTerminal() bool { - return terminal.IsTerminal(int(os.Stdin.Fd())) + return term.IsTerminal(int(os.Stdin.Fd())) } func stdoutIsTerminal() bool { // mintty on windows can use pipes which behave like a posix terminal, // but which are not a terminal handle - return terminal.IsTerminal(int(os.Stdout.Fd())) || stdoutCanUpdateStatus() + return term.IsTerminal(int(os.Stdout.Fd())) || stdoutCanUpdateStatus() } func stdoutCanUpdateStatus() bool { @@ -160,7 +171,7 @@ func stdoutCanUpdateStatus() bool { } func stdoutTerminalWidth() int { - w, _, err := terminal.GetSize(int(os.Stdout.Fd())) + w, _, err := term.GetSize(int(os.Stdout.Fd())) if err != nil { return 0 } @@ -173,12 +184,12 @@ func stdoutTerminalWidth() int { // program execution must revert changes to the terminal configuration itself. // The terminal configuration is only restored while reading a password. func restoreTerminal() { - if !terminal.IsTerminal(int(os.Stdout.Fd())) { + if !term.IsTerminal(int(os.Stdout.Fd())) { return } fd := int(os.Stdout.Fd()) - state, err := terminal.GetState(fd) + state, err := term.GetState(fd) if err != nil { fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err) return @@ -193,7 +204,7 @@ func restoreTerminal() { if !isReadingPassword { return nil } - err := checkErrno(terminal.Restore(fd, state)) + err := checkErrno(term.Restore(fd, state)) if err != nil { fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err) } @@ -296,7 +307,7 @@ func resolvePassword(opts GlobalOptions, envStr string) (string, error) { } if opts.PasswordFile != "" { s, err := textfile.Read(opts.PasswordFile) - if os.IsNotExist(errors.Cause(err)) { + if errors.Is(err, os.ErrNotExist) { return "", errors.Fatalf("%s does not exist", opts.PasswordFile) } return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") @@ -323,7 +334,7 @@ func readPassword(in io.Reader) (password string, err error) { func readPasswordTerminal(in *os.File, out io.Writer, prompt string) (password string, err error) { fmt.Fprint(out, prompt) isReadingPassword = true - buf, err := terminal.ReadPassword(int(in.Fd())) + buf, err := term.ReadPassword(int(in.Fd())) isReadingPassword = false fmt.Fprintln(out) if err != nil { @@ -397,7 +408,7 @@ func ReadRepo(opts GlobalOptions) (string, error) { } s, err := textfile.Read(opts.RepositoryFile) - if os.IsNotExist(errors.Cause(err)) { + if errors.Is(err, os.ErrNotExist) { return "", errors.Fatalf("%s does not exist", opts.RepositoryFile) } if err != nil { @@ -436,7 +447,13 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { } } - s := repository.New(be) + s, err := repository.New(be, repository.Options{ + Compression: opts.Compression, + PackSize: opts.PackSize * 1024 * 1024, + }) + if err != nil { + return nil, err + } passwordTriesLeft := 1 if stdinIsTerminal() && opts.password == "" { @@ -456,7 +473,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { err = s.SearchKey(opts.ctx, opts.password, maxKeys, opts.KeyHint) if err != nil && passwordTriesLeft > 1 { opts.password = "" - fmt.Printf("%s. Try again\n", err) + fmt.Fprintf(os.Stderr, "%s. Try again\n", err) } } if err != nil { @@ -472,7 +489,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { id = id[:8] } if !opts.JSON { - Verbosef("repository %v opened successfully, password is correct\n", id) + Verbosef("repository %v opened (repository version %v) successfully, password is correct\n", id, s.Config().Version) } } @@ -554,13 +571,13 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID") } - if cfg.Secret == "" { - cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY") + if cfg.Secret.String() == "" { + cfg.Secret = options.NewSecretString(os.Getenv("AWS_SECRET_ACCESS_KEY")) } - if cfg.KeyID == "" && cfg.Secret != "" { + if cfg.KeyID == "" && cfg.Secret.String() != "" { return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty") - } else if cfg.KeyID != "" && cfg.Secret == "" { + } else if cfg.KeyID != "" && cfg.Secret.String() == "" { return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty") } @@ -594,8 +611,12 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME") } - if cfg.AccountKey == "" { - cfg.AccountKey = os.Getenv("AZURE_ACCOUNT_KEY") + if cfg.AccountKey.String() == "" { + cfg.AccountKey = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_KEY")) + } + + if cfg.AccountSAS.String() == "" { + cfg.AccountSAS = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_SAS")) } if err := opts.Apply(loc.Scheme, &cfg); err != nil { @@ -630,11 +651,11 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty") } - if cfg.Key == "" { - cfg.Key = os.Getenv("B2_ACCOUNT_KEY") + if cfg.Key.String() == "" { + cfg.Key = options.NewSecretString(os.Getenv("B2_ACCOUNT_KEY")) } - if cfg.Key == "" { + if cfg.Key.String() == "" { return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty") } @@ -683,18 +704,13 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, return nil, err } - tropts := backend.TransportOptions{ - RootCertFilenames: globalOptions.CACerts, - TLSClientCertKeyFilename: globalOptions.TLSClientCert, - InsecureTLS: globalOptions.InsecureTLS, - } - rt, err := backend.Transport(tropts) + rt, err := backend.Transport(globalOptions.TransportOptions) if err != nil { return nil, err } // wrap the transport so that the throughput via HTTP is limited - lim := limiter.NewStaticLimiter(gopts.LimitUploadKb, gopts.LimitDownloadKb) + lim := limiter.NewStaticLimiter(gopts.Limits) rt = lim.Transport(rt) switch loc.Scheme { @@ -724,7 +740,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, } if err != nil { - return nil, errors.Fatalf("unable to open repo at %v: %v", location.StripPassword(s), err) + return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(s), err) } // wrap backend if a test specified an inner hook @@ -766,12 +782,7 @@ func create(s string, opts options.Options) (restic.Backend, error) { return nil, err } - tropts := backend.TransportOptions{ - RootCertFilenames: globalOptions.CACerts, - TLSClientCertKeyFilename: globalOptions.TLSClientCert, - InsecureTLS: globalOptions.InsecureTLS, - } - rt, err := backend.Transport(tropts) + rt, err := backend.Transport(globalOptions.TransportOptions) if err != nil { return nil, err } diff --git a/cmd/restic/global_debug.go b/cmd/restic/global_debug.go index 6f04d047b40..172f3451b85 100644 --- a/cmd/restic/global_debug.go +++ b/cmd/restic/global_debug.go @@ -1,3 +1,4 @@ +//go:build debug || profile // +build debug profile package main diff --git a/cmd/restic/global_release.go b/cmd/restic/global_release.go index f17d9963998..7cb2e6caf3c 100644 --- a/cmd/restic/global_release.go +++ b/cmd/restic/global_release.go @@ -1,3 +1,4 @@ +//go:build !debug && !profile // +build !debug,!profile package main diff --git a/cmd/restic/integration_filter_pattern_test.go b/cmd/restic/integration_filter_pattern_test.go new file mode 100644 index 00000000000..c0c1d932f55 --- /dev/null +++ b/cmd/restic/integration_filter_pattern_test.go @@ -0,0 +1,106 @@ +//go:build go1.16 +// +build go1.16 + +// Before Go 1.16 filepath.Match returned early on a failed match, +// and thus did not report any later syntax error in the pattern. +// https://go.dev/doc/go1.16#path/filepath + +package main + +import ( + "io/ioutil" + "path/filepath" + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + var err error + + // Test --exclude + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + + rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) + + // Test --iexclude + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + + rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) +} + +func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + // Create an exclude file with some invalid patterns + excludeFile := env.base + "/excludefile" + fileErr := ioutil.WriteFile(excludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644) + if fileErr != nil { + t.Fatalf("Could not write exclude file: %v", fileErr) + } + + var err error + + // Test --exclude-file: + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludeFiles: []string{excludeFile}}, env.gopts) + + rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) + + // Test --iexclude-file + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{InsensitiveExcludeFiles: []string{excludeFile}}, env.gopts) + + rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) +} + +func TestRestoreFailsWhenUsingInvalidPatterns(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + var err error + + // Test --exclude + err = testRunRestoreAssumeFailure(t, "latest", RestoreOptions{Exclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + + rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) + + // Test --iexclude + err = testRunRestoreAssumeFailure(t, "latest", RestoreOptions{InsensitiveExclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + + rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) + + // Test --include + err = testRunRestoreAssumeFailure(t, "latest", RestoreOptions{Include: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + + rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) + + // Test --iinclude + err = testRunRestoreAssumeFailure(t, "latest", RestoreOptions{InsensitiveInclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + + rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) +} diff --git a/cmd/restic/integration_fuse_test.go b/cmd/restic/integration_fuse_test.go index 7da85881ea7..6a95ac87daa 100644 --- a/cmd/restic/integration_fuse_test.go +++ b/cmd/restic/integration_fuse_test.go @@ -1,3 +1,4 @@ +//go:build darwin || freebsd || linux // +build darwin freebsd linux package main @@ -54,7 +55,7 @@ func waitForMount(t testing.TB, dir string) { func testRunMount(t testing.TB, gopts GlobalOptions, dir string) { opts := MountOptions{ - SnapshotTemplate: time.RFC3339, + TimeTemplate: time.RFC3339, } rtest.OK(t, runMount(opts, gopts, []string{dir})) } @@ -153,6 +154,8 @@ func TestMount(t *testing.T) { } env, cleanup := withTestEnvironment(t) + // must list snapshots more than once + env.gopts.backendTestHook = nil defer cleanup() testRunInit(t, env.gopts) @@ -196,6 +199,8 @@ func TestMountSameTimestamps(t *testing.T) { } env, cleanup := withTestEnvironment(t) + // must list snapshots more than once + env.gopts.backendTestHook = nil defer cleanup() rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz")) diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index df989335040..e87baddcace 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -198,6 +198,9 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { stdout: os.Stdout, stderr: os.Stderr, extended: make(options.Options), + + // replace this hook with "nil" if listing a filetype more than once is necessary + backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil }, } // always overwrite global options diff --git a/cmd/restic/integration_helpers_unix_test.go b/cmd/restic/integration_helpers_unix_test.go index 1130d2638fb..830d41b3d74 100644 --- a/cmd/restic/integration_helpers_unix_test.go +++ b/cmd/restic/integration_helpers_unix_test.go @@ -1,4 +1,5 @@ -//+build !windows +//go:build !windows +// +build !windows package main diff --git a/cmd/restic/integration_helpers_windows_test.go b/cmd/restic/integration_helpers_windows_test.go index f519a149407..a46d1e5cd81 100644 --- a/cmd/restic/integration_helpers_windows_test.go +++ b/cmd/restic/integration_helpers_windows_test.go @@ -1,4 +1,5 @@ -//+build windows +//go:build windows +// +build windows package main diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 42936d2ea3c..c04a5a2fb2a 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -131,6 +131,12 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()})) } +func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts GlobalOptions) error { + err := runRestore(opts, gopts, []string{snapshotID}) + + return err +} + func testRunCheck(t testing.TB, gopts GlobalOptions) { opts := CheckOptions{ ReadData: true, @@ -275,6 +281,11 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) { } func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) { + oldHook := gopts.backendTestHook + gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil } + defer func() { + gopts.backendTestHook = oldHook + }() rtest.OK(t, runPrune(opts, gopts)) } @@ -738,14 +749,17 @@ func TestBackupTags(t *testing.T) { } func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) { + gopts := srcGopts + gopts.Repo = dstGopts.Repo + gopts.password = dstGopts.password copyOpts := CopyOptions{ secondaryRepoOptions: secondaryRepoOptions{ - Repo: dstGopts.Repo, - password: dstGopts.password, + Repo: srcGopts.Repo, + password: srcGopts.password, }, } - rtest.OK(t, runCopy(copyOpts, srcGopts, nil)) + rtest.OK(t, runCopy(copyOpts, gopts, nil)) } func TestCopy(t *testing.T) { @@ -1035,7 +1049,7 @@ func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) { repo, err := OpenRepository(gopts) rtest.OK(t, err) - key, err := repository.SearchKey(gopts.ctx, repo, testKeyNewPassword, 1, "") + key, err := repository.SearchKey(gopts.ctx, repo, testKeyNewPassword, 2, "") rtest.OK(t, err) rtest.Equals(t, "john", key.Username) @@ -1065,6 +1079,8 @@ func TestKeyAddRemove(t *testing.T) { } env, cleanup := withTestEnvironment(t) + // must list keys more than once + env.gopts.backendTestHook = nil defer cleanup() testRunInit(t, env.gopts) @@ -1463,7 +1479,7 @@ func TestRebuildIndexAlwaysFull(t *testing.T) { defer func() { repository.IndexFull = indexFull }() - repository.IndexFull = func(*repository.Index) bool { return true } + repository.IndexFull = func(*repository.Index, bool) bool { return true } testRebuildIndex(t, nil) } @@ -1566,29 +1582,43 @@ func TestCheckRestoreNoLock(t *testing.T) { } func TestPrune(t *testing.T) { - t.Run("0", func(t *testing.T) { - opts := PruneOptions{MaxUnused: "0%"} + testPruneVariants(t, false) + testPruneVariants(t, true) +} + +func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) { + suffix := "" + if unsafeNoSpaceRecovery { + suffix = "-recovery" + } + t.Run("0"+suffix, func(t *testing.T) { + opts := PruneOptions{MaxUnused: "0%", unsafeRecovery: unsafeNoSpaceRecovery} checkOpts := CheckOptions{ReadData: true, CheckUnused: true} testPrune(t, opts, checkOpts) }) - t.Run("50", func(t *testing.T) { - opts := PruneOptions{MaxUnused: "50%"} + t.Run("50"+suffix, func(t *testing.T) { + opts := PruneOptions{MaxUnused: "50%", unsafeRecovery: unsafeNoSpaceRecovery} checkOpts := CheckOptions{ReadData: true} testPrune(t, opts, checkOpts) }) - t.Run("unlimited", func(t *testing.T) { - opts := PruneOptions{MaxUnused: "unlimited"} + t.Run("unlimited"+suffix, func(t *testing.T) { + opts := PruneOptions{MaxUnused: "unlimited", unsafeRecovery: unsafeNoSpaceRecovery} checkOpts := CheckOptions{ReadData: true} testPrune(t, opts, checkOpts) }) - t.Run("CachableOnly", func(t *testing.T) { - opts := PruneOptions{MaxUnused: "5%", RepackCachableOnly: true} + t.Run("CachableOnly"+suffix, func(t *testing.T) { + opts := PruneOptions{MaxUnused: "5%", RepackCachableOnly: true, unsafeRecovery: unsafeNoSpaceRecovery} checkOpts := CheckOptions{ReadData: true} testPrune(t, opts, checkOpts) }) + t.Run("Small", func(t *testing.T) { + opts := PruneOptions{MaxUnused: "unlimited", RepackSmall: true} + checkOpts := CheckOptions{ReadData: true, CheckUnused: true} + testPrune(t, opts, checkOpts) + }) } func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) { @@ -1659,6 +1689,11 @@ func TestPruneWithDamagedRepository(t *testing.T) { rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs) + oldHook := env.gopts.backendTestHook + env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil } + defer func() { + env.gopts.backendTestHook = oldHook + }() // prune should fail rtest.Assert(t, runPrune(pruneDefaultOptions, env.gopts) == errorPacksMissing, "prune should have reported index not complete error") @@ -1752,12 +1787,22 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o type listOnceBackend struct { restic.Backend listedFileType map[restic.FileType]bool + strictOrder bool } func newListOnceBackend(be restic.Backend) *listOnceBackend { return &listOnceBackend{ Backend: be, listedFileType: make(map[restic.FileType]bool), + strictOrder: false, + } +} + +func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend { + return &listOnceBackend{ + Backend: be, + listedFileType: make(map[restic.FileType]bool), + strictOrder: true, } } @@ -1765,6 +1810,9 @@ func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func( if t != restic.LockFile && be.listedFileType[t] { return errors.Errorf("tried listing type %v the second time", t) } + if be.strictOrder && t == restic.SnapshotFile && be.listedFileType[restic.IndexFile] { + return errors.Errorf("tried listing type snapshots after index") + } be.listedFileType[t] = true return be.Backend.List(ctx, t, fn) } @@ -2135,7 +2183,38 @@ func TestBackendLoadWriteTo(t *testing.T) { firstSnapshot := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(firstSnapshot) == 1, "expected one snapshot, got %v", firstSnapshot) +} - // test readData using the hashing.Reader - testRunCheck(t, env.gopts) +func TestFindListOnce(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + return newListOnceBackend(r), nil + } + + testSetupBackupData(t, env) + opts := BackupOptions{} + + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts) + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts) + secondSnapshot := testRunList(t, "snapshots", env.gopts) + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts) + thirdSnapshot := restic.NewIDSet(testRunList(t, "snapshots", env.gopts)...) + + repo, err := OpenRepository(env.gopts) + rtest.OK(t, err) + + snapshotIDs := restic.NewIDSet() + // specify the two oldest snapshots explicitly and use "latest" to reference the newest one + for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, nil, nil, nil, []string{ + secondSnapshot[0].String(), + secondSnapshot[1].String()[:8], + "latest", + }) { + snapshotIDs.Insert(*sn.ID()) + } + + // the snapshots can only be listed once, if both lists match then the there has been only a single List() call + rtest.Equals(t, thirdSnapshot, snapshotIDs) } diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 0479124b16b..ad3ef89d469 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -98,11 +98,11 @@ func main() { err := cmdRoot.Execute() switch { - case restic.IsAlreadyLocked(errors.Cause(err)): + case restic.IsAlreadyLocked(err): fmt.Fprintf(os.Stderr, "%v\nthe `unlock` command can be used to remove stale locks\n", err) case err == ErrInvalidSourceData: fmt.Fprintf(os.Stderr, "Warning: %v\n", err) - case errors.IsFatal(errors.Cause(err)): + case errors.IsFatal(err): fmt.Fprintf(os.Stderr, "%v\n", err) case err != nil: fmt.Fprintf(os.Stderr, "%+v\n", err) diff --git a/cmd/restic/progress.go b/cmd/restic/progress.go index 5a77bc6eef6..4f33e207287 100644 --- a/cmd/restic/progress.go +++ b/cmd/restic/progress.go @@ -58,7 +58,10 @@ func printProgress(status string, canUpdateStatus bool) { if w < 3 { status = termstatus.Truncate(status, w) } else { - status = termstatus.Truncate(status, w-3) + "..." + trunc := termstatus.Truncate(status, w-3) + if len(trunc) < len(status) { + status = trunc + "..." + } } } diff --git a/cmd/restic/secondary_repo.go b/cmd/restic/secondary_repo.go index fa42f19b36c..7b08004a779 100644 --- a/cmd/restic/secondary_repo.go +++ b/cmd/restic/secondary_repo.go @@ -8,49 +8,98 @@ import ( ) type secondaryRepoOptions struct { + password string + // from-repo options Repo string RepositoryFile string - password string PasswordFile string PasswordCommand string KeyHint string + // repo2 options + LegacyRepo string + LegacyRepositoryFile string + LegacyPasswordFile string + LegacyPasswordCommand string + LegacyKeyHint string } func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repoPrefix string, repoUsage string) { - f.StringVarP(&opts.Repo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)") - f.StringVarP(&opts.RepositoryFile, "repository-file2", "", os.Getenv("RESTIC_REPOSITORY_FILE2"), "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)") - f.StringVarP(&opts.PasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)") - f.StringVarP(&opts.KeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)") - f.StringVarP(&opts.PasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)") + f.StringVarP(&opts.LegacyRepo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)") + f.StringVarP(&opts.LegacyRepositoryFile, "repository-file2", "", os.Getenv("RESTIC_REPOSITORY_FILE2"), "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)") + f.StringVarP(&opts.LegacyPasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)") + f.StringVarP(&opts.LegacyKeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)") + f.StringVarP(&opts.LegacyPasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)") + + // hide repo2 options + _ = f.MarkDeprecated("repo2", "use --repo or --from-repo instead") + _ = f.MarkDeprecated("repository-file2", "use --repository-file or --from-repository-file instead") + _ = f.MarkHidden("password-file2") + _ = f.MarkHidden("key-hint2") + _ = f.MarkHidden("password-command2") + + f.StringVarP(&opts.Repo, "from-repo", "", os.Getenv("RESTIC_FROM_REPOSITORY"), "source `repository` "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY)") + f.StringVarP(&opts.RepositoryFile, "from-repository-file", "", os.Getenv("RESTIC_FROM_REPOSITORY_FILE"), "`file` from which to read the source repository location "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY_FILE)") + f.StringVarP(&opts.PasswordFile, "from-password-file", "", os.Getenv("RESTIC_FROM_PASSWORD_FILE"), "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)") + f.StringVarP(&opts.KeyHint, "from-key-hint", "", os.Getenv("RESTIC_FROM_KEY_HINT"), "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)") + f.StringVarP(&opts.PasswordCommand, "from-password-command", "", os.Getenv("RESTIC_FROM_PASSWORD_COMMAND"), "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)") } -func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, error) { - if opts.Repo == "" && opts.RepositoryFile == "" { - return GlobalOptions{}, errors.Fatal("Please specify a " + repoPrefix + " repository location (--repo2 or --repository-file2)") +func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, bool, error) { + if opts.Repo == "" && opts.RepositoryFile == "" && opts.LegacyRepo == "" && opts.LegacyRepositoryFile == "" { + return GlobalOptions{}, false, errors.Fatal("Please specify a source repository location (--from-repo or --from-repository-file)") } - if opts.Repo != "" && opts.RepositoryFile != "" { - return GlobalOptions{}, errors.Fatal("Options --repo2 and --repository-file2 are mutually exclusive, please specify only one") + hasFromRepo := opts.Repo != "" || opts.RepositoryFile != "" || opts.PasswordFile != "" || + opts.KeyHint != "" || opts.PasswordCommand != "" + hasRepo2 := opts.LegacyRepo != "" || opts.LegacyRepositoryFile != "" || opts.LegacyPasswordFile != "" || + opts.LegacyKeyHint != "" || opts.LegacyPasswordCommand != "" + + if hasFromRepo && hasRepo2 { + return GlobalOptions{}, false, errors.Fatal("Option groups repo2 and from-repo are mutually exclusive, please specify only one") } var err error dstGopts := gopts - dstGopts.Repo = opts.Repo - dstGopts.RepositoryFile = opts.RepositoryFile - dstGopts.PasswordFile = opts.PasswordFile - dstGopts.PasswordCommand = opts.PasswordCommand - dstGopts.KeyHint = opts.KeyHint + var pwdEnv string + + if hasFromRepo { + if opts.Repo != "" && opts.RepositoryFile != "" { + return GlobalOptions{}, false, errors.Fatal("Options --from-repo and --from-repository-file are mutually exclusive, please specify only one") + } + + dstGopts.Repo = opts.Repo + dstGopts.RepositoryFile = opts.RepositoryFile + dstGopts.PasswordFile = opts.PasswordFile + dstGopts.PasswordCommand = opts.PasswordCommand + dstGopts.KeyHint = opts.KeyHint + + pwdEnv = "RESTIC_FROM_PASSWORD" + repoPrefix = "source" + } else { + if opts.LegacyRepo != "" && opts.LegacyRepositoryFile != "" { + return GlobalOptions{}, false, errors.Fatal("Options --repo2 and --repository-file2 are mutually exclusive, please specify only one") + } + + dstGopts.Repo = opts.LegacyRepo + dstGopts.RepositoryFile = opts.LegacyRepositoryFile + dstGopts.PasswordFile = opts.LegacyPasswordFile + dstGopts.PasswordCommand = opts.LegacyPasswordCommand + dstGopts.KeyHint = opts.LegacyKeyHint + + pwdEnv = "RESTIC_PASSWORD2" + } + if opts.password != "" { dstGopts.password = opts.password } else { - dstGopts.password, err = resolvePassword(dstGopts, "RESTIC_PASSWORD2") + dstGopts.password, err = resolvePassword(dstGopts, pwdEnv) if err != nil { - return GlobalOptions{}, err + return GlobalOptions{}, false, err } } dstGopts.password, err = ReadPassword(dstGopts, "enter password for "+repoPrefix+" repository: ") if err != nil { - return GlobalOptions{}, err + return GlobalOptions{}, false, err } - return dstGopts, nil + return dstGopts, hasFromRepo, nil } diff --git a/cmd/restic/secondary_repo_test.go b/cmd/restic/secondary_repo_test.go index 47c1816af21..cb410f1b928 100644 --- a/cmd/restic/secondary_repo_test.go +++ b/cmd/restic/secondary_repo_test.go @@ -8,12 +8,13 @@ import ( rtest "github.com/restic/restic/internal/test" ) -//TestFillSecondaryGlobalOpts tests valid and invalid data on fillSecondaryGlobalOpts-function +// TestFillSecondaryGlobalOpts tests valid and invalid data on fillSecondaryGlobalOpts-function func TestFillSecondaryGlobalOpts(t *testing.T) { //secondaryRepoTestCase defines a struct for test cases type secondaryRepoTestCase struct { Opts secondaryRepoOptions DstGOpts GlobalOptions + FromRepo bool } //validSecondaryRepoTestCases is a list with test cases that must pass @@ -28,6 +29,7 @@ func TestFillSecondaryGlobalOpts(t *testing.T) { Repo: "backupDst", password: "secretDst", }, + FromRepo: true, }, { // Test if RepositoryFile and PasswordFile are parsed correctly. @@ -40,6 +42,7 @@ func TestFillSecondaryGlobalOpts(t *testing.T) { password: "secretDst", PasswordFile: "passwordFileDst", }, + FromRepo: true, }, { // Test if RepositoryFile and PasswordCommand are parsed correctly. @@ -52,6 +55,42 @@ func TestFillSecondaryGlobalOpts(t *testing.T) { password: "secretDst", PasswordCommand: "echo secretDst", }, + FromRepo: true, + }, + { + // Test if LegacyRepo and Password are parsed correctly. + Opts: secondaryRepoOptions{ + LegacyRepo: "backupDst", + password: "secretDst", + }, + DstGOpts: GlobalOptions{ + Repo: "backupDst", + password: "secretDst", + }, + }, + { + // Test if LegacyRepositoryFile and LegacyPasswordFile are parsed correctly. + Opts: secondaryRepoOptions{ + LegacyRepositoryFile: "backupDst", + LegacyPasswordFile: "passwordFileDst", + }, + DstGOpts: GlobalOptions{ + RepositoryFile: "backupDst", + password: "secretDst", + PasswordFile: "passwordFileDst", + }, + }, + { + // Test if LegacyRepositoryFile and LegacyPasswordCommand are parsed correctly. + Opts: secondaryRepoOptions{ + LegacyRepositoryFile: "backupDst", + LegacyPasswordCommand: "echo secretDst", + }, + DstGOpts: GlobalOptions{ + RepositoryFile: "backupDst", + password: "secretDst", + PasswordCommand: "echo secretDst", + }, }, } @@ -96,6 +135,20 @@ func TestFillSecondaryGlobalOpts(t *testing.T) { Repo: "backupDst", }, }, + { + // Test must fail as current and legacy options are mixed + Opts: secondaryRepoOptions{ + Repo: "backupDst", + LegacyRepo: "backupDst", + }, + }, + { + // Test must fail as current and legacy options are mixed + Opts: secondaryRepoOptions{ + Repo: "backupDst", + LegacyPasswordCommand: "notEmpty", + }, + }, } //gOpts defines the Global options used in the secondary repository tests @@ -119,14 +172,15 @@ func TestFillSecondaryGlobalOpts(t *testing.T) { // Test all valid cases for _, testCase := range validSecondaryRepoTestCases { - DstGOpts, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination") + DstGOpts, isFromRepo, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination") rtest.OK(t, err) rtest.Equals(t, DstGOpts, testCase.DstGOpts) + rtest.Equals(t, isFromRepo, testCase.FromRepo) } // Test all invalid cases for _, testCase := range invalidSecondaryRepoTestCases { - _, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination") + _, _, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination") rtest.Assert(t, err != nil, "Expected error, but function did not return an error") } } diff --git a/doc.go b/doc.go index ff0a0796f36..31527a78daa 100644 --- a/doc.go +++ b/doc.go @@ -1,6 +1,6 @@ // Package restic gives a (very brief) introduction to the structure of source code. // -// Overview +// # Overview // // The packages are structured so that cmd/ contains the main package for the // restic binary, and internal/ contains almost all code in library form. We've diff --git a/doc/020_installation.rst b/doc/020_installation.rst index d02feb3d81e..9f6ffa141c5 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -274,7 +274,7 @@ From Source *********** restic is written in the Go programming language and you need at least -Go version 1.14. Building restic may also work with older versions of Go, +Go version 1.15. Building restic may also work with older versions of Go, but that's not supported. See the `Getting started `__ guide of the Go project for instructions how to install Go. @@ -339,6 +339,13 @@ Example for using sudo to write a bash completion script directly to the system- $ sudo ./restic generate --bash-completion /etc/bash_completion.d/restic writing bash completion file to /etc/bash_completion.d/restic +Example for using sudo to write a zsh completion script directly to the system-wide location: + +.. code-block:: console + + $ sudo ./restic generate --zsh-completion /usr/local/share/zsh/site-functions/_restic + writing zsh completion file to /usr/local/share/zsh/site-functions/_restic + .. note:: The path for the ``--bash-completion`` option may vary depending on the operating system used, e.g. ``/usr/share/bash-completion/completions/restic`` in Debian and derivatives. Please look up the correct path in the appropriate diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index d6fb830c6de..dbb641746a3 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -14,18 +14,26 @@ Preparing a new repository ########################## -The place where your backups will be saved is called a "repository". +The place where your backups will be saved is called a "repository". This is +simply a directory containing a set of subdirectories and files created by +restic to store your backups, some corresponding metadata and encryption keys. + +To access the repository, a password (also called a key) must be specified. A +repository can hold multiple keys that can all be used to access the repository. + This chapter explains how to create ("init") such a repository. The repository can be stored locally, or on some remote server or service. We'll first cover using a local repository; the remaining sections of this chapter cover all the other options. You can skip to the next chapter once you've read the relevant section here. -For automated backups, restic accepts the repository location in the +For automated backups, restic supports specifying the repository location in the environment variable ``RESTIC_REPOSITORY``. Restic can also read the repository location from a file specified via the ``--repository-file`` option or the -environment variable ``RESTIC_REPOSITORY_FILE``. For the password, several -options exist: +environment variable ``RESTIC_REPOSITORY_FILE``. + +For automating the supply of the repository password to restic, several options +exist: * Setting the environment variable ``RESTIC_PASSWORD`` @@ -35,6 +43,26 @@ options exist: * Configuring a program to be called when the password is needed via the option ``--password-command`` or the environment variable ``RESTIC_PASSWORD_COMMAND`` + +The ``init`` command has an option called ``--repository-version`` which can +be used to explicitly set the version of the new repository. By default, the +current stable version is used (see table below). The alias ``latest`` will +always resolve to the latest repository version. Have a look at the `design +documentation `__ +for more details. + +The below table shows which restic version is required to use a certain +repository version, as well as notable features introduced in the various +versions. + ++--------------------+-------------------------+---------------------+------------------+ +| Repository version | Required restic version | Major new features | Comment | ++====================+=========================+=====================+==================+ +| ``1`` | Any | | Current default | ++--------------------+-------------------------+---------------------+------------------+ +| ``2`` | 0.14.0 or newer | Compression support | | ++--------------------+-------------------------+---------------------+------------------+ + Local ***** @@ -67,9 +95,9 @@ SFTP **** In order to backup data via SFTP, you must first set up a server with -SSH and let it know your public key. Passwordless login is really -important since restic fails to connect to the repository if the server -prompts for credentials. +SSH and let it know your public key. Passwordless login is important +since automatic backups are not possible if the server prompts for +credentials. Once the server is configured, the setup of the SFTP repository can simply be achieved by changing the URL scheme in the ``init`` command: @@ -148,7 +176,7 @@ SFTP connection, you can specify the command to be run with the option .. note:: Please be aware that sftp servers close connections when no data is received by the client. This can happen when restic is processing huge amounts of unchanged data. To avoid this issue add the following lines - to the client’s .ssh/config file: + to the client's .ssh/config file: :: @@ -167,7 +195,7 @@ scheme like this: .. code-block:: console - $ restic -r rest:http://host:8000/ + $ restic -r rest:http://host:8000/ init Depending on your REST server setup, you can use HTTPS protocol, password protection, multiple repositories or any combination of @@ -176,9 +204,9 @@ are some more examples: .. code-block:: console - $ restic -r rest:https://host:8000/ - $ restic -r rest:https://user:pass@host:8000/ - $ restic -r rest:https://user:pass@host:8000/my_backup_repo/ + $ restic -r rest:https://host:8000/ init + $ restic -r rest:https://user:pass@host:8000/ init + $ restic -r rest:https://user:pass@host:8000/my_backup_repo/ init If you use TLS, restic will use the system's CA certificates to verify the server certificate. When the verification fails, restic refuses to proceed and @@ -226,6 +254,9 @@ parameter like ``-o s3.region="us-east-1"``. If the region is not specified, the default region is used. Afterwards, the S3 server (at least for AWS, ``s3.amazonaws.com``) will redirect restic to the correct endpoint. +When using temporary credentials make sure to include the session token via +then environment variable ``AWS_SESSION_TOKEN``. + Until version 0.8.0, restic used a default prefix of ``restic``, so the files in the bucket were placed in a directory named ``restic``. If you want to access a repository created with an older version of restic, specify the path @@ -482,6 +513,13 @@ account name and key as follows: $ export AZURE_ACCOUNT_NAME= $ export AZURE_ACCOUNT_KEY= +or + +.. code-block:: console + + $ export AZURE_ACCOUNT_NAME= + $ export AZURE_ACCOUNT_SAS= + Afterwards you can initialize a repository in a container called ``foo`` in the root path like this: @@ -501,6 +539,10 @@ established. Google Cloud Storage ******************** +.. note:: Google Cloud Storage is not the same service as Google Drive - to use + the latter, please see :ref:`other-services` for instructions on using + the rclone backend. + Restic supports Google Cloud Storage as a backend and connects via a `service account`_. For normal restic operation, the service account must have the @@ -547,16 +589,18 @@ repository in the bucket ``foo`` at the root path: enter password for new repository: enter password again: - created restic repository bde47d6254 at gs:foo2/ + created restic repository bde47d6254 at gs:foo/ [...] The number of concurrent connections to the GCS service can be set with the ``-o gs.connections=10`` switch. By default, at most five parallel connections are established. -.. _service account: https://cloud.google.com/storage/docs/authentication#service_accounts -.. _create a service account key: https://cloud.google.com/storage/docs/authentication#generating-a-private-key -.. _default authentication material: https://developers.google.com/identity/protocols/application-default-credentials +.. _service account: https://cloud.google.com/iam/docs/service-accounts +.. _create a service account key: https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-console +.. _default authentication material: https://cloud.google.com/docs/authentication/production + +.. _other-services: Other Services via rclone ************************* @@ -566,7 +610,7 @@ store data there. First, you need to install and `configure`_ rclone. The general backend specification format is ``rclone::``, the ``:`` component will be directly passed to rclone. When you configure a remote named ``foo``, you can then call restic as follows to -initiate a new repository in the path ``bar`` in the repo: +initiate a new repository in the path ``bar`` in the remote ``foo``: .. code-block:: console @@ -693,3 +737,55 @@ On MSYS2, you can install ``winpty`` as follows: $ pacman -S winpty $ winpty restic -r /srv/restic-repo init + +Group accessible repositories +***************************** + +Since restic version 0.14 local and SFTP repositories can be made +accessible to members of a system group. To control this we have to change +the group permissions of the top-level ``config`` file and restic will use +this as a hint to determine what permissions to apply to newly created +files. By default ``restic init`` sets repositories up to be group +inaccessible. + +In order to give group members read-only access we simply add the read +permission bit to all repository files with ``chmod``: + +.. code-block:: console + + $ chmod -R g+r /srv/restic-repo + +This serves two purposes: 1) it sets the read permission bit on the +repository config file triggering restic's logic to create new files as +group accessible and 2) it actually allows the group read access to the +files. + +.. note:: By default files on Unix systems are created with a user's + primary group as defined by the gid (group id) field in + ``/etc/passwd``. See `passwd(5) + `_. + +For read-write access things are a bit more complicated. When users other +than the repository creator add new files in the repository they will be +group-owned by this user's primary group by default, not that of the +original repository owner, meaning the original creator wouldn't have +access to these files. That's hardly what you'd want. + +To make this work we can employ the help of the ``setgid`` permission bit +available on Linux and most other Unix systems. This permission bit makes +newly created directories inherit both the group owner (gid) and setgid bit +from the parent directory. Setting this bit requires root but since it +propagates down to any new directories we only have to do this priviledged +setup once: + +.. code-block:: console + + # find /srv/restic-repo -type d -exec chmod g+s '{}' \; + $ chmod -R g+rw /srv/restic-repo + +This sets the ``setgid`` bit on all existing directories in the repository +and then grants read/write permissions for group access. + +.. note:: To manage who has access to the repository you can use + ``usermod`` on Linux systems, to change which group controls + repository access ``chgrp -R`` is your friend. diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 80a14a87a85..891c6820dd7 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -191,7 +191,7 @@ Dry Runs ******** You can perform a backup in dry run mode to see what would happen without -modifying the repo. +modifying the repository. - ``--dry-run``/``-n`` Report what would be done, without writing to the repository @@ -202,7 +202,7 @@ Combined with ``--verbose``, you can see a list of changes: $ restic -r /srv/restic-repo backup ~/work --dry-run -vv | grep "added" modified /plan.txt, saved in 0.000s (9.110 KiB added) modified /archive.tar.gz, saved in 0.140s (25.542 MiB added) - Would be added to the repo: 25.551 MiB + Would be added to the repository: 25.551 MiB Excluding Files *************** @@ -212,7 +212,7 @@ the exclude options are: - ``--exclude`` Specified one or more times to exclude one or more items - ``--iexclude`` Same as ``--exclude`` but ignores the case of paths -- ``--exclude-caches`` Specified once to exclude folders containing a special file +- ``--exclude-caches`` Specified once to exclude folders containing `this special file `__ - ``--exclude-file`` Specified one or more times to exclude items listed in a given file - ``--iexclude-file`` Same as ``exclude-file`` but ignores cases like in ``--iexclude`` - ``--exclude-if-present foo`` Specified one or more times to exclude a folder's content if it contains a file called ``foo`` (optionally having a given header, no wildcards for the file name supported) @@ -346,12 +346,12 @@ option: $ restic -r /srv/restic-repo backup ~/work --exclude-larger-than 1M -This excludes files in ``~/work`` which are larger than 1 MB from the backup. +This excludes files in ``~/work`` which are larger than 1 MiB from the backup. The default unit for the size value is bytes, so e.g. ``--exclude-larger-than 2048`` -would exclude files larger than 2048 bytes (2 kilobytes). To specify other units, -suffix the size value with one of ``k``/``K`` for kilobytes, ``m``/``M`` for megabytes, -``g``/``G`` for gigabytes and ``t``/``T`` for terabytes (e.g. ``1k``, ``10K``, ``20m``, +would exclude files larger than 2048 bytes (2 KiB). To specify other units, +suffix the size value with one of ``k``/``K`` for KiB (1024 bytes), ``m``/``M`` for MiB (1024^2 bytes), +``g``/``G`` for GiB (1024^3 bytes) and ``t``/``T`` for TiB (1024^4 bytes), e.g. ``1k``, ``10K``, ``20m``, ``20M``, ``30g``, ``30G``, ``2t`` or ``2T``). Including Files @@ -552,12 +552,15 @@ environment variables. The following lists these environment variables: RESTIC_PASSWORD_COMMAND Command printing the password for the repository to stdout RESTIC_KEY_HINT ID of key to try decrypting first, before other keys RESTIC_CACHE_DIR Location of the cache directory + RESTIC_COMPRESSION Compression mode (only available for repository format version 2) RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated + RESTIC_PACK_SIZE Target size for pack files TMPDIR Location for temporary files AWS_ACCESS_KEY_ID Amazon S3 access key ID AWS_SECRET_ACCESS_KEY Amazon S3 secret access key + AWS_SESSION_TOKEN Amazon S3 temporary session token AWS_DEFAULT_REGION Amazon S3 default region AWS_PROFILE Amazon credentials profile (alternative to specifying key and region) AWS_SHARED_CREDENTIALS_FILE Location of the AWS CLI shared credentials file (default: ~/.aws/credentials) @@ -593,6 +596,7 @@ environment variables. The following lists these environment variables: AZURE_ACCOUNT_NAME Account name for Azure AZURE_ACCOUNT_KEY Account key for Azure + AZURE_ACCOUNT_SAS Shared access signatures (SAS) for Azure GOOGLE_PROJECT_ID Project ID for Google Cloud Storage GOOGLE_APPLICATION_CREDENTIALS Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 8ba154f30b6..8f702bc6cf5 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -90,7 +90,7 @@ example from a local to a remote repository, you can use the ``copy`` command: .. code-block:: console - $ restic -r /srv/restic-repo copy --repo2 /srv/restic-repo-copy + $ restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo repository d6504c63 opened successfully, password is correct repository 3dd0878c opened successfully, password is correct @@ -117,17 +117,17 @@ be skipped by later copy runs. both the source and destination repository, *may occupy up to twice their space* in the destination repository. See below for how to avoid this. -The destination repository is specified with ``--repo2`` or can be read -from a file specified via ``--repository-file2``. Both of these options -can also set as environment variables ``$RESTIC_REPOSITORY2`` or -``$RESTIC_REPOSITORY_FILE2`` respectively. For the destination repository -the password can be read from a file ``--password-file2`` or from a command -``--password-command2``. -Alternatively the environment variables ``$RESTIC_PASSWORD_COMMAND2`` and -``$RESTIC_PASSWORD_FILE2`` can be used. It is also possible to directly -pass the password via ``$RESTIC_PASSWORD2``. The key which should be used -for decryption can be selected by passing its ID via the flag ``--key-hint2`` -or the environment variable ``$RESTIC_KEY_HINT2``. +The source repository is specified with ``--from-repo`` or can be read +from a file specified via ``--from-repository-file``. Both of these options +can also be set as environment variables ``$RESTIC_FROM_REPOSITORY`` or +``$RESTIC_FROM_REPOSITORY_FILE``, respectively. For the destination repository +the password can be read from a file ``--from-password-file`` or from a command +``--from-password-command``. +Alternatively the environment variables ``$RESTIC_FROM_PASSWORD_COMMAND`` and +``$RESTIC_FROM_PASSWORD_FILE`` can be used. It is also possible to directly +pass the password via ``$RESTIC_FROM_PASSWORD``. The key which should be used +for decryption can be selected by passing its ID via the flag ``--from-key-hint`` +or the environment variable ``$RESTIC_FROM_KEY_HINT``. .. note:: In case the source and destination repository use the same backend, the configuration options and environment variables used to configure the @@ -298,3 +298,29 @@ a file size value the following command may be used: $ restic -r /srv/restic-repo check --read-data-subset=50M $ restic -r /srv/restic-repo check --read-data-subset=10G + + +Upgrading the repository format version +======================================= + +Repositories created using earlier restic versions use an older repository +format version and have to be upgraded to allow using all new features. +Upgrading must be done explicitly as a newer repository version increases the +minimum restic version required to access the repository. For example the +repository format version 2 is only readable using restic 0.14.0 or newer. + +Upgrading to repository version 2 is a two step process: first run +``migrate upgrade_repo_v2`` which will check the repository integrity and +then upgrade the repository version. Repository problems must be corrected +before the migration will be possible. After the migration is complete, run +``prune`` to compress the repository metadata. To limit the amount of data +rewritten in at once, you can use the ``prune --max-repack-size size`` +parameter, see :ref:`customize-pruning` for more details. + +File contents stored in the repository will not be rewritten, data from new +backups will be compressed. Over time more and more of the repository will +be compressed. To speed up this process and compress all not yet compressed +data, you can run ``prune --repack-uncompressed``. When you plan to create +your backups with maximum compression, you should also add the +``--compression max`` flag to the prune command. For already backed up data, +the compression level cannot be changed later on. diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst new file mode 100644 index 00000000000..ecf2bebbb7e --- /dev/null +++ b/doc/047_tuning_backup_parameters.rst @@ -0,0 +1,80 @@ +.. + Normally, there are no heading levels assigned to certain characters as the structure is + determined from the succession of headings. However, this convention is used in Python’s + Style Guide for documenting which you may follow: + # with overline, for parts + * for chapters + = for sections + - for subsections + ^ for subsubsections + " for paragraphs + +######################## +Tuning Backup Parameters +######################## + +Restic offers a few parameters that allow tuning the backup. The default values should +work well in general although specific use cases can benefit from different non-default +values. As the restic commands evolve over time, the optimal value for each parameter +can also change across restic versions. + + +Backend Connections +=================== + +Restic uses a global limit for the number of concurrent connections to a backend. +This limit can be configured using ``-o .connections=5``, for example for +the REST backend the parameter would be ``-o rest.connections=5``. By default restic uses +``5`` connections for each backend, except for the local backend which uses a limit of ``2``. +The defaults should work well in most cases. For high-latency backends it can be beneficial +to increase the number of connections. Please be aware that this increases the resource +consumption of restic and that a too high connection count *will degrade performance*. + + +CPU Usage +========= + +By default, restic uses all available CPU cores. You can set the environment variable +`GOMAXPROCS` to limit the number of used CPU cores. For example to use a single CPU core, +use `GOMAXPROCS=1`. Limiting the number of usable CPU cores, can slightly reduce the memory +usage of restic. + + +Compression +=========== + +For a repository using at least repository format version 2, you can configure how data +is compressed with the option ``--compression``. It can be set to ``auto`` (the default, +which will compress very fast), ``max`` (which will trade backup speed and CPU usage for +slightly better compression), or ``off`` (which disables compression). Each setting is +only applied for the single run of restic. The option can also be set via the environment +variable ``RESTIC_COMPRESSION``. + + +Pack Size +========= + +In certain instances, such as very large repositories (in the TiB range) or very fast +upload connections, it is desirable to use larger pack sizes to reduce the number of +files in the repository and improve upload performance. Notable examples are OpenStack +Swift and some Google Drive Team accounts, where there are hard limits on the total +number of files. Larger pack sizes can also improve the backup speed for a repository +stored on a local HDD. This can be achieved by either using the ``--pack-size`` option +or defining the ``$RESTIC_PACK_SIZE`` environment variable. Restic currently defaults +to a 16 MiB pack size. + +The side effect of increasing the pack size is requiring more disk space for temporary pack +files created before uploading. The space must be available in the system default temp +directory, unless overwritten by setting the ``$TMPDIR`` environment variable. In addition, +depending on the backend the memory usage can also increase by a similar amount. Restic +requires temporary space according to the pack size, multiplied by the number +of backend connections plus one. For example, if the backend uses 5 connections (the default +for most backends), with a target pack size of 64 MiB, you'll need a *minimum* of 384 MiB +of space in the temp directory. A bit of tuning may be required to strike a balance between +resource usage at the backup client and the number of pack files in the repository. + +Note that larger pack files increase the chance that the temporary pack files are written +to disk. An operating system usually caches file write operations in memory and writes +them to disk after a short delay. As larger pack files take longer to upload, this +increases the chance of these files being written to disk. This can increase disk wear +for SSDs. diff --git a/doc/050_restore.rst b/doc/050_restore.rst index 279397e8377..c7f6c0f28f7 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -96,7 +96,7 @@ the data directly. This can be achieved by using the `dump` command, like this: If you have saved multiple different things into the same repo, the ``latest`` snapshot may not be the right one. For example, consider the following -snapshots in a repo: +snapshots in a repository: .. code-block:: console diff --git a/doc/060_forget.rst b/doc/060_forget.rst index 2593716100a..a4205de7545 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -212,12 +212,13 @@ The ``forget`` command accepts the following policy options: .. note:: Specifying ``--keep-tag ''`` will match untagged snapshots only. -When ``forget`` is run with a policy, restic loads the list of all snapshots, -then groups these by host name and list of directories. The grouping options can -be set with ``--group-by``, to e.g. group snapshots by only paths and tags use -``--group-by paths,tags``. The policy is then applied to each group of snapshots -separately. This is a safety feature to prevent accidental removal of unrelated -backup sets. +When ``forget`` is run with a policy, restic first loads the list of all snapshots +and groups them by their host name and paths. The grouping options can be set with +``--group-by``, e.g. using ``--group-by paths,tags`` to instead group snapshots by +paths and tags. The policy is then applied to each group of snapshots individually. +This is a safety feature to prevent accidental removal of unrelated backup sets. To +disable grouping and apply the policy to all snapshots regardless of their host, +paths and tags, use ``--group-by ''`` (that is, an empty value to ``--group-by``). Additionally, you can restrict the policy to only process snapshots which have a particular hostname with the ``--host`` parameter, or tags with the ``--tag`` @@ -387,6 +388,8 @@ the specified duration: if ``forget --keep-within 7d`` is run 8 days after the last good snapshot, then the attacker can still use that opportunity to remove all legitimate snapshots. +.. _customize-pruning: + Customize pruning ***************** @@ -415,9 +418,9 @@ The ``prune`` command accepts the following options: * As an absolute size (e.g. ``200M``). If you want to minimize the space used by your repository, pass ``0`` to this option. - * As a size relative to the total repo size (e.g. ``10%``). This means that - after prune, at most ``10%`` of the total data stored in the repo may be - unused data. If the repo after prune has as size of 500MB, then at most + * As a size relative to the total repository size (e.g. ``10%``). This means that + after prune, at most ``10%`` of the total data stored in the repository may be + unused data. If the repository after prune has a size of 500MB, then at most 50MB may be unused. * If the string ``unlimited`` is passed, there is no limit for partly unused files. This means that as long as some data is still used within @@ -443,3 +446,31 @@ The ``prune`` command accepts the following options: - ``--dry-run`` only show what ``prune`` would do. - ``--verbose`` increased verbosity shows additional statistics for ``prune``. + + +Recovering from "no free space" errors +************************************** + +In some cases when a repository has grown large enough to fill up all disk space or the +allocated quota, then ``prune`` might fail to free space. ``prune`` works in such a way +that a repository remains usable no matter at which point the command is interrupted. +However, this also means that ``prune`` requires some scratch space to work. + +In most cases it is sufficient to instruct ``prune`` to use as little scratch space as +possible by running it as ``prune --max-repack-size 0``. Note that for restic versions +before 0.13.0 ``prune --max-repack-size 1`` must be used. Obviously, this can only work +if several snapshots have been removed using ``forget`` before. This then allows the +``prune`` command to actually remove data from the repository. If the command succeeds, +but there is still little free space, then remove a few more snapshots and run ``prune`` again. + +If ``prune`` fails to complete, then ``prune --unsafe-recover-no-free-space SOME-ID`` +is available as a method of last resort. It allows prune to work with little to no free +space. However, a **failed** ``prune`` run can cause the repository to become +**temporarily unusable**. Therefore, make sure that you have a stable connection to the +repository storage, before running this command. In case the command fails, it may become +necessary to manually remove all files from the `index/` folder of the repository and +run `rebuild-index` afterwards. + +To prevent accidental usages of the ``--unsafe-recover-no-free-space`` option it is +necessary to first run ``prune --unsafe-recover-no-free-space SOME-ID`` and then replace +``SOME-ID`` with the requested ID. diff --git a/doc/080_examples.rst b/doc/080_examples.rst index 5c33dfc6170..74ce3c0b34f 100644 --- a/doc/080_examples.rst +++ b/doc/080_examples.rst @@ -202,11 +202,12 @@ configuration of restic will be placed into environment variables. This will include sensitive information, such as your AWS secret and repository password. Therefore, make sure the next commands **do not** end up in your shell's history file. Adjust the contents of the environment variables to fit your -bucket's name and your user's API credentials. +bucket's name, region, and your user's API credentials. .. code-block:: console $ unset HISTFILE + $ export AWS_DEFAULT_REGION="eu-west-1" $ export RESTIC_REPOSITORY="s3:https://s3.amazonaws.com/restic-demo" $ export AWS_ACCESS_KEY_ID="AKIAJAJSLTZCAZ4SRI5Q" $ export AWS_SECRET_ACCESS_KEY="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" diff --git a/doc/090_participating.rst b/doc/090_participating.rst index 7c7e0e72bf6..00a38797427 100644 --- a/doc/090_participating.rst +++ b/doc/090_participating.rst @@ -14,18 +14,12 @@ Participating ############# -********* -Debugging -********* +********** +Debug Logs +********** -The program can be built with debug support like this: - -.. code-block:: console - - $ go run build.go -tags debug - -Afterwards, extensive debug messages are written to the file in -environment variable ``DEBUG_LOG``, e.g.: +Set the environment variable ``DEBUG_LOG`` to let restic write extensive debug +messages to the specified filed, e.g.: .. code-block:: console @@ -66,6 +60,21 @@ statements originating in functions that match the pattern ``*unlock*`` $ DEBUG_FUNCS=*unlock* restic check +********* +Debugging +********* + +The program can be built with debug support like this: + +.. code-block:: console + + $ go run build.go -tags debug + +This will make the ``restic debug `` available which can be used to +inspect internal data structures. In addition, this enables profiling support +which can help with investigation performance and memory usage issues. + + ************ Contributing ************ diff --git a/doc/bash-completion.sh b/doc/bash-completion.sh index db74119dbe2..f76ae5a7cac 100644 --- a/doc/bash-completion.sh +++ b/doc/bash-completion.sh @@ -2,7 +2,7 @@ __restic_debug() { - if [[ -n ${BASH_COMP_DEBUG_FILE} ]]; then + if [[ -n ${BASH_COMP_DEBUG_FILE:-} ]]; then echo "$*" >> "${BASH_COMP_DEBUG_FILE}" fi } @@ -51,7 +51,8 @@ __restic_handle_go_custom_completion() # Prepare the command to request completions for the program. # Calling ${words[0]} instead of directly restic allows to handle aliases args=("${words[@]:1}") - requestComp="${words[0]} __completeNoDesc ${args[*]}" + # Disable ActiveHelp which is not supported for bash completion v1 + requestComp="RESTIC_ACTIVE_HELP=0 ${words[0]} __completeNoDesc ${args[*]}" lastParam=${words[$((${#words[@]}-1))]} lastChar=${lastParam:$((${#lastParam}-1)):1} @@ -77,7 +78,7 @@ __restic_handle_go_custom_completion() directive=0 fi __restic_debug "${FUNCNAME[0]}: the completion directive is: ${directive}" - __restic_debug "${FUNCNAME[0]}: the completions are: ${out[*]}" + __restic_debug "${FUNCNAME[0]}: the completions are: ${out}" if [ $((directive & shellCompDirectiveError)) -ne 0 ]; then # Error code. No completion. @@ -103,7 +104,7 @@ __restic_handle_go_custom_completion() local fullFilter filter filteringCmd # Do not use quotes around the $out variable or else newline # characters will be kept. - for filter in ${out[*]}; do + for filter in ${out}; do fullFilter+="$filter|" done @@ -112,9 +113,9 @@ __restic_handle_go_custom_completion() $filteringCmd elif [ $((directive & shellCompDirectiveFilterDirs)) -ne 0 ]; then # File completion for directories only - local subDir + local subdir # Use printf to strip any trailing newline - subdir=$(printf "%s" "${out[0]}") + subdir=$(printf "%s" "${out}") if [ -n "$subdir" ]; then __restic_debug "Listing directories in $subdir" __restic_handle_subdirs_in_dir_flag "$subdir" @@ -125,7 +126,7 @@ __restic_handle_go_custom_completion() else while IFS='' read -r comp; do COMPREPLY+=("$comp") - done < <(compgen -W "${out[*]}" -- "$cur") + done < <(compgen -W "${out}" -- "$cur") fi } @@ -165,13 +166,19 @@ __restic_handle_reply() PREFIX="" cur="${cur#*=}" ${flags_completion[${index}]} - if [ -n "${ZSH_VERSION}" ]; then + if [ -n "${ZSH_VERSION:-}" ]; then # zsh completion needs --flag= prefix eval "COMPREPLY=( \"\${COMPREPLY[@]/#/${flag}=}\" )" fi fi fi - return 0; + + if [[ -z "${flag_parsing_disabled}" ]]; then + # If flag parsing is enabled, we have completed the flags and can return. + # If flag parsing is disabled, we may not know all (or any) of the flags, so we fallthrough + # to possibly call handle_go_custom_completion. + return 0; + fi ;; esac @@ -210,13 +217,13 @@ __restic_handle_reply() fi if [[ ${#COMPREPLY[@]} -eq 0 ]]; then - if declare -F __restic_custom_func >/dev/null; then - # try command name qualified custom func - __restic_custom_func - else - # otherwise fall back to unqualified for compatibility - declare -F __custom_func >/dev/null && __custom_func - fi + if declare -F __restic_custom_func >/dev/null; then + # try command name qualified custom func + __restic_custom_func + else + # otherwise fall back to unqualified for compatibility + declare -F __custom_func >/dev/null && __custom_func + fi fi # available in bash-completion >= 2, not always present on macOS @@ -250,7 +257,7 @@ __restic_handle_flag() # if a command required a flag, and we found it, unset must_have_one_flag() local flagname=${words[c]} - local flagvalue + local flagvalue="" # if the word contained an = if [[ ${words[c]} == *"="* ]]; then flagvalue=${flagname#*=} # take in as flagvalue after the = @@ -269,7 +276,7 @@ __restic_handle_flag() # keep flag value with flagname as flaghash # flaghash variable is an associative array which is only supported in bash > 3. - if [[ -z "${BASH_VERSION}" || "${BASH_VERSINFO[0]}" -gt 3 ]]; then + if [[ -z "${BASH_VERSION:-}" || "${BASH_VERSINFO[0]:-}" -gt 3 ]]; then if [ -n "${flagvalue}" ] ; then flaghash[${flagname}]=${flagvalue} elif [ -n "${words[ $((c+1)) ]}" ] ; then @@ -281,7 +288,7 @@ __restic_handle_flag() # skip the argument to a two word flag if [[ ${words[c]} != *"="* ]] && __restic_contains_word "${words[c]}" "${two_word_flags[@]}"; then - __restic_debug "${FUNCNAME[0]}: found a flag ${words[c]}, skip the next argument" + __restic_debug "${FUNCNAME[0]}: found a flag ${words[c]}, skip the next argument" c=$((c+1)) # if we are looking for a flags value, don't show commands if [[ $c -eq $cword ]]; then @@ -341,7 +348,7 @@ __restic_handle_word() __restic_handle_command elif __restic_contains_word "${words[c]}" "${command_aliases[@]}"; then # aliashash variable is an associative array which is only supported in bash > 3. - if [[ -z "${BASH_VERSION}" || "${BASH_VERSINFO[0]}" -gt 3 ]]; then + if [[ -z "${BASH_VERSION:-}" || "${BASH_VERSINFO[0]:-}" -gt 3 ]]; then words[c]=${aliashash[${words[c]}]} __restic_handle_command else @@ -458,6 +465,8 @@ _restic_backup() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -471,6 +480,8 @@ _restic_backup() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -524,6 +535,8 @@ _restic_cache() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -537,6 +550,8 @@ _restic_cache() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -582,6 +597,8 @@ _restic_cat() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -595,6 +612,8 @@ _restic_cat() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -631,8 +650,6 @@ _restic_check() flags_with_completion=() flags_completion=() - flags+=("--check-unused") - local_nonpersistent_flags+=("--check-unused") flags+=("--help") flags+=("-h") local_nonpersistent_flags+=("--help") @@ -650,6 +667,8 @@ _restic_check() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -663,6 +682,8 @@ _restic_check() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -699,6 +720,26 @@ _restic_copy() flags_with_completion=() flags_completion=() + flags+=("--from-key-hint=") + two_word_flags+=("--from-key-hint") + local_nonpersistent_flags+=("--from-key-hint") + local_nonpersistent_flags+=("--from-key-hint=") + flags+=("--from-password-command=") + two_word_flags+=("--from-password-command") + local_nonpersistent_flags+=("--from-password-command") + local_nonpersistent_flags+=("--from-password-command=") + flags+=("--from-password-file=") + two_word_flags+=("--from-password-file") + local_nonpersistent_flags+=("--from-password-file") + local_nonpersistent_flags+=("--from-password-file=") + flags+=("--from-repo=") + two_word_flags+=("--from-repo") + local_nonpersistent_flags+=("--from-repo") + local_nonpersistent_flags+=("--from-repo=") + flags+=("--from-repository-file=") + two_word_flags+=("--from-repository-file") + local_nonpersistent_flags+=("--from-repository-file") + local_nonpersistent_flags+=("--from-repository-file=") flags+=("--help") flags+=("-h") local_nonpersistent_flags+=("--help") @@ -709,30 +750,10 @@ _restic_copy() local_nonpersistent_flags+=("--host") local_nonpersistent_flags+=("--host=") local_nonpersistent_flags+=("-H") - flags+=("--key-hint2=") - two_word_flags+=("--key-hint2") - local_nonpersistent_flags+=("--key-hint2") - local_nonpersistent_flags+=("--key-hint2=") - flags+=("--password-command2=") - two_word_flags+=("--password-command2") - local_nonpersistent_flags+=("--password-command2") - local_nonpersistent_flags+=("--password-command2=") - flags+=("--password-file2=") - two_word_flags+=("--password-file2") - local_nonpersistent_flags+=("--password-file2") - local_nonpersistent_flags+=("--password-file2=") flags+=("--path=") two_word_flags+=("--path") local_nonpersistent_flags+=("--path") local_nonpersistent_flags+=("--path=") - flags+=("--repo2=") - two_word_flags+=("--repo2") - local_nonpersistent_flags+=("--repo2") - local_nonpersistent_flags+=("--repo2=") - flags+=("--repository-file2=") - two_word_flags+=("--repository-file2") - local_nonpersistent_flags+=("--repository-file2") - local_nonpersistent_flags+=("--repository-file2=") flags+=("--tag=") two_word_flags+=("--tag") local_nonpersistent_flags+=("--tag") @@ -742,6 +763,8 @@ _restic_copy() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -755,6 +778,8 @@ _restic_copy() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -802,6 +827,8 @@ _restic_diff() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -815,6 +842,8 @@ _restic_diff() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -880,6 +909,8 @@ _restic_dump() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -893,6 +924,8 @@ _restic_dump() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -986,6 +1019,8 @@ _restic_find() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -999,6 +1034,8 @@ _restic_find() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1137,6 +1174,10 @@ _restic_forget() local_nonpersistent_flags+=("--max-repack-size=") flags+=("--repack-cacheable-only") local_nonpersistent_flags+=("--repack-cacheable-only") + flags+=("--repack-small") + local_nonpersistent_flags+=("--repack-small") + flags+=("--repack-uncompressed") + local_nonpersistent_flags+=("--repack-uncompressed") flags+=("--help") flags+=("-h") local_nonpersistent_flags+=("--help") @@ -1146,6 +1187,8 @@ _restic_forget() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1159,6 +1202,8 @@ _restic_forget() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1220,6 +1265,8 @@ _restic_generate() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1233,6 +1280,8 @@ _restic_generate() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1274,6 +1323,8 @@ _restic_help() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1287,6 +1338,8 @@ _restic_help() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1326,35 +1379,41 @@ _restic_init() flags+=("--copy-chunker-params") local_nonpersistent_flags+=("--copy-chunker-params") + flags+=("--from-key-hint=") + two_word_flags+=("--from-key-hint") + local_nonpersistent_flags+=("--from-key-hint") + local_nonpersistent_flags+=("--from-key-hint=") + flags+=("--from-password-command=") + two_word_flags+=("--from-password-command") + local_nonpersistent_flags+=("--from-password-command") + local_nonpersistent_flags+=("--from-password-command=") + flags+=("--from-password-file=") + two_word_flags+=("--from-password-file") + local_nonpersistent_flags+=("--from-password-file") + local_nonpersistent_flags+=("--from-password-file=") + flags+=("--from-repo=") + two_word_flags+=("--from-repo") + local_nonpersistent_flags+=("--from-repo") + local_nonpersistent_flags+=("--from-repo=") + flags+=("--from-repository-file=") + two_word_flags+=("--from-repository-file") + local_nonpersistent_flags+=("--from-repository-file") + local_nonpersistent_flags+=("--from-repository-file=") flags+=("--help") flags+=("-h") local_nonpersistent_flags+=("--help") local_nonpersistent_flags+=("-h") - flags+=("--key-hint2=") - two_word_flags+=("--key-hint2") - local_nonpersistent_flags+=("--key-hint2") - local_nonpersistent_flags+=("--key-hint2=") - flags+=("--password-command2=") - two_word_flags+=("--password-command2") - local_nonpersistent_flags+=("--password-command2") - local_nonpersistent_flags+=("--password-command2=") - flags+=("--password-file2=") - two_word_flags+=("--password-file2") - local_nonpersistent_flags+=("--password-file2") - local_nonpersistent_flags+=("--password-file2=") - flags+=("--repo2=") - two_word_flags+=("--repo2") - local_nonpersistent_flags+=("--repo2") - local_nonpersistent_flags+=("--repo2=") - flags+=("--repository-file2=") - two_word_flags+=("--repository-file2") - local_nonpersistent_flags+=("--repository-file2") - local_nonpersistent_flags+=("--repository-file2=") + flags+=("--repository-version=") + two_word_flags+=("--repository-version") + local_nonpersistent_flags+=("--repository-version") + local_nonpersistent_flags+=("--repository-version=") flags+=("--cacert=") two_word_flags+=("--cacert") flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1368,6 +1427,8 @@ _restic_init() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1425,6 +1486,8 @@ _restic_key() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1438,6 +1501,8 @@ _restic_key() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1483,6 +1548,8 @@ _restic_list() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1496,6 +1563,8 @@ _restic_list() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1561,6 +1630,8 @@ _restic_ls() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1574,6 +1645,8 @@ _restic_ls() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1623,6 +1696,8 @@ _restic_migrate() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1636,6 +1711,8 @@ _restic_migrate() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1692,19 +1769,25 @@ _restic_mount() two_word_flags+=("--path") local_nonpersistent_flags+=("--path") local_nonpersistent_flags+=("--path=") - flags+=("--snapshot-template=") - two_word_flags+=("--snapshot-template") - local_nonpersistent_flags+=("--snapshot-template") - local_nonpersistent_flags+=("--snapshot-template=") + flags+=("--path-template=") + two_word_flags+=("--path-template") + local_nonpersistent_flags+=("--path-template") + local_nonpersistent_flags+=("--path-template=") flags+=("--tag=") two_word_flags+=("--tag") local_nonpersistent_flags+=("--tag") local_nonpersistent_flags+=("--tag=") + flags+=("--time-template=") + two_word_flags+=("--time-template") + local_nonpersistent_flags+=("--time-template") + local_nonpersistent_flags+=("--time-template=") flags+=("--cacert=") two_word_flags+=("--cacert") flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1718,6 +1801,8 @@ _restic_mount() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1772,11 +1857,21 @@ _restic_prune() local_nonpersistent_flags+=("--max-unused=") flags+=("--repack-cacheable-only") local_nonpersistent_flags+=("--repack-cacheable-only") + flags+=("--repack-small") + local_nonpersistent_flags+=("--repack-small") + flags+=("--repack-uncompressed") + local_nonpersistent_flags+=("--repack-uncompressed") + flags+=("--unsafe-recover-no-free-space=") + two_word_flags+=("--unsafe-recover-no-free-space") + local_nonpersistent_flags+=("--unsafe-recover-no-free-space") + local_nonpersistent_flags+=("--unsafe-recover-no-free-space=") flags+=("--cacert=") two_word_flags+=("--cacert") flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1790,6 +1885,8 @@ _restic_prune() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1837,6 +1934,8 @@ _restic_rebuild-index() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1850,6 +1949,8 @@ _restic_rebuild-index() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1895,6 +1996,8 @@ _restic_recover() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1908,6 +2011,8 @@ _restic_recover() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -1995,6 +2100,8 @@ _restic_restore() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2008,6 +2115,8 @@ _restic_restore() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -2057,6 +2166,8 @@ _restic_self-update() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2070,6 +2181,8 @@ _restic_self-update() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -2143,6 +2256,8 @@ _restic_snapshots() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2156,6 +2271,8 @@ _restic_snapshots() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -2219,6 +2336,8 @@ _restic_stats() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2232,6 +2351,8 @@ _restic_stats() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -2303,6 +2424,8 @@ _restic_tag() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2316,6 +2439,8 @@ _restic_tag() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -2363,6 +2488,8 @@ _restic_unlock() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2376,6 +2503,8 @@ _restic_unlock() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -2421,6 +2550,8 @@ _restic_version() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2434,6 +2565,8 @@ _restic_version() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -2502,6 +2635,8 @@ _restic_root_command() flags+=("--cache-dir=") two_word_flags+=("--cache-dir") flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") flags+=("--help") flags+=("-h") local_nonpersistent_flags+=("--help") @@ -2519,6 +2654,8 @@ _restic_root_command() flags+=("--option=") two_word_flags+=("--option") two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") flags+=("--password-command=") two_word_flags+=("--password-command") flags+=("--password-file=") @@ -2553,6 +2690,7 @@ __start_restic() fi local c=0 + local flag_parsing_disabled= local flags=() local two_word_flags=() local local_nonpersistent_flags=() @@ -2562,8 +2700,8 @@ __start_restic() local command_aliases=() local must_have_one_flag=() local must_have_one_noun=() - local has_completion_function - local last_command + local has_completion_function="" + local last_command="" local nouns=() local noun_aliases=() diff --git a/doc/cache.rst b/doc/cache.rst index 30b601fafe9..7b1cabebb10 100644 --- a/doc/cache.rst +++ b/doc/cache.rst @@ -7,7 +7,7 @@ The location of the cache directory depends on the operating system and the environment; see :ref:`caching`. Each repository has its own cache sub-directory, consisting of the repository ID -which is chosen at ``init``. All cache directories for different repos are +which is chosen at ``init``. All cache directories for different repositories are independent of each other. Snapshots, Data and Indexes @@ -19,8 +19,8 @@ Snapshot, Data and Index files are cached in the sub-directories ``snapshots``, Expiry ====== -Whenever a cache directory for a repo is used, that directory's modification +Whenever a cache directory for a repository is used, that directory's modification timestamp is updated to the current time. By looking at the modification -timestamps of the repo cache directories it is easy to decide which directories +timestamps of the repository cache directories it is easy to decide which directories are old and haven't been used in a long time. Those are probably stale and can be removed. diff --git a/doc/design.rst b/doc/design.rst index 11c3046848c..17ab4c1b59d 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -30,9 +30,10 @@ All data is stored in a restic repository. A repository is able to store data of several different types, which can later be requested based on an ID. This so-called "storage ID" is the SHA-256 hash of the content of a file. All files in a repository are only written once and never -modified afterwards. This allows accessing and even writing to the -repository with multiple clients in parallel. Only the ``prune`` operation -removes data from the repository. +modified afterwards. Writing should occur atomically to prevent concurrent +operations from reading incomplete files. This allows accessing and even +writing to the repository with multiple clients in parallel. Only the ``prune`` +operation removes data from the repository. Repositories consist of several directories and a top-level file called ``config``. For all other files stored in the repository, the name for @@ -61,28 +62,30 @@ like the following: .. code:: json { - "version": 1, + "version": 2, "id": "5956a3f67a6230d4a92cefb29529f10196c7d92582ec305fd71ff6d331d6271b", "chunker_polynomial": "25b468838dcb75" } After decryption, restic first checks that the version field contains a -version number that it understands, otherwise it aborts. At the moment, -the version is expected to be 1. The field ``id`` holds a unique ID -which consists of 32 random bytes, encoded in hexadecimal. This uniquely -identifies the repository, regardless if it is accessed via SFTP or -locally. The field ``chunker_polynomial`` contains a parameter that is -used for splitting large files into smaller chunks (see below). +version number that it understands, otherwise it aborts. At the moment, the +version is expected to be 1 or 2. The list of changes in the repository +format is contained in the section "Changes" below. + +The field ``id`` holds a unique ID which consists of 32 random bytes, encoded +in hexadecimal. This uniquely identifies the repository, regardless if it is +accessed via a remote storage backend or locally. The field +``chunker_polynomial`` contains a parameter that is used for splitting large +files into smaller chunks (see below). Repository Layout ----------------- The ``local`` and ``sftp`` backends are implemented using files and directories stored in a file system. The directory layout is the same -for both backend types. +for both backend types and is also used for all other remote backends. -The basic layout of a repository stored in a ``local`` or ``sftp`` -backend is shown here: +The basic layout of a repository is shown here: :: @@ -108,8 +111,7 @@ backend is shown here: │ └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec └── tmp -A local repository can be initialized with the ``restic init`` command, -e.g.: +A local repository can be initialized with the ``restic init`` command, e.g.: .. code-block:: console @@ -185,40 +187,75 @@ After decryption, a Pack's header consists of the following elements: :: - Type_Blob1 || Length(EncryptedBlob1) || Hash(Plaintext_Blob1) || + Type_Blob1 || Data_Blob1 || [...] - Type_BlobN || Length(EncryptedBlobN) || Hash(Plaintext_Blobn) || + Type_BlobN || Data_BlobN || + +The Blob type field is a single byte. What follows it depends on the type. The +following Blob types are defined: + ++-----------+----------------------+-------------------------------------------------------------------------------+ +| Type | Meaning | Data | ++===========+======================+===============================================================================+ +| 0b00 | data blob | ``Length(encrypted_blob) || Hash(plaintext_blob)`` | ++-----------+----------------------+-------------------------------------------------------------------------------+ +| 0b01 | tree blob | ``Length(encrypted_blob) || Hash(plaintext_blob)`` | ++-----------+----------------------+-------------------------------------------------------------------------------+ +| 0b10 | compressed data blob | ``Length(encrypted_blob) || Length(plaintext_blob) || Hash(plaintext_blob)`` | ++-----------+----------------------+-------------------------------------------------------------------------------+ +| 0b11 | compressed tree blob | ``Length(encrypted_blob) || Length(plaintext_blob) || Hash(plaintext_blob)`` | ++-----------+----------------------+-------------------------------------------------------------------------------+ This is enough to calculate the offsets for all the Blobs in the Pack. -Length is the length of a Blob as a four byte integer in little-endian -format. The type field is a one byte field and labels the content of a -blob according to the following table: +The length fields are encoded as four byte integers in little-endian +format. In the Data column, ``Length(plaintext_blob)`` means the length +of the decrypted and uncompressed data a blob consists of. -+--------+-----------+ -| Type | Meaning | -+========+===========+ -| 0 | data | -+--------+-----------+ -| 1 | tree | -+--------+-----------+ +All other types are invalid, more types may be added in the future. The +compressed types are only valid for repository format version 2. Data and +tree blobs may be compressed with the zstandard compression algorithm. -All other types are invalid, more types may be added in the future. +In repository format version 1, data and tree blobs should be stored in +separate pack files. In version 2, they must be stored in separate files. +Compressed and non-compress blobs of the same type may be mixed in a pack +file. For reconstructing the index or parsing a pack without an index, first the last four bytes must be read in order to find the length of the header. Afterwards, the header can be read and parsed, which yields all plaintext hashes, types, offsets and lengths of all included blobs. +Unpacked Data Format +==================== + +Individual files for the index, locks or snapshots are encrypted +and authenticated like Data and Tree Blobs, so the outer structure is +``IV || Ciphertext || MAC`` again. In repository format version 1 the +plaintext always consists of a JSON document which must either be an +object or an array. + +Repository format version 2 adds support for compression. The plaintext +now starts with a header to indicate the encoding version to distinguish +it from plain JSON and to allow for further evolution of the storage format: +``encoding_version || data`` +The ``encoding_version`` field is encoded as one byte. +For backwards compatibility the encoding versions '[' (0x5b) and '{' (0x7b) +are used to mark that the whole plaintext (including the encoding version +byte) should treated as JSON document. + +For new data the encoding version is currently always ``2``. For that +version ``data`` contains a JSON document compressed using the zstandard +compression algorithm. + Indexing ======== Index files contain information about Data and Tree Blobs and the Packs they are contained in and store this information in the repository. When the local cached index is not accessible any more, the index files can -be downloaded and used to reconstruct the index. The files are encrypted -and authenticated like Data and Tree Blobs, so the outer structure is -``IV || Ciphertext || MAC`` again. The plaintext consists of a JSON -document like the following: +be downloaded and used to reconstruct the index. The file encoding is +described in the "Unpacked Data Format" section. The plaintext consists +of a JSON document like the following: .. code:: json @@ -234,18 +271,22 @@ document like the following: "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", "type": "data", "offset": 0, - "length": 25 - },{ + "length": 38, + // no 'uncompressed_length' as blob is not compressed + }, + { "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", "type": "tree", "offset": 38, - "length": 100 + "length": 112, + "uncompressed_length": 511, }, { "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", "type": "data", "offset": 150, - "length": 123 + "length": 123, + "uncompressed_length": 234, } ] }, [...] @@ -254,7 +295,11 @@ document like the following: This JSON document lists Packs and the blobs contained therein. In this example, the Pack ``73d04e61`` contains two data Blobs and one Tree -blob, the plaintext hashes are listed afterwards. +blob, the plaintext hashes are listed afterwards. The ``length`` field +corresponds to ``Length(encrypted_blob)`` in the pack file header. +Field ``uncompressed_length`` is only present for compressed blobs and +therefore is never present in version 1. It is set to the value of +``Length(blob)``. The field ``supersedes`` lists the storage IDs of index files that have been replaced with the current index file. This happens when index files @@ -271,7 +316,7 @@ Keys, Encryption and MAC All data stored by restic in the repository is encrypted with AES-256 in counter mode and authenticated using Poly1305-AES. For encrypting new data first 16 bytes are read from a cryptographically secure -pseudorandom number generator as a random nonce. This is used both as +pseudo-random number generator as a random nonce. This is used both as the IV for counter mode and the nonce for Poly1305. This operation needs three keys: A 32 byte for AES-256 for encryption, a 16 byte AES key and a 16 byte key for Poly1305. For details see the original paper `The @@ -349,8 +394,9 @@ Snapshots A snapshot represents a directory with all files and sub-directories at a given point in time. For each backup that is made, a new snapshot is -created. A snapshot is a JSON document that is stored in an encrypted -file below the directory ``snapshots`` in the repository. The filename +created. A snapshot is a JSON document that is stored in a file below +the directory ``snapshots`` in the repository. It uses the file encoding +described in the "Unpacked Data Format" section. The filename is the storage ID. This string is unique and used within restic to uniquely identify a snapshot. @@ -411,7 +457,7 @@ Blobs of data. The SHA-256 hashes of all Blobs are saved in an ordered list which then represents the content of the file. In order to relate these plaintext hashes to the actual location within -a Pack file , an index is used. If the index is not available, the +a Pack file, an index is used. If the index is not available, the header of all data Blobs can be read. Trees and Data @@ -516,8 +562,8 @@ time there must not be any other locks (exclusive and non-exclusive). There may be multiple non-exclusive locks in parallel. A lock is a file in the subdir ``locks`` whose filename is the storage -ID of the contents. It is encrypted and authenticated the same way as -other files in the repository and contains the following JSON structure: +ID of the contents. It is stored in the file encoding described in the +"Unpacked Data Format" section and contains the following JSON structure: .. code:: json @@ -544,6 +590,57 @@ detected, restic creates a new lock, waits, and checks if other locks appeared in the repository. Depending on the type of the other locks and the lock to be created, restic either continues or fails. +Read and Write Ordering +======================= +The repository format allows writing (e.g. backup) and reading (e.g. restore) +to happen concurrently. As the data for each snapshot in a repository spans +multiple files (snapshot, index and packs), it is necessary to follow certain +rules regarding the order in which files are read and written. These ordering +rules also guarantee that repository modifications always maintain a correct +repository even if the client or the storage backend crashes for example due +to a power cut or the (network) connection between both is interrupted. + +The correct order to access data in a repository is derived from the following +set of invariants that must be maintained at **any time** in a correct +repository. *Must* in the following is a strict requirement and will lead to +data loss if not followed. *Should* will require steps to fix a repository +(e.g. rebuilding the index) if not followed, but should not cause data loss. +*existing* means that the referenced data is **durably** stored in the repository. + +- A snapshot *must* only reference an existing tree blob. +- A reachable tree blob *must* only reference tree and data blobs that exist + (recursively). *Reachable* means that the tree blob is reachable starting from + a snapshot. +- An index *must* only reference valid blobs in existing packs. +- All blobs referenced by a snapshot *should* be listed in an index. + +This leads to the following recommended order to store data in a repository. +First, pack files, which contain data and tree blobs, must be written. Then the +indexes which reference blobs in these already written pack files. And finally +the corresponding snapshots. + +Note that there is no need for a specific write order of data and tree blobs +during a backup as the blobs only become referenced once the corresponding +snapshot is uploaded. + +Reading data should follow the opposite order compared to writing. Only once a +snapshot was written, it is guaranteed that all required data exists in the +repository. This especially means that the list of snapshots to read should be +collected before loading the repository index. The other way round can lead to +a race condition where a recently written snapshot is loaded but not its +accompanying index, which results in a failure to access the snapshot's tree +blob. + +For removing or rewriting data from a repository the following rules must be +followed, which are derived from the above invariants. + +- A client removing data *must* acquire an exclusive lock first to prevent + conflicts with other clients. +- A pack *must* be removed from the referencing index before it is deleted. +- Rewriting a pack *must* write the new pack, update the index (add an updated + index and delete the old one) and only then delete the old pack. + + Backups and Deduplication ========================= @@ -584,10 +681,10 @@ General assumptions: key management design, it is impossible to securely revoke a leaked key without re-encrypting the whole repository. - Advances in cryptography attacks against the cryptographic primitives used - by restic (i.e, AES-256-CTR-Poly1305-AES and SHA-256) have not occurred. Such + by restic (i.e., AES-256-CTR-Poly1305-AES and SHA-256) have not occurred. Such advances could render the confidentiality or integrity protections provided by restic useless. -- Sufficient advances in computing have not occurred to make bruteforce +- Sufficient advances in computing have not occurred to make brute-force attacks against restic's cryptographic protections feasible. The restic backup program guarantees the following: @@ -669,3 +766,11 @@ An adversary who has a leaked (decrypted) key for a repository could: only be done using the ``copy`` command, which moves the data into a new repository with a new master key, or by making a completely new repository and new backup. + +Changes +======= + +Repository Version 2 +-------------------- + + * Support compression for blobs (data/tree) and index / lock / snapshot files diff --git a/doc/faq.rst b/doc/faq.rst index bad55708c17..6292f2de896 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -51,7 +51,7 @@ looks like this: [0:00] 100.00% 16 / 16 snapshots no errors were found -The message means that there is more data stored in the repo than +The message means that there is more data stored in the repository than strictly necessary. This is uncritical. With high probability this is duplicate data caused by an interrupted backup run or upload operation. In order to clean it up, the command ``restic prune`` can be used. @@ -168,8 +168,8 @@ scheduling algorithm to give it the least favorable niceness (19). The above example makes sure that the system the backup runs on is not slowed down, which is particularly useful for servers. -Creating new repo on a Synology NAS via sftp fails --------------------------------------------------- +Creating new repository on a Synology NAS via sftp fails +-------------------------------------------------------- For using restic with a Synology NAS via sftp, please make sure that the specified path is absolute, it must start with a slash (``/``). diff --git a/doc/fish-completion.fish b/doc/fish-completion.fish index 16ec8f467dc..aa60d536de1 100644 --- a/doc/fish-completion.fish +++ b/doc/fish-completion.fish @@ -18,7 +18,8 @@ function __restic_perform_completion __restic_debug "args: $args" __restic_debug "last arg: $lastArg" - set -l requestComp "$args[1] __complete $args[2..-1] $lastArg" + # Disable ActiveHelp which is not supported for fish shell + set -l requestComp "RESTIC_ACTIVE_HELP=0 $args[1] __complete $args[2..-1] $lastArg" __restic_debug "Calling $requestComp" set -l results (eval $requestComp 2> /dev/null) diff --git a/doc/index.rst b/doc/index.rst index 69bbb84839e..034dbda23a8 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -9,6 +9,7 @@ Restic Documentation 030_preparing_a_new_repo 040_backup 045_working_with_repos + 047_tuning_backup_parameters 050_restore 060_forget 070_encryption diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1 index 4dccd8dadd2..8a7bfc1ce9c 100644 --- a/doc/man/restic-backup.1 +++ b/doc/man/restic-backup.1 @@ -3,12 +3,12 @@ .SH NAME .PP -restic\-backup \- Create a new backup of files and/or directories +restic-backup - Create a new backup of files and/or directories .SH SYNOPSIS .PP -\fBrestic backup [flags] FILE/DIR [FILE/DIR] ...\fP +\fBrestic backup [flags] [FILE/DIR] ...\fP .SH DESCRIPTION @@ -26,170 +26,178 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .SH OPTIONS .PP -\fB\-n\fP, \fB\-\-dry\-run\fP[=false] +\fB-n\fP, \fB--dry-run\fP[=false] do not upload or write any data, just show what would be done .PP -\fB\-e\fP, \fB\-\-exclude\fP=[] +\fB-e\fP, \fB--exclude\fP=[] exclude a \fB\fCpattern\fR (can be specified multiple times) .PP -\fB\-\-exclude\-caches\fP[=false] +\fB--exclude-caches\fP[=false] excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard .PP -\fB\-\-exclude\-file\fP=[] +\fB--exclude-file\fP=[] read exclude patterns from a \fB\fCfile\fR (can be specified multiple times) .PP -\fB\-\-exclude\-if\-present\fP=[] +\fB--exclude-if-present\fP=[] takes \fB\fCfilename[:header]\fR, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times) .PP -\fB\-\-exclude\-larger\-than\fP="" +\fB--exclude-larger-than\fP="" max \fB\fCsize\fR of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T) .PP -\fB\-\-files\-from\fP=[] +\fB--files-from\fP=[] read the files to backup from \fB\fCfile\fR (can be combined with file args; can be specified multiple times) .PP -\fB\-\-files\-from\-raw\fP=[] +\fB--files-from-raw\fP=[] read the files to backup from \fB\fCfile\fR (can be combined with file args; can be specified multiple times) .PP -\fB\-\-files\-from\-verbatim\fP=[] +\fB--files-from-verbatim\fP=[] read the files to backup from \fB\fCfile\fR (can be combined with file args; can be specified multiple times) .PP -\fB\-f\fP, \fB\-\-force\fP[=false] - force re\-reading the target files/directories (overrides the "parent" flag) +\fB-f\fP, \fB--force\fP[=false] + force re-reading the target files/directories (overrides the "parent" flag) .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for backup .PP -\fB\-H\fP, \fB\-\-host\fP="" +\fB-H\fP, \fB--host\fP="" set the \fB\fChostname\fR for the snapshot manually. To prevent an expensive rescan use the "parent" flag .PP -\fB\-\-iexclude\fP=[] - same as \-\-exclude \fB\fCpattern\fR but ignores the casing of filenames +\fB--iexclude\fP=[] + same as --exclude \fB\fCpattern\fR but ignores the casing of filenames .PP -\fB\-\-iexclude\-file\fP=[] - same as \-\-exclude\-file but ignores casing of \fB\fCfile\fRnames in patterns +\fB--iexclude-file\fP=[] + same as --exclude-file but ignores casing of \fB\fCfile\fRnames in patterns .PP -\fB\-\-ignore\-ctime\fP[=false] +\fB--ignore-ctime\fP[=false] ignore ctime changes when checking for modified files .PP -\fB\-\-ignore\-inode\fP[=false] +\fB--ignore-inode\fP[=false] ignore inode number changes when checking for modified files .PP -\fB\-x\fP, \fB\-\-one\-file\-system\fP[=false] +\fB-x\fP, \fB--one-file-system\fP[=false] exclude other file systems, don't cross filesystem boundaries and subvolumes .PP -\fB\-\-parent\fP="" - use this parent \fB\fCsnapshot\fR (default: last snapshot in the repo that has the same target files/directories, and is not newer than the snapshot time) +\fB--parent\fP="" + use this parent \fB\fCsnapshot\fR (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time) .PP -\fB\-\-stdin\fP[=false] +\fB--stdin\fP[=false] read backup from stdin .PP -\fB\-\-stdin\-filename\fP="stdin" +\fB--stdin-filename\fP="stdin" \fB\fCfilename\fR to use when reading from stdin .PP -\fB\-\-tag\fP=[] +\fB--tag\fP=[] add \fB\fCtags\fR for the new snapshot in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times) .PP -\fB\-\-time\fP="" - \fB\fCtime\fR of the backup (ex. '2012\-11\-01 22:08:41') (default: now) +\fB--time\fP="" + \fB\fCtime\fR of the backup (ex. '2012-11-01 22:08:41') (default: now) .PP -\fB\-\-with\-atime\fP[=false] +\fB--with-atime\fP[=false] store the atime for all files and directories .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-cache.1 b/doc/man/restic-cache.1 index cbe24298bd9..bedf699a8bb 100644 --- a/doc/man/restic-cache.1 +++ b/doc/man/restic-cache.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-cache \- Operate on local cache directories +restic-cache - Operate on local cache directories .SH SYNOPSIS @@ -18,99 +18,107 @@ The "cache" command allows listing and cleaning local cache directories. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-\-cleanup\fP[=false] +\fB--cleanup\fP[=false] remove old cache directories .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for cache .PP -\fB\-\-max\-age\fP=30 +\fB--max-age\fP=30 max age in \fB\fCdays\fR for cache directories to be considered old .PP -\fB\-\-no\-size\fP[=false] +\fB--no-size\fP[=false] do not output the size of the cache directories .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-cat.1 b/doc/man/restic-cat.1 index e5f7ddb56a2..4fcc373c33e 100644 --- a/doc/man/restic-cat.1 +++ b/doc/man/restic-cat.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-cat \- Print internal objects to stdout +restic-cat - Print internal objects to stdout .SH SYNOPSIS @@ -18,87 +18,95 @@ The "cat" command is used to print internal objects to stdout. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for cat .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-check.1 b/doc/man/restic-check.1 index 8e004d9ab5e..fd617b5247c 100644 --- a/doc/man/restic-check.1 +++ b/doc/man/restic-check.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-check \- Check the repository for errors +restic-check - Check the repository for errors .SH SYNOPSIS @@ -23,103 +23,107 @@ repository and not use a local cache. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-\-check\-unused\fP[=false] - find unused blobs - -.PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for check .PP -\fB\-\-read\-data\fP[=false] +\fB--read-data\fP[=false] read all data blobs .PP -\fB\-\-read\-data\-subset\fP="" +\fB--read-data-subset\fP="" read a \fB\fCsubset\fR of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset .PP -\fB\-\-with\-cache\fP[=false] +\fB--with-cache\fP[=false] use the cache .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) .PP -\fB\-\-json\fP[=false] +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-copy.1 b/doc/man/restic-copy.1 index f29cdc27052..d87f05d6e74 100644 --- a/doc/man/restic-copy.1 +++ b/doc/man/restic-copy.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-copy \- Copy snapshots from one repository to another +restic-copy - Copy snapshots from one repository to another .SH SYNOPSIS @@ -22,124 +22,132 @@ destination repositories. This /may incur higher bandwidth usage and costs/ than expected during normal backup runs. .PP -NOTE: The copying process does not re\-chunk files, which may break deduplication +NOTE: The copying process does not re-chunk files, which may break deduplication between the files copied and files already stored in the destination repository. This means that copied files, which existed in both the source and destination repository, /may occupy up to twice their space/ in the destination repository. -This can be mitigated by the "\-\-copy\-chunker\-params" option when initializing a +This can be mitigated by the "--copy-chunker-params" option when initializing a new destination repository using the "init" command. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] - help for copy +\fB--from-key-hint\fP="" + key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT) .PP -\fB\-H\fP, \fB\-\-host\fP=[] - only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given (can be specified multiple times) +\fB--from-password-command\fP="" + shell \fB\fCcommand\fR to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND) .PP -\fB\-\-key\-hint2\fP="" - key ID of key to try decrypting the destination repository first (default: $RESTIC\_KEY\_HINT2) +\fB--from-password-file\fP="" + \fB\fCfile\fR to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE) .PP -\fB\-\-password\-command2\fP="" - shell \fB\fCcommand\fR to obtain the destination repository password from (default: $RESTIC\_PASSWORD\_COMMAND2) +\fB--from-repo\fP="" + source \fB\fCrepository\fR to copy snapshots from (default: $RESTIC_FROM_REPOSITORY) .PP -\fB\-\-password\-file2\fP="" - \fB\fCfile\fR to read the destination repository password from (default: $RESTIC\_PASSWORD\_FILE2) +\fB--from-repository-file\fP="" + \fB\fCfile\fR from which to read the source repository location to copy snapshots from (default: $RESTIC_FROM_REPOSITORY_FILE) .PP -\fB\-\-path\fP=[] - only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot ID is given +\fB-h\fP, \fB--help\fP[=false] + help for copy .PP -\fB\-\-repo2\fP="" - destination \fB\fCrepository\fR to copy snapshots to (default: $RESTIC\_REPOSITORY2) +\fB-H\fP, \fB--host\fP=[] + only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given (can be specified multiple times) .PP -\fB\-\-repository\-file2\fP="" - \fB\fCfile\fR from which to read the destination repository location to copy snapshots to (default: $RESTIC\_REPOSITORY\_FILE2) +\fB--path\fP=[] + only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot ID is given .PP -\fB\-\-tag\fP=[] +\fB--tag\fP=[] only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot ID is given .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) .PP -\fB\-\-json\fP[=false] +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-diff.1 b/doc/man/restic-diff.1 index 7b4663ef54f..bb1f8af977d 100644 --- a/doc/man/restic-diff.1 +++ b/doc/man/restic-diff.1 @@ -3,12 +3,12 @@ .SH NAME .PP -restic\-diff \- Show differences between two snapshots +restic-diff - Show differences between two snapshots .SH SYNOPSIS .PP -\fBrestic diff [flags] snapshot\-ID snapshot\-ID\fP +\fBrestic diff [flags] snapshot-ID snapshot-ID\fP .SH DESCRIPTION @@ -21,7 +21,7 @@ directory: .IP \(bu 2 + The item was added .IP \(bu 2 -\- The item was removed +- The item was removed .IP \(bu 2 U The metadata (access mode, timestamps, ...) for the item was updated .IP \(bu 2 @@ -34,91 +34,99 @@ T The type was changed, e.g. a file was made a symlink .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for diff .PP -\fB\-\-metadata\fP[=false] +\fB--metadata\fP[=false] print changes in metadata .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-dump.1 b/doc/man/restic-dump.1 index f172c67333c..f0475cbe3d7 100644 --- a/doc/man/restic-dump.1 +++ b/doc/man/restic-dump.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-dump \- Print a backed\-up file to stdout +restic-dump - Print a backed-up file to stdout .SH SYNOPSIS @@ -25,103 +25,111 @@ repository. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-a\fP, \fB\-\-archive\fP="tar" +\fB-a\fP, \fB--archive\fP="tar" set archive \fB\fCformat\fR as "tar" or "zip" .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for dump .PP -\fB\-H\fP, \fB\-\-host\fP=[] +\fB-H\fP, \fB--host\fP=[] only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times) .PP -\fB\-\-path\fP=[] +\fB--path\fP=[] only consider snapshots which include this (absolute) \fB\fCpath\fR for snapshot ID "latest" .PP -\fB\-\-tag\fP=[] +\fB--tag\fP=[] only consider snapshots which include this \fB\fCtaglist\fR for snapshot ID "latest" .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-find.1 b/doc/man/restic-find.1 index 1a677876480..51185f53b2a 100644 --- a/doc/man/restic-find.1 +++ b/doc/man/restic-find.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-find \- Find a file, a directory or restic IDs +restic-find - Find a file, a directory or restic IDs .SH SYNOPSIS @@ -20,130 +20,138 @@ It can also be used to search for restic blobs or trees for troubleshooting. .SH OPTIONS .PP -\fB\-\-blob\fP[=false] - pattern is a blob\-ID +\fB--blob\fP[=false] + pattern is a blob-ID .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for find .PP -\fB\-H\fP, \fB\-\-host\fP=[] +\fB-H\fP, \fB--host\fP=[] only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given (can be specified multiple times) .PP -\fB\-i\fP, \fB\-\-ignore\-case\fP[=false] +\fB-i\fP, \fB--ignore-case\fP[=false] ignore case for pattern .PP -\fB\-l\fP, \fB\-\-long\fP[=false] +\fB-l\fP, \fB--long\fP[=false] use a long listing format showing size and mode .PP -\fB\-N\fP, \fB\-\-newest\fP="" +\fB-N\fP, \fB--newest\fP="" newest modification date/time .PP -\fB\-O\fP, \fB\-\-oldest\fP="" +\fB-O\fP, \fB--oldest\fP="" oldest modification date/time .PP -\fB\-\-pack\fP[=false] - pattern is a pack\-ID +\fB--pack\fP[=false] + pattern is a pack-ID .PP -\fB\-\-path\fP=[] - only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot\-ID is given +\fB--path\fP=[] + only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot-ID is given .PP -\fB\-\-show\-pack\-id\fP[=false] - display the pack\-ID the blobs belong to (with \-\-blob or \-\-tree) +\fB--show-pack-id\fP[=false] + display the pack-ID the blobs belong to (with --blob or --tree) .PP -\fB\-s\fP, \fB\-\-snapshot\fP=[] +\fB-s\fP, \fB--snapshot\fP=[] snapshot \fB\fCid\fR to search in (can be given multiple times) .PP -\fB\-\-tag\fP=[] - only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot\-ID is given +\fB--tag\fP=[] + only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot-ID is given .PP -\fB\-\-tree\fP[=false] - pattern is a tree\-ID +\fB--tree\fP[=false] + pattern is a tree-ID .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH EXAMPLE @@ -152,16 +160,16 @@ It can also be used to search for restic blobs or trees for troubleshooting. .nf restic find config.json -restic find \-\-json "*.yml" "*.json" -restic find \-\-json \-\-blob 420f620f b46ebe8a ddd38656 -restic find \-\-show\-pack\-id \-\-blob 420f620f -restic find \-\-tree 577c2bc9 f81f2e22 a62827a9 -restic find \-\-pack 025c1d06 +restic find --json "*.yml" "*.json" +restic find --json --blob 420f620f b46ebe8a ddd38656 +restic find --show-pack-id --blob 420f620f +restic find --tree 577c2bc9 f81f2e22 a62827a9 +restic find --pack 025c1d06 EXIT STATUS =========== -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .fi diff --git a/doc/man/restic-forget.1 b/doc/man/restic-forget.1 index 13fb0e5cc26..0be6532161c 100644 --- a/doc/man/restic-forget.1 +++ b/doc/man/restic-forget.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-forget \- Remove snapshots from the repository +restic-forget - Remove snapshots from the repository .SH SYNOPSIS @@ -13,188 +13,211 @@ restic\-forget \- Remove snapshots from the repository .SH DESCRIPTION .PP -The "forget" command removes snapshots according to a policy. Please note that -this command really only deletes the snapshot object in the repository, which -is a reference to data stored there. In order to remove the unreferenced data -after "forget" was run successfully, see the "prune" command. Please also read -the documentation for "forget" to learn about important security considerations. +The "forget" command removes snapshots according to a policy. All snapshots are +first divided into groups according to "--group-by", and after that the policy +specified by the "--keep-*" options is applied to each group individually. + +.PP +Please note that this command really only deletes the snapshot object in the +repository, which is a reference to data stored there. In order to remove the +unreferenced data after "forget" was run successfully, see the "prune" command. + +.PP +Please also read the documentation for "forget" to learn about some important +security considerations. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-l\fP, \fB\-\-keep\-last\fP=0 +\fB-l\fP, \fB--keep-last\fP=0 keep the last \fB\fCn\fR snapshots .PP -\fB\-H\fP, \fB\-\-keep\-hourly\fP=0 +\fB-H\fP, \fB--keep-hourly\fP=0 keep the last \fB\fCn\fR hourly snapshots .PP -\fB\-d\fP, \fB\-\-keep\-daily\fP=0 +\fB-d\fP, \fB--keep-daily\fP=0 keep the last \fB\fCn\fR daily snapshots .PP -\fB\-w\fP, \fB\-\-keep\-weekly\fP=0 +\fB-w\fP, \fB--keep-weekly\fP=0 keep the last \fB\fCn\fR weekly snapshots .PP -\fB\-m\fP, \fB\-\-keep\-monthly\fP=0 +\fB-m\fP, \fB--keep-monthly\fP=0 keep the last \fB\fCn\fR monthly snapshots .PP -\fB\-y\fP, \fB\-\-keep\-yearly\fP=0 +\fB-y\fP, \fB--keep-yearly\fP=0 keep the last \fB\fCn\fR yearly snapshots .PP -\fB\-\-keep\-within\fP= +\fB--keep-within\fP= keep snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP -\fB\-\-keep\-within\-hourly\fP= +\fB--keep-within-hourly\fP= keep hourly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP -\fB\-\-keep\-within\-daily\fP= +\fB--keep-within-daily\fP= keep daily snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP -\fB\-\-keep\-within\-weekly\fP= +\fB--keep-within-weekly\fP= keep weekly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP -\fB\-\-keep\-within\-monthly\fP= +\fB--keep-within-monthly\fP= keep monthly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP -\fB\-\-keep\-within\-yearly\fP= +\fB--keep-within-yearly\fP= keep yearly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot .PP -\fB\-\-keep\-tag\fP=[] +\fB--keep-tag\fP=[] keep snapshots with this \fB\fCtaglist\fR (can be specified multiple times) .PP -\fB\-\-host\fP=[] +\fB--host\fP=[] only consider snapshots with the given \fB\fChost\fR (can be specified multiple times) .PP -\fB\-\-tag\fP=[] +\fB--tag\fP=[] only consider snapshots which include this \fB\fCtaglist\fR in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times) .PP -\fB\-\-path\fP=[] +\fB--path\fP=[] only consider snapshots which include this (absolute) \fB\fCpath\fR (can be specified multiple times) .PP -\fB\-c\fP, \fB\-\-compact\fP[=false] +\fB-c\fP, \fB--compact\fP[=false] use compact output format .PP -\fB\-g\fP, \fB\-\-group\-by\fP="host,paths" - string for grouping snapshots by host,paths,tags +\fB-g\fP, \fB--group-by\fP="host,paths" + \fB\fCgroup\fR snapshots by host, paths and/or tags, separated by comma (disable grouping with '') .PP -\fB\-n\fP, \fB\-\-dry\-run\fP[=false] +\fB-n\fP, \fB--dry-run\fP[=false] do not delete anything, just print what would be done .PP -\fB\-\-prune\fP[=false] +\fB--prune\fP[=false] automatically run the 'prune' command if snapshots have been removed .PP -\fB\-\-max\-unused\fP="5%" +\fB--max-unused\fP="5%" tolerate given \fB\fClimit\fR of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited') .PP -\fB\-\-max\-repack\-size\fP="" +\fB--max-repack-size\fP="" maximum \fB\fCsize\fR to repack (allowed suffixes: k/K, m/M, g/G, t/T) .PP -\fB\-\-repack\-cacheable\-only\fP[=false] +\fB--repack-cacheable-only\fP[=false] only repack packs which are cacheable .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB--repack-small\fP[=false] + repack pack files below 80% of target pack size + +.PP +\fB--repack-uncompressed\fP[=false] + repack all uncompressed data + +.PP +\fB-h\fP, \fB--help\fP[=false] help for forget .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) .PP -\fB\-\-json\fP[=false] +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-generate.1 b/doc/man/restic-generate.1 index 5b555df5695..6bda99f7cfe 100644 --- a/doc/man/restic-generate.1 +++ b/doc/man/restic-generate.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-generate \- Generate manual pages and auto\-completion files (bash, fish, zsh) +restic-generate - Generate manual pages and auto-completion files (bash, fish, zsh) .SH SYNOPSIS @@ -14,108 +14,116 @@ restic\-generate \- Generate manual pages and auto\-completion files (bash, fish .SH DESCRIPTION .PP The "generate" command writes automatically generated files (like the man pages -and the auto\-completion files for bash, fish and zsh). +and the auto-completion files for bash, fish and zsh). .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-\-bash\-completion\fP="" +\fB--bash-completion\fP="" write bash completion \fB\fCfile\fR .PP -\fB\-\-fish\-completion\fP="" +\fB--fish-completion\fP="" write fish completion \fB\fCfile\fR .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for generate .PP -\fB\-\-man\fP="" +\fB--man\fP="" write man pages to \fB\fCdirectory\fR .PP -\fB\-\-zsh\-completion\fP="" +\fB--zsh-completion\fP="" write zsh completion \fB\fCfile\fR .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-init.1 b/doc/man/restic-init.1 index 72fe14ea64a..87ba79a36a5 100644 --- a/doc/man/restic-init.1 +++ b/doc/man/restic-init.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-init \- Initialize a new repository +restic-init - Initialize a new repository .SH SYNOPSIS @@ -18,111 +18,123 @@ The "init" command initializes a new repository. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-\-copy\-chunker\-params\fP[=false] +\fB--copy-chunker-params\fP[=false] copy chunker parameters from the secondary repository (useful with the copy command) .PP -\fB\-h\fP, \fB\-\-help\fP[=false] - help for init +\fB--from-key-hint\fP="" + key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT) + +.PP +\fB--from-password-command\fP="" + shell \fB\fCcommand\fR to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND) .PP -\fB\-\-key\-hint2\fP="" - key ID of key to try decrypting the secondary repository first (default: $RESTIC\_KEY\_HINT2) +\fB--from-password-file\fP="" + \fB\fCfile\fR to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE) .PP -\fB\-\-password\-command2\fP="" - shell \fB\fCcommand\fR to obtain the secondary repository password from (default: $RESTIC\_PASSWORD\_COMMAND2) +\fB--from-repo\fP="" + source \fB\fCrepository\fR to copy chunker parameters from (default: $RESTIC_FROM_REPOSITORY) .PP -\fB\-\-password\-file2\fP="" - \fB\fCfile\fR to read the secondary repository password from (default: $RESTIC\_PASSWORD\_FILE2) +\fB--from-repository-file\fP="" + \fB\fCfile\fR from which to read the source repository location to copy chunker parameters from (default: $RESTIC_FROM_REPOSITORY_FILE) .PP -\fB\-\-repo2\fP="" - secondary \fB\fCrepository\fR to copy chunker parameters from (default: $RESTIC\_REPOSITORY2) +\fB-h\fP, \fB--help\fP[=false] + help for init .PP -\fB\-\-repository\-file2\fP="" - \fB\fCfile\fR from which to read the secondary repository location to copy chunker parameters from (default: $RESTIC\_REPOSITORY\_FILE2) +\fB--repository-version\fP="stable" + repository format version to use, allowed values are a format version, 'latest' and 'stable' .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-key.1 b/doc/man/restic-key.1 index d78a23851d8..4afa3d116c9 100644 --- a/doc/man/restic-key.1 +++ b/doc/man/restic-key.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-key \- Manage keys (passwords) +restic-key - Manage keys (passwords) .SH SYNOPSIS @@ -18,99 +18,107 @@ The "key" command manages keys (passwords) for accessing the repository. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for key .PP -\fB\-\-host\fP="" +\fB--host\fP="" the hostname for new keys .PP -\fB\-\-new\-password\-file\fP="" +\fB--new-password-file\fP="" \fB\fCfile\fR from which to read the new password .PP -\fB\-\-user\fP="" +\fB--user\fP="" the username for new keys .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-list.1 b/doc/man/restic-list.1 index 187ab079126..c87f1d0a6bf 100644 --- a/doc/man/restic-list.1 +++ b/doc/man/restic-list.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-list \- List objects in the repository +restic-list - List objects in the repository .SH SYNOPSIS @@ -18,87 +18,95 @@ The "list" command allows listing objects in the repository based on type. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for list .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-ls.1 b/doc/man/restic-ls.1 index 0e9357ffd2b..9d77ee62d83 100644 --- a/doc/man/restic-ls.1 +++ b/doc/man/restic-ls.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-ls \- List files in a snapshot +restic-ls - List files in a snapshot .SH SYNOPSIS @@ -18,14 +18,14 @@ The "ls" command lists files and directories in a snapshot. .PP The special snapshot ID "latest" can be used to list files and directories of the latest snapshot in the repository. The -\-\-host flag can be used in conjunction to select the latest +--host flag can be used in conjunction to select the latest snapshot originating from a certain host only. .PP File listings can optionally be filtered by directories. Any positional arguments after the snapshot ID are interpreted as absolute directory paths, and only files inside those directories -will be listed. If the \-\-recursive flag is used, then the filter +will be listed. If the --recursive flag is used, then the filter will allow traversing into matching directories' subfolders. Any directory paths specified must be absolute (starting with a path separator); paths use the forward slash '/' as separator. @@ -33,107 +33,115 @@ a path separator); paths use the forward slash '/' as separator. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for ls .PP -\fB\-H\fP, \fB\-\-host\fP=[] +\fB-H\fP, \fB--host\fP=[] only consider snapshots for this \fB\fChost\fR, when snapshot ID "latest" is given (can be specified multiple times) .PP -\fB\-l\fP, \fB\-\-long\fP[=false] +\fB-l\fP, \fB--long\fP[=false] use a long listing format showing size and mode .PP -\fB\-\-path\fP=[] +\fB--path\fP=[] only consider snapshots which include this (absolute) \fB\fCpath\fR, when snapshot ID "latest" is given (can be specified multiple times) .PP -\fB\-\-recursive\fP[=false] +\fB--recursive\fP[=false] include files in subfolders of the listed directories .PP -\fB\-\-tag\fP=[] +\fB--tag\fP=[] only consider snapshots which include this \fB\fCtaglist\fR, when snapshot ID "latest" is given (can be specified multiple times) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-migrate.1 b/doc/man/restic-migrate.1 index c12d7ba9cb7..578c85d692a 100644 --- a/doc/man/restic-migrate.1 +++ b/doc/man/restic-migrate.1 @@ -3,107 +3,116 @@ .SH NAME .PP -restic\-migrate \- Apply migrations +restic-migrate - Apply migrations .SH SYNOPSIS .PP -\fBrestic migrate [flags] [name]\fP +\fBrestic migrate [flags] [migration name] [...]\fP .SH DESCRIPTION .PP -The "migrate" command applies migrations to a repository. When no migration -name is explicitly given, a list of migrations that can be applied is printed. +The "migrate" command checks which migrations can be applied for a repository +and prints a list with available migration names. If one or more migration +names are specified, these migrations are applied. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-f\fP, \fB\-\-force\fP[=false] +\fB-f\fP, \fB--force\fP[=false] apply a migration a second time .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for migrate .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-mount.1 b/doc/man/restic-mount.1 index 8aae16907cc..b2feee01ce5 100644 --- a/doc/man/restic-mount.1 +++ b/doc/man/restic-mount.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-mount \- Mount the repository +restic-mount - Mount the repository .SH SYNOPSIS @@ -14,19 +14,23 @@ restic\-mount \- Mount the repository .SH DESCRIPTION .PP The "mount" command mounts the repository via fuse to a directory. This is a -read\-only mount. +read-only mount. .SH Snapshot Directories .PP -If you need a different template for all directories that contain snapshots, -you can pass a template via \-\-snapshot\-template. Example without colons: +If you need a different template for directories that contain snapshots, +you can pass a time template via --time-template and path templates via +--path-template. + +.PP +Example time template without colons: .PP .RS .nf -\-\-snapshot\-template "2006\-01\-02\_15\-04\-05" +--time-template "2006-01-02_15-04-05" .fi .RE @@ -38,7 +42,7 @@ You need to specify a sample format for exactly the following timestamp: .RS .nf -Mon Jan 2 15:04:05 \-0700 MST 2006 +Mon Jan 2 15:04:05 -0700 MST 2006 .fi .RE @@ -47,118 +51,146 @@ Mon Jan 2 15:04:05 \-0700 MST 2006 For details please see the documentation for time.Format() at: https://godoc.org/time#Time.Format +.PP +For path templates, you can use the following patterns which will be replaced: + %i by short snapshot ID + %I by long snapshot ID + %u by username + %h by hostname + %t by tags + %T by timestamp as specified by --time-template + +.PP +The default path templates are: + "ids/%i" + "snapshots/%T" + "hosts/%h/%T" + "tags/%t/%T" + .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-\-allow\-other\fP[=false] +\fB--allow-other\fP[=false] allow other users to access the data in the mounted directory .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for mount .PP -\fB\-H\fP, \fB\-\-host\fP=[] +\fB-H\fP, \fB--host\fP=[] only consider snapshots for this host (can be specified multiple times) .PP -\fB\-\-no\-default\-permissions\fP[=false] - for 'allow\-other', ignore Unix permissions and allow users to read all snapshot files +\fB--no-default-permissions\fP[=false] + for 'allow-other', ignore Unix permissions and allow users to read all snapshot files .PP -\fB\-\-owner\-root\fP[=false] +\fB--owner-root\fP[=false] use 'root' as the owner of files and dirs .PP -\fB\-\-path\fP=[] +\fB--path\fP=[] only consider snapshots which include this (absolute) \fB\fCpath\fR .PP -\fB\-\-snapshot\-template\fP="2006\-01\-02T15:04:05Z07:00" - set \fB\fCtemplate\fR to use for snapshot dirs +\fB--path-template\fP=[] + set \fB\fCtemplate\fR for path names (can be specified multiple times) .PP -\fB\-\-tag\fP=[] +\fB--tag\fP=[] only consider snapshots which include this \fB\fCtaglist\fR +.PP +\fB--time-template\fP="2006-01-02T15:04:05Z07:00" + set \fB\fCtemplate\fR to use for times + .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) .PP -\fB\-\-json\fP[=false] +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-prune.1 b/doc/man/restic-prune.1 index 70b38df32f9..77a712039d0 100644 --- a/doc/man/restic-prune.1 +++ b/doc/man/restic-prune.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-prune \- Remove unneeded data from the repository +restic-prune - Remove unneeded data from the repository .SH SYNOPSIS @@ -19,103 +19,123 @@ referenced and therefore not needed any more. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-n\fP, \fB\-\-dry\-run\fP[=false] +\fB-n\fP, \fB--dry-run\fP[=false] do not modify the repository, just print what would be done .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for prune .PP -\fB\-\-max\-repack\-size\fP="" +\fB--max-repack-size\fP="" maximum \fB\fCsize\fR to repack (allowed suffixes: k/K, m/M, g/G, t/T) .PP -\fB\-\-max\-unused\fP="5%" +\fB--max-unused\fP="5%" tolerate given \fB\fClimit\fR of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited') .PP -\fB\-\-repack\-cacheable\-only\fP[=false] +\fB--repack-cacheable-only\fP[=false] only repack packs which are cacheable +.PP +\fB--repack-small\fP[=false] + repack pack files below 80% of target pack size + +.PP +\fB--repack-uncompressed\fP[=false] + repack all uncompressed data + +.PP +\fB--unsafe-recover-no-free-space\fP="" + UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first. + .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) .PP -\fB\-\-json\fP[=false] +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-rebuild-index.1 b/doc/man/restic-rebuild-index.1 index 599890054c1..c37f55a18c4 100644 --- a/doc/man/restic-rebuild-index.1 +++ b/doc/man/restic-rebuild-index.1 @@ -3,107 +3,115 @@ .SH NAME .PP -restic\-rebuild\-index \- Build a new index +restic-rebuild-index - Build a new index .SH SYNOPSIS .PP -\fBrestic rebuild\-index [flags]\fP +\fBrestic rebuild-index [flags]\fP .SH DESCRIPTION .PP -The "rebuild\-index" command creates a new index based on the pack files in the +The "rebuild-index" command creates a new index based on the pack files in the repository. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] - help for rebuild\-index +\fB-h\fP, \fB--help\fP[=false] + help for rebuild-index .PP -\fB\-\-read\-all\-packs\fP[=false] +\fB--read-all-packs\fP[=false] read all pack files to generate new index from scratch .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-recover.1 b/doc/man/restic-recover.1 index 0a190e1eed3..cc45eec9aa8 100644 --- a/doc/man/restic-recover.1 +++ b/doc/man/restic-recover.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-recover \- Recover data from the repository not referenced by snapshots +restic-recover - Recover data from the repository not referenced by snapshots .SH SYNOPSIS @@ -20,87 +20,95 @@ It can be used if, for example, a snapshot has been removed by accident with "fo .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for recover .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-restore.1 b/doc/man/restic-restore.1 index 9be8dc10d9a..e96337e7d55 100644 --- a/doc/man/restic-restore.1 +++ b/doc/man/restic-restore.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-restore \- Extract the data from a snapshot +restic-restore - Extract the data from a snapshot .SH SYNOPSIS @@ -23,123 +23,131 @@ repository. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-e\fP, \fB\-\-exclude\fP=[] +\fB-e\fP, \fB--exclude\fP=[] exclude a \fB\fCpattern\fR (can be specified multiple times) .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for restore .PP -\fB\-H\fP, \fB\-\-host\fP=[] +\fB-H\fP, \fB--host\fP=[] only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times) .PP -\fB\-\-iexclude\fP=[] - same as \fB\fC\-\-exclude\fR but ignores the casing of filenames +\fB--iexclude\fP=[] + same as \fB\fC--exclude\fR but ignores the casing of filenames .PP -\fB\-\-iinclude\fP=[] - same as \fB\fC\-\-include\fR but ignores the casing of filenames +\fB--iinclude\fP=[] + same as \fB\fC--include\fR but ignores the casing of filenames .PP -\fB\-i\fP, \fB\-\-include\fP=[] +\fB-i\fP, \fB--include\fP=[] include a \fB\fCpattern\fR, exclude everything else (can be specified multiple times) .PP -\fB\-\-path\fP=[] +\fB--path\fP=[] only consider snapshots which include this (absolute) \fB\fCpath\fR for snapshot ID "latest" .PP -\fB\-\-tag\fP=[] +\fB--tag\fP=[] only consider snapshots which include this \fB\fCtaglist\fR for snapshot ID "latest" .PP -\fB\-t\fP, \fB\-\-target\fP="" +\fB-t\fP, \fB--target\fP="" directory to extract data to .PP -\fB\-\-verify\fP[=false] +\fB--verify\fP[=false] verify restored files content .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-self-update.1 b/doc/man/restic-self-update.1 index 70af52d6deb..9ac97d8c77d 100644 --- a/doc/man/restic-self-update.1 +++ b/doc/man/restic-self-update.1 @@ -3,17 +3,17 @@ .SH NAME .PP -restic\-self\-update \- Update the restic binary +restic-self-update - Update the restic binary .SH SYNOPSIS .PP -\fBrestic self\-update [flags]\fP +\fBrestic self-update [flags]\fP .SH DESCRIPTION .PP -The command "self\-update" downloads the latest stable release of restic from +The command "self-update" downloads the latest stable release of restic from GitHub and replaces the currently running binary. After download, the authenticity of the binary is verified using the GPG signature on the release files. @@ -21,91 +21,99 @@ files. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] - help for self\-update +\fB-h\fP, \fB--help\fP[=false] + help for self-update .PP -\fB\-\-output\fP="" +\fB--output\fP="" Save the downloaded file as \fB\fCfilename\fR (default: running binary itself) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-snapshots.1 b/doc/man/restic-snapshots.1 index 902105134bc..b99ec93d000 100644 --- a/doc/man/restic-snapshots.1 +++ b/doc/man/restic-snapshots.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-snapshots \- List all snapshots +restic-snapshots - List all snapshots .SH SYNOPSIS @@ -18,111 +18,119 @@ The "snapshots" command lists all snapshots stored in the repository. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-c\fP, \fB\-\-compact\fP[=false] +\fB-c\fP, \fB--compact\fP[=false] use compact output format .PP -\fB\-g\fP, \fB\-\-group\-by\fP="" - string for grouping snapshots by host,paths,tags +\fB-g\fP, \fB--group-by\fP="" + \fB\fCgroup\fR snapshots by host, paths and/or tags, separated by comma .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for snapshots .PP -\fB\-H\fP, \fB\-\-host\fP=[] +\fB-H\fP, \fB--host\fP=[] only consider snapshots for this \fB\fChost\fR (can be specified multiple times) .PP -\fB\-\-latest\fP=0 +\fB--latest\fP=0 only show the last \fB\fCn\fR snapshots for each host and path .PP -\fB\-\-path\fP=[] +\fB--path\fP=[] only consider snapshots for this \fB\fCpath\fR (can be specified multiple times) .PP -\fB\-\-tag\fP=[] +\fB--tag\fP=[] only consider snapshots which include this \fB\fCtaglist\fR in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-stats.1 b/doc/man/restic-stats.1 index 6cf921fdfa5..08d058beb55 100644 --- a/doc/man/restic-stats.1 +++ b/doc/man/restic-stats.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-stats \- Scan the repository and show basic statistics +restic-stats - Scan the repository and show basic statistics .SH SYNOPSIS @@ -16,7 +16,7 @@ restic\-stats \- Scan the repository and show basic statistics The "stats" command walks one or multiple snapshots in a repository and accumulates statistics about the data stored therein. It reports on the number of unique files and their sizes, according to one of -the counting modes as given by the \-\-mode flag. +the counting modes as given by the --mode flag. .PP It operates on all snapshots matching the selection criteria or all @@ -30,15 +30,15 @@ The modes are: .RS .IP \(bu 2 -restore\-size: (default) Counts the size of the restored files. +restore-size: (default) Counts the size of the restored files. .IP \(bu 2 -files\-by\-contents: Counts total size of files, where a file is +files-by-contents: Counts total size of files, where a file is considered unique if it has unique contents. .IP \(bu 2 -raw\-data: Counts the size of blobs in the repository, regardless of +raw-data: Counts the size of blobs in the repository, regardless of how many files reference them. .IP \(bu 2 -blobs\-per\-file: A combination of files\-by\-contents and raw\-data. +blobs-per-file: A combination of files-by-contents and raw-data. .RE @@ -48,103 +48,111 @@ Refer to the online manual for more details about each mode. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for stats .PP -\fB\-H\fP, \fB\-\-host\fP=[] +\fB-H\fP, \fB--host\fP=[] only consider snapshots with the given \fB\fChost\fR (can be specified multiple times) .PP -\fB\-\-mode\fP="restore\-size" - counting mode: restore\-size (default), files\-by\-contents, blobs\-per\-file or raw\-data +\fB--mode\fP="restore-size" + counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data .PP -\fB\-\-path\fP=[] +\fB--path\fP=[] only consider snapshots which include this (absolute) \fB\fCpath\fR (can be specified multiple times) .PP -\fB\-\-tag\fP=[] +\fB--tag\fP=[] only consider snapshots which include this \fB\fCtaglist\fR in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times) .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-tag.1 b/doc/man/restic-tag.1 index 00c727b41a8..bb72612f11c 100644 --- a/doc/man/restic-tag.1 +++ b/doc/man/restic-tag.1 @@ -3,12 +3,12 @@ .SH NAME .PP -restic\-tag \- Modify tags on snapshots +restic-tag - Modify tags on snapshots .SH SYNOPSIS .PP -\fBrestic tag [flags] [snapshot\-ID ...]\fP +\fBrestic tag [flags] [snapshot-ID ...]\fP .SH DESCRIPTION @@ -20,116 +20,124 @@ You can either set/replace the entire set of tags on a snapshot, or add tags to/remove tags from the existing set. .PP -When no snapshot\-ID is given, all snapshots matching the host, tag and path filter criteria are modified. +When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-\-add\fP=[] +\fB--add\fP=[] \fB\fCtags\fR which will be added to the existing tags in the format \fB\fCtag[,tag,...]\fR (can be given multiple times) .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for tag .PP -\fB\-H\fP, \fB\-\-host\fP=[] +\fB-H\fP, \fB--host\fP=[] only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given (can be specified multiple times) .PP -\fB\-\-path\fP=[] - only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot\-ID is given +\fB--path\fP=[] + only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot-ID is given .PP -\fB\-\-remove\fP=[] +\fB--remove\fP=[] \fB\fCtags\fR which will be removed from the existing tags in the format \fB\fCtag[,tag,...]\fR (can be given multiple times) .PP -\fB\-\-set\fP=[] +\fB--set\fP=[] \fB\fCtags\fR which will replace the existing tags in the format \fB\fCtag[,tag,...]\fR (can be given multiple times) .PP -\fB\-\-tag\fP=[] - only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot\-ID is given +\fB--tag\fP=[] + only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot-ID is given .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-unlock.1 b/doc/man/restic-unlock.1 index d7a6c1e8196..99a969498f0 100644 --- a/doc/man/restic-unlock.1 +++ b/doc/man/restic-unlock.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-unlock \- Remove locks other processes created +restic-unlock - Remove locks other processes created .SH SYNOPSIS @@ -18,91 +18,99 @@ The "unlock" command removes stale locks that have been created by other restic .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for unlock .PP -\fB\-\-remove\-all\fP[=false] - remove all locks, even non\-stale ones +\fB--remove-all\fP[=false] + remove all locks, even non-stale ones .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic-version.1 b/doc/man/restic-version.1 index 1d3428b6c47..a803cd491e2 100644 --- a/doc/man/restic-version.1 +++ b/doc/man/restic-version.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic\-version \- Print version information +restic-version - Print version information .SH SYNOPSIS @@ -19,87 +19,95 @@ and the version of this software. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non\-zero if there was any error. +Exit status is 0 if the command was successful, and non-zero if there was any error. .SH OPTIONS .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB-h\fP, \fB--help\fP[=false] help for version .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) .PP -\fB\-\-json\fP[=false] +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO diff --git a/doc/man/restic.1 b/doc/man/restic.1 index 72f4bbf325c..9577e012ade 100644 --- a/doc/man/restic.1 +++ b/doc/man/restic.1 @@ -3,7 +3,7 @@ .SH NAME .PP -restic \- Backup and restore files +restic - Backup and restore files .SH SYNOPSIS @@ -19,82 +19,90 @@ directories in an encrypted repository stored on different backends. .SH OPTIONS .PP -\fB\-\-cacert\fP=[] +\fB--cacert\fP=[] \fB\fCfile\fR to load root certificates from (default: use system certificates) .PP -\fB\-\-cache\-dir\fP="" +\fB--cache-dir\fP="" set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory) .PP -\fB\-\-cleanup\-cache\fP[=false] +\fB--cleanup-cache\fP[=false] auto remove old cache directories .PP -\fB\-h\fP, \fB\-\-help\fP[=false] +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) + +.PP +\fB-h\fP, \fB--help\fP[=false] help for restic .PP -\fB\-\-insecure\-tls\fP[=false] - skip TLS certificate verification when connecting to the repo (insecure) +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) .PP -\fB\-\-json\fP[=false] +\fB--json\fP[=false] set output mode to JSON for commands that support it .PP -\fB\-\-key\-hint\fP="" - \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT) +\fB--key-hint\fP="" + \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) .PP -\fB\-\-limit\-download\fP=0 +\fB--limit-download\fP=0 limits downloads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-limit\-upload\fP=0 +\fB--limit-upload\fP=0 limits uploads to a maximum rate in KiB/s. (default: unlimited) .PP -\fB\-\-no\-cache\fP[=false] +\fB--no-cache\fP[=false] do not use a local cache .PP -\fB\-\-no\-lock\fP[=false] - do not lock the repository, this allows some operations on read\-only repositories +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories .PP -\fB\-o\fP, \fB\-\-option\fP=[] +\fB-o\fP, \fB--option\fP=[] set extended option (\fB\fCkey=value\fR, can be specified multiple times) .PP -\fB\-\-password\-command\fP="" - shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND) +\fB--pack-size\fP=0 + set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) .PP -\fB\-p\fP, \fB\-\-password\-file\fP="" - \fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE) +\fB-p\fP, \fB--password-file\fP="" + \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) .PP -\fB\-q\fP, \fB\-\-quiet\fP[=false] +\fB-q\fP, \fB--quiet\fP[=false] do not output comprehensive progress report .PP -\fB\-r\fP, \fB\-\-repo\fP="" - \fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY) +\fB-r\fP, \fB--repo\fP="" + \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) .PP -\fB\-\-repository\-file\fP="" - \fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE) +\fB--repository-file\fP="" + \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) .PP -\fB\-\-tls\-client\-cert\fP="" +\fB--tls-client-cert\fP="" path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key .PP -\fB\-v\fP, \fB\-\-verbose\fP[=0] - be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3) +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3) .SH SEE ALSO .PP -\fBrestic\-backup(1)\fP, \fBrestic\-cache(1)\fP, \fBrestic\-cat(1)\fP, \fBrestic\-check(1)\fP, \fBrestic\-copy(1)\fP, \fBrestic\-diff(1)\fP, \fBrestic\-dump(1)\fP, \fBrestic\-find(1)\fP, \fBrestic\-forget(1)\fP, \fBrestic\-generate(1)\fP, \fBrestic\-init(1)\fP, \fBrestic\-key(1)\fP, \fBrestic\-list(1)\fP, \fBrestic\-ls(1)\fP, \fBrestic\-migrate(1)\fP, \fBrestic\-mount(1)\fP, \fBrestic\-prune(1)\fP, \fBrestic\-rebuild\-index(1)\fP, \fBrestic\-recover(1)\fP, \fBrestic\-restore(1)\fP, \fBrestic\-self\-update(1)\fP, \fBrestic\-snapshots(1)\fP, \fBrestic\-stats(1)\fP, \fBrestic\-tag(1)\fP, \fBrestic\-unlock(1)\fP, \fBrestic\-version(1)\fP +\fBrestic-backup(1)\fP, \fBrestic-cache(1)\fP, \fBrestic-cat(1)\fP, \fBrestic-check(1)\fP, \fBrestic-copy(1)\fP, \fBrestic-diff(1)\fP, \fBrestic-dump(1)\fP, \fBrestic-find(1)\fP, \fBrestic-forget(1)\fP, \fBrestic-generate(1)\fP, \fBrestic-init(1)\fP, \fBrestic-key(1)\fP, \fBrestic-list(1)\fP, \fBrestic-ls(1)\fP, \fBrestic-migrate(1)\fP, \fBrestic-mount(1)\fP, \fBrestic-prune(1)\fP, \fBrestic-rebuild-index(1)\fP, \fBrestic-recover(1)\fP, \fBrestic-restore(1)\fP, \fBrestic-self-update(1)\fP, \fBrestic-snapshots(1)\fP, \fBrestic-stats(1)\fP, \fBrestic-tag(1)\fP, \fBrestic-unlock(1)\fP, \fBrestic-version(1)\fP diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 87f3104a175..e17e5cd8e46 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -36,7 +36,7 @@ Usage help is available: mount Mount the repository prune Remove unneeded data from the repository rebuild-index Build a new index - recover Recover data from the repository + recover Recover data from the repository not referenced by snapshots restore Extract the data from a snapshot self-update Update the restic binary snapshots List all snapshots @@ -49,12 +49,14 @@ Usage help is available: --cacert file file to load root certificates from (default: use system certificates) --cache-dir directory set the cache directory. (default: use system default cache directory) --cleanup-cache auto remove old cache directories + --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default auto) -h, --help help for restic - --insecure-tls skip TLS certificate verification when connecting to the repo (insecure) + --insecure-tls skip TLS certificate verification when connecting to the repository (insecure) --json set output mode to JSON for commands that support it --key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT) --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited) --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited) + --pack-size uint set target pack size in MiB. (default: $RESTIC_PACK_SIZE) --no-cache do not use a local cache --no-lock do not lock the repository, this allows some operations on read-only repositories -o, --option key=value set extended option (key=value, can be specified multiple times) @@ -92,6 +94,7 @@ command: restic backup [flags] FILE/DIR [FILE/DIR] ... Flags: + -n, --dry-run do not upload or write any data, just show what would be done -e, --exclude pattern exclude a pattern (can be specified multiple times) --exclude-caches excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard --exclude-file file read exclude patterns from a file (can be specified multiple times) @@ -105,9 +108,10 @@ command: -H, --host hostname set the hostname for the snapshot manually. To prevent an expensive rescan use the "parent" flag --iexclude pattern same as --exclude pattern but ignores the casing of filenames --iexclude-file file same as --exclude-file but ignores casing of filenames in patterns + --ignore-ctime ignore ctime changes when checking for modified files --ignore-inode ignore inode number changes when checking for modified files -x, --one-file-system exclude other file systems, don't cross filesystem boundaries and subvolumes - --parent snapshot use this parent snapshot (default: last snapshot in the repo that has the same target files/directories) + --parent snapshot use this parent snapshot (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time) --stdin read backup from stdin --stdin-filename filename filename to use when reading from stdin (default "stdin") --tag tags add tags for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times) (default []) @@ -119,11 +123,13 @@ command: --cacert file file to load root certificates from (default: use system certificates) --cache-dir directory set the cache directory. (default: use system default cache directory) --cleanup-cache auto remove old cache directories - --insecure-tls skip TLS certificate verification when connecting to the repo (insecure) + --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default auto) + --insecure-tls skip TLS certificate verification when connecting to the repository (insecure) --json set output mode to JSON for commands that support it --key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT) --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited) --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited) + --pack-size uint set target pack size in MiB. (default: $RESTIC_PACK_SIZE) --no-cache do not use a local cache --no-lock do not lock the repository, this allows some operations on read-only repositories -o, --option key=value set extended option (key=value, can be specified multiple times) @@ -425,13 +431,13 @@ message. The command line parameter ``--cache-dir`` or the environment variable ``$RESTIC_CACHE_DIR`` can be used to override the default cache location. The parameter ``--no-cache`` disables the cache entirely. In this case, all data -is loaded from the repo. +is loaded from the repository. The cache is ephemeral: When a file cannot be read from the cache, it is loaded from the repository. Within the cache directory, there's a sub directory for each repository the -cache was used with. Restic updates the timestamps of a repo directory each +cache was used with. Restic updates the timestamps of a repository directory each time it is used, so by looking at the timestamps of the sub directories of the cache directory it can decide which sub directories are old and probably not needed any more. You can either remove these directories manually, or run a diff --git a/doc/zsh-completion.zsh b/doc/zsh-completion.zsh index d2dca56830c..cea6abb3873 100644 --- a/doc/zsh-completion.zsh +++ b/doc/zsh-completion.zsh @@ -1,4 +1,4 @@ -#compdef _restic restic +#compdef restic # zsh completion for restic -*- shell-script -*- @@ -86,7 +86,24 @@ _restic() return fi + local activeHelpMarker="_activeHelp_ " + local endIndex=${#activeHelpMarker} + local startIndex=$((${#activeHelpMarker}+1)) + local hasActiveHelp=0 while IFS='\n' read -r comp; do + # Check if this is an activeHelp statement (i.e., prefixed with $activeHelpMarker) + if [ "${comp[1,$endIndex]}" = "$activeHelpMarker" ];then + __restic_debug "ActiveHelp found: $comp" + comp="${comp[$startIndex,-1]}" + if [ -n "$comp" ]; then + compadd -x "${comp}" + __restic_debug "ActiveHelp will need delimiter" + hasActiveHelp=1 + fi + + continue + fi + if [ -n "$comp" ]; then # If requested, completions are returned with a description. # The description is preceded by a TAB character. @@ -94,7 +111,7 @@ _restic() # We first need to escape any : as part of the completion itself. comp=${comp//:/\\:} - local tab=$(printf '\t') + local tab="$(printf '\t')" comp=${comp//$tab/:} __restic_debug "Adding completion: ${comp}" @@ -103,6 +120,17 @@ _restic() fi done < <(printf "%s\n" "${out[@]}") + # Add a delimiter after the activeHelp statements, but only if: + # - there are completions following the activeHelp statements, or + # - file completion will be performed (so there will be choices after the activeHelp) + if [ $hasActiveHelp -eq 1 ]; then + if [ ${#completions} -ne 0 ] || [ $((directive & shellCompDirectiveNoFileComp)) -eq 0 ]; then + __restic_debug "Adding activeHelp delimiter" + compadd -x "--" + hasActiveHelp=0 + fi + fi + if [ $((directive & shellCompDirectiveNoSpace)) -ne 0 ]; then __restic_debug "Activating nospace." noSpace="-S ''" @@ -125,7 +153,7 @@ _restic() _arguments '*:filename:'"$filteringCmd" elif [ $((directive & shellCompDirectiveFilterDirs)) -ne 0 ]; then # File completion for directories only - local subDir + local subdir subdir="${completions[1]}" if [ -n "$subdir" ]; then __restic_debug "Listing directories in $subdir" @@ -173,5 +201,5 @@ _restic() # don't run the completion function when being source-ed or eval-ed if [ "$funcstack[1]" = "_restic" ]; then - _restic + _restic fi diff --git a/go.mod b/go.mod index c0742c0909c..5999ec57c68 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/restic/restic replace ( - github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.0.0 - github.com/form3tech-oss/jwt-go => github.com/golang-jwt/jwt/v4 v4.0.0 golang.org/x/text => golang.org/x/text v0.3.7 gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0 ) @@ -11,42 +9,84 @@ require ( bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512 cloud.google.com/go/storage v1.16.0 github.com/Azure/azure-sdk-for-go v55.6.0+incompatible - github.com/Azure/go-autorest/autorest v0.11.21 // indirect - github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/aws/aws-sdk-go v1.38.21 github.com/cenkalti/backoff/v4 v4.1.1 github.com/cespare/xxhash/v2 v2.1.1 - github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/elithrar/simple-scrypt v1.3.0 github.com/go-ole/go-ole v1.2.5 - github.com/gofrs/uuid v4.0.0+incompatible // indirect - github.com/golang-jwt/jwt/v4 v4.1.0 // indirect github.com/google/go-cmp v0.5.6 github.com/hashicorp/golang-lru v0.5.4 github.com/juju/ratelimit v1.0.1 + github.com/klauspost/compress v1.15.11 github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6 github.com/minio/minio-go/v7 v7.0.14 github.com/minio/sha256-simd v1.0.0 github.com/ncw/swift/v2 v2.0.0 - github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.6.0 github.com/pkg/sftp v1.13.2 github.com/pkg/xattr v0.4.5 github.com/restic/chunker v0.4.0 - github.com/smartystreets/assertions v1.2.0 // indirect github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e golang.org/x/net v0.0.0-20210614182718-04defd469f4e - golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 + golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 - golang.org/x/text v0.3.7 - google.golang.org/api v0.58.0 - gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect - gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c + golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 + golang.org/x/text v0.3.6 + google.golang.org/api v0.50.0 +) + +require ( + cloud.google.com/go v0.84.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.19 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dnaeon/go-vcr v1.2.0 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect + github.com/gofrs/uuid v4.0.0+incompatible // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/klauspost/cpuid v1.3.1 // indirect + github.com/klauspost/cpuid/v2 v2.0.4 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/minio/md5-simd v1.1.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.2.1 // indirect + github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/tools v0.1.4 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84 // indirect + google.golang.org/grpc v1.38.0 // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) -go 1.16 +go 1.18 diff --git a/go.sum b/go.sum index 45ebce04503..56789536361 100644 --- a/go.sum +++ b/go.sum @@ -21,12 +21,8 @@ cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECH cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0 h1:hVhK90DwCdOAYGME/FJd9vNIZye9HBR6Yy3fu4js3N8= cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1 h1:DwuSvDZ1pTYGbXo8yOJevCTr3BoBlE+OVkHAKiYQUXc= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -52,10 +48,10 @@ github.com/Azure/azure-sdk-for-go v55.6.0+incompatible h1:SDeTdsn7/wiCDVLiKR1VFD github.com/Azure/azure-sdk-for-go v55.6.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.21 h1:w77zY/9RnUAWcIQyDC0Fc89mCvwftR8F+zsR/OH6enk= -github.com/Azure/go-autorest/autorest v0.11.21/go.mod h1:Do/yuMSW/13ayUkcVREpsMHGG+MvV81uzSCFgYPj4tM= -github.com/Azure/go-autorest/autorest/adal v0.9.14 h1:G8hexQdV5D4khOXrWG2YuLCFKhWYmWD8bHYaXN5ophk= -github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest v0.11.19 h1:7/IqD2fEYVha1EPeaiytVKhzmPV223pfkRIQUGOK2IE= +github.com/Azure/go-autorest/autorest v0.11.19/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= @@ -68,7 +64,6 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -80,8 +75,6 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -91,7 +84,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= @@ -111,9 +103,10 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -125,10 +118,6 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0= -github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -142,7 +131,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -196,17 +184,13 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -244,6 +228,7 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -251,6 +236,8 @@ github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= @@ -258,6 +245,7 @@ github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QH github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= @@ -294,8 +282,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/ncw/swift/v2 v2.0.0 h1:Q1jkMe/yhCkx7yAKq4bBZ/Th3NR+ejRcwbVK8Pi1i/0= github.com/ncw/swift/v2 v2.0.0/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -326,12 +312,10 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= @@ -367,7 +351,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= @@ -406,6 +389,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -417,6 +401,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -471,11 +456,8 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210615190721-d04028783cf1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 h1:3B43BWw0xEBsLZ/NO1VALz6fppU3481pik+2Ksv45z8= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 h1:B333XXssMuKQeBwiNODx4TupZy7bf4sxFZnN2ZOcvUE= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -538,14 +520,9 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 h1:KzbpndAYEM+4oHRp9JmB2ewj0NHHxO3Z0g7Gus2O1kk= -golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -607,8 +584,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4 h1:cVngSRcfgyZCzys3KYOpCFa+4dqX/Oub9tAq00ttGVs= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -639,13 +616,8 @@ google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1Avk google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= google.golang.org/api v0.49.0/go.mod h1:BECiH72wsfwUvOVn3+btPD5WHi0LzavZReBndi42L18= +google.golang.org/api v0.50.0 h1:LX7NFCFYOHzr7WHaYiRUpeipZe9o5L8T+2F4Z798VDw= google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.58.0 h1:MDkAbYIB1JpSgCTOCYYoIec/coMlKK4oVbpnBLLcyT0= -google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -700,18 +672,8 @@ google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxH google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20210624174822-c5cf32407d0a/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84 h1:R1r5J0u6Cx+RNl/6mezTw6oA14cmKC96FeUwL6A9bd4= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 h1:ysnBoUyeL/H6RCvNRhWHjKoDEmguI+mPU+qHgK8qv/w= -google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -733,11 +695,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -750,19 +709,15 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 h1:yiW+nvdHb9LVqSHQBXfZCieqV4fzYhNBql77zY0ykqs= -gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index 0c1e6527c9e..1662ada0b77 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -103,16 +103,13 @@ func build(sourceDir, outputDir, goos, goarch string) (filename string) { ) c.Stdout = os.Stdout c.Stderr = os.Stderr - c.Dir = sourceDir - - verbose("run %v %v in %v", "go", c.Args, c.Dir) - c.Dir = sourceDir c.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+goos, "GOARCH="+goarch, ) + verbose("run %v %v in %v", "go", c.Args, c.Dir) err := c.Run() if err != nil { @@ -151,11 +148,9 @@ func compress(goos, inputDir, filename string) (outputFile string) { case "windows": outputFile = strings.TrimSuffix(filename, ".exe") + ".zip" c = exec.Command("zip", "-q", "-X", outputFile, filename) - c.Dir = inputDir default: outputFile = filename + ".bz2" c = exec.Command("bzip2", filename) - c.Dir = inputDir } rm(filepath.Join(inputDir, outputFile)) @@ -163,7 +158,6 @@ func compress(goos, inputDir, filename string) (outputFile string) { c.Stdout = os.Stdout c.Stderr = os.Stderr c.Dir = inputDir - verbose("run %v %v in %v", "go", c.Args, c.Dir) err := c.Run() @@ -188,14 +182,19 @@ func buildForTarget(sourceDir, outputDir, goos, goarch string) (filename string) func buildTargets(sourceDir, outputDir string, targets map[string][]string) { start := time.Now() - msg("building with %d workers", runtime.NumCPU()) + // the go compiler is already parallelized, thus reduce the concurrency a bit + workers := runtime.GOMAXPROCS(0) / 4 + if workers < 1 { + workers = 1 + } + msg("building with %d workers", workers) type Job struct{ GOOS, GOARCH string } var wg errgroup.Group ch := make(chan Job) - for i := 0; i < runtime.NumCPU(); i++ { + for i := 0; i < workers; i++ { wg.Go(func() error { for job := range ch { start := time.Now() @@ -233,6 +232,18 @@ var defaultBuildTargets = map[string][]string{ "solaris": {"amd64"}, } +func downloadModules(sourceDir string) { + c := exec.Command("go", "mod", "download") + c.Stdout = os.Stdout + c.Stderr = os.Stderr + c.Dir = sourceDir + + err := c.Run() + if err != nil { + die("error downloading modules: %v", err) + } +} + func main() { if len(pflag.Args()) != 0 { die("USAGE: build-release-binaries [OPTIONS]") @@ -242,5 +253,6 @@ func main() { outputDir := abs(opts.OutputDir) mkdir(outputDir) + downloadModules(sourceDir) buildTargets(sourceDir, outputDir, defaultBuildTargets) } diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index e22c097ec80..4fcc8e30c10 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -2,7 +2,6 @@ package archiver import ( "context" - "encoding/json" "os" "path" "runtime" @@ -13,7 +12,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" - tomb "gopkg.in/tomb.v2" + "golang.org/x/sync/errgroup" ) // SelectByNameFunc returns true for all items that should be included (files and @@ -27,22 +26,26 @@ type SelectFunc func(item string, fi os.FileInfo) bool // ErrorFunc is called when an error during archiving occurs. When nil is // returned, the archiver continues, otherwise it aborts and passes the error // up the call stack. -type ErrorFunc func(file string, fi os.FileInfo, err error) error +type ErrorFunc func(file string, err error) error // ItemStats collects some statistics about a particular file or directory. type ItemStats struct { - DataBlobs int // number of new data blobs added for this item - DataSize uint64 // sum of the sizes of all new data blobs - TreeBlobs int // number of new tree blobs added for this item - TreeSize uint64 // sum of the sizes of all new tree blobs + DataBlobs int // number of new data blobs added for this item + DataSize uint64 // sum of the sizes of all new data blobs + DataSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead) + TreeBlobs int // number of new tree blobs added for this item + TreeSize uint64 // sum of the sizes of all new tree blobs + TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead) } // Add adds other to the current ItemStats. func (s *ItemStats) Add(other ItemStats) { s.DataBlobs += other.DataBlobs s.DataSize += other.DataSize + s.DataSizeInRepo += other.DataSizeInRepo s.TreeBlobs += other.TreeBlobs s.TreeSize += other.TreeSize + s.TreeSizeInRepo += other.TreeSizeInRepo } // Archiver saves a directory structure to the repo. @@ -118,13 +121,18 @@ func (o Options) ApplyDefaults() Options { } if o.SaveBlobConcurrency == 0 { - o.SaveBlobConcurrency = uint(runtime.NumCPU()) + // blob saving is CPU bound due to hash checking and encryption + // the actual upload is handled by the repository itself + o.SaveBlobConcurrency = uint(runtime.GOMAXPROCS(0)) } if o.SaveTreeConcurrency == 0 { - // use a relatively high concurrency here, having multiple SaveTree - // workers is cheap - o.SaveTreeConcurrency = o.SaveBlobConcurrency * 20 + // can either wait for a file, wait for a tree, serialize a tree or wait for saveblob + // the last two are cpu-bound and thus mutually exclusive. + // Also allow waiting for FileReadConcurrency files, this is the maximum of FutureFiles + // which currently can be in progress. The main backup loop blocks when trying to queue + // more files to read. + o.SaveTreeConcurrency = uint(runtime.GOMAXPROCS(0)) + o.FileReadConcurrency } return o @@ -148,7 +156,7 @@ func New(repo restic.Repository, fs fs.FS, opts Options) *Archiver { } // error calls arch.Error if it is set and the error is different from context.Canceled. -func (arch *Archiver) error(item string, fi os.FileInfo, err error) error { +func (arch *Archiver) error(item string, err error) error { if arch.Error == nil || err == nil { return err } @@ -157,7 +165,7 @@ func (arch *Archiver) error(item string, fi os.FileInfo, err error) error { return err } - errf := arch.Error(item, fi, err) + errf := arch.Error(item, err) if err != errf { debug.Log("item %v: error was filtered by handler, before: %q, after: %v", item, err, errf) } @@ -166,30 +174,27 @@ func (arch *Archiver) error(item string, fi os.FileInfo, err error) error { // saveTree stores a tree in the repo. It checks the index and the known blobs // before saving anything. -func (arch *Archiver) saveTree(ctx context.Context, t *restic.Tree) (restic.ID, ItemStats, error) { +func (arch *Archiver) saveTree(ctx context.Context, t *restic.TreeJSONBuilder) (restic.ID, ItemStats, error) { var s ItemStats - buf, err := json.Marshal(t) + buf, err := t.Finalize() if err != nil { - return restic.ID{}, s, errors.Wrap(err, "MarshalJSON") + return restic.ID{}, s, err } - // append a newline so that the data is always consistent (json.Encoder - // adds a newline after each object) - buf = append(buf, '\n') - b := &Buffer{Data: buf} res := arch.blobSaver.Save(ctx, restic.TreeBlob, b) - res.Wait(ctx) - if !res.Known() { + sbr := res.Take(ctx) + if !sbr.known { s.TreeBlobs++ - s.TreeSize += uint64(len(buf)) + s.TreeSize += uint64(sbr.length) + s.TreeSizeInRepo += uint64(sbr.sizeInRepo) } - // The context was canceled in the meantime, res.ID() might be invalid + // The context was canceled in the meantime, id might be invalid if ctx.Err() != nil { return restic.ID{}, s, ctx.Err() } - return res.ID(), s, nil + return sbr.id, s, nil } // nodeFromFileInfo returns the restic node from an os.FileInfo. @@ -208,7 +213,7 @@ func (arch *Archiver) loadSubtree(ctx context.Context, node *restic.Node) (*rest return nil, nil } - tree, err := arch.Repo.LoadTree(ctx, *node.Subtree) + tree, err := restic.LoadTree(ctx, arch.Repo, *node.Subtree) if err != nil { debug.Log("unable to load tree %v: %v", node.Subtree.Str(), err) // a tree in the repository is not readable -> warn the user @@ -229,17 +234,17 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { // SaveDir stores a directory in the repo and returns the node. snPath is the // path within the current snapshot. -func (arch *Archiver) SaveDir(ctx context.Context, snPath string, fi os.FileInfo, dir string, previous *restic.Tree, complete CompleteFunc) (d FutureTree, err error) { +func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) { debug.Log("%v %v", snPath, dir) treeNode, err := arch.nodeFromFileInfo(dir, fi) if err != nil { - return FutureTree{}, err + return FutureNode{}, err } names, err := readdirnames(arch.FS, dir, fs.O_NOFOLLOW) if err != nil { - return FutureTree{}, err + return FutureNode{}, err } sort.Strings(names) @@ -249,7 +254,7 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, fi os.FileInfo // test if context has been cancelled if ctx.Err() != nil { debug.Log("context has been cancelled, aborting") - return FutureTree{}, ctx.Err() + return FutureNode{}, ctx.Err() } pathname := arch.FS.Join(dir, name) @@ -259,13 +264,13 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, fi os.FileInfo // return error early if possible if err != nil { - err = arch.error(pathname, fi, err) + err = arch.error(pathname, err) if err == nil { // ignore error continue } - return FutureTree{}, err + return FutureNode{}, err } if excluded { @@ -275,54 +280,58 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, fi os.FileInfo nodes = append(nodes, fn) } - ft := arch.treeSaver.Save(ctx, snPath, treeNode, nodes, complete) + fn := arch.treeSaver.Save(ctx, snPath, dir, treeNode, nodes, complete) - return ft, nil + return fn, nil } -// FutureNode holds a reference to a node, FutureFile, or FutureTree. +// FutureNode holds a reference to a channel that returns a FutureNodeResult +// or a reference to an already existing result. If the result is available +// immediatelly, then storing a reference directly requires less memory than +// using the indirection via a channel. type FutureNode struct { - snPath, target string + ch <-chan futureNodeResult + res *futureNodeResult +} - // kept to call the error callback function - absTarget string - fi os.FileInfo +type futureNodeResult struct { + snPath, target string node *restic.Node stats ItemStats err error +} - isFile bool - file FutureFile - isTree bool - tree FutureTree +func newFutureNode() (FutureNode, chan<- futureNodeResult) { + ch := make(chan futureNodeResult, 1) + return FutureNode{ch: ch}, ch } -func (fn *FutureNode) wait(ctx context.Context) { - switch { - case fn.isFile: - // wait for and collect the data for the file - fn.file.Wait(ctx) - fn.node = fn.file.Node() - fn.err = fn.file.Err() - fn.stats = fn.file.Stats() - - // ensure the other stuff can be garbage-collected - fn.file = FutureFile{} - fn.isFile = false - - case fn.isTree: - // wait for and collect the data for the dir - fn.tree.Wait(ctx) - fn.node = fn.tree.Node() - fn.stats = fn.tree.Stats() - - // ensure the other stuff can be garbage-collected - fn.tree = FutureTree{} - fn.isTree = false +func newFutureNodeWithResult(res futureNodeResult) FutureNode { + return FutureNode{ + res: &res, } } +func (fn *FutureNode) take(ctx context.Context) futureNodeResult { + if fn.res != nil { + res := fn.res + // free result + fn.res = nil + return *res + } + select { + case res, ok := <-fn.ch: + if ok { + // free channel + fn.ch = nil + return res + } + case <-ctx.Done(): + } + return futureNodeResult{} +} + // allBlobsPresent checks if all blobs (contents) of the given node are // present in the index. func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool { @@ -345,19 +354,12 @@ func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool { func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) { start := time.Now() - fn = FutureNode{ - snPath: snPath, - target: target, - } - debug.Log("%v target %q, previous %v", snPath, target, previous) abstarget, err := arch.FS.Abs(target) if err != nil { return FutureNode{}, false, err } - fn.absTarget = abstarget - // exclude files by path before running Lstat to reduce number of lstat calls if !arch.SelectByName(abstarget) { debug.Log("%v is excluded by path", target) @@ -368,7 +370,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous fi, err := arch.FS.Lstat(target) if err != nil { debug.Log("lstat() for %v returned error: %v", target, err) - err = arch.error(abstarget, fi, err) + err = arch.error(abstarget, err) if err != nil { return FutureNode{}, false, errors.Wrap(err, "Lstat") } @@ -391,21 +393,26 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous debug.Log("%v hasn't changed, using old list of blobs", target) arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start)) arch.CompleteBlob(snPath, previous.Size) - fn.node, err = arch.nodeFromFileInfo(target, fi) + node, err := arch.nodeFromFileInfo(target, fi) if err != nil { return FutureNode{}, false, err } // copy list of blobs - fn.node.Content = previous.Content + node.Content = previous.Content + fn = newFutureNodeWithResult(futureNodeResult{ + snPath: snPath, + target: target, + node: node, + }) return fn, false, nil } debug.Log("%v hasn't changed, but contents are missing!", target) // There are contents missing - inform user! err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target) - err = arch.error(abstarget, fi, err) + err = arch.error(abstarget, err) if err != nil { return FutureNode{}, false, err } @@ -416,7 +423,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0) if err != nil { debug.Log("Openfile() for %v returned error: %v", target, err) - err = arch.error(abstarget, fi, err) + err = arch.error(abstarget, err) if err != nil { return FutureNode{}, false, errors.Wrap(err, "Lstat") } @@ -427,7 +434,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous if err != nil { debug.Log("stat() on opened file %v returned error: %v", target, err) _ = file.Close() - err = arch.error(abstarget, fi, err) + err = arch.error(abstarget, err) if err != nil { return FutureNode{}, false, errors.Wrap(err, "Lstat") } @@ -438,16 +445,15 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous if !fs.IsRegularFile(fi) { err = errors.Errorf("file %v changed type, refusing to archive") _ = file.Close() - err = arch.error(abstarget, fi, err) + err = arch.error(abstarget, err) if err != nil { return FutureNode{}, false, err } return FutureNode{}, true, nil } - fn.isFile = true // Save will close the file, we don't need to do that - fn.file = arch.fileSaver.Save(ctx, snPath, file, fi, func() { + fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() { arch.StartFile(snPath) }, func(node *restic.Node, stats ItemStats) { arch.CompleteItem(snPath, previous, node, stats, time.Since(start)) @@ -460,14 +466,13 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous start := time.Now() oldSubtree, err := arch.loadSubtree(ctx, previous) if err != nil { - err = arch.error(abstarget, fi, err) + err = arch.error(abstarget, err) } if err != nil { return FutureNode{}, false, err } - fn.isTree = true - fn.tree, err = arch.SaveDir(ctx, snPath, fi, target, oldSubtree, + fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree, func(node *restic.Node, stats ItemStats) { arch.CompleteItem(snItem, previous, node, stats, time.Since(start)) }) @@ -483,10 +488,15 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous default: debug.Log(" %v other", target) - fn.node, err = arch.nodeFromFileInfo(target, fi) + node, err := arch.nodeFromFileInfo(target, fi) if err != nil { return FutureNode{}, false, err } + fn = newFutureNodeWithResult(futureNodeResult{ + snPath: snPath, + target: target, + node: node, + }) } debug.Log("return after %.3f", time.Since(start).Seconds()) @@ -569,7 +579,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name)) if err != nil { - err = arch.error(subatree.Path, fn.fi, err) + err = arch.error(subatree.Path, err) if err == nil { // ignore error continue @@ -593,7 +603,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, oldNode := previous.Find(name) oldSubtree, err := arch.loadSubtree(ctx, oldNode) if err != nil { - err = arch.error(join(snPath, name), nil, err) + err = arch.error(join(snPath, name), err) } if err != nil { return nil, err @@ -605,7 +615,11 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, return nil, err } - id, nodeStats, err := arch.saveTree(ctx, subtree) + tb, err := restic.TreeToBuilder(subtree) + if err != nil { + return nil, err + } + id, nodeStats, err := arch.saveTree(ctx, tb) if err != nil { return nil, err } @@ -643,28 +657,28 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, // process all futures for name, fn := range futureNodes { - fn.wait(ctx) + fnr := fn.take(ctx) // return the error, or ignore it - if fn.err != nil { - fn.err = arch.error(fn.target, fn.fi, fn.err) - if fn.err == nil { + if fnr.err != nil { + fnr.err = arch.error(fnr.target, fnr.err) + if fnr.err == nil { // ignore error continue } - return nil, fn.err + return nil, fnr.err } // when the error is ignored, the node could not be saved, so ignore it - if fn.node == nil { - debug.Log("%v excluded: %v", fn.snPath, fn.target) + if fnr.node == nil { + debug.Log("%v excluded: %v", fnr.snPath, fnr.target) continue } - fn.node.Name = name + fnr.node.Name = name - err := tree.Insert(fn.node) + err := tree.Insert(fnr.node) if err != nil { return nil, err } @@ -752,27 +766,36 @@ func (arch *Archiver) loadParentTree(ctx context.Context, snapshotID restic.ID) } debug.Log("load parent tree %v", *sn.Tree) - tree, err := arch.Repo.LoadTree(ctx, *sn.Tree) + tree, err := restic.LoadTree(ctx, arch.Repo, *sn.Tree) if err != nil { debug.Log("unable to load tree %v: %v", *sn.Tree, err) - _ = arch.error("/", nil, arch.wrapLoadTreeError(*sn.Tree, err)) + _ = arch.error("/", arch.wrapLoadTreeError(*sn.Tree, err)) return nil } return tree } // runWorkers starts the worker pools, which are stopped when the context is cancelled. -func (arch *Archiver) runWorkers(ctx context.Context, t *tomb.Tomb) { - arch.blobSaver = NewBlobSaver(ctx, t, arch.Repo, arch.Options.SaveBlobConcurrency) +func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group) { + arch.blobSaver = NewBlobSaver(ctx, wg, arch.Repo, arch.Options.SaveBlobConcurrency) - arch.fileSaver = NewFileSaver(ctx, t, + arch.fileSaver = NewFileSaver(ctx, wg, arch.blobSaver.Save, arch.Repo.Config().ChunkerPolynomial, arch.Options.FileReadConcurrency, arch.Options.SaveBlobConcurrency) arch.fileSaver.CompleteBlob = arch.CompleteBlob arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo - arch.treeSaver = NewTreeSaver(ctx, t, arch.Options.SaveTreeConcurrency, arch.saveTree, arch.Error) + arch.treeSaver = NewTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.saveTree, arch.Error) +} + +func (arch *Archiver) stopWorkers() { + arch.blobSaver.TriggerShutdown() + arch.fileSaver.TriggerShutdown() + arch.treeSaver.TriggerShutdown() + arch.blobSaver = nil + arch.fileSaver = nil + arch.treeSaver = nil } // Snapshot saves several targets and returns a snapshot. @@ -787,42 +810,51 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps return nil, restic.ID{}, err } - var t tomb.Tomb - wctx := t.Context(ctx) - start := time.Now() - var rootTreeID restic.ID - var stats ItemStats - t.Go(func() error { - arch.runWorkers(wctx, &t) - debug.Log("starting snapshot") - tree, err := arch.SaveTree(wctx, "/", atree, arch.loadParentTree(wctx, opts.ParentSnapshot)) - if err != nil { - return err - } + wgUp, wgUpCtx := errgroup.WithContext(ctx) + arch.Repo.StartPackUploader(wgUpCtx, wgUp) - if len(tree.Nodes) == 0 { - return errors.New("snapshot is empty") - } + wgUp.Go(func() error { + wg, wgCtx := errgroup.WithContext(wgUpCtx) + start := time.Now() - rootTreeID, stats, err = arch.saveTree(wctx, tree) - // trigger shutdown but don't set an error - t.Kill(nil) - return err - }) + var stats ItemStats + wg.Go(func() error { + arch.runWorkers(wgCtx, wg) - err = t.Wait() - debug.Log("err is %v", err) + debug.Log("starting snapshot") + tree, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot)) + if err != nil { + return err + } - if err != nil { - debug.Log("error while saving tree: %v", err) - return nil, restic.ID{}, err - } + if len(tree.Nodes) == 0 { + return errors.New("snapshot is empty") + } - arch.CompleteItem("/", nil, nil, stats, time.Since(start)) + tb, err := restic.TreeToBuilder(tree) + if err != nil { + return err + } + rootTreeID, stats, err = arch.saveTree(wgCtx, tb) + arch.stopWorkers() + return err + }) + + err = wg.Wait() + debug.Log("err is %v", err) - err = arch.Repo.Flush(ctx) + if err != nil { + debug.Log("error while saving tree: %v", err) + return err + } + + arch.CompleteItem("/", nil, nil, stats, time.Since(start)) + + return arch.Repo.Flush(ctx) + }) + err = wgUp.Wait() if err != nil { return nil, restic.ID{}, err } @@ -839,7 +871,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps } sn.Tree = &rootTreeID - id, err := arch.Repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn) + id, err := restic.SaveSnapshot(ctx, arch.Repo, sn) if err != nil { return nil, restic.ID{}, err } diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index e18156cebd7..a6485234f3a 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -23,7 +23,7 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" restictest "github.com/restic/restic/internal/test" - tomb "gopkg.in/tomb.v2" + "golang.org/x/sync/errgroup" ) func prepareTempdirRepoSrc(t testing.TB, src TestDir) (tempdir string, repo restic.Repository, cleanup func()) { @@ -41,13 +41,13 @@ func prepareTempdirRepoSrc(t testing.TB, src TestDir) (tempdir string, repo rest } func saveFile(t testing.TB, repo restic.Repository, filename string, filesystem fs.FS) (*restic.Node, ItemStats) { - var tmb tomb.Tomb - ctx := tmb.Context(context.Background()) + wg, ctx := errgroup.WithContext(context.TODO()) + repo.StartPackUploader(ctx, wg) arch := New(repo, filesystem, Options{}) - arch.runWorkers(ctx, &tmb) + arch.runWorkers(ctx, wg) - arch.Error = func(item string, fi os.FileInfo, err error) error { + arch.Error = func(item string, err error) error { t.Errorf("archiver error for %v: %v", item, err) return err } @@ -80,21 +80,20 @@ func saveFile(t testing.TB, repo restic.Repository, filename string, filesystem t.Fatal(err) } - res := arch.fileSaver.Save(ctx, "/", file, fi, start, complete) + res := arch.fileSaver.Save(ctx, "/", filename, file, fi, start, complete) - res.Wait(ctx) - if res.Err() != nil { - t.Fatal(res.Err()) + fnr := res.take(ctx) + if fnr.err != nil { + t.Fatal(fnr.err) } - tmb.Kill(nil) - err = tmb.Wait() + arch.stopWorkers() + err = repo.Flush(context.Background()) if err != nil { t.Fatal(err) } - err = repo.Flush(context.Background()) - if err != nil { + if err := wg.Wait(); err != nil { t.Fatal(err) } @@ -110,15 +109,15 @@ func saveFile(t testing.TB, repo restic.Repository, filename string, filesystem t.Errorf("no node returned for complete callback") } - if completeCallbackNode != nil && !res.Node().Equals(*completeCallbackNode) { + if completeCallbackNode != nil && !fnr.node.Equals(*completeCallbackNode) { t.Errorf("different node returned for complete callback") } - if completeCallbackStats != res.Stats() { - t.Errorf("different stats return for complete callback, want:\n %v\ngot:\n %v", res.Stats(), completeCallbackStats) + if completeCallbackStats != fnr.stats { + t.Errorf("different stats return for complete callback, want:\n %v\ngot:\n %v", fnr.stats, completeCallbackStats) } - return res.Node(), res.Stats() + return fnr.node, fnr.stats } func TestArchiverSaveFile(t *testing.T) { @@ -214,14 +213,15 @@ func TestArchiverSave(t *testing.T) { tempdir, repo, cleanup := prepareTempdirRepoSrc(t, TestDir{"file": testfile}) defer cleanup() - var tmb tomb.Tomb + wg, ctx := errgroup.WithContext(ctx) + repo.StartPackUploader(ctx, wg) arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) - arch.Error = func(item string, fi os.FileInfo, err error) error { + arch.Error = func(item string, err error) error { t.Errorf("archiver error for %v: %v", item, err) return err } - arch.runWorkers(tmb.Context(ctx), &tmb) + arch.runWorkers(ctx, wg) node, excluded, err := arch.Save(ctx, "/", filepath.Join(tempdir, "file"), nil) if err != nil { @@ -232,23 +232,24 @@ func TestArchiverSave(t *testing.T) { t.Errorf("Save() excluded the node, that's unexpected") } - node.wait(ctx) - if node.err != nil { - t.Fatal(node.err) + fnr := node.take(ctx) + if fnr.err != nil { + t.Fatal(fnr.err) } - if node.node == nil { + if fnr.node == nil { t.Fatalf("returned node is nil") } - stats := node.stats + stats := fnr.stats + arch.stopWorkers() err = repo.Flush(ctx) if err != nil { t.Fatal(err) } - TestEnsureFileContent(ctx, t, repo, "file", node.node, testfile) + TestEnsureFileContent(ctx, t, repo, "file", fnr.node, testfile) if stats.DataSize != uint64(len(testfile.Content)) { t.Errorf("wrong stats returned in DataSize, want %d, got %d", len(testfile.Content), stats.DataSize) } @@ -281,6 +282,9 @@ func TestArchiverSaveReaderFS(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() + wg, ctx := errgroup.WithContext(ctx) + repo.StartPackUploader(ctx, wg) + ts := time.Now() filename := "xx" readerFs := &fs.Reader{ @@ -290,14 +294,12 @@ func TestArchiverSaveReaderFS(t *testing.T) { ReadCloser: ioutil.NopCloser(strings.NewReader(test.Data)), } - var tmb tomb.Tomb - arch := New(repo, readerFs, Options{}) - arch.Error = func(item string, fi os.FileInfo, err error) error { + arch.Error = func(item string, err error) error { t.Errorf("archiver error for %v: %v", item, err) return err } - arch.runWorkers(tmb.Context(ctx), &tmb) + arch.runWorkers(ctx, wg) node, excluded, err := arch.Save(ctx, "/", filename, nil) t.Logf("Save returned %v %v", node, err) @@ -309,23 +311,24 @@ func TestArchiverSaveReaderFS(t *testing.T) { t.Errorf("Save() excluded the node, that's unexpected") } - node.wait(ctx) - if node.err != nil { - t.Fatal(node.err) + fnr := node.take(ctx) + if fnr.err != nil { + t.Fatal(fnr.err) } - if node.node == nil { + if fnr.node == nil { t.Fatalf("returned node is nil") } - stats := node.stats + stats := fnr.stats + arch.stopWorkers() err = repo.Flush(ctx) if err != nil { t.Fatal(err) } - TestEnsureFileContent(ctx, t, repo, "file", node.node, TestFile{Content: test.Data}) + TestEnsureFileContent(ctx, t, repo, "file", fnr.node, TestFile{Content: test.Data}) if stats.DataSize != uint64(len(test.Data)) { t.Errorf("wrong stats returned in DataSize, want %d, got %d", len(test.Data), stats.DataSize) } @@ -415,20 +418,20 @@ type blobCountingRepo struct { saved map[restic.BlobHandle]uint } -func (repo *blobCountingRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, error) { - id, exists, err := repo.Repository.SaveBlob(ctx, t, buf, id, false) +func (repo *blobCountingRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) { + id, exists, size, err := repo.Repository.SaveBlob(ctx, t, buf, id, false) if exists { - return id, exists, err + return id, exists, size, err } h := restic.BlobHandle{ID: id, Type: t} repo.m.Lock() repo.saved[h]++ repo.m.Unlock() - return id, exists, err + return id, exists, size, err } func (repo *blobCountingRepo) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, error) { - id, err := repo.Repository.SaveTree(ctx, t) + id, err := restic.SaveTree(ctx, repo.Repository, t) h := restic.BlobHandle{ID: id, Type: restic.TreeBlob} repo.m.Lock() repo.saved[h]++ @@ -826,14 +829,14 @@ func TestArchiverSaveDir(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { - var tmb tomb.Tomb - ctx := tmb.Context(context.Background()) - tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src) defer cleanup() + wg, ctx := errgroup.WithContext(context.Background()) + repo.StartPackUploader(ctx, wg) + arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) - arch.runWorkers(ctx, &tmb) + arch.runWorkers(ctx, wg) chdir := tempdir if test.chdir != "" { @@ -848,19 +851,13 @@ func TestArchiverSaveDir(t *testing.T) { t.Fatal(err) } - ft, err := arch.SaveDir(ctx, "/", fi, test.target, nil, nil) + ft, err := arch.SaveDir(ctx, "/", test.target, fi, nil, nil) if err != nil { t.Fatal(err) } - ft.Wait(ctx) - node, stats := ft.Node(), ft.Stats() - - tmb.Kill(nil) - err = tmb.Wait() - if err != nil { - t.Fatal(err) - } + fnr := ft.take(ctx) + node, stats := fnr.node, fnr.stats t.Logf("stats: %v", stats) if stats.DataSize != 0 { @@ -876,24 +873,29 @@ func TestArchiverSaveDir(t *testing.T) { t.Errorf("wrong stats returned in TreeBlobs, want > 0, got %d", stats.TreeBlobs) } - ctx = context.Background() node.Name = targetNodeName tree := &restic.Tree{Nodes: []*restic.Node{node}} - treeID, err := repo.SaveTree(ctx, tree) + treeID, err := restic.SaveTree(ctx, repo, tree) if err != nil { t.Fatal(err) } + arch.stopWorkers() err = repo.Flush(ctx) if err != nil { t.Fatal(err) } + err = wg.Wait() + if err != nil { + t.Fatal(err) + } + want := test.want if want == nil { want = test.src } - TestEnsureTree(ctx, t, "/", repo, treeID, want) + TestEnsureTree(context.TODO(), t, "/", repo, treeID, want) }) } } @@ -915,27 +917,25 @@ func TestArchiverSaveDirIncremental(t *testing.T) { // save the empty directory several times in a row, then have a look if the // archiver did save the same tree several times for i := 0; i < 5; i++ { - var tmb tomb.Tomb - ctx := tmb.Context(context.Background()) + wg, ctx := errgroup.WithContext(context.TODO()) + repo.StartPackUploader(ctx, wg) arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) - arch.runWorkers(ctx, &tmb) + arch.runWorkers(ctx, wg) fi, err := fs.Lstat(tempdir) if err != nil { t.Fatal(err) } - ft, err := arch.SaveDir(ctx, "/", fi, tempdir, nil, nil) + ft, err := arch.SaveDir(ctx, "/", tempdir, fi, nil, nil) if err != nil { t.Fatal(err) } - ft.Wait(ctx) - node, stats := ft.Node(), ft.Stats() + fnr := ft.take(ctx) + node, stats := fnr.node, fnr.stats - tmb.Kill(nil) - err = tmb.Wait() if err != nil { t.Fatal(err) } @@ -972,7 +972,12 @@ func TestArchiverSaveDirIncremental(t *testing.T) { t.Logf("node subtree %v", node.Subtree) - err = repo.Flush(context.Background()) + arch.stopWorkers() + err = repo.Flush(ctx) + if err != nil { + t.Fatal(err) + } + err = wg.Wait() if err != nil { t.Fatal(err) } @@ -1019,7 +1024,7 @@ func TestArchiverSaveTree(t *testing.T) { want: TestDir{ "targetfile": TestFile{Content: string("foobar")}, }, - stat: ItemStats{1, 6, 0, 0}, + stat: ItemStats{1, 6, 32 + 6, 0, 0, 0}, }, { src: TestDir{ @@ -1031,7 +1036,7 @@ func TestArchiverSaveTree(t *testing.T) { "targetfile": TestFile{Content: string("foobar")}, "filesymlink": TestSymlink{Target: "targetfile"}, }, - stat: ItemStats{1, 6, 0, 0}, + stat: ItemStats{1, 6, 32 + 6, 0, 0, 0}, }, { src: TestDir{ @@ -1051,7 +1056,7 @@ func TestArchiverSaveTree(t *testing.T) { "symlink": TestSymlink{Target: "subdir"}, }, }, - stat: ItemStats{0, 0, 1, 0x154}, + stat: ItemStats{0, 0, 0, 1, 0x154, 0x16a}, }, { src: TestDir{ @@ -1075,15 +1080,12 @@ func TestArchiverSaveTree(t *testing.T) { }, }, }, - stat: ItemStats{1, 6, 3, 0x47f}, + stat: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1}, }, } for _, test := range tests { t.Run("", func(t *testing.T) { - var tmb tomb.Tomb - ctx := tmb.Context(context.Background()) - tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src) defer cleanup() @@ -1099,7 +1101,10 @@ func TestArchiverSaveTree(t *testing.T) { stat.Add(s) } - arch.runWorkers(ctx, &tmb) + wg, ctx := errgroup.WithContext(context.TODO()) + repo.StartPackUploader(ctx, wg) + + arch.runWorkers(ctx, wg) back := restictest.Chdir(t, tempdir) defer back() @@ -1118,19 +1123,17 @@ func TestArchiverSaveTree(t *testing.T) { t.Fatal(err) } - treeID, err := repo.SaveTree(ctx, tree) + treeID, err := restic.SaveTree(ctx, repo, tree) if err != nil { t.Fatal(err) } - tmb.Kill(nil) - err = tmb.Wait() + arch.stopWorkers() + err = repo.Flush(ctx) if err != nil { t.Fatal(err) } - - ctx = context.Background() - err = repo.Flush(ctx) + err = wg.Wait() if err != nil { t.Fatal(err) } @@ -1139,11 +1142,12 @@ func TestArchiverSaveTree(t *testing.T) { if want == nil { want = test.src } - TestEnsureTree(ctx, t, "/", repo, treeID, want) + TestEnsureTree(context.TODO(), t, "/", repo, treeID, want) bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs)) bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs)) bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize) - bothZeroOrNeither(t, test.stat.TreeSize, stat.TreeSize) + bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo) + bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo) }) } } @@ -1719,7 +1723,7 @@ func TestArchiverParent(t *testing.T) { func TestArchiverErrorReporting(t *testing.T) { ignoreErrorForBasename := func(basename string) ErrorFunc { - return func(item string, fi os.FileInfo, err error) error { + return func(item string, err error) error { if filepath.Base(item) == "targetfile" { t.Logf("ignoring error for targetfile: %v", err) return nil @@ -1894,7 +1898,7 @@ func TestArchiverContextCanceled(t *testing.T) { defer removeTempdir() // Ensure that the archiver itself reports the canceled context and not just the backend - repo, _ := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}) + repo, _ := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0) back := restictest.Chdir(t, tempdir) defer back() @@ -1947,10 +1951,10 @@ type failSaveRepo struct { err error } -func (f *failSaveRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, error) { +func (f *failSaveRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) { val := atomic.AddInt32(&f.cnt, 1) if val >= f.failAfter { - return restic.ID{}, false, f.err + return restic.ID{}, false, 0, f.err } return f.Repository.SaveBlob(ctx, t, buf, id, storeDuplicate) @@ -2040,8 +2044,8 @@ func TestArchiverAbortEarlyOnError(t *testing.T) { }) _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) - if errors.Cause(err) != test.err { - t.Errorf("expected error (%v) not found, got %v", test.err, errors.Cause(err)) + if !errors.Is(err, test.err) { + t.Errorf("expected error (%v) not found, got %v", test.err, err) } t.Logf("Snapshot return error: %v", err) @@ -2072,7 +2076,7 @@ func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent restic.ID, t.Fatal(err) } - tree, err := repo.LoadTree(ctx, *snapshot.Tree) + tree, err := restic.LoadTree(ctx, repo, *snapshot.Tree) if err != nil { t.Fatal(err) } @@ -2240,14 +2244,15 @@ func TestRacyFileSwap(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - var tmb tomb.Tomb + wg, ctx := errgroup.WithContext(ctx) + repo.StartPackUploader(ctx, wg) arch := New(repo, fs.Track{FS: statfs}, Options{}) - arch.Error = func(item string, fi os.FileInfo, err error) error { + arch.Error = func(item string, err error) error { t.Logf("archiver error as expected for %v: %v", item, err) return err } - arch.runWorkers(tmb.Context(ctx), &tmb) + arch.runWorkers(ctx, wg) // fs.Track will panic if the file was not closed _, excluded, err := arch.Save(ctx, "/", tempfile, nil) diff --git a/internal/archiver/archiver_unix_test.go b/internal/archiver/archiver_unix_test.go index f7e827e7e96..1167f68529c 100644 --- a/internal/archiver/archiver_unix_test.go +++ b/internal/archiver/archiver_unix_test.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package archiver diff --git a/internal/archiver/archiver_windows_test.go b/internal/archiver/archiver_windows_test.go index 9b3d7789844..1254e64ee27 100644 --- a/internal/archiver/archiver_windows_test.go +++ b/internal/archiver/archiver_windows_test.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package archiver diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index c59f0813379..b2b5e59bb81 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -5,13 +5,12 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" - tomb "gopkg.in/tomb.v2" + "golang.org/x/sync/errgroup" ) // Saver allows saving a blob. type Saver interface { - SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, error) - Index() restic.MasterIndex + SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) } // BlobSaver concurrently saves incoming blobs to the repo. @@ -22,7 +21,7 @@ type BlobSaver struct { // NewBlobSaver returns a new blob. A worker pool is started, it is stopped // when ctx is cancelled. -func NewBlobSaver(ctx context.Context, t *tomb.Tomb, repo Saver, workers uint) *BlobSaver { +func NewBlobSaver(ctx context.Context, wg *errgroup.Group, repo Saver, workers uint) *BlobSaver { ch := make(chan saveBlobJob) s := &BlobSaver{ repo: repo, @@ -30,18 +29,22 @@ func NewBlobSaver(ctx context.Context, t *tomb.Tomb, repo Saver, workers uint) * } for i := uint(0); i < workers; i++ { - t.Go(func() error { - return s.worker(t.Context(ctx), ch) + wg.Go(func() error { + return s.worker(ctx, ch) }) } return s } +func (s *BlobSaver) TriggerShutdown() { + close(s.ch) +} + // Save stores a blob in the repo. It checks the index and the known blobs // before saving anything. It takes ownership of the buffer passed in. func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer) FutureBlob { - ch := make(chan saveBlobResponse, 1) + ch := make(chan SaveBlobResponse, 1) select { case s.ch <- saveBlobJob{BlobType: t, buf: buf, ch: ch}: case <-ctx.Done(): @@ -50,74 +53,76 @@ func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer) Fu return FutureBlob{ch: ch} } - return FutureBlob{ch: ch, length: len(buf.Data)} + return FutureBlob{ch: ch} } // FutureBlob is returned by SaveBlob and will return the data once it has been processed. type FutureBlob struct { - ch <-chan saveBlobResponse - length int - res saveBlobResponse + ch <-chan SaveBlobResponse } -// Wait blocks until the result is available or the context is cancelled. -func (s *FutureBlob) Wait(ctx context.Context) { +func (s *FutureBlob) Poll() *SaveBlobResponse { select { - case <-ctx.Done(): - return case res, ok := <-s.ch: if ok { - s.res = res + return &res } + default: } + return nil } -// ID returns the ID of the blob after it has been saved. -func (s *FutureBlob) ID() restic.ID { - return s.res.id -} - -// Known returns whether or not the blob was already known. -func (s *FutureBlob) Known() bool { - return s.res.known -} - -// Length returns the length of the blob. -func (s *FutureBlob) Length() int { - return s.length +// Take blocks until the result is available or the context is cancelled. +func (s *FutureBlob) Take(ctx context.Context) SaveBlobResponse { + select { + case res, ok := <-s.ch: + if ok { + return res + } + case <-ctx.Done(): + } + return SaveBlobResponse{} } type saveBlobJob struct { restic.BlobType buf *Buffer - ch chan<- saveBlobResponse + ch chan<- SaveBlobResponse } -type saveBlobResponse struct { - id restic.ID - known bool +type SaveBlobResponse struct { + id restic.ID + length int + sizeInRepo int + known bool } -func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) { - id, known, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) +func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (SaveBlobResponse, error) { + id, known, sizeInRepo, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) if err != nil { - return saveBlobResponse{}, err + return SaveBlobResponse{}, err } - return saveBlobResponse{ - id: id, - known: known, + return SaveBlobResponse{ + id: id, + length: len(buf), + sizeInRepo: sizeInRepo, + known: known, }, nil } func (s *BlobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error { for { var job saveBlobJob + var ok bool select { case <-ctx.Done(): return nil - case job = <-jobs: + case job, ok = <-jobs: + if !ok { + return nil + } } res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data) diff --git a/internal/archiver/blob_saver_test.go b/internal/archiver/blob_saver_test.go index df63d3209a6..481139a3f17 100644 --- a/internal/archiver/blob_saver_test.go +++ b/internal/archiver/blob_saver_test.go @@ -10,7 +10,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" - tomb "gopkg.in/tomb.v2" + "golang.org/x/sync/errgroup" ) var errTest = errors.New("test error") @@ -21,13 +21,13 @@ type saveFail struct { failAt int32 } -func (b *saveFail) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicates bool) (restic.ID, bool, error) { +func (b *saveFail) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicates bool) (restic.ID, bool, int, error) { val := atomic.AddInt32(&b.cnt, 1) if val == b.failAt { - return restic.ID{}, false, errTest + return restic.ID{}, false, 0, errTest } - return id, false, nil + return id, false, 0, nil } func (b *saveFail) Index() restic.MasterIndex { @@ -38,12 +38,12 @@ func TestBlobSaver(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tmb, ctx := tomb.WithContext(ctx) + wg, ctx := errgroup.WithContext(ctx) saver := &saveFail{ idx: repository.NewMasterIndex(), } - b := NewBlobSaver(ctx, tmb, saver, uint(runtime.NumCPU())) + b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) var results []FutureBlob @@ -54,15 +54,15 @@ func TestBlobSaver(t *testing.T) { } for i, blob := range results { - blob.Wait(ctx) - if blob.Known() { + sbr := blob.Take(ctx) + if sbr.known { t.Errorf("blob %v is known, that should not be the case", i) } } - tmb.Kill(nil) + b.TriggerShutdown() - err := tmb.Wait() + err := wg.Wait() if err != nil { t.Fatal(err) } @@ -84,22 +84,22 @@ func TestBlobSaverError(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tmb, ctx := tomb.WithContext(ctx) + wg, ctx := errgroup.WithContext(ctx) saver := &saveFail{ idx: repository.NewMasterIndex(), failAt: int32(test.failAt), } - b := NewBlobSaver(ctx, tmb, saver, uint(runtime.NumCPU())) + b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) for i := 0; i < test.blobs; i++ { buf := &Buffer{Data: []byte(fmt.Sprintf("foo%d", i))} b.Save(ctx, restic.DataBlob, buf) } - tmb.Kill(nil) + b.TriggerShutdown() - err := tmb.Wait() + err := wg.Wait() if err == nil { t.Errorf("expected error not found") } diff --git a/internal/archiver/buffer.go b/internal/archiver/buffer.go index ef71313222f..39bda26682d 100644 --- a/internal/archiver/buffer.go +++ b/internal/archiver/buffer.go @@ -1,52 +1,44 @@ package archiver -import ( - "context" - "sync" -) - // Buffer is a reusable buffer. After the buffer has been used, Release should // be called so the underlying slice is put back into the pool. type Buffer struct { Data []byte - Put func(*Buffer) + pool *BufferPool } // Release puts the buffer back into the pool it came from. func (b *Buffer) Release() { - if b.Put != nil { - b.Put(b) + pool := b.pool + if pool == nil || cap(b.Data) > pool.defaultSize { + return + } + + select { + case pool.ch <- b: + default: } } // BufferPool implements a limited set of reusable buffers. type BufferPool struct { ch chan *Buffer - chM sync.Mutex defaultSize int - clearOnce sync.Once } -// NewBufferPool initializes a new buffer pool. When the context is cancelled, -// all buffers are released. The pool stores at most max items. New buffers are -// created with defaultSize, buffers that are larger are released and not put -// back. -func NewBufferPool(ctx context.Context, max int, defaultSize int) *BufferPool { +// NewBufferPool initializes a new buffer pool. The pool stores at most max +// items. New buffers are created with defaultSize. Buffers that have grown +// larger are not put back. +func NewBufferPool(max int, defaultSize int) *BufferPool { b := &BufferPool{ ch: make(chan *Buffer, max), defaultSize: defaultSize, } - go func() { - <-ctx.Done() - b.clear() - }() return b } // Get returns a new buffer, either from the pool or newly allocated. func (pool *BufferPool) Get() *Buffer { - pool.chM.Lock() - defer pool.chM.Unlock() select { case buf := <-pool.ch: return buf @@ -54,36 +46,9 @@ func (pool *BufferPool) Get() *Buffer { } b := &Buffer{ - Put: pool.Put, Data: make([]byte, pool.defaultSize), + pool: pool, } return b } - -// Put returns a buffer to the pool for reuse. -func (pool *BufferPool) Put(b *Buffer) { - if cap(b.Data) > pool.defaultSize { - return - } - - pool.chM.Lock() - defer pool.chM.Unlock() - select { - case pool.ch <- b: - default: - } -} - -// clear empties the buffer so that all items can be garbage collected. -func (pool *BufferPool) clear() { - pool.clearOnce.Do(func() { - ch := pool.ch - pool.chM.Lock() - pool.ch = nil - pool.chM.Unlock() - close(ch) - for range ch { - } - }) -} diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index 24cc5e116f7..52dd59113f3 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -10,44 +10,9 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" - tomb "gopkg.in/tomb.v2" + "golang.org/x/sync/errgroup" ) -// FutureFile is returned by Save and will return the data once it -// has been processed. -type FutureFile struct { - ch <-chan saveFileResponse - res saveFileResponse -} - -// Wait blocks until the result of the save operation is received or ctx is -// cancelled. -func (s *FutureFile) Wait(ctx context.Context) { - select { - case res, ok := <-s.ch: - if ok { - s.res = res - } - case <-ctx.Done(): - return - } -} - -// Node returns the node once it is available. -func (s *FutureFile) Node() *restic.Node { - return s.res.node -} - -// Stats returns the stats for the file once they are available. -func (s *FutureFile) Stats() ItemStats { - return s.res.stats -} - -// Err returns the error in case an error occurred. -func (s *FutureFile) Err() error { - return s.res.err -} - // SaveBlobFn saves a blob to a repo. type SaveBlobFn func(context.Context, restic.BlobType, *Buffer) FutureBlob @@ -67,7 +32,7 @@ type FileSaver struct { // NewFileSaver returns a new file saver. A worker pool with fileWorkers is // started, it is stopped when ctx is cancelled. -func NewFileSaver(ctx context.Context, t *tomb.Tomb, save SaveBlobFn, pol chunker.Pol, fileWorkers, blobWorkers uint) *FileSaver { +func NewFileSaver(ctx context.Context, wg *errgroup.Group, save SaveBlobFn, pol chunker.Pol, fileWorkers, blobWorkers uint) *FileSaver { ch := make(chan saveFileJob) debug.Log("new file saver with %v file workers and %v blob workers", fileWorkers, blobWorkers) @@ -76,7 +41,7 @@ func NewFileSaver(ctx context.Context, t *tomb.Tomb, save SaveBlobFn, pol chunke s := &FileSaver{ saveBlob: save, - saveFilePool: NewBufferPool(ctx, int(poolSize), chunker.MaxSize), + saveFilePool: NewBufferPool(int(poolSize), chunker.MaxSize), pol: pol, ch: ch, @@ -84,8 +49,8 @@ func NewFileSaver(ctx context.Context, t *tomb.Tomb, save SaveBlobFn, pol chunke } for i := uint(0); i < fileWorkers; i++ { - t.Go(func() error { - s.worker(t.Context(ctx), ch) + wg.Go(func() error { + s.worker(ctx, ch) return nil }) } @@ -93,15 +58,20 @@ func NewFileSaver(ctx context.Context, t *tomb.Tomb, save SaveBlobFn, pol chunke return s } +func (s *FileSaver) TriggerShutdown() { + close(s.ch) +} + // CompleteFunc is called when the file has been saved. type CompleteFunc func(*restic.Node, ItemStats) // Save stores the file f and returns the data once it has been completed. The // file is closed by Save. -func (s *FileSaver) Save(ctx context.Context, snPath string, file fs.File, fi os.FileInfo, start func(), complete CompleteFunc) FutureFile { - ch := make(chan saveFileResponse, 1) +func (s *FileSaver) Save(ctx context.Context, snPath string, target string, file fs.File, fi os.FileInfo, start func(), complete CompleteFunc) FutureNode { + fn, ch := newFutureNode() job := saveFileJob{ snPath: snPath, + target: target, file: file, fi: fi, start: start, @@ -115,57 +85,66 @@ func (s *FileSaver) Save(ctx context.Context, snPath string, file fs.File, fi os debug.Log("not sending job, context is cancelled: %v", ctx.Err()) _ = file.Close() close(ch) - return FutureFile{ch: ch} } - return FutureFile{ch: ch} + return fn } type saveFileJob struct { snPath string + target string file fs.File fi os.FileInfo - ch chan<- saveFileResponse + ch chan<- futureNodeResult complete CompleteFunc start func() } -type saveFileResponse struct { - node *restic.Node - stats ItemStats - err error -} - // saveFile stores the file f in the repo, then closes it. -func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPath string, f fs.File, fi os.FileInfo, start func()) saveFileResponse { +func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPath string, target string, f fs.File, fi os.FileInfo, start func()) futureNodeResult { start() stats := ItemStats{} + fnr := futureNodeResult{ + snPath: snPath, + target: target, + } debug.Log("%v", snPath) node, err := s.NodeFromFileInfo(f.Name(), fi) if err != nil { _ = f.Close() - return saveFileResponse{err: err} + fnr.err = err + return fnr } if node.Type != "file" { _ = f.Close() - return saveFileResponse{err: errors.Errorf("node type %q is wrong", node.Type)} + fnr.err = errors.Errorf("node type %q is wrong", node.Type) + return fnr } // reuse the chunker chnker.Reset(f, s.pol) var results []FutureBlob + complete := func(sbr SaveBlobResponse) { + if !sbr.known { + stats.DataBlobs++ + stats.DataSize += uint64(sbr.length) + stats.DataSizeInRepo += uint64(sbr.sizeInRepo) + } + + node.Content = append(node.Content, sbr.id) + } node.Content = []restic.ID{} var size uint64 for { buf := s.saveFilePool.Get() chunk, err := chnker.Next(buf.Data) - if errors.Cause(err) == io.EOF { + if err == io.EOF { buf.Release() break } @@ -176,13 +155,15 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat if err != nil { _ = f.Close() - return saveFileResponse{err: err} + fnr.err = err + return fnr } // test if the context has been cancelled, return the error if ctx.Err() != nil { _ = f.Close() - return saveFileResponse{err: ctx.Err()} + fnr.err = ctx.Err() + return fnr } res := s.saveBlob(ctx, restic.DataBlob, buf) @@ -191,33 +172,40 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat // test if the context has been cancelled, return the error if ctx.Err() != nil { _ = f.Close() - return saveFileResponse{err: ctx.Err()} + fnr.err = ctx.Err() + return fnr } s.CompleteBlob(f.Name(), uint64(len(chunk.Data))) + + // collect already completed blobs + for len(results) > 0 { + sbr := results[0].Poll() + if sbr == nil { + break + } + results[0] = FutureBlob{} + results = results[1:] + complete(*sbr) + } } err = f.Close() if err != nil { - return saveFileResponse{err: err} + fnr.err = err + return fnr } - for _, res := range results { - res.Wait(ctx) - if !res.Known() { - stats.DataBlobs++ - stats.DataSize += uint64(res.Length()) - } - - node.Content = append(node.Content, res.ID()) + for i, res := range results { + results[i] = FutureBlob{} + sbr := res.Take(ctx) + complete(sbr) } node.Size = size - - return saveFileResponse{ - node: node, - stats: stats, - } + fnr.node = node + fnr.stats = stats + return fnr } func (s *FileSaver) worker(ctx context.Context, jobs <-chan saveFileJob) { @@ -226,13 +214,17 @@ func (s *FileSaver) worker(ctx context.Context, jobs <-chan saveFileJob) { for { var job saveFileJob + var ok bool select { case <-ctx.Done(): return - case job = <-jobs: + case job, ok = <-jobs: + if !ok { + return + } } - res := s.saveFile(ctx, chnker, job.snPath, job.file, job.fi, job.start) + res := s.saveFile(ctx, chnker, job.snPath, job.target, job.file, job.fi, job.start) if job.complete != nil { job.complete(res.node, res.stats) } diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index d4f4fe82bd5..e4d1dcdb8c8 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -12,7 +12,7 @@ import ( "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" - tomb "gopkg.in/tomb.v2" + "golang.org/x/sync/errgroup" ) func createTestFiles(t testing.TB, num int) (files []string, cleanup func()) { @@ -30,11 +30,11 @@ func createTestFiles(t testing.TB, num int) (files []string, cleanup func()) { return files, cleanup } -func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Context, *tomb.Tomb) { - tmb, ctx := tomb.WithContext(ctx) +func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Context, *errgroup.Group) { + wg, ctx := errgroup.WithContext(ctx) saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer) FutureBlob { - ch := make(chan saveBlobResponse) + ch := make(chan SaveBlobResponse) close(ch) return FutureBlob{ch: ch} } @@ -45,10 +45,10 @@ func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Cont t.Fatal(err) } - s := NewFileSaver(ctx, tmb, saveBlob, pol, workers, workers) + s := NewFileSaver(ctx, wg, saveBlob, pol, workers, workers) s.NodeFromFileInfo = restic.NodeFromFileInfo - return s, ctx, tmb + return s, ctx, wg } func TestFileSaver(t *testing.T) { @@ -62,9 +62,9 @@ func TestFileSaver(t *testing.T) { completeFn := func(*restic.Node, ItemStats) {} testFs := fs.Local{} - s, ctx, tmb := startFileSaver(ctx, t) + s, ctx, wg := startFileSaver(ctx, t) - var results []FutureFile + var results []FutureNode for _, filename := range files { f, err := testFs.Open(filename) @@ -77,20 +77,20 @@ func TestFileSaver(t *testing.T) { t.Fatal(err) } - ff := s.Save(ctx, filename, f, fi, startFn, completeFn) + ff := s.Save(ctx, filename, filename, f, fi, startFn, completeFn) results = append(results, ff) } for _, file := range results { - file.Wait(ctx) - if file.Err() != nil { - t.Errorf("unable to save file: %v", file.Err()) + fnr := file.take(ctx) + if fnr.err != nil { + t.Errorf("unable to save file: %v", fnr.err) } } - tmb.Kill(nil) + s.TriggerShutdown() - err := tmb.Wait() + err := wg.Wait() if err != nil { t.Fatal(err) } diff --git a/internal/archiver/scanner.go b/internal/archiver/scanner.go index 5c847425934..6ce2a47000b 100644 --- a/internal/archiver/scanner.go +++ b/internal/archiver/scanner.go @@ -27,7 +27,7 @@ func NewScanner(fs fs.FS) *Scanner { FS: fs, SelectByName: func(item string) bool { return true }, Select: func(item string, fi os.FileInfo) bool { return true }, - Error: func(item string, fi os.FileInfo, err error) error { return err }, + Error: func(item string, err error) error { return err }, Result: func(item string, s ScanStats) {}, } } @@ -111,7 +111,7 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca // get file information fi, err := s.FS.Lstat(target) if err != nil { - return stats, s.Error(target, fi, err) + return stats, s.Error(target, err) } // run remaining select functions that require file information @@ -126,7 +126,7 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca case fi.Mode().IsDir(): names, err := readdirnames(s.FS, target, fs.O_NOFOLLOW) if err != nil { - return stats, s.Error(target, fi, err) + return stats, s.Error(target, err) } sort.Strings(names) diff --git a/internal/archiver/scanner_test.go b/internal/archiver/scanner_test.go index 6c2d35d81de..87d8c887d61 100644 --- a/internal/archiver/scanner_test.go +++ b/internal/archiver/scanner_test.go @@ -133,7 +133,7 @@ func TestScannerError(t *testing.T) { src TestDir result ScanStats selFn SelectFunc - errFn func(t testing.TB, item string, fi os.FileInfo, err error) error + errFn func(t testing.TB, item string, err error) error resFn func(t testing.TB, item string, s ScanStats) prepare func(t testing.TB) }{ @@ -173,7 +173,7 @@ func TestScannerError(t *testing.T) { t.Fatal(err) } }, - errFn: func(t testing.TB, item string, fi os.FileInfo, err error) error { + errFn: func(t testing.TB, item string, err error) error { if item == filepath.FromSlash("work/subdir") { return nil } @@ -198,7 +198,7 @@ func TestScannerError(t *testing.T) { } } }, - errFn: func(t testing.TB, item string, fi os.FileInfo, err error) error { + errFn: func(t testing.TB, item string, err error) error { if item == "foo" { t.Logf("ignoring error for %v: %v", item, err) return nil @@ -257,13 +257,13 @@ func TestScannerError(t *testing.T) { } } if test.errFn != nil { - sc.Error = func(item string, fi os.FileInfo, err error) error { + sc.Error = func(item string, err error) error { p, relErr := filepath.Rel(cur, item) if relErr != nil { panic(relErr) } - return test.errFn(t, p, fi, err) + return test.errFn(t, p, err) } } diff --git a/internal/archiver/testing.go b/internal/archiver/testing.go index 873e9ab5d8d..35a2d293336 100644 --- a/internal/archiver/testing.go +++ b/internal/archiver/testing.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -225,7 +226,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi return } - content := make([]byte, restic.CiphertextLength(len(file.Content))) + content := make([]byte, crypto.CiphertextLength(len(file.Content))) pos := 0 for _, id := range node.Content { part, err := repo.LoadBlob(ctx, restic.DataBlob, id, content[pos:]) @@ -234,6 +235,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi return } + copy(content[pos:pos+len(part)], part) pos += len(part) } @@ -249,7 +251,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo restic.Repository, treeID restic.ID, dir TestDir) { t.Helper() - tree, err := repo.LoadTree(ctx, treeID) + tree, err := restic.LoadTree(ctx, repo, treeID) if err != nil { t.Fatal(err) return diff --git a/internal/archiver/tree_saver.go b/internal/archiver/tree_saver.go index 867bad6aac7..5aab09b946c 100644 --- a/internal/archiver/tree_saver.go +++ b/internal/archiver/tree_saver.go @@ -5,41 +5,12 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" - tomb "gopkg.in/tomb.v2" + "golang.org/x/sync/errgroup" ) -// FutureTree is returned by Save and will return the data once it -// has been processed. -type FutureTree struct { - ch <-chan saveTreeResponse - res saveTreeResponse -} - -// Wait blocks until the data has been received or ctx is cancelled. -func (s *FutureTree) Wait(ctx context.Context) { - select { - case <-ctx.Done(): - return - case res, ok := <-s.ch: - if ok { - s.res = res - } - } -} - -// Node returns the node. -func (s *FutureTree) Node() *restic.Node { - return s.res.node -} - -// Stats returns the stats for the file. -func (s *FutureTree) Stats() ItemStats { - return s.res.stats -} - // TreeSaver concurrently saves incoming trees to the repo. type TreeSaver struct { - saveTree func(context.Context, *restic.Tree) (restic.ID, ItemStats, error) + saveTree func(context.Context, *restic.TreeJSONBuilder) (restic.ID, ItemStats, error) errFn ErrorFunc ch chan<- saveTreeJob @@ -47,7 +18,7 @@ type TreeSaver struct { // NewTreeSaver returns a new tree saver. A worker pool with treeWorkers is // started, it is stopped when ctx is cancelled. -func NewTreeSaver(ctx context.Context, t *tomb.Tomb, treeWorkers uint, saveTree func(context.Context, *restic.Tree) (restic.ID, ItemStats, error), errFn ErrorFunc) *TreeSaver { +func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveTree func(context.Context, *restic.TreeJSONBuilder) (restic.ID, ItemStats, error), errFn ErrorFunc) *TreeSaver { ch := make(chan saveTreeJob) s := &TreeSaver{ @@ -57,19 +28,24 @@ func NewTreeSaver(ctx context.Context, t *tomb.Tomb, treeWorkers uint, saveTree } for i := uint(0); i < treeWorkers; i++ { - t.Go(func() error { - return s.worker(t.Context(ctx), ch) + wg.Go(func() error { + return s.worker(ctx, ch) }) } return s } +func (s *TreeSaver) TriggerShutdown() { + close(s.ch) +} + // Save stores the dir d and returns the data once it has been completed. -func (s *TreeSaver) Save(ctx context.Context, snPath string, node *restic.Node, nodes []FutureNode, complete CompleteFunc) FutureTree { - ch := make(chan saveTreeResponse, 1) +func (s *TreeSaver) Save(ctx context.Context, snPath string, target string, node *restic.Node, nodes []FutureNode, complete CompleteFunc) FutureNode { + fn, ch := newFutureNode() job := saveTreeJob{ snPath: snPath, + target: target, node: node, nodes: nodes, ch: ch, @@ -80,60 +56,61 @@ func (s *TreeSaver) Save(ctx context.Context, snPath string, node *restic.Node, case <-ctx.Done(): debug.Log("not saving tree, context is cancelled") close(ch) - return FutureTree{ch: ch} } - return FutureTree{ch: ch} + return fn } type saveTreeJob struct { snPath string - nodes []FutureNode + target string node *restic.Node - ch chan<- saveTreeResponse + nodes []FutureNode + ch chan<- futureNodeResult complete CompleteFunc } -type saveTreeResponse struct { - node *restic.Node - stats ItemStats -} - // save stores the nodes as a tree in the repo. -func (s *TreeSaver) save(ctx context.Context, snPath string, node *restic.Node, nodes []FutureNode) (*restic.Node, ItemStats, error) { +func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, ItemStats, error) { var stats ItemStats + node := job.node + nodes := job.nodes + // allow GC of nodes array once the loop is finished + job.nodes = nil - tree := restic.NewTree(len(nodes)) + builder := restic.NewTreeJSONBuilder() - for _, fn := range nodes { - fn.wait(ctx) + for i, fn := range nodes { + // fn is a copy, so clear the original value explicitly + nodes[i] = FutureNode{} + fnr := fn.take(ctx) // return the error if it wasn't ignored - if fn.err != nil { - debug.Log("err for %v: %v", fn.snPath, fn.err) - fn.err = s.errFn(fn.target, fn.fi, fn.err) - if fn.err == nil { + if fnr.err != nil { + debug.Log("err for %v: %v", fnr.snPath, fnr.err) + fnr.err = s.errFn(fnr.target, fnr.err) + if fnr.err == nil { // ignore error continue } - return nil, stats, fn.err + return nil, stats, fnr.err } // when the error is ignored, the node could not be saved, so ignore it - if fn.node == nil { - debug.Log("%v excluded: %v", fn.snPath, fn.target) + if fnr.node == nil { + debug.Log("%v excluded: %v", fnr.snPath, fnr.target) continue } - debug.Log("insert %v", fn.node.Name) - err := tree.Insert(fn.node) + debug.Log("insert %v", fnr.node.Name) + err := builder.AddNode(fnr.node) if err != nil { return nil, stats, err } } - id, treeStats, err := s.saveTree(ctx, tree) + id, treeStats, err := s.saveTree(ctx, builder) stats.Add(treeStats) if err != nil { return nil, stats, err @@ -146,13 +123,17 @@ func (s *TreeSaver) save(ctx context.Context, snPath string, node *restic.Node, func (s *TreeSaver) worker(ctx context.Context, jobs <-chan saveTreeJob) error { for { var job saveTreeJob + var ok bool select { case <-ctx.Done(): return nil - case job = <-jobs: + case job, ok = <-jobs: + if !ok { + return nil + } } - node, stats, err := s.save(ctx, job.snPath, job.node, job.nodes) + node, stats, err := s.save(ctx, &job) if err != nil { debug.Log("error saving tree blob: %v", err) close(job.ch) @@ -162,9 +143,11 @@ func (s *TreeSaver) worker(ctx context.Context, jobs <-chan saveTreeJob) error { if job.complete != nil { job.complete(node, stats) } - job.ch <- saveTreeResponse{ - node: node, - stats: stats, + job.ch <- futureNodeResult{ + snPath: job.snPath, + target: job.target, + node: node, + stats: stats, } close(job.ch) } diff --git a/internal/archiver/tree_saver_test.go b/internal/archiver/tree_saver_test.go index c9c589d1c05..36e585ae1f9 100644 --- a/internal/archiver/tree_saver_test.go +++ b/internal/archiver/tree_saver_test.go @@ -3,50 +3,49 @@ package archiver import ( "context" "fmt" - "os" "runtime" "sync/atomic" "testing" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - tomb "gopkg.in/tomb.v2" + "golang.org/x/sync/errgroup" ) func TestTreeSaver(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tmb, ctx := tomb.WithContext(ctx) + wg, ctx := errgroup.WithContext(ctx) - saveFn := func(context.Context, *restic.Tree) (restic.ID, ItemStats, error) { + saveFn := func(context.Context, *restic.TreeJSONBuilder) (restic.ID, ItemStats, error) { return restic.NewRandomID(), ItemStats{TreeBlobs: 1, TreeSize: 123}, nil } - errFn := func(snPath string, fi os.FileInfo, err error) error { + errFn := func(snPath string, err error) error { return nil } - b := NewTreeSaver(ctx, tmb, uint(runtime.NumCPU()), saveFn, errFn) + b := NewTreeSaver(ctx, wg, uint(runtime.NumCPU()), saveFn, errFn) - var results []FutureTree + var results []FutureNode for i := 0; i < 20; i++ { node := &restic.Node{ Name: fmt.Sprintf("file-%d", i), } - fb := b.Save(ctx, "/", node, nil, nil) + fb := b.Save(ctx, "/", node.Name, node, nil, nil) results = append(results, fb) } for _, tree := range results { - tree.Wait(ctx) + tree.take(ctx) } - tmb.Kill(nil) + b.TriggerShutdown() - err := tmb.Wait() + err := wg.Wait() if err != nil { t.Fatal(err) } @@ -71,10 +70,10 @@ func TestTreeSaverError(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tmb, ctx := tomb.WithContext(ctx) + wg, ctx := errgroup.WithContext(ctx) var num int32 - saveFn := func(context.Context, *restic.Tree) (restic.ID, ItemStats, error) { + saveFn := func(context.Context, *restic.TreeJSONBuilder) (restic.ID, ItemStats, error) { val := atomic.AddInt32(&num, 1) if val == test.failAt { t.Logf("sending error for request %v\n", test.failAt) @@ -83,31 +82,31 @@ func TestTreeSaverError(t *testing.T) { return restic.NewRandomID(), ItemStats{TreeBlobs: 1, TreeSize: 123}, nil } - errFn := func(snPath string, fi os.FileInfo, err error) error { + errFn := func(snPath string, err error) error { t.Logf("ignoring error %v\n", err) return nil } - b := NewTreeSaver(ctx, tmb, uint(runtime.NumCPU()), saveFn, errFn) + b := NewTreeSaver(ctx, wg, uint(runtime.NumCPU()), saveFn, errFn) - var results []FutureTree + var results []FutureNode for i := 0; i < test.trees; i++ { node := &restic.Node{ Name: fmt.Sprintf("file-%d", i), } - fb := b.Save(ctx, "/", node, nil, nil) + fb := b.Save(ctx, "/", node.Name, node, nil, nil) results = append(results, fb) } for _, tree := range results { - tree.Wait(ctx) + tree.take(ctx) } - tmb.Kill(nil) + b.TriggerShutdown() - err := tmb.Wait() + err := wg.Wait() if err == nil { t.Errorf("expected error not found") } diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index e123f516e03..27adac44c9d 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -4,6 +4,7 @@ import ( "context" "crypto/md5" "encoding/base64" + "fmt" "hash" "io" "net/http" @@ -12,6 +13,7 @@ import ( "strings" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -24,7 +26,8 @@ import ( type Backend struct { accountName string container *storage.Container - sem *backend.Semaphore + connections uint + sem sema.Semaphore prefix string listMaxItems int backend.Layout @@ -37,17 +40,41 @@ var _ restic.Backend = &Backend{} func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) - - client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey) - if err != nil { - return nil, errors.Wrap(err, "NewBasicClient") + var client storage.Client + var err error + if cfg.AccountKey.String() != "" { + // We have an account key value, find the BlobServiceClient + // from with a BasicClient + debug.Log(" - using account key") + client, err = storage.NewBasicClient(cfg.AccountName, cfg.AccountKey.Unwrap()) + if err != nil { + return nil, errors.Wrap(err, "NewBasicClient") + } + } else if cfg.AccountSAS.String() != "" { + // Get the client using the SAS Token as authentication, this + // is longer winded than above because the SDK wants a URL for the Account + // if your using a SAS token, and not just the account name + // we (as per the SDK ) assume the default Azure portal. + url := fmt.Sprintf("https://%s.blob.core.windows.net/", cfg.AccountName) + debug.Log(" - using sas token") + sas := cfg.AccountSAS.Unwrap() + // strip query sign prefix + if sas[0] == '?' { + sas = sas[1:] + } + client, err = storage.NewAccountSASClientFromEndpointToken(url, sas) + if err != nil { + return nil, errors.Wrap(err, "NewAccountSASClientFromEndpointToken") + } + } else { + return nil, errors.New("no azure authentication information found") } client.HTTPClient = &http.Client{Transport: rt} service := client.GetBlobService() - sem, err := backend.NewSemaphore(cfg.Connections) + sem, err := sema.New(cfg.Connections) if err != nil { return nil, err } @@ -55,6 +82,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { be := &Backend{ container: service.GetContainerReference(cfg.Container), accountName: cfg.AccountName, + connections: cfg.Connections, sem: sem, prefix: cfg.Prefix, Layout: &backend.DefaultLayout{ @@ -109,6 +137,10 @@ func (be *Backend) Join(p ...string) string { return path.Join(p...) } +func (be *Backend) Connections() uint { + return be.connections +} + // Location returns this backend's location (the container name). func (be *Backend) Location() string { return be.Join(be.container.Name, be.prefix) @@ -119,6 +151,11 @@ func (be *Backend) Hasher() hash.Hash { return md5.New() } +// HasAtomicReplace returns whether Save() can atomically replace files +func (be *Backend) HasAtomicReplace() bool { + return true +} + // Path returns the path in the bucket that is used for this backend. func (be *Backend) Path() string { return be.prefix @@ -226,18 +263,6 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi return errors.Wrap(err, "PutBlockList") } -// wrapReader wraps an io.ReadCloser to run an additional function on Close. -type wrapReader struct { - io.ReadCloser - f func() -} - -func (wr wrapReader) Close() error { - err := wr.ReadCloser.Close() - wr.f() - return err -} - // Load runs fn with a reader that yields the contents of the file at h at the // given offset. func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { @@ -278,15 +303,7 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, return nil, err } - closeRd := wrapReader{ - ReadCloser: rd, - f: func() { - debug.Log("Close()") - be.sem.ReleaseToken() - }, - } - - return closeRd, err + return be.sem.ReleaseTokenOnClose(rd, nil), err } // Stat returns information about a blob. diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index a9ed94cd2e7..f1e58dea37a 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -13,6 +13,7 @@ import ( "github.com/restic/restic/internal/backend/azure" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -36,7 +37,7 @@ func newAzureTestSuite(t testing.TB) *test.Suite { cfg := azcfg.(azure.Config) cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME") - cfg.AccountKey = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY") + cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY")) cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) return cfg, nil }, @@ -146,7 +147,7 @@ func TestUploadLargeFile(t *testing.T) { cfg := azcfg.(azure.Config) cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME") - cfg.AccountKey = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY") + cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY")) cfg.Prefix = fmt.Sprintf("test-upload-large-%d", time.Now().UnixNano()) tr, err := backend.Transport(backend.TransportOptions{}) diff --git a/internal/backend/azure/config.go b/internal/backend/azure/config.go index 682356b01de..cc5169e3e1d 100644 --- a/internal/backend/azure/config.go +++ b/internal/backend/azure/config.go @@ -12,7 +12,8 @@ import ( // server. type Config struct { AccountName string - AccountKey string + AccountSAS options.SecretString + AccountKey options.SecretString Container string Prefix string diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 90aeca3b202..150d396d53f 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -8,6 +8,7 @@ import ( "path" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -23,7 +24,7 @@ type b2Backend struct { cfg Config listMaxItems int backend.Layout - sem *backend.Semaphore + sem sema.Semaphore } const defaultListMaxItems = 1000 @@ -34,7 +35,7 @@ var _ restic.Backend = &b2Backend{} func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) { opts := []b2.ClientOption{b2.Transport(rt)} - c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key, opts...) + c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key.Unwrap(), opts...) if err != nil { return nil, errors.Wrap(err, "b2.NewClient") } @@ -58,7 +59,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend return nil, errors.Wrap(err, "Bucket") } - sem, err := backend.NewSemaphore(cfg.Connections) + sem, err := sema.New(cfg.Connections) if err != nil { return nil, err } @@ -99,7 +100,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe return nil, errors.Wrap(err, "NewBucket") } - sem, err := backend.NewSemaphore(cfg.Connections) + sem, err := sema.New(cfg.Connections) if err != nil { return nil, err } @@ -133,6 +134,10 @@ func (be *b2Backend) SetListMaxItems(i int) { be.listMaxItems = i } +func (be *b2Backend) Connections() uint { + return be.cfg.Connections +} + // Location returns the location for the backend. func (be *b2Backend) Location() string { return be.cfg.Bucket @@ -143,6 +148,11 @@ func (be *b2Backend) Hasher() hash.Hash { return nil } +// HasAtomicReplace returns whether Save() can atomically replace files +func (be *b2Backend) HasAtomicReplace() bool { + return true +} + // IsNotExist returns true if the error is caused by a non-existing file. func (be *b2Backend) IsNotExist(err error) bool { return b2.IsNotExist(errors.Cause(err)) @@ -275,11 +285,11 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { } type semLocker struct { - *backend.Semaphore + sema.Semaphore } -func (sm semLocker) Lock() { sm.GetToken() } -func (sm semLocker) Unlock() { sm.ReleaseToken() } +func (sm *semLocker) Lock() { sm.GetToken() } +func (sm *semLocker) Unlock() { sm.ReleaseToken() } // List returns a channel that yields all names of blobs of type t. func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { @@ -289,7 +299,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic defer cancel() prefix, _ := be.Basedir(t) - iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems), b2.ListLocker(semLocker{be.sem})) + iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems), b2.ListLocker(&semLocker{be.sem})) for iter.Next() { obj := iter.Object() diff --git a/internal/backend/b2/b2_test.go b/internal/backend/b2/b2_test.go index 9f97de4f9ab..123a61d7c2f 100644 --- a/internal/backend/b2/b2_test.go +++ b/internal/backend/b2/b2_test.go @@ -10,6 +10,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/test" + "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -37,7 +38,7 @@ func newB2TestSuite(t testing.TB) *test.Suite { cfg := b2cfg.(b2.Config) cfg.AccountID = os.Getenv("RESTIC_TEST_B2_ACCOUNT_ID") - cfg.Key = os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY") + cfg.Key = options.NewSecretString(os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY")) cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) return cfg, nil }, diff --git a/internal/backend/b2/config.go b/internal/backend/b2/config.go index f10e4d10d73..98e8e144570 100644 --- a/internal/backend/b2/config.go +++ b/internal/backend/b2/config.go @@ -13,7 +13,7 @@ import ( // server. type Config struct { AccountID string - Key string + Key options.SecretString Bucket string Prefix string diff --git a/internal/backend/backend_retry_test.go b/internal/backend/backend_retry_test.go index 4013f4ea50c..e8f4d7315da 100644 --- a/internal/backend/backend_retry_test.go +++ b/internal/backend/backend_retry_test.go @@ -7,8 +7,8 @@ import ( "io/ioutil" "testing" + "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/mock" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) diff --git a/internal/backend/dryrun/dry_backend.go b/internal/backend/dryrun/dry_backend.go index 8412bd26a1a..31012df431e 100644 --- a/internal/backend/dryrun/dry_backend.go +++ b/internal/backend/dryrun/dry_backend.go @@ -45,6 +45,10 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { return nil } +func (be *Backend) Connections() uint { + return be.b.Connections() +} + // Location returns the location of the backend. func (be *Backend) Location() string { return "DRY:" + be.b.Location() @@ -63,6 +67,10 @@ func (be *Backend) Hasher() hash.Hash { return be.b.Hasher() } +func (be *Backend) HasAtomicReplace() bool { + return be.b.HasAtomicReplace() +} + func (be *Backend) IsNotExist(err error) bool { return be.b.IsNotExist(err) } diff --git a/internal/backend/foreground_sysv.go b/internal/backend/foreground_sysv.go index f60e4242e2c..0e88a57a1a1 100644 --- a/internal/backend/foreground_sysv.go +++ b/internal/backend/foreground_sysv.go @@ -1,3 +1,4 @@ +//go:build aix || solaris // +build aix solaris package backend diff --git a/internal/backend/foreground_test.go b/internal/backend/foreground_test.go index 81adefe3259..4f701122d27 100644 --- a/internal/backend/foreground_test.go +++ b/internal/backend/foreground_test.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package backend_test diff --git a/internal/backend/foreground_unix.go b/internal/backend/foreground_unix.go index eb0002dad4f..2b59bdf6c9c 100644 --- a/internal/backend/foreground_unix.go +++ b/internal/backend/foreground_unix.go @@ -1,3 +1,4 @@ +//go:build !aix && !solaris && !windows // +build !aix,!solaris,!windows package backend diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 5f57d9d32b0..aa3aff4c49c 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -14,6 +14,7 @@ import ( "cloud.google.com/go/storage" "github.com/pkg/errors" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -27,14 +28,15 @@ import ( // Backend stores data in a GCS bucket. // // The service account used to access the bucket must have these permissions: -// * storage.objects.create -// * storage.objects.delete -// * storage.objects.get -// * storage.objects.list +// - storage.objects.create +// - storage.objects.delete +// - storage.objects.get +// - storage.objects.list type Backend struct { gcsClient *storage.Client projectID string - sem *backend.Semaphore + connections uint + sem sema.Semaphore bucketName string bucket *storage.BucketHandle prefix string @@ -96,18 +98,19 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { return nil, errors.Wrap(err, "getStorageClient") } - sem, err := backend.NewSemaphore(cfg.Connections) + sem, err := sema.New(cfg.Connections) if err != nil { return nil, err } be := &Backend{ - gcsClient: gcsClient, - projectID: cfg.ProjectID, - sem: sem, - bucketName: cfg.Bucket, - bucket: gcsClient.Bucket(cfg.Bucket), - prefix: cfg.Prefix, + gcsClient: gcsClient, + projectID: cfg.ProjectID, + connections: cfg.Connections, + sem: sem, + bucketName: cfg.Bucket, + bucket: gcsClient.Bucket(cfg.Bucket), + prefix: cfg.Prefix, Layout: &backend.DefaultLayout{ Path: cfg.Prefix, Join: path.Join, @@ -185,6 +188,10 @@ func (be *Backend) Join(p ...string) string { return path.Join(p...) } +func (be *Backend) Connections() uint { + return be.connections +} + // Location returns this backend's location (the bucket name). func (be *Backend) Location() string { return be.Join(be.bucketName, be.prefix) @@ -195,6 +202,11 @@ func (be *Backend) Hasher() hash.Hash { return md5.New() } +// HasAtomicReplace returns whether Save() can atomically replace files +func (be *Backend) HasAtomicReplace() bool { + return true +} + // Path returns the path in the bucket that is used for this backend. func (be *Backend) Path() string { return be.prefix @@ -263,18 +275,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe return nil } -// wrapReader wraps an io.ReadCloser to run an additional function on Close. -type wrapReader struct { - io.ReadCloser - f func() -} - -func (wr wrapReader) Close() error { - err := wr.ReadCloser.Close() - wr.f() - return err -} - // Load runs fn with a reader that yields the contents of the file at h at the // given offset. func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { @@ -303,21 +303,16 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, be.sem.GetToken() + ctx, cancel := context.WithCancel(ctx) + r, err := be.bucket.Object(objName).NewRangeReader(ctx, offset, int64(length)) if err != nil { + cancel() be.sem.ReleaseToken() return nil, err } - closeRd := wrapReader{ - ReadCloser: r, - f: func() { - debug.Log("Close()") - be.sem.ReleaseToken() - }, - } - - return closeRd, err + return be.sem.ReleaseTokenOnClose(r, cancel), err } // Stat returns information about a blob. diff --git a/internal/backend/layout.go b/internal/backend/layout.go index e916df705b6..ebd54e4afba 100644 --- a/internal/backend/layout.go +++ b/internal/backend/layout.go @@ -152,7 +152,7 @@ func ParseLayout(ctx context.Context, repo Filesystem, layout, defaultLayout, pa l, err = DetectLayout(ctx, repo, path) // use the default layout if auto detection failed - if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" { + if errors.Is(err, ErrLayoutDetectionFailed) && defaultLayout != "" { debug.Log("error: %v, use default layout %v", err, defaultLayout) return ParseLayout(ctx, repo, defaultLayout, "", path) } diff --git a/internal/limiter/limiter.go b/internal/backend/limiter/limiter.go similarity index 100% rename from internal/limiter/limiter.go rename to internal/backend/limiter/limiter.go diff --git a/internal/limiter/limiter_backend.go b/internal/backend/limiter/limiter_backend.go similarity index 100% rename from internal/limiter/limiter_backend.go rename to internal/backend/limiter/limiter_backend.go diff --git a/internal/limiter/limiter_backend_test.go b/internal/backend/limiter/limiter_backend_test.go similarity index 94% rename from internal/limiter/limiter_backend_test.go rename to internal/backend/limiter/limiter_backend_test.go index e8f0ae17dd3..1014dbed1d2 100644 --- a/internal/limiter/limiter_backend_test.go +++ b/internal/backend/limiter/limiter_backend_test.go @@ -8,7 +8,7 @@ import ( "io" "testing" - "github.com/restic/restic/internal/mock" + "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -36,7 +36,7 @@ func TestLimitBackendSave(t *testing.T) { } return nil } - limiter := NewStaticLimiter(42*1024, 42*1024) + limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) limbe := LimitBackend(be, limiter) rd := restic.NewByteReader(data, nil) @@ -82,7 +82,7 @@ func TestLimitBackendLoad(t *testing.T) { } return newTracedReadCloser(src), nil } - limiter := NewStaticLimiter(42*1024, 42*1024) + limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) limbe := LimitBackend(be, limiter) err := limbe.Load(context.TODO(), testHandle, 0, 0, func(rd io.Reader) error { diff --git a/internal/limiter/static_limiter.go b/internal/backend/limiter/static_limiter.go similarity index 82% rename from internal/limiter/static_limiter.go rename to internal/backend/limiter/static_limiter.go index 04a9b9d346b..9fb8fbf2463 100644 --- a/internal/limiter/static_limiter.go +++ b/internal/backend/limiter/static_limiter.go @@ -12,20 +12,27 @@ type staticLimiter struct { downstream *ratelimit.Bucket } +// Limits represents static upload and download limits. +// For both, zero means unlimited. +type Limits struct { + UploadKb int + DownloadKb int +} + // NewStaticLimiter constructs a Limiter with a fixed (static) upload and // download rate cap -func NewStaticLimiter(uploadKb, downloadKb int) Limiter { +func NewStaticLimiter(l Limits) Limiter { var ( upstreamBucket *ratelimit.Bucket downstreamBucket *ratelimit.Bucket ) - if uploadKb > 0 { - upstreamBucket = ratelimit.NewBucketWithRate(toByteRate(uploadKb), int64(toByteRate(uploadKb))) + if l.UploadKb > 0 { + upstreamBucket = ratelimit.NewBucketWithRate(toByteRate(l.UploadKb), int64(toByteRate(l.UploadKb))) } - if downloadKb > 0 { - downstreamBucket = ratelimit.NewBucketWithRate(toByteRate(downloadKb), int64(toByteRate(downloadKb))) + if l.DownloadKb > 0 { + downstreamBucket = ratelimit.NewBucketWithRate(toByteRate(l.DownloadKb), int64(toByteRate(l.DownloadKb))) } return staticLimiter{ diff --git a/internal/limiter/static_limiter_test.go b/internal/backend/limiter/static_limiter_test.go similarity index 85% rename from internal/limiter/static_limiter_test.go rename to internal/backend/limiter/static_limiter_test.go index bd3c62ccb36..564b6a00ac3 100644 --- a/internal/limiter/static_limiter_test.go +++ b/internal/backend/limiter/static_limiter_test.go @@ -15,22 +15,19 @@ func TestLimiterWrapping(t *testing.T) { reader := bytes.NewReader([]byte{}) writer := new(bytes.Buffer) - for _, limits := range []struct { - upstream int - downstream int - }{ + for _, limits := range []Limits{ {0, 0}, {42, 0}, {0, 42}, {42, 42}, } { - limiter := NewStaticLimiter(limits.upstream*1024, limits.downstream*1024) + limiter := NewStaticLimiter(limits) - mustWrapUpstream := limits.upstream > 0 + mustWrapUpstream := limits.UploadKb > 0 test.Equals(t, limiter.Upstream(reader) != reader, mustWrapUpstream) test.Equals(t, limiter.UpstreamWriter(writer) != writer, mustWrapUpstream) - mustWrapDownstream := limits.downstream > 0 + mustWrapDownstream := limits.DownloadKb > 0 test.Equals(t, limiter.Downstream(reader) != reader, mustWrapDownstream) test.Equals(t, limiter.DownstreamWriter(writer) != writer, mustWrapDownstream) } @@ -51,12 +48,12 @@ func (r *tracedReadCloser) Close() error { } func TestRoundTripperReader(t *testing.T) { - limiter := NewStaticLimiter(42*1024, 42*1024) + limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) data := make([]byte, 1234) _, err := io.ReadFull(rand.Reader, data) test.OK(t, err) - var send *tracedReadCloser = newTracedReadCloser(bytes.NewReader(data)) + send := newTracedReadCloser(bytes.NewReader(data)) var recv *tracedReadCloser rt := limiter.Transport(roundTripper(func(req *http.Request) (*http.Response, error) { @@ -89,7 +86,7 @@ func TestRoundTripperReader(t *testing.T) { } func TestRoundTripperCornerCases(t *testing.T) { - limiter := NewStaticLimiter(42*1024, 42*1024) + limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) rt := limiter.Transport(roundTripper(func(req *http.Request) (*http.Response, error) { return &http.Response{}, nil diff --git a/internal/backend/local/config.go b/internal/backend/local/config.go index 13b7f67aa44..e59d1f693aa 100644 --- a/internal/backend/local/config.go +++ b/internal/backend/local/config.go @@ -11,6 +11,15 @@ import ( type Config struct { Path string Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` + + Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"` +} + +// NewConfig returns a new config with default options applied. +func NewConfig() Config { + return Config{ + Connections: 2, + } } func init() { @@ -18,10 +27,12 @@ func init() { } // ParseConfig parses a local backend config. -func ParseConfig(cfg string) (interface{}, error) { - if !strings.HasPrefix(cfg, "local:") { +func ParseConfig(s string) (interface{}, error) { + if !strings.HasPrefix(s, "local:") { return nil, errors.New(`invalid format, prefix "local" not found`) } - return Config{Path: cfg[6:]}, nil + cfg := NewConfig() + cfg.Path = s[6:] + return cfg, nil } diff --git a/internal/backend/local/layout_test.go b/internal/backend/local/layout_test.go index 5b1135253e3..9da702877c8 100644 --- a/internal/backend/local/layout_test.go +++ b/internal/backend/local/layout_test.go @@ -37,8 +37,9 @@ func TestLayout(t *testing.T) { repo := filepath.Join(path, "repo") be, err := Open(context.TODO(), Config{ - Path: repo, - Layout: test.layout, + Path: repo, + Layout: test.layout, + Connections: 2, }) if err != nil { t.Fatal(err) diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 833bde26f87..bb644c949c9 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -9,12 +9,12 @@ import ( "path/filepath" "syscall" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" "github.com/cenkalti/backoff/v4" ) @@ -22,7 +22,9 @@ import ( // Local is a backend in a local directory. type Local struct { Config + sem sema.Semaphore backend.Layout + backend.Modes } // ensure statically that *Local implements restic.Backend. @@ -30,15 +32,33 @@ var _ restic.Backend = &Local{} const defaultLayout = "default" -// Open opens the local backend as specified by config. -func Open(ctx context.Context, cfg Config) (*Local, error) { - debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout) +func open(ctx context.Context, cfg Config) (*Local, error) { l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err } - return &Local{Config: cfg, Layout: l}, nil + sem, err := sema.New(cfg.Connections) + if err != nil { + return nil, err + } + + fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile})) + m := backend.DeriveModesFromFileInfo(fi, err) + debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) + + return &Local{ + Config: cfg, + Layout: l, + sem: sem, + Modes: m, + }, nil +} + +// Open opens the local backend as specified by config. +func Open(ctx context.Context, cfg Config) (*Local, error) { + debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout) + return open(ctx, cfg) } // Create creates all the necessary files and directories for a new local @@ -46,16 +66,11 @@ func Open(ctx context.Context, cfg Config) (*Local, error) { func Create(ctx context.Context, cfg Config) (*Local, error) { debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout) - l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) + be, err := open(ctx, cfg) if err != nil { return nil, err } - be := &Local{ - Config: cfg, - Layout: l, - } - // test if config file already exists _, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile})) if err == nil { @@ -64,7 +79,7 @@ func Create(ctx context.Context, cfg Config) (*Local, error) { // create paths for data and refs for _, d := range be.Paths() { - err := fs.MkdirAll(d, backend.Modes.Dir) + err := fs.MkdirAll(d, be.Modes.Dir) if err != nil { return nil, errors.WithStack(err) } @@ -73,6 +88,10 @@ func Create(ctx context.Context, cfg Config) (*Local, error) { return be, nil } +func (b *Local) Connections() uint { + return b.Config.Connections +} + // Location returns this backend's location (the directory name). func (b *Local) Location() string { return b.Path @@ -83,6 +102,11 @@ func (b *Local) Hasher() hash.Hash { return nil } +// HasAtomicReplace returns whether Save() can atomically replace files +func (b *Local) HasAtomicReplace() bool { + return true +} + // IsNotExist returns true if the error is caused by a non existing file. func (b *Local) IsNotExist(err error) bool { return errors.Is(err, os.ErrNotExist) @@ -105,6 +129,9 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade } }() + b.sem.GetToken() + defer b.sem.ReleaseToken() + // Create new file with a temporary name. tmpname := filepath.Base(finalname) + "-tmp-" f, err := tempFile(dir, tmpname) @@ -113,7 +140,7 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade debug.Log("error %v: creating dir", err) // error is caused by a missing directory, try to create it - mkdirErr := fs.MkdirAll(dir, backend.Modes.Dir) + mkdirErr := fs.MkdirAll(dir, b.Modes.Dir) if mkdirErr != nil { debug.Log("error creating dir %v: %v", dir, mkdirErr) } else { @@ -173,7 +200,7 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade // try to mark file as read-only to avoid accidential modifications // ignore if the operation fails as some filesystems don't allow the chmod call // e.g. exfat and network file systems with certain mount options - err = setFileReadonly(finalname, backend.Modes.File) + err = setFileReadonly(finalname, b.Modes.File) if err != nil && !os.IsPermission(err) { return errors.WithStack(err) } @@ -199,24 +226,29 @@ func (b *Local) openReader(ctx context.Context, h restic.Handle, length int, off return nil, errors.New("offset is negative") } + b.sem.GetToken() f, err := fs.Open(b.Filename(h)) if err != nil { + b.sem.ReleaseToken() return nil, err } if offset > 0 { _, err = f.Seek(offset, 0) if err != nil { + b.sem.ReleaseToken() _ = f.Close() return nil, err } } + r := b.sem.ReleaseTokenOnClose(f, nil) + if length > 0 { - return backend.LimitReadCloser(f, int64(length)), nil + return backend.LimitReadCloser(r, int64(length)), nil } - return f, nil + return r, nil } // Stat returns information about a blob. @@ -226,6 +258,9 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err return restic.FileInfo{}, backoff.Permanent(err) } + b.sem.GetToken() + defer b.sem.ReleaseToken() + fi, err := fs.Stat(b.Filename(h)) if err != nil { return restic.FileInfo{}, errors.WithStack(err) @@ -237,6 +272,10 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err // Test returns true if a blob of the given type and name exists in the backend. func (b *Local) Test(ctx context.Context, h restic.Handle) (bool, error) { debug.Log("Test %v", h) + + b.sem.GetToken() + defer b.sem.ReleaseToken() + _, err := fs.Stat(b.Filename(h)) if err != nil { if b.IsNotExist(err) { @@ -253,6 +292,9 @@ func (b *Local) Remove(ctx context.Context, h restic.Handle) error { debug.Log("Remove %v", h) fn := b.Filename(h) + b.sem.GetToken() + defer b.sem.ReleaseToken() + // reset read-only flag err := fs.Chmod(fn, 0666) if err != nil && !os.IsPermission(err) { diff --git a/internal/backend/local/local_internal_test.go b/internal/backend/local/local_internal_test.go index 8d2ec08c321..8de3d3c2fa8 100644 --- a/internal/backend/local/local_internal_test.go +++ b/internal/backend/local/local_internal_test.go @@ -27,7 +27,7 @@ func TestNoSpacePermanent(t *testing.T) { dir, cleanup := rtest.TempDir(t) defer cleanup() - be, err := Open(context.Background(), Config{Path: dir}) + be, err := Open(context.Background(), Config{Path: dir, Connections: 2}) rtest.OK(t, err) defer func() { rtest.OK(t, be.Close()) diff --git a/internal/backend/local/local_test.go b/internal/backend/local/local_test.go index 70b5e771eba..75c3b8ed74f 100644 --- a/internal/backend/local/local_test.go +++ b/internal/backend/local/local_test.go @@ -25,7 +25,8 @@ func newTestSuite(t testing.TB) *test.Suite { t.Logf("create new backend at %v", dir) cfg := local.Config{ - Path: dir, + Path: dir, + Connections: 2, } return cfg, nil }, diff --git a/internal/backend/local/local_unix.go b/internal/backend/local/local_unix.go index 81250a5504f..3dde753a8d2 100644 --- a/internal/backend/local/local_unix.go +++ b/internal/backend/local/local_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package local @@ -18,7 +19,7 @@ func fsyncDir(dir string) error { } err = d.Sync() - if errors.Is(err, syscall.ENOTSUP) { + if errors.Is(err, syscall.ENOTSUP) || errors.Is(err, syscall.ENOENT) || errors.Is(err, syscall.EINVAL) { err = nil } diff --git a/internal/backend/location/location.go b/internal/backend/location/location.go index 64ddd8e8236..07d3c6e5356 100644 --- a/internal/backend/location/location.go +++ b/internal/backend/location/location.go @@ -103,7 +103,7 @@ func Parse(s string) (u Location, err error) { // if s is not a path or contains ":", it's ambiguous if !isPath(s) && strings.ContainsRune(s, ':') { - return Location{}, errors.New("invalid backend\nIf the repo is in a local directory, you need to add a `local:` prefix") + return Location{}, errors.New("invalid backend\nIf the repository is in a local directory, you need to add a `local:` prefix") } u.Scheme = "local" diff --git a/internal/backend/location/location_test.go b/internal/backend/location/location_test.go index 3160a2af7eb..809379850ab 100644 --- a/internal/backend/location/location_test.go +++ b/internal/backend/location/location_test.go @@ -30,7 +30,8 @@ var parseTests = []struct { "local:/srv/repo", Location{Scheme: "local", Config: local.Config{ - Path: "/srv/repo", + Path: "/srv/repo", + Connections: 2, }, }, }, @@ -38,7 +39,8 @@ var parseTests = []struct { "local:dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "dir1/dir2", + Path: "dir1/dir2", + Connections: 2, }, }, }, @@ -46,7 +48,8 @@ var parseTests = []struct { "local:dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "dir1/dir2", + Path: "dir1/dir2", + Connections: 2, }, }, }, @@ -54,7 +57,8 @@ var parseTests = []struct { "dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "dir1/dir2", + Path: "dir1/dir2", + Connections: 2, }, }, }, @@ -62,7 +66,8 @@ var parseTests = []struct { "/dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "/dir1/dir2", + Path: "/dir1/dir2", + Connections: 2, }, }, }, @@ -70,7 +75,8 @@ var parseTests = []struct { "local:../dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "../dir1/dir2", + Path: "../dir1/dir2", + Connections: 2, }, }, }, @@ -78,7 +84,8 @@ var parseTests = []struct { "/dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "/dir1/dir2", + Path: "/dir1/dir2", + Connections: 2, }, }, }, @@ -86,7 +93,8 @@ var parseTests = []struct { "/dir1:foobar/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "/dir1:foobar/dir2", + Path: "/dir1:foobar/dir2", + Connections: 2, }, }, }, @@ -94,7 +102,8 @@ var parseTests = []struct { `\dir1\foobar\dir2`, Location{Scheme: "local", Config: local.Config{ - Path: `\dir1\foobar\dir2`, + Path: `\dir1\foobar\dir2`, + Connections: 2, }, }, }, @@ -102,7 +111,8 @@ var parseTests = []struct { `c:\dir1\foobar\dir2`, Location{Scheme: "local", Config: local.Config{ - Path: `c:\dir1\foobar\dir2`, + Path: `c:\dir1\foobar\dir2`, + Connections: 2, }, }, }, @@ -110,7 +120,8 @@ var parseTests = []struct { `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, Location{Scheme: "local", Config: local.Config{ - Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, + Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, + Connections: 2, }, }, }, @@ -118,7 +129,8 @@ var parseTests = []struct { `c:/dir1/foobar/dir2`, Location{Scheme: "local", Config: local.Config{ - Path: `c:/dir1/foobar/dir2`, + Path: `c:/dir1/foobar/dir2`, + Connections: 2, }, }, }, @@ -126,9 +138,10 @@ var parseTests = []struct { "sftp:user@host:/srv/repo", Location{Scheme: "sftp", Config: sftp.Config{ - User: "user", - Host: "host", - Path: "/srv/repo", + User: "user", + Host: "host", + Path: "/srv/repo", + Connections: 5, }, }, }, @@ -136,9 +149,10 @@ var parseTests = []struct { "sftp:host:/srv/repo", Location{Scheme: "sftp", Config: sftp.Config{ - User: "", - Host: "host", - Path: "/srv/repo", + User: "", + Host: "host", + Path: "/srv/repo", + Connections: 5, }, }, }, @@ -146,9 +160,10 @@ var parseTests = []struct { "sftp://user@host/srv/repo", Location{Scheme: "sftp", Config: sftp.Config{ - User: "user", - Host: "host", - Path: "srv/repo", + User: "user", + Host: "host", + Path: "srv/repo", + Connections: 5, }, }, }, @@ -156,9 +171,10 @@ var parseTests = []struct { "sftp://user@host//srv/repo", Location{Scheme: "sftp", Config: sftp.Config{ - User: "user", - Host: "host", - Path: "/srv/repo", + User: "user", + Host: "host", + Path: "/srv/repo", + Connections: 5, }, }, }, diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 9e3cd0e745a..7e8ae53569c 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -25,17 +26,26 @@ var _ restic.Backend = &MemoryBackend{} var errNotFound = errors.New("not found") +const connectionCount = 2 + // MemoryBackend is a mock backend that uses a map for storing all data in // memory. This should only be used for tests. type MemoryBackend struct { data memMap m sync.Mutex + sem sema.Semaphore } // New returns a new backend that saves all data in a map in memory. func New() *MemoryBackend { + sem, err := sema.New(connectionCount) + if err != nil { + panic(err) + } + be := &MemoryBackend{ data: make(memMap), + sem: sem, } debug.Log("created new memory backend") @@ -45,6 +55,9 @@ func New() *MemoryBackend { // Test returns whether a file exists. func (be *MemoryBackend) Test(ctx context.Context, h restic.Handle) (bool, error) { + be.sem.GetToken() + defer be.sem.ReleaseToken() + be.m.Lock() defer be.m.Unlock() @@ -59,7 +72,7 @@ func (be *MemoryBackend) Test(ctx context.Context, h restic.Handle) (bool, error // IsNotExist returns true if the file does not exist. func (be *MemoryBackend) IsNotExist(err error) bool { - return errors.Cause(err) == errNotFound + return errors.Is(err, errNotFound) } // Save adds new Data to the backend. @@ -68,6 +81,9 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re return backoff.Permanent(err) } + be.sem.GetToken() + defer be.sem.ReleaseToken() + be.m.Lock() defer be.m.Unlock() @@ -120,6 +136,7 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length return nil, backoff.Permanent(err) } + be.sem.GetToken() be.m.Lock() defer be.m.Unlock() @@ -131,15 +148,18 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length debug.Log("Load %v offset %v len %v", h, offset, length) if offset < 0 { + be.sem.ReleaseToken() return nil, errors.New("offset is negative") } if _, ok := be.data[h]; !ok { + be.sem.ReleaseToken() return nil, errNotFound } buf := be.data[h] if offset > int64(len(buf)) { + be.sem.ReleaseToken() return nil, errors.New("offset beyond end of file") } @@ -148,18 +168,21 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length buf = buf[:length] } - return ioutil.NopCloser(bytes.NewReader(buf)), ctx.Err() + return be.sem.ReleaseTokenOnClose(ioutil.NopCloser(bytes.NewReader(buf)), nil), ctx.Err() } // Stat returns information about a file in the backend. func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - be.m.Lock() - defer be.m.Unlock() - if err := h.Valid(); err != nil { return restic.FileInfo{}, backoff.Permanent(err) } + be.sem.GetToken() + defer be.sem.ReleaseToken() + + be.m.Lock() + defer be.m.Unlock() + h.ContainedBlobType = restic.InvalidBlob if h.Type == restic.ConfigFile { h.Name = "" @@ -177,6 +200,9 @@ func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.File // Remove deletes a file from the backend. func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { + be.sem.GetToken() + defer be.sem.ReleaseToken() + be.m.Lock() defer be.m.Unlock() @@ -229,6 +255,10 @@ func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(re return ctx.Err() } +func (be *MemoryBackend) Connections() uint { + return connectionCount +} + // Location returns the location of the backend (RAM). func (be *MemoryBackend) Location() string { return "RAM" @@ -239,6 +269,11 @@ func (be *MemoryBackend) Hasher() hash.Hash { return md5.New() } +// HasAtomicReplace returns whether Save() can atomically replace files +func (be *MemoryBackend) HasAtomicReplace() bool { + return false +} + // Delete removes all data in the backend. func (be *MemoryBackend) Delete(ctx context.Context) error { be.m.Lock() diff --git a/internal/mock/backend.go b/internal/backend/mock/backend.go similarity index 72% rename from internal/mock/backend.go rename to internal/backend/mock/backend.go index 9f6036fdb54..655499b1550 100644 --- a/internal/mock/backend.go +++ b/internal/backend/mock/backend.go @@ -11,17 +11,19 @@ import ( // Backend implements a mock backend. type Backend struct { - CloseFn func() error - IsNotExistFn func(err error) bool - SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error - OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) - StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) - ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error - RemoveFn func(ctx context.Context, h restic.Handle) error - TestFn func(ctx context.Context, h restic.Handle) (bool, error) - DeleteFn func(ctx context.Context) error - LocationFn func() string - HasherFn func() hash.Hash + CloseFn func() error + IsNotExistFn func(err error) bool + SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error + OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) + StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) + ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error + RemoveFn func(ctx context.Context, h restic.Handle) error + TestFn func(ctx context.Context, h restic.Handle) (bool, error) + DeleteFn func(ctx context.Context) error + ConnectionsFn func() uint + LocationFn func() string + HasherFn func() hash.Hash + HasAtomicReplaceFn func() bool } // NewBackend returns new mock Backend instance @@ -39,6 +41,14 @@ func (m *Backend) Close() error { return m.CloseFn() } +func (m *Backend) Connections() uint { + if m.ConnectionsFn == nil { + return 2 + } + + return m.ConnectionsFn() +} + // Location returns a location string. func (m *Backend) Location() string { if m.LocationFn == nil { @@ -57,6 +67,14 @@ func (m *Backend) Hasher() hash.Hash { return m.HasherFn() } +// HasAtomicReplace returns whether Save() can atomically replace files +func (m *Backend) HasAtomicReplace() bool { + if m.HasAtomicReplaceFn == nil { + return false + } + return m.HasAtomicReplaceFn() +} + // IsNotExist returns true if the error is caused by a missing file. func (m *Backend) IsNotExist(err error) bool { if m.IsNotExistFn == nil { diff --git a/internal/backend/ontap/ontap.go b/internal/backend/ontap/ontap.go index 47573fa685a..ab657cc9bca 100644 --- a/internal/backend/ontap/ontap.go +++ b/internal/backend/ontap/ontap.go @@ -5,6 +5,7 @@ import ( "crypto/md5" "crypto/tls" "fmt" + "github.com/restic/restic/internal/backend/sema" "hash" "io" "net/http" @@ -42,11 +43,20 @@ func (wr wrapReader) Close() error { type Backend struct { client s3iface.S3API - sem *backend.Semaphore cfg Config + sem sema.Semaphore backend.Layout } +//Connections returns the max number of back end operations, just pulled 42 out of the book +func (be *Backend) Connections() uint { + return 42 +} + +func (be *Backend) HasAtomicReplace() bool { + return false +} + // Hasher may return a hash function for calculating a content hash for the backend func (be *Backend) Hasher() hash.Hash { return md5.New() @@ -402,12 +412,8 @@ func Open(ctx context.Context, config Config) (*Backend, error) { client := s3.New(newSession) - sem, err := backend.NewSemaphore(config.Connections) - if err != nil { - return nil, err - } - - newBackend := NewBackend(client, sem, config) + sem, _ := sema.New(1) + newBackend := NewBackend(client, sem,config) layout, err := backend.ParseLayout(ctx, newBackend, "default", defaultLayout, config.Prefix) if err != nil { @@ -419,7 +425,8 @@ func Open(ctx context.Context, config Config) (*Backend, error) { return newBackend, nil } -func NewBackend(client s3iface.S3API, sem *backend.Semaphore, config Config) *Backend { +func NewBackend(client s3iface.S3API,sem sema.Semaphore, config Config) *Backend { + return &Backend{ client: client, sem: sem, diff --git a/internal/backend/ontap/ontap_test.go b/internal/backend/ontap/ontap_test.go index 21ca9a23e7c..aacea3efc7e 100644 --- a/internal/backend/ontap/ontap_test.go +++ b/internal/backend/ontap/ontap_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/restic/restic/internal/backend/sema" "io" "net/http" "net/url" @@ -203,7 +204,7 @@ func (client *mockS3Api) GetObjectWithContext(_ aws.Context, input *s3.GetObject func makeBackend() (*ontap.Backend, *mockS3Api) { client := &mockS3Api{} - sem, _ := backend.NewSemaphore(1) + sem, _ := sema.New(1) config := ontap.Config{Bucket: aws.String("foobucket"), Prefix: "fooprefix"} be := ontap.NewBackend(client, sem, config) layout, err := backend.ParseLayout(context.TODO(), be, "default", "default", config.Prefix) diff --git a/internal/backend/paths.go b/internal/backend/paths.go index 940e9fcb92d..eaa1a433aac 100644 --- a/internal/backend/paths.go +++ b/internal/backend/paths.go @@ -21,6 +21,28 @@ var Paths = struct { "config", } -// Modes holds the default modes for directories and files for file-based -// backends. -var Modes = struct{ Dir, File os.FileMode }{0700, 0600} +type Modes struct { + Dir os.FileMode + File os.FileMode +} + +// DefaultModes defines the default permissions to apply to new repository +// files and directories stored on file-based backends. +var DefaultModes = Modes{Dir: 0700, File: 0600} + +// DeriveModesFromFileInfo will, given the mode of a regular file, compute +// the mode we should use for new files and directories. If the passed +// error is non-nil DefaultModes are returned. +func DeriveModesFromFileInfo(fi os.FileInfo, err error) Modes { + m := DefaultModes + if err != nil { + return m + } + + if fi.Mode()&0040 != 0 { // Group has read access + m.Dir |= 0070 + m.File |= 0060 + } + + return m +} diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index 8c1305f7f7b..cccc5238442 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -15,11 +15,12 @@ import ( "sync" "time" + "github.com/cenkalti/backoff/v4" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/limiter" "golang.org/x/net/context/ctxhttp" "golang.org/x/net/http2" ) @@ -174,7 +175,7 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) { debug.Log("new connection requested, %v %v", network, address) if dialCount > 0 { // the connection to the child process is already closed - return nil, errors.New("rclone stdio connection already closed") + return nil, backoff.Permanent(errors.New("rclone stdio connection already closed")) } dialCount++ return conn, nil diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go index 0a8f91aeacc..9708c6af221 100644 --- a/internal/backend/rclone/backend_test.go +++ b/internal/backend/rclone/backend_test.go @@ -29,7 +29,8 @@ func newTestSuite(t testing.TB) *test.Suite { t.Logf("Create()") cfg := config.(rclone.Config) be, err := rclone.Create(context.TODO(), cfg) - if e, ok := errors.Cause(err).(*exec.Error); ok && e.Err == exec.ErrNotFound { + var e *exec.Error + if errors.As(err, &e) && e.Err == exec.ErrNotFound { t.Skipf("program %q not found", e.Name) return nil, nil } diff --git a/internal/backend/rclone/internal_test.go b/internal/backend/rclone/internal_test.go index 8bc661ab734..fe9a63d3069 100644 --- a/internal/backend/rclone/internal_test.go +++ b/internal/backend/rclone/internal_test.go @@ -18,7 +18,8 @@ func TestRcloneExit(t *testing.T) { cfg := NewConfig() cfg.Remote = dir be, err := Open(cfg, nil) - if e, ok := errors.Cause(err).(*exec.Error); ok && e.Err == exec.ErrNotFound { + var e *exec.Error + if errors.As(err, &e) && e.Err == exec.ErrNotFound { t.Skipf("program %q not found", e.Name) return } diff --git a/internal/restic/readerat.go b/internal/backend/readerat.go similarity index 75% rename from internal/restic/readerat.go rename to internal/backend/readerat.go index 1a781c03fea..ff2e4039397 100644 --- a/internal/restic/readerat.go +++ b/internal/backend/readerat.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "context" @@ -6,12 +6,13 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" ) type backendReaderAt struct { ctx context.Context - be Backend - h Handle + be restic.Backend + h restic.Handle } func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { @@ -21,12 +22,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { // ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader // should not escape the caller function to avoid unexpected interactions with the // embedded context -func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt { +func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt { return backendReaderAt{ctx: ctx, be: be, h: h} } // ReadAt reads from the backend handle h at the given position. -func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) { +func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) { debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p)) err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) { diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index c7675cba153..cd41bc0cebc 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -17,6 +17,7 @@ import ( "golang.org/x/net/context/ctxhttp" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -29,9 +30,10 @@ var _ restic.Backend = &Backend{} // Backend uses the REST protocol to access data stored on a server. type Backend struct { - url *url.URL - sem *backend.Semaphore - client *http.Client + url *url.URL + connections uint + sem sema.Semaphore + client *http.Client backend.Layout } @@ -45,7 +47,7 @@ const ( func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { client := &http.Client{Transport: rt} - sem, err := backend.NewSemaphore(cfg.Connections) + sem, err := sema.New(cfg.Connections) if err != nil { return nil, err } @@ -57,10 +59,11 @@ func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { } be := &Backend{ - url: cfg.URL, - client: client, - Layout: &backend.RESTLayout{URL: url, Join: path.Join}, - sem: sem, + url: cfg.URL, + client: client, + Layout: &backend.RESTLayout{URL: url, Join: path.Join}, + connections: cfg.Connections, + sem: sem, } return be, nil @@ -105,6 +108,10 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er return be, nil } +func (b *Backend) Connections() uint { + return b.connections +} + // Location returns this backend's location (the server's URL). func (b *Backend) Location() string { return b.url.String() @@ -115,6 +122,12 @@ func (b *Backend) Hasher() hash.Hash { return nil } +// HasAtomicReplace returns whether Save() can atomically replace files +func (b *Backend) HasAtomicReplace() bool { + // rest-server prevents overwriting + return false +} + // Save stores data in the backend at the handle. func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { if err := h.Valid(); err != nil { @@ -157,21 +170,20 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea return errors.Wrap(cerr, "Close") } -// ErrIsNotExist is returned whenever the requested file does not exist on the +// notExistError is returned whenever the requested file does not exist on the // server. -type ErrIsNotExist struct { +type notExistError struct { restic.Handle } -func (e ErrIsNotExist) Error() string { +func (e *notExistError) Error() string { return fmt.Sprintf("%v does not exist", e.Handle) } // IsNotExist returns true if the error was caused by a non-existing file. func (b *Backend) IsNotExist(err error) bool { - err = errors.Cause(err) - _, ok := err.(ErrIsNotExist) - return ok + var e *notExistError + return errors.As(err, &e) } // Load runs fn with a reader that yields the contents of the file at h at the @@ -284,7 +296,7 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o if resp.StatusCode == http.StatusNotFound { _ = resp.Body.Close() - return nil, ErrIsNotExist{h} + return nil, ¬ExistError{h} } if resp.StatusCode != 200 && resp.StatusCode != 206 { @@ -329,7 +341,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e if resp.StatusCode == http.StatusNotFound { _ = resp.Body.Close() - return restic.FileInfo{}, ErrIsNotExist{h} + return restic.FileInfo{}, ¬ExistError{h} } if resp.StatusCode != 200 { @@ -380,7 +392,7 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { if resp.StatusCode == http.StatusNotFound { _ = resp.Body.Close() - return ErrIsNotExist{h} + return ¬ExistError{h} } if resp.StatusCode != 200 { diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go index 77b712fc7a5..9e83f4004c3 100644 --- a/internal/backend/s3/config.go +++ b/internal/backend/s3/config.go @@ -12,13 +12,14 @@ import ( // Config contains all configuration necessary to connect to an s3 compatible // server. type Config struct { - Endpoint string - UseHTTP bool - KeyID, Secret string - Bucket string - Prefix string - Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"` - StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"` + Endpoint string + UseHTTP bool + KeyID string + Secret options.SecretString + Bucket string + Prefix string + Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"` + StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"` Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` MaxRetries uint `option:"retries" help:"set the number of retries attempted"` diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index d84c00c468e..a22debf2a91 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -13,6 +13,7 @@ import ( "time" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -25,7 +26,7 @@ import ( // Backend stores data on an S3 endpoint. type Backend struct { client *minio.Client - sem *backend.Semaphore + sem sema.Semaphore cfg Config backend.Layout } @@ -56,7 +57,7 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro &credentials.Static{ Value: credentials.Value{ AccessKeyID: cfg.KeyID, - SecretAccessKey: cfg.Secret, + SecretAccessKey: cfg.Secret.Unwrap(), }, }, &credentials.EnvMinio{}, @@ -101,7 +102,7 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro return nil, errors.Wrap(err, "minio.New") } - sem, err := backend.NewSemaphore(cfg.Connections) + sem, err := sema.New(cfg.Connections) if err != nil { return nil, err } @@ -137,7 +138,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe } found, err := be.client.BucketExists(ctx, cfg.Bucket) - if err != nil && be.IsAccessDenied(err) { + if err != nil && isAccessDenied(err) { err = nil found = true } @@ -158,29 +159,23 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe return be, nil } -// IsAccessDenied returns true if the error is caused by Access Denied. -func (be *Backend) IsAccessDenied(err error) bool { - debug.Log("IsAccessDenied(%T, %#v)", err, err) +// isAccessDenied returns true if the error is caused by Access Denied. +func isAccessDenied(err error) bool { + debug.Log("isAccessDenied(%T, %#v)", err, err) - if e, ok := errors.Cause(err).(minio.ErrorResponse); ok && e.Code == "AccessDenied" { - return true - } - - return false + var e minio.ErrorResponse + return errors.As(err, &e) && e.Code == "Access Denied" } // IsNotExist returns true if the error is caused by a not existing file. func (be *Backend) IsNotExist(err error) bool { debug.Log("IsNotExist(%T, %#v)", err, err) - if os.IsNotExist(errors.Cause(err)) { - return true - } - - if e, ok := errors.Cause(err).(minio.ErrorResponse); ok && e.Code == "NoSuchKey" { + if errors.Is(err, os.ErrNotExist) { return true } - return false + var e minio.ErrorResponse + return errors.As(err, &e) && e.Code == "NoSuchKey" } // Join combines path components with slashes. @@ -255,6 +250,10 @@ func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo, return list, nil } +func (be *Backend) Connections() uint { + return be.cfg.Connections +} + // Location returns this backend's location (the bucket name). func (be *Backend) Location() string { return be.Join(be.cfg.Bucket, be.cfg.Prefix) @@ -265,6 +264,11 @@ func (be *Backend) Hasher() hash.Hash { return nil } +// HasAtomicReplace returns whether Save() can atomically replace files +func (be *Backend) HasAtomicReplace() bool { + return true +} + // Path returns the path in the bucket that is used for this backend. func (be *Backend) Path() string { return be.cfg.Prefix @@ -287,6 +291,8 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe opts.ContentType = "application/octet-stream" // the only option with the high-level api is to let the library handle the checksum computation opts.SendContentMd5 = true + // only use multipart uploads for very large files + opts.PartSize = 200 * 1024 * 1024 debug.Log("PutObject(%v, %v, %v)", be.cfg.Bucket, objName, rd.Length()) info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, ioutil.NopCloser(rd), int64(rd.Length()), opts) @@ -301,18 +307,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe return errors.Wrap(err, "client.PutObject") } -// wrapReader wraps an io.ReadCloser to run an additional function on Close. -type wrapReader struct { - io.ReadCloser - f func() -} - -func (wr wrapReader) Close() error { - err := wr.ReadCloser.Close() - wr.f() - return err -} - // Load runs fn with a reader that yields the contents of the file at h at the // given offset. func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { @@ -350,22 +344,17 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, } be.sem.GetToken() + ctx, cancel := context.WithCancel(ctx) + coreClient := minio.Core{Client: be.client} rd, _, _, err := coreClient.GetObject(ctx, be.cfg.Bucket, objName, opts) if err != nil { + cancel() be.sem.ReleaseToken() return nil, err } - closeRd := wrapReader{ - ReadCloser: rd, - f: func() { - debug.Log("Close()") - be.sem.ReleaseToken() - }, - } - - return closeRd, err + return be.sem.ReleaseTokenOnClose(rd, cancel), err } // Stat returns information about a blob. diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index ba7a408dd37..46d96fcbdb8 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -18,6 +18,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/test" + "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -141,7 +142,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { cfg.Config.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) cfg.Config.UseHTTP = true cfg.Config.KeyID = key - cfg.Config.Secret = secret + cfg.Config.Secret = options.NewSecretString(secret) return cfg, nil }, @@ -239,7 +240,7 @@ func newS3TestSuite(t testing.TB) *test.Suite { cfg := s3cfg.(s3.Config) cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY") - cfg.Secret = os.Getenv("RESTIC_TEST_S3_SECRET") + cfg.Secret = options.NewSecretString(os.Getenv("RESTIC_TEST_S3_SECRET")) cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) return cfg, nil }, diff --git a/internal/backend/sema/semaphore.go b/internal/backend/sema/semaphore.go new file mode 100644 index 00000000000..7ee91297923 --- /dev/null +++ b/internal/backend/sema/semaphore.go @@ -0,0 +1,65 @@ +// Package sema implements semaphores. +package sema + +import ( + "context" + "io" + + "github.com/restic/restic/internal/errors" +) + +// A Semaphore limits access to a restricted resource. +type Semaphore struct { + ch chan struct{} +} + +// New returns a new semaphore with capacity n. +func New(n uint) (Semaphore, error) { + if n == 0 { + return Semaphore{}, errors.New("capacity must be a positive number") + } + return Semaphore{ + ch: make(chan struct{}, n), + }, nil +} + +// GetToken blocks until a Token is available. +func (s Semaphore) GetToken() { s.ch <- struct{}{} } + +// ReleaseToken returns a token. +func (s Semaphore) ReleaseToken() { <-s.ch } + +// ReleaseTokenOnClose wraps an io.ReadCloser to return a token on Close. +// Before returning the token, cancel, if not nil, will be run +// to free up context resources. +func (s Semaphore) ReleaseTokenOnClose(rc io.ReadCloser, cancel context.CancelFunc) io.ReadCloser { + return &wrapReader{ReadCloser: rc, sem: s, cancel: cancel} +} + +type wrapReader struct { + io.ReadCloser + eofSeen bool + sem Semaphore + cancel context.CancelFunc +} + +func (wr *wrapReader) Read(p []byte) (int, error) { + if wr.eofSeen { // XXX Why do we do this? + return 0, io.EOF + } + + n, err := wr.ReadCloser.Read(p) + if err == io.EOF { + wr.eofSeen = true + } + return n, err +} + +func (wr *wrapReader) Close() error { + err := wr.ReadCloser.Close() + if wr.cancel != nil { + wr.cancel() + } + wr.sem.ReleaseToken() + return err +} diff --git a/internal/backend/semaphore.go b/internal/backend/semaphore.go deleted file mode 100644 index 28b97472b6c..00000000000 --- a/internal/backend/semaphore.go +++ /dev/null @@ -1,69 +0,0 @@ -package backend - -import ( - "context" - "io" - - "github.com/restic/restic/internal/errors" -) - -// Semaphore limits access to a restricted resource. -type Semaphore struct { - ch chan struct{} -} - -// NewSemaphore returns a new semaphore with capacity n. -func NewSemaphore(n uint) (*Semaphore, error) { - if n == 0 { - return nil, errors.New("must be a positive number") - } - return &Semaphore{ - ch: make(chan struct{}, n), - }, nil -} - -// GetToken blocks until a Token is available. -func (s *Semaphore) GetToken() { - s.ch <- struct{}{} -} - -// ReleaseToken returns a token. -func (s *Semaphore) ReleaseToken() { - <-s.ch -} - -// ReleaseTokenOnClose wraps an io.ReadCloser to return a token on Close. Before returning the token, -// cancel, if provided, will be run to free up context resources. -func (s *Semaphore) ReleaseTokenOnClose(rc io.ReadCloser, cancel context.CancelFunc) io.ReadCloser { - return &wrapReader{rc, false, func() { - if cancel != nil { - cancel() - } - s.ReleaseToken() - }} -} - -// wrapReader wraps an io.ReadCloser to run an additional function on Close. -type wrapReader struct { - io.ReadCloser - eofSeen bool - f func() -} - -func (wr *wrapReader) Read(p []byte) (int, error) { - if wr.eofSeen { - return 0, io.EOF - } - - n, err := wr.ReadCloser.Read(p) - if err == io.EOF { - wr.eofSeen = true - } - return n, err -} - -func (wr *wrapReader) Close() error { - err := wr.ReadCloser.Close() - wr.f() - return err -} diff --git a/internal/backend/sftp/config.go b/internal/backend/sftp/config.go index d5e0e5182a7..ec38ee46770 100644 --- a/internal/backend/sftp/config.go +++ b/internal/backend/sftp/config.go @@ -15,6 +15,15 @@ type Config struct { Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` Command string `option:"command" help:"specify command to create sftp connection"` + + Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` +} + +// NewConfig returns a new config with default options applied. +func NewConfig() Config { + return Config{ + Connections: 5, + } } func init() { @@ -23,9 +32,9 @@ func init() { // ParseConfig parses the string s and extracts the sftp config. The // supported configuration formats are sftp://user@host[:port]/directory -// and sftp:user@host:directory. The directory will be path Cleaned and can -// be an absolute path if it starts with a '/' (e.g. -// sftp://user@host//absolute and sftp:user@host:/absolute). +// and sftp:user@host:directory. The directory will be path Cleaned and can +// be an absolute path if it starts with a '/' (e.g. +// sftp://user@host//absolute and sftp:user@host:/absolute). func ParseConfig(s string) (interface{}, error) { var user, host, port, dir string switch { @@ -75,10 +84,11 @@ func ParseConfig(s string) (interface{}, error) { return nil, errors.Fatal("sftp path starts with the tilde (~) character, that fails for most sftp servers.\nUse a relative directory, most servers interpret this as relative to the user's home directory.") } - return Config{ - User: user, - Host: host, - Port: port, - Path: p, - }, nil + cfg := NewConfig() + cfg.User = user + cfg.Host = host + cfg.Port = port + cfg.Path = p + + return cfg, nil } diff --git a/internal/backend/sftp/config_test.go b/internal/backend/sftp/config_test.go index d785a4113ff..3772c038b2c 100644 --- a/internal/backend/sftp/config_test.go +++ b/internal/backend/sftp/config_test.go @@ -11,68 +11,68 @@ var configTests = []struct { // first form, user specified sftp://user@host/dir { "sftp://user@host/dir/subdir", - Config{User: "user", Host: "host", Path: "dir/subdir"}, + Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, }, { "sftp://host/dir/subdir", - Config{Host: "host", Path: "dir/subdir"}, + Config{Host: "host", Path: "dir/subdir", Connections: 5}, }, { "sftp://host//dir/subdir", - Config{Host: "host", Path: "/dir/subdir"}, + Config{Host: "host", Path: "/dir/subdir", Connections: 5}, }, { "sftp://host:10022//dir/subdir", - Config{Host: "host", Port: "10022", Path: "/dir/subdir"}, + Config{Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5}, }, { "sftp://user@host:10022//dir/subdir", - Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir"}, + Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Connections: 5}, }, { "sftp://user@host/dir/subdir/../other", - Config{User: "user", Host: "host", Path: "dir/other"}, + Config{User: "user", Host: "host", Path: "dir/other", Connections: 5}, }, { "sftp://user@host/dir///subdir", - Config{User: "user", Host: "host", Path: "dir/subdir"}, + Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, }, // IPv6 address. { "sftp://user@[::1]/dir", - Config{User: "user", Host: "::1", Path: "dir"}, + Config{User: "user", Host: "::1", Path: "dir", Connections: 5}, }, // IPv6 address with port. { "sftp://user@[::1]:22/dir", - Config{User: "user", Host: "::1", Port: "22", Path: "dir"}, + Config{User: "user", Host: "::1", Port: "22", Path: "dir", Connections: 5}, }, // second form, user specified sftp:user@host:/dir { "sftp:user@host:/dir/subdir", - Config{User: "user", Host: "host", Path: "/dir/subdir"}, + Config{User: "user", Host: "host", Path: "/dir/subdir", Connections: 5}, }, { "sftp:user@domain@host:/dir/subdir", - Config{User: "user@domain", Host: "host", Path: "/dir/subdir"}, + Config{User: "user@domain", Host: "host", Path: "/dir/subdir", Connections: 5}, }, { "sftp:host:../dir/subdir", - Config{Host: "host", Path: "../dir/subdir"}, + Config{Host: "host", Path: "../dir/subdir", Connections: 5}, }, { "sftp:user@host:dir/subdir:suffix", - Config{User: "user", Host: "host", Path: "dir/subdir:suffix"}, + Config{User: "user", Host: "host", Path: "dir/subdir:suffix", Connections: 5}, }, { "sftp:user@host:dir/subdir/../other", - Config{User: "user", Host: "host", Path: "dir/other"}, + Config{User: "user", Host: "host", Path: "dir/other", Connections: 5}, }, { "sftp:user@host:dir///subdir", - Config{User: "user", Host: "host", Path: "dir/subdir"}, + Config{User: "user", Host: "host", Path: "dir/subdir", Connections: 5}, }, } diff --git a/internal/backend/sftp/layout_test.go b/internal/backend/sftp/layout_test.go index 0d0214669e4..3b654b1bbc6 100644 --- a/internal/backend/sftp/layout_test.go +++ b/internal/backend/sftp/layout_test.go @@ -43,9 +43,10 @@ func TestLayout(t *testing.T) { repo := filepath.Join(path, "repo") be, err := sftp.Open(context.TODO(), sftp.Config{ - Command: fmt.Sprintf("%q -e", sftpServer), - Path: repo, - Layout: test.layout, + Command: fmt.Sprintf("%q -e", sftpServer), + Path: repo, + Layout: test.layout, + Connections: 5, }) if err != nil { t.Fatal(err) diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 3e803a0f401..cc565effb79 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -13,14 +13,15 @@ import ( "path" "time" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" "github.com/cenkalti/backoff/v4" "github.com/pkg/sftp" + "golang.org/x/sync/errgroup" ) // SFTP is a backend in a directory accessed via SFTP. @@ -31,8 +32,12 @@ type SFTP struct { cmd *exec.Cmd result <-chan error + posixRename bool + + sem sema.Semaphore backend.Layout Config + backend.Modes } var _ restic.Backend = &SFTP{} @@ -94,7 +99,8 @@ func startClient(program string, args ...string) (*SFTP, error) { return nil, errors.Wrap(err, "bg") } - return &SFTP{c: client, cmd: cmd, result: ch}, nil + _, posixRename := client.HasExtension("posix-rename@openssh.com") + return &SFTP{c: client, cmd: cmd, result: ch, posixRename: posixRename}, nil } // clientError returns an error if the client has exited. Otherwise, nil is @@ -111,11 +117,15 @@ func (r *SFTP) clientError() error { } // Open opens an sftp backend as described by the config by running -// "ssh" with the appropriate arguments (or cfg.Command, if set). The function -// preExec is run just before, postExec just after starting a program. +// "ssh" with the appropriate arguments (or cfg.Command, if set). func Open(ctx context.Context, cfg Config) (*SFTP, error) { debug.Log("open backend with config %#v", cfg) + sem, err := sema.New(cfg.Connections) + if err != nil { + return nil, err + } + cmd, args, err := buildSSHCommand(cfg) if err != nil { return nil, err @@ -134,20 +144,39 @@ func Open(ctx context.Context, cfg Config) (*SFTP, error) { debug.Log("layout: %v\n", sftp.Layout) + fi, err := sftp.c.Stat(Join(cfg.Path, backend.Paths.Config)) + m := backend.DeriveModesFromFileInfo(fi, err) + debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) + sftp.Config = cfg sftp.p = cfg.Path + sftp.sem = sem + sftp.Modes = m return sftp, nil } -func (r *SFTP) mkdirAllDataSubdirs() error { +func (r *SFTP) mkdirAllDataSubdirs(ctx context.Context, nconn uint) error { + // Run multiple MkdirAll calls concurrently. These involve multiple + // round-trips and we do a lot of them, so this whole operation can be slow + // on high-latency links. + g, _ := errgroup.WithContext(ctx) + // Use errgroup's built-in semaphore, because r.sem is not initialized yet. + for _, d := range r.Paths() { - err := r.c.MkdirAll(d) - if err != nil { - return err - } + d := d + g.Go(func() error { + // First try Mkdir. For most directories in Paths, this takes one + // round trip, not counting duplicate parent creations causes by + // concurrency. MkdirAll first does Stat, then recursive MkdirAll + // on the parent, so calls typically take three round trips. + if err := r.c.Mkdir(d); err == nil { + return nil + } + return r.c.MkdirAll(d) + }) } - return nil + return g.Wait() } // Join combines path components with slashes (according to the sftp spec). @@ -167,7 +196,6 @@ func (r *SFTP) ReadDir(ctx context.Context, dir string) ([]os.FileInfo, error) { // IsNotExist returns true if the error is caused by a not existing file. func (r *SFTP) IsNotExist(err error) bool { - err = errors.Cause(err) return errors.Is(err, os.ErrNotExist) } @@ -199,8 +227,7 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) { } // Create creates an sftp backend as described by the config by running "ssh" -// with the appropriate arguments (or cfg.Command, if set). The function -// preExec is run just before, postExec just after starting a program. +// with the appropriate arguments (or cfg.Command, if set). func Create(ctx context.Context, cfg Config) (*SFTP, error) { cmd, args, err := buildSSHCommand(cfg) if err != nil { @@ -218,6 +245,8 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) { return nil, err } + sftp.Modes = backend.DefaultModes + // test if config file already exists _, err = sftp.c.Lstat(Join(cfg.Path, backend.Paths.Config)) if err == nil { @@ -225,7 +254,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) { } // create paths for data and refs - if err = sftp.mkdirAllDataSubdirs(); err != nil { + if err = sftp.mkdirAllDataSubdirs(ctx, cfg.Connections); err != nil { return nil, err } @@ -238,6 +267,10 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) { return Open(ctx, cfg) } +func (r *SFTP) Connections() uint { + return r.Config.Connections +} + // Location returns this backend's location (the directory name). func (r *SFTP) Location() string { return r.p @@ -248,6 +281,11 @@ func (r *SFTP) Hasher() hash.Hash { return nil } +// HasAtomicReplace returns whether Save() can atomically replace files +func (r *SFTP) HasAtomicReplace() bool { + return r.posixRename +} + // Join joins the given paths and cleans them afterwards. This always uses // forward slashes, which is required by sftp. func Join(parts ...string) string { @@ -280,6 +318,9 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader tmpFilename := filename + "-restic-temp-" + tempSuffix() dirname := r.Dirname(h) + r.sem.GetToken() + defer r.sem.ReleaseToken() + // create new file f, err := r.c.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY) @@ -297,7 +338,7 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader // pkg/sftp doesn't allow creating with a mode. // Chmod while the file is still empty. if err == nil { - err = f.Chmod(backend.Modes.File) + err = f.Chmod(r.Modes.File) } if err != nil { return errors.Wrap(err, "OpenFile") @@ -336,7 +377,12 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader return errors.Wrap(err, "Close") } - err = r.c.Rename(tmpFilename, filename) + // Prefer POSIX atomic rename if available. + if r.posixRename { + err = r.c.PosixRename(tmpFilename, filename) + } else { + err = r.c.Rename(tmpFilename, filename) + } return errors.Wrap(err, "Rename") } @@ -371,6 +417,19 @@ func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int return backend.DefaultLoad(ctx, h, length, offset, r.openReader, fn) } +// wrapReader wraps an io.ReadCloser to run an additional function on Close. +type wrapReader struct { + io.ReadCloser + io.WriterTo + f func() +} + +func (wr *wrapReader) Close() error { + err := wr.ReadCloser.Close() + wr.f() + return err +} + func (r *SFTP) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v", h, length, offset) if err := h.Valid(); err != nil { @@ -381,26 +440,38 @@ func (r *SFTP) openReader(ctx context.Context, h restic.Handle, length int, offs return nil, errors.New("offset is negative") } + r.sem.GetToken() f, err := r.c.Open(r.Filename(h)) if err != nil { + r.sem.ReleaseToken() return nil, err } if offset > 0 { _, err = f.Seek(offset, 0) if err != nil { + r.sem.ReleaseToken() _ = f.Close() return nil, err } } + // use custom close wrapper to also provide WriteTo() on the wrapper + rd := &wrapReader{ + ReadCloser: f, + WriterTo: f, + f: func() { + r.sem.ReleaseToken() + }, + } + if length > 0 { // unlimited reads usually use io.Copy which needs WriteTo support at the underlying reader // limited reads are usually combined with io.ReadFull which reads all required bytes into a buffer in one go - return backend.LimitReadCloser(f, int64(length)), nil + return backend.LimitReadCloser(rd, int64(length)), nil } - return f, nil + return rd, nil } // Stat returns information about a blob. @@ -414,6 +485,9 @@ func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, erro return restic.FileInfo{}, backoff.Permanent(err) } + r.sem.GetToken() + defer r.sem.ReleaseToken() + fi, err := r.c.Lstat(r.Filename(h)) if err != nil { return restic.FileInfo{}, errors.Wrap(err, "Lstat") @@ -429,8 +503,11 @@ func (r *SFTP) Test(ctx context.Context, h restic.Handle) (bool, error) { return false, err } + r.sem.GetToken() + defer r.sem.ReleaseToken() + _, err := r.c.Lstat(r.Filename(h)) - if os.IsNotExist(errors.Cause(err)) { + if errors.Is(err, os.ErrNotExist) { return false, nil } @@ -448,6 +525,9 @@ func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error { return err } + r.sem.GetToken() + defer r.sem.ReleaseToken() + return r.c.Remove(r.Filename(h)) } @@ -458,7 +538,14 @@ func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileI basedir, subdirs := r.Basedir(t) walker := r.c.Walk(basedir) - for walker.Step() { + for { + r.sem.GetToken() + ok := walker.Step() + r.sem.ReleaseToken() + if !ok { + break + } + if walker.Err() != nil { if r.IsNotExist(walker.Err()) { debug.Log("ignoring non-existing directory") diff --git a/internal/backend/sftp/sftp_test.go b/internal/backend/sftp/sftp_test.go index 61bc49dc898..e682b343af6 100644 --- a/internal/backend/sftp/sftp_test.go +++ b/internal/backend/sftp/sftp_test.go @@ -20,7 +20,7 @@ func findSFTPServerBinary() string { for _, dir := range strings.Split(rtest.TestSFTPPath, ":") { testpath := filepath.Join(dir, "sftp-server") _, err := os.Stat(testpath) - if !os.IsNotExist(errors.Cause(err)) { + if !errors.Is(err, os.ErrNotExist) { return testpath } } @@ -42,8 +42,9 @@ func newTestSuite(t testing.TB) *test.Suite { t.Logf("create new backend at %v", dir) cfg := sftp.Config{ - Path: dir, - Command: fmt.Sprintf("%q -e", sftpServer), + Path: dir, + Command: fmt.Sprintf("%q -e", sftpServer), + Connections: 5, } return cfg, nil }, diff --git a/internal/backend/swift/config.go b/internal/backend/swift/config.go index 8ca26a9180a..d2751dd1a5a 100644 --- a/internal/backend/swift/config.go +++ b/internal/backend/swift/config.go @@ -24,12 +24,12 @@ type Config struct { TrustID string StorageURL string - AuthToken string + AuthToken options.SecretString // auth v3 only ApplicationCredentialID string ApplicationCredentialName string - ApplicationCredentialSecret string + ApplicationCredentialSecret options.SecretString Container string Prefix string @@ -111,11 +111,9 @@ func ApplyEnvironment(prefix string, cfg interface{}) error { // Application Credential auth {&c.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"}, {&c.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"}, - {&c.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"}, // Manual authentication {&c.StorageURL, prefix + "OS_STORAGE_URL"}, - {&c.AuthToken, prefix + "OS_AUTH_TOKEN"}, {&c.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"}, } { @@ -123,5 +121,16 @@ func ApplyEnvironment(prefix string, cfg interface{}) error { *val.s = os.Getenv(val.env) } } + for _, val := range []struct { + s *options.SecretString + env string + }{ + {&c.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"}, + {&c.AuthToken, prefix + "OS_AUTH_TOKEN"}, + } { + if val.s.String() == "" { + *val.s = options.NewSecretString(os.Getenv(val.env)) + } + } return nil } diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index 8d82d90cb41..a739b2c6bec 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -14,6 +14,7 @@ import ( "time" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -24,10 +25,11 @@ import ( // beSwift is a backend which stores the data on a swift endpoint. type beSwift struct { - conn *swift.Connection - sem *backend.Semaphore - container string // Container name - prefix string // Prefix of object names in the container + conn *swift.Connection + connections uint + sem sema.Semaphore + container string // Container name + prefix string // Prefix of object names in the container backend.Layout } @@ -39,7 +41,7 @@ var _ restic.Backend = &beSwift{} func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { debug.Log("config %#v", cfg) - sem, err := backend.NewSemaphore(cfg.Connections) + sem, err := sema.New(cfg.Connections) if err != nil { return nil, err } @@ -59,18 +61,19 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend TenantDomainId: cfg.TenantDomainID, TrustId: cfg.TrustID, StorageUrl: cfg.StorageURL, - AuthToken: cfg.AuthToken, + AuthToken: cfg.AuthToken.Unwrap(), ApplicationCredentialId: cfg.ApplicationCredentialID, ApplicationCredentialName: cfg.ApplicationCredentialName, - ApplicationCredentialSecret: cfg.ApplicationCredentialSecret, + ApplicationCredentialSecret: cfg.ApplicationCredentialSecret.Unwrap(), ConnectTimeout: time.Minute, Timeout: time.Minute, Transport: rt, }, - sem: sem, - container: cfg.Container, - prefix: cfg.Prefix, + connections: cfg.Connections, + sem: sem, + container: cfg.Container, + prefix: cfg.Prefix, Layout: &backend.DefaultLayout{ Path: cfg.Prefix, Join: path.Join, @@ -113,6 +116,10 @@ func (be *beSwift) createContainer(ctx context.Context, policy string) error { return be.conn.ContainerCreate(ctx, be.container, h) } +func (be *beSwift) Connections() uint { + return be.connections +} + // Location returns this backend's location (the container name). func (be *beSwift) Location() string { return be.container @@ -123,6 +130,11 @@ func (be *beSwift) Hasher() hash.Hash { return md5.New() } +// HasAtomicReplace returns whether Save() can atomically replace files +func (be *beSwift) HasAtomicReplace() bool { + return true +} + // Load runs fn with a reader that yields the contents of the file at h at the // given offset. func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { @@ -300,11 +312,8 @@ func (be *beSwift) removeKeys(ctx context.Context, t restic.FileType) error { // IsNotExist returns true if the error is caused by a not existing file. func (be *beSwift) IsNotExist(err error) bool { - if e, ok := errors.Cause(err).(*swift.Error); ok { - return e.StatusCode == http.StatusNotFound - } - - return false + var e *swift.Error + return errors.As(err, &e) && e.StatusCode == http.StatusNotFound } // Delete removes all restic objects in the container. diff --git a/internal/backend/test/doc.go b/internal/backend/test/doc.go index c1704d2c914..25bdf0417a8 100644 --- a/internal/backend/test/doc.go +++ b/internal/backend/test/doc.go @@ -1,6 +1,6 @@ // Package test contains a test suite with benchmarks for restic backends. // -// Overview +// # Overview // // For the test suite to work a few functions need to be implemented to create // new config, create a backend, open it and run cleanup tasks afterwards. The @@ -10,30 +10,31 @@ // then the methods RunTests() and RunBenchmarks() can be used to run the // individual tests and benchmarks as subtests/subbenchmarks. // -// Example +// # Example // // Assuming a *Suite is returned by newTestSuite(), the tests and benchmarks // can be run like this: -// func newTestSuite(t testing.TB) *test.Suite { -// return &test.Suite{ -// Create: func(cfg interface{}) (restic.Backend, error) { -// [...] -// }, -// [...] -// } -// } -// -// func TestSuiteBackendMem(t *testing.T) { -// newTestSuite(t).RunTests(t) -// } -// -// func BenchmarkSuiteBackendMem(b *testing.B) { -// newTestSuite(b).RunBenchmarks(b) -// } +// +// func newTestSuite(t testing.TB) *test.Suite { +// return &test.Suite{ +// Create: func(cfg interface{}) (restic.Backend, error) { +// [...] +// }, +// [...] +// } +// } +// +// func TestSuiteBackendMem(t *testing.T) { +// newTestSuite(t).RunTests(t) +// } +// +// func BenchmarkSuiteBackendMem(b *testing.B) { +// newTestSuite(b).RunBenchmarks(b) +// } // // The functions are run in alphabetical order. // -// Add new tests +// # Add new tests // // A new test or benchmark can be added by implementing a method on *Suite // with the name starting with "Test" and a single *testing.T parameter for diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index d0dc2dbf76c..f05366f6d2e 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -10,7 +10,6 @@ import ( "os" "reflect" "sort" - "strings" "testing" "time" @@ -330,11 +329,6 @@ func (s *Suite) TestList(t *testing.T) { } } -func isCanceledError(err error) bool { - cause := strings.ToLower(fmt.Sprint(err)) - return strings.Contains(cause, "cancel") -} - // TestListCancel tests that the context is respected and the error is returned by List. func (s *Suite) TestListCancel(t *testing.T) { seedRand(t) @@ -367,7 +361,7 @@ func (s *Suite) TestListCancel(t *testing.T) { return nil }) - if !isCanceledError(err) { + if !errors.Is(err, context.Canceled) { t.Fatalf("expected error not found, want %v, got %v", context.Canceled, err) } }) @@ -386,7 +380,7 @@ func (s *Suite) TestListCancel(t *testing.T) { return nil }) - if !isCanceledError(err) { + if !errors.Is(err, context.Canceled) { t.Fatalf("expected error not found, want %v, got %v", context.Canceled, err) } @@ -409,7 +403,7 @@ func (s *Suite) TestListCancel(t *testing.T) { return nil }) - if !isCanceledError(err) { + if !errors.Is(err, context.Canceled) { t.Fatalf("expected error not found, want %v, got %v", context.Canceled, err) } @@ -435,7 +429,7 @@ func (s *Suite) TestListCancel(t *testing.T) { return nil }) - if errors.Cause(err) != context.DeadlineExceeded { + if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("expected error not found, want %#v, got %#v", context.DeadlineExceeded, err) } diff --git a/internal/backend/utils.go b/internal/backend/utils.go index 39c68b4ce32..be1c2a9e0ce 100644 --- a/internal/backend/utils.go +++ b/internal/backend/utils.go @@ -3,6 +3,7 @@ package backend import ( "bytes" "context" + "fmt" "io" "github.com/restic/restic/internal/restic" @@ -57,3 +58,44 @@ func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, } return rd.Close() } + +type memorizedLister struct { + fileInfos []restic.FileInfo + tpe restic.FileType +} + +func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + if t != m.tpe { + return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t) + } + for _, fi := range m.fileInfos { + if ctx.Err() != nil { + break + } + err := fn(fi) + if err != nil { + return err + } + } + return ctx.Err() +} + +func MemorizeList(ctx context.Context, be restic.Lister, t restic.FileType) (restic.Lister, error) { + if _, ok := be.(*memorizedLister); ok { + return be, nil + } + + var fileInfos []restic.FileInfo + err := be.List(ctx, t, func(fi restic.FileInfo) error { + fileInfos = append(fileInfos, fi) + return nil + }) + if err != nil { + return nil, err + } + + return &memorizedLister{ + fileInfos: fileInfos, + tpe: t, + }, nil +} diff --git a/internal/backend/utils_test.go b/internal/backend/utils_test.go index 1030537bc9c..2e77fa9bd13 100644 --- a/internal/backend/utils_test.go +++ b/internal/backend/utils_test.go @@ -3,12 +3,14 @@ package backend_test import ( "bytes" "context" + "fmt" "io" "math/rand" "testing" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" + "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -157,3 +159,47 @@ func TestDefaultLoad(t *testing.T) { rtest.Equals(t, true, rd.closed) rtest.Equals(t, "consumer error", err.Error()) } + +func TestMemoizeList(t *testing.T) { + // setup backend to serve as data source for memoized list + be := mock.NewBackend() + files := []restic.FileInfo{ + {Size: 42, Name: restic.NewRandomID().String()}, + {Size: 45, Name: restic.NewRandomID().String()}, + } + be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + for _, fi := range files { + if err := fn(fi); err != nil { + return err + } + } + return nil + } + + mem, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile) + rtest.OK(t, err) + + err = mem.List(context.TODO(), restic.IndexFile, func(fi restic.FileInfo) error { + t.Fatal("file type mismatch") + return nil // the memoized lister must return an error by itself + }) + rtest.Assert(t, err != nil, "missing error on file typ mismatch") + + var memFiles []restic.FileInfo + err = mem.List(context.TODO(), restic.SnapshotFile, func(fi restic.FileInfo) error { + memFiles = append(memFiles, fi) + return nil + }) + rtest.OK(t, err) + rtest.Equals(t, files, memFiles) +} + +func TestMemoizeListError(t *testing.T) { + // setup backend to serve as data source for memoized list + be := mock.NewBackend() + be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + return fmt.Errorf("list error") + } + _, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile) + rtest.Assert(t, err != nil, "missing error on list error") +} diff --git a/internal/bloblru/cache.go b/internal/bloblru/cache.go index b524f870c11..dfd9b2fd17b 100644 --- a/internal/bloblru/cache.go +++ b/internal/bloblru/cache.go @@ -22,7 +22,7 @@ type Cache struct { free, size int // Current and max capacity, in bytes. } -// Construct a blob cache that stores at most size bytes worth of blobs. +// New constructs a blob cache that stores at most size bytes worth of blobs. func New(size int) *Cache { c := &Cache{ free: size, diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 83013603b6d..a43b2bbf2cc 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -59,11 +59,6 @@ func writeCachedirTag(dir string) error { } tagfile := filepath.Join(dir, "CACHEDIR.TAG") - _, err := fs.Lstat(tagfile) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return errors.WithStack(err) - } - f, err := fs.OpenFile(tagfile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, fileMode) if err != nil { if errors.Is(err, os.ErrExist) { diff --git a/internal/cache/file.go b/internal/cache/file.go index 0db1275a3a9..8ed4be77e17 100644 --- a/internal/cache/file.go +++ b/internal/cache/file.go @@ -59,7 +59,7 @@ func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser, return nil, errors.WithStack(err) } - if fi.Size() <= crypto.Extension { + if fi.Size() <= int64(crypto.CiphertextLength(0)) { _ = f.Close() _ = c.remove(h) return nil, errors.Errorf("cached file %v is truncated, removing", h) @@ -117,7 +117,7 @@ func (c *Cache) Save(h restic.Handle, rd io.Reader) error { return errors.Wrap(err, "Copy") } - if n <= crypto.Extension { + if n <= int64(crypto.CiphertextLength(0)) { _ = f.Close() _ = fs.Remove(f.Name()) debug.Log("trying to cache truncated file %v, removing", h) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index e842a08bea8..0e4310c950f 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -1,14 +1,23 @@ package checker import ( + "bufio" + "bytes" "context" "fmt" "io" - "os" + "io/ioutil" + "runtime" + "sort" "sync" + "github.com/minio/sha256-simd" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/s3" + "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -30,6 +39,7 @@ type Checker struct { trackUnused bool masterIndex *repository.MasterIndex + snapshots restic.Lister repo restic.Repository } @@ -48,7 +58,12 @@ func New(repo restic.Repository, trackUnused bool) *Checker { return c } -const defaultParallelism = 5 +// ErrLegacyLayout is returned when the repository uses the S3 legacy layout. +type ErrLegacyLayout struct{} + +func (e *ErrLegacyLayout) Error() string { + return "repository uses S3 legacy layout" +} // ErrDuplicatePacks is returned when a pack is found in more than one index. type ErrDuplicatePacks struct { @@ -56,8 +71,17 @@ type ErrDuplicatePacks struct { Indexes restic.IDSet } -func (e ErrDuplicatePacks) Error() string { - return fmt.Sprintf("pack %v contained in several indexes: %v", e.PackID.Str(), e.Indexes) +func (e *ErrDuplicatePacks) Error() string { + return fmt.Sprintf("pack %v contained in several indexes: %v", e.PackID, e.Indexes) +} + +// ErrMixedPack is returned when a pack is found that contains both tree and data blobs. +type ErrMixedPack struct { + PackID restic.ID +} + +func (e *ErrMixedPack) Error() string { + return fmt.Sprintf("pack %v contains a mix of tree and data blobs", e.PackID.Str()) } // ErrOldIndexFormat is returned when an index with the old format is @@ -66,8 +90,14 @@ type ErrOldIndexFormat struct { restic.ID } -func (err ErrOldIndexFormat) Error() string { - return fmt.Sprintf("index %v has old format", err.ID.Str()) +func (err *ErrOldIndexFormat) Error() string { + return fmt.Sprintf("index %v has old format", err.ID) +} + +func (c *Checker) LoadSnapshots(ctx context.Context) error { + var err error + c.snapshots, err = backend.MemorizeList(ctx, c.repo.Backend(), restic.SnapshotFile) + return err } // LoadIndex loads all index files. @@ -79,11 +109,11 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { debug.Log("process index %v, err %v", id, err) if oldFormat { - debug.Log("index %v has old format", id.Str()) - hints = append(hints, ErrOldIndexFormat{id}) + debug.Log("index %v has old format", id) + hints = append(hints, &ErrOldIndexFormat{id}) } - err = errors.Wrapf(err, "error loading index %v", id.Str()) + err = errors.Wrapf(err, "error loading index %v", id) if err != nil { errs = append(errs, err) @@ -118,17 +148,22 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { } // compute pack size using index entries - c.packs = c.masterIndex.PackSize(ctx, false) + c.packs = pack.Size(ctx, c.masterIndex, false) debug.Log("checking for duplicate packs") for packID := range c.packs { debug.Log(" check pack %v: contained in %d indexes", packID, len(packToIndex[packID])) if len(packToIndex[packID]) > 1 { - hints = append(hints, ErrDuplicatePacks{ + hints = append(hints, &ErrDuplicatePacks{ PackID: packID, Indexes: packToIndex[packID], }) } + if c.masterIndex.IsMixedPack(packID) { + hints = append(hints, &ErrMixedPack{ + PackID: packID, + }) + } } err = c.repo.SetIndex(c.masterIndex) @@ -147,18 +182,29 @@ type PackError struct { Err error } -func (e PackError) Error() string { - return "pack " + e.ID.Str() + ": " + e.Err.Error() +func (e *PackError) Error() string { + return "pack " + e.ID.String() + ": " + e.Err.Error() } // IsOrphanedPack returns true if the error describes a pack which is not // contained in any index. func IsOrphanedPack(err error) bool { - if e, ok := errors.Cause(err).(PackError); ok && e.Orphaned { - return true + var e *PackError + return errors.As(err, &e) && e.Orphaned +} + +func isS3Legacy(b restic.Backend) bool { + // unwrap cache + if be, ok := b.(*cache.Backend); ok { + b = be.Backend + } + + be, ok := b.(*s3.Backend) + if !ok { + return false } - return false + return be.Layout.Name() == "s3legacy" } // Packs checks that all packs referenced in the index are still available and @@ -167,6 +213,10 @@ func IsOrphanedPack(err error) bool { func (c *Checker) Packs(ctx context.Context, errChan chan<- error) { defer close(errChan) + if isS3Legacy(c.repo.Backend()) { + errChan <- &ErrLegacyLayout{} + } + debug.Log("checking for %d packs", len(c.packs)) debug.Log("listing repository packs") @@ -191,7 +241,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) { select { case <-ctx.Done(): return - case errChan <- PackError{ID: id, Err: errors.New("does not exist")}: + case errChan <- &PackError{ID: id, Err: errors.New("does not exist")}: } continue } @@ -201,7 +251,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) { select { case <-ctx.Done(): return - case errChan <- PackError{ID: id, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}: + case errChan <- &PackError{ID: id, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}: } } } @@ -211,7 +261,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) { select { case <-ctx.Done(): return - case errChan <- PackError{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}: + case errChan <- &PackError{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}: } } } @@ -219,20 +269,12 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) { // Error is an error that occurred while checking a repository. type Error struct { TreeID restic.ID - BlobID restic.ID Err error } func (e Error) Error() string { - if !e.BlobID.IsNull() && !e.TreeID.IsNull() { - msg := "tree " + e.TreeID.Str() - msg += ", blob " + e.BlobID.Str() - msg += ": " + e.Err.Error() - return msg - } - if !e.TreeID.IsNull() { - return "tree " + e.TreeID.Str() + ": " + e.Err.Error() + return "tree " + e.TreeID.String() + ": " + e.Err.Error() } return e.Err.Error() @@ -244,8 +286,8 @@ type TreeError struct { Errors []error } -func (e TreeError) Error() string { - return fmt.Sprintf("tree %v: %v", e.ID.Str(), e.Errors) +func (e *TreeError) Error() string { + return fmt.Sprintf("tree %v: %v", e.ID, e.Errors) } // checkTreeWorker checks the trees received and sends out errors to errChan. @@ -263,7 +305,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI if len(errs) == 0 { continue } - treeError := TreeError{ID: job.ID, Errors: errs} + treeError := &TreeError{ID: job.ID, Errors: errs} select { case <-ctx.Done(): return @@ -273,8 +315,8 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI } } -func loadSnapshotTreeIDs(ctx context.Context, repo restic.Repository) (ids restic.IDs, errs []error) { - err := restic.ForAllSnapshots(ctx, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { +func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.Repository) (ids restic.IDs, errs []error) { + err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { if err != nil { errs = append(errs, err) return nil @@ -295,7 +337,7 @@ func loadSnapshotTreeIDs(ctx context.Context, repo restic.Repository) (ids resti // subtrees are available in the index. errChan is closed after all trees have // been traversed. func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan chan<- error) { - trees, errs := loadSnapshotTreeIDs(ctx, c.repo) + trees, errs := loadSnapshotTreeIDs(ctx, c.snapshots, c.repo) p.SetMax(uint64(len(trees))) debug.Log("need to check %d trees from snapshots, %d errs returned", len(trees), len(errs)) @@ -320,7 +362,9 @@ func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan ch }, p) defer close(errChan) - for i := 0; i < defaultParallelism; i++ { + // The checkTree worker only processes already decoded trees and is thus CPU-bound + workerCount := runtime.GOMAXPROCS(0) + for i := 0; i < workerCount; i++ { wg.Go(func() error { c.checkTreeWorker(ctx, treeStream, errChan) return nil @@ -436,99 +480,120 @@ func (c *Checker) GetPacks() map[restic.ID]int64 { } // checkPack reads a pack and checks the integrity of all blobs. -func checkPack(ctx context.Context, r restic.Repository, id restic.ID, size int64) error { - debug.Log("checking pack %v", id) - h := restic.Handle{Type: restic.PackFile, Name: id.String()} +func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []restic.Blob, size int64, bufRd *bufio.Reader) error { + debug.Log("checking pack %v", id.String()) - packfile, hash, realSize, err := repository.DownloadAndHash(ctx, r.Backend(), h) - if err != nil { - return errors.Wrap(err, "checkPack") + if len(blobs) == 0 { + return errors.Errorf("pack %v is empty or not indexed", id) } - defer func() { - _ = packfile.Close() - _ = os.Remove(packfile.Name()) - }() - - debug.Log("hash for pack %v is %v", id, hash) - - if !hash.Equal(id) { - debug.Log("Pack ID does not match, want %v, got %v", id, hash) - return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str()) + // sanity check blobs in index + sort.Slice(blobs, func(i, j int) bool { + return blobs[i].Offset < blobs[j].Offset + }) + idxHdrSize := pack.CalculateHeaderSize(blobs) + lastBlobEnd := 0 + nonContinuousPack := false + for _, blob := range blobs { + if lastBlobEnd != int(blob.Offset) { + nonContinuousPack = true + } + lastBlobEnd = int(blob.Offset + blob.Length) } + // size was calculated by masterindex.PackSize, thus there's no need to recalculate it here - if realSize != size { - debug.Log("Pack size does not match, want %v, got %v", size, realSize) - return errors.Errorf("Pack size does not match, want %v, got %v", size, realSize) + var errs []error + if nonContinuousPack { + debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs) + errs = append(errs, errors.New("Index for pack contains gaps / overlapping blobs")) } - blobs, hdrSize, err := pack.List(r.Key(), packfile, size) - if err != nil { - return err - } + // calculate hash on-the-fly while reading the pack and capture pack header + var hash restic.ID + var hdrBuf []byte + hashingLoader := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error { + hrd := hashing.NewReader(rd, sha256.New()) + bufRd.Reset(hrd) + + // skip to start of first blob, offset == 0 for correct pack files + _, err := bufRd.Discard(int(offset)) + if err != nil { + return err + } - var errs []error - var buf []byte - sizeFromBlobs := uint(hdrSize) - idx := r.Index() - for i, blob := range blobs { - sizeFromBlobs += blob.Length - debug.Log(" check blob %d: %v", i, blob) + err = fn(bufRd) + if err != nil { + return err + } - buf = buf[:cap(buf)] - if uint(len(buf)) < blob.Length { - buf = make([]byte, blob.Length) - } - buf = buf[:blob.Length] + // skip enough bytes until we reach the possible header start + curPos := length + int(offset) + minHdrStart := int(size) - pack.MaxHeaderSize + if minHdrStart > curPos { + _, err := bufRd.Discard(minHdrStart - curPos) + if err != nil { + return err + } + } - _, err := packfile.Seek(int64(blob.Offset), 0) - if err != nil { - return errors.Errorf("Seek(%v): %v", blob.Offset, err) - } + // read remainder, which should be the pack header + hdrBuf, err = ioutil.ReadAll(bufRd) + if err != nil { + return err + } - _, err = io.ReadFull(packfile, buf) - if err != nil { - debug.Log(" error loading blob %v: %v", blob.ID, err) - errs = append(errs, errors.Errorf("blob %v: %v", i, err)) - continue - } + hash = restic.IDFromHash(hrd.Sum(nil)) + return nil + }) + } - nonce, ciphertext := buf[:r.Key().NonceSize()], buf[r.Key().NonceSize():] - plaintext, err := r.Key().Open(ciphertext[:0], nonce, ciphertext, nil) + err := repository.StreamPack(ctx, hashingLoader, r.Key(), id, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + debug.Log(" check blob %v: %v", blob.ID, blob) if err != nil { - debug.Log(" error decrypting blob %v: %v", blob.ID, err) - errs = append(errs, errors.Errorf("blob %v: %v", i, err)) - continue + debug.Log(" error verifying blob %v: %v", blob.ID, err) + errs = append(errs, errors.Errorf("blob %v: %v", blob.ID, err)) } + return nil + }) + if err != nil { + // failed to load the pack file, return as further checks cannot succeed anyways + debug.Log(" error streaming pack: %v", err) + return errors.Errorf("pack %v failed to download: %v", id, err) + } + if !hash.Equal(id) { + debug.Log("Pack ID does not match, want %v, got %v", id, hash) + return errors.Errorf("Pack ID does not match, want %v, got %v", id, hash) + } - hash := restic.Hash(plaintext) - if !hash.Equal(blob.ID) { - debug.Log(" Blob ID does not match, want %v, got %v", blob.ID, hash) - errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())) - continue - } + blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf))) + if err != nil { + return err + } + if uint32(idxHdrSize) != hdrSize { + debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize) + errs = append(errs, errors.Errorf("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)) + } + + idx := r.Index() + for _, blob := range blobs { // Check if blob is contained in index and position is correct idxHas := false for _, pb := range idx.Lookup(blob.BlobHandle) { - if pb.PackID == id && pb.Offset == blob.Offset && pb.Length == blob.Length { + if pb.PackID == id && pb.Blob == blob { idxHas = true break } } if !idxHas { - errs = append(errs, errors.Errorf("Blob %v is not contained in index or position is incorrect", blob.ID.Str())) + errs = append(errs, errors.Errorf("Blob %v is not contained in index or position is incorrect", blob.ID)) continue } } - if int64(sizeFromBlobs) != size { - debug.Log("Pack size does not match, want %v, got %v", size, sizeFromBlobs) - errs = append(errs, errors.Errorf("Pack size does not match, want %v, got %v", size, sizeFromBlobs)) - } - if len(errs) > 0 { - return errors.Errorf("pack %v contains %v errors: %v", id.Str(), len(errs), errs) + return errors.Errorf("pack %v contains %v errors: %v", id, len(errs), errs) } return nil @@ -544,17 +609,23 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p defer close(errChan) g, ctx := errgroup.WithContext(ctx) - type packsize struct { - id restic.ID - size int64 + type checkTask struct { + id restic.ID + size int64 + blobs []restic.Blob } - ch := make(chan packsize) + ch := make(chan checkTask) + // as packs are streamed the concurrency is limited by IO + workerCount := int(c.repo.Connections()) // run workers - for i := 0; i < defaultParallelism; i++ { + for i := 0; i < workerCount; i++ { g.Go(func() error { + // create a buffer that is large enough to be reused by repository.StreamPack + // this ensures that we can read the pack header later on + bufRd := bufio.NewReaderSize(nil, repository.MaxStreamBufferSize) for { - var ps packsize + var ps checkTask var ok bool select { @@ -565,7 +636,8 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p return nil } } - err := checkPack(ctx, c.repo, ps.id, ps.size) + + err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd) p.Add(1) if err == nil { continue @@ -580,10 +652,17 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p }) } + packSet := restic.NewIDSet() + for pack := range packs { + packSet.Insert(pack) + } + // push packs to ch - for pack, size := range packs { + for pbs := range c.repo.Index().ListPacks(ctx, packSet) { + size := packs[pbs.PackID] + debug.Log("listed %v", pbs.PackID) select { - case ch <- packsize{id: pack, size: size}: + case ch <- checkTask{id: pbs.PackID, size: size, blobs: pbs.Blobs}: case <-ctx.Done(): } } diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 6d17a459304..b3a73615284 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -20,6 +20,7 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" + "golang.org/x/sync/errgroup" ) var checkerTestData = filepath.Join("testdata", "checker-test-repo.tar.gz") @@ -44,6 +45,10 @@ func checkPacks(chkr *checker.Checker) []error { } func checkStruct(chkr *checker.Checker) []error { + err := chkr.LoadSnapshots(context.TODO()) + if err != nil { + return []error{err} + } return collectErrors(context.TODO(), func(ctx context.Context, errChan chan<- error) { chkr.Structure(ctx, nil, errChan) }) @@ -58,6 +63,14 @@ func checkData(chkr *checker.Checker) []error { ) } +func assertOnlyMixedPackHints(t *testing.T, hints []error) { + for _, err := range hints { + if _, ok := err.(*checker.ErrMixedPack); !ok { + t.Fatalf("expected mixed pack hint, got %v", err) + } + } +} + func TestCheckRepo(t *testing.T) { repodir, cleanup := test.Env(t, checkerTestData) defer cleanup() @@ -69,9 +82,9 @@ func TestCheckRepo(t *testing.T) { if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } - - if len(hints) > 0 { - t.Errorf("expected no hints, got %v: %v", len(hints), hints) + assertOnlyMixedPackHints(t, hints) + if len(hints) == 0 { + t.Fatal("expected mixed pack warnings, got none") } test.OKs(t, checkPacks(chkr)) @@ -95,17 +108,14 @@ func TestMissingPack(t *testing.T) { if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } - - if len(hints) > 0 { - t.Errorf("expected no hints, got %v: %v", len(hints), hints) - } + assertOnlyMixedPackHints(t, hints) errs = checkPacks(chkr) test.Assert(t, len(errs) == 1, "expected exactly one error, got %v", len(errs)) - if err, ok := errs[0].(checker.PackError); ok { + if err, ok := errs[0].(*checker.PackError); ok { test.Equals(t, packHandle.Name, err.ID.String()) } else { t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err) @@ -131,17 +141,14 @@ func TestUnreferencedPack(t *testing.T) { if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } - - if len(hints) > 0 { - t.Errorf("expected no hints, got %v: %v", len(hints), hints) - } + assertOnlyMixedPackHints(t, hints) errs = checkPacks(chkr) test.Assert(t, len(errs) == 1, "expected exactly one error, got %v", len(errs)) - if err, ok := errs[0].(checker.PackError); ok { + if err, ok := errs[0].(*checker.PackError); ok { test.Equals(t, packID, err.ID.String()) } else { t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err) @@ -176,10 +183,7 @@ func TestUnreferencedBlobs(t *testing.T) { if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } - - if len(hints) > 0 { - t.Errorf("expected no hints, got %v: %v", len(hints), hints) - } + assertOnlyMixedPackHints(t, hints) test.OKs(t, checkPacks(chkr)) test.OKs(t, checkStruct(chkr)) @@ -264,9 +268,7 @@ func TestModifiedIndex(t *testing.T) { t.Logf("found expected error %v", err) } - if len(hints) > 0 { - t.Errorf("expected no hints, got %v: %v", len(hints), hints) - } + assertOnlyMixedPackHints(t, hints) } var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz") @@ -285,7 +287,7 @@ func TestDuplicatePacksInIndex(t *testing.T) { found := false for _, hint := range hints { - if _, ok := hint.(checker.ErrDuplicatePacks); ok { + if _, ok := hint.(*checker.ErrDuplicatePacks); ok { found = true } else { t.Errorf("got unexpected hint: %v", hint) @@ -346,7 +348,8 @@ func TestCheckerModifiedData(t *testing.T) { t.Logf("archived as %v", sn.ID().Str()) beError := &errorBackend{Backend: repo.Backend()} - checkRepo := repository.New(beError) + checkRepo, err := repository.New(beError, repository.Options{}) + test.OK(t, err) test.OK(t, checkRepo.SearchKey(context.TODO(), test.TestPassword, 5, "")) chkr := checker.New(checkRepo, false) @@ -398,7 +401,7 @@ func (r *loadTreesOnceRepository) LoadTree(ctx context.Context, id restic.ID) (* return nil, errors.Errorf("trying to load tree with id %v twice", id) } r.loadedTrees.Insert(id) - return r.Repository.LoadTree(ctx, id) + return restic.LoadTree(ctx, r.Repository, id) } func TestCheckerNoDuplicateTreeDecodes(t *testing.T) { @@ -416,10 +419,7 @@ func TestCheckerNoDuplicateTreeDecodes(t *testing.T) { if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } - - if len(hints) > 0 { - t.Errorf("expected no hints, got %v: %v", len(hints), hints) - } + assertOnlyMixedPackHints(t, hints) test.OKs(t, checkPacks(chkr)) test.OKs(t, checkStruct(chkr)) @@ -438,7 +438,7 @@ func (r *delayRepository) LoadTree(ctx context.Context, id restic.ID) (*restic.T if id == r.DelayTree { <-r.UnblockChannel } - return r.Repository.LoadTree(ctx, id) + return restic.LoadTree(ctx, r.Repository, id) } func (r *delayRepository) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, bool) { @@ -472,14 +472,18 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { Nodes: []*restic.Node{damagedNode}, } - id, err := repo.SaveTree(ctx, damagedTree) + wg, wgCtx := errgroup.WithContext(ctx) + repo.StartPackUploader(wgCtx, wg) + id, err := restic.SaveTree(ctx, repo, damagedTree) test.OK(t, repo.Flush(ctx)) test.OK(t, err) buf, err := repo.LoadBlob(ctx, restic.TreeBlob, id, nil) test.OK(t, err) - _, _, err = repo.SaveBlob(ctx, restic.DataBlob, buf, id, false) + wg, wgCtx = errgroup.WithContext(ctx) + repo.StartPackUploader(wgCtx, wg) + _, _, _, err = repo.SaveBlob(ctx, restic.DataBlob, buf, id, false) test.OK(t, err) malNode := &restic.Node{ @@ -500,18 +504,17 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { Nodes: []*restic.Node{malNode, dirNode}, } - rootID, err := repo.SaveTree(ctx, rootTree) + rootID, err := restic.SaveTree(ctx, repo, rootTree) test.OK(t, err) test.OK(t, repo.Flush(ctx)) - test.OK(t, repo.SaveIndex(ctx)) snapshot, err := restic.NewSnapshot([]string{"/damaged"}, []string{"test"}, "foo", time.Now()) test.OK(t, err) snapshot.Tree = &rootID - snapID, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, snapshot) + snapID, err := restic.SaveSnapshot(ctx, repo, snapshot) test.OK(t, err) t.Logf("saved snapshot %v", snapID.Str()) @@ -564,8 +567,10 @@ func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, fun t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } - if len(hints) > 0 { - t.Errorf("expected no hints, got %v: %v", len(hints), hints) + for _, err := range hints { + if _, ok := err.(*checker.ErrMixedPack); !ok { + t.Fatalf("expected mixed pack hint, got %v", err) + } } return chkr, repo, cleanup } @@ -587,13 +592,12 @@ func benchmarkSnapshotScaling(t *testing.B, newSnapshots int) { chkr, repo, cleanup := loadBenchRepository(t) defer cleanup() - snID, err := restic.FindSnapshot(context.TODO(), repo, "51d249d2") + snID, err := restic.FindSnapshot(context.TODO(), repo.Backend(), "51d249d2") if err != nil { t.Fatal(err) } - var sn2 restic.Snapshot - err = repo.LoadJSONUnpacked(context.TODO(), restic.SnapshotFile, snID, &sn2) + sn2, err := restic.LoadSnapshot(context.TODO(), repo, snID) if err != nil { t.Fatal(err) } @@ -607,7 +611,7 @@ func benchmarkSnapshotScaling(t *testing.B, newSnapshots int) { } sn.Tree = treeID - _, err = repo.SaveJSONUnpacked(context.TODO(), restic.SnapshotFile, sn) + _, err = restic.SaveSnapshot(context.TODO(), repo, sn) if err != nil { t.Fatal(err) } diff --git a/internal/checker/testing.go b/internal/checker/testing.go index d672911b1b5..0668406d83b 100644 --- a/internal/checker/testing.go +++ b/internal/checker/testing.go @@ -20,6 +20,11 @@ func TestCheckRepo(t testing.TB, repo restic.Repository) { t.Fatalf("errors loading index: %v", hints) } + err := chkr.LoadSnapshots(context.TODO()) + if err != nil { + t.Error(err) + } + // packs errChan := make(chan error) go chkr.Packs(context.TODO(), errChan) diff --git a/internal/restic/buffer.go b/internal/crypto/buffer.go similarity index 69% rename from internal/restic/buffer.go rename to internal/crypto/buffer.go index 899f4ea6f88..b098d5c72c9 100644 --- a/internal/restic/buffer.go +++ b/internal/crypto/buffer.go @@ -1,21 +1,19 @@ -package restic - -import "github.com/restic/restic/internal/crypto" +package crypto // NewBlobBuffer returns a buffer that is large enough to hold a blob of size // plaintext bytes, including the crypto overhead. func NewBlobBuffer(size int) []byte { - return make([]byte, size, size+crypto.Extension) + return make([]byte, size, size+Extension) } // PlaintextLength returns the plaintext length of a blob with ciphertextSize // bytes. func PlaintextLength(ciphertextSize int) int { - return ciphertextSize - crypto.Extension + return ciphertextSize - Extension } // CiphertextLength returns the encrypted length of a blob with plaintextSize // bytes. func CiphertextLength(plaintextSize int) int { - return plaintextSize + crypto.Extension + return plaintextSize + Extension } diff --git a/internal/debug/debug.go b/internal/debug/debug.go index 9cfeed1e627..62c145e1a03 100644 --- a/internal/debug/debug.go +++ b/internal/debug/debug.go @@ -1,5 +1,3 @@ -// +build debug - package debug import ( diff --git a/internal/debug/debug_release.go b/internal/debug/debug_release.go deleted file mode 100644 index 9b4259cea57..00000000000 --- a/internal/debug/debug_release.go +++ /dev/null @@ -1,6 +0,0 @@ -// +build !debug - -package debug - -// Log prints a message to the debug log (if debug is enabled). -func Log(fmt string, args ...interface{}) {} diff --git a/internal/debug/hooks.go b/internal/debug/hooks.go deleted file mode 100644 index e47084fee7f..00000000000 --- a/internal/debug/hooks.go +++ /dev/null @@ -1,28 +0,0 @@ -// +build debug - -package debug - -var ( - hooks map[string]func(interface{}) -) - -func init() { - hooks = make(map[string]func(interface{})) -} - -func Hook(name string, f func(interface{})) { - hooks[name] = f -} - -func RunHook(name string, context interface{}) { - f, ok := hooks[name] - if !ok { - return - } - - f(context) -} - -func RemoveHook(name string) { - delete(hooks, name) -} diff --git a/internal/debug/hooks_release.go b/internal/debug/hooks_release.go deleted file mode 100644 index 86efa9f64c2..00000000000 --- a/internal/debug/hooks_release.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build !debug - -package debug - -func Hook(name string, f func(interface{})) {} - -func RunHook(name string, context interface{}) {} - -func RemoveHook(name string) {} diff --git a/internal/debug/round_tripper.go b/internal/debug/round_tripper.go new file mode 100644 index 00000000000..6795d43d00a --- /dev/null +++ b/internal/debug/round_tripper.go @@ -0,0 +1,116 @@ +package debug + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httputil" + "os" + + "github.com/restic/restic/internal/errors" +) + +type eofDetectRoundTripper struct { + http.RoundTripper +} + +type eofDetectReader struct { + eofSeen bool + rd io.ReadCloser +} + +func (rd *eofDetectReader) Read(p []byte) (n int, err error) { + n, err = rd.rd.Read(p) + if err == io.EOF { + rd.eofSeen = true + } + return n, err +} + +func (rd *eofDetectReader) Close() error { + if !rd.eofSeen { + buf, err := ioutil.ReadAll(rd) + msg := fmt.Sprintf("body not drained, %d bytes not read", len(buf)) + if err != nil { + msg += fmt.Sprintf(", error: %v", err) + } + + if len(buf) > 0 { + if len(buf) > 20 { + buf = append(buf[:20], []byte("...")...) + } + msg += fmt.Sprintf(", body: %q", buf) + } + + fmt.Fprintln(os.Stderr, msg) + Log("%s: %+v", msg, errors.New("Close()")) + } + return rd.rd.Close() +} + +func (tr eofDetectRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + res, err = tr.RoundTripper.RoundTrip(req) + if res != nil && res.Body != nil { + res.Body = &eofDetectReader{rd: res.Body} + } + return res, err +} + +type loggingRoundTripper struct { + http.RoundTripper +} + +func redactHeader(header http.Header) map[string][]string { + removedHeaders := make(map[string][]string) + for _, hdr := range []string{ + "Authorization", + "X-Auth-Token", // Swift headers + "X-Auth-Key", + } { + origHeader, hasHeader := header[hdr] + if hasHeader { + removedHeaders[hdr] = origHeader + header[hdr] = []string{"**redacted**"} + } + } + return removedHeaders +} + +func restoreHeader(header http.Header, origHeaders map[string][]string) { + for hdr, val := range origHeaders { + header[hdr] = val + } +} + +func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + // save original auth and redact it + origHeaders := redactHeader(req.Header) + + trace, err := httputil.DumpRequestOut(req, false) + if err != nil { + Log("DumpRequestOut() error: %v\n", err) + } else { + Log("------------ HTTP REQUEST -----------\n%s", trace) + } + + restoreHeader(req.Header, origHeaders) + + res, err = tr.RoundTripper.RoundTrip(req) + if err != nil { + Log("RoundTrip() returned error: %v", err) + } + + if res != nil { + origHeaders := redactHeader(res.Header) + trace, err := httputil.DumpResponse(res, false) + restoreHeader(res.Header, origHeaders) + if err != nil { + Log("DumpResponse() error: %v\n", err) + } else { + Log("------------ HTTP RESPONSE ----------\n%s", trace) + } + } + + return res, err +} diff --git a/internal/debug/round_tripper_debug.go b/internal/debug/round_tripper_debug.go index 5dfbb64c646..df207207bd7 100644 --- a/internal/debug/round_tripper_debug.go +++ b/internal/debug/round_tripper_debug.go @@ -1,67 +1,9 @@ +//go:build debug // +build debug package debug -import ( - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/httputil" - "os" - - "github.com/restic/restic/internal/errors" -) - -type eofDetectRoundTripper struct { - http.RoundTripper -} - -type eofDetectReader struct { - eofSeen bool - rd io.ReadCloser -} - -func (rd *eofDetectReader) Read(p []byte) (n int, err error) { - n, err = rd.rd.Read(p) - if err == io.EOF { - rd.eofSeen = true - } - return n, err -} - -func (rd *eofDetectReader) Close() error { - if !rd.eofSeen { - buf, err := ioutil.ReadAll(rd) - msg := fmt.Sprintf("body not drained, %d bytes not read", len(buf)) - if err != nil { - msg += fmt.Sprintf(", error: %v", err) - } - - if len(buf) > 0 { - if len(buf) > 20 { - buf = append(buf[:20], []byte("...")...) - } - msg += fmt.Sprintf(", body: %q", buf) - } - - fmt.Fprintln(os.Stderr, msg) - Log("%s: %+v", msg, errors.New("Close()")) - } - return rd.rd.Close() -} - -func (tr eofDetectRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { - res, err = tr.RoundTripper.RoundTrip(req) - if res != nil && res.Body != nil { - res.Body = &eofDetectReader{rd: res.Body} - } - return res, err -} - -type loggingRoundTripper struct { - http.RoundTripper -} +import "net/http" // RoundTripper returns a new http.RoundTripper which logs all requests (if // debug is enabled). When debug is not enabled, upstream is returned. @@ -73,28 +15,3 @@ func RoundTripper(upstream http.RoundTripper) http.RoundTripper { } return eofRoundTripper } - -func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { - trace, err := httputil.DumpRequestOut(req, false) - if err != nil { - Log("DumpRequestOut() error: %v\n", err) - } else { - Log("------------ HTTP REQUEST -----------\n%s", trace) - } - - res, err = tr.RoundTripper.RoundTrip(req) - if err != nil { - Log("RoundTrip() returned error: %v", err) - } - - if res != nil { - trace, err := httputil.DumpResponse(res, false) - if err != nil { - Log("DumpResponse() error: %v\n", err) - } else { - Log("------------ HTTP RESPONSE ----------\n%s", trace) - } - } - - return res, err -} diff --git a/internal/debug/round_tripper_release.go b/internal/debug/round_tripper_release.go index 6efff2c2826..6edadb4798e 100644 --- a/internal/debug/round_tripper_release.go +++ b/internal/debug/round_tripper_release.go @@ -1,3 +1,4 @@ +//go:build !debug // +build !debug package debug @@ -7,5 +8,9 @@ import "net/http" // RoundTripper returns a new http.RoundTripper which logs all requests (if // debug is enabled). When debug is not enabled, upstream is returned. func RoundTripper(upstream http.RoundTripper) http.RoundTripper { + if opts.isEnabled { + // only use loggingRoundTripper if the debug log is configured + return loggingRoundTripper{eofDetectRoundTripper{upstream}} + } return upstream } diff --git a/internal/debug/round_tripper_test.go b/internal/debug/round_tripper_test.go new file mode 100644 index 00000000000..cc42a87d159 --- /dev/null +++ b/internal/debug/round_tripper_test.go @@ -0,0 +1,44 @@ +package debug + +import ( + "net/http" + "testing" + + "github.com/restic/restic/internal/test" +) + +func TestRedactHeader(t *testing.T) { + secretHeaders := []string{ + "Authorization", + "X-Auth-Token", + "X-Auth-Key", + } + + header := make(http.Header) + header["Authorization"] = []string{"123"} + header["X-Auth-Token"] = []string{"1234"} + header["X-Auth-Key"] = []string{"12345"} + header["Host"] = []string{"my.host"} + + origHeaders := redactHeader(header) + + for _, hdr := range secretHeaders { + test.Equals(t, "**redacted**", header[hdr][0]) + } + test.Equals(t, "my.host", header["Host"][0]) + + restoreHeader(header, origHeaders) + test.Equals(t, "123", header["Authorization"][0]) + test.Equals(t, "1234", header["X-Auth-Token"][0]) + test.Equals(t, "12345", header["X-Auth-Key"][0]) + test.Equals(t, "my.host", header["Host"][0]) + + delete(header, "X-Auth-Key") + origHeaders = redactHeader(header) + _, hasHeader := header["X-Auth-Key"] + test.Assert(t, !hasHeader, "Unexpected header: %v", header["X-Auth-Key"]) + + restoreHeader(header, origHeaders) + _, hasHeader = header["X-Auth-Key"] + test.Assert(t, !hasHeader, "Unexpected header: %v", header["X-Auth-Key"]) +} diff --git a/internal/dump/common_test.go b/internal/dump/common_test.go index 22d05975193..7892a4fa992 100644 --- a/internal/dump/common_test.go +++ b/internal/dump/common_test.go @@ -88,7 +88,7 @@ func WriteTest(t *testing.T, format string, cd CheckDump) { sn, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{}) rtest.OK(t, err) - tree, err := repo.LoadTree(ctx, *sn.Tree) + tree, err := restic.LoadTree(ctx, repo, *sn.Tree) rtest.OK(t, err) dst := &bytes.Buffer{} diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 09ca656b3e1..021da72c554 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -52,4 +52,6 @@ func Cause(err error) error { // Go 1.13-style error handling. +func As(err error, tgt interface{}) bool { return errors.As(err, tgt) } + func Is(x, y error) bool { return errors.Is(x, y) } diff --git a/internal/errors/fatal.go b/internal/errors/fatal.go index 02ffdaab47c..5fb615cf1ff 100644 --- a/internal/errors/fatal.go +++ b/internal/errors/fatal.go @@ -23,6 +23,8 @@ type Fataler interface { // IsFatal returns true if err is a fatal message that should be printed to the // user. Then, the program should exit. func IsFatal(err error) bool { + // unwrap "Wrap" method + err = Cause(err) e, ok := err.(Fataler) return ok && e.Fatal() } diff --git a/internal/errors/fatal_test.go b/internal/errors/fatal_test.go new file mode 100644 index 00000000000..41da8dee73f --- /dev/null +++ b/internal/errors/fatal_test.go @@ -0,0 +1,22 @@ +package errors_test + +import ( + "testing" + + "github.com/restic/restic/internal/errors" +) + +func TestFatal(t *testing.T) { + for _, v := range []struct { + err error + expected bool + }{ + {errors.Fatal("broken"), true}, + {errors.Fatalf("broken %d", 42), true}, + {errors.New("error"), false}, + } { + if errors.IsFatal(v.err) != v.expected { + t.Fatalf("IsFatal for %q, expected: %v, got: %v", v.err, v.expected, errors.IsFatal(v.err)) + } + } +} diff --git a/internal/filter/filter.go b/internal/filter/filter.go index 3e16c93162f..cfacb8cc5a3 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -18,6 +18,7 @@ type patternPart struct { // Pattern represents a preparsed filter pattern type Pattern struct { + original string parts []patternPart isNegated bool } @@ -31,6 +32,9 @@ func prepareStr(str string) ([]string, error) { func preparePattern(patternStr string) Pattern { var negate bool + + originalPattern := patternStr + if patternStr[0] == '!' { negate = true patternStr = patternStr[1:] @@ -48,7 +52,7 @@ func preparePattern(patternStr string) Pattern { parts[i] = patternPart{part, isSimple} } - return Pattern{parts, negate} + return Pattern{originalPattern, parts, negate} } // Split p into path components. Assuming p has been Cleaned, no component @@ -130,7 +134,7 @@ func childMatch(pattern Pattern, strs []string) (matched bool, err error) { } else { l = len(strs) } - return match(Pattern{pattern.parts[0:l], pattern.isNegated}, strs) + return match(Pattern{pattern.original, pattern.parts[0:l], pattern.isNegated}, strs) } func hasDoubleWildcard(list Pattern) (ok bool, pos int) { @@ -158,7 +162,7 @@ func match(pattern Pattern, strs []string) (matched bool, err error) { } newPat = append(newPat, pattern.parts[pos+1:]...) - matched, err := match(Pattern{newPat, pattern.isNegated}, strs) + matched, err := match(Pattern{pattern.original, newPat, pattern.isNegated}, strs) if err != nil { return false, err } @@ -216,6 +220,27 @@ func match(pattern Pattern, strs []string) (matched bool, err error) { return false, nil } +// ValidatePatterns validates a slice of patterns. +// Returns true if all patterns are valid - false otherwise, along with the invalid patterns. +func ValidatePatterns(patterns []string) (allValid bool, invalidPatterns []string) { + invalidPatterns = make([]string, 0) + + for _, Pattern := range ParsePatterns(patterns) { + // Validate all pattern parts + for _, part := range Pattern.parts { + // Validate the pattern part by trying to match it against itself + if _, validErr := filepath.Match(part.pattern, part.pattern); validErr != nil { + invalidPatterns = append(invalidPatterns, Pattern.original) + + // If a single part is invalid, stop processing this pattern + continue + } + } + } + + return len(invalidPatterns) == 0, invalidPatterns +} + // ParsePatterns prepares a list of patterns for use with List. func ParsePatterns(pattern []string) []Pattern { patpat := make([]Pattern, 0) diff --git a/internal/filter/filter_patterns_test.go b/internal/filter/filter_patterns_test.go new file mode 100644 index 00000000000..215471500cc --- /dev/null +++ b/internal/filter/filter_patterns_test.go @@ -0,0 +1,57 @@ +//go:build go1.16 +// +build go1.16 + +// Before Go 1.16 filepath.Match returned early on a failed match, +// and thus did not report any later syntax error in the pattern. +// https://go.dev/doc/go1.16#path/filepath + +package filter_test + +import ( + "strings" + "testing" + + "github.com/restic/restic/internal/filter" + rtest "github.com/restic/restic/internal/test" +) + +func TestValidPatterns(t *testing.T) { + // Test invalid patterns are detected and returned + t.Run("detect-invalid-patterns", func(t *testing.T) { + allValid, invalidPatterns := filter.ValidatePatterns([]string{"*.foo", "*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}) + + rtest.Assert(t, allValid == false, "Expected invalid patterns to be detected") + + rtest.Equals(t, invalidPatterns, []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}) + }) + + // Test all patterns defined in matchTests are valid + patterns := make([]string, 0) + + for _, data := range matchTests { + patterns = append(patterns, data.pattern) + } + + t.Run("validate-patterns", func(t *testing.T) { + allValid, invalidPatterns := filter.ValidatePatterns(patterns) + + if !allValid { + t.Errorf("Found invalid pattern(s):\n%s", strings.Join(invalidPatterns, "\n")) + } + }) + + // Test all patterns defined in childMatchTests are valid + childPatterns := make([]string, 0) + + for _, data := range childMatchTests { + childPatterns = append(childPatterns, data.pattern) + } + + t.Run("validate-child-patterns", func(t *testing.T) { + allValid, invalidPatterns := filter.ValidatePatterns(childPatterns) + + if !allValid { + t.Errorf("Found invalid child pattern(s):\n%s", strings.Join(invalidPatterns, "\n")) + } + }) +} diff --git a/internal/fs/const_unix.go b/internal/fs/const_unix.go index a90d171b1a5..fe84cda176d 100644 --- a/internal/fs/const_unix.go +++ b/internal/fs/const_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package fs diff --git a/internal/fs/const_windows.go b/internal/fs/const_windows.go index 18c89c27e11..f1b263a54a4 100644 --- a/internal/fs/const_windows.go +++ b/internal/fs/const_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package fs diff --git a/internal/fs/deviceid_unix.go b/internal/fs/deviceid_unix.go index 31efd29ffc9..c366221ab29 100644 --- a/internal/fs/deviceid_unix.go +++ b/internal/fs/deviceid_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package fs diff --git a/internal/fs/deviceid_windows.go b/internal/fs/deviceid_windows.go index 4e2f2f9de53..42355817d5f 100644 --- a/internal/fs/deviceid_windows.go +++ b/internal/fs/deviceid_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package fs diff --git a/internal/fs/file_unix.go b/internal/fs/file_unix.go index f5ea36696f8..3edc60be625 100644 --- a/internal/fs/file_unix.go +++ b/internal/fs/file_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package fs diff --git a/internal/fs/file_windows.go b/internal/fs/file_windows.go index 8a4d01fb065..56effdc18ca 100644 --- a/internal/fs/file_windows.go +++ b/internal/fs/file_windows.go @@ -1,10 +1,14 @@ package fs import ( - "io/ioutil" + "math/rand" "os" "path/filepath" + "strconv" "strings" + "time" + + "golang.org/x/sys/windows" ) // fixpath returns an absolute path on windows, so restic can open long file @@ -30,9 +34,43 @@ func fixpath(name string) string { return name } -// TempFile creates a temporary file. +// TempFile creates a temporary file which is marked as delete-on-close func TempFile(dir, prefix string) (f *os.File, err error) { - return ioutil.TempFile(dir, prefix) + // slightly modified implementation of ioutil.TempFile(dir, prefix) to allow us to add + // the FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE flags. + // These provide two large benefits: + // FILE_ATTRIBUTE_TEMPORARY tells Windows to keep the file in memory only if possible + // which reduces the amount of unnecessary disk writes. + // FILE_FLAG_DELETE_ON_CLOSE instructs Windows to automatically delete the file once + // all file descriptors are closed. + + if dir == "" { + dir = os.TempDir() + } + + access := uint32(windows.GENERIC_READ | windows.GENERIC_WRITE) + creation := uint32(windows.CREATE_NEW) + share := uint32(0) // prevent other processes from accessing the file + flags := uint32(windows.FILE_ATTRIBUTE_TEMPORARY | windows.FILE_FLAG_DELETE_ON_CLOSE) + + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + for i := 0; i < 10000; i++ { + randSuffix := strconv.Itoa(int(1e9 + rnd.Intn(1e9)%1e9))[1:] + path := filepath.Join(dir, prefix+randSuffix) + + ptr, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + h, err := windows.CreateFile(ptr, access, share, nil, creation, flags, 0) + if os.IsExist(err) { + continue + } + return os.NewFile(uintptr(h), path), err + } + + // Proper error handling is still to do + return nil, os.ErrExist } // Chmod changes the mode of the named file to mode. diff --git a/internal/fs/file_windows_test.go b/internal/fs/file_windows_test.go new file mode 100644 index 00000000000..71077709b22 --- /dev/null +++ b/internal/fs/file_windows_test.go @@ -0,0 +1,35 @@ +package fs_test + +import ( + "errors" + "os" + "testing" + + "github.com/restic/restic/internal/fs" + rtest "github.com/restic/restic/internal/test" +) + +func TestTempFile(t *testing.T) { + // create two temp files at the same time to check that the + // collision avoidance works + f, err := fs.TempFile("", "test") + fn := f.Name() + rtest.OK(t, err) + f2, err := fs.TempFile("", "test") + fn2 := f2.Name() + rtest.OK(t, err) + rtest.Assert(t, fn != fn2, "filenames don't differ %s", fn) + + _, err = os.Stat(fn) + rtest.OK(t, err) + _, err = os.Stat(fn2) + rtest.OK(t, err) + + rtest.OK(t, f.Close()) + rtest.OK(t, f2.Close()) + + _, err = os.Stat(fn) + rtest.Assert(t, errors.Is(err, os.ErrNotExist), "err %s", err) + _, err = os.Stat(fn2) + rtest.Assert(t, errors.Is(err, os.ErrNotExist), "err %s", err) +} diff --git a/internal/fs/fs_local_vss.go b/internal/fs/fs_local_vss.go index b3e08fed9f5..aa3522aea5b 100644 --- a/internal/fs/fs_local_vss.go +++ b/internal/fs/fs_local_vss.go @@ -117,7 +117,7 @@ func (fs *LocalVss) snapshotPath(path string) string { fs.msgMessage("creating VSS snapshot for [%s]\n", vssVolume) if snapshot, err := NewVssSnapshot(vssVolume, 120, fs.msgError); err != nil { - _ = fs.msgError(vssVolume, errors.Errorf("failed to create snapshot for [%s]: %s\n", + _ = fs.msgError(vssVolume, errors.Errorf("failed to create snapshot for [%s]: %s", vssVolume, err)) fs.failedSnapshots[volumeNameLower] = struct{}{} } else { diff --git a/internal/fs/stat_bsd.go b/internal/fs/stat_bsd.go index d5e8ce55007..33a7879f4d5 100644 --- a/internal/fs/stat_bsd.go +++ b/internal/fs/stat_bsd.go @@ -1,3 +1,4 @@ +//go:build freebsd || darwin || netbsd // +build freebsd darwin netbsd package fs diff --git a/internal/fs/stat_unix.go b/internal/fs/stat_unix.go index 34b98a31e8c..bf0d5cecae9 100644 --- a/internal/fs/stat_unix.go +++ b/internal/fs/stat_unix.go @@ -1,3 +1,4 @@ +//go:build !windows && !darwin && !freebsd && !netbsd // +build !windows,!darwin,!freebsd,!netbsd package fs diff --git a/internal/fs/stat_windows.go b/internal/fs/stat_windows.go index a8f13cceaa0..ee678d92ac6 100644 --- a/internal/fs/stat_windows.go +++ b/internal/fs/stat_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package fs diff --git a/internal/fs/vss.go b/internal/fs/vss.go index ca0604906d4..9995f2d3edc 100644 --- a/internal/fs/vss.go +++ b/internal/fs/vss.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package fs diff --git a/internal/fs/vss_windows.go b/internal/fs/vss_windows.go index 8cee09fb1f2..bd82f440598 100644 --- a/internal/fs/vss_windows.go +++ b/internal/fs/vss_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package fs diff --git a/internal/fuse/dir.go b/internal/fuse/dir.go index c14e28d27c2..dcacaa96ac1 100644 --- a/internal/fuse/dir.go +++ b/internal/fuse/dir.go @@ -1,3 +1,4 @@ +//go:build darwin || freebsd || linux // +build darwin freebsd linux package fuse @@ -54,7 +55,7 @@ func replaceSpecialNodes(ctx context.Context, repo restic.Repository, node *rest return []*restic.Node{node}, nil } - tree, err := repo.LoadTree(ctx, *node.Subtree) + tree, err := restic.LoadTree(ctx, repo, *node.Subtree) if err != nil { return nil, err } @@ -87,7 +88,7 @@ func (d *dir) open(ctx context.Context) error { debug.Log("open dir %v (%v)", d.node.Name, d.node.Subtree) - tree, err := d.root.repo.LoadTree(ctx, *d.node.Subtree) + tree, err := restic.LoadTree(ctx, d.root.repo, *d.node.Subtree) if err != nil { debug.Log(" error loading tree %v: %v", d.node.Subtree, err) return err diff --git a/internal/fuse/file.go b/internal/fuse/file.go index 2de2660e39a..571d5a865e5 100644 --- a/internal/fuse/file.go +++ b/internal/fuse/file.go @@ -1,3 +1,4 @@ +//go:build darwin || freebsd || linux // +build darwin freebsd linux package fuse diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index 690df770ad6..df24f77afed 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -59,7 +59,7 @@ func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot { } func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree { - tree, err := repo.LoadTree(context.TODO(), id) + tree, err := restic.LoadTree(context.TODO(), repo, id) rtest.OK(t, err) return tree } diff --git a/internal/fuse/meta_dir.go b/internal/fuse/meta_dir.go deleted file mode 100644 index b3644fca115..00000000000 --- a/internal/fuse/meta_dir.go +++ /dev/null @@ -1,84 +0,0 @@ -// +build darwin freebsd linux - -package fuse - -import ( - "context" - "os" - - "github.com/restic/restic/internal/debug" - - "bazil.org/fuse" - "bazil.org/fuse/fs" -) - -// ensure that *DirSnapshots implements these interfaces -var _ = fs.HandleReadDirAller(&MetaDir{}) -var _ = fs.NodeStringLookuper(&MetaDir{}) - -// MetaDir is a fuse directory which contains other directories. -type MetaDir struct { - inode uint64 - root *Root - entries map[string]fs.Node -} - -// NewMetaDir returns a new meta dir. -func NewMetaDir(root *Root, inode uint64, entries map[string]fs.Node) *MetaDir { - debug.Log("new meta dir with %d entries, inode %d", len(entries), inode) - - return &MetaDir{ - root: root, - inode: inode, - entries: entries, - } -} - -// Attr returns the attributes for the root node. -func (d *MetaDir) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Inode = d.inode - attr.Mode = os.ModeDir | 0555 - attr.Uid = d.root.uid - attr.Gid = d.root.gid - - debug.Log("attr: %v", attr) - return nil -} - -// ReadDirAll returns all entries of the root node. -func (d *MetaDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - debug.Log("ReadDirAll()") - items := []fuse.Dirent{ - { - Inode: d.inode, - Name: ".", - Type: fuse.DT_Dir, - }, - { - Inode: d.root.inode, - Name: "..", - Type: fuse.DT_Dir, - }, - } - - for name := range d.entries { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, name), - Name: name, - Type: fuse.DT_Dir, - }) - } - - return items, nil -} - -// Lookup returns a specific entry from the root node. -func (d *MetaDir) Lookup(ctx context.Context, name string) (fs.Node, error) { - debug.Log("Lookup(%s)", name) - - if dir, ok := d.entries[name]; ok { - return dir, nil - } - - return nil, fuse.ENOENT -} diff --git a/internal/fuse/other.go b/internal/fuse/other.go index 575f0257fba..f7745172bcc 100644 --- a/internal/fuse/other.go +++ b/internal/fuse/other.go @@ -1,3 +1,4 @@ +//go:build darwin || freebsd || linux // +build darwin freebsd linux package fuse diff --git a/internal/fuse/root.go b/internal/fuse/root.go index bed760f02d7..63ab96d6d49 100644 --- a/internal/fuse/root.go +++ b/internal/fuse/root.go @@ -5,7 +5,6 @@ package fuse import ( "os" - "time" "github.com/restic/restic/internal/bloblru" "github.com/restic/restic/internal/debug" @@ -16,25 +15,21 @@ import ( // Config holds settings for the fuse mount. type Config struct { - OwnerIsRoot bool - Hosts []string - Tags []restic.TagList - Paths []string - SnapshotTemplate string + OwnerIsRoot bool + Hosts []string + Tags []restic.TagList + Paths []string + TimeTemplate string + PathTemplates []string } // Root is the root node of the fuse mount of a repository. type Root struct { repo restic.Repository cfg Config - inode uint64 - snapshots restic.Snapshots blobCache *bloblru.Cache - snCount int - lastCheck time.Time - - *MetaDir + *SnapshotsDir uid, gid uint32 } @@ -54,7 +49,6 @@ func NewRoot(repo restic.Repository, cfg Config) *Root { root := &Root{ repo: repo, - inode: rootInode, cfg: cfg, blobCache: bloblru.New(blobCacheSize), } @@ -64,14 +58,17 @@ func NewRoot(repo restic.Repository, cfg Config) *Root { root.gid = uint32(os.Getgid()) } - entries := map[string]fs.Node{ - "snapshots": NewSnapshotsDir(root, fs.GenerateDynamicInode(root.inode, "snapshots"), "", ""), - "tags": NewTagsDir(root, fs.GenerateDynamicInode(root.inode, "tags")), - "hosts": NewHostsDir(root, fs.GenerateDynamicInode(root.inode, "hosts")), - "ids": NewSnapshotsIDSDir(root, fs.GenerateDynamicInode(root.inode, "ids")), + // set defaults, if PathTemplates is not set + if len(cfg.PathTemplates) == 0 { + cfg.PathTemplates = []string{ + "ids/%i", + "snapshots/%T", + "hosts/%h/%T", + "tags/%t/%T", + } } - root.MetaDir = NewMetaDir(root, rootInode, entries) + root.SnapshotsDir = NewSnapshotsDir(root, rootInode, rootInode, NewSnapshotsDirStructure(root, cfg.PathTemplates, cfg.TimeTemplate), "") return root } diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 4371f656831..34bcccc4d8f 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -1,12 +1,11 @@ +//go:build darwin || freebsd || linux // +build darwin freebsd linux package fuse import ( "context" - "fmt" "os" - "time" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -15,152 +14,33 @@ import ( "bazil.org/fuse/fs" ) -// SnapshotsDir is a fuse directory which contains snapshots named by timestamp. +// SnapshotsDir is a actual fuse directory generated from SnapshotsDirStructure +// It uses the saved prefix to select the corresponding MetaDirData. type SnapshotsDir struct { - inode uint64 - root *Root - names map[string]*restic.Snapshot - latest string - tag string - host string - snCount int - - template string -} - -// SnapshotsIDSDir is a fuse directory which contains snapshots named by ids. -type SnapshotsIDSDir struct { - inode uint64 - root *Root - names map[string]*restic.Snapshot - snCount int -} - -// HostsDir is a fuse directory which contains hosts. -type HostsDir struct { - inode uint64 - root *Root - hosts map[string]bool - snCount int -} - -// TagsDir is a fuse directory which contains tags. -type TagsDir struct { - inode uint64 - root *Root - tags map[string]bool - snCount int -} - -// SnapshotLink -type snapshotLink struct { - root *Root - inode uint64 - target string - snapshot *restic.Snapshot + root *Root + inode uint64 + parentInode uint64 + dirStruct *SnapshotsDirStructure + prefix string } // ensure that *SnapshotsDir implements these interfaces var _ = fs.HandleReadDirAller(&SnapshotsDir{}) var _ = fs.NodeStringLookuper(&SnapshotsDir{}) -var _ = fs.HandleReadDirAller(&SnapshotsIDSDir{}) -var _ = fs.NodeStringLookuper(&SnapshotsIDSDir{}) -var _ = fs.HandleReadDirAller(&TagsDir{}) -var _ = fs.NodeStringLookuper(&TagsDir{}) -var _ = fs.HandleReadDirAller(&HostsDir{}) -var _ = fs.NodeStringLookuper(&HostsDir{}) -var _ = fs.NodeReadlinker(&snapshotLink{}) - -// read tag names from the current repository-state. -func updateTagNames(d *TagsDir) { - if d.snCount != d.root.snCount { - d.snCount = d.root.snCount - d.tags = make(map[string]bool, len(d.root.snapshots)) - for _, snapshot := range d.root.snapshots { - for _, tag := range snapshot.Tags { - if tag != "" { - d.tags[tag] = true - } - } - } - } -} - -// read host names from the current repository-state. -func updateHostsNames(d *HostsDir) { - if d.snCount != d.root.snCount { - d.snCount = d.root.snCount - d.hosts = make(map[string]bool, len(d.root.snapshots)) - for _, snapshot := range d.root.snapshots { - d.hosts[snapshot.Hostname] = true - } - } -} -// read snapshot id names from the current repository-state. -func updateSnapshotIDSNames(d *SnapshotsIDSDir) { - if d.snCount != d.root.snCount { - d.snCount = d.root.snCount - for _, sn := range d.root.snapshots { - name := sn.ID().Str() - d.names[name] = sn - } - } -} - -// NewSnapshotsDir returns a new directory containing snapshots. -func NewSnapshotsDir(root *Root, inode uint64, tag string, host string) *SnapshotsDir { +// NewSnapshotsDir returns a new directory structure containing snapshots and "latest" links +func NewSnapshotsDir(root *Root, inode, parentInode uint64, dirStruct *SnapshotsDirStructure, prefix string) *SnapshotsDir { debug.Log("create snapshots dir, inode %d", inode) - d := &SnapshotsDir{ - root: root, - inode: inode, - names: make(map[string]*restic.Snapshot), - latest: "", - tag: tag, - host: host, - template: root.cfg.SnapshotTemplate, + return &SnapshotsDir{ + root: root, + inode: inode, + parentInode: parentInode, + dirStruct: dirStruct, + prefix: prefix, } - - return d } -// NewSnapshotsIDSDir returns a new directory containing snapshots named by ids. -func NewSnapshotsIDSDir(root *Root, inode uint64) *SnapshotsIDSDir { - debug.Log("create snapshots ids dir, inode %d", inode) - d := &SnapshotsIDSDir{ - root: root, - inode: inode, - names: make(map[string]*restic.Snapshot), - } - - return d -} - -// NewHostsDir returns a new directory containing host names -func NewHostsDir(root *Root, inode uint64) *HostsDir { - debug.Log("create hosts dir, inode %d", inode) - d := &HostsDir{ - root: root, - inode: inode, - hosts: make(map[string]bool), - } - - return d -} - -// NewTagsDir returns a new directory containing tag names -func NewTagsDir(root *Root, inode uint64) *TagsDir { - debug.Log("create tags dir, inode %d", inode) - d := &TagsDir{ - root: root, - inode: inode, - tags: make(map[string]bool), - } - - return d -} - -// Attr returns the attributes for the root node. +// Attr returns the attributes for any dir in the snapshots directory structure func (d *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error { attr.Inode = d.inode attr.Mode = os.ModeDir | 0555 @@ -171,118 +51,18 @@ func (d *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error { return nil } -// Attr returns the attributes for the SnapshotsDir. -func (d *SnapshotsIDSDir) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Inode = d.inode - attr.Mode = os.ModeDir | 0555 - attr.Uid = d.root.uid - attr.Gid = d.root.gid - - debug.Log("attr: %v", attr) - return nil -} - -// Attr returns the attributes for the HostsDir. -func (d *HostsDir) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Inode = d.inode - attr.Mode = os.ModeDir | 0555 - attr.Uid = d.root.uid - attr.Gid = d.root.gid - - debug.Log("attr: %v", attr) - return nil -} - -// Attr returns the attributes for the TagsDir. -func (d *TagsDir) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Inode = d.inode - attr.Mode = os.ModeDir | 0555 - attr.Uid = d.root.uid - attr.Gid = d.root.gid - - debug.Log("attr: %v", attr) - return nil -} - -// search element in string list. -func isElem(e string, list []string) bool { - for _, x := range list { - if e == x { - return true - } - } - return false -} - -const minSnapshotsReloadTime = 60 * time.Second - -// update snapshots if repository has changed -func updateSnapshots(ctx context.Context, root *Root) error { - if time.Since(root.lastCheck) < minSnapshotsReloadTime { - return nil - } - - snapshots, err := restic.FindFilteredSnapshots(ctx, root.repo, root.cfg.Hosts, root.cfg.Tags, root.cfg.Paths) - if err != nil { - return err - } - - if root.snCount != len(snapshots) { - root.snCount = len(snapshots) - err := root.repo.LoadIndex(ctx) - if err != nil { - return err - } - root.snapshots = snapshots - } - root.lastCheck = time.Now() - - return nil -} - -// read snapshot timestamps from the current repository-state. -func updateSnapshotNames(d *SnapshotsDir, template string) { - if d.snCount != d.root.snCount { - d.snCount = d.root.snCount - var latestTime time.Time - d.latest = "" - d.names = make(map[string]*restic.Snapshot, len(d.root.snapshots)) - for _, sn := range d.root.snapshots { - if d.tag == "" || isElem(d.tag, sn.Tags) { - if d.host == "" || d.host == sn.Hostname { - name := sn.Time.Format(template) - if d.latest == "" || !sn.Time.Before(latestTime) { - latestTime = sn.Time - d.latest = name - } - for i := 1; ; i++ { - if _, ok := d.names[name]; !ok { - break - } - - name = fmt.Sprintf("%s-%d", sn.Time.Format(template), i) - } - - d.names[name] = sn - } - } - } - } -} - // ReadDirAll returns all entries of the SnapshotsDir. func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { debug.Log("ReadDirAll()") // update snapshots - err := updateSnapshots(ctx, d.root) + meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix) if err != nil { return nil, err + } else if meta == nil { + return nil, fuse.ENOENT } - // update snapshot names - updateSnapshotNames(d, d.root.cfg.SnapshotTemplate) - items := []fuse.Dirent{ { Inode: d.inode, @@ -290,141 +70,61 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { Type: fuse.DT_Dir, }, { - Inode: d.root.inode, + Inode: d.parentInode, Name: "..", Type: fuse.DT_Dir, }, } - for name := range d.names { - items = append(items, fuse.Dirent{ + for name, entry := range meta.names { + d := fuse.Dirent{ Inode: fs.GenerateDynamicInode(d.inode, name), Name: name, Type: fuse.DT_Dir, - }) + } + if entry.linkTarget != "" { + d.Type = fuse.DT_Link + } + items = append(items, d) } - // Latest - if d.latest != "" { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, "latest"), - Name: "latest", - Type: fuse.DT_Link, - }) - } return items, nil } -// ReadDirAll returns all entries of the SnapshotsIDSDir. -func (d *SnapshotsIDSDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - debug.Log("ReadDirAll()") +// Lookup returns a specific entry from the SnapshotsDir. +func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { + debug.Log("Lookup(%s)", name) - // update snapshots - err := updateSnapshots(ctx, d.root) + meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix) if err != nil { return nil, err + } else if meta == nil { + return nil, fuse.ENOENT } - // update snapshot ids - updateSnapshotIDSNames(d) - - items := []fuse.Dirent{ - { - Inode: d.inode, - Name: ".", - Type: fuse.DT_Dir, - }, - { - Inode: d.root.inode, - Name: "..", - Type: fuse.DT_Dir, - }, - } - - for name := range d.names { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, name), - Name: name, - Type: fuse.DT_Dir, - }) + entry := meta.names[name] + if entry != nil { + if entry.linkTarget != "" { + return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), entry.linkTarget, entry.snapshot) + } else if entry.snapshot != nil { + return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), entry.snapshot) + } else { + return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.inode, name), d.inode, d.dirStruct, d.prefix+"/"+name), nil + } } - return items, nil + return nil, fuse.ENOENT } -// ReadDirAll returns all entries of the HostsDir. -func (d *HostsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - debug.Log("ReadDirAll()") - - // update snapshots - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update host names - updateHostsNames(d) - - items := []fuse.Dirent{ - { - Inode: d.inode, - Name: ".", - Type: fuse.DT_Dir, - }, - { - Inode: d.root.inode, - Name: "..", - Type: fuse.DT_Dir, - }, - } - - for host := range d.hosts { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, host), - Name: host, - Type: fuse.DT_Dir, - }) - } - - return items, nil +// SnapshotLink +type snapshotLink struct { + root *Root + inode uint64 + target string + snapshot *restic.Snapshot } -// ReadDirAll returns all entries of the TagsDir. -func (d *TagsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - debug.Log("ReadDirAll()") - - // update snapshots - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update tag names - updateTagNames(d) - - items := []fuse.Dirent{ - { - Inode: d.inode, - Name: ".", - Type: fuse.DT_Dir, - }, - { - Inode: d.root.inode, - Name: "..", - Type: fuse.DT_Dir, - }, - } - - for tag := range d.tags { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, tag), - Name: tag, - Type: fuse.DT_Dir, - }) - } - - return items, nil -} +var _ = fs.NodeReadlinker(&snapshotLink{}) // newSnapshotLink func newSnapshotLink(ctx context.Context, root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) { @@ -452,117 +152,3 @@ func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error { return nil } - -// Lookup returns a specific entry from the SnapshotsDir. -func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { - debug.Log("Lookup(%s)", name) - - sn, ok := d.names[name] - if !ok { - // could not find entry. Updating repository-state - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update snapshot names - updateSnapshotNames(d, d.root.cfg.SnapshotTemplate) - - sn, ok := d.names[name] - if ok { - return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) - } - - if name == "latest" && d.latest != "" { - sn, ok := d.names[d.latest] - - // internal error - if !ok { - return nil, fuse.ENOENT - } - - return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), d.latest, sn) - } - return nil, fuse.ENOENT - } - - return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) -} - -// Lookup returns a specific entry from the SnapshotsIDSDir. -func (d *SnapshotsIDSDir) Lookup(ctx context.Context, name string) (fs.Node, error) { - debug.Log("Lookup(%s)", name) - - sn, ok := d.names[name] - if !ok { - // could not find entry. Updating repository-state - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update snapshot ids - updateSnapshotIDSNames(d) - - sn, ok := d.names[name] - if ok { - return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) - } - - return nil, fuse.ENOENT - } - - return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) -} - -// Lookup returns a specific entry from the HostsDir. -func (d *HostsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { - debug.Log("Lookup(%s)", name) - - _, ok := d.hosts[name] - if !ok { - // could not find entry. Updating repository-state - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update host names - updateHostsNames(d) - - _, ok := d.hosts[name] - if ok { - return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), "", name), nil - } - - return nil, fuse.ENOENT - } - - return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), "", name), nil -} - -// Lookup returns a specific entry from the TagsDir. -func (d *TagsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { - debug.Log("Lookup(%s)", name) - - _, ok := d.tags[name] - if !ok { - // could not find entry. Updating repository-state - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update tag names - updateTagNames(d) - - _, ok := d.tags[name] - if ok { - return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), name, ""), nil - } - - return nil, fuse.ENOENT - } - - return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), name, ""), nil -} diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go new file mode 100644 index 00000000000..d0eb080c789 --- /dev/null +++ b/internal/fuse/snapshots_dirstruct.go @@ -0,0 +1,327 @@ +//go:build darwin || freebsd || linux +// +build darwin freebsd linux + +package fuse + +import ( + "context" + "fmt" + "path" + "sort" + "strings" + "sync" + "time" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" +) + +type MetaDirData struct { + // set if this is a symlink or a snapshot mount point + linkTarget string + snapshot *restic.Snapshot + // names is set if this is a pseudo directory + names map[string]*MetaDirData +} + +// SnapshotsDirStructure contains the directory structure for snapshots. +// It uses a paths and time template to generate a map of pathnames +// pointing to the actual snapshots. For templates that end with a time, +// also "latest" links are generated. +type SnapshotsDirStructure struct { + root *Root + pathTemplates []string + timeTemplate string + + mutex sync.Mutex + // "" is the root path, subdirectory paths are assembled as parent+"/"+childFn + // thus all subdirectories are prefixed with a slash as the root is "" + // that way we don't need path processing special cases when using the entries tree + entries map[string]*MetaDirData + + snCount int + lastCheck time.Time +} + +// NewSnapshotsDirStructure returns a new directory structure for snapshots. +func NewSnapshotsDirStructure(root *Root, pathTemplates []string, timeTemplate string) *SnapshotsDirStructure { + return &SnapshotsDirStructure{ + root: root, + pathTemplates: pathTemplates, + timeTemplate: timeTemplate, + snCount: -1, + } +} + +// pathsFromSn generates the paths from pathTemplate and timeTemplate +// where the variables are replaced by the snapshot data. +// The time is given as suffix if the pathTemplate ends with "%T". +func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) (paths []string, timeSuffix string) { + timeformat := sn.Time.Format(timeTemplate) + + inVerb := false + writeTime := false + out := make([]strings.Builder, 1) + for _, c := range pathTemplate { + if writeTime { + for i := range out { + out[i].WriteString(timeformat) + } + writeTime = false + } + + if !inVerb { + if c == '%' { + inVerb = true + } else { + for i := range out { + out[i].WriteRune(c) + } + } + continue + } + + var repl string + inVerb = false + switch c { + case 'T': + // lazy write; time might be returned as suffix + writeTime = true + continue + + case 't': + if len(sn.Tags) == 0 { + return nil, "" + } + if len(sn.Tags) != 1 { + // needs special treatment: Rebuild the string builders + newout := make([]strings.Builder, len(out)*len(sn.Tags)) + for i, tag := range sn.Tags { + tag = filenameFromTag(tag) + for j := range out { + newout[i*len(out)+j].WriteString(out[j].String() + tag) + } + } + out = newout + continue + } + repl = sn.Tags[0] + + case 'i': + repl = sn.ID().Str() + + case 'I': + repl = sn.ID().String() + + case 'u': + repl = sn.Username + + case 'h': + repl = sn.Hostname + + default: + repl = string(c) + } + + // write replacement string to all string builders + for i := range out { + out[i].WriteString(repl) + } + } + + for i := range out { + paths = append(paths, out[i].String()) + } + + if writeTime { + timeSuffix = timeformat + } + + return paths, timeSuffix +} + +// Some tags are problematic when used as filenames: +// +// "" +// ".", ".." +// anything containing '/' +// +// Replace all special character by underscores "_", an empty tag is also represented as a underscore. +func filenameFromTag(tag string) string { + switch tag { + case "", ".": + return "_" + case "..": + return "__" + } + + return strings.ReplaceAll(tag, "/", "_") +} + +// determine static path prefix +func staticPrefix(pathTemplate string) (prefix string) { + inVerb := false + patternStart := -1 +outer: + for i, c := range pathTemplate { + if !inVerb { + if c == '%' { + inVerb = true + } + continue + } + inVerb = false + switch c { + case 'i', 'I', 'u', 'h', 't', 'T': + patternStart = i + break outer + } + } + if patternStart < 0 { + // ignore patterns without template variable + return "" + } + + p := pathTemplate[:patternStart] + idx := strings.LastIndex(p, "/") + if idx < 0 { + return "" + } + return p[:idx] +} + +// uniqueName returns a unique name to be used for prefix+name. +// It appends -number to make the name unique. +func uniqueName(entries map[string]*MetaDirData, prefix, name string) string { + newname := name + for i := 1; ; i++ { + if _, ok := entries[prefix+newname]; !ok { + break + } + newname = fmt.Sprintf("%s-%d", name, i) + } + return newname +} + +// makeDirs inserts all paths generated from pathTemplates and +// TimeTemplate for all given snapshots into d.names. +// Also adds d.latest links if "%T" is at end of a path template +func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) { + entries := make(map[string]*MetaDirData) + + type mountData struct { + sn *restic.Snapshot + linkTarget string // if linkTarget!= "", this is a symlink + childFn string + child *MetaDirData + } + + // recursively build tree structure + var mount func(path string, data mountData) + mount = func(path string, data mountData) { + e := entries[path] + if e == nil { + e = &MetaDirData{} + } + if data.sn != nil { + e.snapshot = data.sn + e.linkTarget = data.linkTarget + } else { + // intermediate directory, register as a child directory + if e.names == nil { + e.names = make(map[string]*MetaDirData) + } + if data.child != nil { + e.names[data.childFn] = data.child + } + } + entries[path] = e + + slashIdx := strings.LastIndex(path, "/") + if slashIdx >= 0 { + // add to parent dir, but without snapshot + mount(path[:slashIdx], mountData{childFn: path[slashIdx+1:], child: e}) + } + } + + // root directory + mount("", mountData{}) + + // insert pure directories; needed to get empty structure even if there + // are no snapshots in these dirs + for _, p := range d.pathTemplates { + p = staticPrefix(p) + if p != "" { + mount(path.Clean("/"+p), mountData{}) + } + } + + latestTime := make(map[string]time.Time) + for _, sn := range snapshots { + for _, templ := range d.pathTemplates { + paths, timeSuffix := pathsFromSn(templ, d.timeTemplate, sn) + for _, p := range paths { + if p != "" { + p = "/" + p + } + suffix := uniqueName(entries, p, timeSuffix) + mount(path.Clean(p+suffix), mountData{sn: sn}) + if timeSuffix != "" { + lt, ok := latestTime[p] + if !ok || !sn.Time.Before(lt) { + debug.Log("link (update) %v -> %v\n", p, suffix) + // inject symlink + mount(path.Clean(p+"/latest"), mountData{sn: sn, linkTarget: suffix}) + latestTime[p] = sn.Time + } + } + } + } + } + + d.entries = entries +} + +const minSnapshotsReloadTime = 60 * time.Second + +// update snapshots if repository has changed +func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { + d.mutex.Lock() + defer d.mutex.Unlock() + if time.Since(d.lastCheck) < minSnapshotsReloadTime { + return nil + } + + snapshots, err := restic.FindFilteredSnapshots(ctx, d.root.repo.Backend(), d.root.repo, d.root.cfg.Hosts, d.root.cfg.Tags, d.root.cfg.Paths) + if err != nil { + return err + } + // sort snapshots ascending by time (default order is descending) + sort.Sort(sort.Reverse(snapshots)) + + if d.snCount == len(snapshots) { + d.lastCheck = time.Now() + return nil + } + + err = d.root.repo.LoadIndex(ctx) + if err != nil { + return err + } + + d.lastCheck = time.Now() + d.snCount = len(snapshots) + d.makeDirs(snapshots) + return nil +} + +func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, error) { + err := d.updateSnapshots(ctx) + if err != nil { + return nil, err + } + + d.mutex.Lock() + defer d.mutex.Unlock() + return d.entries[prefix], nil +} diff --git a/internal/fuse/snapshots_dirstruct_test.go b/internal/fuse/snapshots_dirstruct_test.go new file mode 100644 index 00000000000..1e823a475ac --- /dev/null +++ b/internal/fuse/snapshots_dirstruct_test.go @@ -0,0 +1,291 @@ +//go:build darwin || freebsd || linux +// +build darwin freebsd linux + +package fuse + +import ( + "strings" + "testing" + "time" + + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" +) + +func TestPathsFromSn(t *testing.T) { + id1, _ := restic.ParseID("1234567812345678123456781234567812345678123456781234567812345678") + time1, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T00:00:01") + sn1 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time1} + restic.TestSetSnapshotID(t, sn1, id1) + + var p []string + var s string + + p, s = pathsFromSn("ids/%i", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"ids/12345678"}, p) + test.Equals(t, "", s) + + p, s = pathsFromSn("snapshots/%T", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"snapshots/"}, p) + test.Equals(t, "2021-01-01T00:00:01", s) + + p, s = pathsFromSn("hosts/%h/%T", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"hosts/host/"}, p) + test.Equals(t, "2021-01-01T00:00:01", s) + + p, s = pathsFromSn("tags/%t/%T", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"tags/tag1/", "tags/tag2/"}, p) + test.Equals(t, "2021-01-01T00:00:01", s) + + p, s = pathsFromSn("users/%u/%T", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"users/user/"}, p) + test.Equals(t, "2021-01-01T00:00:01", s) + + p, s = pathsFromSn("longids/%I", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"longids/1234567812345678123456781234567812345678123456781234567812345678"}, p) + test.Equals(t, "", s) + + p, s = pathsFromSn("%T/%h", "2006/01/02", sn1) + test.Equals(t, []string{"2021/01/01/host"}, p) + test.Equals(t, "", s) + + p, s = pathsFromSn("%T/%i", "2006/01", sn1) + test.Equals(t, []string{"2021/01/12345678"}, p) + test.Equals(t, "", s) +} + +func TestMakeDirs(t *testing.T) { + pathTemplates := []string{"ids/%i", "snapshots/%T", "hosts/%h/%T", + "tags/%t/%T", "users/%u/%T", "longids/%I", "%T/%h", "%T/%i", + } + timeTemplate := "2006/01/02" + + sds := &SnapshotsDirStructure{ + pathTemplates: pathTemplates, + timeTemplate: timeTemplate, + } + + id0, _ := restic.ParseID("0000000012345678123456781234567812345678123456781234567812345678") + time0, _ := time.Parse("2006-01-02T15:04:05", "2020-12-31T00:00:01") + sn0 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time0} + restic.TestSetSnapshotID(t, sn0, id0) + + id1, _ := restic.ParseID("1234567812345678123456781234567812345678123456781234567812345678") + time1, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T00:00:01") + sn1 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time1} + restic.TestSetSnapshotID(t, sn1, id1) + + id2, _ := restic.ParseID("8765432112345678123456781234567812345678123456781234567812345678") + time2, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T01:02:03") + sn2 := &restic.Snapshot{Hostname: "host2", Username: "user2", Tags: []string{"tag2", "tag3", "tag4"}, Time: time2} + restic.TestSetSnapshotID(t, sn2, id2) + + id3, _ := restic.ParseID("aaaaaaaa12345678123456781234567812345678123456781234567812345678") + time3, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T01:02:03") + sn3 := &restic.Snapshot{Hostname: "host", Username: "user2", Tags: []string{}, Time: time3} + restic.TestSetSnapshotID(t, sn3, id3) + + sds.makeDirs(restic.Snapshots{sn0, sn1, sn2, sn3}) + + expNames := make(map[string]*restic.Snapshot) + expLatest := make(map[string]string) + + // entries for sn0 + expNames["/ids/00000000"] = sn0 + expNames["/snapshots/2020/12/31"] = sn0 + expNames["/hosts/host/2020/12/31"] = sn0 + expNames["/tags/tag1/2020/12/31"] = sn0 + expNames["/tags/tag2/2020/12/31"] = sn0 + expNames["/users/user/2020/12/31"] = sn0 + expNames["/longids/0000000012345678123456781234567812345678123456781234567812345678"] = sn0 + expNames["/2020/12/31/host"] = sn0 + expNames["/2020/12/31/00000000"] = sn0 + + // entries for sn1 + expNames["/ids/12345678"] = sn1 + expNames["/snapshots/2021/01/01"] = sn1 + expNames["/hosts/host/2021/01/01"] = sn1 + expNames["/tags/tag1/2021/01/01"] = sn1 + expNames["/tags/tag2/2021/01/01"] = sn1 + expNames["/users/user/2021/01/01"] = sn1 + expNames["/longids/1234567812345678123456781234567812345678123456781234567812345678"] = sn1 + expNames["/2021/01/01/host"] = sn1 + expNames["/2021/01/01/12345678"] = sn1 + + // entries for sn2 + expNames["/ids/87654321"] = sn2 + expNames["/snapshots/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string + expNames["/hosts/host2/2021/01/01"] = sn2 + expNames["/tags/tag2/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string + expNames["/tags/tag3/2021/01/01"] = sn2 + expNames["/tags/tag4/2021/01/01"] = sn2 + expNames["/users/user2/2021/01/01"] = sn2 + expNames["/longids/8765432112345678123456781234567812345678123456781234567812345678"] = sn2 + expNames["/2021/01/01/host2"] = sn2 + expNames["/2021/01/01/87654321"] = sn2 + + // entries for sn3 + expNames["/ids/aaaaaaaa"] = sn3 + expNames["/snapshots/2021/01/01-2"] = sn3 // sn1 - sn3 have same time string + expNames["/hosts/host/2021/01/01-1"] = sn3 // sn1 and sn3 have same time string + expNames["/users/user2/2021/01/01-1"] = sn3 // sn2 and sn3 have same time string + expNames["/longids/aaaaaaaa12345678123456781234567812345678123456781234567812345678"] = sn3 + expNames["/2021/01/01/host-1"] = sn3 // sn1 and sn3 have same time string and identical host + expNames["/2021/01/01/aaaaaaaa"] = sn3 + + // intermediate directories + // sn0 + expNames["/ids"] = nil + expNames[""] = nil + expNames["/snapshots/2020/12"] = nil + expNames["/snapshots/2020"] = nil + expNames["/snapshots"] = nil + expNames["/hosts/host/2020/12"] = nil + expNames["/hosts/host/2020"] = nil + expNames["/hosts/host"] = nil + expNames["/hosts"] = nil + expNames["/tags/tag1/2020/12"] = nil + expNames["/tags/tag1/2020"] = nil + expNames["/tags/tag1"] = nil + expNames["/tags"] = nil + expNames["/tags/tag2/2020/12"] = nil + expNames["/tags/tag2/2020"] = nil + expNames["/tags/tag2"] = nil + expNames["/users/user/2020/12"] = nil + expNames["/users/user/2020"] = nil + expNames["/users/user"] = nil + expNames["/users"] = nil + expNames["/longids"] = nil + expNames["/2020/12/31"] = nil + expNames["/2020/12"] = nil + expNames["/2020"] = nil + + // sn1 + expNames["/snapshots/2021/01"] = nil + expNames["/snapshots/2021"] = nil + expNames["/hosts/host/2021/01"] = nil + expNames["/hosts/host/2021"] = nil + expNames["/tags/tag1/2021/01"] = nil + expNames["/tags/tag1/2021"] = nil + expNames["/tags/tag2/2021/01"] = nil + expNames["/tags/tag2/2021"] = nil + expNames["/users/user/2021/01"] = nil + expNames["/users/user/2021"] = nil + expNames["/2021/01/01"] = nil + expNames["/2021/01"] = nil + expNames["/2021"] = nil + + // sn2 + expNames["/hosts/host2/2021/01"] = nil + expNames["/hosts/host2/2021"] = nil + expNames["/hosts/host2"] = nil + expNames["/tags/tag3/2021/01"] = nil + expNames["/tags/tag3/2021"] = nil + expNames["/tags/tag3"] = nil + expNames["/tags/tag4/2021/01"] = nil + expNames["/tags/tag4/2021"] = nil + expNames["/tags/tag4"] = nil + expNames["/users/user2/2021/01"] = nil + expNames["/users/user2/2021"] = nil + expNames["/users/user2"] = nil + + // target snapshots for links + expNames["/snapshots/latest"] = sn3 // sn1 - sn3 have same time string + expNames["/hosts/host/latest"] = sn3 + expNames["/hosts/host2/latest"] = sn2 + expNames["/tags/tag1/latest"] = sn1 + expNames["/tags/tag2/latest"] = sn2 // sn1 and sn2 have same time string + expNames["/tags/tag3/latest"] = sn2 + expNames["/tags/tag4/latest"] = sn2 + expNames["/users/user/latest"] = sn1 + expNames["/users/user2/latest"] = sn3 // sn2 and sn3 have same time string + + // latest links + expLatest["/snapshots/latest"] = "2021/01/01-2" // sn1 - sn3 have same time string + expLatest["/hosts/host/latest"] = "2021/01/01-1" + expLatest["/hosts/host2/latest"] = "2021/01/01" + expLatest["/tags/tag1/latest"] = "2021/01/01" + expLatest["/tags/tag2/latest"] = "2021/01/01-1" // sn1 and sn2 have same time string + expLatest["/tags/tag3/latest"] = "2021/01/01" + expLatest["/tags/tag4/latest"] = "2021/01/01" + expLatest["/users/user/latest"] = "2021/01/01" + expLatest["/users/user2/latest"] = "2021/01/01-1" // sn2 and sn3 have same time string + + verifyEntries(t, expNames, expLatest, sds.entries) +} + +func verifyEntries(t *testing.T, expNames map[string]*restic.Snapshot, expLatest map[string]string, entries map[string]*MetaDirData) { + actNames := make(map[string]*restic.Snapshot) + actLatest := make(map[string]string) + for path, entry := range entries { + actNames[path] = entry.snapshot + if entry.linkTarget != "" { + actLatest[path] = entry.linkTarget + } + } + + test.Equals(t, expNames, actNames) + test.Equals(t, expLatest, actLatest) + + // verify tree integrity + for path, entry := range entries { + // check that all children are actually contained in entry.names + for otherPath := range entries { + if strings.HasPrefix(otherPath, path+"/") { + sub := otherPath[len(path)+1:] + // remaining path does not contain a directory + test.Assert(t, strings.Contains(sub, "/") || (entry.names != nil && entry.names[sub] != nil), "missing entry %v in %v", sub, path) + } + } + if entry.names == nil { + continue + } + // child entries reference the correct MetaDirData + for elem, subentry := range entry.names { + test.Equals(t, entries[path+"/"+elem], subentry) + } + } +} + +func TestMakeEmptyDirs(t *testing.T) { + pathTemplates := []string{"ids/%i", "snapshots/%T", "hosts/%h/%T", + "tags/%t/%T", "users/%u/%T", "longids/id-%I", "%T/%h", "%T/%i", "id-%i", + } + timeTemplate := "2006/01/02" + + sds := &SnapshotsDirStructure{ + pathTemplates: pathTemplates, + timeTemplate: timeTemplate, + } + sds.makeDirs(restic.Snapshots{}) + + expNames := make(map[string]*restic.Snapshot) + expLatest := make(map[string]string) + + // empty entries for dir structure + expNames["/ids"] = nil + expNames["/snapshots"] = nil + expNames["/hosts"] = nil + expNames["/tags"] = nil + expNames["/users"] = nil + expNames["/longids"] = nil + expNames[""] = nil + + verifyEntries(t, expNames, expLatest, sds.entries) +} + +func TestFilenameFromTag(t *testing.T) { + for _, c := range []struct { + tag, filename string + }{ + {"", "_"}, + {".", "_"}, + {"..", "__"}, + {"%.", "%."}, + {"foo", "foo"}, + {"foo ", "foo "}, + {"foo/bar_baz", "foo_bar_baz"}, + } { + test.Equals(t, c.filename, filenameFromTag(c.tag)) + } +} diff --git a/internal/hashing/reader.go b/internal/hashing/reader.go index ea45dcd24ac..073d609329f 100644 --- a/internal/hashing/reader.go +++ b/internal/hashing/reader.go @@ -5,47 +5,24 @@ import ( "io" ) -// ReadSumer hashes all data read from the underlying reader. -type ReadSumer interface { - io.Reader - // Sum returns the hash of the data read so far. - Sum(d []byte) []byte -} - -type reader struct { - io.Reader +// A Reader hashes all data read from the underlying reader. +type Reader struct { + r io.Reader h hash.Hash } -type readWriterTo struct { - reader - writerTo io.WriterTo +// NewReader returns a new Reader that uses the hash h. +func NewReader(r io.Reader, h hash.Hash) *Reader { + return &Reader{r: r, h: h} } -// NewReader returns a new ReadSummer that uses the hash h. If the underlying -// reader supports WriteTo then the returned reader will do so too. -func NewReader(r io.Reader, h hash.Hash) ReadSumer { - rs := reader{ - Reader: io.TeeReader(r, h), - h: h, - } - - if _, ok := r.(io.WriterTo); ok { - return &readWriterTo{ - reader: rs, - writerTo: r.(io.WriterTo), - } - } - - return &rs +func (h *Reader) Read(p []byte) (int, error) { + n, err := h.r.Read(p) + _, _ = h.h.Write(p[:n]) // Never returns an error. + return n, err } // Sum returns the hash of the data read so far. -func (h *reader) Sum(d []byte) []byte { +func (h *Reader) Sum(d []byte) []byte { return h.h.Sum(d) } - -// WriteTo reads all data into the passed writer -func (h *readWriterTo) WriteTo(w io.Writer) (int64, error) { - return h.writerTo.WriteTo(NewWriter(w, h.h)) -} diff --git a/internal/hashing/reader_test.go b/internal/hashing/reader_test.go index d7bdc2e024f..d17f264de00 100644 --- a/internal/hashing/reader_test.go +++ b/internal/hashing/reader_test.go @@ -7,26 +7,8 @@ import ( "io" "io/ioutil" "testing" - - rtest "github.com/restic/restic/internal/test" ) -// only expose Read method -type onlyReader struct { - io.Reader -} - -type traceWriterTo struct { - io.Reader - writerTo io.WriterTo - Traced bool -} - -func (r *traceWriterTo) WriteTo(w io.Writer) (n int64, err error) { - r.Traced = true - return r.writerTo.WriteTo(w) -} - func TestReader(t *testing.T) { tests := []int{5, 23, 2<<18 + 23, 1 << 20} @@ -39,44 +21,22 @@ func TestReader(t *testing.T) { expectedHash := sha256.Sum256(data) - for _, test := range []struct { - innerWriteTo, outerWriteTo bool - }{{false, false}, {false, true}, {true, false}, {true, true}} { - // test both code paths in WriteTo - src := bytes.NewReader(data) - rawSrc := &traceWriterTo{Reader: src, writerTo: src} - innerSrc := io.Reader(rawSrc) - if !test.innerWriteTo { - innerSrc = &onlyReader{Reader: rawSrc} - } - - rd := NewReader(innerSrc, sha256.New()) - // test both Read and WriteTo - outerSrc := io.Reader(rd) - if !test.outerWriteTo { - outerSrc = &onlyReader{Reader: outerSrc} - } - - n, err := io.Copy(ioutil.Discard, outerSrc) - if err != nil { - t.Fatal(err) - } - - if n != int64(size) { - t.Errorf("Reader: invalid number of bytes written: got %d, expected %d", - n, size) - } + rd := NewReader(bytes.NewReader(data), sha256.New()) + n, err := io.Copy(ioutil.Discard, rd) + if err != nil { + t.Fatal(err) + } - resultingHash := rd.Sum(nil) + if n != int64(size) { + t.Errorf("Reader: invalid number of bytes written: got %d, expected %d", + n, size) + } - if !bytes.Equal(expectedHash[:], resultingHash) { - t.Errorf("Reader: hashes do not match: expected %02x, got %02x", - expectedHash, resultingHash) - } + resultingHash := rd.Sum(nil) - rtest.Assert(t, rawSrc.Traced == (test.innerWriteTo && test.outerWriteTo), - "unexpected/missing writeTo call innerWriteTo %v outerWriteTo %v", - test.innerWriteTo, test.outerWriteTo) + if !bytes.Equal(expectedHash[:], resultingHash) { + t.Errorf("Reader: hashes do not match: expected %02x, got %02x", + expectedHash, resultingHash) } } } diff --git a/internal/migrations/interface.go b/internal/migrations/interface.go index 9d9eedba1cb..99100bce3c2 100644 --- a/internal/migrations/interface.go +++ b/internal/migrations/interface.go @@ -11,6 +11,8 @@ type Migration interface { // Check returns true if the migration can be applied to a repo. Check(context.Context, restic.Repository) (bool, error) + RepoCheck() bool + // Apply runs the migration. Apply(context.Context, restic.Repository) error diff --git a/internal/migrations/s3_layout.go b/internal/migrations/s3_layout.go index 877b44c84d1..afb14b8487c 100644 --- a/internal/migrations/s3_layout.go +++ b/internal/migrations/s3_layout.go @@ -8,6 +8,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/s3" + "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -21,10 +22,25 @@ func init() { // "default" layout. type S3Layout struct{} +func toS3Backend(repo restic.Repository) *s3.Backend { + b := repo.Backend() + // unwrap cache + if be, ok := b.(*cache.Backend); ok { + b = be.Backend + } + + be, ok := b.(*s3.Backend) + if !ok { + debug.Log("backend is not s3") + return nil + } + return be +} + // Check tests whether the migration can be applied. func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, error) { - be, ok := repo.Backend().(*s3.Backend) - if !ok { + be := toS3Backend(repo) + if be == nil { debug.Log("backend is not s3") return false, nil } @@ -37,6 +53,10 @@ func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, err return true, nil } +func (m *S3Layout) RepoCheck() bool { + return false +} + func retry(max int, fail func(err error), f func() error) error { var err error for i := 0; i < max; i++ { @@ -71,8 +91,8 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layo // Apply runs the migration. func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { - be, ok := repo.Backend().(*s3.Backend) - if !ok { + be := toS3Backend(repo) + if be == nil { debug.Log("backend is not s3") return errors.New("backend is not s3") } diff --git a/internal/migrations/upgrade_repo_v2.go b/internal/migrations/upgrade_repo_v2.go new file mode 100644 index 00000000000..b29fbcdcc44 --- /dev/null +++ b/internal/migrations/upgrade_repo_v2.go @@ -0,0 +1,126 @@ +package migrations + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/restic/restic/internal/restic" +) + +func init() { + register(&UpgradeRepoV2{}) +} + +type UpgradeRepoV2Error struct { + UploadNewConfigError error + ReuploadOldConfigError error + + BackupFilePath string +} + +func (err *UpgradeRepoV2Error) Error() string { + if err.ReuploadOldConfigError != nil { + return fmt.Sprintf("error uploading config (%v), re-uploading old config filed failed as well (%v), but there is a backup of the config file in %v", err.UploadNewConfigError, err.ReuploadOldConfigError, err.BackupFilePath) + } + + return fmt.Sprintf("error uploading config (%v), re-uploaded old config was successful, there is a backup of the config file in %v", err.UploadNewConfigError, err.BackupFilePath) +} + +func (err *UpgradeRepoV2Error) Unwrap() error { + // consider the original upload error as the primary cause + return err.UploadNewConfigError +} + +type UpgradeRepoV2 struct{} + +func (*UpgradeRepoV2) Name() string { + return "upgrade_repo_v2" +} + +func (*UpgradeRepoV2) Desc() string { + return "upgrade a repository to version 2" +} + +func (*UpgradeRepoV2) Check(ctx context.Context, repo restic.Repository) (bool, error) { + isV1 := repo.Config().Version == 1 + return isV1, nil +} + +func (*UpgradeRepoV2) RepoCheck() bool { + return true +} +func (*UpgradeRepoV2) upgrade(ctx context.Context, repo restic.Repository) error { + h := restic.Handle{Type: restic.ConfigFile} + + if !repo.Backend().HasAtomicReplace() { + // remove the original file for backends which do not support atomic overwriting + err := repo.Backend().Remove(ctx, h) + if err != nil { + return fmt.Errorf("remove config failed: %w", err) + } + } + + // upgrade config + cfg := repo.Config() + cfg.Version = 2 + + err := restic.SaveConfig(ctx, repo, cfg) + if err != nil { + return fmt.Errorf("save new config file failed: %w", err) + } + + return nil +} + +func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error { + tempdir, err := ioutil.TempDir("", "restic-migrate-upgrade-repo-v2-") + if err != nil { + return fmt.Errorf("create temp dir failed: %w", err) + } + + h := restic.Handle{Type: restic.ConfigFile} + + // read raw config file and save it to a temp dir, just in case + var rawConfigFile []byte + err = repo.Backend().Load(ctx, h, 0, 0, func(rd io.Reader) (err error) { + rawConfigFile, err = ioutil.ReadAll(rd) + return err + }) + if err != nil { + return fmt.Errorf("load config file failed: %w", err) + } + + backupFileName := filepath.Join(tempdir, "config") + err = ioutil.WriteFile(backupFileName, rawConfigFile, 0600) + if err != nil { + return fmt.Errorf("write config file backup to %v failed: %w", tempdir, err) + } + + // run the upgrade + err = m.upgrade(ctx, repo) + if err != nil { + + // build an error we can return to the caller + repoError := &UpgradeRepoV2Error{ + UploadNewConfigError: err, + BackupFilePath: backupFileName, + } + + // try contingency methods, reupload the original file + _ = repo.Backend().Remove(ctx, h) + err = repo.Backend().Save(ctx, h, restic.NewByteReader(rawConfigFile, nil)) + if err != nil { + repoError.ReuploadOldConfigError = err + } + + return repoError + } + + _ = os.Remove(backupFileName) + _ = os.Remove(tempdir) + return nil +} diff --git a/internal/migrations/upgrade_repo_v2_test.go b/internal/migrations/upgrade_repo_v2_test.go new file mode 100644 index 00000000000..0d86d265cd2 --- /dev/null +++ b/internal/migrations/upgrade_repo_v2_test.go @@ -0,0 +1,112 @@ +package migrations + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" +) + +func TestUpgradeRepoV2(t *testing.T) { + repo, cleanup := repository.TestRepositoryWithVersion(t, 1) + defer cleanup() + + if repo.Config().Version != 1 { + t.Fatal("test repo has wrong version") + } + + m := &UpgradeRepoV2{} + + ok, err := m.Check(context.Background(), repo) + if err != nil { + t.Fatal(err) + } + + if !ok { + t.Fatal("migration check returned false") + } + + err = m.Apply(context.Background(), repo) + if err != nil { + t.Fatal(err) + } +} + +type failBackend struct { + restic.Backend + + mu sync.Mutex + ConfigFileSavesUntilError uint +} + +func (be *failBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + if h.Type != restic.ConfigFile { + return be.Backend.Save(ctx, h, rd) + } + + be.mu.Lock() + if be.ConfigFileSavesUntilError == 0 { + be.mu.Unlock() + return errors.New("failure induced for testing") + } + + be.ConfigFileSavesUntilError-- + be.mu.Unlock() + + return be.Backend.Save(ctx, h, rd) +} + +func TestUpgradeRepoV2Failure(t *testing.T) { + be, cleanup := repository.TestBackend(t) + defer cleanup() + + // wrap backend so that it fails upgrading the config after the initial write + be = &failBackend{ + ConfigFileSavesUntilError: 1, + Backend: be, + } + + repo, cleanup := repository.TestRepositoryWithBackend(t, be, 1) + defer cleanup() + + if repo.Config().Version != 1 { + t.Fatal("test repo has wrong version") + } + + m := &UpgradeRepoV2{} + + ok, err := m.Check(context.Background(), repo) + if err != nil { + t.Fatal(err) + } + + if !ok { + t.Fatal("migration check returned false") + } + + err = m.Apply(context.Background(), repo) + if err == nil { + t.Fatal("expected error returned from Apply(), got nil") + } + + upgradeErr := err.(*UpgradeRepoV2Error) + if upgradeErr.UploadNewConfigError == nil { + t.Fatal("expected upload error, got nil") + } + + if upgradeErr.ReuploadOldConfigError == nil { + t.Fatal("expected reupload error, got nil") + } + + if upgradeErr.BackupFilePath == "" { + t.Fatal("no backup file path found") + } + test.OK(t, os.Remove(upgradeErr.BackupFilePath)) + test.OK(t, os.Remove(filepath.Dir(upgradeErr.BackupFilePath))) +} diff --git a/internal/options/secret_string.go b/internal/options/secret_string.go new file mode 100644 index 00000000000..107127612f6 --- /dev/null +++ b/internal/options/secret_string.go @@ -0,0 +1,27 @@ +package options + +type SecretString struct { + s *string +} + +func NewSecretString(s string) SecretString { + return SecretString{s: &s} +} + +func (s SecretString) GoString() string { + return `"` + s.String() + `"` +} + +func (s SecretString) String() string { + if s.s == nil || len(*s.s) == 0 { + return `` + } + return `**redacted**` +} + +func (s *SecretString) Unwrap() string { + if s.s == nil { + return "" + } + return *s.s +} diff --git a/internal/options/secret_string_test.go b/internal/options/secret_string_test.go new file mode 100644 index 00000000000..a2ca017462e --- /dev/null +++ b/internal/options/secret_string_test.go @@ -0,0 +1,63 @@ +package options_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/test" +) + +type secretTest struct { + str options.SecretString +} + +func assertNotIn(t *testing.T, str string, substr string) { + if strings.Contains(str, substr) { + t.Fatalf("'%s' should not contain '%s'", str, substr) + } +} + +func TestSecretString(t *testing.T) { + keyStr := "secret-key" + secret := options.NewSecretString(keyStr) + + test.Equals(t, "**redacted**", secret.String()) + test.Equals(t, `"**redacted**"`, secret.GoString()) + test.Equals(t, "**redacted**", fmt.Sprint(secret)) + test.Equals(t, "**redacted**", fmt.Sprintf("%v", secret)) + test.Equals(t, `"**redacted**"`, fmt.Sprintf("%#v", secret)) + test.Equals(t, keyStr, secret.Unwrap()) +} + +func TestSecretStringStruct(t *testing.T) { + keyStr := "secret-key" + secretStruct := &secretTest{ + str: options.NewSecretString(keyStr), + } + + assertNotIn(t, fmt.Sprint(secretStruct), keyStr) + assertNotIn(t, fmt.Sprintf("%v", secretStruct), keyStr) + assertNotIn(t, fmt.Sprintf("%#v", secretStruct), keyStr) +} + +func TestSecretStringEmpty(t *testing.T) { + keyStr := "" + secret := options.NewSecretString(keyStr) + + test.Equals(t, "", secret.String()) + test.Equals(t, `""`, secret.GoString()) + test.Equals(t, "", fmt.Sprint(secret)) + test.Equals(t, "", fmt.Sprintf("%v", secret)) + test.Equals(t, `""`, fmt.Sprintf("%#v", secret)) + test.Equals(t, keyStr, secret.Unwrap()) +} + +func TestSecretStringDefault(t *testing.T) { + secretStruct := &secretTest{} + + test.Equals(t, "", secretStruct.str.String()) + test.Equals(t, `""`, secretStruct.str.GoString()) + test.Equals(t, "", secretStruct.str.Unwrap()) +} diff --git a/internal/pack/pack.go b/internal/pack/pack.go index d679c658b0c..11be4169742 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -1,6 +1,7 @@ package pack import ( + "context" "encoding/binary" "fmt" "io" @@ -30,8 +31,8 @@ func NewPacker(k *crypto.Key, wr io.Writer) *Packer { } // Add saves the data read from rd as a new blob to the packer. Returned is the -// number of bytes written to the pack. -func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte) (int, error) { +// number of bytes written to the pack plus the pack header entry size. +func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedLength int) (int, error) { p.m.Lock() defer p.m.Unlock() @@ -40,13 +41,16 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte) (int, error) n, err := p.wr.Write(data) c.Length = uint(n) c.Offset = p.bytes + c.UncompressedLength = uint(uncompressedLength) p.bytes += uint(n) p.blobs = append(p.blobs, c) + n += CalculateEntrySize(c) return n, errors.Wrap(err, "Write") } -var EntrySize = uint(binary.Size(restic.BlobType(0)) + headerLengthSize + len(restic.ID{})) +var entrySize = uint(binary.Size(restic.BlobType(0)) + 2*headerLengthSize + len(restic.ID{})) +var plainEntrySize = uint(binary.Size(restic.BlobType(0)) + headerLengthSize + len(restic.ID{})) // headerEntry describes the format of header entries. It serves only as // documentation. @@ -56,20 +60,26 @@ type headerEntry struct { ID restic.ID } +// compressedHeaderEntry describes the format of header entries for compressed blobs. +// It serves only as documentation. +type compressedHeaderEntry struct { + Type uint8 + Length uint32 + UncompressedLength uint32 + ID restic.ID +} + // Finalize writes the header for all added blobs and finalizes the pack. -// Returned are the number of bytes written, including the header. -func (p *Packer) Finalize() (uint, error) { +func (p *Packer) Finalize() error { p.m.Lock() defer p.m.Unlock() - bytesWritten := p.bytes - header, err := p.makeHeader() if err != nil { - return 0, err + return err } - encryptedHeader := make([]byte, 0, len(header)+p.k.Overhead()+p.k.NonceSize()) + encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header))) nonce := crypto.NewRandomNonce() encryptedHeader = append(encryptedHeader, nonce...) encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil) @@ -77,37 +87,43 @@ func (p *Packer) Finalize() (uint, error) { // append the header n, err := p.wr.Write(encryptedHeader) if err != nil { - return 0, errors.Wrap(err, "Write") + return errors.Wrap(err, "Write") } - hdrBytes := restic.CiphertextLength(len(header)) + hdrBytes := len(encryptedHeader) if n != hdrBytes { - return 0, errors.New("wrong number of bytes written") + return errors.New("wrong number of bytes written") } - bytesWritten += uint(hdrBytes) - // write length - err = binary.Write(p.wr, binary.LittleEndian, uint32(restic.CiphertextLength(len(p.blobs)*int(EntrySize)))) + err = binary.Write(p.wr, binary.LittleEndian, uint32(hdrBytes)) if err != nil { - return 0, errors.Wrap(err, "binary.Write") + return errors.Wrap(err, "binary.Write") } - bytesWritten += uint(binary.Size(uint32(0))) + p.bytes += uint(hdrBytes + binary.Size(uint32(0))) - p.bytes = uint(bytesWritten) - return bytesWritten, nil + return nil +} + +// HeaderOverhead returns an estimate of the number of bytes written by a call to Finalize. +func (p *Packer) HeaderOverhead() int { + return crypto.CiphertextLength(0) + binary.Size(uint32(0)) } // makeHeader constructs the header for p. func (p *Packer) makeHeader() ([]byte, error) { - buf := make([]byte, 0, len(p.blobs)*int(EntrySize)) + buf := make([]byte, 0, len(p.blobs)*int(entrySize)) for _, b := range p.blobs { - switch b.Type { - case restic.DataBlob: + switch { + case b.Type == restic.DataBlob && b.UncompressedLength == 0: buf = append(buf, 0) - case restic.TreeBlob: + case b.Type == restic.TreeBlob && b.UncompressedLength == 0: buf = append(buf, 1) + case b.Type == restic.DataBlob && b.UncompressedLength != 0: + buf = append(buf, 2) + case b.Type == restic.TreeBlob && b.UncompressedLength != 0: + buf = append(buf, 3) default: return nil, errors.Errorf("invalid blob type %v", b.Type) } @@ -115,6 +131,10 @@ func (p *Packer) makeHeader() ([]byte, error) { var lenLE [4]byte binary.LittleEndian.PutUint32(lenLE[:], uint32(b.Length)) buf = append(buf, lenLE[:]...) + if b.UncompressedLength != 0 { + binary.LittleEndian.PutUint32(lenLE[:], uint32(b.UncompressedLength)) + buf = append(buf, lenLE[:]...) + } buf = append(buf, b.ID[:]...) } @@ -137,6 +157,13 @@ func (p *Packer) Count() int { return len(p.blobs) } +// HeaderFull returns true if the pack header is full. +func (p *Packer) HeaderFull() bool { + p.m.Lock() + defer p.m.Unlock() + return headerSize+uint(len(p.blobs)+1)*entrySize > MaxHeaderSize +} + // Blobs returns the slice of blobs that have been written. func (p *Packer) Blobs() []restic.Blob { p.m.Lock() @@ -151,30 +178,26 @@ func (p *Packer) String() string { var ( // we require at least one entry in the header, and one blob for a pack file - minFileSize = EntrySize + crypto.Extension + uint(headerLengthSize) + minFileSize = plainEntrySize + crypto.Extension + uint(headerLengthSize) ) const ( // size of the header-length field at the end of the file; it is a uint32 headerLengthSize = 4 - // HeaderSize is the header's constant overhead (independent of #entries) - HeaderSize = headerLengthSize + crypto.Extension + // headerSize is the header's constant overhead (independent of #entries) + headerSize = headerLengthSize + crypto.Extension - maxHeaderSize = 16 * 1024 * 1024 + // MaxHeaderSize is the max size of header including header-length field + MaxHeaderSize = 16*1024*1024 + headerLengthSize // number of header enries to download as part of header-length request eagerEntries = 15 ) -// readRecords reads up to max records from the underlying ReaderAt, returning -// the raw header, the total number of records in the header, and any error. -// If the header contains fewer than max entries, the header is truncated to +// readRecords reads up to bufsize bytes from the underlying ReaderAt, returning +// the raw header, the total number of bytes in the header, and any error. +// If the header contains fewer than bufsize bytes, the header is truncated to // the appropriate size. -func readRecords(rd io.ReaderAt, size int64, max int) ([]byte, int, error) { - var bufsize int - bufsize += max * int(EntrySize) - bufsize += crypto.Extension - bufsize += headerLengthSize - +func readRecords(rd io.ReaderAt, size int64, bufsize int) ([]byte, int, error) { if bufsize > int(size) { bufsize = int(size) } @@ -195,19 +218,17 @@ func readRecords(rd io.ReaderAt, size int64, max int) ([]byte, int, error) { err = InvalidFileError{Message: "header length is zero"} case hlen < crypto.Extension: err = InvalidFileError{Message: "header length is too small"} - case (hlen-crypto.Extension)%uint32(EntrySize) != 0: - err = InvalidFileError{Message: "header length is invalid"} case int64(hlen) > size-int64(headerLengthSize): err = InvalidFileError{Message: "header is larger than file"} - case int64(hlen) > maxHeaderSize: + case int64(hlen) > MaxHeaderSize-int64(headerLengthSize): err = InvalidFileError{Message: "header is larger than maxHeaderSize"} } if err != nil { return nil, 0, errors.Wrap(err, "readHeader") } - total := (int(hlen) - crypto.Extension) / int(EntrySize) - if total < max { + total := int(hlen + headerLengthSize) + if total < bufsize { // truncate to the beginning of the pack header b = b[len(b)-int(hlen):] } @@ -228,11 +249,12 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) { // eagerly download eagerEntries header entries as part of header-length request. // only make second request if actual number of entries is greater than eagerEntries - b, c, err := readRecords(rd, size, eagerEntries) + eagerSize := eagerEntries*int(entrySize) + headerSize + b, c, err := readRecords(rd, size, eagerSize) if err != nil { return nil, err } - if c <= eagerEntries { + if c <= eagerSize { // eager read sufficed, return what we got return b, nil } @@ -260,7 +282,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, hdr return nil, 0, err } - if len(buf) < k.NonceSize()+k.Overhead() { + if len(buf) < crypto.CiphertextLength(0) { return nil, 0, errors.New("invalid header, too small") } @@ -272,11 +294,12 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, hdr return nil, 0, err } - entries = make([]restic.Blob, 0, uint(len(buf))/EntrySize) + // might over allocate a bit if all blobs have EntrySize but only by a few percent + entries = make([]restic.Blob, 0, uint(len(buf))/plainEntrySize) pos := uint(0) for len(buf) > 0 { - entry, err := parseHeaderEntry(buf) + entry, headerSize, err := parseHeaderEntry(buf) if err != nil { return nil, 0, err } @@ -284,35 +307,79 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, hdr entries = append(entries, entry) pos += entry.Length - buf = buf[EntrySize:] + buf = buf[headerSize:] } return entries, hdrSize, nil } -// PackedSizeOfBlob returns the size a blob actually uses when saved in a pack -func PackedSizeOfBlob(blobLength uint) uint { - return blobLength + EntrySize -} - -func parseHeaderEntry(p []byte) (b restic.Blob, err error) { - if uint(len(p)) < EntrySize { +func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) { + l := uint(len(p)) + size = plainEntrySize + if l < plainEntrySize { err = errors.Errorf("parseHeaderEntry: buffer of size %d too short", len(p)) - return b, err + return b, size, err } - p = p[:EntrySize] + tpe := p[0] - switch p[0] { - case 0: + switch tpe { + case 0, 2: b.Type = restic.DataBlob - case 1: + case 1, 3: b.Type = restic.TreeBlob default: - return b, errors.Errorf("invalid type %d", p[0]) + return b, size, errors.Errorf("invalid type %d", tpe) } b.Length = uint(binary.LittleEndian.Uint32(p[1:5])) - copy(b.ID[:], p[5:]) + p = p[5:] + if tpe == 2 || tpe == 3 { + size = entrySize + if l < entrySize { + err = errors.Errorf("parseHeaderEntry: buffer of size %d too short", len(p)) + return b, size, err + } + b.UncompressedLength = uint(binary.LittleEndian.Uint32(p[0:4])) + p = p[4:] + } - return b, nil + copy(b.ID[:], p[:]) + + return b, size, nil +} + +func CalculateEntrySize(blob restic.Blob) int { + if blob.UncompressedLength != 0 { + return int(entrySize) + } + return int(plainEntrySize) +} + +func CalculateHeaderSize(blobs []restic.Blob) int { + size := headerSize + for _, blob := range blobs { + size += CalculateEntrySize(blob) + } + return size +} + +// Size returns the size of all packs computed by index information. +// If onlyHdr is set to true, only the size of the header is returned +// Note that this function only gives correct sizes, if there are no +// duplicates in the index. +func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.ID]int64 { + packSize := make(map[restic.ID]int64) + + for blob := range mi.Each(ctx) { + size, ok := packSize[blob.PackID] + if !ok { + size = headerSize + } + if !onlyHdr { + size += int64(blob.Length) + } + packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob)) + } + + return packSize } diff --git a/internal/pack/pack_internal_test.go b/internal/pack/pack_internal_test.go index b0a5d2862b9..c1a4867ea4d 100644 --- a/internal/pack/pack_internal_test.go +++ b/internal/pack/pack_internal_test.go @@ -13,7 +13,7 @@ import ( func TestParseHeaderEntry(t *testing.T) { h := headerEntry{ - Type: 0, // Blob. + Type: 0, // Blob Length: 100, } for i := range h.ID { @@ -23,25 +23,58 @@ func TestParseHeaderEntry(t *testing.T) { buf := new(bytes.Buffer) _ = binary.Write(buf, binary.LittleEndian, &h) - b, err := parseHeaderEntry(buf.Bytes()) + b, size, err := parseHeaderEntry(buf.Bytes()) rtest.OK(t, err) rtest.Equals(t, restic.DataBlob, b.Type) + rtest.Equals(t, plainEntrySize, size) t.Logf("%v %v", h.ID, b.ID) - rtest.Assert(t, bytes.Equal(h.ID[:], b.ID[:]), "id mismatch") + rtest.Equals(t, h.ID[:], b.ID[:]) rtest.Equals(t, uint(h.Length), b.Length) + rtest.Equals(t, uint(0), b.UncompressedLength) + + c := compressedHeaderEntry{ + Type: 2, // compressed Blob + Length: 100, + UncompressedLength: 200, + } + for i := range c.ID { + c.ID[i] = byte(i) + } + + buf = new(bytes.Buffer) + _ = binary.Write(buf, binary.LittleEndian, &c) + + b, size, err = parseHeaderEntry(buf.Bytes()) + rtest.OK(t, err) + rtest.Equals(t, restic.DataBlob, b.Type) + rtest.Equals(t, entrySize, size) + t.Logf("%v %v", c.ID, b.ID) + rtest.Equals(t, c.ID[:], b.ID[:]) + rtest.Equals(t, uint(c.Length), b.Length) + rtest.Equals(t, uint(c.UncompressedLength), b.UncompressedLength) +} + +func TestParseHeaderEntryErrors(t *testing.T) { + h := headerEntry{ + Type: 0, // Blob + Length: 100, + } + for i := range h.ID { + h.ID[i] = byte(i) + } h.Type = 0xae - buf.Reset() + buf := new(bytes.Buffer) _ = binary.Write(buf, binary.LittleEndian, &h) - b, err = parseHeaderEntry(buf.Bytes()) + _, _, err := parseHeaderEntry(buf.Bytes()) rtest.Assert(t, err != nil, "no error for invalid type") h.Type = 0 buf.Reset() _ = binary.Write(buf, binary.LittleEndian, &h) - b, err = parseHeaderEntry(buf.Bytes()[:EntrySize-1]) + _, _, err = parseHeaderEntry(buf.Bytes()[:plainEntrySize-1]) rtest.Assert(t, err != nil, "no error for short input") } @@ -58,7 +91,7 @@ func (rd *countingReaderAt) ReadAt(p []byte, off int64) (n int, err error) { func TestReadHeaderEagerLoad(t *testing.T) { testReadHeader := func(dataSize, entryCount, expectedReadInvocationCount int) { - expectedHeader := rtest.Random(0, entryCount*int(EntrySize)+crypto.Extension) + expectedHeader := rtest.Random(0, entryCount*int(entrySize)+crypto.Extension) buf := &bytes.Buffer{} buf.Write(rtest.Random(0, dataSize)) // pack blobs data @@ -83,8 +116,8 @@ func TestReadHeaderEagerLoad(t *testing.T) { testReadHeader(100, eagerEntries+1, 2) // file size == eager header load size - eagerLoadSize := int((eagerEntries * EntrySize) + crypto.Extension) - headerSize := int(1*EntrySize) + crypto.Extension + eagerLoadSize := int((eagerEntries * entrySize) + crypto.Extension) + headerSize := int(1*entrySize) + crypto.Extension dataSize := eagerLoadSize - headerSize - binary.Size(uint32(0)) testReadHeader(dataSize-1, 1, 1) testReadHeader(dataSize, 1, 1) @@ -96,8 +129,9 @@ func TestReadHeaderEagerLoad(t *testing.T) { func TestReadRecords(t *testing.T) { testReadRecords := func(dataSize, entryCount, totalRecords int) { - totalHeader := rtest.Random(0, totalRecords*int(EntrySize)+crypto.Extension) - off := len(totalHeader) - (entryCount*int(EntrySize) + crypto.Extension) + totalHeader := rtest.Random(0, totalRecords*int(entrySize)+crypto.Extension) + bufSize := entryCount*int(entrySize) + crypto.Extension + off := len(totalHeader) - bufSize if off < 0 { off = 0 } @@ -110,10 +144,10 @@ func TestReadRecords(t *testing.T) { rd := bytes.NewReader(buf.Bytes()) - header, count, err := readRecords(rd, int64(rd.Len()), entryCount) + header, count, err := readRecords(rd, int64(rd.Len()), bufSize+4) rtest.OK(t, err) + rtest.Equals(t, len(totalHeader)+4, count) rtest.Equals(t, expectedHeader, header) - rtest.Equals(t, totalRecords, count) } // basic @@ -127,8 +161,8 @@ func TestReadRecords(t *testing.T) { testReadRecords(100, eagerEntries, eagerEntries+1) // file size == eager header load size - eagerLoadSize := int((eagerEntries * EntrySize) + crypto.Extension) - headerSize := int(1*EntrySize) + crypto.Extension + eagerLoadSize := int((eagerEntries * entrySize) + crypto.Extension) + headerSize := int(1*entrySize) + crypto.Extension dataSize := eagerLoadSize - headerSize - binary.Size(uint32(0)) testReadRecords(dataSize-1, 1, 1) testReadRecords(dataSize, 1, 1) diff --git a/internal/pack/pack_test.go b/internal/pack/pack_test.go index c789e472b79..3f7077390bb 100644 --- a/internal/pack/pack_test.go +++ b/internal/pack/pack_test.go @@ -5,11 +5,11 @@ import ( "context" "crypto/rand" "crypto/sha256" - "encoding/binary" "encoding/json" "io" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/pack" @@ -39,11 +39,11 @@ func newPack(t testing.TB, k *crypto.Key, lengths []int) ([]Buf, []byte, uint) { var buf bytes.Buffer p := pack.NewPacker(k, &buf) for _, b := range bufs { - _, err := p.Add(restic.TreeBlob, b.id, b.data) + _, err := p.Add(restic.TreeBlob, b.id, b.data, 2*len(b.data)) rtest.OK(t, err) } - _, err := p.Finalize() + err := p.Finalize() rtest.OK(t, err) return bufs, buf.Bytes(), p.Size() @@ -54,17 +54,18 @@ func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReaderAt, packSi for _, buf := range bufs { written += len(buf.data) } - // header length + header + header crypto - headerSize := binary.Size(uint32(0)) + restic.CiphertextLength(len(bufs)*int(pack.EntrySize)) - written += headerSize - - // check length - rtest.Equals(t, uint(written), packSize) // read and parse it again entries, hdrSize, err := pack.List(k, rd, int64(packSize)) rtest.OK(t, err) rtest.Equals(t, len(entries), len(bufs)) + + // check the head size calculation for consistency + headerSize := pack.CalculateHeaderSize(entries) + written += headerSize + + // check length + rtest.Equals(t, uint(written), packSize) rtest.Equals(t, headerSize, int(hdrSize)) var buf []byte @@ -128,7 +129,7 @@ func TestUnpackReadSeeker(t *testing.T) { handle := restic.Handle{Type: restic.PackFile, Name: id.String()} rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) - verifyBlobs(t, bufs, k, restic.ReaderAt(context.TODO(), b, handle), packSize) + verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) } func TestShortPack(t *testing.T) { @@ -141,5 +142,5 @@ func TestShortPack(t *testing.T) { handle := restic.Handle{Type: restic.PackFile, Name: id.String()} rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) - verifyBlobs(t, bufs, k, restic.ReaderAt(context.TODO(), b, handle), packSize) + verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) } diff --git a/internal/repository/doc.go b/internal/repository/doc.go index cb98334c4eb..9e5a2c2bcbc 100644 --- a/internal/repository/doc.go +++ b/internal/repository/doc.go @@ -2,7 +2,7 @@ // the following the abstractions used for this package are listed. More // information can be found in the restic design document. // -// File +// # File // // A file is a named handle for some data saved in the backend. For the local // backend, this corresponds to actual files saved to disk. Usually, the SHA256 @@ -11,18 +11,17 @@ // encrypted before being saved in a backend. This means that the name is the // hash of the ciphertext. // -// Blob +// # Blob // // A blob is a number of bytes that has a type (data or tree). Blobs are // identified by an ID, which is the SHA256 hash of the blobs' contents. One or // more blobs are bundled together in a Pack and then saved to the backend. // Blobs are always encrypted before being bundled in a Pack. // -// Pack +// # Pack // // A Pack is a File in the backend that contains one or more (encrypted) blobs, // followed by a header at the end of the Pack. The header is encrypted and // contains the ID, type, length and offset for each blob contained in the // Pack. -// package repository diff --git a/internal/repository/fuzz_test.go b/internal/repository/fuzz_test.go new file mode 100644 index 00000000000..7a98477b6bf --- /dev/null +++ b/internal/repository/fuzz_test.go @@ -0,0 +1,47 @@ +//go:build go1.18 +// +build go1.18 + +package repository + +import ( + "context" + "testing" + + "github.com/restic/restic/internal/backend/mem" + "github.com/restic/restic/internal/restic" + "golang.org/x/sync/errgroup" +) + +// Test saving a blob and loading it again, with varying buffer sizes. +// Also a regression test for #3783. +func FuzzSaveLoadBlob(f *testing.F) { + f.Fuzz(func(t *testing.T, blob []byte, buflen uint) { + if buflen > 64<<20 { + // Don't allocate enormous buffers. We're not testing the allocator. + t.Skip() + } + + id := restic.Hash(blob) + repo, _ := TestRepositoryWithBackend(t, mem.New(), 2) + + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + + _, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, blob, id, false) + if err != nil { + t.Fatal(err) + } + err = repo.Flush(context.TODO()) + if err != nil { + t.Fatal(err) + } + + buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, make([]byte, buflen)) + if err != nil { + t.Fatal(err) + } + if restic.Hash(buf) != id { + t.Fatal("mismatch") + } + }) +} diff --git a/internal/repository/index.go b/internal/repository/index.go index fdd57b052ca..28863436b62 100644 --- a/internal/repository/index.go +++ b/internal/repository/index.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -46,8 +47,6 @@ type Index struct { byType [restic.NumBlobTypes]indexMap packs restic.IDs mixedPacks restic.IDSet - // only used by Store, StorePacks does not check for already saved packIDs - packIDToIndex map[restic.ID]int final bool // set to true for all indexes read from the backend ("finalized") ids restic.IDs // set to the IDs of the contained finalized indexes @@ -58,9 +57,8 @@ type Index struct { // NewIndex returns a new index. func NewIndex() *Index { return &Index{ - packIDToIndex: make(map[restic.ID]int), - mixedPacks: restic.NewIDSet(), - created: time.Now(), + mixedPacks: restic.NewIDSet(), + created: time.Now(), } } @@ -75,12 +73,12 @@ const maxuint32 = 1<<32 - 1 func (idx *Index) store(packIndex int, blob restic.Blob) { // assert that offset and length fit into uint32! - if blob.Offset > maxuint32 || blob.Length > maxuint32 { + if blob.Offset > maxuint32 || blob.Length > maxuint32 || blob.UncompressedLength > maxuint32 { panic("offset or length does not fit in uint32. You have packs > 4GB!") } m := &idx.byType[blob.Type] - m.add(blob.ID, packIndex, uint32(blob.Offset), uint32(blob.Length)) + m.add(blob.ID, packIndex, uint32(blob.Offset), uint32(blob.Length), uint32(blob.UncompressedLength)) } // Final returns true iff the index is already written to the repository, it is @@ -93,12 +91,13 @@ func (idx *Index) Final() bool { } const ( - indexMaxBlobs = 50000 - indexMaxAge = 10 * time.Minute + indexMaxBlobs = 50000 + indexMaxBlobsCompressed = 3 * indexMaxBlobs + indexMaxAge = 10 * time.Minute ) // IndexFull returns true iff the index is "full enough" to be saved as a preliminary index. -var IndexFull = func(idx *Index) bool { +var IndexFull = func(idx *Index, compress bool) bool { idx.m.Lock() defer idx.m.Unlock() @@ -109,12 +108,18 @@ var IndexFull = func(idx *Index) bool { blobs += idx.byType[typ].len() } age := time.Since(idx.created) + var maxBlobs uint + if compress { + maxBlobs = indexMaxBlobsCompressed + } else { + maxBlobs = indexMaxBlobs + } switch { case age >= indexMaxAge: debug.Log("index %p is old enough", idx, age) return true - case blobs >= indexMaxBlobs: + case blobs >= maxBlobs: debug.Log("index %p has %d blobs", idx, blobs) return true } @@ -124,27 +129,6 @@ var IndexFull = func(idx *Index) bool { } -// Store remembers the id and pack in the index. -func (idx *Index) Store(pb restic.PackedBlob) { - idx.m.Lock() - defer idx.m.Unlock() - - if idx.final { - panic("store new item in finalized index") - } - - debug.Log("%v", pb) - - // get packIndex and save if new packID - packIndex, ok := idx.packIDToIndex[pb.PackID] - if !ok { - packIndex = idx.addToPacks(pb.PackID) - idx.packIDToIndex[pb.PackID] = packIndex - } - - idx.store(packIndex, pb.Blob) -} - // StorePack remembers the ids of all blobs of a given pack // in the index func (idx *Index) StorePack(id restic.ID, blobs []restic.Blob) { @@ -169,8 +153,9 @@ func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) restic.PackedBl BlobHandle: restic.BlobHandle{ ID: e.id, Type: t}, - Length: uint(e.length), - Offset: uint(e.offset), + Length: uint(e.length), + Offset: uint(e.offset), + UncompressedLength: uint(e.uncompressedLength), }, PackID: idx.packs[e.packIndex], } @@ -189,24 +174,6 @@ func (idx *Index) Lookup(bh restic.BlobHandle, pbs []restic.PackedBlob) []restic return pbs } -// ListPack returns a list of blobs contained in a pack. -func (idx *Index) ListPack(id restic.ID) (pbs []restic.PackedBlob) { - idx.m.Lock() - defer idx.m.Unlock() - - for typ := range idx.byType { - m := &idx.byType[typ] - m.foreach(func(e *indexEntry) bool { - if idx.packs[e.packIndex] == id { - pbs = append(pbs, idx.toPackedBlob(e, restic.BlobType(typ))) - } - return true - }) - } - - return pbs -} - // Has returns true iff the id is listed in the index. func (idx *Index) Has(bh restic.BlobHandle) bool { idx.m.Lock() @@ -225,7 +192,10 @@ func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found if e == nil { return 0, false } - return uint(restic.PlaintextLength(int(e.length))), true + if e.uncompressedLength != 0 { + return uint(e.uncompressedLength), true + } + return uint(crypto.PlaintextLength(int(e.length))), true } // Supersedes returns the list of indexes this index supersedes, if any. @@ -278,8 +248,8 @@ func (idx *Index) Each(ctx context.Context) <-chan restic.PackedBlob { } type EachByPackResult struct { - packID restic.ID - blobs []restic.Blob + PackID restic.ID + Blobs []restic.Blob } // EachByPack returns a channel that yields all blobs known to the index @@ -300,29 +270,35 @@ func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <- close(ch) }() + byPack := make(map[restic.ID][][]*indexEntry) + for typ := range idx.byType { - byPack := make(map[restic.ID][]*indexEntry) m := &idx.byType[typ] m.foreach(func(e *indexEntry) bool { packID := idx.packs[e.packIndex] if !idx.final || !packBlacklist.Has(packID) { - byPack[packID] = append(byPack[packID], e) + if _, ok := byPack[packID]; !ok { + byPack[packID] = make([][]*indexEntry, restic.NumBlobTypes) + } + byPack[packID][typ] = append(byPack[packID][typ], e) } return true }) + } - for packID, pack := range byPack { - var result EachByPackResult - result.packID = packID + for packID, packByType := range byPack { + var result EachByPackResult + result.PackID = packID + for typ, pack := range packByType { for _, e := range pack { - result.blobs = append(result.blobs, idx.toPackedBlob(e, restic.BlobType(typ)).Blob) - } - select { - case <-ctx.Done(): - return - case ch <- result: + result.Blobs = append(result.Blobs, idx.toPackedBlob(e, restic.BlobType(typ)).Blob) } } + select { + case <-ctx.Done(): + return + case ch <- result: + } } }() @@ -342,25 +318,17 @@ func (idx *Index) Packs() restic.IDSet { return packs } -// Count returns the number of blobs of type t in the index. -func (idx *Index) Count(t restic.BlobType) (n uint) { - debug.Log("counting blobs of type %v", t) - idx.m.Lock() - defer idx.m.Unlock() - - return idx.byType[t].len() -} - type packJSON struct { ID restic.ID `json:"id"` Blobs []blobJSON `json:"blobs"` } type blobJSON struct { - ID restic.ID `json:"id"` - Type restic.BlobType `json:"type"` - Offset uint `json:"offset"` - Length uint `json:"length"` + ID restic.ID `json:"id"` + Type restic.BlobType `json:"type"` + Offset uint `json:"offset"` + Length uint `json:"length"` + UncompressedLength uint `json:"uncompressed_length,omitempty"` } // generatePackList returns a list of packs. @@ -391,10 +359,11 @@ func (idx *Index) generatePackList() ([]*packJSON, error) { // add blob p.Blobs = append(p.Blobs, blobJSON{ - ID: e.id, - Type: restic.BlobType(typ), - Offset: uint(e.offset), - Length: uint(e.length), + ID: e.id, + Type: restic.BlobType(typ), + Offset: uint(e.offset), + Length: uint(e.length), + UncompressedLength: uint(e.uncompressedLength), }) return true @@ -417,13 +386,6 @@ func (idx *Index) Encode(w io.Writer) error { idx.m.Lock() defer idx.m.Unlock() - return idx.encode(w) -} - -// encode writes the JSON serialization of the index to the writer w. -func (idx *Index) encode(w io.Writer) error { - debug.Log("encoding index") - list, err := idx.generatePackList() if err != nil { return err @@ -444,8 +406,6 @@ func (idx *Index) Finalize() { defer idx.m.Unlock() idx.final = true - // clear packIDToIndex as no more elements will be added - idx.packIDToIndex = nil } // IDs returns the IDs of the index, if available. If the index is not yet @@ -543,7 +503,7 @@ func (idx *Index) merge(idx2 *Index) error { m.foreachWithID(e2.id, func(e *indexEntry) { b := idx.toPackedBlob(e, restic.BlobType(typ)) b2 := idx2.toPackedBlob(e2, restic.BlobType(typ)) - if b.Length == b2.Length && b.Offset == b2.Offset && b.PackID == b2.PackID { + if b == b2 { found = true } }) @@ -553,7 +513,7 @@ func (idx *Index) merge(idx2 *Index) error { m2.foreach(func(e2 *indexEntry) bool { if !hasIdenticalEntry(e2) { // packIndex needs to be changed as idx2.pack was appended to idx.pack, see above - m.add(e2.id, e2.packIndex+packlen, e2.offset, e2.length) + m.add(e2.id, e2.packIndex+packlen, e2.offset, e2.length, e2.uncompressedLength) } return true }) @@ -601,8 +561,9 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, oldFormat bool, err erro BlobHandle: restic.BlobHandle{ Type: blob.Type, ID: blob.ID}, - Offset: blob.Offset, - Length: blob.Length, + Offset: blob.Offset, + Length: blob.Length, + UncompressedLength: blob.UncompressedLength, }) switch blob.Type { @@ -648,6 +609,7 @@ func decodeOldIndex(buf []byte) (idx *Index, err error) { ID: blob.ID}, Offset: blob.Offset, Length: blob.Length, + // no compressed length in the old index format }) switch blob.Type { diff --git a/internal/repository/index_parallel.go b/internal/repository/index_parallel.go index ffe03d42731..dcf33113e3c 100644 --- a/internal/repository/index_parallel.go +++ b/internal/repository/index_parallel.go @@ -2,6 +2,7 @@ package repository import ( "context" + "runtime" "sync" "github.com/restic/restic/internal/debug" @@ -9,8 +10,6 @@ import ( "golang.org/x/sync/errgroup" ) -const loadIndexParallelism = 5 - // ForAllIndexes loads all index files in parallel and calls the given callback. // It is guaranteed that the function is not run concurrently. If the callback // returns an error, this function is cancelled and also returns that error. @@ -37,7 +36,7 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository, return repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { select { case <-ctx.Done(): - return nil + return ctx.Err() case ch <- FileInfo{id, size}: } return nil @@ -53,7 +52,7 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository, var idx *Index oldFormat := false - buf, err = repo.LoadAndDecrypt(ctx, buf[:0], restic.IndexFile, fi.ID) + buf, err = repo.LoadUnpacked(ctx, restic.IndexFile, fi.ID, buf[:0]) if err == nil { idx, oldFormat, err = DecodeIndex(buf, fi.ID) } @@ -68,10 +67,13 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository, return nil } + // decoding an index can take quite some time such that this can be both CPU- or IO-bound + // as the whole index is kept in memory anyways, a few workers too much don't matter + workerCount := int(repo.Connections()) + runtime.GOMAXPROCS(0) // run workers on ch - wg.Go(func() error { - return RunWorkers(loadIndexParallelism, worker) - }) + for i := 0; i < workerCount; i++ { + wg.Go(worker) + } return wg.Wait() } diff --git a/internal/repository/index_test.go b/internal/repository/index_test.go index c4f0179dbab..6bc7afca27b 100644 --- a/internal/repository/index_test.go +++ b/internal/repository/index_test.go @@ -2,6 +2,7 @@ package repository_test import ( "bytes" + "context" "math/rand" "sync" "testing" @@ -19,22 +20,30 @@ func TestIndexSerialize(t *testing.T) { // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { packID := restic.NewRandomID() + var blobs []restic.Blob pos := uint(0) for j := 0; j < 20; j++ { length := uint(i*100 + j) + uncompressedLength := uint(0) + if i >= 25 { + // test a mix of compressed and uncompressed packs + uncompressedLength = 2 * length + } pb := restic.PackedBlob{ Blob: restic.Blob{ - BlobHandle: restic.NewRandomBlobHandle(), - Offset: pos, - Length: length, + BlobHandle: restic.NewRandomBlobHandle(), + Offset: pos, + Length: length, + UncompressedLength: uncompressedLength, }, PackID: packID, } - idx.Store(pb) + blobs = append(blobs, pb.Blob) tests = append(tests, pb) pos += length } + idx.StorePack(packID, blobs) } wr := bytes.NewBuffer(nil) @@ -77,6 +86,7 @@ func TestIndexSerialize(t *testing.T) { newtests := []restic.PackedBlob{} for i := 0; i < 10; i++ { packID := restic.NewRandomID() + var blobs []restic.Blob pos := uint(0) for j := 0; j < 10; j++ { @@ -89,10 +99,11 @@ func TestIndexSerialize(t *testing.T) { }, PackID: packID, } - idx.Store(pb) + blobs = append(blobs, pb.Blob) newtests = append(newtests, pb) pos += length } + idx.StorePack(packID, blobs) } // finalize; serialize idx, unserialize to idx3 @@ -135,24 +146,23 @@ func TestIndexSize(t *testing.T) { idx := repository.NewIndex() packs := 200 - blobs := 100 + blobCount := 100 for i := 0; i < packs; i++ { packID := restic.NewRandomID() + var blobs []restic.Blob pos := uint(0) - for j := 0; j < blobs; j++ { + for j := 0; j < blobCount; j++ { length := uint(i*100 + j) - idx.Store(restic.PackedBlob{ - Blob: restic.Blob{ - BlobHandle: restic.NewRandomBlobHandle(), - Offset: pos, - Length: length, - }, - PackID: packID, + blobs = append(blobs, restic.Blob{ + BlobHandle: restic.NewRandomBlobHandle(), + Offset: pos, + Length: length, }) pos += length } + idx.StorePack(packID, blobs) } wr := bytes.NewBuffer(nil) @@ -160,11 +170,11 @@ func TestIndexSize(t *testing.T) { err := idx.Encode(wr) rtest.OK(t, err) - t.Logf("Index file size for %d blobs in %d packs is %d", blobs*packs, packs, wr.Len()) + t.Logf("Index file size for %d blobs in %d packs is %d", blobCount*packs, packs, wr.Len()) } // example index serialization from doc/Design.rst -var docExample = []byte(` +var docExampleV1 = []byte(` { "supersedes": [ "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452" @@ -177,12 +187,12 @@ var docExample = []byte(` "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", "type": "data", "offset": 0, - "length": 25 + "length": 38 },{ "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", "type": "tree", "offset": 38, - "length": 100 + "length": 112 }, { "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", @@ -196,6 +206,41 @@ var docExample = []byte(` } `) +var docExampleV2 = []byte(` +{ + "supersedes": [ + "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452" + ], + "packs": [ + { + "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c", + "blobs": [ + { + "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", + "type": "data", + "offset": 0, + "length": 38 + }, + { + "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", + "type": "tree", + "offset": 38, + "length": 112, + "uncompressed_length": 511 + }, + { + "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", + "type": "data", + "offset": 150, + "length": 123, + "uncompressed_length": 234 + } + ] + } + ] + } +`) + var docOldExample = []byte(` [ { "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c", @@ -204,12 +249,12 @@ var docOldExample = []byte(` "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", "type": "data", "offset": 0, - "length": 25 + "length": 38 },{ "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", "type": "tree", "offset": 38, - "length": 100 + "length": 112 }, { "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", @@ -222,22 +267,23 @@ var docOldExample = []byte(` `) var exampleTests = []struct { - id, packID restic.ID - tpe restic.BlobType - offset, length uint + id, packID restic.ID + tpe restic.BlobType + offset, length uint + uncompressedLength uint }{ { restic.TestParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"), restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), - restic.DataBlob, 0, 25, + restic.DataBlob, 0, 38, 0, }, { restic.TestParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"), restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), - restic.TreeBlob, 38, 100, + restic.TreeBlob, 38, 112, 511, }, { restic.TestParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"), restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), - restic.DataBlob, 150, 123, + restic.DataBlob, 150, 123, 234, }, } @@ -254,43 +300,67 @@ var exampleLookupTest = struct { } func TestIndexUnserialize(t *testing.T) { - oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")} - - idx, oldFormat, err := repository.DecodeIndex(docExample, restic.NewRandomID()) - rtest.OK(t, err) - rtest.Assert(t, !oldFormat, "new index format recognized as old format") - - for _, test := range exampleTests { - list := idx.Lookup(restic.BlobHandle{ID: test.id, Type: test.tpe}, nil) - if len(list) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", test.id.Str(), len(list), list) + for _, task := range []struct { + idxBytes []byte + version int + }{ + {docExampleV1, 1}, + {docExampleV2, 2}, + } { + oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")} + + idx, oldFormat, err := repository.DecodeIndex(task.idxBytes, restic.NewRandomID()) + rtest.OK(t, err) + rtest.Assert(t, !oldFormat, "new index format recognized as old format") + + for _, test := range exampleTests { + list := idx.Lookup(restic.BlobHandle{ID: test.id, Type: test.tpe}, nil) + if len(list) != 1 { + t.Errorf("expected one result for blob %v, got %v: %v", test.id.Str(), len(list), list) + } + blob := list[0] + + t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob) + + rtest.Equals(t, test.packID, blob.PackID) + rtest.Equals(t, test.tpe, blob.Type) + rtest.Equals(t, test.offset, blob.Offset) + rtest.Equals(t, test.length, blob.Length) + if task.version == 1 { + rtest.Equals(t, uint(0), blob.UncompressedLength) + } else if task.version == 2 { + rtest.Equals(t, test.uncompressedLength, blob.UncompressedLength) + } else { + t.Fatal("Invalid index version") + } } - blob := list[0] - - t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob) - rtest.Equals(t, test.packID, blob.PackID) - rtest.Equals(t, test.tpe, blob.Type) - rtest.Equals(t, test.offset, blob.Offset) - rtest.Equals(t, test.length, blob.Length) - } + rtest.Equals(t, oldIdx, idx.Supersedes()) - rtest.Equals(t, oldIdx, idx.Supersedes()) + blobs := listPack(idx, exampleLookupTest.packID) + if len(blobs) != len(exampleLookupTest.blobs) { + t.Fatalf("expected %d blobs in pack, got %d", len(exampleLookupTest.blobs), len(blobs)) + } - blobs := idx.ListPack(exampleLookupTest.packID) - if len(blobs) != len(exampleLookupTest.blobs) { - t.Fatalf("expected %d blobs in pack, got %d", len(exampleLookupTest.blobs), len(blobs)) + for _, blob := range blobs { + b, ok := exampleLookupTest.blobs[blob.ID] + if !ok { + t.Errorf("unexpected blob %v found", blob.ID.Str()) + } + if blob.Type != b { + t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type) + } + } } +} - for _, blob := range blobs { - b, ok := exampleLookupTest.blobs[blob.ID] - if !ok { - t.Errorf("unexpected blob %v found", blob.ID.Str()) - } - if blob.Type != b { - t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type) +func listPack(idx *repository.Index, id restic.ID) (pbs []restic.PackedBlob) { + for pb := range idx.Each(context.TODO()) { + if pb.PackID.Equal(id) { + pbs = append(pbs, pb) } } + return pbs } var ( @@ -362,13 +432,12 @@ func TestIndexPacks(t *testing.T) { for i := 0; i < 20; i++ { packID := restic.NewRandomID() - idx.Store(restic.PackedBlob{ - Blob: restic.Blob{ + idx.StorePack(packID, []restic.Blob{ + { BlobHandle: restic.NewRandomBlobHandle(), Offset: 0, Length: 23, }, - PackID: packID, }) packs.Insert(packID) @@ -403,8 +472,9 @@ func createRandomIndex(rng *rand.Rand, packfiles int) (idx *repository.Index, lo Type: restic.DataBlob, ID: id, }, - Length: uint(size), - Offset: uint(offset), + Length: uint(size), + UncompressedLength: uint(2 * size), + Offset: uint(offset), }) offset += size @@ -471,22 +541,30 @@ func TestIndexHas(t *testing.T) { // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { packID := restic.NewRandomID() + var blobs []restic.Blob pos := uint(0) for j := 0; j < 20; j++ { length := uint(i*100 + j) + uncompressedLength := uint(0) + if i >= 25 { + // test a mix of compressed and uncompressed packs + uncompressedLength = 2 * length + } pb := restic.PackedBlob{ Blob: restic.Blob{ - BlobHandle: restic.NewRandomBlobHandle(), - Offset: pos, - Length: length, + BlobHandle: restic.NewRandomBlobHandle(), + Offset: pos, + Length: length, + UncompressedLength: uncompressedLength, }, PackID: packID, } - idx.Store(pb) + blobs = append(blobs, pb.Blob) tests = append(tests, pb) pos += length } + idx.StorePack(packID, blobs) } for _, testBlob := range tests { @@ -496,3 +574,76 @@ func TestIndexHas(t *testing.T) { rtest.Assert(t, !idx.Has(restic.NewRandomBlobHandle()), "Index reports having a data blob not added to it") rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob") } + +func TestMixedEachByPack(t *testing.T) { + idx := repository.NewIndex() + + expected := make(map[restic.ID]int) + // create 50 packs with 2 blobs each + for i := 0; i < 50; i++ { + packID := restic.NewRandomID() + expected[packID] = 1 + blobs := []restic.Blob{ + { + BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()}, + Offset: 0, + Length: 42, + }, + { + BlobHandle: restic.BlobHandle{Type: restic.TreeBlob, ID: restic.NewRandomID()}, + Offset: 42, + Length: 43, + }, + } + idx.StorePack(packID, blobs) + } + + reported := make(map[restic.ID]int) + for bp := range idx.EachByPack(context.TODO(), restic.NewIDSet()) { + reported[bp.PackID]++ + + rtest.Equals(t, 2, len(bp.Blobs)) // correct blob count + if bp.Blobs[0].Offset > bp.Blobs[1].Offset { + bp.Blobs[1], bp.Blobs[0] = bp.Blobs[0], bp.Blobs[1] + } + b0 := bp.Blobs[0] + rtest.Assert(t, b0.Type == restic.DataBlob && b0.Offset == 0 && b0.Length == 42, "wrong blob", b0) + b1 := bp.Blobs[1] + rtest.Assert(t, b1.Type == restic.TreeBlob && b1.Offset == 42 && b1.Length == 43, "wrong blob", b1) + } + rtest.Equals(t, expected, reported) +} + +func TestEachByPackIgnoes(t *testing.T) { + idx := repository.NewIndex() + + ignores := restic.NewIDSet() + expected := make(map[restic.ID]int) + // create 50 packs with one blob each + for i := 0; i < 50; i++ { + packID := restic.NewRandomID() + if i < 3 { + ignores.Insert(packID) + } else { + expected[packID] = 1 + } + blobs := []restic.Blob{ + { + BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()}, + Offset: 0, + Length: 42, + }, + } + idx.StorePack(packID, blobs) + } + idx.Finalize() + + reported := make(map[restic.ID]int) + for bp := range idx.EachByPack(context.TODO(), ignores) { + reported[bp.PackID]++ + rtest.Equals(t, 1, len(bp.Blobs)) // correct blob count + b0 := bp.Blobs[0] + rtest.Assert(t, b0.Type == restic.DataBlob && b0.Offset == 0 && b0.Length == 42, "wrong blob", b0) + } + rtest.Equals(t, expected, reported) +} diff --git a/internal/repository/indexmap.go b/internal/repository/indexmap.go index f713a33045f..99c3fd33171 100644 --- a/internal/repository/indexmap.go +++ b/internal/repository/indexmap.go @@ -32,7 +32,7 @@ const ( // add inserts an indexEntry for the given arguments into the map, // using id as the key. -func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32) { +func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32, uncompressedLength uint32) { switch { case m.numentries == 0: // Lazy initialization. m.init() @@ -47,6 +47,7 @@ func (m *indexMap) add(id restic.ID, packIdx int, offset, length uint32) { e.packIndex = packIdx e.offset = offset e.length = length + e.uncompressedLength = uncompressedLength m.buckets[h] = e m.numentries++ @@ -129,32 +130,41 @@ func (m *indexMap) init() { func (m *indexMap) len() uint { return m.numentries } func (m *indexMap) newEntry() *indexEntry { - // Allocating in batches means that we get closer to optimal space usage, - // as Go's malloc will overallocate for structures of size 56 (indexEntry - // on amd64). + // We keep a free list of objects to speed up allocation and GC. + // There's an obvious trade-off here: allocating in larger batches + // means we allocate faster and the GC has to keep fewer bits to track + // what we have in use, but it means we waste some space. // - // 256*56 and 256*48 both have minimal malloc overhead among reasonable sizes. - // See src/runtime/sizeclasses.go in the standard library. - const entryAllocBatch = 256 + // Then again, allocating each indexEntry separately also wastes space + // on 32-bit platforms, because the Go malloc has no size class for + // exactly 52 bytes, so it puts the indexEntry in a 64-byte slot instead. + // See src/runtime/sizeclasses.go in the Go source repo. + // + // The batch size of 4 means we hit the size classes for 4×64=256 bytes + // (64-bit) and 4×52=208 bytes (32-bit), wasting nothing in malloc on + // 64-bit and relatively little on 32-bit. + const entryAllocBatch = 4 - if m.free == nil { + e := m.free + if e != nil { + m.free = e.next + } else { free := new([entryAllocBatch]indexEntry) - for i := range free[:len(free)-1] { + e = &free[0] + for i := 1; i < len(free)-1; i++ { free[i].next = &free[i+1] } - m.free = &free[0] + m.free = &free[1] } - e := m.free - m.free = m.free.next - return e } type indexEntry struct { - id restic.ID - next *indexEntry - packIndex int // Position in containing Index's packs field. - offset uint32 - length uint32 + id restic.ID + next *indexEntry + packIndex int // Position in containing Index's packs field. + offset uint32 + length uint32 + uncompressedLength uint32 } diff --git a/internal/repository/indexmap_test.go b/internal/repository/indexmap_test.go index d803bf3c52f..6699b360164 100644 --- a/internal/repository/indexmap_test.go +++ b/internal/repository/indexmap_test.go @@ -22,7 +22,7 @@ func TestIndexMapBasic(t *testing.T) { r.Read(id[:]) rtest.Assert(t, m.get(id) == nil, "%v retrieved but not added", id) - m.add(id, 0, 0, 0) + m.add(id, 0, 0, 0, 0) rtest.Assert(t, m.get(id) != nil, "%v added but not retrieved", id) rtest.Equals(t, uint(i), m.len()) } @@ -41,7 +41,7 @@ func TestIndexMapForeach(t *testing.T) { for i := 0; i < N; i++ { var id restic.ID id[0] = byte(i) - m.add(id, i, uint32(i), uint32(i)) + m.add(id, i, uint32(i), uint32(i), uint32(i/2)) } seen := make(map[int]struct{}) @@ -51,6 +51,7 @@ func TestIndexMapForeach(t *testing.T) { rtest.Equals(t, i, e.packIndex) rtest.Equals(t, i, int(e.length)) rtest.Equals(t, i, int(e.offset)) + rtest.Equals(t, i/2, int(e.uncompressedLength)) seen[i] = struct{}{} return true @@ -85,13 +86,13 @@ func TestIndexMapForeachWithID(t *testing.T) { // Test insertion and retrieval of duplicates. for i := 0; i < ndups; i++ { - m.add(id, i, 0, 0) + m.add(id, i, 0, 0, 0) } for i := 0; i < 100; i++ { var otherid restic.ID r.Read(otherid[:]) - m.add(otherid, -1, 0, 0) + m.add(otherid, -1, 0, 0, 0) } n = 0 @@ -109,7 +110,7 @@ func TestIndexMapForeachWithID(t *testing.T) { func BenchmarkIndexMapHash(b *testing.B) { var m indexMap - m.add(restic.ID{}, 0, 0, 0) // Trigger lazy initialization. + m.add(restic.ID{}, 0, 0, 0, 0) // Trigger lazy initialization. ids := make([]restic.ID, 128) // 4 KiB. r := rand.New(rand.NewSource(time.Now().UnixNano())) diff --git a/internal/repository/key.go b/internal/repository/key.go index 5de154195ca..4ce59a1f554 100644 --- a/internal/repository/key.go +++ b/internal/repository/key.go @@ -137,6 +137,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int, // try at most maxKeys keys in repo err = s.Backend().List(listCtx, restic.KeyFile, func(fi restic.FileInfo) error { + checked++ if maxKeys > 0 && checked > maxKeys { return ErrMaxKeysReached } @@ -153,7 +154,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int, debug.Log("key %v returned error %v", fi.Name, err) // ErrUnauthenticated means the password is wrong, try the next key - if errors.Cause(err) == crypto.ErrUnauthenticated { + if errors.Is(err, crypto.ErrUnauthenticated) { return nil } @@ -262,7 +263,7 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str } nonce := crypto.NewRandomNonce() - ciphertext := make([]byte, 0, len(buf)+newkey.user.Overhead()+newkey.user.NonceSize()) + ciphertext := make([]byte, 0, crypto.CiphertextLength(len(buf))) ciphertext = append(ciphertext, nonce...) ciphertext = newkey.user.Seal(ciphertext, nonce, buf, nil) newkey.Data = ciphertext diff --git a/internal/repository/master_index.go b/internal/repository/master_index.go index 5e099a3a580..955080e8224 100644 --- a/internal/repository/master_index.go +++ b/internal/repository/master_index.go @@ -1,12 +1,13 @@ package repository import ( + "bytes" "context" "fmt" + "runtime" "sync" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" @@ -17,6 +18,7 @@ type MasterIndex struct { idx []*Index pendingBlobs restic.BlobSet idxMutex sync.RWMutex + compress bool } // NewMasterIndex creates a new master index. @@ -29,6 +31,10 @@ func NewMasterIndex() *MasterIndex { return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()} } +func (mi *MasterIndex) markCompressed() { + mi.compress = true +} + // Lookup queries all known Indexes for the ID and returns all matches. func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) { mi.idxMutex.RLock() @@ -112,6 +118,28 @@ func (mi *MasterIndex) IsMixedPack(packID restic.ID) bool { return false } +// IDs returns the IDs of all indexes contained in the index. +func (mi *MasterIndex) IDs() restic.IDSet { + mi.idxMutex.RLock() + defer mi.idxMutex.RUnlock() + + ids := restic.NewIDSet() + for _, idx := range mi.idx { + if !idx.Final() { + continue + } + indexIDs, err := idx.IDs() + if err != nil { + debug.Log("not using index, ID() returned error %v", err) + continue + } + for _, id := range indexIDs { + ids.Insert(id) + } + } + return ids +} + // Packs returns all packs that are covered by the index. // If packBlacklist is given, those packs are only contained in the // resulting IDSet if they are contained in a non-final (newly written) index. @@ -131,40 +159,6 @@ func (mi *MasterIndex) Packs(packBlacklist restic.IDSet) restic.IDSet { return packs } -// PackSize returns the size of all packs computed by index information. -// If onlyHdr is set to true, only the size of the header is returned -// Note that this function only gives correct sizes, if there are no -// duplicates in the index. -func (mi *MasterIndex) PackSize(ctx context.Context, onlyHdr bool) map[restic.ID]int64 { - packSize := make(map[restic.ID]int64) - - for blob := range mi.Each(ctx) { - size, ok := packSize[blob.PackID] - if !ok { - size = pack.HeaderSize - } - if !onlyHdr { - size += int64(blob.Length) - } - packSize[blob.PackID] = size + int64(pack.EntrySize) - } - - return packSize -} - -// Count returns the number of blobs of type t in the index. -func (mi *MasterIndex) Count(t restic.BlobType) (n uint) { - mi.idxMutex.RLock() - defer mi.idxMutex.RUnlock() - - var sum uint - for _, idx := range mi.idx { - sum += idx.Count(t) - } - - return sum -} - // Insert adds a new index to the MasterIndex. func (mi *MasterIndex) Insert(idx *Index) { mi.idxMutex.Lock() @@ -195,9 +189,9 @@ func (mi *MasterIndex) StorePack(id restic.ID, blobs []restic.Blob) { mi.idx = append(mi.idx, newIdx) } -// FinalizeNotFinalIndexes finalizes all indexes that +// finalizeNotFinalIndexes finalizes all indexes that // have not yet been saved and returns that list -func (mi *MasterIndex) FinalizeNotFinalIndexes() []*Index { +func (mi *MasterIndex) finalizeNotFinalIndexes() []*Index { mi.idxMutex.Lock() defer mi.idxMutex.Unlock() @@ -214,8 +208,8 @@ func (mi *MasterIndex) FinalizeNotFinalIndexes() []*Index { return list } -// FinalizeFullIndexes finalizes all indexes that are full and returns that list. -func (mi *MasterIndex) FinalizeFullIndexes() []*Index { +// finalizeFullIndexes finalizes all indexes that are full and returns that list. +func (mi *MasterIndex) finalizeFullIndexes() []*Index { mi.idxMutex.Lock() defer mi.idxMutex.Unlock() @@ -224,11 +218,10 @@ func (mi *MasterIndex) FinalizeFullIndexes() []*Index { debug.Log("checking %d indexes", len(mi.idx)) for _, idx := range mi.idx { if idx.Final() { - debug.Log("index %p is final", idx) continue } - if IndexFull(idx) { + if IndexFull(idx, mi.compress) { debug.Log("index %p is full", idx) idx.Finalize() list = append(list, idx) @@ -241,14 +234,6 @@ func (mi *MasterIndex) FinalizeFullIndexes() []*Index { return list } -// All returns all indexes. -func (mi *MasterIndex) All() []*Index { - mi.idxMutex.Lock() - defer mi.idxMutex.Unlock() - - return mi.idx -} - // Each returns a channel that yields all blobs known to the index. When the // context is cancelled, the background goroutine terminates. This blocks any // modification of the index. @@ -259,13 +244,10 @@ func (mi *MasterIndex) Each(ctx context.Context) <-chan restic.PackedBlob { go func() { defer mi.idxMutex.RUnlock() - defer func() { - close(ch) - }() + defer close(ch) for _, idx := range mi.idx { - idxCh := idx.Each(ctx) - for pb := range idxCh { + for pb := range idx.Each(ctx) { select { case <-ctx.Done(): return @@ -294,7 +276,9 @@ func (mi *MasterIndex) MergeFinalIndexes() error { idx := mi.idx[i] // clear reference in masterindex as it may become stale mi.idx[i] = nil - if !idx.Final() { + // do not merge indexes that have no id set + ids, _ := idx.IDs() + if !idx.Final() || len(ids) == 0 { newIdx = append(newIdx, idx) } else { err := mi.idx[0].merge(idx) @@ -308,14 +292,14 @@ func (mi *MasterIndex) MergeFinalIndexes() error { return nil } -const saveIndexParallelism = 4 - // Save saves all known indexes to index files, leaving out any // packs whose ID is contained in packBlacklist from finalized indexes. // The new index contains the IDs of all known indexes in the "supersedes" // field. The IDs are also returned in the IDSet obsolete. // After calling this function, you should remove the obsolete index files. -func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, packBlacklist restic.IDSet, extraObsolete restic.IDs, p *progress.Counter) (obsolete restic.IDSet, err error) { +func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, packBlacklist restic.IDSet, extraObsolete restic.IDs, p *progress.Counter) (obsolete restic.IDSet, err error) { + p.SetMax(uint64(len(mi.Packs(packBlacklist)))) + mi.idxMutex.Lock() defer mi.idxMutex.Unlock() @@ -354,13 +338,13 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, packBla debug.Log("adding index %d", i) for pbs := range idx.EachByPack(ctx, packBlacklist) { - newIndex.StorePack(pbs.packID, pbs.blobs) + newIndex.StorePack(pbs.PackID, pbs.Blobs) p.Add(1) - if IndexFull(newIndex) { + if IndexFull(newIndex, mi.compress) { select { case ch <- newIndex: case <-ctx.Done(): - return nil + return ctx.Err() } newIndex = NewIndex() } @@ -391,12 +375,96 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, packBla return nil } + // encoding an index can take quite some time such that this can be both CPU- or IO-bound + workerCount := int(repo.Connections()) + runtime.GOMAXPROCS(0) // run workers on ch - wg.Go(func() error { - return RunWorkers(saveIndexParallelism, worker) - }) - + for i := 0; i < workerCount; i++ { + wg.Go(worker) + } err = wg.Wait() return obsolete, err } + +// SaveIndex saves an index in the repository. +func SaveIndex(ctx context.Context, repo restic.SaverUnpacked, index *Index) (restic.ID, error) { + buf := bytes.NewBuffer(nil) + + err := index.Encode(buf) + if err != nil { + return restic.ID{}, err + } + + id, err := repo.SaveUnpacked(ctx, restic.IndexFile, buf.Bytes()) + ierr := index.SetID(id) + if ierr != nil { + // logic bug + panic(ierr) + } + return id, err +} + +// saveIndex saves all indexes in the backend. +func (mi *MasterIndex) saveIndex(ctx context.Context, r restic.SaverUnpacked, indexes ...*Index) error { + for i, idx := range indexes { + debug.Log("Saving index %d", i) + + sid, err := SaveIndex(ctx, r, idx) + if err != nil { + return err + } + + debug.Log("Saved index %d as %v", i, sid) + } + + return mi.MergeFinalIndexes() +} + +// SaveIndex saves all new indexes in the backend. +func (mi *MasterIndex) SaveIndex(ctx context.Context, r restic.SaverUnpacked) error { + return mi.saveIndex(ctx, r, mi.finalizeNotFinalIndexes()...) +} + +// SaveFullIndex saves all full indexes in the backend. +func (mi *MasterIndex) SaveFullIndex(ctx context.Context, r restic.SaverUnpacked) error { + return mi.saveIndex(ctx, r, mi.finalizeFullIndexes()...) +} + +// ListPacks returns the blobs of the specified pack files grouped by pack file. +func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { + out := make(chan restic.PackBlobs) + go func() { + defer close(out) + // only resort a part of the index to keep the memory overhead bounded + for i := byte(0); i < 16; i++ { + if ctx.Err() != nil { + return + } + + packBlob := make(map[restic.ID][]restic.Blob) + for pack := range packs { + if pack[0]&0xf == i { + packBlob[pack] = nil + } + } + if len(packBlob) == 0 { + continue + } + for pb := range mi.Each(ctx) { + if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i { + packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob) + } + } + + // pass on packs + for packID, pbs := range packBlob { + select { + case out <- restic.PackBlobs{PackID: packID, Blobs: pbs}: + case <-ctx.Done(): + return + } + } + } + }() + return out +} diff --git a/internal/repository/master_index_test.go b/internal/repository/master_index_test.go index f1fe9af7eae..2430c83dc60 100644 --- a/internal/repository/master_index_test.go +++ b/internal/repository/master_index_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/restic/restic/internal/checker" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -22,7 +23,7 @@ func TestMasterIndex(t *testing.T) { PackID: restic.NewRandomID(), Blob: restic.Blob{ BlobHandle: bhInIdx1, - Length: uint(restic.CiphertextLength(10)), + Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } @@ -30,9 +31,10 @@ func TestMasterIndex(t *testing.T) { blob2 := restic.PackedBlob{ PackID: restic.NewRandomID(), Blob: restic.Blob{ - BlobHandle: bhInIdx2, - Length: uint(restic.CiphertextLength(100)), - Offset: 10, + BlobHandle: bhInIdx2, + Length: uint(crypto.CiphertextLength(100)), + Offset: 10, + UncompressedLength: 200, }, } @@ -40,7 +42,7 @@ func TestMasterIndex(t *testing.T) { PackID: restic.NewRandomID(), Blob: restic.Blob{ BlobHandle: bhInIdx12, - Length: uint(restic.CiphertextLength(123)), + Length: uint(crypto.CiphertextLength(123)), Offset: 110, }, } @@ -48,19 +50,20 @@ func TestMasterIndex(t *testing.T) { blob12b := restic.PackedBlob{ PackID: restic.NewRandomID(), Blob: restic.Blob{ - BlobHandle: bhInIdx12, - Length: uint(restic.CiphertextLength(123)), - Offset: 50, + BlobHandle: bhInIdx12, + Length: uint(crypto.CiphertextLength(123)), + Offset: 50, + UncompressedLength: 80, }, } idx1 := repository.NewIndex() - idx1.Store(blob1) - idx1.Store(blob12a) + idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob}) + idx1.StorePack(blob12a.PackID, []restic.Blob{blob12a.Blob}) idx2 := repository.NewIndex() - idx2.Store(blob2) - idx2.Store(blob12b) + idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) + idx2.StorePack(blob12b.PackID, []restic.Blob{blob12b.Blob}) mIdx := repository.NewMasterIndex() mIdx.Insert(idx1) @@ -86,7 +89,7 @@ func TestMasterIndex(t *testing.T) { size, found = mIdx.LookupSize(bhInIdx2) rtest.Equals(t, true, found) - rtest.Equals(t, uint(100), size) + rtest.Equals(t, uint(200), size) // test idInIdx12 found = mIdx.Has(bhInIdx12) @@ -120,12 +123,6 @@ func TestMasterIndex(t *testing.T) { rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id") _, found = mIdx.LookupSize(restic.NewRandomBlobHandle()) rtest.Assert(t, !found, "Expected no blobs when fetching with a random id") - - // Test Count - num := mIdx.Count(restic.DataBlob) - rtest.Equals(t, uint(2), num) - num = mIdx.Count(restic.TreeBlob) - rtest.Equals(t, uint(2), num) } func TestMasterMergeFinalIndexes(t *testing.T) { @@ -144,32 +141,26 @@ func TestMasterMergeFinalIndexes(t *testing.T) { blob2 := restic.PackedBlob{ PackID: restic.NewRandomID(), Blob: restic.Blob{ - BlobHandle: bhInIdx2, - Length: 100, - Offset: 10, + BlobHandle: bhInIdx2, + Length: 100, + Offset: 10, + UncompressedLength: 200, }, } idx1 := repository.NewIndex() - idx1.Store(blob1) + idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob}) idx2 := repository.NewIndex() - idx2.Store(blob2) + idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) mIdx := repository.NewMasterIndex() mIdx.Insert(idx1) mIdx.Insert(idx2) - finalIndexes := mIdx.FinalizeNotFinalIndexes() + finalIndexes, idxCount := repository.TestMergeIndex(t, mIdx) rtest.Equals(t, []*repository.Index{idx1, idx2}, finalIndexes) - - err := mIdx.MergeFinalIndexes() - if err != nil { - t.Fatal(err) - } - - allIndexes := mIdx.All() - rtest.Equals(t, 1, len(allIndexes)) + rtest.Equals(t, 1, idxCount) blobCount := 0 for range mIdx.Each(context.TODO()) { @@ -188,20 +179,13 @@ func TestMasterMergeFinalIndexes(t *testing.T) { // merge another index containing identical blobs idx3 := repository.NewIndex() - idx3.Store(blob1) - idx3.Store(blob2) + idx3.StorePack(blob1.PackID, []restic.Blob{blob1.Blob}) + idx3.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) mIdx.Insert(idx3) - finalIndexes = mIdx.FinalizeNotFinalIndexes() + finalIndexes, idxCount = repository.TestMergeIndex(t, mIdx) rtest.Equals(t, []*repository.Index{idx3}, finalIndexes) - - err = mIdx.MergeFinalIndexes() - if err != nil { - t.Fatal(err) - } - - allIndexes = mIdx.All() - rtest.Equals(t, 1, len(allIndexes)) + rtest.Equals(t, 1, idxCount) // Index should have same entries as before! blobs = mIdx.Lookup(bhInIdx1) @@ -226,11 +210,7 @@ func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*repo idx1, lookupBh := createRandomIndex(rng, size) mIdx.Insert(idx1) - mIdx.FinalizeNotFinalIndexes() - err := mIdx.MergeFinalIndexes() - if err != nil { - t.Fatal(err) - } + repository.TestMergeIndex(t, mIdx) return mIdx, lookupBh } @@ -288,14 +268,12 @@ func BenchmarkMasterIndexLookupMultipleIndexUnknown(b *testing.B) { } func BenchmarkMasterIndexLookupParallel(b *testing.B) { - mIdx := repository.NewMasterIndex() - for _, numindices := range []int{25, 50, 100} { var lookupBh restic.BlobHandle b.StopTimer() rng := rand.New(rand.NewSource(0)) - mIdx, lookupBh = createRandomMasterIndex(b, rng, numindices, 10000) + mIdx, lookupBh := createRandomMasterIndex(b, rng, numindices, 10000) b.StartTimer() name := fmt.Sprintf("known,indices=%d", numindices) @@ -335,8 +313,8 @@ var ( depth = 3 ) -func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Repository, func()) { - repo, cleanup := repository.TestRepository(t) +func createFilledRepo(t testing.TB, snapshots int, dup float32, version uint) (restic.Repository, func()) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) for i := 0; i < 3; i++ { restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth, dup) @@ -346,7 +324,11 @@ func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Reposito } func TestIndexSave(t *testing.T) { - repo, cleanup := createFilledRepo(t, 3, 0) + repository.TestAllVersions(t, testIndexSave) +} + +func testIndexSave(t *testing.T, version uint) { + repo, cleanup := createFilledRepo(t, 3, 0, version) defer cleanup() err := repo.LoadIndex(context.TODO()) @@ -354,7 +336,7 @@ func TestIndexSave(t *testing.T) { t.Fatal(err) } - obsoletes, err := repo.Index().(*repository.MasterIndex).Save(context.TODO(), repo, nil, nil, nil) + obsoletes, err := repo.Index().Save(context.TODO(), repo, nil, nil, nil) if err != nil { t.Fatalf("unable to save new index: %v", err) } @@ -369,6 +351,11 @@ func TestIndexSave(t *testing.T) { } checker := checker.New(repo, false) + err = checker.LoadSnapshots(context.TODO()) + if err != nil { + t.Error(err) + } + hints, errs := checker.LoadIndex(context.TODO()) for _, h := range hints { t.Logf("hint: %v\n", h) diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index 73c0d52404d..e83bf876973 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -1,10 +1,12 @@ package repository import ( + "bufio" "context" - "hash" "io" + "io/ioutil" "os" + "runtime" "sync" "github.com/restic/restic/internal/errors" @@ -19,112 +21,154 @@ import ( "github.com/minio/sha256-simd" ) -// Saver implements saving data in a backend. -type Saver interface { - Save(context.Context, restic.Handle, restic.RewindReader) error - Hasher() hash.Hash -} - // Packer holds a pack.Packer together with a hash writer. type Packer struct { *pack.Packer - hw *hashing.Writer - beHw *hashing.Writer tmpfile *os.File + bufWr *bufio.Writer } // packerManager keeps a list of open packs and creates new on demand. type packerManager struct { - be Saver + tpe restic.BlobType key *crypto.Key - pm sync.Mutex - packers []*Packer -} + queueFn func(ctx context.Context, t restic.BlobType, p *Packer) error -const minPackSize = 4 * 1024 * 1024 + pm sync.Mutex + packer *Packer + packSize uint +} // newPackerManager returns an new packer manager which writes temporary files // to a temporary directory -func newPackerManager(be Saver, key *crypto.Key) *packerManager { +func newPackerManager(key *crypto.Key, tpe restic.BlobType, packSize uint, queueFn func(ctx context.Context, t restic.BlobType, p *Packer) error) *packerManager { return &packerManager{ - be: be, - key: key, + tpe: tpe, + key: key, + queueFn: queueFn, + packSize: packSize, } } -// findPacker returns a packer for a new blob of size bytes. Either a new one is -// created or one is returned that already has some blobs. -func (r *packerManager) findPacker() (packer *Packer, err error) { +func (r *packerManager) Flush(ctx context.Context) error { + r.pm.Lock() + defer r.pm.Unlock() + + if r.packer != nil { + debug.Log("manually flushing pending pack") + err := r.queueFn(ctx, r.tpe, r.packer) + if err != nil { + return err + } + r.packer = nil + } + return nil +} + +func (r *packerManager) SaveBlob(ctx context.Context, t restic.BlobType, id restic.ID, ciphertext []byte, uncompressedLength int) (int, error) { r.pm.Lock() defer r.pm.Unlock() - // search for a suitable packer - if len(r.packers) > 0 { - p := r.packers[0] - last := len(r.packers) - 1 - r.packers[0] = r.packers[last] - r.packers[last] = nil // Allow GC of stale reference. - r.packers = r.packers[:last] - return p, nil + var err error + packer := r.packer + if r.packer == nil { + packer, err = r.newPacker() + if err != nil { + return 0, err + } } + // remember packer + r.packer = packer - // no suitable packer found, return new + // save ciphertext + // Add only appends bytes in memory to avoid being a scaling bottleneck + size, err := packer.Add(t, id, ciphertext, uncompressedLength) + if err != nil { + return 0, err + } + + // if the pack and header is not full enough, put back to the list + if packer.Size() < r.packSize && !packer.HeaderFull() { + debug.Log("pack is not full enough (%d bytes)", packer.Size()) + return size, nil + } + // forget full packer + r.packer = nil + + // call while holding lock to prevent findPacker from creating new packers if the uploaders are busy + // else write the pack to the backend + err = r.queueFn(ctx, t, packer) + if err != nil { + return 0, err + } + + return size + packer.HeaderOverhead(), nil +} + +// findPacker returns a packer for a new blob of size bytes. Either a new one is +// created or one is returned that already has some blobs. +func (r *packerManager) newPacker() (packer *Packer, err error) { debug.Log("create new pack") tmpfile, err := fs.TempFile("", "restic-temp-pack-") if err != nil { return nil, errors.Wrap(err, "fs.TempFile") } - w := io.Writer(tmpfile) - beHasher := r.be.Hasher() - var beHw *hashing.Writer - if beHasher != nil { - beHw = hashing.NewWriter(w, beHasher) - w = beHw - } - - hw := hashing.NewWriter(w, sha256.New()) - p := pack.NewPacker(r.key, hw) + bufWr := bufio.NewWriter(tmpfile) + p := pack.NewPacker(r.key, bufWr) packer = &Packer{ Packer: p, - beHw: beHw, - hw: hw, tmpfile: tmpfile, + bufWr: bufWr, } return packer, nil } -// insertPacker appends p to s.packs. -func (r *packerManager) insertPacker(p *Packer) { - r.pm.Lock() - defer r.pm.Unlock() - - r.packers = append(r.packers, p) - debug.Log("%d packers\n", len(r.packers)) -} - // savePacker stores p in the backend. func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packer) error { debug.Log("save packer for %v with %d blobs (%d bytes)\n", t, p.Packer.Count(), p.Packer.Size()) - _, err := p.Packer.Finalize() + err := p.Packer.Finalize() + if err != nil { + return err + } + err = p.bufWr.Flush() + if err != nil { + return err + } + + // calculate sha256 hash in a second pass + var rd io.Reader + rd, err = restic.NewFileReader(p.tmpfile, nil) + if err != nil { + return err + } + beHasher := r.be.Hasher() + var beHr *hashing.Reader + if beHasher != nil { + beHr = hashing.NewReader(rd, beHasher) + rd = beHr + } + + hr := hashing.NewReader(rd, sha256.New()) + _, err = io.Copy(ioutil.Discard, hr) if err != nil { return err } - id := restic.IDFromHash(p.hw.Sum(nil)) + id := restic.IDFromHash(hr.Sum(nil)) h := restic.Handle{Type: restic.PackFile, Name: id.String(), ContainedBlobType: t} var beHash []byte - if p.beHw != nil { - beHash = p.beHw.Sum(nil) + if beHr != nil { + beHash = beHr.Sum(nil) } - rd, err := restic.NewFileReader(p.tmpfile, beHash) + rrd, err := restic.NewFileReader(p.tmpfile, beHash) if err != nil { return err } - err = r.be.Save(ctx, h, rd) + err = r.be.Save(ctx, h, rrd) if err != nil { debug.Log("Save(%v) error: %v", h, err) return err @@ -137,9 +181,12 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe return errors.Wrap(err, "close tempfile") } - err = fs.RemoveIfExists(p.tmpfile.Name()) - if err != nil { - return errors.Wrap(err, "Remove") + // on windows the tempfile is automatically deleted on close + if runtime.GOOS != "windows" { + err = fs.RemoveIfExists(p.tmpfile.Name()) + if err != nil { + return errors.Wrap(err, "Remove") + } } // update blobs in the index @@ -150,13 +197,5 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe if r.noAutoIndexUpdate { return nil } - return r.SaveFullIndex(ctx) -} - -// countPacker returns the number of open (unfinished) packers. -func (r *packerManager) countPacker() int { - r.pm.Lock() - defer r.pm.Unlock() - - return len(r.packers) + return r.idx.SaveFullIndex(ctx, r) } diff --git a/internal/repository/packer_manager_test.go b/internal/repository/packer_manager_test.go index 1a810ab619a..90f716e0da8 100644 --- a/internal/repository/packer_manager_test.go +++ b/internal/repository/packer_manager_test.go @@ -4,15 +4,12 @@ import ( "context" "io" "math/rand" - "os" "sync" "testing" - "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/crypto" - "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/mock" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" ) func randomID(rd io.Reader) restic.ID { @@ -33,91 +30,27 @@ func min(a, b int) int { return b } -func saveFile(t testing.TB, be Saver, length int, f *os.File, id restic.ID, hash []byte) { - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - t.Logf("save file %v", h) - - rd, err := restic.NewFileReader(f, hash) - if err != nil { - t.Fatal(err) - } - - err = be.Save(context.TODO(), h, rd) - if err != nil { - t.Fatal(err) - } - - if err := f.Close(); err != nil { - t.Fatal(err) - } - - if err := fs.RemoveIfExists(f.Name()); err != nil { - t.Fatal(err) - } -} - -func fillPacks(t testing.TB, rnd *rand.Rand, be Saver, pm *packerManager, buf []byte) (bytes int) { - for i := 0; i < 100; i++ { +func fillPacks(t testing.TB, rnd *rand.Rand, pm *packerManager, buf []byte) (bytes int) { + for i := 0; i < 102; i++ { l := rnd.Intn(maxBlobSize) - - packer, err := pm.findPacker() - if err != nil { - t.Fatal(err) - } - id := randomID(rnd) buf = buf[:l] // Only change a few bytes so we know we're not benchmarking the RNG. rnd.Read(buf[:min(l, 4)]) - n, err := packer.Add(restic.DataBlob, id, buf) + n, err := pm.SaveBlob(context.TODO(), restic.DataBlob, id, buf, 0) if err != nil { t.Fatal(err) } - if n != l { - t.Errorf("Add() returned invalid number of bytes: want %v, got %v", n, l) + if n != l+37 && n != l+37+36 { + t.Errorf("Add() returned invalid number of bytes: want %v, got %v", l, n) } - bytes += l - - if packer.Size() < minPackSize { - pm.insertPacker(packer) - continue - } - - _, err = packer.Finalize() - if err != nil { - t.Fatal(err) - } - - packID := restic.IDFromHash(packer.hw.Sum(nil)) - var beHash []byte - if packer.beHw != nil { - beHash = packer.beHw.Sum(nil) - } - saveFile(t, be, int(packer.Size()), packer.tmpfile, packID, beHash) + bytes += n } - - return bytes -} - -func flushRemainingPacks(t testing.TB, be Saver, pm *packerManager) (bytes int) { - if pm.countPacker() > 0 { - for _, packer := range pm.packers { - n, err := packer.Finalize() - if err != nil { - t.Fatal(err) - } - bytes += int(n) - - packID := restic.IDFromHash(packer.hw.Sum(nil)) - var beHash []byte - if packer.beHw != nil { - beHash = packer.beHw.Sum(nil) - } - saveFile(t, be, int(packer.Size()), packer.tmpfile, packID, beHash) - } + err := pm.Flush(context.TODO()) + if err != nil { + t.Fatal(err) } - return bytes } @@ -136,13 +69,21 @@ func TestPackerManager(t *testing.T) { func testPackerManager(t testing.TB) int64 { rnd := rand.New(rand.NewSource(randomSeed)) - be := mem.New() - pm := newPackerManager(be, crypto.NewRandomKey()) + savedBytes := int(0) + pm := newPackerManager(crypto.NewRandomKey(), restic.DataBlob, DefaultPackSize, func(ctx context.Context, tp restic.BlobType, p *Packer) error { + err := p.Finalize() + if err != nil { + return err + } + savedBytes += int(p.Size()) + return nil + }) blobBuf := make([]byte, maxBlobSize) - bytes := fillPacks(t, rnd, be, pm, blobBuf) - bytes += flushRemainingPacks(t, be, pm) + bytes := fillPacks(t, rnd, pm, blobBuf) + // bytes does not include the last packs header + test.Equals(t, savedBytes, bytes+36) t.Logf("saved %d bytes", bytes) return int64(bytes) @@ -155,10 +96,6 @@ func BenchmarkPackerManager(t *testing.B) { }) rnd := rand.New(rand.NewSource(randomSeed)) - - be := &mock.Backend{ - SaveFn: func(context.Context, restic.Handle, restic.RewindReader) error { return nil }, - } blobBuf := make([]byte, maxBlobSize) t.ReportAllocs() @@ -167,8 +104,9 @@ func BenchmarkPackerManager(t *testing.B) { for i := 0; i < t.N; i++ { rnd.Seed(randomSeed) - pm := newPackerManager(be, crypto.NewRandomKey()) - fillPacks(t, rnd, be, pm, blobBuf) - flushRemainingPacks(t, be, pm) + pm := newPackerManager(crypto.NewRandomKey(), restic.DataBlob, DefaultPackSize, func(ctx context.Context, t restic.BlobType, p *Packer) error { + return nil + }) + fillPacks(t, rnd, pm, blobBuf) } } diff --git a/internal/repository/packer_uploader.go b/internal/repository/packer_uploader.go new file mode 100644 index 00000000000..30c8f77afc6 --- /dev/null +++ b/internal/repository/packer_uploader.go @@ -0,0 +1,63 @@ +package repository + +import ( + "context" + + "github.com/restic/restic/internal/restic" + "golang.org/x/sync/errgroup" +) + +// SavePacker implements saving a pack in the repository. +type SavePacker interface { + savePacker(ctx context.Context, t restic.BlobType, p *Packer) error +} + +type uploadTask struct { + packer *Packer + tpe restic.BlobType +} + +type packerUploader struct { + uploadQueue chan uploadTask +} + +func newPackerUploader(ctx context.Context, wg *errgroup.Group, repo SavePacker, connections uint) *packerUploader { + pu := &packerUploader{ + uploadQueue: make(chan uploadTask), + } + + for i := 0; i < int(connections); i++ { + wg.Go(func() error { + for { + select { + case t, ok := <-pu.uploadQueue: + if !ok { + return nil + } + err := repo.savePacker(ctx, t.tpe, t.packer) + if err != nil { + return err + } + case <-ctx.Done(): + return ctx.Err() + } + } + }) + } + + return pu +} + +func (pu *packerUploader) QueuePacker(ctx context.Context, t restic.BlobType, p *Packer) (err error) { + select { + case <-ctx.Done(): + return ctx.Err() + case pu.uploadQueue <- uploadTask{tpe: t, packer: p}: + } + + return nil +} + +func (pu *packerUploader) TriggerShutdown() { + close(pu.uploadQueue) +} diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 423f3c831ce..bf6b65c8f91 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -2,21 +2,16 @@ package repository import ( "context" - "os" "sync" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) -const numRepackWorkers = 8 - // Repack takes a list of packs together with a list of blobs contained in // these packs. Each pack is loaded and the blobs listed in keepBlobs is saved // into a new pack. Returned is the list of obsolete packs which can then @@ -24,157 +19,102 @@ const numRepackWorkers = 8 // // The map keepBlobs is modified by Repack, it is used to keep track of which // blobs have been processed. -func Repack(ctx context.Context, repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet, p *progress.Counter) (obsoletePacks restic.IDSet, err error) { +func Repack(ctx context.Context, repo restic.Repository, dstRepo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet, p *progress.Counter) (obsoletePacks restic.IDSet, err error) { debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs)) + if repo == dstRepo && dstRepo.Backend().Connections() < 2 { + return nil, errors.Fatal("repack step requires a backend connection limit of at least two") + } + wg, wgCtx := errgroup.WithContext(ctx) - downloadQueue := make(chan restic.ID) + dstRepo.StartPackUploader(wgCtx, wg) wg.Go(func() error { - defer close(downloadQueue) - for packID := range packs { - select { - case downloadQueue <- packID: - case <-wgCtx.Done(): - return wgCtx.Err() - } - } - return nil + var err error + obsoletePacks, err = repack(wgCtx, repo, dstRepo, packs, keepBlobs, p) + return err }) - type repackJob struct { - tempfile *os.File - hash restic.ID - packLength int64 + if err := wg.Wait(); err != nil { + return nil, err } - processQueue := make(chan repackJob) - // used to close processQueue once all downloaders have finished - var downloadWG sync.WaitGroup - - downloader := func() error { - defer downloadWG.Done() - for packID := range downloadQueue { - // load the complete pack into a temp file - h := restic.Handle{Type: restic.PackFile, Name: packID.String()} - - tempfile, hash, packLength, err := DownloadAndHash(wgCtx, repo.Backend(), h) - if err != nil { - return errors.Wrap(err, "Repack") - } + return obsoletePacks, nil +} - debug.Log("pack %v loaded (%d bytes), hash %v", packID, packLength, hash) +func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet, p *progress.Counter) (obsoletePacks restic.IDSet, err error) { + wg, wgCtx := errgroup.WithContext(ctx) - if !packID.Equal(hash) { - return errors.Errorf("hash does not match id: want %v, got %v", packID, hash) + var keepMutex sync.Mutex + downloadQueue := make(chan restic.PackBlobs) + wg.Go(func() error { + defer close(downloadQueue) + for pbs := range repo.Index().ListPacks(wgCtx, packs) { + var packBlobs []restic.Blob + keepMutex.Lock() + // filter out unnecessary blobs + for _, entry := range pbs.Blobs { + h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} + if keepBlobs.Has(h) { + packBlobs = append(packBlobs, entry) + } } + keepMutex.Unlock() select { - case processQueue <- repackJob{tempfile, hash, packLength}: + case downloadQueue <- restic.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}: case <-wgCtx.Done(): return wgCtx.Err() } } return nil - } - - downloadWG.Add(numRepackWorkers) - for i := 0; i < numRepackWorkers; i++ { - wg.Go(downloader) - } - wg.Go(func() error { - downloadWG.Wait() - close(processQueue) - return nil }) - var keepMutex sync.Mutex worker := func() error { - for job := range processQueue { - tempfile, packID, packLength := job.tempfile, job.hash, job.packLength - - blobs, _, err := pack.List(repo.Key(), tempfile, packLength) - if err != nil { - return err - } - - debug.Log("processing pack %v, blobs: %v", packID, len(blobs)) - var buf []byte - for _, entry := range blobs { - h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} - - keepMutex.Lock() - shouldKeep := keepBlobs.Has(h) - keepMutex.Unlock() - - if !shouldKeep { - continue - } - - debug.Log(" process blob %v", h) - - if uint(cap(buf)) < entry.Length { - buf = make([]byte, entry.Length) - } - buf = buf[:entry.Length] - - n, err := tempfile.ReadAt(buf, int64(entry.Offset)) + for t := range downloadQueue { + err := StreamPack(wgCtx, repo.Backend().Load, repo.Key(), t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { - return errors.Wrap(err, "ReadAt") - } - - if n != len(buf) { - return errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v", - h, tempfile.Name(), len(buf), n) - } - - nonce, ciphertext := buf[:repo.Key().NonceSize()], buf[repo.Key().NonceSize():] - plaintext, err := repo.Key().Open(ciphertext[:0], nonce, ciphertext, nil) - if err != nil { - return err - } - - id := restic.Hash(plaintext) - if !id.Equal(entry.ID) { - debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v", - h.Type, h.ID, tempfile.Name(), id) - return errors.Errorf("read blob %v from %v: wrong data returned, hash is %v", - h, tempfile.Name(), id) + var ierr error + // check whether we can get a valid copy somewhere else + buf, ierr = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) + if ierr != nil { + // no luck, return the original error + return err + } } keepMutex.Lock() // recheck whether some other worker was faster - shouldKeep = keepBlobs.Has(h) + shouldKeep := keepBlobs.Has(blob) if shouldKeep { - keepBlobs.Delete(h) + keepBlobs.Delete(blob) } keepMutex.Unlock() if !shouldKeep { - continue + return nil } // We do want to save already saved blobs! - _, _, err = repo.SaveBlob(wgCtx, entry.Type, plaintext, entry.ID, true) + _, _, _, err = dstRepo.SaveBlob(wgCtx, blob.Type, buf, blob.ID, true) if err != nil { return err } - debug.Log(" saved blob %v", entry.ID) - } - - if err = tempfile.Close(); err != nil { - return errors.Wrap(err, "Close") - } - - if err = fs.RemoveIfExists(tempfile.Name()); err != nil { - return errors.Wrap(err, "Remove") + debug.Log(" saved blob %v", blob.ID) + return nil + }) + if err != nil { + return err } p.Add(1) } return nil } - for i := 0; i < numRepackWorkers; i++ { + // as packs are streamed the concurrency is limited by IO + // reduce by one to ensure that uploading is always possible + repackWorkerCount := int(repo.Connections() - 1) + for i := 0; i < repackWorkerCount; i++ { wg.Go(worker) } @@ -182,7 +122,7 @@ func Repack(ctx context.Context, repo restic.Repository, packs restic.IDSet, kee return nil, err } - if err := repo.Flush(ctx); err != nil { + if err := dstRepo.Flush(ctx); err != nil { return nil, err } diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 108c167d973..f8cefc00b43 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -8,6 +8,8 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" + "golang.org/x/sync/errgroup" ) func randomSize(min, max int) int { @@ -15,6 +17,9 @@ func randomSize(min, max int) int { } func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData float32) { + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + for i := 0; i < blobs; i++ { var ( tpe restic.BlobType @@ -32,7 +37,7 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl buf := make([]byte, length) rand.Read(buf) - id, exists, err := repo.SaveBlob(context.TODO(), tpe, buf, restic.ID{}, false) + id, exists, _, err := repo.SaveBlob(context.TODO(), tpe, buf, restic.ID{}, false) if err != nil { t.Fatalf("SaveFrom() error %v", err) } @@ -46,6 +51,7 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl if err = repo.Flush(context.Background()); err != nil { t.Fatalf("repo.Flush() returned error %v", err) } + repo.StartPackUploader(context.TODO(), &wg) } } @@ -62,7 +68,9 @@ func createRandomWrongBlob(t testing.TB, repo restic.Repository) { // invert first data byte buf[0] ^= 0xff - _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, id, false) + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + _, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, id, false) if err != nil { t.Fatalf("SaveFrom() error %v", err) } @@ -142,7 +150,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe } func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs restic.BlobSet) { - repackedBlobs, err := repository.Repack(context.TODO(), repo, packs, blobs, nil) + repackedBlobs, err := repository.Repack(context.TODO(), repo, repo, packs, blobs, nil) if err != nil { t.Fatal(err) } @@ -155,8 +163,8 @@ func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs rest } } -func saveIndex(t *testing.T, repo restic.Repository) { - if err := repo.SaveIndex(context.TODO()); err != nil { +func flush(t *testing.T, repo restic.Repository) { + if err := repo.Flush(context.TODO()); err != nil { t.Fatalf("repo.SaveIndex() %v", err) } } @@ -192,9 +200,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) { t.Fatal(err) } - _, err = (repo.Index()).(*repository.MasterIndex). - Save(context.TODO(), repo, restic.NewIDSet(), nil, nil) - + _, err = repo.Index().Save(context.TODO(), repo, restic.NewIDSet(), nil, nil) if err != nil { t.Fatal(err) } @@ -212,7 +218,11 @@ func reloadIndex(t *testing.T, repo restic.Repository) { } func TestRepack(t *testing.T) { - repo, cleanup := repository.TestRepository(t) + repository.TestAllVersions(t, testRepack) +} + +func testRepack(t *testing.T, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) defer cleanup() seed := time.Now().UnixNano() @@ -233,7 +243,7 @@ func TestRepack(t *testing.T) { packsBefore, packsAfter) } - saveIndex(t, repo) + flush(t, repo) removeBlobs, keepBlobs := selectBlobs(t, repo, 0.2) @@ -278,8 +288,55 @@ func TestRepack(t *testing.T) { } } +func TestRepackCopy(t *testing.T) { + repository.TestAllVersions(t, testRepackCopy) +} + +func testRepackCopy(t *testing.T, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) + defer cleanup() + dstRepo, dstCleanup := repository.TestRepositoryWithVersion(t, version) + defer dstCleanup() + + seed := time.Now().UnixNano() + rand.Seed(seed) + t.Logf("rand seed is %v", seed) + + createRandomBlobs(t, repo, 100, 0.7) + flush(t, repo) + + _, keepBlobs := selectBlobs(t, repo, 0.2) + copyPacks := findPacksForBlobs(t, repo, keepBlobs) + + _, err := repository.Repack(context.TODO(), repo, dstRepo, copyPacks, keepBlobs, nil) + if err != nil { + t.Fatal(err) + } + rebuildIndex(t, dstRepo) + reloadIndex(t, dstRepo) + + idx := dstRepo.Index() + + for h := range keepBlobs { + list := idx.Lookup(h) + if len(list) == 0 { + t.Errorf("unable to find blob %v in repo", h.ID.Str()) + continue + } + + if len(list) != 1 { + t.Errorf("expected one pack in the list, got: %v", list) + continue + } + } +} + func TestRepackWrongBlob(t *testing.T) { - repo, cleanup := repository.TestRepository(t) + repository.TestAllVersions(t, testRepackWrongBlob) +} + +func testRepackWrongBlob(t *testing.T, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) defer cleanup() seed := time.Now().UnixNano() @@ -293,9 +350,58 @@ func TestRepackWrongBlob(t *testing.T) { _, keepBlobs := selectBlobs(t, repo, 0) rewritePacks := findPacksForBlobs(t, repo, keepBlobs) - _, err := repository.Repack(context.TODO(), repo, rewritePacks, keepBlobs, nil) + _, err := repository.Repack(context.TODO(), repo, repo, rewritePacks, keepBlobs, nil) if err == nil { t.Fatal("expected repack to fail but got no error") } t.Logf("found expected error: %v", err) } + +func TestRepackBlobFallback(t *testing.T) { + repository.TestAllVersions(t, testRepackBlobFallback) +} + +func testRepackBlobFallback(t *testing.T, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) + defer cleanup() + + seed := time.Now().UnixNano() + rand.Seed(seed) + t.Logf("rand seed is %v", seed) + + length := randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data + buf := make([]byte, length) + rand.Read(buf) + id := restic.Hash(buf) + + // corrupted copy + modbuf := make([]byte, len(buf)) + copy(modbuf, buf) + // invert first data byte + modbuf[0] ^= 0xff + + // create pack with broken copy + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + _, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, modbuf, id, false) + rtest.OK(t, err) + rtest.OK(t, repo.Flush(context.Background())) + + // find pack with damaged blob + keepBlobs := restic.NewBlobSet(restic.BlobHandle{Type: restic.DataBlob, ID: id}) + rewritePacks := findPacksForBlobs(t, repo, keepBlobs) + + // create pack with valid copy + repo.StartPackUploader(context.TODO(), &wg) + _, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, id, true) + rtest.OK(t, err) + rtest.OK(t, repo.Flush(context.Background())) + + // repack must fallback to valid copy + _, err = repository.Repack(context.TODO(), repo, repo, rewritePacks, keepBlobs, nil) + rtest.OK(t, err) + + keepBlobs = restic.NewBlobSet(restic.BlobHandle{Type: restic.DataBlob, ID: id}) + packs := findPacksForBlobs(t, repo, keepBlobs) + rtest.Assert(t, len(packs) == 3, "unexpected number of copies: %v", len(packs)) +} diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 90e9d5d8298..1dd574baa1a 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -1,30 +1,37 @@ package repository import ( + "bufio" "bytes" "context" - "encoding/json" "fmt" "io" "os" + "sort" "sync" + "github.com/cenkalti/backoff/v4" + "github.com/klauspost/compress/zstd" "github.com/restic/chunker" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/dryrun" "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" - "github.com/minio/sha256-simd" "golang.org/x/sync/errgroup" ) +const MaxStreamBufferSize = 4 * 1024 * 1024 + +const MinPackSize = 4 * 1024 * 1024 +const DefaultPackSize = 16 * 1024 * 1024 +const MaxPackSize = 128 * 1024 * 1024 + // Repository is used to access a repository in a backend. type Repository struct { be restic.Backend @@ -34,22 +41,87 @@ type Repository struct { idx *MasterIndex Cache *cache.Cache + opts Options + noAutoIndexUpdate bool - treePM *packerManager - dataPM *packerManager + packerWg *errgroup.Group + uploader *packerUploader + treePM *packerManager + dataPM *packerManager + + allocEnc sync.Once + allocDec sync.Once + enc *zstd.Encoder + dec *zstd.Decoder +} + +type Options struct { + Compression CompressionMode + PackSize uint +} + +// CompressionMode configures if data should be compressed. +type CompressionMode uint + +// Constants for the different compression levels. +const ( + CompressionAuto CompressionMode = 0 + CompressionOff CompressionMode = 1 + CompressionMax CompressionMode = 2 +) + +// Set implements the method needed for pflag command flag parsing. +func (c *CompressionMode) Set(s string) error { + switch s { + case "auto": + *c = CompressionAuto + case "off": + *c = CompressionOff + case "max": + *c = CompressionMax + default: + return fmt.Errorf("invalid compression mode %q, must be one of (auto|off|max)", s) + } + + return nil +} + +func (c *CompressionMode) String() string { + switch *c { + case CompressionAuto: + return "auto" + case CompressionOff: + return "off" + case CompressionMax: + return "max" + default: + return "invalid" + } + +} +func (c *CompressionMode) Type() string { + return "mode" } // New returns a new repository with backend be. -func New(be restic.Backend) *Repository { +func New(be restic.Backend, opts Options) (*Repository, error) { + if opts.PackSize == 0 { + opts.PackSize = DefaultPackSize + } + if opts.PackSize > MaxPackSize { + return nil, errors.Fatalf("pack size larger than limit of %v MiB", MaxPackSize/1024/1024) + } else if opts.PackSize < MinPackSize { + return nil, errors.Fatalf("pack size smaller than minimum of %v MiB", MinPackSize/1024/1024) + } + repo := &Repository{ - be: be, - idx: NewMasterIndex(), - dataPM: newPackerManager(be, nil), - treePM: newPackerManager(be, nil), + be: be, + opts: opts, + idx: NewMasterIndex(), } - return repo + return repo, nil } // DisableAutoIndexUpdate deactives the automatic finalization and upload of new @@ -58,11 +130,24 @@ func (r *Repository) DisableAutoIndexUpdate() { r.noAutoIndexUpdate = true } +// setConfig assigns the given config and updates the repository parameters accordingly +func (r *Repository) setConfig(cfg restic.Config) { + r.cfg = cfg + if r.cfg.Version >= 2 { + r.idx.markCompressed() + } +} + // Config returns the repository configuration. func (r *Repository) Config() restic.Config { return r.cfg } +// PackSize return the target size of a pack file when uploading +func (r *Repository) PackSize() uint { + return r.opts.PackSize +} + // UseCache replaces the backend with the wrapped cache. func (r *Repository) UseCache(c *cache.Cache) { if c == nil { @@ -84,10 +169,10 @@ func (r *Repository) PrefixLength(ctx context.Context, t restic.FileType) (int, return restic.PrefixLength(ctx, r.be, t) } -// LoadAndDecrypt loads and decrypts the file with the given type and ID, using +// LoadUnpacked loads and decrypts the file with the given type and ID, using // the supplied buffer (which must be empty). If the buffer is nil, a new // buffer will be allocated and returned. -func (r *Repository) LoadAndDecrypt(ctx context.Context, buf []byte, t restic.FileType, id restic.ID) ([]byte, error) { +func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id restic.ID, buf []byte) ([]byte, error) { if len(buf) != 0 { panic("buf is not empty") } @@ -123,6 +208,9 @@ func (r *Repository) LoadAndDecrypt(ctx context.Context, buf []byte, t restic.Fi if err != nil { return nil, err } + if t != restic.ConfigFile { + return r.decompressUnpacked(plaintext) + } return plaintext, nil } @@ -194,7 +282,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. buf = buf[:blob.Length] } - n, err := restic.ReadAt(ctx, r.be, h, int64(blob.Offset), buf) + n, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf) if err != nil { debug.Log("error loading blob %v: %v", blob, err) lastError = err @@ -216,15 +304,27 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. continue } + if blob.IsCompressed() { + plaintext, err = r.getZstdDecoder().DecodeAll(plaintext, make([]byte, 0, blob.DataLength())) + if err != nil { + lastError = errors.Errorf("decompressing blob %v failed: %v", id, err) + continue + } + } + // check hash if !restic.Hash(plaintext).Equal(id) { lastError = errors.Errorf("blob %v returned invalid hash", id) continue } + if len(plaintext) > cap(buf) { + return plaintext, nil + } // move decrypted data to the start of the buffer + buf = buf[:len(plaintext)] copy(buf, plaintext) - return buf[:len(plaintext)], nil + return buf, nil } if lastError != nil { @@ -234,31 +334,79 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. return nil, errors.Errorf("loading blob %v from %v packs failed", id.Str(), len(blobs)) } -// LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on -// the item. -func (r *Repository) LoadJSONUnpacked(ctx context.Context, t restic.FileType, id restic.ID, item interface{}) (err error) { - buf, err := r.LoadAndDecrypt(ctx, nil, t, id) - if err != nil { - return err - } - - return json.Unmarshal(buf, item) -} - // LookupBlobSize returns the size of blob id. func (r *Repository) LookupBlobSize(id restic.ID, tpe restic.BlobType) (uint, bool) { return r.idx.LookupSize(restic.BlobHandle{ID: id, Type: tpe}) } -// SaveAndEncrypt encrypts data and stores it to the backend as type t. If data -// is small enough, it will be packed together with other small blobs. -// The caller must ensure that the id matches the data. -func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data []byte, id restic.ID) error { +func (r *Repository) getZstdEncoder() *zstd.Encoder { + r.allocEnc.Do(func() { + level := zstd.SpeedDefault + if r.opts.Compression == CompressionMax { + level = zstd.SpeedBestCompression + } + + opts := []zstd.EOption{ + // Set the compression level configured. + zstd.WithEncoderLevel(level), + // Disable CRC, we have enough checks in place, makes the + // compressed data four bytes shorter. + zstd.WithEncoderCRC(false), + // Set a window of 512kbyte, so we have good lookbehind for usual + // blob sizes. + zstd.WithWindowSize(512 * 1024), + } + + enc, err := zstd.NewWriter(nil, opts...) + if err != nil { + panic(err) + } + r.enc = enc + }) + return r.enc +} + +func (r *Repository) getZstdDecoder() *zstd.Decoder { + r.allocDec.Do(func() { + opts := []zstd.DOption{ + // Use all available cores. + zstd.WithDecoderConcurrency(0), + // Limit the maximum decompressed memory. Set to a very high, + // conservative value. + zstd.WithDecoderMaxMemory(16 * 1024 * 1024 * 1024), + } + + dec, err := zstd.NewReader(nil, opts...) + if err != nil { + panic(err) + } + r.dec = dec + }) + return r.dec +} + +// saveAndEncrypt encrypts data and stores it to the backend as type t. If data +// is small enough, it will be packed together with other small blobs. The +// caller must ensure that the id matches the data. Returned is the size data +// occupies in the repo (compressed or not, including the encryption overhead). +func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data []byte, id restic.ID) (size int, err error) { debug.Log("save id %v (%v, %d bytes)", id, t, len(data)) + uncompressedLength := 0 + if r.cfg.Version > 1 { + + // we have a repo v2, so compression is available. if the user opts to + // not compress, we won't compress any data, but everything else is + // compressed. + if r.opts.Compression != CompressionOff || t != restic.DataBlob { + uncompressedLength = len(data) + data = r.getZstdEncoder().EncodeAll(data, nil) + } + } + nonce := crypto.NewRandomNonce() - ciphertext := make([]byte, 0, restic.CiphertextLength(len(data))) + ciphertext := make([]byte, 0, crypto.CiphertextLength(len(data))) ciphertext = append(ciphertext, nonce...) // encrypt blob @@ -276,44 +424,54 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data panic(fmt.Sprintf("invalid type: %v", t)) } - packer, err := pm.findPacker() - if err != nil { - return err - } - - // save ciphertext - _, err = packer.Add(t, id, ciphertext) - if err != nil { - return err - } + return pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength) +} - // if the pack is not full enough, put back to the list - if packer.Size() < minPackSize { - debug.Log("pack is not full enough (%d bytes)", packer.Size()) - pm.insertPacker(packer) - return nil +func (r *Repository) compressUnpacked(p []byte) ([]byte, error) { + // compression is only available starting from version 2 + if r.cfg.Version < 2 { + return p, nil } - // else write the pack to the backend - return r.savePacker(ctx, t, packer) + // version byte + out := []byte{2} + out = r.getZstdEncoder().EncodeAll(p, out) + return out, nil } -// SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the -// backend as type t, without a pack. It returns the storage hash. -func (r *Repository) SaveJSONUnpacked(ctx context.Context, t restic.FileType, item interface{}) (restic.ID, error) { - debug.Log("save new blob %v", t) - plaintext, err := json.Marshal(item) - if err != nil { - return restic.ID{}, errors.Wrap(err, "json.Marshal") +func (r *Repository) decompressUnpacked(p []byte) ([]byte, error) { + // compression is only available starting from version 2 + if r.cfg.Version < 2 { + return p, nil + } + + if len(p) == 0 { + // too short for version header + return p, nil + } + if p[0] == '[' || p[0] == '{' { + // probably raw JSON + return p, nil + } + // version + if p[0] != 2 { + return nil, errors.New("not supported encoding format") } - return r.SaveUnpacked(ctx, t, plaintext) + return r.getZstdDecoder().DecodeAll(p[1:], nil) } // SaveUnpacked encrypts data and stores it in the backend. Returned is the // storage hash. func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) { - ciphertext := restic.NewBlobBuffer(len(p)) + if t != restic.ConfigFile { + p, err = r.compressUnpacked(p) + if err != nil { + return restic.ID{}, err + } + } + + ciphertext := crypto.NewBlobBuffer(len(p)) ciphertext = ciphertext[:0] nonce := crypto.NewRandomNonce() ciphertext = append(ciphertext, nonce...) @@ -339,7 +497,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by // Flush saves all remaining packs and the index func (r *Repository) Flush(ctx context.Context) error { - if err := r.FlushPacks(ctx); err != nil { + if err := r.flushPacks(ctx); err != nil { return err } @@ -347,34 +505,48 @@ func (r *Repository) Flush(ctx context.Context) error { if r.noAutoIndexUpdate { return nil } - return r.SaveIndex(ctx) + return r.idx.SaveIndex(ctx, r) } -// FlushPacks saves all remaining packs. -func (r *Repository) FlushPacks(ctx context.Context) error { - pms := []struct { - t restic.BlobType - pm *packerManager - }{ - {restic.DataBlob, r.dataPM}, - {restic.TreeBlob, r.treePM}, +func (r *Repository) StartPackUploader(ctx context.Context, wg *errgroup.Group) { + if r.packerWg != nil { + panic("uploader already started") } - for _, p := range pms { - p.pm.pm.Lock() + innerWg, ctx := errgroup.WithContext(ctx) + r.packerWg = innerWg + r.uploader = newPackerUploader(ctx, innerWg, r, r.be.Connections()) + r.treePM = newPackerManager(r.key, restic.TreeBlob, r.PackSize(), r.uploader.QueuePacker) + r.dataPM = newPackerManager(r.key, restic.DataBlob, r.PackSize(), r.uploader.QueuePacker) - debug.Log("manually flushing %d packs", len(p.pm.packers)) - for _, packer := range p.pm.packers { - err := r.savePacker(ctx, p.t, packer) - if err != nil { - p.pm.pm.Unlock() - return err - } - } - p.pm.packers = p.pm.packers[:0] - p.pm.pm.Unlock() + wg.Go(func() error { + return innerWg.Wait() + }) +} + +// FlushPacks saves all remaining packs. +func (r *Repository) flushPacks(ctx context.Context) error { + if r.packerWg == nil { + return nil } - return nil + + err := r.treePM.Flush(ctx) + if err != nil { + return err + } + err = r.dataPM.Flush(ctx) + if err != nil { + return err + } + r.uploader.TriggerShutdown() + err = r.packerWg.Wait() + + r.treePM = nil + r.dataPM = nil + r.uploader = nil + r.packerWg = nil + + return err } // Backend returns the backend for the repository. @@ -382,6 +554,10 @@ func (r *Repository) Backend() restic.Backend { return r.be } +func (r *Repository) Connections() uint { + return r.be.Connections() +} + // Index returns the currently used MasterIndex. func (r *Repository) Index() restic.MasterIndex { return r.idx @@ -390,58 +566,7 @@ func (r *Repository) Index() restic.MasterIndex { // SetIndex instructs the repository to use the given index. func (r *Repository) SetIndex(i restic.MasterIndex) error { r.idx = i.(*MasterIndex) - - ids := restic.NewIDSet() - for _, idx := range r.idx.All() { - indexIDs, err := idx.IDs() - if err != nil { - debug.Log("not using index, ID() returned error %v", err) - continue - } - for _, id := range indexIDs { - ids.Insert(id) - } - } - - return r.PrepareCache(ids) -} - -// SaveIndex saves an index in the repository. -func SaveIndex(ctx context.Context, repo restic.Repository, index *Index) (restic.ID, error) { - buf := bytes.NewBuffer(nil) - - err := index.Encode(buf) - if err != nil { - return restic.ID{}, err - } - - return repo.SaveUnpacked(ctx, restic.IndexFile, buf.Bytes()) -} - -// saveIndex saves all indexes in the backend. -func (r *Repository) saveIndex(ctx context.Context, indexes ...*Index) error { - for i, idx := range indexes { - debug.Log("Saving index %d", i) - - sid, err := SaveIndex(ctx, r, idx) - if err != nil { - return err - } - - debug.Log("Saved index %d as %v", i, sid) - } - - return r.idx.MergeFinalIndexes() -} - -// SaveIndex saves all new indexes in the backend. -func (r *Repository) SaveIndex(ctx context.Context) error { - return r.saveIndex(ctx, r.idx.FinalizeNotFinalIndexes()...) -} - -// SaveFullIndex saves all full indexes in the backend. -func (r *Repository) SaveFullIndex(ctx context.Context) error { - return r.saveIndex(ctx, r.idx.FinalizeFullIndexes()...) + return r.PrepareCache() } // LoadIndex loads all index files from the backend in parallel and stores them @@ -449,20 +574,10 @@ func (r *Repository) SaveFullIndex(ctx context.Context) error { func (r *Repository) LoadIndex(ctx context.Context) error { debug.Log("Loading index") - validIndex := restic.NewIDSet() err := ForAllIndexes(ctx, r, func(id restic.ID, idx *Index, oldFormat bool, err error) error { if err != nil { return err } - - ids, err := idx.IDs() - if err != nil { - return err - } - - for _, id := range ids { - validIndex.Insert(id) - } r.idx.Insert(idx) return nil }) @@ -476,12 +591,21 @@ func (r *Repository) LoadIndex(ctx context.Context) error { return err } + if r.cfg.Version < 2 { + // sanity check + ctx, cancel := context.WithCancel(ctx) + defer cancel() + for blob := range r.idx.Each(ctx) { + if blob.IsCompressed() { + return errors.Fatal("index uses feature not supported by repository version 1") + } + } + } + // remove index files from the cache which have been removed in the repo - return r.PrepareCache(validIndex) + return r.PrepareCache() } -const listPackParallelism = 10 - // CreateIndexFromPacks creates a new index by reading all given pack files (with sizes). // The index is added to the MasterIndex but not marked as finalized. // Returned is the list of pack files which could not be read. @@ -506,14 +630,13 @@ func (r *Repository) CreateIndexFromPacks(ctx context.Context, packsize map[rest for id, size := range packsize { select { case <-ctx.Done(): - return nil + return ctx.Err() case ch <- FileInfo{id, size}: } } return nil }) - idx := NewIndex() // a worker receives an pack ID from ch, reads the pack contents, and adds them to idx worker := func() error { for fi := range ch { @@ -524,36 +647,36 @@ func (r *Repository) CreateIndexFromPacks(ctx context.Context, packsize map[rest invalid = append(invalid, fi.ID) m.Unlock() } - idx.StorePack(fi.ID, entries) + r.idx.StorePack(fi.ID, entries) p.Add(1) } return nil } + // decoding the pack header is usually quite fast, thus we are primarily IO-bound + workerCount := int(r.Connections()) // run workers on ch - wg.Go(func() error { - return RunWorkers(listPackParallelism, worker) - }) + for i := 0; i < workerCount; i++ { + wg.Go(worker) + } err = wg.Wait() if err != nil { return invalid, errors.Fatal(err.Error()) } - // Add idx to MasterIndex - r.idx.Insert(idx) - return invalid, nil } // PrepareCache initializes the local cache. indexIDs is the list of IDs of // index files still present in the repo. -func (r *Repository) PrepareCache(indexIDs restic.IDSet) error { +func (r *Repository) PrepareCache() error { if r.Cache == nil { return nil } + indexIDs := r.idx.IDs() debug.Log("prepare cache with %d index files", len(indexIDs)) // clear old index files @@ -562,12 +685,7 @@ func (r *Repository) PrepareCache(indexIDs restic.IDSet) error { fmt.Fprintf(os.Stderr, "error clearing index files in cache: %v\n", err) } - packs := restic.NewIDSet() - for _, idx := range r.idx.All() { - for id := range idx.Packs() { - packs.Insert(id) - } - } + packs := r.idx.Packs(restic.NewIDSet()) // clear old packs err = r.Cache.Clear(restic.PackFile, packs) @@ -587,19 +705,29 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int } r.key = key.master - r.dataPM.key = key.master - r.treePM.key = key.master r.keyName = key.Name() - r.cfg, err = restic.LoadConfig(ctx, r) - if err != nil { + cfg, err := restic.LoadConfig(ctx, r) + if err == crypto.ErrUnauthenticated { + return errors.Fatalf("config or key %v is damaged: %v", key.Name(), err) + } else if err != nil { return errors.Fatalf("config cannot be loaded: %v", err) } + + r.setConfig(cfg) return nil } // Init creates a new master key with the supplied password, initializes and // saves the repository config. -func (r *Repository) Init(ctx context.Context, password string, chunkerPolynomial *chunker.Pol) (bool, error) { +func (r *Repository) Init(ctx context.Context, version uint, password string, chunkerPolynomial *chunker.Pol)(bool, error) { + if version > restic.MaxRepoVersion { + return false, fmt.Errorf("repository version %v too high", version) + } + + if version < restic.MinRepoVersion { + return false, fmt.Errorf("repository version %v too low", version) + } + has, err := r.be.Test(ctx, restic.Handle{Type: restic.ConfigFile}) if err != nil { return false, err @@ -612,7 +740,7 @@ func (r *Repository) Init(ctx context.Context, password string, chunkerPolynomia return false, r.be.List(ctx, restic.LockFile, func(_ restic.FileInfo) error { return nil }) } - cfg, err := restic.CreateConfig() + cfg, err := restic.CreateConfig(version) if err != nil { return false, err } @@ -636,12 +764,9 @@ func (r *Repository) init(ctx context.Context, password string, cfg restic.Confi } r.key = key.master - r.dataPM.key = key.master - r.treePM.key = key.master r.keyName = key.Name() - r.cfg = cfg - _, err = r.SaveJSONUnpacked(ctx, restic.ConfigFile, cfg) - return err + r.setConfig(cfg) + return restic.SaveConfig(ctx, r, cfg) } // Key returns the current master key. @@ -671,7 +796,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) { h := restic.Handle{Type: restic.PackFile, Name: id.String()} - return pack.List(r.Key(), restic.ReaderAt(ctx, r.Backend(), h), size) + return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size) } // Delete calls backend.Delete() if implemented, and returns an error @@ -689,8 +814,10 @@ func (r *Repository) Close() error { // It takes care that no duplicates are saved; this can be overwritten // by setting storeDuplicate to true. // If id is the null id, it will be computed and returned. -// Also returns if the blob was already known before -func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, err error) { +// Also returns if the blob was already known before. +// If the blob was not known before, it returns the number of bytes the blob +// occupies in the repo (compressed or not, including encryption overhead). +func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) { // compute plaintext hash if not already set if id.IsNull() { @@ -704,89 +831,147 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte // only save when needed or explicitly told if !known || storeDuplicate { - err = r.SaveAndEncrypt(ctx, t, buf, newID) + size, err = r.saveAndEncrypt(ctx, t, buf, newID) } - return newID, known, err + return newID, known, size, err } -// LoadTree loads a tree from the repository. -func (r *Repository) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) { - debug.Log("load tree %v", id) +type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error - buf, err := r.LoadBlob(ctx, restic.TreeBlob, id, nil) - if err != nil { - return nil, err - } +// Skip sections with more than 4MB unused blobs +const maxUnusedRange = 4 * 1024 * 1024 - t := &restic.Tree{} - err = json.Unmarshal(buf, t) - if err != nil { - return nil, err +// StreamPack loads the listed blobs from the specified pack file. The plaintext blob is passed to +// the handleBlobFn callback or an error if decryption failed or the blob hash does not match. In +// case of download errors handleBlobFn might be called multiple times for the same blob. If the +// callback returns an error, then StreamPack will abort and not retry it. +func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + if len(blobs) == 0 { + // nothing to do + return nil } - return t, nil -} + sort.Slice(blobs, func(i, j int) bool { + return blobs[i].Offset < blobs[j].Offset + }) -// SaveTree stores a tree into the repository and returns the ID. The ID is -// checked against the index. The tree is only stored when the index does not -// contain the ID. -func (r *Repository) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, error) { - buf, err := json.Marshal(t) - if err != nil { - return restic.ID{}, errors.Wrap(err, "MarshalJSON") + lowerIdx := 0 + lastPos := blobs[0].Offset + for i := 0; i < len(blobs); i++ { + if blobs[i].Offset < lastPos { + // don't wait for streamPackPart to fail + return errors.Errorf("overlapping blobs in pack %v", packID) + } + if blobs[i].Offset-lastPos > maxUnusedRange { + // load everything up to the skipped file section + err := streamPackPart(ctx, beLoad, key, packID, blobs[lowerIdx:i], handleBlobFn) + if err != nil { + return err + } + lowerIdx = i + } + lastPos = blobs[i].Offset + blobs[i].Length } + // load remainder + return streamPackPart(ctx, beLoad, key, packID, blobs[lowerIdx:], handleBlobFn) +} - // append a newline so that the data is always consistent (json.Encoder - // adds a newline after each object) - buf = append(buf, '\n') +func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + h := restic.Handle{Type: restic.PackFile, Name: packID.String(), ContainedBlobType: restic.DataBlob} - id, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false) - return id, err -} + dataStart := blobs[0].Offset + dataEnd := blobs[len(blobs)-1].Offset + blobs[len(blobs)-1].Length -// Loader allows loading data from a backend. -type Loader interface { - Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error -} + debug.Log("streaming pack %v (%d to %d bytes), blobs: %v", packID, dataStart, dataEnd, len(blobs)) -// DownloadAndHash is all-in-one helper to download content of the file at h to a temporary filesystem location -// and calculate ID of the contents. Returned (temporary) file is positioned at the beginning of the file; -// it is the reponsibility of the caller to close and delete the file. -func DownloadAndHash(ctx context.Context, be Loader, h restic.Handle) (tmpfile *os.File, hash restic.ID, size int64, err error) { - tmpfile, err = fs.TempFile("", "restic-temp-") + dec, err := zstd.NewReader(nil) if err != nil { - return nil, restic.ID{}, -1, errors.Wrap(err, "TempFile") + panic(dec) } + defer dec.Close() - err = be.Load(ctx, h, 0, 0, func(rd io.Reader) (ierr error) { - _, ierr = tmpfile.Seek(0, io.SeekStart) - if ierr == nil { - ierr = tmpfile.Truncate(0) + ctx, cancel := context.WithCancel(ctx) + // stream blobs in pack + err = beLoad(ctx, h, int(dataEnd-dataStart), int64(dataStart), func(rd io.Reader) error { + // prevent callbacks after cancelation + if ctx.Err() != nil { + return ctx.Err() } - if ierr != nil { - return ierr + bufferSize := int(dataEnd - dataStart) + if bufferSize > MaxStreamBufferSize { + bufferSize = MaxStreamBufferSize } - hrd := hashing.NewReader(rd, sha256.New()) - size, ierr = io.Copy(tmpfile, hrd) - hash = restic.IDFromHash(hrd.Sum(nil)) - return ierr - }) + // create reader here to allow reusing the buffered reader from checker.checkData + bufRd := bufio.NewReaderSize(rd, bufferSize) + currentBlobEnd := dataStart + var buf []byte + var decode []byte + for _, entry := range blobs { + skipBytes := int(entry.Offset - currentBlobEnd) + if skipBytes < 0 { + return errors.Errorf("overlapping blobs in pack %v", packID) + } - if err != nil { - // ignore subsequent errors - _ = tmpfile.Close() - _ = os.Remove(tmpfile.Name()) - return nil, restic.ID{}, -1, errors.Wrap(err, "Load") - } + _, err := bufRd.Discard(skipBytes) + if err != nil { + return err + } - _, err = tmpfile.Seek(0, io.SeekStart) - if err != nil { - // ignore subsequent errors - _ = tmpfile.Close() - _ = os.Remove(tmpfile.Name()) - return nil, restic.ID{}, -1, errors.Wrap(err, "Seek") - } + h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} + debug.Log(" process blob %v, skipped %d, %v", h, skipBytes, entry) + + if uint(cap(buf)) < entry.Length { + buf = make([]byte, entry.Length) + } + buf = buf[:entry.Length] + + n, err := io.ReadFull(bufRd, buf) + if err != nil { + debug.Log(" read error %v", err) + return errors.Wrap(err, "ReadFull") + } - return tmpfile, hash, size, err + if n != len(buf) { + return errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v", + h, packID.Str(), len(buf), n) + } + currentBlobEnd = entry.Offset + entry.Length + + if int(entry.Length) <= key.NonceSize() { + debug.Log("%v", blobs) + return errors.Errorf("invalid blob length %v", entry) + } + + // decryption errors are likely permanent, give the caller a chance to skip them + nonce, ciphertext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err := key.Open(ciphertext[:0], nonce, ciphertext, nil) + if err == nil && entry.IsCompressed() { + // DecodeAll will allocate a slice if it is not large enough since it + // knows the decompressed size (because we're using EncodeAll) + decode, err = dec.DecodeAll(plaintext, decode[:0]) + plaintext = decode + if err != nil { + err = errors.Errorf("decompressing blob %v failed: %v", h, err) + } + } + if err == nil { + id := restic.Hash(plaintext) + if !id.Equal(entry.ID) { + debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v", + h.Type, h.ID, packID.Str(), id) + err = errors.Errorf("read blob %v from %v: wrong data returned, hash is %v", + h, packID.Str(), id) + } + } + + err = handleBlobFn(entry.BlobHandle, plaintext, err) + if err != nil { + cancel() + return backoff.Permanent(err) + } + } + return nil + }) + return errors.Wrap(err, "StreamPack") } diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 22a4245561f..b5b0ff92d9a 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -4,20 +4,24 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/json" "fmt" "io" "math/rand" "os" "path/filepath" + "strings" "testing" "time" - "github.com/restic/restic/internal/archiver" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" + "github.com/google/go-cmp/cmp" + "github.com/klauspost/compress/zstd" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" + "golang.org/x/sync/errgroup" ) var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} @@ -25,7 +29,11 @@ var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) func TestSave(t *testing.T) { - repo, cleanup := repository.TestRepository(t) + repository.TestAllVersions(t, testSave) +} + +func testSave(t *testing.T, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) defer cleanup() for _, size := range testSizes { @@ -35,8 +43,11 @@ func TestSave(t *testing.T) { id := restic.Hash(data) + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + // save - sid, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false) + sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false) rtest.OK(t, err) rtest.Equals(t, id, sid) @@ -60,7 +71,11 @@ func TestSave(t *testing.T) { } func TestSaveFrom(t *testing.T) { - repo, cleanup := repository.TestRepository(t) + repository.TestAllVersions(t, testSaveFrom) +} + +func testSaveFrom(t *testing.T, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) defer cleanup() for _, size := range testSizes { @@ -70,8 +85,11 @@ func TestSaveFrom(t *testing.T) { id := restic.Hash(data) + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + // save - id2, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false) + id2, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false) rtest.OK(t, err) rtest.Equals(t, id, id2) @@ -93,7 +111,11 @@ func TestSaveFrom(t *testing.T) { } func BenchmarkSaveAndEncrypt(t *testing.B) { - repo, cleanup := repository.TestRepository(t) + repository.BenchmarkAllVersions(t, benchmarkSaveAndEncrypt) +} + +func benchmarkSaveAndEncrypt(t *testing.B, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) defer cleanup() size := 4 << 20 // 4MiB @@ -109,61 +131,32 @@ func BenchmarkSaveAndEncrypt(t *testing.B) { t.SetBytes(int64(size)) for i := 0; i < t.N; i++ { - _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, true) + _, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, true) rtest.OK(t, err) } } -func TestLoadTree(t *testing.T) { - repo, cleanup := repository.TestRepository(t) - defer cleanup() - - if rtest.BenchArchiveDirectory == "" { - t.Skip("benchdir not set, skipping") - } - - // archive a few files - sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil) - rtest.OK(t, repo.Flush(context.Background())) - - _, err := repo.LoadTree(context.TODO(), *sn.Tree) - rtest.OK(t, err) -} - -func BenchmarkLoadTree(t *testing.B) { - repo, cleanup := repository.TestRepository(t) - defer cleanup() - - if rtest.BenchArchiveDirectory == "" { - t.Skip("benchdir not set, skipping") - } - - // archive a few files - sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil) - rtest.OK(t, repo.Flush(context.Background())) - - t.ResetTimer() - - for i := 0; i < t.N; i++ { - _, err := repo.LoadTree(context.TODO(), *sn.Tree) - rtest.OK(t, err) - } +func TestLoadBlob(t *testing.T) { + repository.TestAllVersions(t, testLoadBlob) } -func TestLoadBlob(t *testing.T) { - repo, cleanup := repository.TestRepository(t) +func testLoadBlob(t *testing.T, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) defer cleanup() length := 1000000 - buf := restic.NewBlobBuffer(length) + buf := crypto.NewBlobBuffer(length) _, err := io.ReadFull(rnd, buf) rtest.OK(t, err) - id, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + + id, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) rtest.OK(t, err) rtest.OK(t, repo.Flush(context.Background())) - base := restic.CiphertextLength(length) + base := crypto.CiphertextLength(length) for _, testlength := range []int{0, base - 20, base - 1, base, base + 7, base + 15, base + 1000} { buf = make([]byte, 0, testlength) buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) @@ -180,15 +173,22 @@ func TestLoadBlob(t *testing.T) { } func BenchmarkLoadBlob(b *testing.B) { - repo, cleanup := repository.TestRepository(b) + repository.BenchmarkAllVersions(b, benchmarkLoadBlob) +} + +func benchmarkLoadBlob(b *testing.B, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(b, version) defer cleanup() length := 1000000 - buf := restic.NewBlobBuffer(length) + buf := crypto.NewBlobBuffer(length) _, err := io.ReadFull(rnd, buf) rtest.OK(b, err) - id, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + + id, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) rtest.OK(b, err) rtest.OK(b, repo.Flush(context.Background())) @@ -215,12 +215,16 @@ func BenchmarkLoadBlob(b *testing.B) { } } -func BenchmarkLoadAndDecrypt(b *testing.B) { - repo, cleanup := repository.TestRepository(b) +func BenchmarkLoadUnpacked(b *testing.B) { + repository.BenchmarkAllVersions(b, benchmarkLoadUnpacked) +} + +func benchmarkLoadUnpacked(b *testing.B, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(b, version) defer cleanup() length := 1000000 - buf := restic.NewBlobBuffer(length) + buf := crypto.NewBlobBuffer(length) _, err := io.ReadFull(rnd, buf) rtest.OK(b, err) @@ -234,7 +238,7 @@ func BenchmarkLoadAndDecrypt(b *testing.B) { b.SetBytes(int64(length)) for i := 0; i < b.N; i++ { - data, err := repo.LoadAndDecrypt(context.TODO(), nil, restic.PackFile, storageID) + data, err := repo.LoadUnpacked(context.TODO(), restic.PackFile, storageID, nil) rtest.OK(b, err) // See comment in BenchmarkLoadBlob. @@ -251,40 +255,6 @@ func BenchmarkLoadAndDecrypt(b *testing.B) { } } -func TestLoadJSONUnpacked(t *testing.T) { - repo, cleanup := repository.TestRepository(t) - defer cleanup() - - if rtest.BenchArchiveDirectory == "" { - t.Skip("benchdir not set, skipping") - } - - // archive a snapshot - sn := restic.Snapshot{} - sn.Hostname = "foobar" - sn.Username = "test!" - - id, err := repo.SaveJSONUnpacked(context.TODO(), restic.SnapshotFile, &sn) - rtest.OK(t, err) - - var sn2 restic.Snapshot - - // restore - err = repo.LoadJSONUnpacked(context.TODO(), restic.SnapshotFile, id, &sn2) - rtest.OK(t, err) - - rtest.Equals(t, sn.Hostname, sn2.Hostname) - rtest.Equals(t, sn.Username, sn2.Username) - - var cf restic.Config - - // load and check Config - err = repo.LoadJSONUnpacked(context.TODO(), restic.ConfigFile, id, &cf) - rtest.OK(t, err) - - rtest.Equals(t, cf.ChunkerPolynomial, repository.TestChunkerPol) -} - var repoFixture = filepath.Join("testdata", "test-repo.tar.gz") func TestRepositoryLoadIndex(t *testing.T) { @@ -297,7 +267,7 @@ func TestRepositoryLoadIndex(t *testing.T) { // loadIndex loads the index id from backend and returns it. func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*repository.Index, error) { - buf, err := repo.LoadAndDecrypt(ctx, nil, restic.IndexFile, id) + buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil) if err != nil { return nil, err } @@ -310,28 +280,31 @@ func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*repo } func BenchmarkLoadIndex(b *testing.B) { + repository.BenchmarkAllVersions(b, benchmarkLoadIndex) +} + +func benchmarkLoadIndex(b *testing.B, version uint) { repository.TestUseLowSecurityKDFParameters(b) - repo, cleanup := repository.TestRepository(b) + repo, cleanup := repository.TestRepositoryWithVersion(b, version) defer cleanup() idx := repository.NewIndex() for i := 0; i < 5000; i++ { - idx.Store(restic.PackedBlob{ - Blob: restic.Blob{ + idx.StorePack(restic.NewRandomID(), []restic.Blob{ + { BlobHandle: restic.NewRandomBlobHandle(), Length: 1234, Offset: 1235, }, - PackID: restic.NewRandomID(), }) } id, err := repository.SaveIndex(context.TODO(), repo, idx) rtest.OK(b, err) - b.Logf("index saved as %v (%v entries)", id.Str(), idx.Count(restic.DataBlob)) + b.Logf("index saved as %v", id.Str()) fi, err := repo.Backend().Stat(context.TODO(), restic.Handle{Type: restic.IndexFile, Name: id.String()}) rtest.OK(b, err) b.Logf("filesize is %v", fi.Size) @@ -346,6 +319,9 @@ func BenchmarkLoadIndex(b *testing.B) { // saveRandomDataBlobs generates random data blobs and saves them to the repository. func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) { + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + for i := 0; i < num; i++ { size := rand.Int() % sizeMax @@ -353,38 +329,32 @@ func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax _, err := io.ReadFull(rnd, buf) rtest.OK(t, err) - _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) + _, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) rtest.OK(t, err) } } func TestRepositoryIncrementalIndex(t *testing.T) { - r, cleanup := repository.TestRepository(t) + repository.TestAllVersions(t, testRepositoryIncrementalIndex) +} + +func testRepositoryIncrementalIndex(t *testing.T, version uint) { + r, cleanup := repository.TestRepositoryWithVersion(t, version) defer cleanup() repo := r.(*repository.Repository) - repository.IndexFull = func(*repository.Index) bool { return true } + repository.IndexFull = func(*repository.Index, bool) bool { return true } - // add 15 packs + // add a few rounds of packs for j := 0; j < 5; j++ { - // add 3 packs, write intermediate index - for i := 0; i < 3; i++ { - saveRandomDataBlobs(t, repo, 5, 1<<15) - rtest.OK(t, repo.FlushPacks(context.Background())) - } - - rtest.OK(t, repo.SaveFullIndex(context.TODO())) - } - - // add another 5 packs - for i := 0; i < 5; i++ { - saveRandomDataBlobs(t, repo, 5, 1<<15) - rtest.OK(t, repo.FlushPacks(context.Background())) + // add some packs, write intermediate index + saveRandomDataBlobs(t, repo, 20, 1<<15) + rtest.OK(t, repo.Flush(context.TODO())) } // save final index - rtest.OK(t, repo.SaveIndex(context.TODO())) + rtest.OK(t, repo.Flush(context.TODO())) packEntries := make(map[restic.ID]map[restic.ID]struct{}) @@ -410,108 +380,245 @@ func TestRepositoryIncrementalIndex(t *testing.T) { t.Errorf("pack %v listed in %d indexes\n", packID, len(ids)) } } -} -type backend struct { - rd io.Reader } -func (be backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return fn(be.rd) -} +// buildPackfileWithoutHeader returns a manually built pack file without a header. +func buildPackfileWithoutHeader(t testing.TB, blobSizes []int, key *crypto.Key, compress bool) (blobs []restic.Blob, packfile []byte) { + opts := []zstd.EOption{ + // Set the compression level configured. + zstd.WithEncoderLevel(zstd.SpeedDefault), + // Disable CRC, we have enough checks in place, makes the + // compressed data four bytes shorter. + zstd.WithEncoderCRC(false), + // Set a window of 512kbyte, so we have good lookbehind for usual + // blob sizes. + zstd.WithWindowSize(512 * 1024), + } + enc, err := zstd.NewWriter(nil, opts...) + if err != nil { + panic(err) + } -type retryBackend struct { - buf []byte -} + var offset uint + for i, size := range blobSizes { + plaintext := test.Random(800+i, size) + id := restic.Hash(plaintext) + uncompressedLength := uint(0) + if compress { + uncompressedLength = uint(len(plaintext)) + plaintext = enc.EncodeAll(plaintext, nil) + } -func (be retryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - err := fn(bytes.NewReader(be.buf[:len(be.buf)/2])) - if err != nil { - return err + // we use a deterministic nonce here so the whole process is + // deterministic, last byte is the blob index + var nonce = []byte{ + 0x15, 0x98, 0xc0, 0xf7, 0xb9, 0x65, 0x97, 0x74, + 0x12, 0xdc, 0xd3, 0x62, 0xa9, 0x6e, 0x20, byte(i), + } + + before := len(packfile) + packfile = append(packfile, nonce...) + packfile = key.Seal(packfile, nonce, plaintext, nil) + after := len(packfile) + + ciphertextLength := after - before + + blobs = append(blobs, restic.Blob{ + BlobHandle: restic.BlobHandle{ + Type: restic.DataBlob, + ID: id, + }, + Length: uint(ciphertextLength), + UncompressedLength: uncompressedLength, + Offset: offset, + }) + + offset = uint(len(packfile)) } - return fn(bytes.NewReader(be.buf)) + return blobs, packfile } -func TestDownloadAndHash(t *testing.T) { - buf := make([]byte, 5*1024*1024+881) - _, err := io.ReadFull(rnd, buf) +func TestStreamPack(t *testing.T) { + repository.TestAllVersions(t, testStreamPack) +} + +func testStreamPack(t *testing.T, version uint) { + // always use the same key for deterministic output + const jsonKey = `{"mac":{"k":"eQenuI8adktfzZMuC8rwdA==","r":"k8cfAly2qQSky48CQK7SBA=="},"encrypt":"MKO9gZnRiQFl8mDUurSDa9NMjiu9MUifUrODTHS05wo="}` + + var key crypto.Key + err := json.Unmarshal([]byte(jsonKey), &key) if err != nil { t.Fatal(err) } - var tests = []struct { - be repository.Loader - want []byte - }{ - { - be: backend{rd: bytes.NewReader(buf)}, - want: buf, - }, - { - be: retryBackend{buf: buf}, - want: buf, - }, + blobSizes := []int{ + 5522811, + 10, + 5231, + 18812, + 123123, + 13522811, + 12301, + 892242, + 28616, + 13351, + 252287, + 188883, + 3522811, + 18883, } - for _, test := range tests { - t.Run("", func(t *testing.T) { - f, id, size, err := repository.DownloadAndHash(context.TODO(), test.be, restic.Handle{}) - if err != nil { - t.Error(err) - } + var compress bool + switch version { + case 1: + compress = false + case 2: + compress = true + default: + t.Fatal("test does not suport repository version", version) + } - want := restic.Hash(test.want) - if !want.Equal(id) { - t.Errorf("wrong hash returned, want %v, got %v", want.Str(), id.Str()) - } + packfileBlobs, packfile := buildPackfileWithoutHeader(t, blobSizes, &key, compress) - if size != int64(len(test.want)) { - t.Errorf("wrong size returned, want %v, got %v", test.want, size) - } + loadCalls := 0 + load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + data := packfile - err = f.Close() - if err != nil { - t.Error(err) - } + if offset > int64(len(data)) { + offset = 0 + length = 0 + } + data = data[offset:] - err = fs.RemoveIfExists(f.Name()) - if err != nil { - t.Fatal(err) - } - }) - } -} + if length > len(data) { + length = len(data) + } -type errorReader struct { - err error -} + data = data[:length] + loadCalls++ -func (er errorReader) Read(p []byte) (n int, err error) { - return 0, er.err -} + return fn(bytes.NewReader(data)) -func TestDownloadAndHashErrors(t *testing.T) { - var tests = []struct { - be repository.Loader - err string - }{ - { - be: backend{rd: errorReader{errors.New("test error 1")}}, - err: "test error 1", - }, } - for _, test := range tests { - t.Run("", func(t *testing.T) { - _, _, _, err := repository.DownloadAndHash(context.TODO(), test.be, restic.Handle{}) - if err == nil { - t.Fatalf("wanted error %q, got nil", test.err) - } + // first, test regular usage + t.Run("regular", func(t *testing.T) { + tests := []struct { + blobs []restic.Blob + calls int + }{ + {packfileBlobs[1:2], 1}, + {packfileBlobs[2:5], 1}, + {packfileBlobs[2:8], 1}, + {[]restic.Blob{ + packfileBlobs[0], + packfileBlobs[4], + packfileBlobs[2], + }, 1}, + {[]restic.Blob{ + packfileBlobs[0], + packfileBlobs[len(packfileBlobs)-1], + }, 2}, + } - if errors.Cause(err).Error() != test.err { - t.Fatalf("wanted error %q, got %q", test.err, err) - } - }) - } + for _, test := range tests { + t.Run("", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + gotBlobs := make(map[restic.ID]int) + + handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { + gotBlobs[blob.ID]++ + + id := restic.Hash(buf) + if !id.Equal(blob.ID) { + t.Fatalf("wrong id %v for blob %s returned", id, blob.ID) + } + + return err + } + + wantBlobs := make(map[restic.ID]int) + for _, blob := range test.blobs { + wantBlobs[blob.ID] = 1 + } + + loadCalls = 0 + err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) + if err != nil { + t.Fatal(err) + } + + if !cmp.Equal(wantBlobs, gotBlobs) { + t.Fatal(cmp.Diff(wantBlobs, gotBlobs)) + } + rtest.Equals(t, test.calls, loadCalls) + }) + } + }) + + // next, test invalid uses, which should return an error + t.Run("invalid", func(t *testing.T) { + tests := []struct { + blobs []restic.Blob + err string + }{ + { + // pass one blob several times + blobs: []restic.Blob{ + packfileBlobs[3], + packfileBlobs[8], + packfileBlobs[3], + packfileBlobs[4], + }, + err: "overlapping blobs in pack", + }, + + { + // pass something that's not a valid blob in the current pack file + blobs: []restic.Blob{ + { + Offset: 123, + Length: 20000, + }, + }, + err: "ciphertext verification failed", + }, + + { + // pass a blob that's too small + blobs: []restic.Blob{ + { + Offset: 123, + Length: 10, + }, + }, + err: "invalid blob length", + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { + return err + } + + err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) + if err == nil { + t.Fatalf("wanted error %v, got nil", test.err) + } + + if !strings.Contains(err.Error(), test.err) { + t.Fatalf("wrong error returned, it should contain %q but was %q", test.err, err) + } + }) + } + }) } diff --git a/internal/repository/testdata/fuzz/FuzzSaveLoadBlob/62d79435b9ad1777d0562c405e1ab2e1ef5d11d07c8aa4fe6814010294bffd33 b/internal/repository/testdata/fuzz/FuzzSaveLoadBlob/62d79435b9ad1777d0562c405e1ab2e1ef5d11d07c8aa4fe6814010294bffd33 new file mode 100644 index 00000000000..c6fcd9e9af4 --- /dev/null +++ b/internal/repository/testdata/fuzz/FuzzSaveLoadBlob/62d79435b9ad1777d0562c405e1ab2e1ef5d11d07c8aa4fe6814010294bffd33 @@ -0,0 +1,3 @@ +go test fuzz v1 +[]byte("\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6\xc6") +uint(109) diff --git a/internal/repository/testing.go b/internal/repository/testing.go index 899b8a7e334..380a47d0410 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -2,6 +2,7 @@ package repository import ( "context" + "fmt" "os" "testing" @@ -41,7 +42,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173) // TestRepositoryWithBackend returns a repository initialized with a test // password. If be is nil, an in-memory backend is used. A constant polynomial // is used for the chunker and low-security test parameters. -func TestRepositoryWithBackend(t testing.TB, be restic.Backend) (r restic.Repository, cleanup func()) { +func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) (r restic.Repository, cleanup func()) { t.Helper() TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) @@ -51,10 +52,13 @@ func TestRepositoryWithBackend(t testing.TB, be restic.Backend) (r restic.Reposi be, beCleanup = TestBackend(t) } - repo := New(be) + repo, err := New(be, Options{}) + if err != nil { + t.Fatalf("TestRepository(): new repo failed: %v", err) + } - cfg := restic.TestCreateConfig(t, TestChunkerPol) - err := repo.init(context.TODO(), test.TestPassword, cfg) + cfg := restic.TestCreateConfig(t, TestChunkerPol, version) + err = repo.init(context.TODO(), test.TestPassword, cfg) if err != nil { t.Fatalf("TestRepository(): initialize repo failed: %v", err) } @@ -71,6 +75,11 @@ func TestRepositoryWithBackend(t testing.TB, be restic.Backend) (r restic.Reposi // a non-existing directory, a local backend is created there and this is used // instead. The directory is not removed, but left there for inspection. func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) { + t.Helper() + return TestRepositoryWithVersion(t, 0) +} + +func TestRepositoryWithVersion(t testing.TB, version uint) (r restic.Repository, cleanup func()) { t.Helper() dir := os.Getenv("RESTIC_TEST_REPO") if dir != "" { @@ -80,7 +89,7 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) { if err != nil { t.Fatalf("error creating local backend at %v: %v", dir, err) } - return TestRepositoryWithBackend(t, be) + return TestRepositoryWithBackend(t, be, version) } if err == nil { @@ -88,17 +97,20 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) { } } - return TestRepositoryWithBackend(t, nil) + return TestRepositoryWithBackend(t, nil, version) } // TestOpenLocal opens a local repository. func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { - be, err := local.Open(context.TODO(), local.Config{Path: dir}) + be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2}) if err != nil { t.Fatal(err) } - repo := New(be) + repo, err := New(be, Options{}) + if err != nil { + t.Fatal(err) + } err = repo.SearchKey(context.TODO(), test.TestPassword, 10, "") if err != nil { t.Fatal(err) @@ -106,3 +118,33 @@ func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { return repo } + +type VersionedTest func(t *testing.T, version uint) + +func TestAllVersions(t *testing.T, test VersionedTest) { + for version := restic.MinRepoVersion; version <= restic.MaxRepoVersion; version++ { + t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) { + test(t, uint(version)) + }) + } +} + +type VersionedBenchmark func(b *testing.B, version uint) + +func BenchmarkAllVersions(b *testing.B, bench VersionedBenchmark) { + for version := restic.MinRepoVersion; version <= restic.MaxRepoVersion; version++ { + b.Run(fmt.Sprintf("v%d", version), func(b *testing.B) { + bench(b, uint(version)) + }) + } +} + +func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int) { + finalIndexes := mi.finalizeNotFinalIndexes() + for _, idx := range finalIndexes { + test.OK(t, idx.SetID(restic.NewRandomID())) + } + + test.OK(t, mi.MergeFinalIndexes()) + return finalIndexes, len(mi.idx) +} diff --git a/internal/repository/worker_group.go b/internal/repository/worker_group.go deleted file mode 100644 index c612d1d228b..00000000000 --- a/internal/repository/worker_group.go +++ /dev/null @@ -1,18 +0,0 @@ -package repository - -import ( - "golang.org/x/sync/errgroup" -) - -// RunWorkers runs count instances of workerFunc using an errgroup.Group. -// If an error occurs in one of the workers, it is returned. -func RunWorkers(count int, workerFunc func() error) error { - var wg errgroup.Group - - // run workers - for i := 0; i < count; i++ { - wg.Go(workerFunc) - } - - return wg.Wait() -} diff --git a/internal/restic/backend.go b/internal/restic/backend.go index 41292470a4b..6ec10e6859f 100644 --- a/internal/restic/backend.go +++ b/internal/restic/backend.go @@ -18,9 +18,15 @@ type Backend interface { // repository. Location() string + // Connections returns the maxmimum number of concurrent backend operations. + Connections() uint + // Hasher may return a hash function for calculating a content hash for the backend Hasher() hash.Hash + // HasAtomicReplace returns whether Save() can atomically replace files + HasAtomicReplace() bool + // Test a boolean value whether a File with the name and type exists. Test(ctx context.Context, h Handle) (bool, error) diff --git a/internal/restic/blob.go b/internal/restic/blob.go index d365bb92ebf..4ac149adb97 100644 --- a/internal/restic/blob.go +++ b/internal/restic/blob.go @@ -3,19 +3,32 @@ package restic import ( "fmt" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" ) // Blob is one part of a file or a tree. type Blob struct { BlobHandle - Length uint - Offset uint + Length uint + Offset uint + UncompressedLength uint } func (b Blob) String() string { - return fmt.Sprintf("", - b.Type, b.ID.Str(), b.Offset, b.Length) + return fmt.Sprintf("", + b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength) +} + +func (b Blob) DataLength() uint { + if b.UncompressedLength != 0 { + return b.UncompressedLength + } + return uint(crypto.PlaintextLength(int(b.Length))) +} + +func (b Blob) IsCompressed() bool { + return b.UncompressedLength != 0 } // PackedBlob is a blob stored within a file. diff --git a/internal/restic/config.go b/internal/restic/config.go index 4f3c6c4bcf4..67ee190bc6c 100644 --- a/internal/restic/config.go +++ b/internal/restic/config.go @@ -18,9 +18,12 @@ type Config struct { ChunkerPolynomial chunker.Pol `json:"chunker_polynomial"` } -// RepoVersion is the version that is written to the config when a repository +const MinRepoVersion = 1 +const MaxRepoVersion = 2 + +// StableRepoVersion is the version that is written to the config when a repository // is newly created with Init(). -const RepoVersion = 1 +const StableRepoVersion = 2 // JSONUnpackedLoader loads unpacked JSON. type JSONUnpackedLoader interface { @@ -29,7 +32,7 @@ type JSONUnpackedLoader interface { // CreateConfig creates a config file with a randomly selected polynomial and // ID. -func CreateConfig() (Config, error) { +func CreateConfig(version uint) (Config, error) { var ( err error cfg Config @@ -41,18 +44,24 @@ func CreateConfig() (Config, error) { } cfg.ID = NewRandomID().String() - cfg.Version = RepoVersion + cfg.Version = version debug.Log("New config: %#v", cfg) return cfg, nil } // TestCreateConfig creates a config for use within tests. -func TestCreateConfig(t testing.TB, pol chunker.Pol) (cfg Config) { +func TestCreateConfig(t testing.TB, pol chunker.Pol, version uint) (cfg Config) { cfg.ChunkerPolynomial = pol cfg.ID = NewRandomID().String() - cfg.Version = RepoVersion + if version == 0 { + version = StableRepoVersion + } + if version < MinRepoVersion || version > MaxRepoVersion { + t.Fatalf("version %d is out of range", version) + } + cfg.Version = version return cfg } @@ -67,18 +76,18 @@ func TestDisableCheckPolynomial(t testing.TB) { } // LoadConfig returns loads, checks and returns the config for a repository. -func LoadConfig(ctx context.Context, r JSONUnpackedLoader) (Config, error) { +func LoadConfig(ctx context.Context, r LoaderUnpacked) (Config, error) { var ( cfg Config ) - err := r.LoadJSONUnpacked(ctx, ConfigFile, ID{}, &cfg) + err := LoadJSONUnpacked(ctx, r, ConfigFile, ID{}, &cfg) if err != nil { return Config{}, err } - if cfg.Version != RepoVersion { - return Config{}, errors.New("unsupported repository version") + if cfg.Version < MinRepoVersion || cfg.Version > MaxRepoVersion { + return Config{}, errors.Errorf("unsupported repository version %v", cfg.Version) } if checkPolynomial { @@ -89,3 +98,8 @@ func LoadConfig(ctx context.Context, r JSONUnpackedLoader) (Config, error) { return cfg, nil } + +func SaveConfig(ctx context.Context, r SaverUnpacked, cfg Config) error { + _, err := SaveJSONUnpacked(ctx, r, ConfigFile, cfg) + return err +} diff --git a/internal/restic/config_test.go b/internal/restic/config_test.go index 5063819651d..662a2e69e8c 100644 --- a/internal/restic/config_test.go +++ b/internal/restic/config_test.go @@ -8,47 +8,56 @@ import ( rtest "github.com/restic/restic/internal/test" ) -type saver func(restic.FileType, interface{}) (restic.ID, error) +type saver struct { + fn func(restic.FileType, []byte) (restic.ID, error) +} + +func (s saver) SaveUnpacked(ctx context.Context, t restic.FileType, buf []byte) (restic.ID, error) { + return s.fn(t, buf) +} -func (s saver) SaveJSONUnpacked(t restic.FileType, arg interface{}) (restic.ID, error) { - return s(t, arg) +func (s saver) Connections() uint { + return 2 } -type loader func(context.Context, restic.FileType, restic.ID, interface{}) error +type loader struct { + fn func(restic.FileType, restic.ID, []byte) ([]byte, error) +} + +func (l loader) LoadUnpacked(ctx context.Context, t restic.FileType, id restic.ID, buf []byte) (data []byte, err error) { + return l.fn(t, id, buf) +} -func (l loader) LoadJSONUnpacked(ctx context.Context, t restic.FileType, id restic.ID, arg interface{}) error { - return l(ctx, t, id, arg) +func (l loader) Connections() uint { + return 2 } func TestConfig(t *testing.T) { - resultConfig := restic.Config{} - save := func(tpe restic.FileType, arg interface{}) (restic.ID, error) { + var resultBuf []byte + save := func(tpe restic.FileType, buf []byte) (restic.ID, error) { rtest.Assert(t, tpe == restic.ConfigFile, "wrong backend type: got %v, wanted %v", tpe, restic.ConfigFile) - cfg := arg.(restic.Config) - resultConfig = cfg + resultBuf = buf return restic.ID{}, nil } - cfg1, err := restic.CreateConfig() + cfg1, err := restic.CreateConfig(restic.MaxRepoVersion) rtest.OK(t, err) - _, err = saver(save).SaveJSONUnpacked(restic.ConfigFile, cfg1) + err = restic.SaveConfig(context.TODO(), saver{save}, cfg1) rtest.OK(t, err) - load := func(ctx context.Context, tpe restic.FileType, id restic.ID, arg interface{}) error { + load := func(tpe restic.FileType, id restic.ID, in []byte) ([]byte, error) { rtest.Assert(t, tpe == restic.ConfigFile, "wrong backend type: got %v, wanted %v", tpe, restic.ConfigFile) - cfg := arg.(*restic.Config) - *cfg = resultConfig - return nil + return resultBuf, nil } - cfg2, err := restic.LoadConfig(context.TODO(), loader(load)) + cfg2, err := restic.LoadConfig(context.TODO(), loader{load}) rtest.OK(t, err) rtest.Assert(t, cfg1 == cfg2, diff --git a/internal/restic/duration.go b/internal/restic/duration.go index ad56bcc81be..831971fe0f5 100644 --- a/internal/restic/duration.go +++ b/internal/restic/duration.go @@ -76,8 +76,7 @@ func nextNumber(input string) (num int, rest string, err error) { return num, rest, nil } -// ParseDuration parses a duration from a string. The format is: -// 6y5m234d37h +// ParseDuration parses a duration from a string. The format is `6y5m234d37h` func ParseDuration(s string) (Duration, error) { var ( d Duration @@ -106,6 +105,8 @@ func ParseDuration(s string) (Duration, error) { d.Days = num case 'h': d.Hours = num + default: + return Duration{}, errors.Errorf("invalid unit %q found after number %d", s[0], num) } s = s[1:] diff --git a/internal/restic/duration_test.go b/internal/restic/duration_test.go index 716c00cc9fc..f03aa5553ba 100644 --- a/internal/restic/duration_test.go +++ b/internal/restic/duration_test.go @@ -63,24 +63,34 @@ func TestParseDuration(t *testing.T) { input string d Duration output string + err bool }{ - {"9h", Duration{Hours: 9}, "9h"}, - {"3d", Duration{Days: 3}, "3d"}, - {"4d2h", Duration{Days: 4, Hours: 2}, "4d2h"}, - {"7m5d", Duration{Months: 7, Days: 5}, "7m5d"}, - {"6m4d8h", Duration{Months: 6, Days: 4, Hours: 8}, "6m4d8h"}, - {"5d7m", Duration{Months: 7, Days: 5}, "7m5d"}, - {"4h3d9m", Duration{Months: 9, Days: 3, Hours: 4}, "9m3d4h"}, - {"-7m5d", Duration{Months: -7, Days: 5}, "-7m5d"}, - {"1y4m-5d-3h", Duration{Years: 1, Months: 4, Days: -5, Hours: -3}, "1y4m-5d-3h"}, - {"2y7m-5d", Duration{Years: 2, Months: 7, Days: -5}, "2y7m-5d"}, + {input: "9h", d: Duration{Hours: 9}, output: "9h"}, + {input: "3d", d: Duration{Days: 3}, output: "3d"}, + {input: "4d2h", d: Duration{Days: 4, Hours: 2}, output: "4d2h"}, + {input: "7m5d", d: Duration{Months: 7, Days: 5}, output: "7m5d"}, + {input: "6m4d8h", d: Duration{Months: 6, Days: 4, Hours: 8}, output: "6m4d8h"}, + {input: "5d7m", d: Duration{Months: 7, Days: 5}, output: "7m5d"}, + {input: "4h3d9m", d: Duration{Months: 9, Days: 3, Hours: 4}, output: "9m3d4h"}, + {input: "-7m5d", d: Duration{Months: -7, Days: 5}, output: "-7m5d"}, + {input: "1y4m-5d-3h", d: Duration{Years: 1, Months: 4, Days: -5, Hours: -3}, output: "1y4m-5d-3h"}, + {input: "2y7m-5d", d: Duration{Years: 2, Months: 7, Days: -5}, output: "2y7m-5d"}, + {input: "2w", err: true}, + {input: "1y4m3w1d", err: true}, + {input: "s", err: true}, } for _, test := range tests { t.Run("", func(t *testing.T) { d, err := ParseDuration(test.input) - if err != nil { - t.Fatal(err) + if test.err { + if err == nil { + t.Fatalf("Missing error for %v", test.input) + } + } else { + if err != nil { + t.Fatal(err) + } } if !cmp.Equal(d, test.d) { diff --git a/internal/restic/find.go b/internal/restic/find.go index c7406d75029..6544f2b3d8b 100644 --- a/internal/restic/find.go +++ b/internal/restic/find.go @@ -8,15 +8,16 @@ import ( "golang.org/x/sync/errgroup" ) -// TreeLoader loads a tree from a repository. -type TreeLoader interface { - LoadTree(context.Context, ID) (*Tree, error) +// Loader loads a blob from a repository. +type Loader interface { + LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) LookupBlobSize(id ID, tpe BlobType) (uint, bool) + Connections() uint } // FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data // blobs) to the set blobs. Already seen tree blobs will not be visited again. -func FindUsedBlobs(ctx context.Context, repo TreeLoader, treeIDs IDs, blobs BlobSet, p *progress.Counter) error { +func FindUsedBlobs(ctx context.Context, repo Loader, treeIDs IDs, blobs BlobSet, p *progress.Counter) error { var lock sync.Mutex wg, ctx := errgroup.WithContext(ctx) diff --git a/internal/restic/find_test.go b/internal/restic/find_test.go index 4d9bc5a1318..b415501dc26 100644 --- a/internal/restic/find_test.go +++ b/internal/restic/find_test.go @@ -162,7 +162,7 @@ func TestMultiFindUsedBlobs(t *testing.T) { type ForbiddenRepo struct{} -func (r ForbiddenRepo) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) { +func (r ForbiddenRepo) LoadBlob(context.Context, restic.BlobType, restic.ID, []byte) ([]byte, error) { return nil, errors.New("should not be called") } @@ -170,6 +170,10 @@ func (r ForbiddenRepo) LookupBlobSize(id restic.ID, tpe restic.BlobType) (uint, return 0, false } +func (r ForbiddenRepo) Connections() uint { + return 2 +} + func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() diff --git a/internal/restic/id.go b/internal/restic/id.go index c098dbfb32d..6d85ed68a54 100644 --- a/internal/restic/id.go +++ b/internal/restic/id.go @@ -3,7 +3,6 @@ package restic import ( "crypto/rand" "encoding/hex" - "encoding/json" "fmt" "io" @@ -98,7 +97,13 @@ func (id ID) EqualString(other string) (bool, error) { // MarshalJSON returns the JSON encoding of id. func (id ID) MarshalJSON() ([]byte, error) { - return json.Marshal(id.String()) + buf := make([]byte, 2+hex.EncodedLen(len(id))) + + buf[0] = '"' + hex.Encode(buf[1:], id[:]) + buf[len(buf)-1] = '"' + + return buf, nil } // UnmarshalJSON parses the JSON-encoded data and stores the result in id. diff --git a/internal/restic/json.go b/internal/restic/json.go new file mode 100644 index 00000000000..6ad4b5f3968 --- /dev/null +++ b/internal/restic/json.go @@ -0,0 +1,32 @@ +package restic + +import ( + "context" + "encoding/json" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" +) + +// LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on +// the item. +func LoadJSONUnpacked(ctx context.Context, repo LoaderUnpacked, t FileType, id ID, item interface{}) (err error) { + buf, err := repo.LoadUnpacked(ctx, t, id, nil) + if err != nil { + return err + } + + return json.Unmarshal(buf, item) +} + +// SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the +// backend as type t, without a pack. It returns the storage hash. +func SaveJSONUnpacked(ctx context.Context, repo SaverUnpacked, t FileType, item interface{}) (ID, error) { + debug.Log("save new blob %v", t) + plaintext, err := json.Marshal(item) + if err != nil { + return ID{}, errors.Wrap(err, "json.Marshal") + } + + return repo.SaveUnpacked(ctx, t, plaintext) +} diff --git a/internal/restic/lock.go b/internal/restic/lock.go index 9a5fd841de0..3f233483f21 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -38,13 +38,13 @@ type Lock struct { lockID *ID } -// ErrAlreadyLocked is returned when NewLock or NewExclusiveLock are unable to +// alreadyLockedError is returned when NewLock or NewExclusiveLock are unable to // acquire the desired lock. -type ErrAlreadyLocked struct { +type alreadyLockedError struct { otherLock *Lock } -func (e ErrAlreadyLocked) Error() string { +func (e *alreadyLockedError) Error() string { s := "" if e.otherLock.Exclusive { s = "exclusively " @@ -52,25 +52,23 @@ func (e ErrAlreadyLocked) Error() string { return fmt.Sprintf("repository is already locked %sby %v", s, e.otherLock) } -// IsAlreadyLocked returns true iff err is an instance of ErrAlreadyLocked. +// IsAlreadyLocked returns true iff err indicates that a repository is +// already locked. func IsAlreadyLocked(err error) bool { - if _, ok := errors.Cause(err).(ErrAlreadyLocked); ok { - return true - } - - return false + var e *alreadyLockedError + return errors.As(err, &e) } // NewLock returns a new, non-exclusive lock for the repository. If an -// exclusive lock is already held by another process, ErrAlreadyLocked is -// returned. +// exclusive lock is already held by another process, it returns an error +// that satisfies IsAlreadyLocked. func NewLock(ctx context.Context, repo Repository) (*Lock, error) { return newLock(ctx, repo, false) } // NewExclusiveLock returns a new, exclusive lock for the repository. If // another lock (normal and exclusive) is already held by another process, -// ErrAlreadyLocked is returned. +// it returns an error that satisfies IsAlreadyLocked. func NewExclusiveLock(ctx context.Context, repo Repository) (*Lock, error) { return newLock(ctx, repo, true) } @@ -147,11 +145,11 @@ func (l *Lock) checkForOtherLocks(ctx context.Context) error { } if l.Exclusive { - return ErrAlreadyLocked{otherLock: lock} + return &alreadyLockedError{otherLock: lock} } if !l.Exclusive && lock.Exclusive { - return ErrAlreadyLocked{otherLock: lock} + return &alreadyLockedError{otherLock: lock} } return nil @@ -160,7 +158,7 @@ func (l *Lock) checkForOtherLocks(ctx context.Context) error { // createLock acquires the lock by creating a file in the repository. func (l *Lock) createLock(ctx context.Context) (ID, error) { - id, err := l.repo.SaveJSONUnpacked(ctx, LockFile, l) + id, err := SaveJSONUnpacked(ctx, l.repo, LockFile, l) if err != nil { return ID{}, err } @@ -257,7 +255,7 @@ func init() { // LoadLock loads and unserializes a lock from a repository. func LoadLock(ctx context.Context, repo Repository, id ID) (*Lock, error) { lock := &Lock{} - if err := repo.LoadJSONUnpacked(ctx, LockFile, id, lock); err != nil { + if err := LoadJSONUnpacked(ctx, repo, LockFile, id, lock); err != nil { return nil, err } lock.lockID = &id @@ -289,8 +287,6 @@ func RemoveAllLocks(ctx context.Context, repo Repository) error { }) } -const loadLockParallelism = 5 - // ForAllLocks reads all locks in parallel and calls the given callback. // It is guaranteed that the function is not run concurrently. If the // callback returns an error, this function is cancelled and also returns that error. @@ -338,7 +334,8 @@ func ForAllLocks(ctx context.Context, repo Repository, excludeID *ID, fn func(ID return nil } - for i := 0; i < loadLockParallelism; i++ { + // For locks decoding is nearly for free, thus just assume were only limited by IO + for i := 0; i < int(repo.Connections()); i++ { wg.Go(worker) } diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go index 09dce3c78a4..b92eace7064 100644 --- a/internal/restic/lock_test.go +++ b/internal/restic/lock_test.go @@ -99,7 +99,7 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er } newLock := &restic.Lock{Time: t, PID: pid, Hostname: hostname} - return repo.SaveJSONUnpacked(context.TODO(), restic.LockFile, &newLock) + return restic.SaveJSONUnpacked(context.TODO(), repo, restic.LockFile, &newLock) } func removeLock(repo restic.Repository, id restic.ID) error { diff --git a/internal/restic/lock_unix.go b/internal/restic/lock_unix.go index 266f5558001..dbf23fc6c9b 100644 --- a/internal/restic/lock_unix.go +++ b/internal/restic/lock_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package restic diff --git a/internal/restic/mknod_unix.go b/internal/restic/mknod_unix.go index 7021bf35cb1..7dd6c60d0b9 100644 --- a/internal/restic/mknod_unix.go +++ b/internal/restic/mknod_unix.go @@ -1,3 +1,4 @@ +//go:build !freebsd && !windows // +build !freebsd,!windows package restic diff --git a/internal/restic/node.go b/internal/restic/node.go index cd237ac43ce..54b67c6721f 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -166,7 +166,7 @@ func (node *Node) CreateAt(ctx context.Context, path string, repo Repository) er case "socket": return nil default: - return errors.Errorf("filetype %q not implemented!\n", node.Type) + return errors.Errorf("filetype %q not implemented", node.Type) } return nil diff --git a/internal/restic/node_aix.go b/internal/restic/node_aix.go index 65914411c14..572e33a6508 100644 --- a/internal/restic/node_aix.go +++ b/internal/restic/node_aix.go @@ -1,3 +1,4 @@ +//go:build aix // +build aix package restic diff --git a/internal/restic/node_freebsd.go b/internal/restic/node_freebsd.go index c06701d244f..34d5b272c01 100644 --- a/internal/restic/node_freebsd.go +++ b/internal/restic/node_freebsd.go @@ -1,3 +1,4 @@ +//go:build freebsd // +build freebsd package restic diff --git a/internal/restic/node_unix.go b/internal/restic/node_unix.go index 05d577a9886..976cd7b0366 100644 --- a/internal/restic/node_unix.go +++ b/internal/restic/node_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package restic diff --git a/internal/restic/node_unix_test.go b/internal/restic/node_unix_test.go index 20433082488..c4fef371020 100644 --- a/internal/restic/node_unix_test.go +++ b/internal/restic/node_unix_test.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package restic diff --git a/internal/restic/node_xattr.go b/internal/restic/node_xattr.go index 642eb8df4cb..a2eed39c065 100644 --- a/internal/restic/node_xattr.go +++ b/internal/restic/node_xattr.go @@ -13,31 +13,37 @@ import ( // Getxattr retrieves extended attribute data associated with path. func Getxattr(path, name string) ([]byte, error) { - b, e := xattr.Get(path, name) - if err, ok := e.(*xattr.Error); ok && - (err.Err == syscall.ENOTSUP || err.Err == xattr.ENOATTR) { - return nil, nil - } - return b, errors.Wrap(e, "Getxattr") + b, err := xattr.Get(path, name) + return b, handleXattrErr(err) } // Listxattr retrieves a list of names of extended attributes associated with the // given path in the file system. func Listxattr(path string) ([]string, error) { - s, e := xattr.List(path) - if err, ok := e.(*xattr.Error); ok && - (err.Err == syscall.ENOTSUP || err.Err == xattr.ENOATTR) { - return nil, nil - } - return s, errors.Wrap(e, "Listxattr") + l, err := xattr.List(path) + return l, handleXattrErr(err) } // Setxattr associates name and data together as an attribute of path. func Setxattr(path, name string, data []byte) error { - e := xattr.Set(path, name, data) - if err, ok := e.(*xattr.Error); ok && - (err.Err == syscall.ENOTSUP || err.Err == xattr.ENOATTR) { + return handleXattrErr(xattr.Set(path, name, data)) +} + +func handleXattrErr(err error) error { + switch e := err.(type) { + case nil: return nil + + case *xattr.Error: + // On Linux, xattr calls on files in an SMB/CIFS mount can return + // ENOATTR instead of ENOTSUP. + switch e.Err { + case syscall.ENOTSUP, xattr.ENOATTR: + return nil + } + return errors.WithStack(e) + + default: + return errors.WithStack(e) } - return errors.Wrap(e, "Setxattr") } diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 38c611ce6b4..36f5a73bf5f 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -4,6 +4,8 @@ import ( "context" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/ui/progress" + "golang.org/x/sync/errgroup" ) // Repository stores data in a backend. It provides high-level functions and @@ -12,19 +14,18 @@ type Repository interface { // Backend returns the backend used by the repository Backend() Backend + // Connections returns the maximum number of concurrent backend operations + Connections() uint Key() *crypto.Key - SetIndex(MasterIndex) error - Index() MasterIndex - SaveFullIndex(context.Context) error - SaveIndex(context.Context) error LoadIndex(context.Context) error + SetIndex(MasterIndex) error + LookupBlobSize(ID, BlobType) (uint, bool) Config() Config - - LookupBlobSize(ID, BlobType) (uint, bool) + PackSize() uint // List calls the function fn for each file of type t in the repository. // When an error is returned by fn, processing stops and List() returns the @@ -37,22 +38,20 @@ type Repository interface { // the the pack header. ListPack(context.Context, ID, int64) ([]Blob, uint32, error) - Flush(context.Context) error + LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) + SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, int, error) - SaveUnpacked(context.Context, FileType, []byte) (ID, error) - SaveJSONUnpacked(context.Context, FileType, interface{}) (ID, error) + // StartPackUploader start goroutines to upload new pack files. The errgroup + // is used to immediately notify about an upload error. Flush() will also return + // that error. + StartPackUploader(ctx context.Context, wg *errgroup.Group) + Flush(context.Context) error - LoadJSONUnpacked(ctx context.Context, t FileType, id ID, dest interface{}) error - // LoadAndDecrypt loads and decrypts the file with the given type and ID, + // LoadUnpacked loads and decrypts the file with the given type and ID, // using the supplied buffer (which must be empty). If the buffer is nil, a // new buffer will be allocated and returned. - LoadAndDecrypt(ctx context.Context, buf []byte, t FileType, id ID) (data []byte, err error) - - LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) - SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, error) - - LoadTree(context.Context, ID) (*Tree, error) - SaveTree(context.Context, *Tree) (ID, error) + LoadUnpacked(ctx context.Context, t FileType, id ID, buf []byte) (data []byte, err error) + SaveUnpacked(context.Context, FileType, []byte) (ID, error) } // Lister allows listing files in a backend. @@ -60,15 +59,35 @@ type Lister interface { List(context.Context, FileType, func(FileInfo) error) error } +// LoaderUnpacked allows loading a blob not stored in a pack file +type LoaderUnpacked interface { + // Connections returns the maximum number of concurrent backend operations + Connections() uint + LoadUnpacked(ctx context.Context, t FileType, id ID, buf []byte) (data []byte, err error) +} + +// SaverUnpacked allows saving a blob not stored in a pack file +type SaverUnpacked interface { + // Connections returns the maximum number of concurrent backend operations + Connections() uint + SaveUnpacked(context.Context, FileType, []byte) (ID, error) +} + +type PackBlobs struct { + PackID ID + Blobs []Blob +} + // MasterIndex keeps track of the blobs are stored within files. type MasterIndex interface { Has(BlobHandle) bool Lookup(BlobHandle) []PackedBlob - Count(BlobType) uint - PackSize(ctx context.Context, onlyHdr bool) map[ID]int64 // Each returns a channel that yields all blobs known to the index. When // the context is cancelled, the background goroutine terminates. This // blocks any modification of the index. Each(ctx context.Context) <-chan PackedBlob + ListPacks(ctx context.Context, packs IDSet) <-chan PackBlobs + + Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error) } diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index ac5f2cf44be..10c4f218ea8 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -59,9 +59,9 @@ func NewSnapshot(paths []string, tags []string, hostname string, time time.Time) } // LoadSnapshot loads the snapshot with the id and returns it. -func LoadSnapshot(ctx context.Context, repo Repository, id ID) (*Snapshot, error) { +func LoadSnapshot(ctx context.Context, loader LoaderUnpacked, id ID) (*Snapshot, error) { sn := &Snapshot{id: &id} - err := repo.LoadJSONUnpacked(ctx, SnapshotFile, id, sn) + err := LoadJSONUnpacked(ctx, loader, SnapshotFile, id, sn) if err != nil { return nil, err } @@ -69,14 +69,17 @@ func LoadSnapshot(ctx context.Context, repo Repository, id ID) (*Snapshot, error return sn, nil } -const loadSnapshotParallelism = 5 +// SaveSnapshot saves the snapshot sn and returns its ID. +func SaveSnapshot(ctx context.Context, repo SaverUnpacked, sn *Snapshot) (ID, error) { + return SaveJSONUnpacked(ctx, repo, SnapshotFile, sn) +} // ForAllSnapshots reads all snapshots in parallel and calls the // given function. It is guaranteed that the function is not run concurrently. // If the called function returns an error, this function is cancelled and // also returns this error. // If a snapshot ID is in excludeIDs, it will be ignored. -func ForAllSnapshots(ctx context.Context, repo Repository, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error { +func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error { var m sync.Mutex // track spawned goroutines using wg, create a new context which is @@ -88,7 +91,13 @@ func ForAllSnapshots(ctx context.Context, repo Repository, excludeIDs IDSet, fn // send list of snapshot files through ch, which is closed afterwards wg.Go(func() error { defer close(ch) - return repo.List(ctx, SnapshotFile, func(id ID, size int64) error { + return be.List(ctx, SnapshotFile, func(fi FileInfo) error { + id, err := ParseID(fi.Name) + if err != nil { + debug.Log("unable to parse %v as an ID", fi.Name) + return nil + } + if excludeIDs.Has(id) { return nil } @@ -107,7 +116,7 @@ func ForAllSnapshots(ctx context.Context, repo Repository, excludeIDs IDSet, fn worker := func() error { for id := range ch { debug.Log("load snapshot %v", id) - sn, err := LoadSnapshot(ctx, repo, id) + sn, err := LoadSnapshot(ctx, loader, id) m.Lock() err = fn(id, sn, err) @@ -119,7 +128,8 @@ func ForAllSnapshots(ctx context.Context, repo Repository, excludeIDs IDSet, fn return nil } - for i := 0; i < loadSnapshotParallelism; i++ { + // For most snapshots decoding is nearly for free, thus just assume were only limited by IO + for i := 0; i < int(loader.Connections()); i++ { wg.Go(worker) } @@ -207,9 +217,9 @@ func (sn *Snapshot) HasTags(l []string) bool { } // HasTagList returns true if either -// - the snapshot satisfies at least one TagList, so there is a TagList in l -// for which all tags are included in sn, or -// - l is empty +// - the snapshot satisfies at least one TagList, so there is a TagList in l +// for which all tags are included in sn, or +// - l is empty func (sn *Snapshot) HasTagList(l []TagList) bool { debug.Log("testing snapshot with tags %v against list: %v", sn.Tags, l) diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index 1f309f0bd9b..49f9d62bf97 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -14,7 +14,9 @@ import ( var ErrNoSnapshotFound = errors.New("no snapshot found") // FindLatestSnapshot finds latest snapshot with optional target/directory, tags, hostname, and timestamp filters. -func FindLatestSnapshot(ctx context.Context, repo Repository, targets []string, tagLists []TagList, hostnames []string, timeStampLimit *time.Time) (ID, error) { +func FindLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, targets []string, + tagLists []TagList, hostnames []string, timeStampLimit *time.Time) (ID, error) { + var err error absTargets := make([]string, 0, len(targets)) for _, target := range targets { @@ -33,7 +35,7 @@ func FindLatestSnapshot(ctx context.Context, repo Repository, targets []string, found bool ) - err = ForAllSnapshots(ctx, repo, nil, func(id ID, snapshot *Snapshot, err error) error { + err = ForAllSnapshots(ctx, be, loader, nil, func(id ID, snapshot *Snapshot, err error) error { if err != nil { return errors.Errorf("Error loading snapshot %v: %v", id.Str(), err) } @@ -77,10 +79,10 @@ func FindLatestSnapshot(ctx context.Context, repo Repository, targets []string, // FindSnapshot takes a string and tries to find a snapshot whose ID matches // the string as closely as possible. -func FindSnapshot(ctx context.Context, repo Repository, s string) (ID, error) { +func FindSnapshot(ctx context.Context, be Lister, s string) (ID, error) { // find snapshot id with prefix - name, err := Find(ctx, repo.Backend(), SnapshotFile, s) + name, err := Find(ctx, be, SnapshotFile, s) if err != nil { return ID{}, err } @@ -90,10 +92,10 @@ func FindSnapshot(ctx context.Context, repo Repository, s string) (ID, error) { // FindFilteredSnapshots yields Snapshots filtered from the list of all // snapshots. -func FindFilteredSnapshots(ctx context.Context, repo Repository, hosts []string, tags []TagList, paths []string) (Snapshots, error) { +func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, tags []TagList, paths []string) (Snapshots, error) { results := make(Snapshots, 0, 20) - err := ForAllSnapshots(ctx, repo, nil, func(id ID, sn *Snapshot, err error) error { + err := ForAllSnapshots(ctx, be, loader, nil, func(id ID, sn *Snapshot, err error) error { if err != nil { fmt.Fprintf(os.Stderr, "could not load snapshot %v: %v\n", id.Str(), err) return nil diff --git a/internal/restic/snapshot_find_test.go b/internal/restic/snapshot_find_test.go index d4fedcc9d6b..534eb456d71 100644 --- a/internal/restic/snapshot_find_test.go +++ b/internal/restic/snapshot_find_test.go @@ -16,7 +16,7 @@ func TestFindLatestSnapshot(t *testing.T) { restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) - id, err := restic.FindLatestSnapshot(context.TODO(), repo, []string{}, []restic.TagList{}, []string{"foo"}, nil) + id, err := restic.FindLatestSnapshot(context.TODO(), repo.Backend(), repo, []string{}, []restic.TagList{}, []string{"foo"}, nil) if err != nil { t.Fatalf("FindLatestSnapshot returned error: %v", err) } @@ -36,7 +36,7 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) { maxTimestamp := parseTimeUTC("2018-08-08 08:08:08") - id, err := restic.FindLatestSnapshot(context.TODO(), repo, []string{}, []restic.TagList{}, []string{"foo"}, &maxTimestamp) + id, err := restic.FindLatestSnapshot(context.TODO(), repo.Backend(), repo, []string{}, []restic.TagList{}, []string{"foo"}, &maxTimestamp) if err != nil { t.Fatalf("FindLatestSnapshot returned error: %v", err) } diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index 8ce6cb2e34e..3271140aa0d 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -190,7 +190,7 @@ type KeepReason struct { // according to the policy p. list is sorted in the process. reasons contains // the reasons to keep each snapshot, it is in the same order as keep. func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots, reasons []KeepReason) { - sort.Sort(list) + sort.Stable(list) if p.Empty() { for _, sn := range list { diff --git a/internal/restic/snapshot_test.go b/internal/restic/snapshot_test.go index efcc0096062..96325debf3a 100644 --- a/internal/restic/snapshot_test.go +++ b/internal/restic/snapshot_test.go @@ -1,9 +1,11 @@ package restic_test import ( + "context" "testing" "time" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -24,3 +26,27 @@ func TestTagList(t *testing.T) { r := sn.HasTags(tags) rtest.Assert(t, r, "Failed to match untagged snapshot") } + +func TestLoadJSONUnpacked(t *testing.T) { + repository.TestAllVersions(t, testLoadJSONUnpacked) +} + +func testLoadJSONUnpacked(t *testing.T, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) + defer cleanup() + + // archive a snapshot + sn := restic.Snapshot{} + sn.Hostname = "foobar" + sn.Username = "test!" + + id, err := restic.SaveSnapshot(context.TODO(), repo, &sn) + rtest.OK(t, err) + + // restore + sn2, err := restic.LoadSnapshot(context.TODO(), repo, id) + rtest.OK(t, err) + + rtest.Equals(t, sn.Hostname, sn2.Hostname) + rtest.Equals(t, sn.Username, sn2.Username) +} diff --git a/internal/restic/testdata/policy_keep_snapshots_0 b/internal/restic/testdata/policy_keep_snapshots_0 index 1290b88cffe..11ca587c8af 100644 --- a/internal/restic/testdata/policy_keep_snapshots_0 +++ b/internal/restic/testdata/policy_keep_snapshots_0 @@ -153,14 +153,7 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] + "paths": null }, { "time": "2015-10-22T10:20:30Z", @@ -174,12 +167,19 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" @@ -915,14 +915,7 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] + "paths": null }, "matches": [ "policy is empty" @@ -948,7 +941,11 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, "matches": [ "policy is empty" @@ -959,7 +956,10 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" diff --git a/internal/restic/testdata/policy_keep_snapshots_18 b/internal/restic/testdata/policy_keep_snapshots_18 index cf63c45b81a..feb9ac20b02 100644 --- a/internal/restic/testdata/policy_keep_snapshots_18 +++ b/internal/restic/testdata/policy_keep_snapshots_18 @@ -3,10 +3,7 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], + "paths": null, "tags": [ "foo", "bar" @@ -24,7 +21,10 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" @@ -157,10 +157,7 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], + "paths": null, "tags": [ "foo", "bar" @@ -190,7 +187,10 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" diff --git a/internal/restic/testdata/policy_keep_snapshots_19 b/internal/restic/testdata/policy_keep_snapshots_19 index 81a43831388..440315b1045 100644 --- a/internal/restic/testdata/policy_keep_snapshots_19 +++ b/internal/restic/testdata/policy_keep_snapshots_19 @@ -3,10 +3,7 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], + "paths": null, "tags": [ "foo", "bar" @@ -24,7 +21,10 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" @@ -45,10 +45,7 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], + "paths": null, "tags": [ "foo", "bar" @@ -78,7 +75,10 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" diff --git a/internal/restic/testdata/policy_keep_snapshots_20 b/internal/restic/testdata/policy_keep_snapshots_20 index a57fcf024ae..f4febe82cfe 100644 --- a/internal/restic/testdata/policy_keep_snapshots_20 +++ b/internal/restic/testdata/policy_keep_snapshots_20 @@ -3,10 +3,7 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], + "paths": null, "tags": [ "foo", "bar" @@ -24,7 +21,10 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" @@ -165,10 +165,7 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], + "paths": null, "tags": [ "foo", "bar" @@ -200,7 +197,10 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" diff --git a/internal/restic/testdata/policy_keep_snapshots_26 b/internal/restic/testdata/policy_keep_snapshots_26 index 61703f8fe9b..af3bc0ced79 100644 --- a/internal/restic/testdata/policy_keep_snapshots_26 +++ b/internal/restic/testdata/policy_keep_snapshots_26 @@ -153,14 +153,7 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] + "paths": null }, { "time": "2015-10-22T10:20:30Z", @@ -174,12 +167,19 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" @@ -666,14 +666,7 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] + "paths": null }, "matches": [ "within 1y1m1d" @@ -699,7 +692,11 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, "matches": [ "within 1y1m1d" @@ -710,7 +707,10 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" diff --git a/internal/restic/testdata/policy_keep_snapshots_29 b/internal/restic/testdata/policy_keep_snapshots_29 index 172d3000ffa..2a82ecabd8d 100644 --- a/internal/restic/testdata/policy_keep_snapshots_29 +++ b/internal/restic/testdata/policy_keep_snapshots_29 @@ -153,14 +153,7 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] + "paths": null }, { "time": "2015-10-22T10:20:30Z", @@ -174,12 +167,19 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" @@ -695,14 +695,7 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] + "paths": null }, "matches": [ "within 1y2m3d3h" @@ -728,7 +721,11 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, "matches": [ "within 1y2m3d3h" @@ -739,7 +736,10 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" diff --git a/internal/restic/testdata/policy_keep_snapshots_3 b/internal/restic/testdata/policy_keep_snapshots_3 index 265e5213084..b497c690245 100644 --- a/internal/restic/testdata/policy_keep_snapshots_3 +++ b/internal/restic/testdata/policy_keep_snapshots_3 @@ -153,14 +153,7 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] + "paths": null }, { "time": "2015-10-22T10:20:30Z", @@ -174,12 +167,19 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" @@ -955,14 +955,7 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] + "paths": null }, "matches": [ "last snapshot" @@ -992,7 +985,11 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, "matches": [ "last snapshot" @@ -1005,7 +1002,10 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" diff --git a/internal/restic/testdata/policy_keep_snapshots_4 b/internal/restic/testdata/policy_keep_snapshots_4 index 8657da8c845..ff572d6a09a 100644 --- a/internal/restic/testdata/policy_keep_snapshots_4 +++ b/internal/restic/testdata/policy_keep_snapshots_4 @@ -153,14 +153,7 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] + "paths": null }, { "time": "2015-10-22T10:20:30Z", @@ -174,12 +167,19 @@ { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" @@ -975,14 +975,7 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] + "paths": null }, "matches": [ "last snapshot" @@ -1012,7 +1005,11 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, "matches": [ "last snapshot" @@ -1025,7 +1022,10 @@ "snapshot": { "time": "2015-10-22T10:20:30Z", "tree": null, - "paths": null, + "paths": [ + "path1", + "path2" + ], "tags": [ "foo", "bar" diff --git a/internal/restic/testing.go b/internal/restic/testing.go index f6965070bad..ebafdf65148 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -9,9 +9,8 @@ import ( "testing" "time" - "github.com/restic/restic/internal/errors" - "github.com/restic/chunker" + "golang.org/x/sync/errgroup" ) // fakeFile returns a reader which yields deterministic pseudo-random data. @@ -44,7 +43,7 @@ func (fs *fakeFileSystem) saveFile(ctx context.Context, rd io.Reader) (blobs IDs blobs = IDs{} for { chunk, err := fs.chunker.Next(fs.buf) - if errors.Cause(err) == io.EOF { + if err == io.EOF { break } @@ -54,7 +53,7 @@ func (fs *fakeFileSystem) saveFile(ctx context.Context, rd io.Reader) (blobs IDs id := Hash(chunk.Data) if !fs.blobIsKnown(BlobHandle{ID: id, Type: DataBlob}) { - _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, id, true) + _, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, id, true) if err != nil { fs.t.Fatalf("error saving chunk: %v", err) } @@ -140,7 +139,7 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I return id } - _, _, err := fs.repo.SaveBlob(ctx, TreeBlob, buf, id, false) + _, _, _, err := fs.repo.SaveBlob(ctx, TreeBlob, buf, id, false) if err != nil { fs.t.Fatal(err) } @@ -171,23 +170,26 @@ func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int, rand: rand.New(rand.NewSource(seed)), } + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + treeID := fs.saveTree(context.TODO(), seed, depth) snapshot.Tree = &treeID - id, err := repo.SaveJSONUnpacked(context.TODO(), SnapshotFile, snapshot) + err = repo.Flush(context.Background()) if err != nil { t.Fatal(err) } - snapshot.id = &id - - t.Logf("saved snapshot %v", id.Str()) - - err = repo.Flush(context.Background()) + id, err := SaveSnapshot(context.TODO(), repo, snapshot) if err != nil { t.Fatal(err) } + snapshot.id = &id + + t.Logf("saved snapshot %v", id.Str()) + return snapshot } @@ -205,3 +207,8 @@ func TestParseID(s string) ID { func TestParseHandle(s string, t BlobType) BlobHandle { return BlobHandle{ID: TestParseID(s), Type: t} } + +// TestSetSnapshotID sets the snapshot's ID. +func TestSetSnapshotID(t testing.TB, sn *Snapshot, id ID) { + sn.id = &id +} diff --git a/internal/restic/testing_test.go b/internal/restic/testing_test.go index 5607dc5fa68..7ee7461a5f6 100644 --- a/internal/restic/testing_test.go +++ b/internal/restic/testing_test.go @@ -20,7 +20,7 @@ const ( // LoadAllSnapshots returns a list of all snapshots in the repo. // If a snapshot ID is in excludeIDs, it will not be included in the result. func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs restic.IDSet) (snapshots restic.Snapshots, err error) { - err = restic.ForAllSnapshots(ctx, repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error { + err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error { if err != nil { return err } diff --git a/internal/restic/tree.go b/internal/restic/tree.go index 09ea44dfa53..d1264074c24 100644 --- a/internal/restic/tree.go +++ b/internal/restic/tree.go @@ -1,6 +1,9 @@ package restic import ( + "bytes" + "context" + "encoding/json" "fmt" "sort" @@ -98,3 +101,95 @@ func (t *Tree) Subtrees() (trees IDs) { return trees } + +type BlobLoader interface { + LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) +} + +// LoadTree loads a tree from the repository. +func LoadTree(ctx context.Context, r BlobLoader, id ID) (*Tree, error) { + debug.Log("load tree %v", id) + + buf, err := r.LoadBlob(ctx, TreeBlob, id, nil) + if err != nil { + return nil, err + } + + t := &Tree{} + err = json.Unmarshal(buf, t) + if err != nil { + return nil, err + } + + return t, nil +} + +type BlobSaver interface { + SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, int, error) +} + +// SaveTree stores a tree into the repository and returns the ID. The ID is +// checked against the index. The tree is only stored when the index does not +// contain the ID. +func SaveTree(ctx context.Context, r BlobSaver, t *Tree) (ID, error) { + buf, err := json.Marshal(t) + if err != nil { + return ID{}, errors.Wrap(err, "MarshalJSON") + } + + // append a newline so that the data is always consistent (json.Encoder + // adds a newline after each object) + buf = append(buf, '\n') + + id, _, _, err := r.SaveBlob(ctx, TreeBlob, buf, ID{}, false) + return id, err +} + +type TreeJSONBuilder struct { + buf bytes.Buffer + lastName string +} + +func NewTreeJSONBuilder() *TreeJSONBuilder { + tb := &TreeJSONBuilder{} + _, _ = tb.buf.WriteString(`{"nodes":[`) + return tb +} + +func (builder *TreeJSONBuilder) AddNode(node *Node) error { + if node.Name <= builder.lastName { + return errors.Errorf("nodes are not ordered got %q, last %q", node.Name, builder.lastName) + } + if builder.lastName != "" { + _ = builder.buf.WriteByte(',') + } + builder.lastName = node.Name + + val, err := json.Marshal(node) + if err != nil { + return err + } + _, _ = builder.buf.Write(val) + return nil +} + +func (builder *TreeJSONBuilder) Finalize() ([]byte, error) { + // append a newline so that the data is always consistent (json.Encoder + // adds a newline after each object) + _, _ = builder.buf.WriteString("]}\n") + buf := builder.buf.Bytes() + // drop reference to buffer + builder.buf = bytes.Buffer{} + return buf, nil +} + +func TreeToBuilder(t *Tree) (*TreeJSONBuilder, error) { + builder := NewTreeJSONBuilder() + for _, node := range t.Nodes { + err := builder.AddNode(node) + if err != nil { + return nil, err + } + } + return builder, nil +} diff --git a/internal/restic/tree_stream.go b/internal/restic/tree_stream.go index f6982efc2c9..4110a5e8d83 100644 --- a/internal/restic/tree_stream.go +++ b/internal/restic/tree_stream.go @@ -3,6 +3,7 @@ package restic import ( "context" "errors" + "runtime" "sync" "github.com/restic/restic/internal/debug" @@ -10,8 +11,6 @@ import ( "golang.org/x/sync/errgroup" ) -const streamTreeParallelism = 6 - // TreeItem is used to return either an error or the tree for a tree id type TreeItem struct { ID @@ -30,11 +29,11 @@ type trackedID struct { } // loadTreeWorker loads trees from repo and sends them to out. -func loadTreeWorker(ctx context.Context, repo TreeLoader, +func loadTreeWorker(ctx context.Context, repo Loader, in <-chan trackedID, out chan<- trackedTreeItem) { for treeID := range in { - tree, err := repo.LoadTree(ctx, treeID.ID) + tree, err := LoadTree(ctx, repo, treeID.ID) debug.Log("load tree %v (%v) returned err: %v", tree, treeID, err) job := trackedTreeItem{TreeItem: TreeItem{ID: treeID.ID, Error: err, Tree: tree}, rootIdx: treeID.rootIdx} @@ -46,7 +45,7 @@ func loadTreeWorker(ctx context.Context, repo TreeLoader, } } -func filterTrees(ctx context.Context, repo TreeLoader, trees IDs, loaderChan chan<- trackedID, hugeTreeLoaderChan chan<- trackedID, +func filterTrees(ctx context.Context, repo Loader, trees IDs, loaderChan chan<- trackedID, hugeTreeLoaderChan chan<- trackedID, in <-chan trackedTreeItem, out chan<- TreeItem, skip func(tree ID) bool, p *progress.Counter) { var ( @@ -155,7 +154,7 @@ func filterTrees(ctx context.Context, repo TreeLoader, trees IDs, loaderChan cha // is guaranteed to always be called from the same goroutine. To shutdown the started // goroutines, either read all items from the channel or cancel the context. Then `Wait()` // on the errgroup until all goroutines were stopped. -func StreamTrees(ctx context.Context, wg *errgroup.Group, repo TreeLoader, trees IDs, skip func(tree ID) bool, p *progress.Counter) <-chan TreeItem { +func StreamTrees(ctx context.Context, wg *errgroup.Group, repo Loader, trees IDs, skip func(tree ID) bool, p *progress.Counter) <-chan TreeItem { loaderChan := make(chan trackedID) hugeTreeChan := make(chan trackedID, 10) loadedTreeChan := make(chan trackedTreeItem) @@ -163,7 +162,10 @@ func StreamTrees(ctx context.Context, wg *errgroup.Group, repo TreeLoader, trees var loadTreeWg sync.WaitGroup - for i := 0; i < streamTreeParallelism; i++ { + // decoding a tree can take quite some time such that this can be both CPU- or IO-bound + // one extra worker to handle huge tree blobs + workerCount := int(repo.Connections()) + runtime.GOMAXPROCS(0) + 1 + for i := 0; i < workerCount; i++ { workerLoaderChan := loaderChan if i == 0 { workerLoaderChan = hugeTreeChan diff --git a/internal/restic/tree_test.go b/internal/restic/tree_test.go index 598a0ad4bef..811f0c6c6fb 100644 --- a/internal/restic/tree_test.go +++ b/internal/restic/tree_test.go @@ -9,9 +9,11 @@ import ( "strconv" "testing" + "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "golang.org/x/sync/errgroup" ) var testFiles = []struct { @@ -94,20 +96,22 @@ func TestNodeComparison(t *testing.T) { rtest.Assert(t, !node.Equals(n2), "nodes are equal") } -func TestLoadTree(t *testing.T) { +func TestEmptyLoadTree(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) // save tree tree := restic.NewTree(0) - id, err := repo.SaveTree(context.TODO(), tree) + id, err := restic.SaveTree(context.TODO(), repo, tree) rtest.OK(t, err) // save packs rtest.OK(t, repo.Flush(context.Background())) // load tree again - tree2, err := repo.LoadTree(context.TODO(), id) + tree2, err := restic.LoadTree(context.TODO(), repo, id) rtest.OK(t, err) rtest.Assert(t, tree.Equals(tree2), @@ -115,6 +119,37 @@ func TestLoadTree(t *testing.T) { tree, tree2) } +func TestTreeEqualSerialization(t *testing.T) { + files := []string{"node.go", "tree.go", "tree_test.go"} + for i := 1; i <= len(files); i++ { + tree := restic.NewTree(i) + builder := restic.NewTreeJSONBuilder() + + for _, fn := range files[:i] { + fi, err := os.Lstat(fn) + rtest.OK(t, err) + node, err := restic.NodeFromFileInfo(fn, fi) + rtest.OK(t, err) + + rtest.OK(t, tree.Insert(node)) + rtest.OK(t, builder.AddNode(node)) + + rtest.Assert(t, tree.Insert(node) != nil, "no error on duplicate node") + rtest.Assert(t, builder.AddNode(node) != nil, "no error on duplicate node") + } + + treeBytes, err := json.Marshal(tree) + treeBytes = append(treeBytes, '\n') + rtest.OK(t, err) + + stiBytes, err := builder.Finalize() + rtest.OK(t, err) + + // compare serialization of an individual node and the SaveTreeIterator + rtest.Equals(t, treeBytes, stiBytes) + } +} + func BenchmarkBuildTree(b *testing.B) { const size = 100 // Directories of this size are not uncommon. @@ -135,3 +170,47 @@ func BenchmarkBuildTree(b *testing.B) { } } } + +func TestLoadTree(t *testing.T) { + repository.TestAllVersions(t, testLoadTree) +} + +func testLoadTree(t *testing.T, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) + defer cleanup() + + if rtest.BenchArchiveDirectory == "" { + t.Skip("benchdir not set, skipping") + } + + // archive a few files + sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil) + rtest.OK(t, repo.Flush(context.Background())) + + _, err := restic.LoadTree(context.TODO(), repo, *sn.Tree) + rtest.OK(t, err) +} + +func BenchmarkLoadTree(t *testing.B) { + repository.BenchmarkAllVersions(t, benchmarkLoadTree) +} + +func benchmarkLoadTree(t *testing.B, version uint) { + repo, cleanup := repository.TestRepositoryWithVersion(t, version) + defer cleanup() + + if rtest.BenchArchiveDirectory == "" { + t.Skip("benchdir not set, skipping") + } + + // archive a few files + sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil) + rtest.OK(t, repo.Flush(context.Background())) + + t.ResetTimer() + + for i := 0; i < t.N; i++ { + _, err := restic.LoadTree(context.TODO(), repo, *sn.Tree) + rtest.OK(t, err) + } +} diff --git a/internal/restorer/doc.go b/internal/restorer/doc.go index 680e30c9076..e230f23f0b3 100644 --- a/internal/restorer/doc.go +++ b/internal/restorer/doc.go @@ -8,10 +8,10 @@ // Here is high-level pseudo-code of how the Restorer attempts to achieve // these goals: // -// while there are packs to process -// choose a pack to process [1] -// retrieve the pack from the backend [2] -// write pack blobs to the files that need them [3] +// while there are packs to process +// choose a pack to process [1] +// retrieve the pack from the backend [2] +// write pack blobs to the files that need them [3] // // Retrieval of repository packs (step [2]) and writing target files (step [3]) // are performed concurrently on multiple goroutines. diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index d3d52f13a39..362d821d2a0 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -1,12 +1,8 @@ package restorer import ( - "bufio" "context" - "io" - "math" "path/filepath" - "sort" "sync" "golang.org/x/sync/errgroup" @@ -14,6 +10,7 @@ import ( "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" ) @@ -23,8 +20,6 @@ import ( // con: each worker needs to keep one pack in memory const ( - workerCount = 8 - largeFileBlobCount = 25 ) @@ -52,8 +47,9 @@ type packInfo struct { type fileRestorer struct { key *crypto.Key idx func(restic.BlobHandle) []restic.PackedBlob - packLoader func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error + packLoader repository.BackendLoadFn + workerCount int filesWriter *filesWriter dst string @@ -62,15 +58,20 @@ type fileRestorer struct { } func newFileRestorer(dst string, - packLoader func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error, + packLoader repository.BackendLoadFn, key *crypto.Key, - idx func(restic.BlobHandle) []restic.PackedBlob) *fileRestorer { + idx func(restic.BlobHandle) []restic.PackedBlob, + connections uint) *fileRestorer { + + // as packs are streamed the concurrency is limited by IO + workerCount := int(connections) return &fileRestorer{ key: key, idx: idx, packLoader: packLoader, filesWriter: newFilesWriter(workerCount), + workerCount: workerCount, dst: dst, Error: restorerAbortOnAllErrors, } @@ -120,7 +121,7 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob) { if largeFile { packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.ID, offset: fileOffset}) - fileOffset += int64(blob.Length) - crypto.Extension + fileOffset += int64(blob.DataLength()) } pack, ok := packs[packID] if !ok { @@ -153,7 +154,7 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { } return nil } - for i := 0; i < workerCount; i++ { + for i := 0; i < r.workerCount; i++ { wg.Go(worker) } @@ -175,30 +176,19 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { return wg.Wait() } -const maxBufferSize = 4 * 1024 * 1024 - func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { - // calculate pack byte range and blob->[]files->[]offsets mappings - start, end := int64(math.MaxInt64), int64(0) + // calculate blob->[]files->[]offsets mappings blobs := make(map[restic.ID]struct { - offset int64 // offset of the blob in the pack - length int // length of the blob - files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file + files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file }) + var blobList []restic.Blob for file := range pack.files { addBlob := func(blob restic.Blob, fileOffset int64) { - if start > int64(blob.Offset) { - start = int64(blob.Offset) - } - if end < int64(blob.Offset+blob.Length) { - end = int64(blob.Offset + blob.Length) - } blobInfo, ok := blobs[blob.ID] if !ok { - blobInfo.offset = int64(blob.Offset) - blobInfo.length = int(blob.Length) blobInfo.files = make(map[*fileInfo][]int64) + blobList = append(blobList, blob) blobs[blob.ID] = blobInfo } blobInfo.files[file] = append(blobInfo.files[file], fileOffset) @@ -209,7 +199,7 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { if packID.Equal(pack.id) { addBlob(blob, fileOffset) } - fileOffset += int64(blob.Length) - crypto.Extension + fileOffset += int64(blob.DataLength()) }) if err != nil { // restoreFiles should have caught this error before @@ -228,14 +218,6 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { } } - sortedBlobs := make([]restic.ID, 0, len(blobs)) - for blobID := range blobs { - sortedBlobs = append(sortedBlobs, blobID) - } - sort.Slice(sortedBlobs, func(i, j int) bool { - return blobs[sortedBlobs[i]].offset < blobs[sortedBlobs[j]].offset - }) - sanitizeError := func(file *fileInfo, err error) error { if err != nil { err = r.Error(file.location, err) @@ -243,59 +225,39 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { return err } - h := restic.Handle{Type: restic.PackFile, Name: pack.id.String(), ContainedBlobType: restic.DataBlob} - err := r.packLoader(ctx, h, int(end-start), start, func(rd io.Reader) error { - bufferSize := int(end - start) - if bufferSize > maxBufferSize { - bufferSize = maxBufferSize - } - bufRd := bufio.NewReaderSize(rd, bufferSize) - currentBlobEnd := start - var blobData, buf []byte - for _, blobID := range sortedBlobs { - blob := blobs[blobID] - _, err := bufRd.Discard(int(blob.offset - currentBlobEnd)) - if err != nil { - return err - } - buf, err = r.downloadBlob(bufRd, blobID, blob.length, buf) - if err != nil { - return err - } - blobData, err = r.decryptBlob(blobID, buf) - if err != nil { - for file := range blob.files { - if errFile := sanitizeError(file, err); errFile != nil { - return errFile - } + err := repository.StreamPack(ctx, r.packLoader, r.key, pack.id, blobList, func(h restic.BlobHandle, blobData []byte, err error) error { + blob := blobs[h.ID] + if err != nil { + for file := range blob.files { + if errFile := sanitizeError(file, err); errFile != nil { + return errFile } - continue } - currentBlobEnd = blob.offset + int64(blob.length) - for file, offsets := range blob.files { - for _, offset := range offsets { - writeToFile := func() error { - // this looks overly complicated and needs explanation - // two competing requirements: - // - must create the file once and only once - // - should allow concurrent writes to the file - // so write the first blob while holding file lock - // write other blobs after releasing the lock - createSize := int64(-1) - file.lock.Lock() - if file.inProgress { - file.lock.Unlock() - } else { - defer file.lock.Unlock() - file.inProgress = true - createSize = file.size - } - return r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize) - } - err := sanitizeError(file, writeToFile()) - if err != nil { - return err + return nil + } + for file, offsets := range blob.files { + for _, offset := range offsets { + writeToFile := func() error { + // this looks overly complicated and needs explanation + // two competing requirements: + // - must create the file once and only once + // - should allow concurrent writes to the file + // so write the first blob while holding file lock + // write other blobs after releasing the lock + createSize := int64(-1) + file.lock.Lock() + if file.inProgress { + file.lock.Unlock() + } else { + defer file.lock.Unlock() + file.inProgress = true + createSize = file.size } + return r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize) + } + err := sanitizeError(file, writeToFile()) + if err != nil { + return err } } } @@ -312,41 +274,3 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { return nil } - -func (r *fileRestorer) downloadBlob(rd io.Reader, blobID restic.ID, length int, buf []byte) ([]byte, error) { - // TODO reconcile with Repository#loadBlob implementation - - if cap(buf) < length { - buf = make([]byte, length) - } else { - buf = buf[:length] - } - - n, err := io.ReadFull(rd, buf) - if err != nil { - return nil, err - } - - if n != length { - return nil, errors.Errorf("error loading blob %v: wrong length returned, want %d, got %d", blobID.Str(), length, n) - } - return buf, nil -} - -func (r *fileRestorer) decryptBlob(blobID restic.ID, buf []byte) ([]byte, error) { - // TODO reconcile with Repository#loadBlob implementation - - // decrypt - nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] - plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil) - if err != nil { - return nil, errors.Errorf("decrypting blob %v failed: %v", blobID, err) - } - - // check hash - if !restic.Hash(plaintext).Equal(blobID) { - return nil, errors.Errorf("blob %v returned invalid hash", blobID) - } - - return plaintext, nil -} diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index 333420b70cf..fa781f8c8c9 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -10,6 +10,7 @@ import ( "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -38,7 +39,7 @@ type TestRepo struct { filesPathToContent map[string]string // - loader func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error + loader repository.BackendLoadFn } func (i *TestRepo) Lookup(bh restic.BlobHandle) []restic.PackedBlob { @@ -60,7 +61,7 @@ func newTestRepo(content []TestFile) *TestRepo { key := crypto.NewRandomKey() seal := func(data []byte) []byte { - ciphertext := restic.NewBlobBuffer(len(data)) + ciphertext := crypto.NewBlobBuffer(len(data)) ciphertext = ciphertext[:0] // truncate the slice nonce := crypto.NewRandomNonce() ciphertext = append(ciphertext, nonce...) @@ -149,7 +150,7 @@ func newTestRepo(content []TestFile) *TestRepo { func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files map[string]bool) { repo := newTestRepo(content) - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup) + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2) if files == nil { r.files = repo.files @@ -263,11 +264,11 @@ func TestErrorRestoreFiles(t *testing.T) { return loadError } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup) + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2) r.files = repo.files err := r.restoreFiles(context.TODO()) - rtest.Equals(t, loadError, err) + rtest.Assert(t, errors.Is(err, loadError), "got %v, expected contained error %v", err, loadError) } func TestDownloadError(t *testing.T) { @@ -303,7 +304,7 @@ func testPartialDownloadError(t *testing.T, part int) { return loader(ctx, h, length, offset, fn) } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup) + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2) r.files = repo.files r.Error = func(s string, e error) error { // ignore errors as in the `restore` command diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index 542a15f0218..8b7ee435338 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -12,7 +12,7 @@ import ( // multiple files can be written to concurrently. // multiple blobs can be concurrently written to the same file. // TODO I am not 100% convinced this is necessary, i.e. it may be okay -// to use multiple os.File to write to the same target file +// to use multiple os.File to write to the same target file type filesWriter struct { buckets []filesWriterBucket } diff --git a/internal/restic/hardlinks_index.go b/internal/restorer/hardlinks_index.go similarity index 98% rename from internal/restic/hardlinks_index.go rename to internal/restorer/hardlinks_index.go index 0874f32a450..9cf45975a8d 100644 --- a/internal/restic/hardlinks_index.go +++ b/internal/restorer/hardlinks_index.go @@ -1,4 +1,4 @@ -package restic +package restorer import ( "sync" diff --git a/internal/restic/hardlinks_index_test.go b/internal/restorer/hardlinks_index_test.go similarity index 85% rename from internal/restic/hardlinks_index_test.go rename to internal/restorer/hardlinks_index_test.go index 8040d657f00..75a2b83ee00 100644 --- a/internal/restic/hardlinks_index_test.go +++ b/internal/restorer/hardlinks_index_test.go @@ -1,16 +1,16 @@ -package restic_test +package restorer_test import ( "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/restorer" rtest "github.com/restic/restic/internal/test" ) // TestHardLinks contains various tests for HardlinkIndex. func TestHardLinks(t *testing.T) { - idx := restic.NewHardlinkIndex() + idx := restorer.NewHardlinkIndex() idx.Add(1, 2, "inode1-file1-on-device2") idx.Add(2, 3, "inode2-file2-on-device3") diff --git a/internal/restorer/preallocate_other.go b/internal/restorer/preallocate_other.go index b43afc33561..f01757bf4c3 100644 --- a/internal/restorer/preallocate_other.go +++ b/internal/restorer/preallocate_other.go @@ -1,3 +1,4 @@ +//go:build !linux && !darwin // +build !linux,!darwin package restorer diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index a1e5b362851..829e5aedc9e 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -53,7 +53,7 @@ type treeVisitor struct { // target is the path in the file system, location within the snapshot. func (res *Restorer) traverseTree(ctx context.Context, target, location string, treeID restic.ID, visitor treeVisitor) (hasRestored bool, err error) { debug.Log("%v %v %v", target, location, treeID) - tree, err := res.repo.LoadTree(ctx, treeID) + tree, err := restic.LoadTree(ctx, res.repo, treeID) if err != nil { debug.Log("error loading tree %v: %v", treeID, err) return hasRestored, res.Error(location, err) @@ -218,8 +218,8 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } } - idx := restic.NewHardlinkIndex() - filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup) + idx := NewHardlinkIndex() + filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, res.repo.Connections()) filerestorer.Error = res.Error debug.Log("first pass for %q", dst) diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index a5a3bb5ba19..2eea1a6fd65 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "golang.org/x/sync/errgroup" ) type Node interface{} @@ -41,7 +42,7 @@ func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - id, _, err := repo.SaveBlob(ctx, restic.DataBlob, []byte(node.Data), restic.ID{}, false) + id, _, _, err := repo.SaveBlob(ctx, restic.DataBlob, []byte(node.Data), restic.ID{}, false) if err != nil { t.Fatal(err) } @@ -110,7 +111,7 @@ func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode } } - id, err := repo.SaveTree(ctx, tree) + id, err := restic.SaveTree(ctx, repo, tree) if err != nil { t.Fatal(err) } @@ -122,8 +123,9 @@ func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*res ctx, cancel := context.WithCancel(context.Background()) defer cancel() + wg, wgCtx := errgroup.WithContext(ctx) + repo.StartPackUploader(wgCtx, wg) treeID := saveDir(t, repo, snapshot.Nodes, 1000) - err := repo.Flush(ctx) if err != nil { t.Fatal(err) @@ -135,7 +137,7 @@ func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*res } sn.Tree = &treeID - id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn) + id, err := restic.SaveSnapshot(ctx, repo, sn) if err != nil { t.Fatal(err) } diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index f3f23cd1600..13e318c986c 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -1,4 +1,5 @@ -//+build !windows +//go:build !windows +// +build !windows package restorer diff --git a/internal/selfupdate/download.go b/internal/selfupdate/download.go index 888007c4c26..b12afd61c97 100644 --- a/internal/selfupdate/download.go +++ b/internal/selfupdate/download.go @@ -10,6 +10,7 @@ import ( "encoding/hex" "fmt" "io" + "io/ioutil" "os" "path/filepath" "runtime" @@ -40,14 +41,6 @@ func findHash(buf []byte, filename string) (hash []byte, err error) { } func extractToFile(buf []byte, filename, target string, printf func(string, ...interface{})) error { - var mode = os.FileMode(0755) - - // get information about the target file - fi, err := os.Lstat(target) - if err == nil { - mode = fi.Mode() - } - var rd io.Reader = bytes.NewReader(buf) switch filepath.Ext(filename) { case ".bz2": @@ -74,33 +67,44 @@ func extractToFile(buf []byte, filename, target string, printf func(string, ...i rd = file } - err = os.Remove(target) - if os.IsNotExist(err) { - err = nil - } + // Write everything to a temp file + dir := filepath.Dir(target) + new, err := ioutil.TempFile(dir, "restic") if err != nil { - return fmt.Errorf("unable to remove target file: %v", err) + return err } - dest, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode) + n, err := io.Copy(new, rd) if err != nil { + _ = new.Close() + _ = os.Remove(new.Name()) + return err + } + if err = new.Sync(); err != nil { + return err + } + if err = new.Close(); err != nil { return err } - n, err := io.Copy(dest, rd) - if err != nil { - _ = dest.Close() - _ = os.Remove(dest.Name()) + mode := os.FileMode(0755) + // attempt to find the original mode + if fi, err := os.Lstat(target); err == nil { + mode = fi.Mode() + } + + // Remove the original binary. + if err := removeResticBinary(dir, target); err != nil { return err } - err = dest.Close() - if err != nil { + // Rename the temp file to the final location atomically. + if err := os.Rename(new.Name(), target); err != nil { return err } - printf("saved %d bytes in %v\n", n, dest.Name()) - return nil + printf("saved %d bytes in %v\n", n, target) + return os.Chmod(target, mode) } // DownloadLatestStableRelease downloads the latest stable released version of diff --git a/internal/selfupdate/download_unix.go b/internal/selfupdate/download_unix.go new file mode 100644 index 00000000000..c6189e9d943 --- /dev/null +++ b/internal/selfupdate/download_unix.go @@ -0,0 +1,10 @@ +//go:build !windows +// +build !windows + +package selfupdate + +// Remove the target binary. +func removeResticBinary(dir, target string) error { + // removed on rename on this platform + return nil +} diff --git a/internal/selfupdate/download_windows.go b/internal/selfupdate/download_windows.go new file mode 100644 index 00000000000..4f27973852d --- /dev/null +++ b/internal/selfupdate/download_windows.go @@ -0,0 +1,23 @@ +//go:build windows +// +build windows + +package selfupdate + +import ( + "fmt" + "os" + "path/filepath" +) + +// Rename (rather than remove) the running version. The running binary will be locked +// on Windows and cannot be removed while still executing. +func removeResticBinary(dir, target string) error { + backup := filepath.Join(dir, filepath.Base(target)+".bak") + if _, err := os.Stat(backup); err == nil { + _ = os.Remove(backup) + } + if err := os.Rename(target, backup); err != nil { + return fmt.Errorf("unable to rename target file: %v", err) + } + return nil +} diff --git a/internal/test/helpers.go b/internal/test/helpers.go index 1e127f7863a..9ace2df5e10 100644 --- a/internal/test/helpers.go +++ b/internal/test/helpers.go @@ -175,7 +175,7 @@ func ResetReadOnly(t testing.TB, dir string) { return nil }) - if os.IsNotExist(errors.Cause(err)) { + if errors.Is(err, os.ErrNotExist) { err = nil } OK(t, err) @@ -186,7 +186,7 @@ func ResetReadOnly(t testing.TB, dir string) { func RemoveAll(t testing.TB, path string) { ResetReadOnly(t, path) err := os.RemoveAll(path) - if os.IsNotExist(errors.Cause(err)) { + if errors.Is(err, os.ErrNotExist) { err = nil } OK(t, err) diff --git a/internal/ui/backup/json.go b/internal/ui/backup/json.go index fcc200b2a9b..1cbd0c19714 100644 --- a/internal/ui/backup/json.go +++ b/internal/ui/backup/json.go @@ -3,7 +3,6 @@ package backup import ( "bytes" "encoding/json" - "os" "sort" "time" @@ -79,7 +78,7 @@ func (b *JSONProgress) Update(total, processed Counter, errors uint, currentFile // ScannerError is the error callback function for the scanner, it prints the // error in verbose mode and returns nil. -func (b *JSONProgress) ScannerError(item string, fi os.FileInfo, err error) error { +func (b *JSONProgress) ScannerError(item string, err error) error { b.error(errorUpdate{ MessageType: "error", Error: err, @@ -90,7 +89,7 @@ func (b *JSONProgress) ScannerError(item string, fi os.FileInfo, err error) erro } // Error is the error callback function for the archiver, it prints the error and returns nil. -func (b *JSONProgress) Error(item string, fi os.FileInfo, err error) error { +func (b *JSONProgress) Error(item string, err error) error { b.error(errorUpdate{ MessageType: "error", Error: err, @@ -110,12 +109,14 @@ func (b *JSONProgress) CompleteItem(messageType, item string, previous, current switch messageType { case "dir new": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "new", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, - MetadataSize: s.TreeSize, + MessageType: "verbose_status", + Action: "new", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, + MetadataSize: s.TreeSize, + MetadataSizeInRepo: s.TreeSizeInRepo, }) case "dir unchanged": b.print(verboseUpdate{ @@ -125,20 +126,23 @@ func (b *JSONProgress) CompleteItem(messageType, item string, previous, current }) case "dir modified": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "modified", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, - MetadataSize: s.TreeSize, + MessageType: "verbose_status", + Action: "modified", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, + MetadataSize: s.TreeSize, + MetadataSizeInRepo: s.TreeSizeInRepo, }) case "file new": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "new", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, + MessageType: "verbose_status", + Action: "new", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, }) case "file unchanged": b.print(verboseUpdate{ @@ -148,11 +152,12 @@ func (b *JSONProgress) CompleteItem(messageType, item string, previous, current }) case "file modified": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "modified", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, + MessageType: "verbose_status", + Action: "modified", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, }) } } @@ -216,13 +221,15 @@ type errorUpdate struct { } type verboseUpdate struct { - MessageType string `json:"message_type"` // "verbose_status" - Action string `json:"action"` - Item string `json:"item"` - Duration float64 `json:"duration"` // in seconds - DataSize uint64 `json:"data_size"` - MetadataSize uint64 `json:"metadata_size"` - TotalFiles uint `json:"total_files"` + MessageType string `json:"message_type"` // "verbose_status" + Action string `json:"action"` + Item string `json:"item"` + Duration float64 `json:"duration"` // in seconds + DataSize uint64 `json:"data_size"` + DataSizeInRepo uint64 `json:"data_size_in_repo"` + MetadataSize uint64 `json:"metadata_size"` + MetadataSizeInRepo uint64 `json:"metadata_size_in_repo"` + TotalFiles uint `json:"total_files"` } type summaryOutput struct { diff --git a/internal/ui/backup/progress.go b/internal/ui/backup/progress.go index 781ac289b18..a4b641fe9ce 100644 --- a/internal/ui/backup/progress.go +++ b/internal/ui/backup/progress.go @@ -3,7 +3,6 @@ package backup import ( "context" "io" - "os" "sync" "time" @@ -14,8 +13,8 @@ import ( type ProgressPrinter interface { Update(total, processed Counter, errors uint, currentFiles map[string]struct{}, start time.Time, secs uint64) - Error(item string, fi os.FileInfo, err error) error - ScannerError(item string, fi os.FileInfo, err error) error + Error(item string, err error) error + ScannerError(item string, err error) error CompleteItem(messageType string, item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) ReportTotal(item string, start time.Time, s archiver.ScanStats) Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool) @@ -44,11 +43,11 @@ type ProgressReporter interface { CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) StartFile(filename string) CompleteBlob(filename string, bytes uint64) - ScannerError(item string, fi os.FileInfo, err error) error + ScannerError(item string, err error) error ReportTotal(item string, s archiver.ScanStats) SetMinUpdatePause(d time.Duration) Run(ctx context.Context) error - Error(item string, fi os.FileInfo, err error) error + Error(item string, err error) error Finish(snapshotID restic.ID) } @@ -173,13 +172,13 @@ func (p *Progress) Run(ctx context.Context) error { // ScannerError is the error callback function for the scanner, it prints the // error in verbose mode and returns nil. -func (p *Progress) ScannerError(item string, fi os.FileInfo, err error) error { - return p.printer.ScannerError(item, fi, err) +func (p *Progress) ScannerError(item string, err error) error { + return p.printer.ScannerError(item, err) } // Error is the error callback function for the archiver, it prints the error and returns nil. -func (p *Progress) Error(item string, fi os.FileInfo, err error) error { - cbErr := p.printer.Error(item, fi, err) +func (p *Progress) Error(item string, err error) error { + cbErr := p.printer.Error(item, err) select { case p.errCh <- struct{}{}: diff --git a/internal/ui/backup/text.go b/internal/ui/backup/text.go index 998805865f1..03013bec174 100644 --- a/internal/ui/backup/text.go +++ b/internal/ui/backup/text.go @@ -2,7 +2,6 @@ package backup import ( "fmt" - "os" "sort" "time" @@ -75,13 +74,13 @@ func (b *TextProgress) Update(total, processed Counter, errors uint, currentFile // ScannerError is the error callback function for the scanner, it prints the // error in verbose mode and returns nil. -func (b *TextProgress) ScannerError(item string, fi os.FileInfo, err error) error { +func (b *TextProgress) ScannerError(item string, err error) error { b.V("scan: %v\n", err) return nil } // Error is the error callback function for the archiver, it prints the error and returns nil. -func (b *TextProgress) Error(item string, fi os.FileInfo, err error) error { +func (b *TextProgress) Error(item string, err error) error { b.E("error: %v\n", err) return nil } @@ -138,17 +137,17 @@ func formatBytes(c uint64) string { func (b *TextProgress) CompleteItem(messageType, item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) { switch messageType { case "dir new": - b.VV("new %v, saved in %.3fs (%v added, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.TreeSize)) + b.VV("new %v, saved in %.3fs (%v added, %v stored, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo), formatBytes(s.TreeSizeInRepo)) case "dir unchanged": b.VV("unchanged %v", item) case "dir modified": - b.VV("modified %v, saved in %.3fs (%v added, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.TreeSize)) + b.VV("modified %v, saved in %.3fs (%v added, %v stored, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo), formatBytes(s.TreeSizeInRepo)) case "file new": b.VV("new %v, saved in %.3fs (%v added)", item, d.Seconds(), formatBytes(s.DataSize)) case "file unchanged": b.VV("unchanged %v", item) case "file modified": - b.VV("modified %v, saved in %.3fs (%v added)", item, d.Seconds(), formatBytes(s.DataSize)) + b.VV("modified %v, saved in %.3fs (%v added, %v stored)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo)) } } @@ -178,7 +177,7 @@ func (b *TextProgress) Finish(snapshotID restic.ID, start time.Time, summary *Su if dryRun { verb = "Would add" } - b.P("%s to the repo: %-5s\n", verb, formatBytes(summary.ItemStats.DataSize+summary.ItemStats.TreeSize)) + b.P("%s to the repository: %-5s (%-5s stored)\n", verb, formatBytes(summary.ItemStats.DataSize+summary.ItemStats.TreeSize), formatBytes(summary.ItemStats.DataSizeInRepo+summary.ItemStats.TreeSizeInRepo)) b.P("\n") b.P("processed %v files, %v in %s", summary.Files.New+summary.Files.Changed+summary.Files.Unchanged, diff --git a/internal/ui/progress/counter_test.go b/internal/ui/progress/counter_test.go index 49a99f7ee42..85695d209d2 100644 --- a/internal/ui/progress/counter_test.go +++ b/internal/ui/progress/counter_test.go @@ -61,7 +61,7 @@ func TestCounter(t *testing.T) { func TestCounterNil(t *testing.T) { // Shouldn't panic. - var c *progress.Counter = nil + var c *progress.Counter c.Add(1) c.Done() } diff --git a/internal/ui/signals/signals_bsd.go b/internal/ui/signals/signals_bsd.go index be3ab8882cf..d96e48c4ecb 100644 --- a/internal/ui/signals/signals_bsd.go +++ b/internal/ui/signals/signals_bsd.go @@ -1,3 +1,4 @@ +//go:build darwin || dragonfly || freebsd || netbsd || openbsd // +build darwin dragonfly freebsd netbsd openbsd package signals diff --git a/internal/ui/signals/signals_sysv.go b/internal/ui/signals/signals_sysv.go index a3b4eb29ea2..9480c1c9959 100644 --- a/internal/ui/signals/signals_sysv.go +++ b/internal/ui/signals/signals_sysv.go @@ -1,3 +1,4 @@ +//go:build aix || linux || solaris // +build aix linux solaris package signals diff --git a/internal/ui/termstatus/background.go b/internal/ui/termstatus/background.go index 8c1e9f1628e..4834a460f70 100644 --- a/internal/ui/termstatus/background.go +++ b/internal/ui/termstatus/background.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package termstatus diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index ce6593f37c0..88c4f898e8b 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -10,7 +10,7 @@ import ( "strings" "unicode" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" "golang.org/x/text/width" ) @@ -321,7 +321,7 @@ func (t *Terminal) SetStatus(lines []string) { var width int if t.canUpdateStatus { var err error - width, _, err = terminal.GetSize(int(t.fd)) + width, _, err = term.GetSize(int(t.fd)) if err != nil || width <= 0 { // use 80 columns by default width = 80 diff --git a/internal/ui/termstatus/terminal_unix.go b/internal/ui/termstatus/terminal_unix.go index 67ce06b0bc2..7190169394a 100644 --- a/internal/ui/termstatus/terminal_unix.go +++ b/internal/ui/termstatus/terminal_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package termstatus @@ -6,7 +7,7 @@ import ( "io" "os" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) // clearCurrentLine removes all characters from the current line and resets the @@ -23,7 +24,7 @@ func moveCursorUp(wr io.Writer, fd uintptr) func(io.Writer, uintptr, int) { // CanUpdateStatus returns true if status lines can be printed, the process // output is not redirected to a file or pipe. func CanUpdateStatus(fd uintptr) bool { - if !terminal.IsTerminal(int(fd)) { + if !term.IsTerminal(int(fd)) { return false } term := os.Getenv("TERM") diff --git a/internal/ui/termstatus/terminal_windows.go b/internal/ui/termstatus/terminal_windows.go index f7217e35dc9..d1358c02219 100644 --- a/internal/ui/termstatus/terminal_windows.go +++ b/internal/ui/termstatus/terminal_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package termstatus diff --git a/internal/walker/walker.go b/internal/walker/walker.go index 3c8e723a827..4c4e7f5abc3 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -10,11 +10,6 @@ import ( "github.com/restic/restic/internal/restic" ) -// TreeLoader loads a tree from a repository. -type TreeLoader interface { - LoadTree(context.Context, restic.ID) (*restic.Tree, error) -} - // ErrSkipNode is returned by WalkFunc when a dir node should not be walked. var ErrSkipNode = errors.New("skip this node") @@ -38,8 +33,8 @@ type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeE // Walk calls walkFn recursively for each node in root. If walkFn returns an // error, it is passed up the call stack. The trees in ignoreTrees are not // walked. If walkFn ignores trees, these are added to the set. -func Walk(ctx context.Context, repo TreeLoader, root restic.ID, ignoreTrees restic.IDSet, walkFn WalkFunc) error { - tree, err := repo.LoadTree(ctx, root) +func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, ignoreTrees restic.IDSet, walkFn WalkFunc) error { + tree, err := restic.LoadTree(ctx, repo, root) _, err = walkFn(root, "/", nil, err) if err != nil { @@ -60,7 +55,7 @@ func Walk(ctx context.Context, repo TreeLoader, root restic.ID, ignoreTrees rest // walk recursively traverses the tree, ignoring subtrees when the ID of the // subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID // will be added to ignoreTrees by walk. -func walk(ctx context.Context, repo TreeLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) { +func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) { var allNodesIgnored = true if len(tree.Nodes) == 0 { @@ -104,7 +99,7 @@ func walk(ctx context.Context, repo TreeLoader, prefix string, parentTreeID rest continue } - subtree, err := repo.LoadTree(ctx, *node.Subtree) + subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) ignore, err := walkFn(parentTreeID, p, node, err) if err != nil { if err == ErrSkipNode { diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 7a939b1e2dd..90ca7c2b8b1 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -67,13 +67,26 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID { // TreeMap returns the trees from the map on LoadTree. type TreeMap map[restic.ID]*restic.Tree -func (t TreeMap) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) { +func (t TreeMap) LoadBlob(ctx context.Context, tpe restic.BlobType, id restic.ID, buf []byte) ([]byte, error) { + if tpe != restic.TreeBlob { + return nil, errors.New("can only load trees") + } tree, ok := t[id] if !ok { return nil, errors.New("tree not found") } - return tree, nil + tbuf, err := json.Marshal(tree) + if err != nil { + panic(err) + } + tbuf = append(tbuf, '\n') + + return tbuf, nil +} + +func (t TreeMap) Connections() uint { + return 2 } // checkFunc returns a function suitable for walking the tree to check