diff --git a/CHANGELOG.md b/CHANGELOG.md index bd14e711d..1d31fec8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Syncthing upgraded to v1.26.1 -- Restic upgraded to v0.16.3 +- Restic upgraded to v0.16.4 ## [0.8.0] diff --git a/mover-restic/SOURCE_VERSIONS b/mover-restic/SOURCE_VERSIONS index 6da975cad..f09b410a2 100644 --- a/mover-restic/SOURCE_VERSIONS +++ b/mover-restic/SOURCE_VERSIONS @@ -1,2 +1,2 @@ -https://github.com/restic/restic.git v0.16.3 abca1124041f9c83227d253142dcc713e7b3914c +https://github.com/restic/restic.git v0.16.4 3786536dc18ef27aedcfa8e4c6953b48353eee79 https://github.com/minio/minio-go.git v7.0.66 5415e6c72a71610108fe05ee747ac760dd40094f diff --git a/mover-restic/restic/CHANGELOG.md b/mover-restic/restic/CHANGELOG.md index ba6fa7a11..b8969a443 100644 --- a/mover-restic/restic/CHANGELOG.md +++ b/mover-restic/restic/CHANGELOG.md @@ -1,5 +1,6 @@ # Table of Contents +* [Changelog for 0.16.4](#changelog-for-restic-0164-2024-02-04) * [Changelog for 0.16.3](#changelog-for-restic-0163-2024-01-14) * [Changelog for 0.16.2](#changelog-for-restic-0162-2023-10-29) * [Changelog for 0.16.1](#changelog-for-restic-0161-2023-10-24) @@ -32,6 +33,57 @@ * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) +# Changelog for restic 0.16.4 (2024-02-04) +The following sections list the changes in restic 0.16.4 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #4677: Downgrade zstd library to fix rare data corruption at max. compression + * Enh #4529: Add extra verification of data integrity before upload + +## Details + + * Bugfix #4677: Downgrade zstd library to fix rare data corruption at max. compression + + In restic 0.16.3, backups where the compression level was set to `max` (using + `--compression max`) could in rare and very specific circumstances result in + data corruption due to a bug in the library used for compressing data. Restic + 0.16.1 and 0.16.2 were not affected. + + Restic now uses the previous version of the library used to compress data, the + same version used by restic 0.16.2. Please note that the `auto` compression + level (which restic uses by default) was never affected, and even if you used + `max` compression, chances of being affected by this issue are small. + + To check a repository for any corruption, run `restic check --read-data`. This + will download and verify the whole repository and can be used at any time to + completely verify the integrity of a repository. If the `check` command detects + anomalies, follow the suggested steps. + + https://github.com/restic/restic/issues/4677 + https://github.com/restic/restic/pull/4679 + + * Enhancement #4529: Add extra verification of data integrity before upload + + Hardware issues, or a bug in restic or its dependencies, could previously cause + corruption in the files restic created and stored in the repository. Detecting + such corruption previously required explicitly running the `check --read-data` + or `check --read-data-subset` commands. + + To further ensure data integrity, even in the case of hardware issues or + software bugs, restic now performs additional verification of the files about to + be uploaded to the repository. + + These extra checks will increase CPU usage during backups. They can therefore, + if absolutely necessary, be disabled using the `--no-extra-verify` global + option. Please note that this should be combined with more active checking using + the previously mentioned check commands. + + https://github.com/restic/restic/issues/4529 + https://github.com/restic/restic/pull/4681 + + # Changelog for restic 0.16.3 (2024-01-14) The following sections list the changes in restic 0.16.3 relevant to restic users. The changes are ordered by importance. diff --git a/mover-restic/restic/VERSION b/mover-restic/restic/VERSION index 7eb3095a3..5f2491c5a 100644 --- a/mover-restic/restic/VERSION +++ b/mover-restic/restic/VERSION @@ -1 +1 @@ -0.16.3 +0.16.4 diff --git a/mover-restic/restic/changelog/0.16.4_2024-02-04/issue-4529 b/mover-restic/restic/changelog/0.16.4_2024-02-04/issue-4529 new file mode 100644 index 000000000..fed726d2d --- /dev/null +++ b/mover-restic/restic/changelog/0.16.4_2024-02-04/issue-4529 @@ -0,0 +1,18 @@ +Enhancement: Add extra verification of data integrity before upload + +Hardware issues, or a bug in restic or its dependencies, could previously cause +corruption in the files restic created and stored in the repository. Detecting +such corruption previously required explicitly running the `check --read-data` +or `check --read-data-subset` commands. + +To further ensure data integrity, even in the case of hardware issues or +software bugs, restic now performs additional verification of the files about +to be uploaded to the repository. + +These extra checks will increase CPU usage during backups. They can therefore, +if absolutely necessary, be disabled using the `--no-extra-verify` global +option. Please note that this should be combined with more active checking +using the previously mentioned check commands. + +https://github.com/restic/restic/issues/4529 +https://github.com/restic/restic/pull/4681 diff --git a/mover-restic/restic/changelog/0.16.4_2024-02-04/issue-4677 b/mover-restic/restic/changelog/0.16.4_2024-02-04/issue-4677 new file mode 100644 index 000000000..8fa6cf65b --- /dev/null +++ b/mover-restic/restic/changelog/0.16.4_2024-02-04/issue-4677 @@ -0,0 +1,19 @@ +Bugfix: Downgrade zstd library to fix rare data corruption at max. compression + +In restic 0.16.3, backups where the compression level was set to `max` (using +`--compression max`) could in rare and very specific circumstances result in +data corruption due to a bug in the library used for compressing data. Restic +0.16.1 and 0.16.2 were not affected. + +Restic now uses the previous version of the library used to compress data, the +same version used by restic 0.16.2. Please note that the `auto` compression +level (which restic uses by default) was never affected, and even if you used +`max` compression, chances of being affected by this issue are small. + +To check a repository for any corruption, run `restic check --read-data`. This +will download and verify the whole repository and can be used at any time to +completely verify the integrity of a repository. If the `check` command detects +anomalies, follow the suggested steps. + +https://github.com/restic/restic/issues/4677 +https://github.com/restic/restic/pull/4679 diff --git a/mover-restic/restic/cmd/restic/global.go b/mover-restic/restic/cmd/restic/global.go index c11aca615..e979dcc2b 100644 --- a/mover-restic/restic/cmd/restic/global.go +++ b/mover-restic/restic/cmd/restic/global.go @@ -43,7 +43,7 @@ import ( "golang.org/x/term" ) -var version = "0.16.3" +var version = "0.16.4" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" @@ -67,6 +67,7 @@ type GlobalOptions struct { CleanupCache bool Compression repository.CompressionMode PackSize uint + NoExtraVerify bool backend.TransportOptions limiter.Limits @@ -139,6 +140,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") + f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip additional verification of data before upload (see documentation)") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") @@ -453,8 +455,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } s, err := repository.New(be, repository.Options{ - Compression: opts.Compression, - PackSize: opts.PackSize * 1024 * 1024, + Compression: opts.Compression, + PackSize: opts.PackSize * 1024 * 1024, + NoExtraVerify: opts.NoExtraVerify, }) if err != nil { return nil, errors.Fatal(err.Error()) diff --git a/mover-restic/restic/doc/047_tuning_backup_parameters.rst b/mover-restic/restic/doc/047_tuning_backup_parameters.rst index 6ea39dc75..d8fb2c9b6 100644 --- a/mover-restic/restic/doc/047_tuning_backup_parameters.rst +++ b/mover-restic/restic/doc/047_tuning_backup_parameters.rst @@ -60,6 +60,20 @@ only applied for the single run of restic. The option can also be set via the en variable ``RESTIC_COMPRESSION``. +Data Verification +================= + +To prevent the upload of corrupted data to the repository, which can happen due +to hardware issues or software bugs, restic verifies that generated files can +be decoded and contain the correct data beforehand. This increases the CPU usage +during backups. If necessary, you can disable this verification using the +``--no-extra-verify`` option of the ``backup`` command. However, in this case +you should verify the repository integrity more actively using +``restic check --read-data`` (or the similar ``--read-data-subset`` option). +Otherwise, data corruption due to hardware issues or software bugs might go +unnoticed. + + File Read Concurrency ===================== diff --git a/mover-restic/restic/doc/bash-completion.sh b/mover-restic/restic/doc/bash-completion.sh index e691af363..cae37a6ca 100644 --- a/mover-restic/restic/doc/bash-completion.sh +++ b/mover-restic/restic/doc/bash-completion.sh @@ -488,6 +488,7 @@ _restic_backup() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -560,6 +561,7 @@ _restic_cache() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -624,6 +626,7 @@ _restic_cat() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -696,6 +699,7 @@ _restic_check() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -794,6 +798,7 @@ _restic_copy() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -860,6 +865,7 @@ _restic_diff() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -944,6 +950,7 @@ _restic_dump() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1058,6 +1065,7 @@ _restic_find() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1228,6 +1236,7 @@ _restic_forget() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1312,6 +1321,7 @@ _restic_generate() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1372,6 +1382,7 @@ _restic_help() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1463,6 +1474,7 @@ _restic_init() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1539,6 +1551,7 @@ _restic_key() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1603,6 +1616,7 @@ _restic_list() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1689,6 +1703,7 @@ _restic_ls() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1757,6 +1772,7 @@ _restic_migrate() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1849,6 +1865,7 @@ _restic_mount() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1935,6 +1952,7 @@ _restic_prune() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1999,6 +2017,7 @@ _restic_recover() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2059,6 +2078,7 @@ _restic_repair_help() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2126,6 +2146,7 @@ _restic_repair_index() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2190,6 +2211,7 @@ _restic_repair_packs() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2274,6 +2296,7 @@ _restic_repair_snapshots() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2342,6 +2365,7 @@ _restic_repair() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2450,6 +2474,7 @@ _restic_restore() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2552,6 +2577,7 @@ _restic_rewrite() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2620,6 +2646,7 @@ _restic_self-update() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2712,6 +2739,7 @@ _restic_snapshots() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2794,6 +2822,7 @@ _restic_stats() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2884,6 +2913,7 @@ _restic_tag() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2950,6 +2980,7 @@ _restic_unlock() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -3014,6 +3045,7 @@ _restic_version() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -3106,6 +3138,7 @@ _restic_root_command() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") diff --git a/mover-restic/restic/doc/man/restic-backup.1 b/mover-restic/restic/doc/man/restic-backup.1 index c3bccdfa5..730685271 100644 --- a/mover-restic/restic/doc/man/restic-backup.1 +++ b/mover-restic/restic/doc/man/restic-backup.1 @@ -171,6 +171,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-cache.1 b/mover-restic/restic/doc/man/restic-cache.1 index 3ae27ea57..c170c1624 100644 --- a/mover-restic/restic/doc/man/restic-cache.1 +++ b/mover-restic/restic/doc/man/restic-cache.1 @@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-cat.1 b/mover-restic/restic/doc/man/restic-cat.1 index c1df138aa..b42a58e14 100644 --- a/mover-restic/restic/doc/man/restic-cat.1 +++ b/mover-restic/restic/doc/man/restic-cat.1 @@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-check.1 b/mover-restic/restic/doc/man/restic-check.1 index 17eb972bc..9c1dc77e5 100644 --- a/mover-restic/restic/doc/man/restic-check.1 +++ b/mover-restic/restic/doc/man/restic-check.1 @@ -85,6 +85,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-copy.1 b/mover-restic/restic/doc/man/restic-copy.1 index be8f21e25..bd9795f44 100644 --- a/mover-restic/restic/doc/man/restic-copy.1 +++ b/mover-restic/restic/doc/man/restic-copy.1 @@ -109,6 +109,10 @@ new destination repository using the "init" command. \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-diff.1 b/mover-restic/restic/doc/man/restic-diff.1 index a01a2562b..28f3a4838 100644 --- a/mover-restic/restic/doc/man/restic-diff.1 +++ b/mover-restic/restic/doc/man/restic-diff.1 @@ -93,6 +93,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-dump.1 b/mover-restic/restic/doc/man/restic-dump.1 index 6fa1f8200..7fa3f777d 100644 --- a/mover-restic/restic/doc/man/restic-dump.1 +++ b/mover-restic/restic/doc/man/restic-dump.1 @@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-find.1 b/mover-restic/restic/doc/man/restic-find.1 index 72bc3a0b6..c3297c43f 100644 --- a/mover-restic/restic/doc/man/restic-find.1 +++ b/mover-restic/restic/doc/man/restic-find.1 @@ -117,6 +117,10 @@ It can also be used to search for restic blobs or trees for troubleshooting. \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-forget.1 b/mover-restic/restic/doc/man/restic-forget.1 index 757022a21..d0c4cfc74 100644 --- a/mover-restic/restic/doc/man/restic-forget.1 +++ b/mover-restic/restic/doc/man/restic-forget.1 @@ -179,6 +179,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-generate.1 b/mover-restic/restic/doc/man/restic-generate.1 index aef3a5e55..84f659ef2 100644 --- a/mover-restic/restic/doc/man/restic-generate.1 +++ b/mover-restic/restic/doc/man/restic-generate.1 @@ -89,6 +89,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-init.1 b/mover-restic/restic/doc/man/restic-init.1 index 27d7f5874..5f19c8f8c 100644 --- a/mover-restic/restic/doc/man/restic-init.1 +++ b/mover-restic/restic/doc/man/restic-init.1 @@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-key.1 b/mover-restic/restic/doc/man/restic-key.1 index 855ef5443..8d1813188 100644 --- a/mover-restic/restic/doc/man/restic-key.1 +++ b/mover-restic/restic/doc/man/restic-key.1 @@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-list.1 b/mover-restic/restic/doc/man/restic-list.1 index 95eeac5f7..e399038a2 100644 --- a/mover-restic/restic/doc/man/restic-list.1 +++ b/mover-restic/restic/doc/man/restic-list.1 @@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-ls.1 b/mover-restic/restic/doc/man/restic-ls.1 index 0cd0f5a88..10b0657a3 100644 --- a/mover-restic/restic/doc/man/restic-ls.1 +++ b/mover-restic/restic/doc/man/restic-ls.1 @@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-migrate.1 b/mover-restic/restic/doc/man/restic-migrate.1 index eca0ef8e1..7e48f726c 100644 --- a/mover-restic/restic/doc/man/restic-migrate.1 +++ b/mover-restic/restic/doc/man/restic-migrate.1 @@ -74,6 +74,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-mount.1 b/mover-restic/restic/doc/man/restic-mount.1 index 33c016ffa..aab607fcf 100644 --- a/mover-restic/restic/doc/man/restic-mount.1 +++ b/mover-restic/restic/doc/man/restic-mount.1 @@ -144,6 +144,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-prune.1 b/mover-restic/restic/doc/man/restic-prune.1 index e4a32cac3..c54d5d7ff 100644 --- a/mover-restic/restic/doc/man/restic-prune.1 +++ b/mover-restic/restic/doc/man/restic-prune.1 @@ -97,6 +97,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-recover.1 b/mover-restic/restic/doc/man/restic-recover.1 index 26d2fc7bd..010fbafd7 100644 --- a/mover-restic/restic/doc/man/restic-recover.1 +++ b/mover-restic/restic/doc/man/restic-recover.1 @@ -70,6 +70,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-repair-index.1 b/mover-restic/restic/doc/man/restic-repair-index.1 index 35e2845b8..f06be64c0 100644 --- a/mover-restic/restic/doc/man/restic-repair-index.1 +++ b/mover-restic/restic/doc/man/restic-repair-index.1 @@ -73,6 +73,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-repair-packs.1 b/mover-restic/restic/doc/man/restic-repair-packs.1 index b21211925..f3671fe18 100644 --- a/mover-restic/restic/doc/man/restic-repair-packs.1 +++ b/mover-restic/restic/doc/man/restic-repair-packs.1 @@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-repair-snapshots.1 b/mover-restic/restic/doc/man/restic-repair-snapshots.1 index f59067f05..9369f25f2 100644 --- a/mover-restic/restic/doc/man/restic-repair-snapshots.1 +++ b/mover-restic/restic/doc/man/restic-repair-snapshots.1 @@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-repair.1 b/mover-restic/restic/doc/man/restic-repair.1 index dbe783df4..77aecc173 100644 --- a/mover-restic/restic/doc/man/restic-repair.1 +++ b/mover-restic/restic/doc/man/restic-repair.1 @@ -63,6 +63,10 @@ Repair the repository \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-restore.1 b/mover-restic/restic/doc/man/restic-restore.1 index d8c1b72e1..4635b1e43 100644 --- a/mover-restic/restic/doc/man/restic-restore.1 +++ b/mover-restic/restic/doc/man/restic-restore.1 @@ -117,6 +117,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-rewrite.1 b/mover-restic/restic/doc/man/restic-rewrite.1 index 8a06aef40..d63c653e6 100644 --- a/mover-restic/restic/doc/man/restic-rewrite.1 +++ b/mover-restic/restic/doc/man/restic-rewrite.1 @@ -121,6 +121,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-self-update.1 b/mover-restic/restic/doc/man/restic-self-update.1 index 28fd24a92..92ab5add3 100644 --- a/mover-restic/restic/doc/man/restic-self-update.1 +++ b/mover-restic/restic/doc/man/restic-self-update.1 @@ -75,6 +75,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-snapshots.1 b/mover-restic/restic/doc/man/restic-snapshots.1 index cb34d6c8e..6203bbf2b 100644 --- a/mover-restic/restic/doc/man/restic-snapshots.1 +++ b/mover-restic/restic/doc/man/restic-snapshots.1 @@ -92,6 +92,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-stats.1 b/mover-restic/restic/doc/man/restic-stats.1 index cf0374351..9d37163de 100644 --- a/mover-restic/restic/doc/man/restic-stats.1 +++ b/mover-restic/restic/doc/man/restic-stats.1 @@ -114,6 +114,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-tag.1 b/mover-restic/restic/doc/man/restic-tag.1 index 162d50d29..b1468c74d 100644 --- a/mover-restic/restic/doc/man/restic-tag.1 +++ b/mover-restic/restic/doc/man/restic-tag.1 @@ -99,6 +99,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-unlock.1 b/mover-restic/restic/doc/man/restic-unlock.1 index 0274c56e8..0b3b43f2a 100644 --- a/mover-restic/restic/doc/man/restic-unlock.1 +++ b/mover-restic/restic/doc/man/restic-unlock.1 @@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic-version.1 b/mover-restic/restic/doc/man/restic-version.1 index 774e19453..ccc23038f 100644 --- a/mover-restic/restic/doc/man/restic-version.1 +++ b/mover-restic/restic/doc/man/restic-version.1 @@ -69,6 +69,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/doc/man/restic.1 b/mover-restic/restic/doc/man/restic.1 index 427ce7c65..333eab76a 100644 --- a/mover-restic/restic/doc/man/restic.1 +++ b/mover-restic/restic/doc/man/restic.1 @@ -65,6 +65,10 @@ The full documentation can be found at https://restic.readthedocs.io/ . \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/mover-restic/restic/go.mod b/mover-restic/restic/go.mod index 5b9645561..90c00e1a9 100644 --- a/mover-restic/restic/go.mod +++ b/mover-restic/restic/go.mod @@ -35,6 +35,8 @@ require ( google.golang.org/api v0.149.0 ) +replace github.com/klauspost/compress => github.com/klauspost/compress v1.17.2 + require ( cloud.google.com/go v0.110.9 // indirect cloud.google.com/go/compute v1.23.1 // indirect diff --git a/mover-restic/restic/go.sum b/mover-restic/restic/go.sum index 1080ff1f0..1597aef2e 100644 --- a/mover-restic/restic/go.sum +++ b/mover-restic/restic/go.sum @@ -108,8 +108,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= diff --git a/mover-restic/restic/internal/archiver/archiver_test.go b/mover-restic/restic/internal/archiver/archiver_test.go index 3c87055d8..5a9896a48 100644 --- a/mover-restic/restic/internal/archiver/archiver_test.go +++ b/mover-restic/restic/internal/archiver/archiver_test.go @@ -1879,7 +1879,7 @@ func TestArchiverContextCanceled(t *testing.T) { }) // Ensure that the archiver itself reports the canceled context and not just the backend - repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0) + repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{}) back := restictest.Chdir(t, tempdir) defer back() diff --git a/mover-restic/restic/internal/migrations/upgrade_repo_v2_test.go b/mover-restic/restic/internal/migrations/upgrade_repo_v2_test.go index 96fc7788e..7f251de93 100644 --- a/mover-restic/restic/internal/migrations/upgrade_repo_v2_test.go +++ b/mover-restic/restic/internal/migrations/upgrade_repo_v2_test.go @@ -69,7 +69,7 @@ func TestUpgradeRepoV2Failure(t *testing.T) { Backend: be, } - repo := repository.TestRepositoryWithBackend(t, be, 1) + repo := repository.TestRepositoryWithBackend(t, be, 1, repository.Options{}) if repo.Config().Version != 1 { t.Fatal("test repo has wrong version") } diff --git a/mover-restic/restic/internal/pack/pack.go b/mover-restic/restic/internal/pack/pack.go index 34ad9d071..f9e7896e0 100644 --- a/mover-restic/restic/internal/pack/pack.go +++ b/mover-restic/restic/internal/pack/pack.go @@ -1,6 +1,7 @@ package pack import ( + "bytes" "context" "encoding/binary" "fmt" @@ -74,7 +75,7 @@ func (p *Packer) Finalize() error { p.m.Lock() defer p.m.Unlock() - header, err := p.makeHeader() + header, err := makeHeader(p.blobs) if err != nil { return err } @@ -83,6 +84,12 @@ func (p *Packer) Finalize() error { nonce := crypto.NewRandomNonce() encryptedHeader = append(encryptedHeader, nonce...) encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil) + encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) + + if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil { + //nolint:revive // ignore linter warnings about error message spelling + return fmt.Errorf("Detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", err) + } // append the header n, err := p.wr.Write(encryptedHeader) @@ -90,18 +97,33 @@ func (p *Packer) Finalize() error { return errors.Wrap(err, "Write") } - hdrBytes := len(encryptedHeader) - if n != hdrBytes { + if n != len(encryptedHeader) { return errors.New("wrong number of bytes written") } + p.bytes += uint(len(encryptedHeader)) + + return nil +} + +func verifyHeader(k *crypto.Key, header []byte, expected []restic.Blob) error { + // do not offer a way to skip the pack header verification, as pack headers are usually small enough + // to not result in a significant performance impact - // write length - err = binary.Write(p.wr, binary.LittleEndian, uint32(hdrBytes)) + decoded, hdrSize, err := List(k, bytes.NewReader(header), int64(len(header))) if err != nil { - return errors.Wrap(err, "binary.Write") + return fmt.Errorf("header decoding failed: %w", err) + } + if hdrSize != uint32(len(header)) { + return fmt.Errorf("unexpected header size %v instead of %v", hdrSize, len(header)) + } + if len(decoded) != len(expected) { + return fmt.Errorf("pack header size mismatch") + } + for i := 0; i < len(decoded); i++ { + if decoded[i] != expected[i] { + return fmt.Errorf("pack header entry mismatch got %v instead of %v", decoded[i], expected[i]) + } } - p.bytes += uint(hdrBytes + binary.Size(uint32(0))) - return nil } @@ -111,10 +133,10 @@ func (p *Packer) HeaderOverhead() int { } // makeHeader constructs the header for p. -func (p *Packer) makeHeader() ([]byte, error) { - buf := make([]byte, 0, len(p.blobs)*int(entrySize)) +func makeHeader(blobs []restic.Blob) ([]byte, error) { + buf := make([]byte, 0, len(blobs)*int(entrySize)) - for _, b := range p.blobs { + for _, b := range blobs { switch { case b.Type == restic.DataBlob && b.UncompressedLength == 0: buf = append(buf, 0) diff --git a/mover-restic/restic/internal/pack/pack_internal_test.go b/mover-restic/restic/internal/pack/pack_internal_test.go index c1a4867ea..2e7400ad0 100644 --- a/mover-restic/restic/internal/pack/pack_internal_test.go +++ b/mover-restic/restic/internal/pack/pack_internal_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "io" + "strings" "testing" "github.com/restic/restic/internal/crypto" @@ -177,3 +178,60 @@ func TestReadRecords(t *testing.T) { } } } + +func TestUnpackedVerification(t *testing.T) { + // create random keys + k := crypto.NewRandomKey() + blobs := []restic.Blob{ + { + BlobHandle: restic.NewRandomBlobHandle(), + Length: 42, + Offset: 0, + UncompressedLength: 2 * 42, + }, + } + + type DamageType string + const ( + damageData DamageType = "data" + damageCiphertext DamageType = "ciphertext" + damageLength DamageType = "length" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "pack header entry mismatch"}, + {damageCiphertext, "ciphertext verification failed"}, + {damageLength, "header decoding failed"}, + } { + header, err := makeHeader(blobs) + rtest.OK(t, err) + + if test.damage == damageData { + header[8] ^= 0x42 + } + + encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header))) + nonce := crypto.NewRandomNonce() + encryptedHeader = append(encryptedHeader, nonce...) + encryptedHeader = k.Seal(encryptedHeader, nonce, header, nil) + encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) + + if test.damage == damageCiphertext { + encryptedHeader[8] ^= 0x42 + } + if test.damage == damageLength { + encryptedHeader[len(encryptedHeader)-1] ^= 0x42 + } + + err = verifyHeader(k, encryptedHeader, blobs) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} diff --git a/mover-restic/restic/internal/repository/fuzz_test.go b/mover-restic/restic/internal/repository/fuzz_test.go index b4036288c..80372f8e0 100644 --- a/mover-restic/restic/internal/repository/fuzz_test.go +++ b/mover-restic/restic/internal/repository/fuzz_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" ) @@ -19,7 +18,7 @@ func FuzzSaveLoadBlob(f *testing.F) { } id := restic.Hash(blob) - repo := TestRepositoryWithBackend(t, mem.New(), 2) + repo := TestRepositoryWithVersion(t, 2) var wg errgroup.Group repo.StartPackUploader(context.TODO(), &wg) diff --git a/mover-restic/restic/internal/repository/repack_test.go b/mover-restic/restic/internal/repository/repack_test.go index c8570a9d4..1ecbf3e1c 100644 --- a/mover-restic/restic/internal/repository/repack_test.go +++ b/mover-restic/restic/internal/repository/repack_test.go @@ -346,7 +346,8 @@ func TestRepackWrongBlob(t *testing.T) { } func testRepackWrongBlob(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) @@ -371,7 +372,8 @@ func TestRepackBlobFallback(t *testing.T) { } func testRepackBlobFallback(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/mover-restic/restic/internal/repository/repository.go b/mover-restic/restic/internal/repository/repository.go index 0b50382b8..40794508f 100644 --- a/mover-restic/restic/internal/repository/repository.go +++ b/mover-restic/restic/internal/repository/repository.go @@ -59,8 +59,9 @@ type Repository struct { } type Options struct { - Compression CompressionMode - PackSize uint + Compression CompressionMode + PackSize uint + NoExtraVerify bool } // CompressionMode configures if data should be compressed. @@ -423,6 +424,11 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data // encrypt blob ciphertext = r.key.Seal(ciphertext, nonce, data, nil) + if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { + //nolint:revive // ignore linter warnings about error message spelling + return 0, fmt.Errorf("Detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", id, err) + } + // find suitable packer and add blob var pm *packerManager @@ -438,6 +444,31 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data return pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength) } +func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { + if r.opts.NoExtraVerify { + return nil + } + + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] + plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + if uncompressedLength != 0 { + // DecodeAll will allocate a slice if it is not large enough since it + // knows the decompressed size (because we're using EncodeAll) + plaintext, err = r.getZstdDecoder().DecodeAll(plaintext, nil) + if err != nil { + return fmt.Errorf("decompression failed: %w", err) + } + } + if !restic.Hash(plaintext).Equal(id) { + return errors.New("hash mismatch") + } + + return nil +} + func (r *Repository) compressUnpacked(p []byte) ([]byte, error) { // compression is only available starting from version 2 if r.cfg.Version < 2 { @@ -474,7 +505,8 @@ func (r *Repository) decompressUnpacked(p []byte) ([]byte, error) { // SaveUnpacked encrypts data and stores it in the backend. Returned is the // storage hash. -func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) { +func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []byte) (id restic.ID, err error) { + p := buf if t != restic.ConfigFile { p, err = r.compressUnpacked(p) if err != nil { @@ -489,6 +521,11 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by ciphertext = r.key.Seal(ciphertext, nonce, p, nil) + if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { + //nolint:revive // ignore linter warnings about error message spelling + return restic.ID{}, fmt.Errorf("Detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", t, err) + } + if t == restic.ConfigFile { id = restic.ID{} } else { @@ -506,6 +543,29 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by return id, nil } +func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { + if r.opts.NoExtraVerify { + return nil + } + + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] + plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + if t != restic.ConfigFile { + plaintext, err = r.decompressUnpacked(plaintext) + if err != nil { + return fmt.Errorf("decompression failed: %w", err) + } + } + + if !bytes.Equal(plaintext, expected) { + return errors.New("data mismatch") + } + return nil +} + // Flush saves all remaining packs and the index func (r *Repository) Flush(ctx context.Context) error { if err := r.flushPacks(ctx); err != nil { diff --git a/mover-restic/restic/internal/repository/repository_internal_test.go b/mover-restic/restic/internal/repository/repository_internal_test.go index e5ab6e5b7..2a9976ace 100644 --- a/mover-restic/restic/internal/repository/repository_internal_test.go +++ b/mover-restic/restic/internal/repository/repository_internal_test.go @@ -3,8 +3,10 @@ package repository import ( "math/rand" "sort" + "strings" "testing" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -72,3 +74,101 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { sortCachedPacksFirst(cache, cpy[:]) } } + +func TestBlobVerification(t *testing.T) { + repo := TestRepository(t).(*Repository) + + type DamageType string + const ( + damageData DamageType = "data" + damageCompressed DamageType = "compressed" + damageCiphertext DamageType = "ciphertext" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "hash mismatch"}, + {damageCompressed, "decompression failed"}, + {damageCiphertext, "ciphertext verification failed"}, + } { + plaintext := rtest.Random(800, 1234) + id := restic.Hash(plaintext) + if test.damage == damageData { + plaintext[42] ^= 0x42 + } + + uncompressedLength := uint(len(plaintext)) + plaintext = repo.getZstdEncoder().EncodeAll(plaintext, nil) + + if test.damage == damageCompressed { + plaintext = plaintext[:len(plaintext)-8] + } + + nonce := crypto.NewRandomNonce() + ciphertext := append([]byte{}, nonce...) + ciphertext = repo.Key().Seal(ciphertext, nonce, plaintext, nil) + + if test.damage == damageCiphertext { + ciphertext[42] ^= 0x42 + } + + err := repo.verifyCiphertext(ciphertext, int(uncompressedLength), id) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} + +func TestUnpackedVerification(t *testing.T) { + repo := TestRepository(t).(*Repository) + + type DamageType string + const ( + damageData DamageType = "data" + damageCompressed DamageType = "compressed" + damageCiphertext DamageType = "ciphertext" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "data mismatch"}, + {damageCompressed, "decompression failed"}, + {damageCiphertext, "ciphertext verification failed"}, + } { + plaintext := rtest.Random(800, 1234) + orig := append([]byte{}, plaintext...) + if test.damage == damageData { + plaintext[42] ^= 0x42 + } + + compressed := []byte{2} + compressed = repo.getZstdEncoder().EncodeAll(plaintext, compressed) + + if test.damage == damageCompressed { + compressed = compressed[:len(compressed)-8] + } + + nonce := crypto.NewRandomNonce() + ciphertext := append([]byte{}, nonce...) + ciphertext = repo.Key().Seal(ciphertext, nonce, compressed, nil) + + if test.damage == damageCiphertext { + ciphertext[42] ^= 0x42 + } + + err := repo.verifyUnpacked(ciphertext, restic.IndexFile, orig) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} diff --git a/mover-restic/restic/internal/repository/testing.go b/mover-restic/restic/internal/repository/testing.go index 4936cc368..9bdd65901 100644 --- a/mover-restic/restic/internal/repository/testing.go +++ b/mover-restic/restic/internal/repository/testing.go @@ -43,7 +43,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173) // TestRepositoryWithBackend returns a repository initialized with a test // password. If be is nil, an in-memory backend is used. A constant polynomial // is used for the chunker and low-security test parameters. -func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository { +func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint, opts Options) restic.Repository { t.Helper() TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) @@ -52,7 +52,7 @@ func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) re be = TestBackend(t) } - repo, err := New(be, Options{}) + repo, err := New(be, opts) if err != nil { t.Fatalf("TestRepository(): new repo failed: %v", err) } @@ -78,6 +78,7 @@ func TestRepository(t testing.TB) restic.Repository { func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { t.Helper() dir := os.Getenv("RESTIC_TEST_REPO") + opts := Options{} if dir != "" { _, err := os.Stat(dir) if err != nil { @@ -85,7 +86,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { if err != nil { t.Fatalf("error creating local backend at %v: %v", dir, err) } - return TestRepositoryWithBackend(t, be, version) + return TestRepositoryWithBackend(t, be, version, opts) } if err == nil { @@ -93,7 +94,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { } } - return TestRepositoryWithBackend(t, nil, version) + return TestRepositoryWithBackend(t, nil, version, opts) } // TestOpenLocal opens a local repository. diff --git a/mover-restic/restic/internal/restic/lock_test.go b/mover-restic/restic/internal/restic/lock_test.go index f3c405c9c..2eb22be1b 100644 --- a/mover-restic/restic/internal/restic/lock_test.go +++ b/mover-restic/restic/internal/restic/lock_test.go @@ -65,7 +65,7 @@ func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, len func TestMultipleLockFailure(t *testing.T) { be := &failLockLoadingBackend{Backend: mem.New()} - repo := repository.TestRepositoryWithBackend(t, be, 0) + repo := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{}) restic.TestSetLockTimeout(t, 5*time.Millisecond) lock1, err := restic.NewLock(context.TODO(), repo)