From e8f5539bb146ab788f7fc310b7058ce964e57274 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Sun, 10 Sep 2023 22:26:06 +0000 Subject: [PATCH] Bug 1852172 - Update base64 to 0.21.3. r=emilio,supply-chain-reviewers Differential Revision: https://phabricator.services.mozilla.com/D187731 --- Cargo.lock | 22 +- supply-chain/imports.lock | 15 + third_party/rust/base64/.cargo-checksum.json | 2 +- third_party/rust/base64/Cargo.lock | 256 +++++++----- third_party/rust/base64/Cargo.toml | 14 +- third_party/rust/base64/README.md | 10 +- third_party/rust/base64/RELEASE-NOTES.md | 24 +- third_party/rust/base64/clippy.toml | 2 +- third_party/rust/base64/src/alphabet.rs | 37 +- .../rust/base64/src/chunked_encoder.rs | 95 +---- third_party/rust/base64/src/decode.rs | 13 +- third_party/rust/base64/src/encode.rs | 34 +- .../src/engine/general_purpose/decode.rs | 59 ++- .../engine/general_purpose/decode_suffix.rs | 17 +- .../base64/src/engine/general_purpose/mod.rs | 5 +- third_party/rust/base64/src/engine/mod.rs | 219 ++++++---- third_party/rust/base64/src/engine/naive.rs | 4 +- third_party/rust/base64/src/engine/tests.rs | 393 +++++++++++++++--- third_party/rust/base64/src/read/decoder.rs | 65 ++- .../rust/base64/src/read/decoder_tests.rs | 157 ++++++- .../base64/src/write/encoder_string_writer.rs | 45 +- .../rust/base64/src/write/encoder_tests.rs | 2 +- third_party/rust/base64/tests/encode.rs | 22 +- 23 files changed, 1053 insertions(+), 459 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82e42632c5f8..da919e9281db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -300,7 +300,7 @@ version = "0.4.0-alpha.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6552b35e6a39b2e059b8107e45abf2adfa7cd3d20d294f5a315996da38559cbe" dependencies = [ - "base64 0.21.0", + "base64 0.21.3", "bitflags 1.3.2", "cfg-if 1.0.0", "core-foundation", @@ -326,7 +326,7 @@ name = "authrs_bridge" version = "0.1.0" dependencies = [ "authenticator", - "base64 0.21.0", + "base64 0.21.3", "log", "moz_task", "nserror", @@ -346,14 +346,14 @@ version = "1.1.0" name = "base64" version = "0.13.999" dependencies = [ - "base64 0.21.0", + "base64 0.21.3", ] [[package]] name = "base64" -version = "0.21.0" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "basic-toml" @@ -645,7 +645,7 @@ checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" name = "cert_storage" version = "0.0.1" dependencies = [ - "base64 0.21.0", + "base64 0.21.3", "byteorder", "crossbeam-utils", "cstr", @@ -1012,7 +1012,7 @@ dependencies = [ name = "crypto_hash" version = "0.1.0" dependencies = [ - "base64 0.21.0", + "base64 0.21.3", "digest", "libc", "md-5", @@ -2005,7 +2005,7 @@ dependencies = [ name = "geckodriver" version = "0.33.0" dependencies = [ - "base64 0.21.0", + "base64 0.21.3", "chrono", "clap", "hyper", @@ -2525,7 +2525,7 @@ dependencies = [ name = "http3server" version = "0.1.1" dependencies = [ - "base64 0.21.0", + "base64 0.21.3", "bindgen 0.66.1", "cfg-if 1.0.0", "http", @@ -4335,7 +4335,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3675d093a7713f2b861f77b16c3c33fadd6de0a69bf7203014d938b9d5daa6f7" dependencies = [ - "base64 0.21.0", + "base64 0.21.3", "byteorder", "getrandom", "serde", @@ -4642,7 +4642,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ - "base64 0.21.0", + "base64 0.21.3", "bitflags 2.999.999", "serde", "serde_derive", diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 096b34b2c05e..5bff65f7ca00 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -1189,6 +1189,21 @@ user-id = 101233 # le-automaton start = "2020-09-28" end = "2024-03-23" +[[audits.isrg.audits.base64]] +who = "Tim Geoghegan " +criteria = "safe-to-deploy" +delta = "0.21.0 -> 0.21.1" + +[[audits.isrg.audits.base64]] +who = "Brandon Pitman " +criteria = "safe-to-deploy" +delta = "0.21.1 -> 0.21.2" + +[[audits.isrg.audits.base64]] +who = "David Cook " +criteria = "safe-to-deploy" +delta = "0.21.2 -> 0.21.3" + [[audits.isrg.audits.block-buffer]] who = "David Cook " criteria = "safe-to-deploy" diff --git a/third_party/rust/base64/.cargo-checksum.json b/third_party/rust/base64/.cargo-checksum.json index ccb0bff19c2f..22dff2d17809 100644 --- a/third_party/rust/base64/.cargo-checksum.json +++ b/third_party/rust/base64/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.lock":"ee9a902629f1a6cc9308158590fc298c628f323383c4fb9a5ab9fd51011b268e","Cargo.toml":"37ffe4d4bdbd21f5a1cc78596abf6e704e4131dbec830fcd6c8bec33d4caf76b","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"0dd882e53de11566d50f8e8e2d5a651bcf3fabee4987d70f306233cf39094ba7","README.md":"99e61de0bafd6985761b596f9aa970dee5b4d0cfbfc05cd6565b5ffa139cb34f","RELEASE-NOTES.md":"c8b9e21adecb3a89928cdfbe55d184cc234e0e6cf8f61fb8ebab48982b3a6f9c","benches/benchmarks.rs":"faf63c3d83fe1568927288cfcc7f9bd4bd15c6b531450c53d2c064386fc5c652","clippy.toml":"ee3dedc35eb156cbfbe836692846cd492f59acc7443dd43cc6d71f087d80051c","examples/base64.rs":"8c48673029aeeb1e06a2ecfd237acf8ef24349990e97f6d2c4d0fa2af36c94b3","icon_CLion.svg":"cffa044ba75cb998ee3306991dc4a3755ec2f39ab95ddd4b74bc21988389020f","src/alphabet.rs":"420b5e23da0702c401489c53721696c5d5f69631f4ca9462f4c5ef3bdc77114e","src/chunked_encoder.rs":"4dfad5b47da1c35deaaa6ed2bb1efa51d98d6d9a7ca85a37ef4a02dfa846e723","src/decode.rs":"c293bf40a821795909a458aa8d7e76005a46e6953eed7ea284da1334e117ae74","src/display.rs":"31bf3e19274a0b80dd8948a81ea535944f756ef5b88736124c940f5fe1e8c71c","src/encode.rs":"34c800de1576f425cc48fa7ed9486d7c925cf7215dfc0634d2349c97b5199595","src/engine/general_purpose/decode.rs":"be237ac84b6a1deafd335896959302b7cf9337868cd718ebf7d249ccdc43a84f","src/engine/general_purpose/decode_suffix.rs":"797729cc1d56e9786f65e765cc5bb9ab2799f9140db4b412b919512fd7542355","src/engine/general_purpose/mod.rs":"2c6fbe61fae32800d30be5dc5bde429b8a07a5f027d0d2d1227a24ed13b1b461","src/engine/mod.rs":"7cd78bb317074a6e5439e272e4943d59d6bd47b149ed76b52e6f75b45909ce52","src/engine/naive.rs":"dcebcc41a0f4a156dd516ae89824748f5a4eedeabfe8d92f6f5bd3a6d5ceb5fb","src/engine/tests.rs":"4a8ff2ab7700b49e5b33606a93af04a5f93b18ca48e760ab6ced6337ba3a4847","src/lib.rs":"b4699408a9356f88fd8a3aeffae97e54e7a249afe5d919ecf9d4092d1c8efde1","src/prelude.rs":"f82fcf5e31921060929f9e10efb2868ba7339b085ee76fc5e7077f6030fbb2cc","src/read/decoder.rs":"65f03af1f4eb8d466a9a800dc6414678195b4ac6c579cd747b5632eee219f5a4","src/read/decoder_tests.rs":"ebf40a5722a58dbe74f013a4163ab20f5ce42ceb4beaefede07562079d596604","src/read/mod.rs":"e0b714eda02d16b1ffa6f78fd09b2f963e01c881b1f7c17b39db4e904be5e746","src/tests.rs":"90cb9f8a1ccb7c4ddc4f8618208e0031fc97e0df0e5aa466d6a5cf45d25967d8","src/write/encoder.rs":"c889c853249220fe2ddaeb77ee6e2ee2945f7db88cd6658ef89ff71b81255ea8","src/write/encoder_string_writer.rs":"ac3702b1a846fd0664e78d2dd82c939073ca00577d3201a8f1fbe17a9bf85e70","src/write/encoder_tests.rs":"39572f11fdf63af47f13bb58be280221322c669504a1b4a30a9181fe465e0f90","src/write/mod.rs":"73cd98dadc9d712b3fefd9449d97e825e097397441b90588e0051e4d3b0911b9","tests/encode.rs":"072f079f0250d542ff964e8e72b7d13e2336fbee7395367ff737e140c38ac459","tests/tests.rs":"78efcf0dc4bb6ae52f7a91fcad89e44e4dce578224c36b4e6c1c306459be8500"},"package":"a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"} \ No newline at end of file +{"files":{"Cargo.lock":"8b23494504aad8968c998b54d9201489ac9e8d3fa792e7c86fc3c0d9419d5722","Cargo.toml":"d7c38395300d1d122c8cfa70f902469edac765aebbc93d39356c2c634d0dd182","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"0dd882e53de11566d50f8e8e2d5a651bcf3fabee4987d70f306233cf39094ba7","README.md":"df01f5b4317d601e7de86743f9818aec9196abf9e298f5e47679b7a966ecd945","RELEASE-NOTES.md":"8dd8a47135cb7660a4dd26fa13be26e97d1b4989ebcd2e0f3e6c35baec0a3821","benches/benchmarks.rs":"faf63c3d83fe1568927288cfcc7f9bd4bd15c6b531450c53d2c064386fc5c652","clippy.toml":"b26be4d15ed059985ce6994f11817fd7562046f46e460a0dc64dbb71cfc246d1","examples/base64.rs":"8c48673029aeeb1e06a2ecfd237acf8ef24349990e97f6d2c4d0fa2af36c94b3","icon_CLion.svg":"cffa044ba75cb998ee3306991dc4a3755ec2f39ab95ddd4b74bc21988389020f","src/alphabet.rs":"f0cba9462692db0bc9572e3d03c01ac77ff705fa9e664db4162da1a279a871e1","src/chunked_encoder.rs":"35b435042074f2ddd7dac6dae89c2ab6e5dfe173c73ccd1504c4f916f49151ef","src/decode.rs":"a43c5dc72524d49803f4170a84a975aa78040f457b4432ed22f04ab79bc1f3f4","src/display.rs":"31bf3e19274a0b80dd8948a81ea535944f756ef5b88736124c940f5fe1e8c71c","src/encode.rs":"8893d154878f894b3a8930e817ac8f0e4e8f67736f75200aa5eeee2ee813f626","src/engine/general_purpose/decode.rs":"ba8a76d333ab96dd07b3f84bd6d405d690d2d17e84bd0878f05245a82dc16853","src/engine/general_purpose/decode_suffix.rs":"71ceb066b73e8cc833916e2cedbf0a01b07c2f16e30b2b2f63aff1c823874b51","src/engine/general_purpose/mod.rs":"4acf7293e5bb83faf01edf2618bcb2b2aff2c2a3bcb85ddc815baa96e5751bb2","src/engine/mod.rs":"f1a307dadcab561719c868595cae5be0e37cdba337c2f8959d6ed1367030ed75","src/engine/naive.rs":"4ebd14e28502700d5de3e2aa193b6b384ad1189c6aa9368be3ab35775777aa4a","src/engine/tests.rs":"37bee2de07343bf5d37720f29cda291e8562f2363704e0ad91862d5991568d22","src/lib.rs":"b4699408a9356f88fd8a3aeffae97e54e7a249afe5d919ecf9d4092d1c8efde1","src/prelude.rs":"f82fcf5e31921060929f9e10efb2868ba7339b085ee76fc5e7077f6030fbb2cc","src/read/decoder.rs":"cc87daa4c52a23d1275352bccf07468baf2b60e90b2ac14f89a94254697cb83c","src/read/decoder_tests.rs":"cc3c2273867972a835f459073e3982f20a690c3b7d5f7546042e417d11c97a94","src/read/mod.rs":"e0b714eda02d16b1ffa6f78fd09b2f963e01c881b1f7c17b39db4e904be5e746","src/tests.rs":"90cb9f8a1ccb7c4ddc4f8618208e0031fc97e0df0e5aa466d6a5cf45d25967d8","src/write/encoder.rs":"c889c853249220fe2ddaeb77ee6e2ee2945f7db88cd6658ef89ff71b81255ea8","src/write/encoder_string_writer.rs":"0326c9d120369b9bbc35697b5b9b141bed24283374c93d5af1052eb042e47799","src/write/encoder_tests.rs":"28695a485b17cf5db73656aae5d90127f726e02c6d70efd83e5ab53a4cc17b38","src/write/mod.rs":"73cd98dadc9d712b3fefd9449d97e825e097397441b90588e0051e4d3b0911b9","tests/encode.rs":"ca8fb0c03f71563788cced06deb335f2c4bace0c875696662340c86ccedfdc7f","tests/tests.rs":"78efcf0dc4bb6ae52f7a91fcad89e44e4dce578224c36b4e6c1c306459be8500"},"package":"414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53"} \ No newline at end of file diff --git a/third_party/rust/base64/Cargo.lock b/third_party/rust/base64/Cargo.lock index 739c8c4147af..990f04fc0811 100644 --- a/third_party/rust/base64/Cargo.lock +++ b/third_party/rust/base64/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +dependencies = [ + "memchr", +] + [[package]] name = "anes" version = "0.1.6" @@ -36,9 +45,10 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.3" dependencies = [ "criterion", + "lazy_static", "rand", "rstest", "rstest_reuse", @@ -53,9 +63,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "cast" @@ -71,9 +81,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "ciborium" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" dependencies = [ "ciborium-io", "ciborium-ll", @@ -82,15 +92,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" [[package]] name = "ciborium-ll" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" dependencies = [ "ciborium-io", "half", @@ -113,9 +123,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.23" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "bitflags", "clap_lex", @@ -142,7 +152,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap 3.2.23", + "clap 3.2.25", "criterion-plot", "itertools", "lazy_static", @@ -170,9 +180,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -180,9 +190,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -191,9 +201,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if", @@ -204,24 +214,24 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] [[package]] name = "either" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", @@ -260,18 +270,15 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "indexmap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown", @@ -288,15 +295,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -309,52 +316,55 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.2", "libc", ] [[package]] name = "once_cell" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "oorandom" @@ -364,15 +374,15 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "os_str_bytes" -version = "6.4.1" +version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" +checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" [[package]] name = "plotters" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" dependencies = [ "num-traits", "plotters-backend", @@ -383,15 +393,15 @@ dependencies = [ [[package]] name = "plotters-backend" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" [[package]] name = "plotters-svg" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" dependencies = [ "plotters-backend", ] @@ -411,7 +421,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -428,18 +438,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.23" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -476,9 +486,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -486,9 +496,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -498,18 +508,32 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", ] [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "rstest" @@ -521,7 +545,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 1.0.109", ] [[package]] @@ -532,7 +556,7 @@ checksum = "b29d3117bce27ea307d1fb7ce12c64ba11b3fd04311a42d32bc5f0072e6e3d4d" dependencies = [ "quote", "rustc_version", - "syn", + "syn 1.0.109", ] [[package]] @@ -546,9 +570,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -561,41 +585,41 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.16" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.29", ] [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" dependencies = [ "itoa", "ryu", @@ -629,14 +653,25 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" dependencies = [ "proc-macro2", "quote", @@ -670,15 +705,15 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-segmentation" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" @@ -700,12 +735,11 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -717,9 +751,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -727,24 +761,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.29", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -752,28 +786,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.29", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/third_party/rust/base64/Cargo.toml b/third_party/rust/base64/Cargo.toml index 83f3da1e5d33..c551663d43e9 100644 --- a/third_party/rust/base64/Cargo.toml +++ b/third_party/rust/base64/Cargo.toml @@ -10,10 +10,10 @@ # See Cargo.toml.orig for the original contents. [package] -edition = "2021" -rust-version = "1.57.0" +edition = "2018" +rust-version = "1.48.0" name = "base64" -version = "0.21.0" +version = "0.21.3" authors = [ "Alice Maz ", "Marshall Pierce ", @@ -32,8 +32,11 @@ categories = ["encoding"] license = "MIT OR Apache-2.0" repository = "https://github.com/marshallpierce/rust-base64" +[package.metadata.docs.rs] +rustdoc-args = ["--generate-link-to-definition"] + [profile.bench] -debug = true +debug = 2 [profile.test] opt-level = 3 @@ -45,6 +48,9 @@ harness = false [dev-dependencies.criterion] version = "0.4.0" +[dev-dependencies.lazy_static] +version = "1.4.0" + [dev-dependencies.rand] version = "0.8.5" features = ["small_rng"] diff --git a/third_party/rust/base64/README.md b/third_party/rust/base64/README.md index d7b0885534a2..f566756d51fa 100644 --- a/third_party/rust/base64/README.md +++ b/third_party/rust/base64/README.md @@ -63,7 +63,7 @@ optionally may allow other behaviors. ## Rust version compatibility -The minimum supported Rust version is 1.57.0. +The minimum supported Rust version is 1.48.0. # Contributing @@ -76,10 +76,10 @@ free time to give each PR the attention it deserves. I will get to everyone even ## Developing -Benchmarks are in `benches/`. Running them requires nightly rust, but `rustup` makes it easy: +Benchmarks are in `benches/`. ```bash -rustup run nightly cargo bench +cargo bench ``` ## no_std @@ -92,12 +92,12 @@ to bring back the support for heap allocations. ## Profiling On Linux, you can use [perf](https://perf.wiki.kernel.org/index.php/Main_Page) for profiling. Then compile the -benchmarks with `rustup nightly run cargo bench --no-run`. +benchmarks with `cargo bench --no-run`. Run the benchmark binary with `perf` (shown here filtering to one particular benchmark, which will make the results easier to read). `perf` is only available to the root user on most systems as it fiddles with event counters in your CPU, so use `sudo`. We need to run the actual benchmark binary, hence the path into `target`. You can see the actual -full path with `rustup run nightly cargo bench -v`; it will print out the commands it runs. If you use the exact path +full path with `cargo bench -v`; it will print out the commands it runs. If you use the exact path that `bench` outputs, make sure you get the one that's for the benchmarks, not the tests. You may also want to `cargo clean` so you have only one `benchmarks-` binary (they tend to accumulate). diff --git a/third_party/rust/base64/RELEASE-NOTES.md b/third_party/rust/base64/RELEASE-NOTES.md index 4fcaddab3615..6fa3e90acd40 100644 --- a/third_party/rust/base64/RELEASE-NOTES.md +++ b/third_party/rust/base64/RELEASE-NOTES.md @@ -1,8 +1,26 @@ +# 0.21.3 + +- Implement `source` instead of `cause` on Error types +- Roll back MSRV to 1.48.0 so Debian can continue to live in a time warp +- Slightly faster chunked encoding for short inputs +- Decrease binary size + +# 0.21.2 + +- Rollback MSRV to 1.57.0 -- only dev dependencies need 1.60, not the main code + +# 0.21.1 + +- Remove the possibility of panicking during decoded length calculations +- `DecoderReader` no longer sometimes erroneously ignores padding [#226](https://github.com/marshallpierce/rust-base64/issues/226) + +## Breaking changes + +- `Engine.internal_decode` return type changed +- Update MSRV to 1.60.0 + # 0.21.0 -(not yet released) - - ## Migration ### Functions diff --git a/third_party/rust/base64/clippy.toml b/third_party/rust/base64/clippy.toml index 23b32c17c3ce..11d46a73f332 100644 --- a/third_party/rust/base64/clippy.toml +++ b/third_party/rust/base64/clippy.toml @@ -1 +1 @@ -msrv = "1.57.0" +msrv = "1.48.0" diff --git a/third_party/rust/base64/src/alphabet.rs b/third_party/rust/base64/src/alphabet.rs index 7cd1b57073c2..f7cd81911346 100644 --- a/third_party/rust/base64/src/alphabet.rs +++ b/third_party/rust/base64/src/alphabet.rs @@ -1,7 +1,7 @@ //! Provides [Alphabet] and constants for alphabets commonly used in the wild. use crate::PAD_BYTE; -use core::fmt; +use core::{convert, fmt}; #[cfg(any(feature = "std", test))] use std::error; @@ -12,6 +12,10 @@ const ALPHABET_SIZE: usize = 64; /// Common alphabets are provided as constants, and custom alphabets /// can be made via `from_str` or the `TryFrom` implementation. /// +/// # Examples +/// +/// Building and using a custom Alphabet: +/// /// ``` /// let custom = base64::alphabet::Alphabet::new("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/").unwrap(); /// @@ -19,6 +23,33 @@ const ALPHABET_SIZE: usize = 64; /// &custom, /// base64::engine::general_purpose::PAD); /// ``` +/// +/// Building a const: +/// +/// ``` +/// use base64::alphabet::Alphabet; +/// +/// static CUSTOM: Alphabet = { +/// // Result::unwrap() isn't const yet, but panic!() is OK +/// match Alphabet::new("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/") { +/// Ok(x) => x, +/// Err(_) => panic!("creation of alphabet failed"), +/// } +/// }; +/// ``` +/// +/// Building a lazy_static: +/// +/// ``` +/// use base64::{ +/// alphabet::Alphabet, +/// engine::{general_purpose::GeneralPurpose, GeneralPurposeConfig}, +/// }; +/// +/// lazy_static::lazy_static! { +/// static ref CUSTOM: Alphabet = Alphabet::new("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/").unwrap(); +/// } +/// ``` #[derive(Clone, Debug, Eq, PartialEq)] pub struct Alphabet { pub(crate) symbols: [u8; ALPHABET_SIZE], @@ -93,7 +124,7 @@ impl Alphabet { } } -impl TryFrom<&str> for Alphabet { +impl convert::TryFrom<&str> for Alphabet { type Error = ParseAlphabetError; fn try_from(value: &str) -> Result { @@ -171,7 +202,7 @@ pub const BIN_HEX: Alphabet = Alphabet::from_str_unchecked( #[cfg(test)] mod tests { use crate::alphabet::*; - use std::convert::TryFrom as _; + use core::convert::TryFrom as _; #[test] fn detects_duplicate_start() { diff --git a/third_party/rust/base64/src/chunked_encoder.rs b/third_party/rust/base64/src/chunked_encoder.rs index 045725974444..69bc7457e1db 100644 --- a/third_party/rust/base64/src/chunked_encoder.rs +++ b/third_party/rust/base64/src/chunked_encoder.rs @@ -1,12 +1,12 @@ +use crate::{ + encode::add_padding, + engine::{Config, Engine}, +}; #[cfg(any(feature = "alloc", feature = "std", test))] use alloc::string::String; -use core::cmp; #[cfg(any(feature = "alloc", feature = "std", test))] use core::str; -use crate::encode::add_padding; -use crate::engine::{Config, Engine}; - /// The output mechanism for ChunkedEncoder's encoded bytes. pub trait Sink { type Error; @@ -15,71 +15,37 @@ pub trait Sink { fn write_encoded_bytes(&mut self, encoded: &[u8]) -> Result<(), Self::Error>; } -const BUF_SIZE: usize = 1024; - /// A base64 encoder that emits encoded bytes in chunks without heap allocation. pub struct ChunkedEncoder<'e, E: Engine + ?Sized> { engine: &'e E, - max_input_chunk_len: usize, } impl<'e, E: Engine + ?Sized> ChunkedEncoder<'e, E> { pub fn new(engine: &'e E) -> ChunkedEncoder<'e, E> { - ChunkedEncoder { - engine, - max_input_chunk_len: max_input_length(BUF_SIZE, engine.config().encode_padding()), - } + ChunkedEncoder { engine } } pub fn encode(&self, bytes: &[u8], sink: &mut S) -> Result<(), S::Error> { - let mut encode_buf: [u8; BUF_SIZE] = [0; BUF_SIZE]; - let mut input_index = 0; + const BUF_SIZE: usize = 1024; + const CHUNK_SIZE: usize = BUF_SIZE / 4 * 3; - while input_index < bytes.len() { - // either the full input chunk size, or it's the last iteration - let input_chunk_len = cmp::min(self.max_input_chunk_len, bytes.len() - input_index); - - let chunk = &bytes[input_index..(input_index + input_chunk_len)]; - - let mut b64_bytes_written = self.engine.internal_encode(chunk, &mut encode_buf); - - input_index += input_chunk_len; - let more_input_left = input_index < bytes.len(); - - if self.engine.config().encode_padding() && !more_input_left { - // no more input, add padding if needed. Buffer will have room because - // max_input_length leaves room for it. - b64_bytes_written += add_padding(bytes.len(), &mut encode_buf[b64_bytes_written..]); + let mut buf = [0; BUF_SIZE]; + for chunk in bytes.chunks(CHUNK_SIZE) { + let mut len = self.engine.internal_encode(chunk, &mut buf); + if chunk.len() != CHUNK_SIZE && self.engine.config().encode_padding() { + // Final, potentially partial, chunk. + // Only need to consider if padding is needed on a partial chunk since full chunk + // is a multiple of 3, which therefore won't be padded. + // Pad output to multiple of four bytes if required by config. + len += add_padding(len, &mut buf[len..]); } - - sink.write_encoded_bytes(&encode_buf[0..b64_bytes_written])?; + sink.write_encoded_bytes(&buf[..len])?; } Ok(()) } } -/// Calculate the longest input that can be encoded for the given output buffer size. -/// -/// If the config requires padding, two bytes of buffer space will be set aside so that the last -/// chunk of input can be encoded safely. -/// -/// The input length will always be a multiple of 3 so that no encoding state has to be carried over -/// between chunks. -fn max_input_length(encoded_buf_len: usize, padded: bool) -> usize { - let effective_buf_len = if padded { - // make room for padding - encoded_buf_len - .checked_sub(2) - .expect("Don't use a tiny buffer") - } else { - encoded_buf_len - }; - - // No padding, so just normal base64 expansion. - (effective_buf_len / 4) * 3 -} - // A really simple sink that just appends to a string #[cfg(any(feature = "alloc", feature = "std", test))] pub(crate) struct StringSink<'a> { @@ -151,38 +117,13 @@ pub mod tests { chunked_encode_matches_normal_encode_random(&helper); } - #[test] - fn max_input_length_no_pad() { - assert_eq!(768, max_input_length(1024, false)); - } - - #[test] - fn max_input_length_with_pad_decrements_one_triple() { - assert_eq!(765, max_input_length(1024, true)); - } - - #[test] - fn max_input_length_with_pad_one_byte_short() { - assert_eq!(765, max_input_length(1025, true)); - } - - #[test] - fn max_input_length_with_pad_fits_exactly() { - assert_eq!(768, max_input_length(1026, true)); - } - - #[test] - fn max_input_length_cant_use_extra_single_encoded_byte() { - assert_eq!(300, max_input_length(401, false)); - } - pub fn chunked_encode_matches_normal_encode_random(sink_test_helper: &S) { let mut input_buf: Vec = Vec::new(); let mut output_buf = String::new(); let mut rng = rand::rngs::SmallRng::from_entropy(); let input_len_range = Uniform::new(1, 10_000); - for _ in 0..5_000 { + for _ in 0..20_000 { input_buf.clear(); output_buf.clear(); diff --git a/third_party/rust/base64/src/decode.rs b/third_party/rust/base64/src/decode.rs index 047151840c83..f590cbdd04d9 100644 --- a/third_party/rust/base64/src/decode.rs +++ b/third_party/rust/base64/src/decode.rs @@ -41,11 +41,7 @@ impl fmt::Display for DecodeError { } #[cfg(any(feature = "std", test))] -impl error::Error for DecodeError { - fn cause(&self) -> Option<&dyn error::Error> { - None - } -} +impl error::Error for DecodeError {} /// Errors that can occur while decoding into a slice. #[derive(Clone, Debug, PartialEq, Eq)] @@ -69,7 +65,7 @@ impl fmt::Display for DecodeSliceError { #[cfg(any(feature = "std", test))] impl error::Error for DecodeSliceError { - fn cause(&self) -> Option<&dyn error::Error> { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { match self { DecodeSliceError::DecodeError(e) => Some(e), DecodeSliceError::OutputSliceTooSmall => None, @@ -148,11 +144,6 @@ pub fn decode_engine_slice>( /// // start of the next quad of encoded symbols /// assert_eq!(6, decoded_len_estimate(5)); /// ``` -/// -/// # Panics -/// -/// Panics if decoded length estimation overflows. -/// This would happen for sizes within a few bytes of the maximum value of `usize`. pub fn decoded_len_estimate(encoded_len: usize) -> usize { STANDARD .internal_decoded_len_estimate(encoded_len) diff --git a/third_party/rust/base64/src/encode.rs b/third_party/rust/base64/src/encode.rs index cb176504a15d..00ade7472060 100644 --- a/third_party/rust/base64/src/encode.rs +++ b/third_party/rust/base64/src/encode.rs @@ -77,7 +77,7 @@ pub(crate) fn encode_with_padding( let b64_bytes_written = engine.internal_encode(input, output); let padding_bytes = if engine.config().encode_padding() { - add_padding(input.len(), &mut output[b64_bytes_written..]) + add_padding(b64_bytes_written, &mut output[b64_bytes_written..]) } else { 0 }; @@ -117,20 +117,20 @@ pub fn encoded_len(bytes_len: usize, padding: bool) -> Option { } /// Write padding characters. -/// `input_len` is the size of the original, not encoded, input. +/// `unpadded_output_len` is the size of the unpadded but base64 encoded data. /// `output` is the slice where padding should be written, of length at least 2. /// /// Returns the number of padding bytes written. -pub(crate) fn add_padding(input_len: usize, output: &mut [u8]) -> usize { - // TODO base on encoded len to use cheaper mod by 4 (aka & 7) - let rem = input_len % 3; - let mut bytes_written = 0; - for _ in 0..((3 - rem) % 3) { - output[bytes_written] = PAD_BYTE; - bytes_written += 1; +pub(crate) fn add_padding(unpadded_output_len: usize, output: &mut [u8]) -> usize { + let pad_bytes = (4 - (unpadded_output_len % 4)) % 4; + // for just a couple bytes, this has better performance than using + // .fill(), or iterating over mutable refs, which call memset() + #[allow(clippy::needless_range_loop)] + for i in 0..pad_bytes { + output[i] = PAD_BYTE; } - bytes_written + pad_bytes } /// Errors that can occur while encoding into a slice. @@ -149,11 +149,7 @@ impl fmt::Display for EncodeSliceError { } #[cfg(any(feature = "std", test))] -impl error::Error for EncodeSliceError { - fn cause(&self) -> Option<&dyn error::Error> { - None - } -} +impl error::Error for EncodeSliceError {} #[cfg(test)] mod tests { @@ -434,18 +430,18 @@ mod tests { let mut rng = rand::rngs::SmallRng::from_entropy(); - // cover our bases for length % 3 - for input_len in 0..10 { + // cover our bases for length % 4 + for unpadded_output_len in 0..20 { output.clear(); // fill output with random - for _ in 0..10 { + for _ in 0..100 { output.push(rng.gen()); } let orig_output_buf = output.clone(); - let bytes_written = add_padding(input_len, &mut output); + let bytes_written = add_padding(unpadded_output_len, &mut output); // make sure the part beyond bytes_written is the same garbage it was before assert_eq!(orig_output_buf[bytes_written..], output[bytes_written..]); diff --git a/third_party/rust/base64/src/engine/general_purpose/decode.rs b/third_party/rust/base64/src/engine/general_purpose/decode.rs index e9fd78877b64..21a386fd6591 100644 --- a/third_party/rust/base64/src/engine/general_purpose/decode.rs +++ b/third_party/rust/base64/src/engine/general_purpose/decode.rs @@ -1,5 +1,5 @@ use crate::{ - engine::{general_purpose::INVALID_VALUE, DecodeEstimate, DecodePaddingMode}, + engine::{general_purpose::INVALID_VALUE, DecodeEstimate, DecodeMetadata, DecodePaddingMode}, DecodeError, PAD_BYTE, }; @@ -30,16 +30,11 @@ pub struct GeneralPurposeEstimate { impl GeneralPurposeEstimate { pub(crate) fn new(encoded_len: usize) -> Self { + // Formulas that won't overflow Self { - num_chunks: encoded_len - .checked_add(INPUT_CHUNK_LEN - 1) - .expect("Overflow when calculating number of chunks in input") - / INPUT_CHUNK_LEN, - decoded_len_estimate: encoded_len - .checked_add(3) - .expect("Overflow when calculating decoded len estimate") - / 4 - * 3, + num_chunks: encoded_len / INPUT_CHUNK_LEN + + (encoded_len % INPUT_CHUNK_LEN > 0) as usize, + decoded_len_estimate: (encoded_len / 4 + (encoded_len % 4 > 0) as usize) * 3, } } } @@ -51,7 +46,7 @@ impl DecodeEstimate for GeneralPurposeEstimate { } /// Helper to avoid duplicating num_chunks calculation, which is costly on short inputs. -/// Returns the number of bytes written, or an error. +/// Returns the decode metadata, or an error. // We're on the fragile edge of compiler heuristics here. If this is not inlined, slow. If this is // inlined(always), a different slow. plain ol' inline makes the benchmarks happiest at the moment, // but this is fragile and the best setting changes with only minor code modifications. @@ -63,7 +58,7 @@ pub(crate) fn decode_helper( decode_table: &[u8; 256], decode_allow_trailing_bits: bool, padding_mode: DecodePaddingMode, -) -> Result { +) -> Result { let remainder_len = input.len() % INPUT_CHUNK_LEN; // Because the fast decode loop writes in groups of 8 bytes (unrolled to @@ -345,4 +340,44 @@ mod tests { decode_chunk(&input[..], 0, &STANDARD.decode_table, &mut output).unwrap(); assert_eq!(&vec![b'f', b'o', b'o', b'b', b'a', b'r', 0, 0], &output); } + + #[test] + fn estimate_short_lengths() { + for (range, (num_chunks, decoded_len_estimate)) in [ + (0..=0, (0, 0)), + (1..=4, (1, 3)), + (5..=8, (1, 6)), + (9..=12, (2, 9)), + (13..=16, (2, 12)), + (17..=20, (3, 15)), + ] { + for encoded_len in range { + let estimate = GeneralPurposeEstimate::new(encoded_len); + assert_eq!(num_chunks, estimate.num_chunks); + assert_eq!(decoded_len_estimate, estimate.decoded_len_estimate); + } + } + } + + #[test] + fn estimate_via_u128_inflation() { + // cover both ends of usize + (0..1000) + .chain(usize::MAX - 1000..=usize::MAX) + .for_each(|encoded_len| { + // inflate to 128 bit type to be able to safely use the easy formulas + let len_128 = encoded_len as u128; + + let estimate = GeneralPurposeEstimate::new(encoded_len); + assert_eq!( + ((len_128 + (INPUT_CHUNK_LEN - 1) as u128) / (INPUT_CHUNK_LEN as u128)) + as usize, + estimate.num_chunks + ); + assert_eq!( + ((len_128 + 3) / 4 * 3) as usize, + estimate.decoded_len_estimate + ); + }) + } } diff --git a/third_party/rust/base64/src/engine/general_purpose/decode_suffix.rs b/third_party/rust/base64/src/engine/general_purpose/decode_suffix.rs index 5652035d0ee7..e1e005d25b50 100644 --- a/third_party/rust/base64/src/engine/general_purpose/decode_suffix.rs +++ b/third_party/rust/base64/src/engine/general_purpose/decode_suffix.rs @@ -1,13 +1,13 @@ use crate::{ - engine::{general_purpose::INVALID_VALUE, DecodePaddingMode}, + engine::{general_purpose::INVALID_VALUE, DecodeMetadata, DecodePaddingMode}, DecodeError, PAD_BYTE, }; /// Decode the last 1-8 bytes, checking for trailing set bits and padding per the provided /// parameters. /// -/// Returns the total number of bytes decoded, including the ones indicated as already written by -/// `output_index`. +/// Returns the decode metadata representing the total number of bytes decoded, including the ones +/// indicated as already written by `output_index`. pub(crate) fn decode_suffix( input: &[u8], input_index: usize, @@ -16,7 +16,7 @@ pub(crate) fn decode_suffix( decode_table: &[u8; 256], decode_allow_trailing_bits: bool, padding_mode: DecodePaddingMode, -) -> Result { +) -> Result { // Decode any leftovers that aren't a complete input block of 8 bytes. // Use a u64 as a stack-resident 8 byte buffer. let mut leftover_bits: u64 = 0; @@ -157,5 +157,12 @@ pub(crate) fn decode_suffix( leftover_bits_appended_to_buf += 8; } - Ok(output_index) + Ok(DecodeMetadata::new( + output_index, + if padding_bytes > 0 { + Some(input_index + first_padding_index) + } else { + None + }, + )) } diff --git a/third_party/rust/base64/src/engine/general_purpose/mod.rs b/third_party/rust/base64/src/engine/general_purpose/mod.rs index af8897bc2be2..01d22049a8a4 100644 --- a/third_party/rust/base64/src/engine/general_purpose/mod.rs +++ b/third_party/rust/base64/src/engine/general_purpose/mod.rs @@ -2,13 +2,14 @@ use crate::{ alphabet, alphabet::Alphabet, - engine::{Config, DecodePaddingMode}, + engine::{Config, DecodeMetadata, DecodePaddingMode}, DecodeError, }; use core::convert::TryInto; mod decode; pub(crate) mod decode_suffix; + pub use decode::GeneralPurposeEstimate; pub(crate) const INVALID_VALUE: u8 = 255; @@ -170,7 +171,7 @@ impl super::Engine for GeneralPurpose { input: &[u8], output: &mut [u8], estimate: Self::DecodeEstimate, - ) -> Result { + ) -> Result { decode::decode_helper( input, estimate, diff --git a/third_party/rust/base64/src/engine/mod.rs b/third_party/rust/base64/src/engine/mod.rs index 12dfaa884534..e10d66bb2215 100644 --- a/third_party/rust/base64/src/engine/mod.rs +++ b/third_party/rust/base64/src/engine/mod.rs @@ -62,10 +62,6 @@ pub trait Engine: Send + Sync { /// As an optimization to prevent the decoded length from being calculated twice, it is /// sometimes helpful to have a conservative estimate of the decoded size before doing the /// decoding, so this calculation is done separately and passed to [Engine::decode()] as needed. - /// - /// # Panics - /// - /// Panics if decoded length estimation overflows. #[doc(hidden)] fn internal_decoded_len_estimate(&self, input_len: usize) -> Self::DecodeEstimate; @@ -77,8 +73,6 @@ pub trait Engine: Send + Sync { /// `decode_estimate` is the result of [Engine::internal_decoded_len_estimate()], which is passed in to avoid /// calculating it again (expensive on short inputs).` /// - /// Returns the number of bytes written to `output`. - /// /// Each complete 4-byte chunk of encoded data decodes to 3 bytes of decoded data, but this /// function must also handle the final possibly partial chunk. /// If the input length is not a multiple of 4, or uses padding bytes to reach a multiple of 4, @@ -99,7 +93,7 @@ pub trait Engine: Send + Sync { input: &[u8], output: &mut [u8], decode_estimate: Self::DecodeEstimate, - ) -> Result; + ) -> Result; /// Returns the config for this engine. fn config(&self) -> &Self::Config; @@ -120,14 +114,23 @@ pub trait Engine: Send + Sync { /// /// let b64_url = CUSTOM_ENGINE.encode(b"hello internet~"); #[cfg(any(feature = "alloc", feature = "std", test))] + #[inline] fn encode>(&self, input: T) -> String { - let encoded_size = encoded_len(input.as_ref().len(), self.config().encode_padding()) - .expect("integer overflow when calculating buffer size"); - let mut buf = vec![0; encoded_size]; + fn inner(engine: &E, input_bytes: &[u8]) -> String + where + E: Engine + ?Sized, + { + let encoded_size = encoded_len(input_bytes.len(), engine.config().encode_padding()) + .expect("integer overflow when calculating buffer size"); - encode_with_padding(input.as_ref(), &mut buf[..], self, encoded_size); + let mut buf = vec![0; encoded_size]; - String::from_utf8(buf).expect("Invalid UTF8") + encode_with_padding(input_bytes, &mut buf[..], engine, encoded_size); + + String::from_utf8(buf).expect("Invalid UTF8") + } + + inner(self, input.as_ref()) } /// Encode arbitrary octets as base64 into a supplied `String`. @@ -151,16 +154,20 @@ pub trait Engine: Send + Sync { /// } /// ``` #[cfg(any(feature = "alloc", feature = "std", test))] + #[inline] fn encode_string>(&self, input: T, output_buf: &mut String) { - let input_bytes = input.as_ref(); - + fn inner(engine: &E, input_bytes: &[u8], output_buf: &mut String) + where + E: Engine + ?Sized, { let mut sink = chunked_encoder::StringSink::new(output_buf); - chunked_encoder::ChunkedEncoder::new(self) + chunked_encoder::ChunkedEncoder::new(engine) .encode(input_bytes, &mut sink) .expect("Writing to a String shouldn't fail"); } + + inner(self, input.as_ref(), output_buf) } /// Encode arbitrary octets as base64 into a supplied slice. @@ -185,29 +192,38 @@ pub trait Engine: Send + Sync { /// /// assert_eq!(s, general_purpose::STANDARD.decode(&buf).unwrap().as_slice()); /// ``` + #[inline] fn encode_slice>( &self, input: T, output_buf: &mut [u8], ) -> Result { - let input_bytes = input.as_ref(); + fn inner( + engine: &E, + input_bytes: &[u8], + output_buf: &mut [u8], + ) -> Result + where + E: Engine + ?Sized, + { + let encoded_size = encoded_len(input_bytes.len(), engine.config().encode_padding()) + .expect("usize overflow when calculating buffer size"); - let encoded_size = encoded_len(input_bytes.len(), self.config().encode_padding()) - .expect("usize overflow when calculating buffer size"); + if output_buf.len() < encoded_size { + return Err(EncodeSliceError::OutputSliceTooSmall); + } - if output_buf.len() < encoded_size { - return Err(EncodeSliceError::OutputSliceTooSmall); + let b64_output = &mut output_buf[0..encoded_size]; + + encode_with_padding(input_bytes, b64_output, engine, encoded_size); + + Ok(encoded_size) } - let b64_output = &mut output_buf[0..encoded_size]; - - encode_with_padding(input_bytes, b64_output, self, encoded_size); - - Ok(encoded_size) + inner(self, input.as_ref(), output_buf) } - /// Decode from string reference as octets using the specified [Engine]. - /// Returns a `Result` containing a `Vec`. + /// Decode the input into a new `Vec`. /// /// # Example /// @@ -225,25 +241,30 @@ pub trait Engine: Send + Sync { /// .decode("aGVsbG8gaW50ZXJuZXR-Cg").unwrap(); /// println!("{:?}", bytes_url); /// ``` - /// - /// # Panics - /// - /// Panics if decoded length estimation overflows. - /// This would happen for sizes within a few bytes of the maximum value of `usize`. #[cfg(any(feature = "alloc", feature = "std", test))] + #[inline] fn decode>(&self, input: T) -> Result, DecodeError> { - let input_bytes = input.as_ref(); + fn inner(engine: &E, input_bytes: &[u8]) -> Result, DecodeError> + where + E: Engine + ?Sized, + { + let estimate = engine.internal_decoded_len_estimate(input_bytes.len()); + let mut buffer = vec![0; estimate.decoded_len_estimate()]; - let estimate = self.internal_decoded_len_estimate(input_bytes.len()); - let mut buffer = vec![0; estimate.decoded_len_estimate()]; + let bytes_written = engine + .internal_decode(input_bytes, &mut buffer, estimate)? + .decoded_len; - let bytes_written = self.internal_decode(input_bytes, &mut buffer, estimate)?; - buffer.truncate(bytes_written); + buffer.truncate(bytes_written); - Ok(buffer) + Ok(buffer) + } + + inner(self, input.as_ref()) } - /// Decode from string reference as octets. + /// Decode the `input` into the supplied `buffer`. + /// /// Writes into the supplied `Vec`, which may allocate if its internal buffer isn't big enough. /// Returns a `Result` containing an empty tuple, aka `()`. /// @@ -272,39 +293,45 @@ pub trait Engine: Send + Sync { /// println!("{:?}", buffer); /// } /// ``` - /// - /// # Panics - /// - /// Panics if decoded length estimation overflows. - /// This would happen for sizes within a few bytes of the maximum value of `usize`. #[cfg(any(feature = "alloc", feature = "std", test))] + #[inline] fn decode_vec>( &self, input: T, buffer: &mut Vec, ) -> Result<(), DecodeError> { - let input_bytes = input.as_ref(); + fn inner(engine: &E, input_bytes: &[u8], buffer: &mut Vec) -> Result<(), DecodeError> + where + E: Engine + ?Sized, + { + let starting_output_len = buffer.len(); + let estimate = engine.internal_decoded_len_estimate(input_bytes.len()); - let starting_output_len = buffer.len(); + let total_len_estimate = estimate + .decoded_len_estimate() + .checked_add(starting_output_len) + .expect("Overflow when calculating output buffer length"); - let estimate = self.internal_decoded_len_estimate(input_bytes.len()); - let total_len_estimate = estimate - .decoded_len_estimate() - .checked_add(starting_output_len) - .expect("Overflow when calculating output buffer length"); - buffer.resize(total_len_estimate, 0); + buffer.resize(total_len_estimate, 0); - let buffer_slice = &mut buffer.as_mut_slice()[starting_output_len..]; - let bytes_written = self.internal_decode(input_bytes, buffer_slice, estimate)?; + let buffer_slice = &mut buffer.as_mut_slice()[starting_output_len..]; - buffer.truncate(starting_output_len + bytes_written); + let bytes_written = engine + .internal_decode(input_bytes, buffer_slice, estimate)? + .decoded_len; - Ok(()) + buffer.truncate(starting_output_len + bytes_written); + + Ok(()) + } + + inner(self, input.as_ref(), buffer) } /// Decode the input into the provided output slice. /// - /// Returns an error if `output` is smaller than the estimated decoded length. + /// Returns the number of bytes written to the slice, or an error if `output` is smaller than + /// the estimated decoded length. /// /// This will not write any bytes past exactly what is decoded (no stray garbage bytes at the end). /// @@ -312,29 +339,39 @@ pub trait Engine: Send + Sync { /// /// See [Engine::decode_slice_unchecked] for a version that panics instead of returning an error /// if the output buffer is too small. - /// - /// # Panics - /// - /// Panics if decoded length estimation overflows. - /// This would happen for sizes within a few bytes of the maximum value of `usize`. + #[inline] fn decode_slice>( &self, input: T, output: &mut [u8], ) -> Result { - let input_bytes = input.as_ref(); + fn inner( + engine: &E, + input_bytes: &[u8], + output: &mut [u8], + ) -> Result + where + E: Engine + ?Sized, + { + let estimate = engine.internal_decoded_len_estimate(input_bytes.len()); - let estimate = self.internal_decoded_len_estimate(input_bytes.len()); - if output.len() < estimate.decoded_len_estimate() { - return Err(DecodeSliceError::OutputSliceTooSmall); + if output.len() < estimate.decoded_len_estimate() { + return Err(DecodeSliceError::OutputSliceTooSmall); + } + + engine + .internal_decode(input_bytes, output, estimate) + .map_err(|e| e.into()) + .map(|dm| dm.decoded_len) } - self.internal_decode(input_bytes, output, estimate) - .map_err(|e| e.into()) + inner(self, input.as_ref(), output) } /// Decode the input into the provided output slice. /// + /// Returns the number of bytes written to the slice. + /// /// This will not write any bytes past exactly what is decoded (no stray garbage bytes at the end). /// /// See [crate::decoded_len_estimate] for calculating buffer sizes. @@ -344,22 +381,27 @@ pub trait Engine: Send + Sync { /// /// # Panics /// - /// Panics if decoded length estimation overflows. - /// This would happen for sizes within a few bytes of the maximum value of `usize`. - /// /// Panics if the provided output buffer is too small for the decoded data. + #[inline] fn decode_slice_unchecked>( &self, input: T, output: &mut [u8], ) -> Result { - let input_bytes = input.as_ref(); + fn inner(engine: &E, input_bytes: &[u8], output: &mut [u8]) -> Result + where + E: Engine + ?Sized, + { + engine + .internal_decode( + input_bytes, + output, + engine.internal_decoded_len_estimate(input_bytes.len()), + ) + .map(|dm| dm.decoded_len) + } - self.internal_decode( - input_bytes, - output, - self.internal_decoded_len_estimate(input_bytes.len()), - ) + inner(self, input.as_ref(), output) } } @@ -387,11 +429,6 @@ pub trait DecodeEstimate { /// /// The estimate must be no larger than the next largest complete triple of decoded bytes. /// That is, the final quad of tokens to decode may be assumed to be complete with no padding. - /// - /// # Panics - /// - /// Panics if decoded length estimation overflows. - /// This would happen for sizes within a few bytes of the maximum value of `usize`. fn decoded_len_estimate(&self) -> usize; } @@ -408,3 +445,21 @@ pub enum DecodePaddingMode { /// Padding must be absent -- for when you want predictable padding, without any wasted bytes. RequireNone, } + +/// Metadata about the result of a decode operation +#[derive(PartialEq, Eq, Debug)] +pub struct DecodeMetadata { + /// Number of decoded bytes output + pub(crate) decoded_len: usize, + /// Offset of the first padding byte in the input, if any + pub(crate) padding_offset: Option, +} + +impl DecodeMetadata { + pub(crate) fn new(decoded_bytes: usize, padding_index: Option) -> Self { + Self { + decoded_len: decoded_bytes, + padding_offset: padding_index, + } + } +} diff --git a/third_party/rust/base64/src/engine/naive.rs b/third_party/rust/base64/src/engine/naive.rs index 6665c5eb4114..42b6085bd638 100644 --- a/third_party/rust/base64/src/engine/naive.rs +++ b/third_party/rust/base64/src/engine/naive.rs @@ -2,7 +2,7 @@ use crate::{ alphabet::Alphabet, engine::{ general_purpose::{self, decode_table, encode_table}, - Config, DecodeEstimate, DecodePaddingMode, Engine, + Config, DecodeEstimate, DecodeMetadata, DecodePaddingMode, Engine, }, DecodeError, PAD_BYTE, }; @@ -112,7 +112,7 @@ impl Engine for Naive { input: &[u8], output: &mut [u8], estimate: Self::DecodeEstimate, - ) -> Result { + ) -> Result { if estimate.rem == 1 { // trailing whitespace is so common that it's worth it to check the last byte to // possibly return a better error message diff --git a/third_party/rust/base64/src/engine/tests.rs b/third_party/rust/base64/src/engine/tests.rs index 906bba04d88e..b048005520d8 100644 --- a/third_party/rust/base64/src/engine/tests.rs +++ b/third_party/rust/base64/src/engine/tests.rs @@ -8,13 +8,16 @@ use rand::{ }; use rstest::rstest; use rstest_reuse::{apply, template}; -use std::{collections, fmt}; +use std::{collections, fmt, io::Read as _}; use crate::{ alphabet::{Alphabet, STANDARD}, encode::add_padding, encoded_len, - engine::{general_purpose, naive, Config, DecodeEstimate, DecodePaddingMode, Engine}, + engine::{ + general_purpose, naive, Config, DecodeEstimate, DecodeMetadata, DecodePaddingMode, Engine, + }, + read::DecoderReader, tests::{assert_encode_sanity, random_alphabet, random_config}, DecodeError, PAD_BYTE, }; @@ -24,9 +27,20 @@ use crate::{ #[rstest(engine_wrapper, case::general_purpose(GeneralPurposeWrapper {}), case::naive(NaiveWrapper {}), +case::decoder_reader(DecoderReaderEngineWrapper {}), )] fn all_engines(engine_wrapper: E) {} +/// Some decode tests don't make sense for use with `DecoderReader` as they are difficult to +/// reason about or otherwise inapplicable given how DecoderReader slice up its input along +/// chunk boundaries. +#[template] +#[rstest(engine_wrapper, +case::general_purpose(GeneralPurposeWrapper {}), +case::naive(NaiveWrapper {}), +)] +fn all_engines_except_decoder_reader(engine_wrapper: E) {} + #[apply(all_engines)] fn rfc_test_vectors_std_alphabet(engine_wrapper: E) { let data = vec![ @@ -86,7 +100,7 @@ fn rfc_test_vectors_std_alphabet(engine_wrapper: E) { &encoded_without_padding, &std::str::from_utf8(&encode_buf[0..encode_len]).unwrap() ); - let pad_len = add_padding(orig.len(), &mut encode_buf[encode_len..]); + let pad_len = add_padding(encode_len, &mut encode_buf[encode_len..]); assert_eq!(encoded.as_bytes(), &encode_buf[..encode_len + pad_len]); let decode_len = engine @@ -195,7 +209,10 @@ fn encode_doesnt_write_extra_bytes(engine_wrapper: E) { // pad so we can decode it in case our random engine requires padding let pad_len = if padded { - add_padding(orig_len, &mut encode_buf[prefix_len + encoded_len_no_pad..]) + add_padding( + encoded_len_no_pad, + &mut encode_buf[prefix_len + encoded_len_no_pad..], + ) } else { 0 }; @@ -382,7 +399,7 @@ fn decode_detect_invalid_last_symbol_every_possible_two_symbols(engine_wrapper: E) { let len_range = distributions::Uniform::new(1, 1_000); - for _ in 0..10_000 { + for _ in 0..100_000 { let alphabet = random_alphabet(&mut rng); let engine = E::random_alphabet(&mut rng, alphabet); @@ -573,7 +590,7 @@ fn decode_invalid_byte_error(engine_wrapper: E) { let invalid_byte: u8 = loop { let byte: u8 = rng.gen(); - if alphabet.symbols.contains(&byte) { + if alphabet.symbols.contains(&byte) || byte == PAD_BYTE { continue; } else { break byte; @@ -597,14 +614,16 @@ fn decode_invalid_byte_error(engine_wrapper: E) { /// Any amount of padding anywhere before the final non padding character = invalid byte at first /// pad byte. /// From this, we know padding must extend to the end of the input. -#[apply(all_engines)] +// DecoderReader pseudo-engine detects InvalidLastSymbol instead of InvalidLength because it +// can end a decode on the quad that happens to contain the start of the padding +#[apply(all_engines_except_decoder_reader)] fn decode_padding_before_final_non_padding_char_error_invalid_byte( engine_wrapper: E, ) { let mut rng = seeded_rng(); // the different amounts of proper padding, w/ offset from end for the last non-padding char - let suffixes = vec![("/w==", 2), ("iYu=", 1), ("zzzz", 0)]; + let suffixes = [("/w==", 2), ("iYu=", 1), ("zzzz", 0)]; let prefix_quads_range = distributions::Uniform::from(0..=256); @@ -641,10 +660,13 @@ fn decode_padding_before_final_non_padding_char_error_invalid_byte( engine_wrapper: E, ) { @@ -652,8 +674,8 @@ fn decode_padding_starts_before_final_chunk_error_invalid_byte // must have at least one prefix quad let prefix_quads_range = distributions::Uniform::from(1..256); - // including 1 just to make sure that it really does produce invalid length - let suffix_pad_len_range = distributions::Uniform::from(1..=4); + // excluding 1 since we don't care about invalid length in this test + let suffix_pad_len_range = distributions::Uniform::from(2..=4); for mode in all_pad_modes() { // we don't encode so we don't care about encode padding let engine = E::standard_with_pad_mode(true, mode); @@ -671,14 +693,48 @@ fn decode_padding_starts_before_final_chunk_error_invalid_byte let padding_start = encoded.len() - padding_len; encoded[padding_start..].fill(PAD_BYTE); - if suffix_len == 1 { - assert_eq!(Err(DecodeError::InvalidLength), engine.decode(&encoded),); - } else { - assert_eq!( - Err(DecodeError::InvalidByte(padding_start, PAD_BYTE)), - engine.decode(&encoded), - ); - } + assert_eq!( + Err(DecodeError::InvalidByte(padding_start, PAD_BYTE)), + engine.decode(&encoded), + "suffix_len: {}, padding_len: {}, b64: {}", + suffix_len, + padding_len, + std::str::from_utf8(&encoded).unwrap() + ); + } + } +} + +/// Any amount of padding before final chunk that crosses over into final chunk with 1 byte = +/// invalid length. +/// From this we know the padding must start in the final chunk. +// DecoderReader pseudo-engine detects InvalidByte instead of InvalidLength because it starts by +// decoding only the available complete quads +#[apply(all_engines_except_decoder_reader)] +fn decode_padding_starts_before_final_chunk_error_invalid_length( + engine_wrapper: E, +) { + let mut rng = seeded_rng(); + + // must have at least one prefix quad + let prefix_quads_range = distributions::Uniform::from(1..256); + for mode in all_pad_modes() { + // we don't encode so we don't care about encode padding + let engine = E::standard_with_pad_mode(true, mode); + for _ in 0..100_000 { + let mut encoded = "ABCD" + .repeat(prefix_quads_range.sample(&mut rng)) + .into_bytes(); + encoded.resize(encoded.len() + 1, PAD_BYTE); + + // amount of padding must be long enough to extend back from suffix into previous + // quads + let padding_len = rng.gen_range(1 + 1..encoded.len()); + // no non-padding after padding in this test, so padding goes to the end + let padding_start = encoded.len() - padding_len; + encoded[padding_start..].fill(PAD_BYTE); + + assert_eq!(Err(DecodeError::InvalidLength), engine.decode(&encoded),); } } } @@ -787,7 +843,9 @@ fn decode_malleability_test_case_2_byte_suffix_no_padding(engi } // https://eprint.iacr.org/2022/361.pdf table 2, test 7 -#[apply(all_engines)] +// DecoderReader pseudo-engine gets InvalidByte at 8 (extra padding) since it decodes the first +// two complete quads correctly. +#[apply(all_engines_except_decoder_reader)] fn decode_malleability_test_case_2_byte_suffix_too_much_padding( engine_wrapper: E, ) { @@ -811,7 +869,7 @@ fn decode_pad_mode_requires_canonical_accepts_canonical(engine fn decode_pad_mode_requires_canonical_rejects_non_canonical(engine_wrapper: E) { let engine = E::standard_with_pad_mode(true, DecodePaddingMode::RequireCanonical); - let suffixes = vec!["/w", "/w=", "iYU"]; + let suffixes = ["/w", "/w=", "iYU"]; for num_prefix_quads in 0..256 { for &suffix in suffixes.iter() { let mut encoded = "AAAA".repeat(num_prefix_quads); @@ -838,7 +896,7 @@ fn decode_pad_mode_requires_no_padding_accepts_no_padding(engi fn decode_pad_mode_requires_no_padding_rejects_any_padding(engine_wrapper: E) { let engine = E::standard_with_pad_mode(true, DecodePaddingMode::RequireNone); - let suffixes = vec!["/w=", "/w==", "iYU="]; + let suffixes = ["/w=", "/w==", "iYU="]; for num_prefix_quads in 0..256 { for &suffix in suffixes.iter() { let mut encoded = "AAAA".repeat(num_prefix_quads); @@ -861,7 +919,11 @@ fn decode_pad_mode_indifferent_padding_accepts_anything(engine } //this is a MAY in the rfc: https://tools.ietf.org/html/rfc4648#section-3.3 -#[apply(all_engines)] +// DecoderReader pseudo-engine finds the first padding, but doesn't report it as an error, +// because in the next decode it finds more padding, which is reported as InvalidByte, just +// with an offset at its position in the second decode, rather than being linked to the start +// of the padding that was first seen in the previous decode. +#[apply(all_engines_except_decoder_reader)] fn decode_pad_byte_in_penultimate_quad_error(engine_wrapper: E) { for mode in all_pad_modes() { // we don't encode so we don't care about encode padding @@ -895,7 +957,7 @@ fn decode_pad_byte_in_penultimate_quad_error(engine_wrapper: E num_prefix_quads * 4 + num_valid_bytes_penultimate_quad, b'=', ), - engine.decode(&s).unwrap_err() + engine.decode(&s).unwrap_err(), ); } } @@ -955,7 +1017,9 @@ fn decode_absurd_pad_error(engine_wrapper: E) { } } -#[apply(all_engines)] +// DecoderReader pseudo-engine detects InvalidByte instead of InvalidLength because it starts by +// decoding only the available complete quads +#[apply(all_engines_except_decoder_reader)] fn decode_too_much_padding_returns_error(engine_wrapper: E) { for mode in all_pad_modes() { // we don't encode so we don't care about encode padding @@ -981,7 +1045,9 @@ fn decode_too_much_padding_returns_error(engine_wrapper: E) { } } -#[apply(all_engines)] +// DecoderReader pseudo-engine detects InvalidByte instead of InvalidLength because it starts by +// decoding only the available complete quads +#[apply(all_engines_except_decoder_reader)] fn decode_padding_followed_by_non_padding_returns_error(engine_wrapper: E) { for mode in all_pad_modes() { // we don't encode so we don't care about encode padding @@ -1079,27 +1145,43 @@ fn decode_too_few_symbols_in_final_quad_error(engine_wrapper: } } -#[apply(all_engines)] +// DecoderReader pseudo-engine can't handle DecodePaddingMode::RequireNone since it will decode +// a complete quad with padding in it before encountering the stray byte that makes it an invalid +// length +#[apply(all_engines_except_decoder_reader)] fn decode_invalid_trailing_bytes(engine_wrapper: E) { for mode in all_pad_modes() { - // we don't encode so we don't care about encode padding - let engine = E::standard_with_pad_mode(true, mode); + do_invalid_trailing_byte(E::standard_with_pad_mode(true, mode), mode); + } +} - for num_prefix_quads in 0..256 { - let mut s: String = "ABCD".repeat(num_prefix_quads); - s.push_str("Cg==\n"); +#[apply(all_engines)] +fn decode_invalid_trailing_bytes_all_modes(engine_wrapper: E) { + // excluding no padding mode because the DecoderWrapper pseudo-engine will fail with + // InvalidPadding because it will decode the last complete quad with padding first + for mode in pad_modes_allowing_padding() { + do_invalid_trailing_byte(E::standard_with_pad_mode(true, mode), mode); + } +} - // The case of trailing newlines is common enough to warrant a test for a good error - // message. - assert_eq!( - Err(DecodeError::InvalidByte(num_prefix_quads * 4 + 4, b'\n')), - engine.decode(&s) - ); +#[apply(all_engines)] +fn decode_invalid_trailing_padding_as_invalid_length(engine_wrapper: E) { + // excluding no padding mode because the DecoderWrapper pseudo-engine will fail with + // InvalidPadding because it will decode the last complete quad with padding first + for mode in pad_modes_allowing_padding() { + do_invalid_trailing_padding_as_invalid_length(E::standard_with_pad_mode(true, mode), mode); + } +} - // extra padding, however, is still InvalidLength - let s = s.replace('\n', "="); - assert_eq!(Err(DecodeError::InvalidLength), engine.decode(s)); - } +// DecoderReader pseudo-engine can't handle DecodePaddingMode::RequireNone since it will decode +// a complete quad with padding in it before encountering the stray byte that makes it an invalid +// length +#[apply(all_engines_except_decoder_reader)] +fn decode_invalid_trailing_padding_as_invalid_length_all_modes( + engine_wrapper: E, +) { + for mode in all_pad_modes() { + do_invalid_trailing_padding_as_invalid_length(E::standard_with_pad_mode(true, mode), mode); } } @@ -1177,6 +1259,53 @@ fn decode_into_slice_fits_in_precisely_sized_slice(engine_wrap } } +#[apply(all_engines)] +fn inner_decode_reports_padding_position(engine_wrapper: E) { + let mut b64 = String::new(); + let mut decoded = Vec::new(); + let engine = E::standard(); + + for pad_position in 1..10_000 { + b64.clear(); + decoded.clear(); + // plenty of room for original data + decoded.resize(pad_position, 0); + + for _ in 0..pad_position { + b64.push('A'); + } + // finish the quad with padding + for _ in 0..(4 - (pad_position % 4)) { + b64.push('='); + } + + let decode_res = engine.internal_decode( + b64.as_bytes(), + &mut decoded[..], + engine.internal_decoded_len_estimate(b64.len()), + ); + if pad_position % 4 < 2 { + // impossible padding + assert_eq!( + Err(DecodeError::InvalidByte(pad_position, PAD_BYTE)), + decode_res + ); + } else { + let decoded_bytes = pad_position / 4 * 3 + + match pad_position % 4 { + 0 => 0, + 2 => 1, + 3 => 2, + _ => unreachable!(), + }; + assert_eq!( + Ok(DecodeMetadata::new(decoded_bytes, Some(pad_position))), + decode_res + ); + } + } +} + #[apply(all_engines)] fn decode_length_estimate_delta(engine_wrapper: E) { for engine in [E::standard(), E::standard_unpadded()] { @@ -1200,6 +1329,64 @@ fn decode_length_estimate_delta(engine_wrapper: E) { } } +#[apply(all_engines)] +fn estimate_via_u128_inflation(engine_wrapper: E) { + // cover both ends of usize + (0..1000) + .chain(usize::MAX - 1000..=usize::MAX) + .for_each(|encoded_len| { + // inflate to 128 bit type to be able to safely use the easy formulas + let len_128 = encoded_len as u128; + + let estimate = E::standard() + .internal_decoded_len_estimate(encoded_len) + .decoded_len_estimate(); + + // This check is a little too strict: it requires using the (len + 3) / 4 * 3 formula + // or equivalent, but until other engines come along that use a different formula + // requiring that we think more carefully about what the allowable criteria are, this + // will do. + assert_eq!( + ((len_128 + 3) / 4 * 3) as usize, + estimate, + "enc len {}", + encoded_len + ); + }) +} + +fn do_invalid_trailing_byte(engine: impl Engine, mode: DecodePaddingMode) { + for num_prefix_quads in 0..256 { + let mut s: String = "ABCD".repeat(num_prefix_quads); + s.push_str("Cg==\n"); + + // The case of trailing newlines is common enough to warrant a test for a good error + // message. + assert_eq!( + Err(DecodeError::InvalidByte(num_prefix_quads * 4 + 4, b'\n')), + engine.decode(&s), + "mode: {:?}, input: {}", + mode, + s + ); + } +} + +fn do_invalid_trailing_padding_as_invalid_length(engine: impl Engine, mode: DecodePaddingMode) { + for num_prefix_quads in 0..256 { + let mut s: String = "ABCD".repeat(num_prefix_quads); + s.push_str("Cg==="); + + assert_eq!( + Err(DecodeError::InvalidLength), + engine.decode(&s), + "mode: {:?}, input: {}", + mode, + s + ); + } +} + /// Returns a tuple of the original data length, the encoded data length (just data), and the length including padding. /// /// Vecs provided should be empty. @@ -1219,7 +1406,7 @@ fn generate_random_encoded_data(vec: &mut Vec, rng: &mut R, len: usize) { } } -fn prefixed_data<'i, 'd>( - input_with_prefix: &'i mut String, - prefix_len: usize, - data: &'d str, -) -> &'i str { +fn prefixed_data<'i>(input_with_prefix: &'i mut String, prefix_len: usize, data: &str) -> &'i str { input_with_prefix.truncate(prefix_len); input_with_prefix.push_str(data); input_with_prefix.as_str() @@ -1405,6 +1588,103 @@ impl EngineWrapper for NaiveWrapper { } } +/// A pseudo-Engine that routes all decoding through [DecoderReader] +struct DecoderReaderEngine { + engine: E, +} + +impl From for DecoderReaderEngine { + fn from(value: E) -> Self { + Self { engine: value } + } +} + +impl Engine for DecoderReaderEngine { + type Config = E::Config; + type DecodeEstimate = E::DecodeEstimate; + + fn internal_encode(&self, input: &[u8], output: &mut [u8]) -> usize { + self.engine.internal_encode(input, output) + } + + fn internal_decoded_len_estimate(&self, input_len: usize) -> Self::DecodeEstimate { + self.engine.internal_decoded_len_estimate(input_len) + } + + fn internal_decode( + &self, + input: &[u8], + output: &mut [u8], + decode_estimate: Self::DecodeEstimate, + ) -> Result { + let mut reader = DecoderReader::new(input, &self.engine); + let mut buf = vec![0; input.len()]; + // to avoid effects like not detecting invalid length due to progressively growing + // the output buffer in read_to_end etc, read into a big enough buffer in one go + // to make behavior more consistent with normal engines + let _ = reader + .read(&mut buf) + .and_then(|len| { + buf.truncate(len); + // make sure we got everything + reader.read_to_end(&mut buf) + }) + .map_err(|io_error| { + *io_error + .into_inner() + .and_then(|inner| inner.downcast::().ok()) + .unwrap() + })?; + output[..buf.len()].copy_from_slice(&buf); + Ok(DecodeMetadata::new( + buf.len(), + input + .iter() + .enumerate() + .filter(|(_offset, byte)| **byte == PAD_BYTE) + .map(|(offset, _byte)| offset) + .next(), + )) + } + + fn config(&self) -> &Self::Config { + self.engine.config() + } +} + +struct DecoderReaderEngineWrapper {} + +impl EngineWrapper for DecoderReaderEngineWrapper { + type Engine = DecoderReaderEngine; + + fn standard() -> Self::Engine { + GeneralPurposeWrapper::standard().into() + } + + fn standard_unpadded() -> Self::Engine { + GeneralPurposeWrapper::standard_unpadded().into() + } + + fn standard_with_pad_mode( + encode_pad: bool, + decode_pad_mode: DecodePaddingMode, + ) -> Self::Engine { + GeneralPurposeWrapper::standard_with_pad_mode(encode_pad, decode_pad_mode).into() + } + + fn standard_allow_trailing_bits() -> Self::Engine { + GeneralPurposeWrapper::standard_allow_trailing_bits().into() + } + + fn random(rng: &mut R) -> Self::Engine { + GeneralPurposeWrapper::random(rng).into() + } + + fn random_alphabet(rng: &mut R, alphabet: &Alphabet) -> Self::Engine { + GeneralPurposeWrapper::random_alphabet(rng, alphabet).into() + } +} + fn seeded_rng() -> impl rand::Rng { rngs::SmallRng::from_entropy() } @@ -1417,6 +1697,13 @@ fn all_pad_modes() -> Vec { ] } +fn pad_modes_allowing_padding() -> Vec { + vec![ + DecodePaddingMode::Indifferent, + DecodePaddingMode::RequireCanonical, + ] +} + fn assert_all_suffixes_ok(engine: E, suffixes: Vec<&str>) { for num_prefix_quads in 0..256 { for &suffix in suffixes.iter() { diff --git a/third_party/rust/base64/src/read/decoder.rs b/third_party/rust/base64/src/read/decoder.rs index 4888c9c4e71a..b656ae3d2678 100644 --- a/third_party/rust/base64/src/read/decoder.rs +++ b/third_party/rust/base64/src/read/decoder.rs @@ -1,4 +1,4 @@ -use crate::{engine::Engine, DecodeError}; +use crate::{engine::Engine, DecodeError, PAD_BYTE}; use std::{cmp, fmt, io}; // This should be large, but it has to fit on the stack. @@ -46,13 +46,15 @@ pub struct DecoderReader<'e, E: Engine, R: io::Read> { // Technically we only need to hold 2 bytes but then we'd need a separate temporary buffer to // decode 3 bytes into and then juggle copying one byte into the provided read buf and the rest // into here, which seems like a lot of complexity for 1 extra byte of storage. - decoded_buffer: [u8; 3], + decoded_buffer: [u8; DECODED_CHUNK_SIZE], // index of start of decoded data decoded_offset: usize, // length of decoded data decoded_len: usize, // used to provide accurate offsets in errors total_b64_decoded: usize, + // offset of previously seen padding, if any + padding_offset: Option, } impl<'e, E: Engine, R: io::Read> fmt::Debug for DecoderReader<'e, E, R> { @@ -64,6 +66,7 @@ impl<'e, E: Engine, R: io::Read> fmt::Debug for DecoderReader<'e, E, R> { .field("decoded_offset", &self.decoded_offset) .field("decoded_len", &self.decoded_len) .field("total_b64_decoded", &self.total_b64_decoded) + .field("padding_offset", &self.padding_offset) .finish() } } @@ -81,6 +84,7 @@ impl<'e, E: Engine, R: io::Read> DecoderReader<'e, E, R> { decoded_offset: 0, decoded_len: 0, total_b64_decoded: 0, + padding_offset: None, } } @@ -127,20 +131,28 @@ impl<'e, E: Engine, R: io::Read> DecoderReader<'e, E, R> { /// caller's responsibility to choose the number of b64 bytes to decode correctly. /// /// Returns a Result with the number of decoded bytes written to `buf`. - fn decode_to_buf(&mut self, num_bytes: usize, buf: &mut [u8]) -> io::Result { - debug_assert!(self.b64_len >= num_bytes); + fn decode_to_buf(&mut self, b64_len_to_decode: usize, buf: &mut [u8]) -> io::Result { + debug_assert!(self.b64_len >= b64_len_to_decode); debug_assert!(self.b64_offset + self.b64_len <= BUF_SIZE); debug_assert!(!buf.is_empty()); - let decoded = self + let b64_to_decode = &self.b64_buffer[self.b64_offset..self.b64_offset + b64_len_to_decode]; + let decode_metadata = self .engine .internal_decode( - &self.b64_buffer[self.b64_offset..self.b64_offset + num_bytes], + b64_to_decode, buf, - self.engine.internal_decoded_len_estimate(num_bytes), + self.engine.internal_decoded_len_estimate(b64_len_to_decode), ) .map_err(|e| match e { DecodeError::InvalidByte(offset, byte) => { + // This can be incorrect, but not in a way that probably matters to anyone: + // if there was padding handled in a previous decode, and we are now getting + // InvalidByte due to more padding, we should arguably report InvalidByte with + // PAD_BYTE at the original padding position (`self.padding_offset`), but we + // don't have a good way to tie those two cases together, so instead we + // just report the invalid byte as if the previous padding, and its possibly + // related downgrade to a now invalid byte, didn't happen. DecodeError::InvalidByte(self.total_b64_decoded + offset, byte) } DecodeError::InvalidLength => DecodeError::InvalidLength, @@ -151,13 +163,27 @@ impl<'e, E: Engine, R: io::Read> DecoderReader<'e, E, R> { }) .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; - self.total_b64_decoded += num_bytes; - self.b64_offset += num_bytes; - self.b64_len -= num_bytes; + if let Some(offset) = self.padding_offset { + // we've already seen padding + if decode_metadata.decoded_len > 0 { + // we read more after already finding padding; report error at first padding byte + return Err(io::Error::new( + io::ErrorKind::InvalidData, + DecodeError::InvalidByte(offset, PAD_BYTE), + )); + } + } + + self.padding_offset = self.padding_offset.or(decode_metadata + .padding_offset + .map(|offset| self.total_b64_decoded + offset)); + self.total_b64_decoded += b64_len_to_decode; + self.b64_offset += b64_len_to_decode; + self.b64_len -= b64_len_to_decode; debug_assert!(self.b64_offset + self.b64_len <= BUF_SIZE); - Ok(decoded) + Ok(decode_metadata.decoded_len) } /// Unwraps this `DecoderReader`, returning the base reader which it reads base64 encoded @@ -205,9 +231,9 @@ impl<'e, E: Engine, R: io::Read> io::Read for DecoderReader<'e, E, R> { self.decoded_offset < DECODED_CHUNK_SIZE }); - // We shouldn't ever decode into here when we can't immediately write at least one byte into - // the provided buf, so the effective length should only be 3 momentarily between when we - // decode and when we copy into the target buffer. + // We shouldn't ever decode into decoded_buffer when we can't immediately write at least one + // byte into the provided buf, so the effective length should only be 3 momentarily between + // when we decode and when we copy into the target buffer. debug_assert!(self.decoded_len < DECODED_CHUNK_SIZE); debug_assert!(self.decoded_len + self.decoded_offset <= DECODED_CHUNK_SIZE); @@ -217,20 +243,15 @@ impl<'e, E: Engine, R: io::Read> io::Read for DecoderReader<'e, E, R> { } else { let mut at_eof = false; while self.b64_len < BASE64_CHUNK_SIZE { - // Work around lack of copy_within, which is only present in 1.37 // Copy any bytes we have to the start of the buffer. - // We know we have < 1 chunk, so we can use a tiny tmp buffer. - let mut memmove_buf = [0_u8; BASE64_CHUNK_SIZE]; - memmove_buf[..self.b64_len].copy_from_slice( - &self.b64_buffer[self.b64_offset..self.b64_offset + self.b64_len], - ); - self.b64_buffer[0..self.b64_len].copy_from_slice(&memmove_buf[..self.b64_len]); + self.b64_buffer + .copy_within(self.b64_offset..self.b64_offset + self.b64_len, 0); self.b64_offset = 0; // then fill in more data let read = self.read_from_delegate()?; if read == 0 { - // we never pass in an empty buf, so 0 => we've hit EOF + // we never read into an empty buf, so 0 => we've hit EOF at_eof = true; break; } diff --git a/third_party/rust/base64/src/read/decoder_tests.rs b/third_party/rust/base64/src/read/decoder_tests.rs index 65d58d8e3ffe..625a07dbd584 100644 --- a/third_party/rust/base64/src/read/decoder_tests.rs +++ b/third_party/rust/base64/src/read/decoder_tests.rs @@ -8,9 +8,10 @@ use rand::{Rng as _, RngCore as _}; use super::decoder::{DecoderReader, BUF_SIZE}; use crate::{ + alphabet, engine::{general_purpose::STANDARD, Engine, GeneralPurpose}, tests::{random_alphabet, random_config, random_engine}, - DecodeError, + DecodeError, PAD_BYTE, }; #[test] @@ -247,19 +248,21 @@ fn reports_invalid_byte_correctly() { let mut rng = rand::thread_rng(); let mut bytes = Vec::new(); let mut b64 = String::new(); - let mut decoded = Vec::new(); + let mut stream_decoded = Vec::new(); + let mut bulk_decoded = Vec::new(); for _ in 0..10_000 { bytes.clear(); b64.clear(); - decoded.clear(); + stream_decoded.clear(); + bulk_decoded.clear(); let size = rng.gen_range(1..(10 * BUF_SIZE)); bytes.extend(iter::repeat(0).take(size)); rng.fill_bytes(&mut bytes[..size]); assert_eq!(size, bytes.len()); - let engine = random_engine(&mut rng); + let engine = GeneralPurpose::new(&alphabet::STANDARD, random_config(&mut rng)); engine.encode_string(&bytes[..], &mut b64); // replace one byte, somewhere, with '*', which is invalid @@ -270,9 +273,8 @@ fn reports_invalid_byte_correctly() { let mut wrapped_reader = io::Cursor::new(b64_bytes.clone()); let mut decoder = DecoderReader::new(&mut wrapped_reader, &engine); - // some gymnastics to avoid double-moving the io::Error, which is not Copy let read_decode_err = decoder - .read_to_end(&mut decoded) + .read_to_end(&mut stream_decoded) .map_err(|e| { let kind = e.kind(); let inner = e @@ -283,8 +285,7 @@ fn reports_invalid_byte_correctly() { .err() .and_then(|o| o); - let mut bulk_buf = Vec::new(); - let bulk_decode_err = engine.decode_vec(&b64_bytes[..], &mut bulk_buf).err(); + let bulk_decode_err = engine.decode_vec(&b64_bytes[..], &mut bulk_decoded).err(); // it's tricky to predict where the invalid data's offset will be since if it's in the last // chunk it will be reported at the first padding location because it's treated as invalid @@ -296,6 +297,134 @@ fn reports_invalid_byte_correctly() { } } +#[test] +fn internal_padding_error_with_short_read_concatenated_texts_invalid_byte_error() { + let mut rng = rand::thread_rng(); + let mut bytes = Vec::new(); + let mut b64 = String::new(); + let mut reader_decoded = Vec::new(); + let mut bulk_decoded = Vec::new(); + + // encodes with padding, requires that padding be present so we don't get InvalidPadding + // just because padding is there at all + let engine = STANDARD; + + for _ in 0..10_000 { + bytes.clear(); + b64.clear(); + reader_decoded.clear(); + bulk_decoded.clear(); + + // at least 2 bytes so there can be a split point between bytes + let size = rng.gen_range(2..(10 * BUF_SIZE)); + bytes.resize(size, 0); + rng.fill_bytes(&mut bytes[..size]); + + // Concatenate two valid b64s, yielding padding in the middle. + // This avoids scenarios that are challenging to assert on, like random padding location + // that might be InvalidLastSymbol when decoded at certain buffer sizes but InvalidByte + // when done all at once. + let split = loop { + // find a split point that will produce padding on the first part + let s = rng.gen_range(1..size); + if s % 3 != 0 { + // short enough to need padding + break s; + }; + }; + + engine.encode_string(&bytes[..split], &mut b64); + assert!(b64.contains('='), "split: {}, b64: {}", split, b64); + let bad_byte_pos = b64.find('=').unwrap(); + engine.encode_string(&bytes[split..], &mut b64); + let b64_bytes = b64.as_bytes(); + + // short read to make it plausible for padding to happen on a read boundary + let read_len = rng.gen_range(1..10); + let mut wrapped_reader = ShortRead { + max_read_len: read_len, + delegate: io::Cursor::new(&b64_bytes), + }; + + let mut decoder = DecoderReader::new(&mut wrapped_reader, &engine); + + let read_decode_err = decoder + .read_to_end(&mut reader_decoded) + .map_err(|e| { + *e.into_inner() + .and_then(|e| e.downcast::().ok()) + .unwrap() + }) + .unwrap_err(); + + let bulk_decode_err = engine.decode_vec(b64_bytes, &mut bulk_decoded).unwrap_err(); + + assert_eq!( + bulk_decode_err, + read_decode_err, + "read len: {}, bad byte pos: {}, b64: {}", + read_len, + bad_byte_pos, + std::str::from_utf8(b64_bytes).unwrap() + ); + assert_eq!( + DecodeError::InvalidByte( + split / 3 * 4 + + match split % 3 { + 1 => 2, + 2 => 3, + _ => unreachable!(), + }, + PAD_BYTE + ), + read_decode_err + ); + } +} + +#[test] +fn internal_padding_anywhere_error() { + let mut rng = rand::thread_rng(); + let mut bytes = Vec::new(); + let mut b64 = String::new(); + let mut reader_decoded = Vec::new(); + + // encodes with padding, requires that padding be present so we don't get InvalidPadding + // just because padding is there at all + let engine = STANDARD; + + for _ in 0..10_000 { + bytes.clear(); + b64.clear(); + reader_decoded.clear(); + + bytes.resize(10 * BUF_SIZE, 0); + rng.fill_bytes(&mut bytes[..]); + + // Just shove a padding byte in there somewhere. + // The specific error to expect is challenging to predict precisely because it + // will vary based on the position of the padding in the quad and the read buffer + // length, but SOMETHING should go wrong. + + engine.encode_string(&bytes[..], &mut b64); + let mut b64_bytes = b64.as_bytes().to_vec(); + // put padding somewhere other than the last quad + b64_bytes[rng.gen_range(0..bytes.len() - 4)] = PAD_BYTE; + + // short read to make it plausible for padding to happen on a read boundary + let read_len = rng.gen_range(1..10); + let mut wrapped_reader = ShortRead { + max_read_len: read_len, + delegate: io::Cursor::new(&b64_bytes), + }; + + let mut decoder = DecoderReader::new(&mut wrapped_reader, &engine); + + let result = decoder.read_to_end(&mut reader_decoded); + assert!(result.is_err()); + } +} + fn consume_with_short_reads_and_validate( rng: &mut rand::rngs::ThreadRng, expected_bytes: &[u8], @@ -344,3 +473,15 @@ impl<'a, 'b, R: io::Read, N: rand::Rng> io::Read for RandomShortRead<'a, 'b, R, self.delegate.read(&mut buf[..effective_len]) } } + +struct ShortRead { + delegate: R, + max_read_len: usize, +} + +impl io::Read for ShortRead { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let len = self.max_read_len.max(buf.len()); + self.delegate.read(&mut buf[..len]) + } +} diff --git a/third_party/rust/base64/src/write/encoder_string_writer.rs b/third_party/rust/base64/src/write/encoder_string_writer.rs index 9394dc9bf777..9c02bcde84fb 100644 --- a/third_party/rust/base64/src/write/encoder_string_writer.rs +++ b/third_party/rust/base64/src/write/encoder_string_writer.rs @@ -44,11 +44,6 @@ use std::io; /// assert_eq!("base64: YXNkZg==", &buf); /// ``` /// -/// # Panics -/// -/// Calling `write()` (or related methods) or `finish()` after `finish()` has completed without -/// error is invalid and will panic. -/// /// # Performance /// /// Because it has to validate that the base64 is UTF-8, it is about 80% as fast as writing plain @@ -144,6 +139,7 @@ mod tests { engine::Engine, tests::random_engine, write::encoder_string_writer::EncoderStringWriter, }; use rand::Rng; + use std::cmp; use std::io::Write; #[test] @@ -158,9 +154,8 @@ mod tests { orig_data.clear(); normal_encoded.clear(); - for _ in 0..size { - orig_data.push(rng.gen()); - } + orig_data.resize(size, 0); + rng.fill(&mut orig_data[..]); let engine = random_engine(&mut rng); engine.encode_string(&orig_data, &mut normal_encoded); @@ -172,6 +167,40 @@ mod tests { let stream_encoded = stream_encoder.into_inner(); + assert_eq!(normal_encoded, stream_encoded); + } + } + #[test] + fn incremental_writes() { + let mut rng = rand::thread_rng(); + let mut orig_data = Vec::::new(); + let mut normal_encoded = String::new(); + + let size = 5_000; + + for _ in 0..size { + orig_data.clear(); + normal_encoded.clear(); + + orig_data.resize(size, 0); + rng.fill(&mut orig_data[..]); + + let engine = random_engine(&mut rng); + engine.encode_string(&orig_data, &mut normal_encoded); + + let mut stream_encoder = EncoderStringWriter::new(&engine); + // write small nibbles of data + let mut offset = 0; + while offset < size { + let nibble_size = cmp::min(rng.gen_range(0..=64), size - offset); + let len = stream_encoder + .write(&orig_data[offset..offset + nibble_size]) + .unwrap(); + offset += len; + } + + let stream_encoded = stream_encoder.into_inner(); + assert_eq!(normal_encoded, stream_encoded); } } diff --git a/third_party/rust/base64/src/write/encoder_tests.rs b/third_party/rust/base64/src/write/encoder_tests.rs index ce76d631e5cb..1f1a1650a6b4 100644 --- a/third_party/rust/base64/src/write/encoder_tests.rs +++ b/third_party/rust/base64/src/write/encoder_tests.rs @@ -358,7 +358,7 @@ fn retrying_writes_that_error_with_interrupted_works() { Ok(_) => break, Err(e) => match e.kind() { io::ErrorKind::Interrupted => continue, - _ => Err(e).unwrap(), // bail + _ => panic!("{:?}", e), // bail }, } } diff --git a/third_party/rust/base64/tests/encode.rs b/third_party/rust/base64/tests/encode.rs index 2e1f89348ae0..cf61e9ac900b 100644 --- a/third_party/rust/base64/tests/encode.rs +++ b/third_party/rust/base64/tests/encode.rs @@ -8,11 +8,7 @@ fn compare_encode(expected: &str, target: &[u8]) { #[test] fn encode_all_ascii() { - let mut ascii = Vec::::with_capacity(128); - - for i in 0..128 { - ascii.push(i); - } + let ascii: Vec = (0..=127).collect(); compare_encode( "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7P\ @@ -24,12 +20,7 @@ fn encode_all_ascii() { #[test] fn encode_all_bytes() { - let mut bytes = Vec::::with_capacity(256); - - for i in 0..255 { - bytes.push(i); - } - bytes.push(255); //bug with "overflowing" ranges? + let bytes: Vec = (0..=255).collect(); compare_encode( "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7P\ @@ -42,12 +33,7 @@ fn encode_all_bytes() { #[test] fn encode_all_bytes_url() { - let mut bytes = Vec::::with_capacity(256); - - for i in 0..255 { - bytes.push(i); - } - bytes.push(255); //bug with "overflowing" ranges? + let bytes: Vec = (0..=255).collect(); assert_eq!( "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0\ @@ -55,6 +41,6 @@ fn encode_all_bytes_url() { -AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq\ -wsbKztLW2t7i5uru8vb6_wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t_g4eLj5OXm5-jp6uvs7e7v8PHy\ 8_T19vf4-fr7_P3-_w==", - &engine::GeneralPurpose::new(&URL_SAFE, PAD).encode(&bytes) + &engine::GeneralPurpose::new(&URL_SAFE, PAD).encode(bytes) ); }