Bug 1852172 - Update base64 to 0.21.3. r=emilio,supply-chain-reviewers

Differential Revision: https://phabricator.services.mozilla.com/D187731
This commit is contained in:
Mike Hommey 2023-09-10 22:26:06 +00:00
parent 9c6f94b8b4
commit e8f5539bb1
23 changed files with 1053 additions and 459 deletions

22
Cargo.lock generated
View File

@ -300,7 +300,7 @@ version = "0.4.0-alpha.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6552b35e6a39b2e059b8107e45abf2adfa7cd3d20d294f5a315996da38559cbe"
dependencies = [
"base64 0.21.0",
"base64 0.21.3",
"bitflags 1.3.2",
"cfg-if 1.0.0",
"core-foundation",
@ -326,7 +326,7 @@ name = "authrs_bridge"
version = "0.1.0"
dependencies = [
"authenticator",
"base64 0.21.0",
"base64 0.21.3",
"log",
"moz_task",
"nserror",
@ -346,14 +346,14 @@ version = "1.1.0"
name = "base64"
version = "0.13.999"
dependencies = [
"base64 0.21.0",
"base64 0.21.3",
]
[[package]]
name = "base64"
version = "0.21.0"
version = "0.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53"
[[package]]
name = "basic-toml"
@ -645,7 +645,7 @@ checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
name = "cert_storage"
version = "0.0.1"
dependencies = [
"base64 0.21.0",
"base64 0.21.3",
"byteorder",
"crossbeam-utils",
"cstr",
@ -1012,7 +1012,7 @@ dependencies = [
name = "crypto_hash"
version = "0.1.0"
dependencies = [
"base64 0.21.0",
"base64 0.21.3",
"digest",
"libc",
"md-5",
@ -2005,7 +2005,7 @@ dependencies = [
name = "geckodriver"
version = "0.33.0"
dependencies = [
"base64 0.21.0",
"base64 0.21.3",
"chrono",
"clap",
"hyper",
@ -2525,7 +2525,7 @@ dependencies = [
name = "http3server"
version = "0.1.1"
dependencies = [
"base64 0.21.0",
"base64 0.21.3",
"bindgen 0.66.1",
"cfg-if 1.0.0",
"http",
@ -4335,7 +4335,7 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3675d093a7713f2b861f77b16c3c33fadd6de0a69bf7203014d938b9d5daa6f7"
dependencies = [
"base64 0.21.0",
"base64 0.21.3",
"byteorder",
"getrandom",
"serde",
@ -4642,7 +4642,7 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94"
dependencies = [
"base64 0.21.0",
"base64 0.21.3",
"bitflags 2.999.999",
"serde",
"serde_derive",

View File

@ -1189,6 +1189,21 @@ user-id = 101233 # le-automaton
start = "2020-09-28"
end = "2024-03-23"
[[audits.isrg.audits.base64]]
who = "Tim Geoghegan <timg@letsencrypt.org>"
criteria = "safe-to-deploy"
delta = "0.21.0 -> 0.21.1"
[[audits.isrg.audits.base64]]
who = "Brandon Pitman <bran@bran.land>"
criteria = "safe-to-deploy"
delta = "0.21.1 -> 0.21.2"
[[audits.isrg.audits.base64]]
who = "David Cook <dcook@divviup.org>"
criteria = "safe-to-deploy"
delta = "0.21.2 -> 0.21.3"
[[audits.isrg.audits.block-buffer]]
who = "David Cook <dcook@divviup.org>"
criteria = "safe-to-deploy"

View File

@ -1 +1 @@
{"files":{"Cargo.lock":"ee9a902629f1a6cc9308158590fc298c628f323383c4fb9a5ab9fd51011b268e","Cargo.toml":"37ffe4d4bdbd21f5a1cc78596abf6e704e4131dbec830fcd6c8bec33d4caf76b","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"0dd882e53de11566d50f8e8e2d5a651bcf3fabee4987d70f306233cf39094ba7","README.md":"99e61de0bafd6985761b596f9aa970dee5b4d0cfbfc05cd6565b5ffa139cb34f","RELEASE-NOTES.md":"c8b9e21adecb3a89928cdfbe55d184cc234e0e6cf8f61fb8ebab48982b3a6f9c","benches/benchmarks.rs":"faf63c3d83fe1568927288cfcc7f9bd4bd15c6b531450c53d2c064386fc5c652","clippy.toml":"ee3dedc35eb156cbfbe836692846cd492f59acc7443dd43cc6d71f087d80051c","examples/base64.rs":"8c48673029aeeb1e06a2ecfd237acf8ef24349990e97f6d2c4d0fa2af36c94b3","icon_CLion.svg":"cffa044ba75cb998ee3306991dc4a3755ec2f39ab95ddd4b74bc21988389020f","src/alphabet.rs":"420b5e23da0702c401489c53721696c5d5f69631f4ca9462f4c5ef3bdc77114e","src/chunked_encoder.rs":"4dfad5b47da1c35deaaa6ed2bb1efa51d98d6d9a7ca85a37ef4a02dfa846e723","src/decode.rs":"c293bf40a821795909a458aa8d7e76005a46e6953eed7ea284da1334e117ae74","src/display.rs":"31bf3e19274a0b80dd8948a81ea535944f756ef5b88736124c940f5fe1e8c71c","src/encode.rs":"34c800de1576f425cc48fa7ed9486d7c925cf7215dfc0634d2349c97b5199595","src/engine/general_purpose/decode.rs":"be237ac84b6a1deafd335896959302b7cf9337868cd718ebf7d249ccdc43a84f","src/engine/general_purpose/decode_suffix.rs":"797729cc1d56e9786f65e765cc5bb9ab2799f9140db4b412b919512fd7542355","src/engine/general_purpose/mod.rs":"2c6fbe61fae32800d30be5dc5bde429b8a07a5f027d0d2d1227a24ed13b1b461","src/engine/mod.rs":"7cd78bb317074a6e5439e272e4943d59d6bd47b149ed76b52e6f75b45909ce52","src/engine/naive.rs":"dcebcc41a0f4a156dd516ae89824748f5a4eedeabfe8d92f6f5bd3a6d5ceb5fb","src/engine/tests.rs":"4a8ff2ab7700b49e5b33606a93af04a5f93b18ca48e760ab6ced6337ba3a4847","src/lib.rs":"b4699408a9356f88fd8a3aeffae97e54e7a249afe5d919ecf9d4092d1c8efde1","src/prelude.rs":"f82fcf5e31921060929f9e10efb2868ba7339b085ee76fc5e7077f6030fbb2cc","src/read/decoder.rs":"65f03af1f4eb8d466a9a800dc6414678195b4ac6c579cd747b5632eee219f5a4","src/read/decoder_tests.rs":"ebf40a5722a58dbe74f013a4163ab20f5ce42ceb4beaefede07562079d596604","src/read/mod.rs":"e0b714eda02d16b1ffa6f78fd09b2f963e01c881b1f7c17b39db4e904be5e746","src/tests.rs":"90cb9f8a1ccb7c4ddc4f8618208e0031fc97e0df0e5aa466d6a5cf45d25967d8","src/write/encoder.rs":"c889c853249220fe2ddaeb77ee6e2ee2945f7db88cd6658ef89ff71b81255ea8","src/write/encoder_string_writer.rs":"ac3702b1a846fd0664e78d2dd82c939073ca00577d3201a8f1fbe17a9bf85e70","src/write/encoder_tests.rs":"39572f11fdf63af47f13bb58be280221322c669504a1b4a30a9181fe465e0f90","src/write/mod.rs":"73cd98dadc9d712b3fefd9449d97e825e097397441b90588e0051e4d3b0911b9","tests/encode.rs":"072f079f0250d542ff964e8e72b7d13e2336fbee7395367ff737e140c38ac459","tests/tests.rs":"78efcf0dc4bb6ae52f7a91fcad89e44e4dce578224c36b4e6c1c306459be8500"},"package":"a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"}
{"files":{"Cargo.lock":"8b23494504aad8968c998b54d9201489ac9e8d3fa792e7c86fc3c0d9419d5722","Cargo.toml":"d7c38395300d1d122c8cfa70f902469edac765aebbc93d39356c2c634d0dd182","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"0dd882e53de11566d50f8e8e2d5a651bcf3fabee4987d70f306233cf39094ba7","README.md":"df01f5b4317d601e7de86743f9818aec9196abf9e298f5e47679b7a966ecd945","RELEASE-NOTES.md":"8dd8a47135cb7660a4dd26fa13be26e97d1b4989ebcd2e0f3e6c35baec0a3821","benches/benchmarks.rs":"faf63c3d83fe1568927288cfcc7f9bd4bd15c6b531450c53d2c064386fc5c652","clippy.toml":"b26be4d15ed059985ce6994f11817fd7562046f46e460a0dc64dbb71cfc246d1","examples/base64.rs":"8c48673029aeeb1e06a2ecfd237acf8ef24349990e97f6d2c4d0fa2af36c94b3","icon_CLion.svg":"cffa044ba75cb998ee3306991dc4a3755ec2f39ab95ddd4b74bc21988389020f","src/alphabet.rs":"f0cba9462692db0bc9572e3d03c01ac77ff705fa9e664db4162da1a279a871e1","src/chunked_encoder.rs":"35b435042074f2ddd7dac6dae89c2ab6e5dfe173c73ccd1504c4f916f49151ef","src/decode.rs":"a43c5dc72524d49803f4170a84a975aa78040f457b4432ed22f04ab79bc1f3f4","src/display.rs":"31bf3e19274a0b80dd8948a81ea535944f756ef5b88736124c940f5fe1e8c71c","src/encode.rs":"8893d154878f894b3a8930e817ac8f0e4e8f67736f75200aa5eeee2ee813f626","src/engine/general_purpose/decode.rs":"ba8a76d333ab96dd07b3f84bd6d405d690d2d17e84bd0878f05245a82dc16853","src/engine/general_purpose/decode_suffix.rs":"71ceb066b73e8cc833916e2cedbf0a01b07c2f16e30b2b2f63aff1c823874b51","src/engine/general_purpose/mod.rs":"4acf7293e5bb83faf01edf2618bcb2b2aff2c2a3bcb85ddc815baa96e5751bb2","src/engine/mod.rs":"f1a307dadcab561719c868595cae5be0e37cdba337c2f8959d6ed1367030ed75","src/engine/naive.rs":"4ebd14e28502700d5de3e2aa193b6b384ad1189c6aa9368be3ab35775777aa4a","src/engine/tests.rs":"37bee2de07343bf5d37720f29cda291e8562f2363704e0ad91862d5991568d22","src/lib.rs":"b4699408a9356f88fd8a3aeffae97e54e7a249afe5d919ecf9d4092d1c8efde1","src/prelude.rs":"f82fcf5e31921060929f9e10efb2868ba7339b085ee76fc5e7077f6030fbb2cc","src/read/decoder.rs":"cc87daa4c52a23d1275352bccf07468baf2b60e90b2ac14f89a94254697cb83c","src/read/decoder_tests.rs":"cc3c2273867972a835f459073e3982f20a690c3b7d5f7546042e417d11c97a94","src/read/mod.rs":"e0b714eda02d16b1ffa6f78fd09b2f963e01c881b1f7c17b39db4e904be5e746","src/tests.rs":"90cb9f8a1ccb7c4ddc4f8618208e0031fc97e0df0e5aa466d6a5cf45d25967d8","src/write/encoder.rs":"c889c853249220fe2ddaeb77ee6e2ee2945f7db88cd6658ef89ff71b81255ea8","src/write/encoder_string_writer.rs":"0326c9d120369b9bbc35697b5b9b141bed24283374c93d5af1052eb042e47799","src/write/encoder_tests.rs":"28695a485b17cf5db73656aae5d90127f726e02c6d70efd83e5ab53a4cc17b38","src/write/mod.rs":"73cd98dadc9d712b3fefd9449d97e825e097397441b90588e0051e4d3b0911b9","tests/encode.rs":"ca8fb0c03f71563788cced06deb335f2c4bace0c875696662340c86ccedfdc7f","tests/tests.rs":"78efcf0dc4bb6ae52f7a91fcad89e44e4dce578224c36b4e6c1c306459be8500"},"package":"414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53"}

256
third_party/rust/base64/Cargo.lock generated vendored
View File

@ -2,6 +2,15 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "aho-corasick"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a"
dependencies = [
"memchr",
]
[[package]]
name = "anes"
version = "0.1.6"
@ -36,9 +45,10 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "base64"
version = "0.21.0"
version = "0.21.3"
dependencies = [
"criterion",
"lazy_static",
"rand",
"rstest",
"rstest_reuse",
@ -53,9 +63,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bumpalo"
version = "3.11.1"
version = "3.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
[[package]]
name = "cast"
@ -71,9 +81,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "ciborium"
version = "0.2.0"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926"
dependencies = [
"ciborium-io",
"ciborium-ll",
@ -82,15 +92,15 @@ dependencies = [
[[package]]
name = "ciborium-io"
version = "0.2.0"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656"
[[package]]
name = "ciborium-ll"
version = "0.2.0"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b"
dependencies = [
"ciborium-io",
"half",
@ -113,9 +123,9 @@ dependencies = [
[[package]]
name = "clap"
version = "3.2.23"
version = "3.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
dependencies = [
"bitflags",
"clap_lex",
@ -142,7 +152,7 @@ dependencies = [
"atty",
"cast",
"ciborium",
"clap 3.2.23",
"clap 3.2.25",
"criterion-plot",
"itertools",
"lazy_static",
@ -170,9 +180,9 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
version = "0.5.6"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
dependencies = [
"cfg-if",
"crossbeam-utils",
@ -180,9 +190,9 @@ dependencies = [
[[package]]
name = "crossbeam-deque"
version = "0.8.2"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
dependencies = [
"cfg-if",
"crossbeam-epoch",
@ -191,9 +201,9 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
version = "0.9.13"
version = "0.9.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
dependencies = [
"autocfg",
"cfg-if",
@ -204,24 +214,24 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.14"
version = "0.8.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
dependencies = [
"cfg-if",
]
[[package]]
name = "either"
version = "1.8.0"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
[[package]]
name = "getrandom"
version = "0.2.8"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
dependencies = [
"cfg-if",
"libc",
@ -260,18 +270,15 @@ dependencies = [
[[package]]
name = "hermit-abi"
version = "0.2.6"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
dependencies = [
"libc",
]
checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
[[package]]
name = "indexmap"
version = "1.9.2"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown",
@ -288,15 +295,15 @@ dependencies = [
[[package]]
name = "itoa"
version = "1.0.5"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
[[package]]
name = "js-sys"
version = "0.3.60"
version = "0.3.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
dependencies = [
"wasm-bindgen",
]
@ -309,52 +316,55 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.139"
version = "0.2.147"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
[[package]]
name = "log"
version = "0.4.17"
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if",
]
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "memchr"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memoffset"
version = "0.7.1"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
dependencies = [
"autocfg",
]
[[package]]
name = "num-traits"
version = "0.2.15"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.15.0"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi 0.2.6",
"hermit-abi 0.3.2",
"libc",
]
[[package]]
name = "once_cell"
version = "1.17.0"
version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "oorandom"
@ -364,15 +374,15 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "os_str_bytes"
version = "6.4.1"
version = "6.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac"
[[package]]
name = "plotters"
version = "0.3.4"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97"
checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45"
dependencies = [
"num-traits",
"plotters-backend",
@ -383,15 +393,15 @@ dependencies = [
[[package]]
name = "plotters-backend"
version = "0.3.4"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609"
[[package]]
name = "plotters-svg"
version = "0.3.3"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f"
checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab"
dependencies = [
"plotters-backend",
]
@ -411,7 +421,7 @@ dependencies = [
"proc-macro-error-attr",
"proc-macro2",
"quote",
"syn",
"syn 1.0.109",
"version_check",
]
@ -428,18 +438,18 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.49"
version = "1.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5"
checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.23"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
dependencies = [
"proc-macro2",
]
@ -476,9 +486,9 @@ dependencies = [
[[package]]
name = "rayon"
version = "1.6.1"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
dependencies = [
"either",
"rayon-core",
@ -486,9 +496,9 @@ dependencies = [
[[package]]
name = "rayon-core"
version = "1.10.1"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
@ -498,18 +508,32 @@ dependencies = [
[[package]]
name = "regex"
version = "1.7.0"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.6.28"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
[[package]]
name = "rstest"
@ -521,7 +545,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustc_version",
"syn",
"syn 1.0.109",
]
[[package]]
@ -532,7 +556,7 @@ checksum = "b29d3117bce27ea307d1fb7ce12c64ba11b3fd04311a42d32bc5f0072e6e3d4d"
dependencies = [
"quote",
"rustc_version",
"syn",
"syn 1.0.109",
]
[[package]]
@ -546,9 +570,9 @@ dependencies = [
[[package]]
name = "ryu"
version = "1.0.12"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
[[package]]
name = "same-file"
@ -561,41 +585,41 @@ dependencies = [
[[package]]
name = "scopeguard"
version = "1.1.0"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "semver"
version = "1.0.16"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
[[package]]
name = "serde"
version = "1.0.152"
version = "1.0.188"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.152"
version = "1.0.188"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
]
[[package]]
name = "serde_json"
version = "1.0.91"
version = "1.0.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360"
dependencies = [
"itoa",
"ryu",
@ -629,14 +653,25 @@ dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
"syn",
"syn 1.0.109",
]
[[package]]
name = "syn"
version = "1.0.107"
version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
dependencies = [
"proc-macro2",
"quote",
@ -670,15 +705,15 @@ dependencies = [
[[package]]
name = "unicode-ident"
version = "1.0.6"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
[[package]]
name = "unicode-segmentation"
version = "1.10.0"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"
checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
[[package]]
name = "unicode-width"
@ -700,12 +735,11 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "walkdir"
version = "2.3.2"
version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698"
dependencies = [
"same-file",
"winapi",
"winapi-util",
]
@ -717,9 +751,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.83"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@ -727,24 +761,24 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.83"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.83"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -752,28 +786,28 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.83"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
"syn",
"syn 2.0.29",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.83"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
[[package]]
name = "web-sys"
version = "0.3.60"
version = "0.3.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"
checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b"
dependencies = [
"js-sys",
"wasm-bindgen",

View File

@ -10,10 +10,10 @@
# See Cargo.toml.orig for the original contents.
[package]
edition = "2021"
rust-version = "1.57.0"
edition = "2018"
rust-version = "1.48.0"
name = "base64"
version = "0.21.0"
version = "0.21.3"
authors = [
"Alice Maz <alice@alicemaz.com>",
"Marshall Pierce <marshall@mpierce.org>",
@ -32,8 +32,11 @@ categories = ["encoding"]
license = "MIT OR Apache-2.0"
repository = "https://github.com/marshallpierce/rust-base64"
[package.metadata.docs.rs]
rustdoc-args = ["--generate-link-to-definition"]
[profile.bench]
debug = true
debug = 2
[profile.test]
opt-level = 3
@ -45,6 +48,9 @@ harness = false
[dev-dependencies.criterion]
version = "0.4.0"
[dev-dependencies.lazy_static]
version = "1.4.0"
[dev-dependencies.rand]
version = "0.8.5"
features = ["small_rng"]

View File

@ -63,7 +63,7 @@ optionally may allow other behaviors.
## Rust version compatibility
The minimum supported Rust version is 1.57.0.
The minimum supported Rust version is 1.48.0.
# Contributing
@ -76,10 +76,10 @@ free time to give each PR the attention it deserves. I will get to everyone even
## Developing
Benchmarks are in `benches/`. Running them requires nightly rust, but `rustup` makes it easy:
Benchmarks are in `benches/`.
```bash
rustup run nightly cargo bench
cargo bench
```
## no_std
@ -92,12 +92,12 @@ to bring back the support for heap allocations.
## Profiling
On Linux, you can use [perf](https://perf.wiki.kernel.org/index.php/Main_Page) for profiling. Then compile the
benchmarks with `rustup nightly run cargo bench --no-run`.
benchmarks with `cargo bench --no-run`.
Run the benchmark binary with `perf` (shown here filtering to one particular benchmark, which will make the results
easier to read). `perf` is only available to the root user on most systems as it fiddles with event counters in your
CPU, so use `sudo`. We need to run the actual benchmark binary, hence the path into `target`. You can see the actual
full path with `rustup run nightly cargo bench -v`; it will print out the commands it runs. If you use the exact path
full path with `cargo bench -v`; it will print out the commands it runs. If you use the exact path
that `bench` outputs, make sure you get the one that's for the benchmarks, not the tests. You may also want
to `cargo clean` so you have only one `benchmarks-` binary (they tend to accumulate).

View File

@ -1,8 +1,26 @@
# 0.21.3
- Implement `source` instead of `cause` on Error types
- Roll back MSRV to 1.48.0 so Debian can continue to live in a time warp
- Slightly faster chunked encoding for short inputs
- Decrease binary size
# 0.21.2
- Rollback MSRV to 1.57.0 -- only dev dependencies need 1.60, not the main code
# 0.21.1
- Remove the possibility of panicking during decoded length calculations
- `DecoderReader` no longer sometimes erroneously ignores padding [#226](https://github.com/marshallpierce/rust-base64/issues/226)
## Breaking changes
- `Engine.internal_decode` return type changed
- Update MSRV to 1.60.0
# 0.21.0
(not yet released)
## Migration
### Functions

View File

@ -1 +1 @@
msrv = "1.57.0"
msrv = "1.48.0"

View File

@ -1,7 +1,7 @@
//! Provides [Alphabet] and constants for alphabets commonly used in the wild.
use crate::PAD_BYTE;
use core::fmt;
use core::{convert, fmt};
#[cfg(any(feature = "std", test))]
use std::error;
@ -12,6 +12,10 @@ const ALPHABET_SIZE: usize = 64;
/// Common alphabets are provided as constants, and custom alphabets
/// can be made via `from_str` or the `TryFrom<str>` implementation.
///
/// # Examples
///
/// Building and using a custom Alphabet:
///
/// ```
/// let custom = base64::alphabet::Alphabet::new("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/").unwrap();
///
@ -19,6 +23,33 @@ const ALPHABET_SIZE: usize = 64;
/// &custom,
/// base64::engine::general_purpose::PAD);
/// ```
///
/// Building a const:
///
/// ```
/// use base64::alphabet::Alphabet;
///
/// static CUSTOM: Alphabet = {
/// // Result::unwrap() isn't const yet, but panic!() is OK
/// match Alphabet::new("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/") {
/// Ok(x) => x,
/// Err(_) => panic!("creation of alphabet failed"),
/// }
/// };
/// ```
///
/// Building a lazy_static:
///
/// ```
/// use base64::{
/// alphabet::Alphabet,
/// engine::{general_purpose::GeneralPurpose, GeneralPurposeConfig},
/// };
///
/// lazy_static::lazy_static! {
/// static ref CUSTOM: Alphabet = Alphabet::new("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/").unwrap();
/// }
/// ```
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Alphabet {
pub(crate) symbols: [u8; ALPHABET_SIZE],
@ -93,7 +124,7 @@ impl Alphabet {
}
}
impl TryFrom<&str> for Alphabet {
impl convert::TryFrom<&str> for Alphabet {
type Error = ParseAlphabetError;
fn try_from(value: &str) -> Result<Self, Self::Error> {
@ -171,7 +202,7 @@ pub const BIN_HEX: Alphabet = Alphabet::from_str_unchecked(
#[cfg(test)]
mod tests {
use crate::alphabet::*;
use std::convert::TryFrom as _;
use core::convert::TryFrom as _;
#[test]
fn detects_duplicate_start() {

View File

@ -1,12 +1,12 @@
use crate::{
encode::add_padding,
engine::{Config, Engine},
};
#[cfg(any(feature = "alloc", feature = "std", test))]
use alloc::string::String;
use core::cmp;
#[cfg(any(feature = "alloc", feature = "std", test))]
use core::str;
use crate::encode::add_padding;
use crate::engine::{Config, Engine};
/// The output mechanism for ChunkedEncoder's encoded bytes.
pub trait Sink {
type Error;
@ -15,71 +15,37 @@ pub trait Sink {
fn write_encoded_bytes(&mut self, encoded: &[u8]) -> Result<(), Self::Error>;
}
const BUF_SIZE: usize = 1024;
/// A base64 encoder that emits encoded bytes in chunks without heap allocation.
pub struct ChunkedEncoder<'e, E: Engine + ?Sized> {
engine: &'e E,
max_input_chunk_len: usize,
}
impl<'e, E: Engine + ?Sized> ChunkedEncoder<'e, E> {
pub fn new(engine: &'e E) -> ChunkedEncoder<'e, E> {
ChunkedEncoder {
engine,
max_input_chunk_len: max_input_length(BUF_SIZE, engine.config().encode_padding()),
}
ChunkedEncoder { engine }
}
pub fn encode<S: Sink>(&self, bytes: &[u8], sink: &mut S) -> Result<(), S::Error> {
let mut encode_buf: [u8; BUF_SIZE] = [0; BUF_SIZE];
let mut input_index = 0;
const BUF_SIZE: usize = 1024;
const CHUNK_SIZE: usize = BUF_SIZE / 4 * 3;
while input_index < bytes.len() {
// either the full input chunk size, or it's the last iteration
let input_chunk_len = cmp::min(self.max_input_chunk_len, bytes.len() - input_index);
let chunk = &bytes[input_index..(input_index + input_chunk_len)];
let mut b64_bytes_written = self.engine.internal_encode(chunk, &mut encode_buf);
input_index += input_chunk_len;
let more_input_left = input_index < bytes.len();
if self.engine.config().encode_padding() && !more_input_left {
// no more input, add padding if needed. Buffer will have room because
// max_input_length leaves room for it.
b64_bytes_written += add_padding(bytes.len(), &mut encode_buf[b64_bytes_written..]);
let mut buf = [0; BUF_SIZE];
for chunk in bytes.chunks(CHUNK_SIZE) {
let mut len = self.engine.internal_encode(chunk, &mut buf);
if chunk.len() != CHUNK_SIZE && self.engine.config().encode_padding() {
// Final, potentially partial, chunk.
// Only need to consider if padding is needed on a partial chunk since full chunk
// is a multiple of 3, which therefore won't be padded.
// Pad output to multiple of four bytes if required by config.
len += add_padding(len, &mut buf[len..]);
}
sink.write_encoded_bytes(&encode_buf[0..b64_bytes_written])?;
sink.write_encoded_bytes(&buf[..len])?;
}
Ok(())
}
}
/// Calculate the longest input that can be encoded for the given output buffer size.
///
/// If the config requires padding, two bytes of buffer space will be set aside so that the last
/// chunk of input can be encoded safely.
///
/// The input length will always be a multiple of 3 so that no encoding state has to be carried over
/// between chunks.
fn max_input_length(encoded_buf_len: usize, padded: bool) -> usize {
let effective_buf_len = if padded {
// make room for padding
encoded_buf_len
.checked_sub(2)
.expect("Don't use a tiny buffer")
} else {
encoded_buf_len
};
// No padding, so just normal base64 expansion.
(effective_buf_len / 4) * 3
}
// A really simple sink that just appends to a string
#[cfg(any(feature = "alloc", feature = "std", test))]
pub(crate) struct StringSink<'a> {
@ -151,38 +117,13 @@ pub mod tests {
chunked_encode_matches_normal_encode_random(&helper);
}
#[test]
fn max_input_length_no_pad() {
assert_eq!(768, max_input_length(1024, false));
}
#[test]
fn max_input_length_with_pad_decrements_one_triple() {
assert_eq!(765, max_input_length(1024, true));
}
#[test]
fn max_input_length_with_pad_one_byte_short() {
assert_eq!(765, max_input_length(1025, true));
}
#[test]
fn max_input_length_with_pad_fits_exactly() {
assert_eq!(768, max_input_length(1026, true));
}
#[test]
fn max_input_length_cant_use_extra_single_encoded_byte() {
assert_eq!(300, max_input_length(401, false));
}
pub fn chunked_encode_matches_normal_encode_random<S: SinkTestHelper>(sink_test_helper: &S) {
let mut input_buf: Vec<u8> = Vec::new();
let mut output_buf = String::new();
let mut rng = rand::rngs::SmallRng::from_entropy();
let input_len_range = Uniform::new(1, 10_000);
for _ in 0..5_000 {
for _ in 0..20_000 {
input_buf.clear();
output_buf.clear();

View File

@ -41,11 +41,7 @@ impl fmt::Display for DecodeError {
}
#[cfg(any(feature = "std", test))]
impl error::Error for DecodeError {
fn cause(&self) -> Option<&dyn error::Error> {
None
}
}
impl error::Error for DecodeError {}
/// Errors that can occur while decoding into a slice.
#[derive(Clone, Debug, PartialEq, Eq)]
@ -69,7 +65,7 @@ impl fmt::Display for DecodeSliceError {
#[cfg(any(feature = "std", test))]
impl error::Error for DecodeSliceError {
fn cause(&self) -> Option<&dyn error::Error> {
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
match self {
DecodeSliceError::DecodeError(e) => Some(e),
DecodeSliceError::OutputSliceTooSmall => None,
@ -148,11 +144,6 @@ pub fn decode_engine_slice<E: Engine, T: AsRef<[u8]>>(
/// // start of the next quad of encoded symbols
/// assert_eq!(6, decoded_len_estimate(5));
/// ```
///
/// # Panics
///
/// Panics if decoded length estimation overflows.
/// This would happen for sizes within a few bytes of the maximum value of `usize`.
pub fn decoded_len_estimate(encoded_len: usize) -> usize {
STANDARD
.internal_decoded_len_estimate(encoded_len)

View File

@ -77,7 +77,7 @@ pub(crate) fn encode_with_padding<E: Engine + ?Sized>(
let b64_bytes_written = engine.internal_encode(input, output);
let padding_bytes = if engine.config().encode_padding() {
add_padding(input.len(), &mut output[b64_bytes_written..])
add_padding(b64_bytes_written, &mut output[b64_bytes_written..])
} else {
0
};
@ -117,20 +117,20 @@ pub fn encoded_len(bytes_len: usize, padding: bool) -> Option<usize> {
}
/// Write padding characters.
/// `input_len` is the size of the original, not encoded, input.
/// `unpadded_output_len` is the size of the unpadded but base64 encoded data.
/// `output` is the slice where padding should be written, of length at least 2.
///
/// Returns the number of padding bytes written.
pub(crate) fn add_padding(input_len: usize, output: &mut [u8]) -> usize {
// TODO base on encoded len to use cheaper mod by 4 (aka & 7)
let rem = input_len % 3;
let mut bytes_written = 0;
for _ in 0..((3 - rem) % 3) {
output[bytes_written] = PAD_BYTE;
bytes_written += 1;
pub(crate) fn add_padding(unpadded_output_len: usize, output: &mut [u8]) -> usize {
let pad_bytes = (4 - (unpadded_output_len % 4)) % 4;
// for just a couple bytes, this has better performance than using
// .fill(), or iterating over mutable refs, which call memset()
#[allow(clippy::needless_range_loop)]
for i in 0..pad_bytes {
output[i] = PAD_BYTE;
}
bytes_written
pad_bytes
}
/// Errors that can occur while encoding into a slice.
@ -149,11 +149,7 @@ impl fmt::Display for EncodeSliceError {
}
#[cfg(any(feature = "std", test))]
impl error::Error for EncodeSliceError {
fn cause(&self) -> Option<&dyn error::Error> {
None
}
}
impl error::Error for EncodeSliceError {}
#[cfg(test)]
mod tests {
@ -434,18 +430,18 @@ mod tests {
let mut rng = rand::rngs::SmallRng::from_entropy();
// cover our bases for length % 3
for input_len in 0..10 {
// cover our bases for length % 4
for unpadded_output_len in 0..20 {
output.clear();
// fill output with random
for _ in 0..10 {
for _ in 0..100 {
output.push(rng.gen());
}
let orig_output_buf = output.clone();
let bytes_written = add_padding(input_len, &mut output);
let bytes_written = add_padding(unpadded_output_len, &mut output);
// make sure the part beyond bytes_written is the same garbage it was before
assert_eq!(orig_output_buf[bytes_written..], output[bytes_written..]);

View File

@ -1,5 +1,5 @@
use crate::{
engine::{general_purpose::INVALID_VALUE, DecodeEstimate, DecodePaddingMode},
engine::{general_purpose::INVALID_VALUE, DecodeEstimate, DecodeMetadata, DecodePaddingMode},
DecodeError, PAD_BYTE,
};
@ -30,16 +30,11 @@ pub struct GeneralPurposeEstimate {
impl GeneralPurposeEstimate {
pub(crate) fn new(encoded_len: usize) -> Self {
// Formulas that won't overflow
Self {
num_chunks: encoded_len
.checked_add(INPUT_CHUNK_LEN - 1)
.expect("Overflow when calculating number of chunks in input")
/ INPUT_CHUNK_LEN,
decoded_len_estimate: encoded_len
.checked_add(3)
.expect("Overflow when calculating decoded len estimate")
/ 4
* 3,
num_chunks: encoded_len / INPUT_CHUNK_LEN
+ (encoded_len % INPUT_CHUNK_LEN > 0) as usize,
decoded_len_estimate: (encoded_len / 4 + (encoded_len % 4 > 0) as usize) * 3,
}
}
}
@ -51,7 +46,7 @@ impl DecodeEstimate for GeneralPurposeEstimate {
}
/// Helper to avoid duplicating num_chunks calculation, which is costly on short inputs.
/// Returns the number of bytes written, or an error.
/// Returns the decode metadata, or an error.
// We're on the fragile edge of compiler heuristics here. If this is not inlined, slow. If this is
// inlined(always), a different slow. plain ol' inline makes the benchmarks happiest at the moment,
// but this is fragile and the best setting changes with only minor code modifications.
@ -63,7 +58,7 @@ pub(crate) fn decode_helper(
decode_table: &[u8; 256],
decode_allow_trailing_bits: bool,
padding_mode: DecodePaddingMode,
) -> Result<usize, DecodeError> {
) -> Result<DecodeMetadata, DecodeError> {
let remainder_len = input.len() % INPUT_CHUNK_LEN;
// Because the fast decode loop writes in groups of 8 bytes (unrolled to
@ -345,4 +340,44 @@ mod tests {
decode_chunk(&input[..], 0, &STANDARD.decode_table, &mut output).unwrap();
assert_eq!(&vec![b'f', b'o', b'o', b'b', b'a', b'r', 0, 0], &output);
}
#[test]
fn estimate_short_lengths() {
for (range, (num_chunks, decoded_len_estimate)) in [
(0..=0, (0, 0)),
(1..=4, (1, 3)),
(5..=8, (1, 6)),
(9..=12, (2, 9)),
(13..=16, (2, 12)),
(17..=20, (3, 15)),
] {
for encoded_len in range {
let estimate = GeneralPurposeEstimate::new(encoded_len);
assert_eq!(num_chunks, estimate.num_chunks);
assert_eq!(decoded_len_estimate, estimate.decoded_len_estimate);
}
}
}
#[test]
fn estimate_via_u128_inflation() {
// cover both ends of usize
(0..1000)
.chain(usize::MAX - 1000..=usize::MAX)
.for_each(|encoded_len| {
// inflate to 128 bit type to be able to safely use the easy formulas
let len_128 = encoded_len as u128;
let estimate = GeneralPurposeEstimate::new(encoded_len);
assert_eq!(
((len_128 + (INPUT_CHUNK_LEN - 1) as u128) / (INPUT_CHUNK_LEN as u128))
as usize,
estimate.num_chunks
);
assert_eq!(
((len_128 + 3) / 4 * 3) as usize,
estimate.decoded_len_estimate
);
})
}
}

View File

@ -1,13 +1,13 @@
use crate::{
engine::{general_purpose::INVALID_VALUE, DecodePaddingMode},
engine::{general_purpose::INVALID_VALUE, DecodeMetadata, DecodePaddingMode},
DecodeError, PAD_BYTE,
};
/// Decode the last 1-8 bytes, checking for trailing set bits and padding per the provided
/// parameters.
///
/// Returns the total number of bytes decoded, including the ones indicated as already written by
/// `output_index`.
/// Returns the decode metadata representing the total number of bytes decoded, including the ones
/// indicated as already written by `output_index`.
pub(crate) fn decode_suffix(
input: &[u8],
input_index: usize,
@ -16,7 +16,7 @@ pub(crate) fn decode_suffix(
decode_table: &[u8; 256],
decode_allow_trailing_bits: bool,
padding_mode: DecodePaddingMode,
) -> Result<usize, DecodeError> {
) -> Result<DecodeMetadata, DecodeError> {
// Decode any leftovers that aren't a complete input block of 8 bytes.
// Use a u64 as a stack-resident 8 byte buffer.
let mut leftover_bits: u64 = 0;
@ -157,5 +157,12 @@ pub(crate) fn decode_suffix(
leftover_bits_appended_to_buf += 8;
}
Ok(output_index)
Ok(DecodeMetadata::new(
output_index,
if padding_bytes > 0 {
Some(input_index + first_padding_index)
} else {
None
},
))
}

View File

@ -2,13 +2,14 @@
use crate::{
alphabet,
alphabet::Alphabet,
engine::{Config, DecodePaddingMode},
engine::{Config, DecodeMetadata, DecodePaddingMode},
DecodeError,
};
use core::convert::TryInto;
mod decode;
pub(crate) mod decode_suffix;
pub use decode::GeneralPurposeEstimate;
pub(crate) const INVALID_VALUE: u8 = 255;
@ -170,7 +171,7 @@ impl super::Engine for GeneralPurpose {
input: &[u8],
output: &mut [u8],
estimate: Self::DecodeEstimate,
) -> Result<usize, DecodeError> {
) -> Result<DecodeMetadata, DecodeError> {
decode::decode_helper(
input,
estimate,

View File

@ -62,10 +62,6 @@ pub trait Engine: Send + Sync {
/// As an optimization to prevent the decoded length from being calculated twice, it is
/// sometimes helpful to have a conservative estimate of the decoded size before doing the
/// decoding, so this calculation is done separately and passed to [Engine::decode()] as needed.
///
/// # Panics
///
/// Panics if decoded length estimation overflows.
#[doc(hidden)]
fn internal_decoded_len_estimate(&self, input_len: usize) -> Self::DecodeEstimate;
@ -77,8 +73,6 @@ pub trait Engine: Send + Sync {
/// `decode_estimate` is the result of [Engine::internal_decoded_len_estimate()], which is passed in to avoid
/// calculating it again (expensive on short inputs).`
///
/// Returns the number of bytes written to `output`.
///
/// Each complete 4-byte chunk of encoded data decodes to 3 bytes of decoded data, but this
/// function must also handle the final possibly partial chunk.
/// If the input length is not a multiple of 4, or uses padding bytes to reach a multiple of 4,
@ -99,7 +93,7 @@ pub trait Engine: Send + Sync {
input: &[u8],
output: &mut [u8],
decode_estimate: Self::DecodeEstimate,
) -> Result<usize, DecodeError>;
) -> Result<DecodeMetadata, DecodeError>;
/// Returns the config for this engine.
fn config(&self) -> &Self::Config;
@ -120,14 +114,23 @@ pub trait Engine: Send + Sync {
///
/// let b64_url = CUSTOM_ENGINE.encode(b"hello internet~");
#[cfg(any(feature = "alloc", feature = "std", test))]
#[inline]
fn encode<T: AsRef<[u8]>>(&self, input: T) -> String {
let encoded_size = encoded_len(input.as_ref().len(), self.config().encode_padding())
.expect("integer overflow when calculating buffer size");
let mut buf = vec![0; encoded_size];
fn inner<E>(engine: &E, input_bytes: &[u8]) -> String
where
E: Engine + ?Sized,
{
let encoded_size = encoded_len(input_bytes.len(), engine.config().encode_padding())
.expect("integer overflow when calculating buffer size");
encode_with_padding(input.as_ref(), &mut buf[..], self, encoded_size);
let mut buf = vec![0; encoded_size];
String::from_utf8(buf).expect("Invalid UTF8")
encode_with_padding(input_bytes, &mut buf[..], engine, encoded_size);
String::from_utf8(buf).expect("Invalid UTF8")
}
inner(self, input.as_ref())
}
/// Encode arbitrary octets as base64 into a supplied `String`.
@ -151,16 +154,20 @@ pub trait Engine: Send + Sync {
/// }
/// ```
#[cfg(any(feature = "alloc", feature = "std", test))]
#[inline]
fn encode_string<T: AsRef<[u8]>>(&self, input: T, output_buf: &mut String) {
let input_bytes = input.as_ref();
fn inner<E>(engine: &E, input_bytes: &[u8], output_buf: &mut String)
where
E: Engine + ?Sized,
{
let mut sink = chunked_encoder::StringSink::new(output_buf);
chunked_encoder::ChunkedEncoder::new(self)
chunked_encoder::ChunkedEncoder::new(engine)
.encode(input_bytes, &mut sink)
.expect("Writing to a String shouldn't fail");
}
inner(self, input.as_ref(), output_buf)
}
/// Encode arbitrary octets as base64 into a supplied slice.
@ -185,29 +192,38 @@ pub trait Engine: Send + Sync {
///
/// assert_eq!(s, general_purpose::STANDARD.decode(&buf).unwrap().as_slice());
/// ```
#[inline]
fn encode_slice<T: AsRef<[u8]>>(
&self,
input: T,
output_buf: &mut [u8],
) -> Result<usize, EncodeSliceError> {
let input_bytes = input.as_ref();
fn inner<E>(
engine: &E,
input_bytes: &[u8],
output_buf: &mut [u8],
) -> Result<usize, EncodeSliceError>
where
E: Engine + ?Sized,
{
let encoded_size = encoded_len(input_bytes.len(), engine.config().encode_padding())
.expect("usize overflow when calculating buffer size");
let encoded_size = encoded_len(input_bytes.len(), self.config().encode_padding())
.expect("usize overflow when calculating buffer size");
if output_buf.len() < encoded_size {
return Err(EncodeSliceError::OutputSliceTooSmall);
}
if output_buf.len() < encoded_size {
return Err(EncodeSliceError::OutputSliceTooSmall);
let b64_output = &mut output_buf[0..encoded_size];
encode_with_padding(input_bytes, b64_output, engine, encoded_size);
Ok(encoded_size)
}
let b64_output = &mut output_buf[0..encoded_size];
encode_with_padding(input_bytes, b64_output, self, encoded_size);
Ok(encoded_size)
inner(self, input.as_ref(), output_buf)
}
/// Decode from string reference as octets using the specified [Engine].
/// Returns a `Result` containing a `Vec<u8>`.
/// Decode the input into a new `Vec`.
///
/// # Example
///
@ -225,25 +241,30 @@ pub trait Engine: Send + Sync {
/// .decode("aGVsbG8gaW50ZXJuZXR-Cg").unwrap();
/// println!("{:?}", bytes_url);
/// ```
///
/// # Panics
///
/// Panics if decoded length estimation overflows.
/// This would happen for sizes within a few bytes of the maximum value of `usize`.
#[cfg(any(feature = "alloc", feature = "std", test))]
#[inline]
fn decode<T: AsRef<[u8]>>(&self, input: T) -> Result<Vec<u8>, DecodeError> {
let input_bytes = input.as_ref();
fn inner<E>(engine: &E, input_bytes: &[u8]) -> Result<Vec<u8>, DecodeError>
where
E: Engine + ?Sized,
{
let estimate = engine.internal_decoded_len_estimate(input_bytes.len());
let mut buffer = vec![0; estimate.decoded_len_estimate()];
let estimate = self.internal_decoded_len_estimate(input_bytes.len());
let mut buffer = vec![0; estimate.decoded_len_estimate()];
let bytes_written = engine
.internal_decode(input_bytes, &mut buffer, estimate)?
.decoded_len;
let bytes_written = self.internal_decode(input_bytes, &mut buffer, estimate)?;
buffer.truncate(bytes_written);
buffer.truncate(bytes_written);
Ok(buffer)
Ok(buffer)
}
inner(self, input.as_ref())
}
/// Decode from string reference as octets.
/// Decode the `input` into the supplied `buffer`.
///
/// Writes into the supplied `Vec`, which may allocate if its internal buffer isn't big enough.
/// Returns a `Result` containing an empty tuple, aka `()`.
///
@ -272,39 +293,45 @@ pub trait Engine: Send + Sync {
/// println!("{:?}", buffer);
/// }
/// ```
///
/// # Panics
///
/// Panics if decoded length estimation overflows.
/// This would happen for sizes within a few bytes of the maximum value of `usize`.
#[cfg(any(feature = "alloc", feature = "std", test))]
#[inline]
fn decode_vec<T: AsRef<[u8]>>(
&self,
input: T,
buffer: &mut Vec<u8>,
) -> Result<(), DecodeError> {
let input_bytes = input.as_ref();
fn inner<E>(engine: &E, input_bytes: &[u8], buffer: &mut Vec<u8>) -> Result<(), DecodeError>
where
E: Engine + ?Sized,
{
let starting_output_len = buffer.len();
let estimate = engine.internal_decoded_len_estimate(input_bytes.len());
let starting_output_len = buffer.len();
let total_len_estimate = estimate
.decoded_len_estimate()
.checked_add(starting_output_len)
.expect("Overflow when calculating output buffer length");
let estimate = self.internal_decoded_len_estimate(input_bytes.len());
let total_len_estimate = estimate
.decoded_len_estimate()
.checked_add(starting_output_len)
.expect("Overflow when calculating output buffer length");
buffer.resize(total_len_estimate, 0);
buffer.resize(total_len_estimate, 0);
let buffer_slice = &mut buffer.as_mut_slice()[starting_output_len..];
let bytes_written = self.internal_decode(input_bytes, buffer_slice, estimate)?;
let buffer_slice = &mut buffer.as_mut_slice()[starting_output_len..];
buffer.truncate(starting_output_len + bytes_written);
let bytes_written = engine
.internal_decode(input_bytes, buffer_slice, estimate)?
.decoded_len;
Ok(())
buffer.truncate(starting_output_len + bytes_written);
Ok(())
}
inner(self, input.as_ref(), buffer)
}
/// Decode the input into the provided output slice.
///
/// Returns an error if `output` is smaller than the estimated decoded length.
/// Returns the number of bytes written to the slice, or an error if `output` is smaller than
/// the estimated decoded length.
///
/// This will not write any bytes past exactly what is decoded (no stray garbage bytes at the end).
///
@ -312,29 +339,39 @@ pub trait Engine: Send + Sync {
///
/// See [Engine::decode_slice_unchecked] for a version that panics instead of returning an error
/// if the output buffer is too small.
///
/// # Panics
///
/// Panics if decoded length estimation overflows.
/// This would happen for sizes within a few bytes of the maximum value of `usize`.
#[inline]
fn decode_slice<T: AsRef<[u8]>>(
&self,
input: T,
output: &mut [u8],
) -> Result<usize, DecodeSliceError> {
let input_bytes = input.as_ref();
fn inner<E>(
engine: &E,
input_bytes: &[u8],
output: &mut [u8],
) -> Result<usize, DecodeSliceError>
where
E: Engine + ?Sized,
{
let estimate = engine.internal_decoded_len_estimate(input_bytes.len());
let estimate = self.internal_decoded_len_estimate(input_bytes.len());
if output.len() < estimate.decoded_len_estimate() {
return Err(DecodeSliceError::OutputSliceTooSmall);
if output.len() < estimate.decoded_len_estimate() {
return Err(DecodeSliceError::OutputSliceTooSmall);
}
engine
.internal_decode(input_bytes, output, estimate)
.map_err(|e| e.into())
.map(|dm| dm.decoded_len)
}
self.internal_decode(input_bytes, output, estimate)
.map_err(|e| e.into())
inner(self, input.as_ref(), output)
}
/// Decode the input into the provided output slice.
///
/// Returns the number of bytes written to the slice.
///
/// This will not write any bytes past exactly what is decoded (no stray garbage bytes at the end).
///
/// See [crate::decoded_len_estimate] for calculating buffer sizes.
@ -344,22 +381,27 @@ pub trait Engine: Send + Sync {
///
/// # Panics
///
/// Panics if decoded length estimation overflows.
/// This would happen for sizes within a few bytes of the maximum value of `usize`.
///
/// Panics if the provided output buffer is too small for the decoded data.
#[inline]
fn decode_slice_unchecked<T: AsRef<[u8]>>(
&self,
input: T,
output: &mut [u8],
) -> Result<usize, DecodeError> {
let input_bytes = input.as_ref();
fn inner<E>(engine: &E, input_bytes: &[u8], output: &mut [u8]) -> Result<usize, DecodeError>
where
E: Engine + ?Sized,
{
engine
.internal_decode(
input_bytes,
output,
engine.internal_decoded_len_estimate(input_bytes.len()),
)
.map(|dm| dm.decoded_len)
}
self.internal_decode(
input_bytes,
output,
self.internal_decoded_len_estimate(input_bytes.len()),
)
inner(self, input.as_ref(), output)
}
}
@ -387,11 +429,6 @@ pub trait DecodeEstimate {
///
/// The estimate must be no larger than the next largest complete triple of decoded bytes.
/// That is, the final quad of tokens to decode may be assumed to be complete with no padding.
///
/// # Panics
///
/// Panics if decoded length estimation overflows.
/// This would happen for sizes within a few bytes of the maximum value of `usize`.
fn decoded_len_estimate(&self) -> usize;
}
@ -408,3 +445,21 @@ pub enum DecodePaddingMode {
/// Padding must be absent -- for when you want predictable padding, without any wasted bytes.
RequireNone,
}
/// Metadata about the result of a decode operation
#[derive(PartialEq, Eq, Debug)]
pub struct DecodeMetadata {
/// Number of decoded bytes output
pub(crate) decoded_len: usize,
/// Offset of the first padding byte in the input, if any
pub(crate) padding_offset: Option<usize>,
}
impl DecodeMetadata {
pub(crate) fn new(decoded_bytes: usize, padding_index: Option<usize>) -> Self {
Self {
decoded_len: decoded_bytes,
padding_offset: padding_index,
}
}
}

View File

@ -2,7 +2,7 @@ use crate::{
alphabet::Alphabet,
engine::{
general_purpose::{self, decode_table, encode_table},
Config, DecodeEstimate, DecodePaddingMode, Engine,
Config, DecodeEstimate, DecodeMetadata, DecodePaddingMode, Engine,
},
DecodeError, PAD_BYTE,
};
@ -112,7 +112,7 @@ impl Engine for Naive {
input: &[u8],
output: &mut [u8],
estimate: Self::DecodeEstimate,
) -> Result<usize, DecodeError> {
) -> Result<DecodeMetadata, DecodeError> {
if estimate.rem == 1 {
// trailing whitespace is so common that it's worth it to check the last byte to
// possibly return a better error message

View File

@ -8,13 +8,16 @@ use rand::{
};
use rstest::rstest;
use rstest_reuse::{apply, template};
use std::{collections, fmt};
use std::{collections, fmt, io::Read as _};
use crate::{
alphabet::{Alphabet, STANDARD},
encode::add_padding,
encoded_len,
engine::{general_purpose, naive, Config, DecodeEstimate, DecodePaddingMode, Engine},
engine::{
general_purpose, naive, Config, DecodeEstimate, DecodeMetadata, DecodePaddingMode, Engine,
},
read::DecoderReader,
tests::{assert_encode_sanity, random_alphabet, random_config},
DecodeError, PAD_BYTE,
};
@ -24,9 +27,20 @@ use crate::{
#[rstest(engine_wrapper,
case::general_purpose(GeneralPurposeWrapper {}),
case::naive(NaiveWrapper {}),
case::decoder_reader(DecoderReaderEngineWrapper {}),
)]
fn all_engines<E: EngineWrapper>(engine_wrapper: E) {}
/// Some decode tests don't make sense for use with `DecoderReader` as they are difficult to
/// reason about or otherwise inapplicable given how DecoderReader slice up its input along
/// chunk boundaries.
#[template]
#[rstest(engine_wrapper,
case::general_purpose(GeneralPurposeWrapper {}),
case::naive(NaiveWrapper {}),
)]
fn all_engines_except_decoder_reader<E: EngineWrapper>(engine_wrapper: E) {}
#[apply(all_engines)]
fn rfc_test_vectors_std_alphabet<E: EngineWrapper>(engine_wrapper: E) {
let data = vec![
@ -86,7 +100,7 @@ fn rfc_test_vectors_std_alphabet<E: EngineWrapper>(engine_wrapper: E) {
&encoded_without_padding,
&std::str::from_utf8(&encode_buf[0..encode_len]).unwrap()
);
let pad_len = add_padding(orig.len(), &mut encode_buf[encode_len..]);
let pad_len = add_padding(encode_len, &mut encode_buf[encode_len..]);
assert_eq!(encoded.as_bytes(), &encode_buf[..encode_len + pad_len]);
let decode_len = engine
@ -195,7 +209,10 @@ fn encode_doesnt_write_extra_bytes<E: EngineWrapper>(engine_wrapper: E) {
// pad so we can decode it in case our random engine requires padding
let pad_len = if padded {
add_padding(orig_len, &mut encode_buf[prefix_len + encoded_len_no_pad..])
add_padding(
encoded_len_no_pad,
&mut encode_buf[prefix_len + encoded_len_no_pad..],
)
} else {
0
};
@ -382,7 +399,7 @@ fn decode_detect_invalid_last_symbol_every_possible_two_symbols<E: EngineWrapper
for b in 0_u8..=255 {
let mut b64 = vec![0_u8; 4];
assert_eq!(2, engine.internal_encode(&[b], &mut b64[..]));
let _ = add_padding(1, &mut b64[2..]);
let _ = add_padding(2, &mut b64[2..]);
assert!(base64_to_bytes.insert(b64, vec![b]).is_none());
}
@ -442,7 +459,7 @@ fn decode_detect_invalid_last_symbol_every_possible_three_symbols<E: EngineWrapp
bytes[1] = b2;
let mut b64 = vec![0_u8; 4];
assert_eq!(3, engine.internal_encode(&bytes, &mut b64[..]));
let _ = add_padding(2, &mut b64[3..]);
let _ = add_padding(3, &mut b64[3..]);
let mut v = Vec::with_capacity(2);
v.extend_from_slice(&bytes[..]);
@ -549,7 +566,7 @@ fn decode_invalid_byte_error<E: EngineWrapper>(engine_wrapper: E) {
let len_range = distributions::Uniform::new(1, 1_000);
for _ in 0..10_000 {
for _ in 0..100_000 {
let alphabet = random_alphabet(&mut rng);
let engine = E::random_alphabet(&mut rng, alphabet);
@ -573,7 +590,7 @@ fn decode_invalid_byte_error<E: EngineWrapper>(engine_wrapper: E) {
let invalid_byte: u8 = loop {
let byte: u8 = rng.gen();
if alphabet.symbols.contains(&byte) {
if alphabet.symbols.contains(&byte) || byte == PAD_BYTE {
continue;
} else {
break byte;
@ -597,14 +614,16 @@ fn decode_invalid_byte_error<E: EngineWrapper>(engine_wrapper: E) {
/// Any amount of padding anywhere before the final non padding character = invalid byte at first
/// pad byte.
/// From this, we know padding must extend to the end of the input.
#[apply(all_engines)]
// DecoderReader pseudo-engine detects InvalidLastSymbol instead of InvalidLength because it
// can end a decode on the quad that happens to contain the start of the padding
#[apply(all_engines_except_decoder_reader)]
fn decode_padding_before_final_non_padding_char_error_invalid_byte<E: EngineWrapper>(
engine_wrapper: E,
) {
let mut rng = seeded_rng();
// the different amounts of proper padding, w/ offset from end for the last non-padding char
let suffixes = vec![("/w==", 2), ("iYu=", 1), ("zzzz", 0)];
let suffixes = [("/w==", 2), ("iYu=", 1), ("zzzz", 0)];
let prefix_quads_range = distributions::Uniform::from(0..=256);
@ -641,10 +660,13 @@ fn decode_padding_before_final_non_padding_char_error_invalid_byte<E: EngineWrap
}
}
/// Any amount of padding before final chunk that crosses over into final chunk with 1-4 bytes =
/// invalid byte at first pad byte (except for 1 byte suffix = invalid length).
/// From this we know the padding must start in the final chunk.
#[apply(all_engines)]
/// Any amount of padding before final chunk that crosses over into final chunk with 2-4 bytes =
/// invalid byte at first pad byte.
/// From this and [decode_padding_starts_before_final_chunk_error_invalid_length] we know the
/// padding must start in the final chunk.
// DecoderReader pseudo-engine detects InvalidLastSymbol instead of InvalidLength because it
// can end a decode on the quad that happens to contain the start of the padding
#[apply(all_engines_except_decoder_reader)]
fn decode_padding_starts_before_final_chunk_error_invalid_byte<E: EngineWrapper>(
engine_wrapper: E,
) {
@ -652,8 +674,8 @@ fn decode_padding_starts_before_final_chunk_error_invalid_byte<E: EngineWrapper>
// must have at least one prefix quad
let prefix_quads_range = distributions::Uniform::from(1..256);
// including 1 just to make sure that it really does produce invalid length
let suffix_pad_len_range = distributions::Uniform::from(1..=4);
// excluding 1 since we don't care about invalid length in this test
let suffix_pad_len_range = distributions::Uniform::from(2..=4);
for mode in all_pad_modes() {
// we don't encode so we don't care about encode padding
let engine = E::standard_with_pad_mode(true, mode);
@ -671,14 +693,48 @@ fn decode_padding_starts_before_final_chunk_error_invalid_byte<E: EngineWrapper>
let padding_start = encoded.len() - padding_len;
encoded[padding_start..].fill(PAD_BYTE);
if suffix_len == 1 {
assert_eq!(Err(DecodeError::InvalidLength), engine.decode(&encoded),);
} else {
assert_eq!(
Err(DecodeError::InvalidByte(padding_start, PAD_BYTE)),
engine.decode(&encoded),
);
}
assert_eq!(
Err(DecodeError::InvalidByte(padding_start, PAD_BYTE)),
engine.decode(&encoded),
"suffix_len: {}, padding_len: {}, b64: {}",
suffix_len,
padding_len,
std::str::from_utf8(&encoded).unwrap()
);
}
}
}
/// Any amount of padding before final chunk that crosses over into final chunk with 1 byte =
/// invalid length.
/// From this we know the padding must start in the final chunk.
// DecoderReader pseudo-engine detects InvalidByte instead of InvalidLength because it starts by
// decoding only the available complete quads
#[apply(all_engines_except_decoder_reader)]
fn decode_padding_starts_before_final_chunk_error_invalid_length<E: EngineWrapper>(
engine_wrapper: E,
) {
let mut rng = seeded_rng();
// must have at least one prefix quad
let prefix_quads_range = distributions::Uniform::from(1..256);
for mode in all_pad_modes() {
// we don't encode so we don't care about encode padding
let engine = E::standard_with_pad_mode(true, mode);
for _ in 0..100_000 {
let mut encoded = "ABCD"
.repeat(prefix_quads_range.sample(&mut rng))
.into_bytes();
encoded.resize(encoded.len() + 1, PAD_BYTE);
// amount of padding must be long enough to extend back from suffix into previous
// quads
let padding_len = rng.gen_range(1 + 1..encoded.len());
// no non-padding after padding in this test, so padding goes to the end
let padding_start = encoded.len() - padding_len;
encoded[padding_start..].fill(PAD_BYTE);
assert_eq!(Err(DecodeError::InvalidLength), engine.decode(&encoded),);
}
}
}
@ -787,7 +843,9 @@ fn decode_malleability_test_case_2_byte_suffix_no_padding<E: EngineWrapper>(engi
}
// https://eprint.iacr.org/2022/361.pdf table 2, test 7
#[apply(all_engines)]
// DecoderReader pseudo-engine gets InvalidByte at 8 (extra padding) since it decodes the first
// two complete quads correctly.
#[apply(all_engines_except_decoder_reader)]
fn decode_malleability_test_case_2_byte_suffix_too_much_padding<E: EngineWrapper>(
engine_wrapper: E,
) {
@ -811,7 +869,7 @@ fn decode_pad_mode_requires_canonical_accepts_canonical<E: EngineWrapper>(engine
fn decode_pad_mode_requires_canonical_rejects_non_canonical<E: EngineWrapper>(engine_wrapper: E) {
let engine = E::standard_with_pad_mode(true, DecodePaddingMode::RequireCanonical);
let suffixes = vec!["/w", "/w=", "iYU"];
let suffixes = ["/w", "/w=", "iYU"];
for num_prefix_quads in 0..256 {
for &suffix in suffixes.iter() {
let mut encoded = "AAAA".repeat(num_prefix_quads);
@ -838,7 +896,7 @@ fn decode_pad_mode_requires_no_padding_accepts_no_padding<E: EngineWrapper>(engi
fn decode_pad_mode_requires_no_padding_rejects_any_padding<E: EngineWrapper>(engine_wrapper: E) {
let engine = E::standard_with_pad_mode(true, DecodePaddingMode::RequireNone);
let suffixes = vec!["/w=", "/w==", "iYU="];
let suffixes = ["/w=", "/w==", "iYU="];
for num_prefix_quads in 0..256 {
for &suffix in suffixes.iter() {
let mut encoded = "AAAA".repeat(num_prefix_quads);
@ -861,7 +919,11 @@ fn decode_pad_mode_indifferent_padding_accepts_anything<E: EngineWrapper>(engine
}
//this is a MAY in the rfc: https://tools.ietf.org/html/rfc4648#section-3.3
#[apply(all_engines)]
// DecoderReader pseudo-engine finds the first padding, but doesn't report it as an error,
// because in the next decode it finds more padding, which is reported as InvalidByte, just
// with an offset at its position in the second decode, rather than being linked to the start
// of the padding that was first seen in the previous decode.
#[apply(all_engines_except_decoder_reader)]
fn decode_pad_byte_in_penultimate_quad_error<E: EngineWrapper>(engine_wrapper: E) {
for mode in all_pad_modes() {
// we don't encode so we don't care about encode padding
@ -895,7 +957,7 @@ fn decode_pad_byte_in_penultimate_quad_error<E: EngineWrapper>(engine_wrapper: E
num_prefix_quads * 4 + num_valid_bytes_penultimate_quad,
b'=',
),
engine.decode(&s).unwrap_err()
engine.decode(&s).unwrap_err(),
);
}
}
@ -955,7 +1017,9 @@ fn decode_absurd_pad_error<E: EngineWrapper>(engine_wrapper: E) {
}
}
#[apply(all_engines)]
// DecoderReader pseudo-engine detects InvalidByte instead of InvalidLength because it starts by
// decoding only the available complete quads
#[apply(all_engines_except_decoder_reader)]
fn decode_too_much_padding_returns_error<E: EngineWrapper>(engine_wrapper: E) {
for mode in all_pad_modes() {
// we don't encode so we don't care about encode padding
@ -981,7 +1045,9 @@ fn decode_too_much_padding_returns_error<E: EngineWrapper>(engine_wrapper: E) {
}
}
#[apply(all_engines)]
// DecoderReader pseudo-engine detects InvalidByte instead of InvalidLength because it starts by
// decoding only the available complete quads
#[apply(all_engines_except_decoder_reader)]
fn decode_padding_followed_by_non_padding_returns_error<E: EngineWrapper>(engine_wrapper: E) {
for mode in all_pad_modes() {
// we don't encode so we don't care about encode padding
@ -1079,27 +1145,43 @@ fn decode_too_few_symbols_in_final_quad_error<E: EngineWrapper>(engine_wrapper:
}
}
#[apply(all_engines)]
// DecoderReader pseudo-engine can't handle DecodePaddingMode::RequireNone since it will decode
// a complete quad with padding in it before encountering the stray byte that makes it an invalid
// length
#[apply(all_engines_except_decoder_reader)]
fn decode_invalid_trailing_bytes<E: EngineWrapper>(engine_wrapper: E) {
for mode in all_pad_modes() {
// we don't encode so we don't care about encode padding
let engine = E::standard_with_pad_mode(true, mode);
do_invalid_trailing_byte(E::standard_with_pad_mode(true, mode), mode);
}
}
for num_prefix_quads in 0..256 {
let mut s: String = "ABCD".repeat(num_prefix_quads);
s.push_str("Cg==\n");
#[apply(all_engines)]
fn decode_invalid_trailing_bytes_all_modes<E: EngineWrapper>(engine_wrapper: E) {
// excluding no padding mode because the DecoderWrapper pseudo-engine will fail with
// InvalidPadding because it will decode the last complete quad with padding first
for mode in pad_modes_allowing_padding() {
do_invalid_trailing_byte(E::standard_with_pad_mode(true, mode), mode);
}
}
// The case of trailing newlines is common enough to warrant a test for a good error
// message.
assert_eq!(
Err(DecodeError::InvalidByte(num_prefix_quads * 4 + 4, b'\n')),
engine.decode(&s)
);
#[apply(all_engines)]
fn decode_invalid_trailing_padding_as_invalid_length<E: EngineWrapper>(engine_wrapper: E) {
// excluding no padding mode because the DecoderWrapper pseudo-engine will fail with
// InvalidPadding because it will decode the last complete quad with padding first
for mode in pad_modes_allowing_padding() {
do_invalid_trailing_padding_as_invalid_length(E::standard_with_pad_mode(true, mode), mode);
}
}
// extra padding, however, is still InvalidLength
let s = s.replace('\n', "=");
assert_eq!(Err(DecodeError::InvalidLength), engine.decode(s));
}
// DecoderReader pseudo-engine can't handle DecodePaddingMode::RequireNone since it will decode
// a complete quad with padding in it before encountering the stray byte that makes it an invalid
// length
#[apply(all_engines_except_decoder_reader)]
fn decode_invalid_trailing_padding_as_invalid_length_all_modes<E: EngineWrapper>(
engine_wrapper: E,
) {
for mode in all_pad_modes() {
do_invalid_trailing_padding_as_invalid_length(E::standard_with_pad_mode(true, mode), mode);
}
}
@ -1177,6 +1259,53 @@ fn decode_into_slice_fits_in_precisely_sized_slice<E: EngineWrapper>(engine_wrap
}
}
#[apply(all_engines)]
fn inner_decode_reports_padding_position<E: EngineWrapper>(engine_wrapper: E) {
let mut b64 = String::new();
let mut decoded = Vec::new();
let engine = E::standard();
for pad_position in 1..10_000 {
b64.clear();
decoded.clear();
// plenty of room for original data
decoded.resize(pad_position, 0);
for _ in 0..pad_position {
b64.push('A');
}
// finish the quad with padding
for _ in 0..(4 - (pad_position % 4)) {
b64.push('=');
}
let decode_res = engine.internal_decode(
b64.as_bytes(),
&mut decoded[..],
engine.internal_decoded_len_estimate(b64.len()),
);
if pad_position % 4 < 2 {
// impossible padding
assert_eq!(
Err(DecodeError::InvalidByte(pad_position, PAD_BYTE)),
decode_res
);
} else {
let decoded_bytes = pad_position / 4 * 3
+ match pad_position % 4 {
0 => 0,
2 => 1,
3 => 2,
_ => unreachable!(),
};
assert_eq!(
Ok(DecodeMetadata::new(decoded_bytes, Some(pad_position))),
decode_res
);
}
}
}
#[apply(all_engines)]
fn decode_length_estimate_delta<E: EngineWrapper>(engine_wrapper: E) {
for engine in [E::standard(), E::standard_unpadded()] {
@ -1200,6 +1329,64 @@ fn decode_length_estimate_delta<E: EngineWrapper>(engine_wrapper: E) {
}
}
#[apply(all_engines)]
fn estimate_via_u128_inflation<E: EngineWrapper>(engine_wrapper: E) {
// cover both ends of usize
(0..1000)
.chain(usize::MAX - 1000..=usize::MAX)
.for_each(|encoded_len| {
// inflate to 128 bit type to be able to safely use the easy formulas
let len_128 = encoded_len as u128;
let estimate = E::standard()
.internal_decoded_len_estimate(encoded_len)
.decoded_len_estimate();
// This check is a little too strict: it requires using the (len + 3) / 4 * 3 formula
// or equivalent, but until other engines come along that use a different formula
// requiring that we think more carefully about what the allowable criteria are, this
// will do.
assert_eq!(
((len_128 + 3) / 4 * 3) as usize,
estimate,
"enc len {}",
encoded_len
);
})
}
fn do_invalid_trailing_byte(engine: impl Engine, mode: DecodePaddingMode) {
for num_prefix_quads in 0..256 {
let mut s: String = "ABCD".repeat(num_prefix_quads);
s.push_str("Cg==\n");
// The case of trailing newlines is common enough to warrant a test for a good error
// message.
assert_eq!(
Err(DecodeError::InvalidByte(num_prefix_quads * 4 + 4, b'\n')),
engine.decode(&s),
"mode: {:?}, input: {}",
mode,
s
);
}
}
fn do_invalid_trailing_padding_as_invalid_length(engine: impl Engine, mode: DecodePaddingMode) {
for num_prefix_quads in 0..256 {
let mut s: String = "ABCD".repeat(num_prefix_quads);
s.push_str("Cg===");
assert_eq!(
Err(DecodeError::InvalidLength),
engine.decode(&s),
"mode: {:?}, input: {}",
mode,
s
);
}
}
/// Returns a tuple of the original data length, the encoded data length (just data), and the length including padding.
///
/// Vecs provided should be empty.
@ -1219,7 +1406,7 @@ fn generate_random_encoded_data<E: Engine, R: rand::Rng, D: distributions::Distr
let base_encoded_len = engine.internal_encode(&orig_data[..], &mut encode_buf[..]);
let enc_len_with_padding = if padding {
base_encoded_len + add_padding(orig_len, &mut encode_buf[base_encoded_len..])
base_encoded_len + add_padding(base_encoded_len, &mut encode_buf[base_encoded_len..])
} else {
base_encoded_len
};
@ -1249,11 +1436,7 @@ fn fill_rand_len<R: rand::Rng>(vec: &mut Vec<u8>, rng: &mut R, len: usize) {
}
}
fn prefixed_data<'i, 'd>(
input_with_prefix: &'i mut String,
prefix_len: usize,
data: &'d str,
) -> &'i str {
fn prefixed_data<'i>(input_with_prefix: &'i mut String, prefix_len: usize, data: &str) -> &'i str {
input_with_prefix.truncate(prefix_len);
input_with_prefix.push_str(data);
input_with_prefix.as_str()
@ -1405,6 +1588,103 @@ impl EngineWrapper for NaiveWrapper {
}
}
/// A pseudo-Engine that routes all decoding through [DecoderReader]
struct DecoderReaderEngine<E: Engine> {
engine: E,
}
impl<E: Engine> From<E> for DecoderReaderEngine<E> {
fn from(value: E) -> Self {
Self { engine: value }
}
}
impl<E: Engine> Engine for DecoderReaderEngine<E> {
type Config = E::Config;
type DecodeEstimate = E::DecodeEstimate;
fn internal_encode(&self, input: &[u8], output: &mut [u8]) -> usize {
self.engine.internal_encode(input, output)
}
fn internal_decoded_len_estimate(&self, input_len: usize) -> Self::DecodeEstimate {
self.engine.internal_decoded_len_estimate(input_len)
}
fn internal_decode(
&self,
input: &[u8],
output: &mut [u8],
decode_estimate: Self::DecodeEstimate,
) -> Result<DecodeMetadata, DecodeError> {
let mut reader = DecoderReader::new(input, &self.engine);
let mut buf = vec![0; input.len()];
// to avoid effects like not detecting invalid length due to progressively growing
// the output buffer in read_to_end etc, read into a big enough buffer in one go
// to make behavior more consistent with normal engines
let _ = reader
.read(&mut buf)
.and_then(|len| {
buf.truncate(len);
// make sure we got everything
reader.read_to_end(&mut buf)
})
.map_err(|io_error| {
*io_error
.into_inner()
.and_then(|inner| inner.downcast::<DecodeError>().ok())
.unwrap()
})?;
output[..buf.len()].copy_from_slice(&buf);
Ok(DecodeMetadata::new(
buf.len(),
input
.iter()
.enumerate()
.filter(|(_offset, byte)| **byte == PAD_BYTE)
.map(|(offset, _byte)| offset)
.next(),
))
}
fn config(&self) -> &Self::Config {
self.engine.config()
}
}
struct DecoderReaderEngineWrapper {}
impl EngineWrapper for DecoderReaderEngineWrapper {
type Engine = DecoderReaderEngine<general_purpose::GeneralPurpose>;
fn standard() -> Self::Engine {
GeneralPurposeWrapper::standard().into()
}
fn standard_unpadded() -> Self::Engine {
GeneralPurposeWrapper::standard_unpadded().into()
}
fn standard_with_pad_mode(
encode_pad: bool,
decode_pad_mode: DecodePaddingMode,
) -> Self::Engine {
GeneralPurposeWrapper::standard_with_pad_mode(encode_pad, decode_pad_mode).into()
}
fn standard_allow_trailing_bits() -> Self::Engine {
GeneralPurposeWrapper::standard_allow_trailing_bits().into()
}
fn random<R: rand::Rng>(rng: &mut R) -> Self::Engine {
GeneralPurposeWrapper::random(rng).into()
}
fn random_alphabet<R: rand::Rng>(rng: &mut R, alphabet: &Alphabet) -> Self::Engine {
GeneralPurposeWrapper::random_alphabet(rng, alphabet).into()
}
}
fn seeded_rng() -> impl rand::Rng {
rngs::SmallRng::from_entropy()
}
@ -1417,6 +1697,13 @@ fn all_pad_modes() -> Vec<DecodePaddingMode> {
]
}
fn pad_modes_allowing_padding() -> Vec<DecodePaddingMode> {
vec![
DecodePaddingMode::Indifferent,
DecodePaddingMode::RequireCanonical,
]
}
fn assert_all_suffixes_ok<E: Engine>(engine: E, suffixes: Vec<&str>) {
for num_prefix_quads in 0..256 {
for &suffix in suffixes.iter() {

View File

@ -1,4 +1,4 @@
use crate::{engine::Engine, DecodeError};
use crate::{engine::Engine, DecodeError, PAD_BYTE};
use std::{cmp, fmt, io};
// This should be large, but it has to fit on the stack.
@ -46,13 +46,15 @@ pub struct DecoderReader<'e, E: Engine, R: io::Read> {
// Technically we only need to hold 2 bytes but then we'd need a separate temporary buffer to
// decode 3 bytes into and then juggle copying one byte into the provided read buf and the rest
// into here, which seems like a lot of complexity for 1 extra byte of storage.
decoded_buffer: [u8; 3],
decoded_buffer: [u8; DECODED_CHUNK_SIZE],
// index of start of decoded data
decoded_offset: usize,
// length of decoded data
decoded_len: usize,
// used to provide accurate offsets in errors
total_b64_decoded: usize,
// offset of previously seen padding, if any
padding_offset: Option<usize>,
}
impl<'e, E: Engine, R: io::Read> fmt::Debug for DecoderReader<'e, E, R> {
@ -64,6 +66,7 @@ impl<'e, E: Engine, R: io::Read> fmt::Debug for DecoderReader<'e, E, R> {
.field("decoded_offset", &self.decoded_offset)
.field("decoded_len", &self.decoded_len)
.field("total_b64_decoded", &self.total_b64_decoded)
.field("padding_offset", &self.padding_offset)
.finish()
}
}
@ -81,6 +84,7 @@ impl<'e, E: Engine, R: io::Read> DecoderReader<'e, E, R> {
decoded_offset: 0,
decoded_len: 0,
total_b64_decoded: 0,
padding_offset: None,
}
}
@ -127,20 +131,28 @@ impl<'e, E: Engine, R: io::Read> DecoderReader<'e, E, R> {
/// caller's responsibility to choose the number of b64 bytes to decode correctly.
///
/// Returns a Result with the number of decoded bytes written to `buf`.
fn decode_to_buf(&mut self, num_bytes: usize, buf: &mut [u8]) -> io::Result<usize> {
debug_assert!(self.b64_len >= num_bytes);
fn decode_to_buf(&mut self, b64_len_to_decode: usize, buf: &mut [u8]) -> io::Result<usize> {
debug_assert!(self.b64_len >= b64_len_to_decode);
debug_assert!(self.b64_offset + self.b64_len <= BUF_SIZE);
debug_assert!(!buf.is_empty());
let decoded = self
let b64_to_decode = &self.b64_buffer[self.b64_offset..self.b64_offset + b64_len_to_decode];
let decode_metadata = self
.engine
.internal_decode(
&self.b64_buffer[self.b64_offset..self.b64_offset + num_bytes],
b64_to_decode,
buf,
self.engine.internal_decoded_len_estimate(num_bytes),
self.engine.internal_decoded_len_estimate(b64_len_to_decode),
)
.map_err(|e| match e {
DecodeError::InvalidByte(offset, byte) => {
// This can be incorrect, but not in a way that probably matters to anyone:
// if there was padding handled in a previous decode, and we are now getting
// InvalidByte due to more padding, we should arguably report InvalidByte with
// PAD_BYTE at the original padding position (`self.padding_offset`), but we
// don't have a good way to tie those two cases together, so instead we
// just report the invalid byte as if the previous padding, and its possibly
// related downgrade to a now invalid byte, didn't happen.
DecodeError::InvalidByte(self.total_b64_decoded + offset, byte)
}
DecodeError::InvalidLength => DecodeError::InvalidLength,
@ -151,13 +163,27 @@ impl<'e, E: Engine, R: io::Read> DecoderReader<'e, E, R> {
})
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
self.total_b64_decoded += num_bytes;
self.b64_offset += num_bytes;
self.b64_len -= num_bytes;
if let Some(offset) = self.padding_offset {
// we've already seen padding
if decode_metadata.decoded_len > 0 {
// we read more after already finding padding; report error at first padding byte
return Err(io::Error::new(
io::ErrorKind::InvalidData,
DecodeError::InvalidByte(offset, PAD_BYTE),
));
}
}
self.padding_offset = self.padding_offset.or(decode_metadata
.padding_offset
.map(|offset| self.total_b64_decoded + offset));
self.total_b64_decoded += b64_len_to_decode;
self.b64_offset += b64_len_to_decode;
self.b64_len -= b64_len_to_decode;
debug_assert!(self.b64_offset + self.b64_len <= BUF_SIZE);
Ok(decoded)
Ok(decode_metadata.decoded_len)
}
/// Unwraps this `DecoderReader`, returning the base reader which it reads base64 encoded
@ -205,9 +231,9 @@ impl<'e, E: Engine, R: io::Read> io::Read for DecoderReader<'e, E, R> {
self.decoded_offset < DECODED_CHUNK_SIZE
});
// We shouldn't ever decode into here when we can't immediately write at least one byte into
// the provided buf, so the effective length should only be 3 momentarily between when we
// decode and when we copy into the target buffer.
// We shouldn't ever decode into decoded_buffer when we can't immediately write at least one
// byte into the provided buf, so the effective length should only be 3 momentarily between
// when we decode and when we copy into the target buffer.
debug_assert!(self.decoded_len < DECODED_CHUNK_SIZE);
debug_assert!(self.decoded_len + self.decoded_offset <= DECODED_CHUNK_SIZE);
@ -217,20 +243,15 @@ impl<'e, E: Engine, R: io::Read> io::Read for DecoderReader<'e, E, R> {
} else {
let mut at_eof = false;
while self.b64_len < BASE64_CHUNK_SIZE {
// Work around lack of copy_within, which is only present in 1.37
// Copy any bytes we have to the start of the buffer.
// We know we have < 1 chunk, so we can use a tiny tmp buffer.
let mut memmove_buf = [0_u8; BASE64_CHUNK_SIZE];
memmove_buf[..self.b64_len].copy_from_slice(
&self.b64_buffer[self.b64_offset..self.b64_offset + self.b64_len],
);
self.b64_buffer[0..self.b64_len].copy_from_slice(&memmove_buf[..self.b64_len]);
self.b64_buffer
.copy_within(self.b64_offset..self.b64_offset + self.b64_len, 0);
self.b64_offset = 0;
// then fill in more data
let read = self.read_from_delegate()?;
if read == 0 {
// we never pass in an empty buf, so 0 => we've hit EOF
// we never read into an empty buf, so 0 => we've hit EOF
at_eof = true;
break;
}

View File

@ -8,9 +8,10 @@ use rand::{Rng as _, RngCore as _};
use super::decoder::{DecoderReader, BUF_SIZE};
use crate::{
alphabet,
engine::{general_purpose::STANDARD, Engine, GeneralPurpose},
tests::{random_alphabet, random_config, random_engine},
DecodeError,
DecodeError, PAD_BYTE,
};
#[test]
@ -247,19 +248,21 @@ fn reports_invalid_byte_correctly() {
let mut rng = rand::thread_rng();
let mut bytes = Vec::new();
let mut b64 = String::new();
let mut decoded = Vec::new();
let mut stream_decoded = Vec::new();
let mut bulk_decoded = Vec::new();
for _ in 0..10_000 {
bytes.clear();
b64.clear();
decoded.clear();
stream_decoded.clear();
bulk_decoded.clear();
let size = rng.gen_range(1..(10 * BUF_SIZE));
bytes.extend(iter::repeat(0).take(size));
rng.fill_bytes(&mut bytes[..size]);
assert_eq!(size, bytes.len());
let engine = random_engine(&mut rng);
let engine = GeneralPurpose::new(&alphabet::STANDARD, random_config(&mut rng));
engine.encode_string(&bytes[..], &mut b64);
// replace one byte, somewhere, with '*', which is invalid
@ -270,9 +273,8 @@ fn reports_invalid_byte_correctly() {
let mut wrapped_reader = io::Cursor::new(b64_bytes.clone());
let mut decoder = DecoderReader::new(&mut wrapped_reader, &engine);
// some gymnastics to avoid double-moving the io::Error, which is not Copy
let read_decode_err = decoder
.read_to_end(&mut decoded)
.read_to_end(&mut stream_decoded)
.map_err(|e| {
let kind = e.kind();
let inner = e
@ -283,8 +285,7 @@ fn reports_invalid_byte_correctly() {
.err()
.and_then(|o| o);
let mut bulk_buf = Vec::new();
let bulk_decode_err = engine.decode_vec(&b64_bytes[..], &mut bulk_buf).err();
let bulk_decode_err = engine.decode_vec(&b64_bytes[..], &mut bulk_decoded).err();
// it's tricky to predict where the invalid data's offset will be since if it's in the last
// chunk it will be reported at the first padding location because it's treated as invalid
@ -296,6 +297,134 @@ fn reports_invalid_byte_correctly() {
}
}
#[test]
fn internal_padding_error_with_short_read_concatenated_texts_invalid_byte_error() {
let mut rng = rand::thread_rng();
let mut bytes = Vec::new();
let mut b64 = String::new();
let mut reader_decoded = Vec::new();
let mut bulk_decoded = Vec::new();
// encodes with padding, requires that padding be present so we don't get InvalidPadding
// just because padding is there at all
let engine = STANDARD;
for _ in 0..10_000 {
bytes.clear();
b64.clear();
reader_decoded.clear();
bulk_decoded.clear();
// at least 2 bytes so there can be a split point between bytes
let size = rng.gen_range(2..(10 * BUF_SIZE));
bytes.resize(size, 0);
rng.fill_bytes(&mut bytes[..size]);
// Concatenate two valid b64s, yielding padding in the middle.
// This avoids scenarios that are challenging to assert on, like random padding location
// that might be InvalidLastSymbol when decoded at certain buffer sizes but InvalidByte
// when done all at once.
let split = loop {
// find a split point that will produce padding on the first part
let s = rng.gen_range(1..size);
if s % 3 != 0 {
// short enough to need padding
break s;
};
};
engine.encode_string(&bytes[..split], &mut b64);
assert!(b64.contains('='), "split: {}, b64: {}", split, b64);
let bad_byte_pos = b64.find('=').unwrap();
engine.encode_string(&bytes[split..], &mut b64);
let b64_bytes = b64.as_bytes();
// short read to make it plausible for padding to happen on a read boundary
let read_len = rng.gen_range(1..10);
let mut wrapped_reader = ShortRead {
max_read_len: read_len,
delegate: io::Cursor::new(&b64_bytes),
};
let mut decoder = DecoderReader::new(&mut wrapped_reader, &engine);
let read_decode_err = decoder
.read_to_end(&mut reader_decoded)
.map_err(|e| {
*e.into_inner()
.and_then(|e| e.downcast::<DecodeError>().ok())
.unwrap()
})
.unwrap_err();
let bulk_decode_err = engine.decode_vec(b64_bytes, &mut bulk_decoded).unwrap_err();
assert_eq!(
bulk_decode_err,
read_decode_err,
"read len: {}, bad byte pos: {}, b64: {}",
read_len,
bad_byte_pos,
std::str::from_utf8(b64_bytes).unwrap()
);
assert_eq!(
DecodeError::InvalidByte(
split / 3 * 4
+ match split % 3 {
1 => 2,
2 => 3,
_ => unreachable!(),
},
PAD_BYTE
),
read_decode_err
);
}
}
#[test]
fn internal_padding_anywhere_error() {
let mut rng = rand::thread_rng();
let mut bytes = Vec::new();
let mut b64 = String::new();
let mut reader_decoded = Vec::new();
// encodes with padding, requires that padding be present so we don't get InvalidPadding
// just because padding is there at all
let engine = STANDARD;
for _ in 0..10_000 {
bytes.clear();
b64.clear();
reader_decoded.clear();
bytes.resize(10 * BUF_SIZE, 0);
rng.fill_bytes(&mut bytes[..]);
// Just shove a padding byte in there somewhere.
// The specific error to expect is challenging to predict precisely because it
// will vary based on the position of the padding in the quad and the read buffer
// length, but SOMETHING should go wrong.
engine.encode_string(&bytes[..], &mut b64);
let mut b64_bytes = b64.as_bytes().to_vec();
// put padding somewhere other than the last quad
b64_bytes[rng.gen_range(0..bytes.len() - 4)] = PAD_BYTE;
// short read to make it plausible for padding to happen on a read boundary
let read_len = rng.gen_range(1..10);
let mut wrapped_reader = ShortRead {
max_read_len: read_len,
delegate: io::Cursor::new(&b64_bytes),
};
let mut decoder = DecoderReader::new(&mut wrapped_reader, &engine);
let result = decoder.read_to_end(&mut reader_decoded);
assert!(result.is_err());
}
}
fn consume_with_short_reads_and_validate<R: io::Read>(
rng: &mut rand::rngs::ThreadRng,
expected_bytes: &[u8],
@ -344,3 +473,15 @@ impl<'a, 'b, R: io::Read, N: rand::Rng> io::Read for RandomShortRead<'a, 'b, R,
self.delegate.read(&mut buf[..effective_len])
}
}
struct ShortRead<R: io::Read> {
delegate: R,
max_read_len: usize,
}
impl<R: io::Read> io::Read for ShortRead<R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
let len = self.max_read_len.max(buf.len());
self.delegate.read(&mut buf[..len])
}
}

View File

@ -44,11 +44,6 @@ use std::io;
/// assert_eq!("base64: YXNkZg==", &buf);
/// ```
///
/// # Panics
///
/// Calling `write()` (or related methods) or `finish()` after `finish()` has completed without
/// error is invalid and will panic.
///
/// # Performance
///
/// Because it has to validate that the base64 is UTF-8, it is about 80% as fast as writing plain
@ -144,6 +139,7 @@ mod tests {
engine::Engine, tests::random_engine, write::encoder_string_writer::EncoderStringWriter,
};
use rand::Rng;
use std::cmp;
use std::io::Write;
#[test]
@ -158,9 +154,8 @@ mod tests {
orig_data.clear();
normal_encoded.clear();
for _ in 0..size {
orig_data.push(rng.gen());
}
orig_data.resize(size, 0);
rng.fill(&mut orig_data[..]);
let engine = random_engine(&mut rng);
engine.encode_string(&orig_data, &mut normal_encoded);
@ -172,6 +167,40 @@ mod tests {
let stream_encoded = stream_encoder.into_inner();
assert_eq!(normal_encoded, stream_encoded);
}
}
#[test]
fn incremental_writes() {
let mut rng = rand::thread_rng();
let mut orig_data = Vec::<u8>::new();
let mut normal_encoded = String::new();
let size = 5_000;
for _ in 0..size {
orig_data.clear();
normal_encoded.clear();
orig_data.resize(size, 0);
rng.fill(&mut orig_data[..]);
let engine = random_engine(&mut rng);
engine.encode_string(&orig_data, &mut normal_encoded);
let mut stream_encoder = EncoderStringWriter::new(&engine);
// write small nibbles of data
let mut offset = 0;
while offset < size {
let nibble_size = cmp::min(rng.gen_range(0..=64), size - offset);
let len = stream_encoder
.write(&orig_data[offset..offset + nibble_size])
.unwrap();
offset += len;
}
let stream_encoded = stream_encoder.into_inner();
assert_eq!(normal_encoded, stream_encoded);
}
}

View File

@ -358,7 +358,7 @@ fn retrying_writes_that_error_with_interrupted_works() {
Ok(_) => break,
Err(e) => match e.kind() {
io::ErrorKind::Interrupted => continue,
_ => Err(e).unwrap(), // bail
_ => panic!("{:?}", e), // bail
},
}
}

View File

@ -8,11 +8,7 @@ fn compare_encode(expected: &str, target: &[u8]) {
#[test]
fn encode_all_ascii() {
let mut ascii = Vec::<u8>::with_capacity(128);
for i in 0..128 {
ascii.push(i);
}
let ascii: Vec<u8> = (0..=127).collect();
compare_encode(
"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7P\
@ -24,12 +20,7 @@ fn encode_all_ascii() {
#[test]
fn encode_all_bytes() {
let mut bytes = Vec::<u8>::with_capacity(256);
for i in 0..255 {
bytes.push(i);
}
bytes.push(255); //bug with "overflowing" ranges?
let bytes: Vec<u8> = (0..=255).collect();
compare_encode(
"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7P\
@ -42,12 +33,7 @@ fn encode_all_bytes() {
#[test]
fn encode_all_bytes_url() {
let mut bytes = Vec::<u8>::with_capacity(256);
for i in 0..255 {
bytes.push(i);
}
bytes.push(255); //bug with "overflowing" ranges?
let bytes: Vec<u8> = (0..=255).collect();
assert_eq!(
"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0\
@ -55,6 +41,6 @@ fn encode_all_bytes_url() {
-AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq\
-wsbKztLW2t7i5uru8vb6_wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t_g4eLj5OXm5-jp6uvs7e7v8PHy\
8_T19vf4-fr7_P3-_w==",
&engine::GeneralPurpose::new(&URL_SAFE, PAD).encode(&bytes)
&engine::GeneralPurpose::new(&URL_SAFE, PAD).encode(bytes)
);
}