From 42af0e0dfc8f08d36abb4593f2707cb4470e181c Mon Sep 17 00:00:00 2001 From: Matt McCoy Date: Wed, 1 Apr 2015 22:15:37 -0600 Subject: [PATCH] servo: Merge #5299 - Add HTTP compression (from mattnenterprise:http_compression); r=jdm Source-Repo: https://github.com/servo/servo Source-Revision: dbd16932e67c63a778362f71be028132279d5e91 --- servo/components/net/Cargo.toml | 1 + servo/components/net/http_loader.rs | 72 ++++++++++++++++++++--------- servo/components/net/lib.rs | 1 + servo/components/servo/Cargo.lock | 19 ++++++++ servo/ports/cef/Cargo.lock | 19 ++++++++ servo/ports/gonk/Cargo.lock | 19 ++++++++ 6 files changed, 110 insertions(+), 21 deletions(-) diff --git a/servo/components/net/Cargo.toml b/servo/components/net/Cargo.toml index 8f8c43b41393..f98ec042b44f 100644 --- a/servo/components/net/Cargo.toml +++ b/servo/components/net/Cargo.toml @@ -31,3 +31,4 @@ cookie="*" regex = "0.1.14" regex_macros = "0.1.8" hyper = "0.3" +flate2 = "0.2.0" diff --git a/servo/components/net/http_loader.rs b/servo/components/net/http_loader.rs index 6bc5e9900965..57e65a02b583 100644 --- a/servo/components/net/http_loader.rs +++ b/servo/components/net/http_loader.rs @@ -3,15 +3,16 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use cookie_storage::CookieSource; -use resource_task::{Metadata, TargetedLoadResponse, LoadData, start_sending_opt, ResponseSenders}; +use resource_task::{Metadata, TargetedLoadResponse, LoadData, start_sending_opt, ResponseSenders, ProgressMsg}; use resource_task::ControlMsg; use resource_task::ProgressMsg::{Payload, Done}; use log; use std::collections::HashSet; use file_loader; +use flate2::read::{DeflateDecoder, GzDecoder}; use hyper::client::Request; -use hyper::header::{ContentLength, ContentType, Host, Location}; +use hyper::header::{AcceptEncoding, ContentLength, ContentType, Host, Location}; use hyper::HttpError; use hyper::method::Method; use hyper::mime::{Mime, TopLevel, SubLevel}; @@ -160,11 +161,9 @@ reason: \"certificate verify failed\" }]"; req.headers_mut().set_raw("Cookie".to_owned(), v); } - // FIXME(seanmonstar): use AcceptEncoding from Hyper once available - //if !req.headers.has::() { - // We currently don't support HTTP Compression (FIXME #2587) - req.headers_mut().set_raw("Accept-Encoding".to_owned(), vec![b"identity".to_vec()]); - //} + if !req.headers().has::() { + req.headers_mut().set_raw("Accept-Encoding".to_owned(), vec![b"gzip, deflate".to_vec()]); + } if log_enabled!(log::INFO) { info!("{}", load_data.method); for header in req.headers().iter() { @@ -296,24 +295,32 @@ reason: \"certificate verify failed\" }]"; Ok(p) => p, _ => return }; - loop { - let mut buf = Vec::with_capacity(1024); - unsafe { buf.set_len(1024); } - match response.read(buf.as_mut_slice()) { - Ok(len) if len > 0 => { - unsafe { buf.set_len(len); } - if progress_chan.send(Payload(buf)).is_err() { - // The send errors when the receiver is out of scope, - // which will happen if the fetch has timed out (or has been aborted) - // so we don't need to continue with the loading of the file here. - return; + let mut encoding_str: Option = None; + //FIXME: Implement Content-Encoding Header https://github.com/hyperium/hyper/issues/391 + if let Some(encodings) = response.headers.get_raw("content-encoding") { + for encoding in encodings.iter() { + if let Ok(encodings) = String::from_utf8(encoding.clone()) { + if encodings == "gzip" || encodings == "deflate" { + encoding_str = Some(encodings); + break; } } - Ok(_) | Err(_) => { - let _ = progress_chan.send(Done(Ok(()))); - break; + } + } + + match encoding_str { + Some(encoding) => { + if encoding == "gzip" { + let mut response_decoding = GzDecoder::new(response).unwrap(); + send_data(&mut response_decoding, progress_chan); + } else if encoding == "deflate" { + let mut response_decoding = DeflateDecoder::new(response); + send_data(&mut response_decoding, progress_chan); } + }, + None => { + send_data(&mut response, progress_chan); } } @@ -321,3 +328,26 @@ reason: \"certificate verify failed\" }]"; break; } } + +fn send_data(reader: &mut R, progress_chan: Sender) { + loop { + let mut buf = Vec::with_capacity(1024); + + unsafe { buf.set_len(1024); } + match reader.read(buf.as_mut_slice()) { + Ok(len) if len > 0 => { + unsafe { buf.set_len(len); } + if progress_chan.send(Payload(buf)).is_err() { + // The send errors when the receiver is out of scope, + // which will happen if the fetch has timed out (or has been aborted) + // so we don't need to continue with the loading of the file here. + return; + } + } + Ok(_) | Err(_) => { + let _ = progress_chan.send(Done(Ok(()))); + break; + } + } + } +} diff --git a/servo/components/net/lib.rs b/servo/components/net/lib.rs index 6eb504ff2d31..4ebb5ef092ce 100644 --- a/servo/components/net/lib.rs +++ b/servo/components/net/lib.rs @@ -20,6 +20,7 @@ extern crate "cookie" as cookie_rs; extern crate collections; +extern crate flate2; extern crate geom; extern crate hyper; extern crate png; diff --git a/servo/components/servo/Cargo.lock b/servo/components/servo/Cargo.lock index 4ed7d0fee113..b8bd6086f936 100644 --- a/servo/components/servo/Cargo.lock +++ b/servo/components/servo/Cargo.lock @@ -240,6 +240,15 @@ name = "expat-sys" version = "2.1.0" source = "git+https://github.com/servo/libexpat#523a2f2f51b41adf7bb5c4c65e80db0cb615d70b" +[[package]] +name = "flate2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "miniz-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "fontconfig" version = "0.1.0" @@ -586,6 +595,15 @@ dependencies = [ "log 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "miniz-sys" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "mod_path" version = "0.1.4" @@ -617,6 +635,7 @@ name = "net" version = "0.0.1" dependencies = [ "cookie 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "geom 0.1.0 (git+https://github.com/servo/rust-geom)", "hyper 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "openssl 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/servo/ports/cef/Cargo.lock b/servo/ports/cef/Cargo.lock index 0fbbade1a283..0d5cb75c18d4 100644 --- a/servo/ports/cef/Cargo.lock +++ b/servo/ports/cef/Cargo.lock @@ -245,6 +245,15 @@ name = "expat-sys" version = "2.1.0" source = "git+https://github.com/servo/libexpat#523a2f2f51b41adf7bb5c4c65e80db0cb615d70b" +[[package]] +name = "flate2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "miniz-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "fontconfig" version = "0.1.0" @@ -591,6 +600,15 @@ dependencies = [ "log 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "miniz-sys" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "mod_path" version = "0.1.4" @@ -622,6 +640,7 @@ name = "net" version = "0.0.1" dependencies = [ "cookie 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "geom 0.1.0 (git+https://github.com/servo/rust-geom)", "hyper 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "openssl 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/servo/ports/gonk/Cargo.lock b/servo/ports/gonk/Cargo.lock index 49e49ef1338f..d836bf44eb4e 100644 --- a/servo/ports/gonk/Cargo.lock +++ b/servo/ports/gonk/Cargo.lock @@ -224,6 +224,15 @@ name = "expat-sys" version = "2.1.0" source = "git+https://github.com/servo/libexpat#523a2f2f51b41adf7bb5c4c65e80db0cb615d70b" +[[package]] +name = "flate2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "miniz-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "fontconfig" version = "0.1.0" @@ -516,6 +525,15 @@ dependencies = [ "log 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "miniz-sys" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "mod_path" version = "0.1.4" @@ -547,6 +565,7 @@ name = "net" version = "0.0.1" dependencies = [ "cookie 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "geom 0.1.0 (git+https://github.com/servo/rust-geom)", "hyper 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "openssl 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",