Bug 1389470 - use fallible memory allocation to avoid OOM. r=kinetik

MozReview-Commit-ID: BvqIe685Rfs

--HG--
extra : rebase_source : b030ba607474a285c8fb50e9198f126c36a011f9
This commit is contained in:
Alfredo.Yang 2017-09-13 14:59:45 +08:00
parent 3bcdf661d6
commit 43be656b03
12 changed files with 236 additions and 50 deletions

View File

@ -763,6 +763,8 @@ MP4MetadataRust::Init()
mp4parse_log(true);
}
mp4parse_fallible_allocation(true);
mp4parse_status rv = mp4parse_read(mRustParser.get());
MOZ_LOG(sLog, LogLevel::Debug, ("rust parser returned %d\n", rv));
Telemetry::Accumulate(Telemetry::MEDIA_RUST_MP4PARSE_SUCCESS,

View File

@ -24,6 +24,7 @@ typedef enum mp4parse_status {
mp4parse_status_EOF = 4,
mp4parse_status_IO = 5,
mp4parse_status_TABLE_TOO_LARGE = 6,
mp4parse_status_OOM = 7,
} mp4parse_status;
typedef enum mp4parse_track_type {
@ -118,6 +119,8 @@ void mp4parse_free(mp4parse_parser* parser);
/// Enable `mp4_parser` log.
void mp4parse_log(bool enable);
void mp4parse_fallible_allocation(bool enable);
/// Run the `mp4parse_parser*` allocated by `mp4parse_new()` until EOF or error.
mp4parse_status mp4parse_read(mp4parse_parser* parser);

View File

@ -2,7 +2,7 @@ diff --git a/media/libstagefright/binding/mp4parse/Cargo.toml b/media/libstagefr
index ff9422c..814c4c6 100644
--- a/media/libstagefright/binding/mp4parse/Cargo.toml
+++ b/media/libstagefright/binding/mp4parse/Cargo.toml
@@ -20,19 +20,11 @@ exclude = [
@@ -20,20 +20,12 @@ exclude = [
]
-[badges]
@ -15,9 +15,11 @@ index ff9422c..814c4c6 100644
-abort_on_panic = { version = "1.0.0", optional = true }
-bitreader = { version = "0.3.0" }
-num-traits = "0.1.37"
-mp4parse_fallible = { path = "../mp4parse_fallible" }
+byteorder = "1.0.0"
+bitreader = { version = "0.3.0" }
+num-traits = "0.1.37"
+mp4parse_fallible = { path = "../mp4parse_fallible" }
[dev-dependencies]
test-assembler = "0.1.2"

View File

@ -24,6 +24,7 @@ exclude = [
byteorder = "1.0.0"
bitreader = { version = "0.3.0" }
num-traits = "0.1.37"
mp4parse_fallible = { path = "../mp4parse_fallible" }
[dev-dependencies]
test-assembler = "0.1.2"

View File

@ -11,12 +11,14 @@ extern crate afl;
extern crate byteorder;
extern crate bitreader;
extern crate num_traits;
extern crate mp4parse_fallible;
use byteorder::{ReadBytesExt, WriteBytesExt};
use bitreader::{BitReader, ReadInto};
use std::io::{Read, Take};
use std::io::Cursor;
use std::cmp;
use num_traits::Num;
use mp4parse_fallible::FallibleVec;
mod boxes;
use boxes::{BoxType, FourCC};
@ -30,15 +32,20 @@ const BUF_SIZE_LIMIT: usize = 1024 * 1024;
// Max table length. Calculating in worth case for one week long video, one
// frame per table entry in 30 fps.
#[cfg(target_pointer_width = "64")]
const TABLE_SIZE_LIMIT: u32 = 30 * 60 * 60 * 24 * 7;
// Reduce max table length if it is in 32 arch for memory problem.
#[cfg(target_pointer_width = "32")]
const TABLE_SIZE_LIMIT: u32 = 30 * 60 * 60 * 24;
static DEBUG_MODE: std::sync::atomic::AtomicBool = std::sync::atomic::ATOMIC_BOOL_INIT;
static FALLIBLE_ALLOCATION: std::sync::atomic::AtomicBool = std::sync::atomic::ATOMIC_BOOL_INIT;
pub fn set_fallible_allocation_mode(fallible: bool) {
FALLIBLE_ALLOCATION.store(fallible, std::sync::atomic::Ordering::SeqCst);
}
fn get_fallible_allocation_mode() -> bool {
FALLIBLE_ALLOCATION.load(std::sync::atomic::Ordering::Relaxed)
}
pub fn set_debug_mode(mode: bool) {
DEBUG_MODE.store(mode, std::sync::atomic::Ordering::SeqCst);
}
@ -56,6 +63,37 @@ macro_rules! log {
)
}
// TODO: vec_push() and vec_reserve() needs to be replaced when Rust supports
// fallible memory allocation in raw_vec.
pub fn vec_push<T>(vec: &mut Vec<T>, val: T) -> std::result::Result<(), ()> {
if get_fallible_allocation_mode() {
return vec.try_push(val);
}
vec.push(val);
Ok(())
}
pub fn vec_reserve<T>(vec: &mut Vec<T>, size: usize) -> std::result::Result<(), ()> {
if get_fallible_allocation_mode() {
return vec.try_reserve(size);
}
vec.reserve(size);
Ok(())
}
fn reserve_read_buf(size: usize) -> std::result::Result<Vec<u8>, ()> {
if get_fallible_allocation_mode() {
let mut buf: Vec<u8> = Vec::new();
buf.try_reserve(size)?;
unsafe { buf.set_len(size); }
return Ok(buf);
}
Ok(vec![0; size])
}
/// Describes parser failures.
///
/// This enum wraps the standard `io::Error` type, unified with
@ -74,6 +112,8 @@ pub enum Error {
NoMoov,
/// Parse error caused by table size is over limitation.
TableTooLarge,
/// Out of memory
OutOfMemory,
}
impl From<bitreader::BitReaderError> for Error {
@ -97,6 +137,12 @@ impl From<std::string::FromUtf8Error> for Error {
}
}
impl From<()> for Error {
fn from(_: ()) -> Error {
Error::OutOfMemory
}
}
/// Result shorthand using our Error enum.
pub type Result<T> = std::result::Result<T, Error>;
@ -689,7 +735,7 @@ fn read_moov<T: Read>(f: &mut BMFFBox<T>, context: &mut MediaContext) -> Result<
BoxType::TrackBox => {
let mut track = Track::new(context.tracks.len());
read_trak(&mut b, &mut track)?;
context.tracks.push(track);
vec_push(&mut context.tracks, track)?;
}
BoxType::MovieExtendsBox => {
let mvex = read_mvex(&mut b)?;
@ -699,7 +745,7 @@ fn read_moov<T: Read>(f: &mut BMFFBox<T>, context: &mut MediaContext) -> Result<
BoxType::ProtectionSystemSpecificHeaderBox => {
let pssh = read_pssh(&mut b)?;
log!("{:?}", pssh);
context.psshs.push(pssh);
vec_push(&mut context.psshs, pssh)?;
}
_ => skip_box_content(&mut b)?,
};
@ -723,7 +769,7 @@ fn read_pssh<T: Read>(src: &mut BMFFBox<T>) -> Result<ProtectionSystemSpecificHe
let count = be_u32_with_limit(pssh)?;
for _ in 0..count {
let item = read_buf(pssh, 16)?;
kid.push(item);
vec_push(&mut kid, item)?;
}
}
@ -735,7 +781,7 @@ fn read_pssh<T: Read>(src: &mut BMFFBox<T>) -> Result<ProtectionSystemSpecificHe
let mut pssh_box = Vec::new();
write_be_u32(&mut pssh_box, src.head.size as u32)?;
pssh_box.append(&mut b"pssh".to_vec());
pssh_box.extend_from_slice(b"pssh");
pssh_box.append(&mut box_content);
Ok(ProtectionSystemSpecificHeaderBox {
@ -940,7 +986,7 @@ fn read_ftyp<T: Read>(src: &mut BMFFBox<T>) -> Result<FileTypeBox> {
let brand_count = bytes_left / 4;
let mut brands = Vec::new();
for _ in 0..brand_count {
brands.push(From::from(be_u32(src)?));
vec_push(&mut brands, From::from(be_u32(src)?))?;
}
Ok(FileTypeBox {
major_brand: From::from(major),
@ -1049,12 +1095,12 @@ fn read_elst<T: Read>(src: &mut BMFFBox<T>) -> Result<EditListBox> {
};
let media_rate_integer = be_i16(src)?;
let media_rate_fraction = be_i16(src)?;
edits.push(Edit {
vec_push(&mut edits, Edit {
segment_duration: segment_duration,
media_time: media_time,
media_rate_integer: media_rate_integer,
media_rate_fraction: media_rate_fraction,
})
})?;
}
Ok(EditListBox {
@ -1110,7 +1156,7 @@ fn read_stco<T: Read>(src: &mut BMFFBox<T>) -> Result<ChunkOffsetBox> {
let offset_count = be_u32_with_limit(src)?;
let mut offsets = Vec::new();
for _ in 0..offset_count {
offsets.push(be_u32(src)? as u64);
vec_push(&mut offsets, be_u32(src)? as u64)?;
}
// Padding could be added in some contents.
@ -1127,7 +1173,7 @@ fn read_co64<T: Read>(src: &mut BMFFBox<T>) -> Result<ChunkOffsetBox> {
let offset_count = be_u32_with_limit(src)?;
let mut offsets = Vec::new();
for _ in 0..offset_count {
offsets.push(be_u64(src)?);
vec_push(&mut offsets, be_u64(src)?)?;
}
// Padding could be added in some contents.
@ -1144,7 +1190,7 @@ fn read_stss<T: Read>(src: &mut BMFFBox<T>) -> Result<SyncSampleBox> {
let sample_count = be_u32_with_limit(src)?;
let mut samples = Vec::new();
for _ in 0..sample_count {
samples.push(be_u32(src)?);
vec_push(&mut samples, be_u32(src)?)?;
}
// Padding could be added in some contents.
@ -1164,11 +1210,11 @@ fn read_stsc<T: Read>(src: &mut BMFFBox<T>) -> Result<SampleToChunkBox> {
let first_chunk = be_u32(src)?;
let samples_per_chunk = be_u32_with_limit(src)?;
let sample_description_index = be_u32(src)?;
samples.push(SampleToChunk {
vec_push(&mut samples, SampleToChunk {
first_chunk: first_chunk,
samples_per_chunk: samples_per_chunk,
sample_description_index: sample_description_index,
});
})?;
}
// Padding could be added in some contents.
@ -1203,10 +1249,10 @@ fn read_ctts<T: Read>(src: &mut BMFFBox<T>) -> Result<CompositionOffsetBox> {
return Err(Error::InvalidData("unsupported version in 'ctts' box"));
}
};
offsets.push(TimeOffset {
vec_push(&mut offsets, TimeOffset {
sample_count: sample_count,
time_offset: time_offset,
});
})?;
}
skip_box_remain(src)?;
@ -1224,7 +1270,7 @@ fn read_stsz<T: Read>(src: &mut BMFFBox<T>) -> Result<SampleSizeBox> {
let mut sample_sizes = Vec::new();
if sample_size == 0 {
for _ in 0..sample_count {
sample_sizes.push(be_u32(src)?);
vec_push(&mut sample_sizes, be_u32(src)?)?;
}
}
@ -1245,10 +1291,10 @@ fn read_stts<T: Read>(src: &mut BMFFBox<T>) -> Result<TimeToSampleBox> {
for _ in 0..sample_count {
let sample_count = be_u32_with_limit(src)?;
let sample_delta = be_u32(src)?;
samples.push(Sample {
vec_push(&mut samples, Sample {
sample_count: sample_count,
sample_delta: sample_delta,
});
})?;
}
// Padding could be added in some contents.
@ -1455,7 +1501,7 @@ fn read_ds_descriptor(data: &[u8], esds: &mut ES_Descriptor) -> Result<()> {
esds.audio_sample_rate = sample_frequency;
esds.audio_channel_count = Some(channel_counts);
assert!(esds.decoder_specific_data.is_empty());
esds.decoder_specific_data.extend(data.iter());
esds.decoder_specific_data.extend_from_slice(data);
Ok(())
}
@ -1543,7 +1589,7 @@ fn read_dfla<T: Read>(src: &mut BMFFBox<T>) -> Result<FLACSpecificBox> {
let mut blocks = Vec::new();
while src.bytes_left() > 0 {
let block = read_flac_metadata(src)?;
blocks.push(block);
vec_push(&mut blocks, block)?;
}
// The box must have at least one meta block, and the first block
// must be the METADATA_BLOCK_STREAMINFO
@ -1737,7 +1783,7 @@ fn read_video_sample_entry<T: Read>(src: &mut BMFFBox<T>) -> Result<(CodecType,
}
let sinf = read_sinf(&mut b)?;
log!("{:?} (sinf)", sinf);
protection_info.push(sinf);
vec_push(&mut protection_info, sinf)?;
}
_ => {
log!("Unsupported video codec, box {:?} found", b.head.name);
@ -1862,7 +1908,7 @@ fn read_audio_sample_entry<T: Read>(src: &mut BMFFBox<T>) -> Result<(CodecType,
let sinf = read_sinf(&mut b)?;
log!("{:?} (sinf)", sinf);
codec_type = CodecType::EncryptedAudio;
protection_info.push(sinf);
vec_push(&mut protection_info, sinf)?;
}
_ => {
log!("Unsupported audio codec, box {:?} found", b.head.name);
@ -1920,7 +1966,7 @@ fn read_stsd<T: Read>(src: &mut BMFFBox<T>, track: &mut Track) -> Result<SampleD
} else {
log!("** don't know how to handle multiple descriptions **");
}
descriptions.push(description);
vec_push(&mut descriptions, description)?;
check_parser_state!(b.content);
if descriptions.len() == description_count as usize {
break;
@ -2015,12 +2061,15 @@ fn read_buf<T: ReadBytesExt>(src: &mut T, size: usize) -> Result<Vec<u8>> {
if size > BUF_SIZE_LIMIT {
return Err(Error::InvalidData("read_buf size exceeds BUF_SIZE_LIMIT"));
}
let mut buf = vec![0; size];
let r = src.read(&mut buf)?;
if r != size {
return Err(Error::InvalidData("failed buffer read"));
if let Ok(mut buf) = reserve_read_buf(size) {
let r = src.read(&mut buf)?;
if r != size {
return Err(Error::InvalidData("failed buffer read"));
}
return Ok(buf);
}
Ok(buf)
Err(Error::OutOfMemory)
}
fn be_i16<T: ReadBytesExt>(src: &mut T) -> Result<i16> {

View File

@ -58,6 +58,7 @@ use mp4parse::TrackScaledTime;
use mp4parse::serialize_opus_header;
use mp4parse::CodecType;
use mp4parse::Track;
use mp4parse::vec_push;
#[allow(non_camel_case_types)]
#[repr(C)]
@ -70,6 +71,7 @@ pub enum mp4parse_status {
EOF = 4,
IO = 5,
TABLE_TOO_LARGE = 6,
OOM = 7,
}
#[allow(non_camel_case_types)]
@ -308,6 +310,11 @@ pub unsafe extern fn mp4parse_log(enable: bool) {
mp4parse::set_debug_mode(enable);
}
#[no_mangle]
pub unsafe extern fn mp4parse_fallible_allocation(enable: bool) {
mp4parse::set_fallible_allocation_mode(enable);
}
/// Run the `mp4parse_parser*` allocated by `mp4parse_new()` until EOF or error.
#[no_mangle]
pub unsafe extern fn mp4parse_read(parser: *mut mp4parse_parser) -> mp4parse_status {
@ -316,8 +323,8 @@ pub unsafe extern fn mp4parse_read(parser: *mut mp4parse_parser) -> mp4parse_sta
return mp4parse_status::BAD_ARG;
}
let mut context = (*parser).context_mut();
let mut io = (*parser).io_mut();
let context = (*parser).context_mut();
let io = (*parser).io_mut();
let r = read_mp4(io, context);
match r {
@ -338,6 +345,7 @@ pub unsafe extern fn mp4parse_read(parser: *mut mp4parse_parser) -> mp4parse_sta
mp4parse_status::IO
},
Err(Error::TableTooLarge) => mp4parse_status::TABLE_TOO_LARGE,
Err(Error::OutOfMemory) => mp4parse_status::OOM,
}
}
@ -900,16 +908,17 @@ fn create_sample_table(track: &Track, track_offset_time: i64) -> Option<Vec<mp4p
}
cur_position = end_offset;
sample_table.push(
mp4parse_indice {
start_offset: start_offset,
end_offset: end_offset,
start_composition: 0,
end_composition: 0,
start_decode: 0,
sync: !has_sync_table,
}
);
let res = vec_push(&mut sample_table, mp4parse_indice {
start_offset: start_offset,
end_offset: end_offset,
start_composition: 0,
end_composition: 0,
start_decode: 0,
sync: !has_sync_table,
});
if res.is_err() {
return None;
}
}
}
@ -977,12 +986,13 @@ fn create_sample_table(track: &Track, track_offset_time: i64) -> Option<Vec<mp4p
//
// Composition end time is not in specification. However, gecko needs it, so we need to
// calculate to correct the composition end time.
if track.ctts.is_some() {
if sample_table.len() > 0 {
// Create an index table refers to sample_table and sorted by start_composisiton time.
let mut sort_table = Vec::new();
sort_table.reserve(sample_table.len());
for i in 0 .. sample_table.len() {
sort_table.push(i);
if vec_push(&mut sort_table, i).is_err() {
return None;
}
}
sort_table.sort_by_key(|i| {
@ -1116,7 +1126,7 @@ extern fn error_read(_: *mut u8, _: usize, _: *mut std::os::raw::c_void) -> isiz
#[cfg(test)]
extern fn valid_read(buf: *mut u8, size: usize, userdata: *mut std::os::raw::c_void) -> isize {
let mut input: &mut std::fs::File = unsafe { &mut *(userdata as *mut _) };
let input: &mut std::fs::File = unsafe { &mut *(userdata as *mut _) };
let mut buf = unsafe { std::slice::from_raw_parts_mut(buf, size) };
match input.read(&mut buf) {
@ -1280,6 +1290,8 @@ fn get_track_count_poisoned_parser() {
let mut count: u32 = 0;
let rv = mp4parse_get_track_count(parser, &mut count);
assert_eq!(rv, mp4parse_status::BAD_ARG);
mp4parse_free(parser);
}
}

View File

@ -0,0 +1,10 @@
[package]
name = "mp4parse_fallible"
version = "0.0.1"
authors = ["The Servo Project Developers"]
license = "MPL-2.0"
publish = false
[lib]
name = "mp4parse_fallible"
path = "lib.rs"

View File

@ -0,0 +1,2 @@
This is from https://github.com/servo/servo/tree/master/components/fallible
with modificaion for mp4 demuxer.

View File

@ -0,0 +1,92 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use std::mem;
use std::vec::Vec;
extern "C" {
fn realloc(ptr: *mut u8, bytes: usize) -> *mut u8;
fn malloc(bytes: usize) -> *mut u8;
}
pub trait FallibleVec<T> {
/// Append |val| to the end of |vec|. Returns Ok(()) on success,
/// Err(()) if it fails, which can only be due to lack of memory.
fn try_push(&mut self, value: T) -> Result<(), ()>;
/// Expand the vector size. Return Ok(()) on success, Err(()) if it
/// fails.
fn try_reserve(&mut self, new_cap: usize) -> Result<(), ()>;
}
/////////////////////////////////////////////////////////////////
// Vec
impl<T> FallibleVec<T> for Vec<T> {
#[inline]
fn try_push(&mut self, val: T) -> Result<(), ()> {
if self.capacity() == self.len() {
let old_cap: usize = self.capacity();
let new_cap: usize
= if old_cap == 0 { 4 } else { old_cap.checked_mul(2).ok_or(()) ? };
try_extend_vec(self, new_cap)?;
debug_assert!(self.capacity() > self.len());
}
self.push(val);
Ok(())
}
#[inline]
fn try_reserve(&mut self, cap: usize) -> Result<(), ()> {
let new_cap = cap + self.capacity();
try_extend_vec(self, new_cap)?;
debug_assert!(self.capacity() == new_cap);
Ok(())
}
}
#[inline(never)]
#[cold]
fn try_extend_vec<T>(vec: &mut Vec<T>, new_cap: usize) -> Result<(), ()> {
let old_ptr = vec.as_mut_ptr();
let old_len = vec.len();
let old_cap: usize = vec.capacity();
if old_cap > new_cap {
return Ok(());
}
let new_size_bytes
= new_cap.checked_mul(mem::size_of::<T>()).ok_or(()) ? ;
let new_ptr = unsafe {
if old_cap == 0 {
malloc(new_size_bytes)
} else {
realloc(old_ptr as *mut u8, new_size_bytes)
}
};
if new_ptr.is_null() {
return Err(());
}
let new_vec = unsafe {
Vec::from_raw_parts(new_ptr as *mut T, old_len, new_cap)
};
mem::forget(mem::replace(vec, new_vec));
Ok(())
}
#[test]
fn oom_test() {
let mut vec: Vec<char> = Vec::new();
match vec.try_reserve(std::usize::MAX) {
Ok(_) => panic!("it should be OOM"),
_ => (),
}
}

View File

@ -2,7 +2,7 @@
# Script to update mp4parse-rust sources to latest upstream
# Default version.
VER=81260ded506dce968716720e10544c510f37d222
VER=759cf90957b1fd5a01632b1aa90cead16c26d0db
# Accept version or commit from the command line.
if test -n "$1"; then
@ -38,6 +38,9 @@ cp _upstream/mp4parse/mp4parse_capi/Cargo.toml mp4parse_capi/
cp _upstream/mp4parse/mp4parse_capi/build.rs mp4parse_capi/
cp _upstream/mp4parse/mp4parse_capi/include/mp4parse.h include/
cp _upstream/mp4parse/mp4parse_capi/src/*.rs mp4parse_capi/src/
rm -rf mp4parse_fallible
mkdir -p mp4parse_fallible
cp _upstream/mp4parse/mp4parse_fallible/* mp4parse_fallible/
echo "Applying patches..."
patch -p4 < mp4parse-cargo.patch

View File

@ -802,6 +802,7 @@ version = "0.8.0"
dependencies = [
"bitreader 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"mp4parse_fallible 0.0.1",
"num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -818,6 +819,10 @@ dependencies = [
"num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "mp4parse_fallible"
version = "0.0.1"
[[package]]
name = "net2"
version = "0.2.31"

View File

@ -800,6 +800,7 @@ version = "0.8.0"
dependencies = [
"bitreader 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"mp4parse_fallible 0.0.1",
"num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -812,6 +813,10 @@ dependencies = [
"num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "mp4parse_fallible"
version = "0.0.1"
[[package]]
name = "net2"
version = "0.2.31"