bug 1207298 - update brotli library r=jfkthame

This commit is contained in:
Patrick McManus 2015-09-28 13:10:09 -04:00
parent 863594b840
commit 0ff66ad510
17 changed files with 10333 additions and 10030 deletions

View File

@ -14,4 +14,4 @@ The in-tree copy is updated by running
sh update.sh
from within the modules/brotli directory.
Current version: [commit 1dd66ef114fd244778d9dcb5da09c28b49a0df33].
Current version: [commit 933bb9bd800c8f5f7f6a02382d33c902a98ef73a].

View File

@ -4,7 +4,7 @@ include ../shared.mk
CFLAGS += -Wall
OBJS = bit_reader.o decode.o huffman.o state.o streams.o
OBJS = bit_reader.o decode.o dictionary.o huffman.o state.o streams.o
all : $(OBJS)

View File

@ -29,25 +29,28 @@ void BrotliInitBitReader(BrotliBitReader* const br, BrotliInput input) {
br->input_ = input;
br->val_ = 0;
br->bit_pos_ = 0;
br->bit_pos_ = sizeof(br->val_) << 3;
br->avail_in = 0;
br->eos_ = 0;
br->next_in = br->buf_;
}
void BrotliWarmupBitReader(BrotliBitReader* const br) {
size_t i;
br->val_ = 0;
for (i = 0; i < sizeof(br->val_); ++i) {
#if (BROTLI_64_BITS_LITTLE_ENDIAN)
br->val_ |= ((uint64_t)*br->next_in) << (8 * i);
#else
br->val_ |= ((uint32_t)*br->next_in) << (8 * i);
#endif
++br->next_in;
--br->avail_in;
int BrotliWarmupBitReader(BrotliBitReader* const br) {
size_t aligned_read_mask = (sizeof(br->val_) >> 1) - 1;
/* Fixing alignment after unaligned BrotliFillWindow would result accumulator
overflow. If unalignment is caused by BrotliSafeReadBits, then there is
enough space in accumulator to fix aligment. */
if (!BROTLI_ALIGNED_READ) {
aligned_read_mask = 0;
}
while (br->bit_pos_ == (sizeof(br->val_) << 3) ||
(((size_t)br->next_in) & aligned_read_mask) != 0) {
if (!br->avail_in) {
return 0;
}
BrotliPullByte(br);
}
return 1;
}
#if defined(__cplusplus) || defined(c_plusplus)

View File

@ -27,9 +27,10 @@
extern "C" {
#endif
#define BROTLI_MAX_NUM_BIT_READ 25
#define BROTLI_READ_SIZE 1024
#define BROTLI_IMPLICIT_ZEROES 128
/* 128 bytes, plus 8 bytes slack for valid 128-byte BrotliCheckInputAmount with
some bytes read in val_ of bit reader. */
#define BROTLI_IMPLICIT_ZEROES 136
#define BROTLI_IBUF_SIZE (BROTLI_READ_SIZE + BROTLI_IMPLICIT_ZEROES)
#define BROTLI_IBUF_MASK (BROTLI_READ_SIZE - 1)
@ -54,13 +55,14 @@ typedef struct {
uint8_t buf_[BROTLI_IBUF_SIZE];
} BrotliBitReader;
/* Initializes the bitreader fields. After this, BrotliReadInput then
BrotliWarmupBitReader must be used. */
/* Initializes the bitreader fields. */
void BrotliInitBitReader(BrotliBitReader* const br, BrotliInput input);
/* Initializes bit reading and bit position with the first input data available.
Requires that there is enough input available (BrotliCheckInputAmount). */
void BrotliWarmupBitReader(BrotliBitReader* const br);
/* Ensures that accumulator is not empty. May consume one byte of input.
Returns 0 if data is required but there is no input available.
For BROTLI_BUILD_PORTABLE this function also prepares bit reader for aligned
reading. */
int BrotliWarmupBitReader(BrotliBitReader* const br);
/* Pulls data from the input to the the read buffer.
@ -109,7 +111,14 @@ static BROTLI_INLINE int BrotliReadInput(
/* Returns amount of unread bytes the bit reader still has buffered from the
BrotliInput, including whole bytes in br->val_. */
static BROTLI_INLINE size_t BrotliGetRemainingBytes(BrotliBitReader* br) {
return br->avail_in + sizeof(br->val_) - (br->bit_pos_ >> 3);
size_t result = br->avail_in + sizeof(br->val_) - (br->bit_pos_ >> 3);
if (!br->eos_) {
return result;
}
if (result <= BROTLI_IMPLICIT_ZEROES) {
return 0;
}
return result - BROTLI_IMPLICIT_ZEROES;
}
/* Checks if there is at least num bytes left in the input ringbuffer (excluding
@ -120,12 +129,14 @@ static BROTLI_INLINE int BrotliCheckInputAmount(
return br->avail_in >= num;
}
/* Guarantees that there are at least n_bits in the buffer.
n_bits should be in the range [1..24] */
/* Guarantees that there are at least n_bits + 1 bits in accumulator.
Precondition: accumulator contains at least 1 bit.
n_bits should be in the range [1..24] for regular build. For portable
non-64-bit little endian build only 16 bits are safe to request. */
static BROTLI_INLINE void BrotliFillBitWindow(
BrotliBitReader* const br, int n_bits) {
#if (BROTLI_64_BITS_LITTLE_ENDIAN)
if (IS_CONSTANT(n_bits) && n_bits <= 8) {
if (!BROTLI_ALIGNED_READ && IS_CONSTANT(n_bits) && (n_bits <= 8)) {
if (br->bit_pos_ >= 56) {
br->val_ >>= 56;
br->bit_pos_ ^= 56; /* here same as -= 56 because of the if condition */
@ -133,7 +144,7 @@ static BROTLI_INLINE void BrotliFillBitWindow(
br->avail_in -= 7;
br->next_in += 7;
}
} else if (IS_CONSTANT(n_bits) && n_bits <= 16) {
} else if (!BROTLI_ALIGNED_READ && IS_CONSTANT(n_bits) && (n_bits <= 16)) {
if (br->bit_pos_ >= 48) {
br->val_ >>= 48;
br->bit_pos_ ^= 48; /* here same as -= 48 because of the if condition */
@ -151,7 +162,7 @@ static BROTLI_INLINE void BrotliFillBitWindow(
}
}
#elif (BROTLI_LITTLE_ENDIAN)
if (IS_CONSTANT(n_bits) && n_bits <= 8) {
if (!BROTLI_ALIGNED_READ && IS_CONSTANT(n_bits) && (n_bits <= 8)) {
if (br->bit_pos_ >= 24) {
br->val_ >>= 24;
br->bit_pos_ ^= 24; /* here same as -= 24 because of the if condition */
@ -167,27 +178,27 @@ static BROTLI_INLINE void BrotliFillBitWindow(
br->avail_in -= 2;
br->next_in += 2;
}
if (!IS_CONSTANT(n_bits) || (n_bits > 16)) {
if (br->bit_pos_ >= 8) {
br->val_ >>= 8;
br->bit_pos_ ^= 8; /* here same as -= 8 because of the if condition */
br->val_ |= ((uint32_t)*br->next_in) << 24;
--br->avail_in;
++br->next_in;
}
}
}
#else
while (br->bit_pos_ >= 8) {
br->val_ >>= 8;
br->val_ |= ((uint32_t)*br->next_in) << 24;
br->bit_pos_ -= 8;
--br->avail_in;
++br->next_in;
while (br->bit_pos_ >= 16) {
BrotliPullByte(br);
}
#endif
}
/* Pulls one byte of input to accumulator. */
static BROTLI_INLINE void BrotliPullByte(BrotliBitReader* const br) {
br->val_ >>= 8;
#if (BROTLI_64_BITS_LITTLE_ENDIAN)
br->val_ |= ((uint64_t)*br->next_in) << 56;
#else
br->val_ |= ((uint32_t)*br->next_in) << 24;
#endif
br->bit_pos_ -= 8;
--br->avail_in;
++br->next_in;
}
/* Like BrotliGetBits, but does not mask the result, it is only guaranteed
that it has minimum n_bits. */
static BROTLI_INLINE uint32_t BrotliGetBitsUnmasked(
@ -209,28 +220,81 @@ static BROTLI_INLINE void BrotliDropBits(
br->bit_pos_ += (uint32_t)n_bits;
}
/* Reads the specified number of bits from br and advances the bit pos. */
static BROTLI_INLINE uint32_t BrotliReadBits(
BrotliBitReader* const br, int n_bits) {
uint32_t val;
BrotliFillBitWindow(br, n_bits);
val = (uint32_t)(br->val_ >> br->bit_pos_) & BitMask(n_bits);
/* Reads the specified number of bits from br and advances the bit pos.
Precondition: accumulator MUST contain at least n_bits. */
static BROTLI_INLINE void BrotliTakeBits(
BrotliBitReader* const br, int n_bits, uint32_t* val) {
*val = (uint32_t)(br->val_ >> br->bit_pos_) & BitMask(n_bits);
#ifdef BROTLI_DECODE_DEBUG
printf("[BrotliReadBits] %d %d %d val: %6x\n",
(int)br->avail_in, (int)br->bit_pos_, n_bits, val);
(int)br->avail_in, (int)br->bit_pos_, n_bits, (int)*val);
#endif
br->bit_pos_ += (uint32_t)n_bits;
return val;
}
/* Reads the specified number of bits from br and advances the bit pos.
Assumes that there is enough input to perform BrotliFillBitWindow. */
static BROTLI_INLINE uint32_t BrotliReadBits(
BrotliBitReader* const br, int n_bits) {
if (BROTLI_64_BITS_LITTLE_ENDIAN || (n_bits <= 16)) {
uint32_t val;
BrotliFillBitWindow(br, n_bits);
BrotliTakeBits(br, n_bits, &val);
return val;
} else {
uint32_t low_val;
uint32_t high_val;
BrotliFillBitWindow(br, 16);
BrotliTakeBits(br, 16, &low_val);
BrotliFillBitWindow(br, 8);
BrotliTakeBits(br, n_bits - 16, &high_val);
return low_val | (high_val << 16);
}
}
/* Tries to read the specified amount of bits. Returns 0, if there is not
enough input. */
static BROTLI_INLINE int BrotliSafeReadBits(
BrotliBitReader* const br, int n_bits, uint32_t* val) {
while (br->bit_pos_ + (uint32_t)n_bits > (sizeof(br->val_) << 3)) {
if (br->avail_in == 0) {
return 0;
}
BrotliPullByte(br);
}
BrotliTakeBits(br, n_bits, val);
return 1;
}
/* Advances the bit reader position to the next byte boundary and verifies
that any skipped bits are set to zero. */
static BROTLI_INLINE int BrotliJumpToByteBoundary(BrotliBitReader* br) {
uint32_t new_bit_pos = (br->bit_pos_ + 7) & (uint32_t)(~7UL);
uint32_t pad_bits = BrotliReadBits(br, (int)(new_bit_pos - br->bit_pos_));
int pad_bits_count = (64 - (int)br->bit_pos_) & 0x7;
uint32_t pad_bits = 0;
if (pad_bits_count != 0) {
BrotliTakeBits(br, pad_bits_count, &pad_bits);
}
return pad_bits == 0;
}
/* Peeks a byte at specified offset.
Precondition: bit reader is parked to a byte boundary.
Returns -1 if operation is not feasible. */
static BROTLI_INLINE int BrotliPeekByte(BrotliBitReader* br, int offset) {
int bytes_left = (int)(sizeof(br->val_) - (br->bit_pos_ >> 3));
if (br->bit_pos_ & 7) {
return -1;
}
if (offset < bytes_left) {
return (br->val_ >> (br->bit_pos_ + (unsigned)(offset << 3))) & 0xFF;
}
offset -= bytes_left;
if (offset < br->avail_in) {
return br->next_in[offset];
}
return -1;
}
/* Copies remaining input bytes stored in the bit reader to the output. Value
num may not be larger than BrotliGetRemainingBytes. The bit reader must be
warmed up again after this. */
@ -246,9 +310,20 @@ static BROTLI_INLINE void BrotliCopyBytes(uint8_t* dest,
memcpy(dest, br->next_in, num);
br->avail_in -= (uint32_t)num;
br->next_in += num;
br->bit_pos_ = 0;
}
/* Checks that bit reader hasn't read after the end of input.
Returns 0 if bit reader has used implicit zeroes after the end of input. */
static BROTLI_INLINE int BrotliIsBitReaderOK(BrotliBitReader* br) {
size_t remaining_bytes =
br->avail_in + sizeof(br->val_) - (br->bit_pos_ >> 3);
return !br->eos_ || (remaining_bytes >= BROTLI_IMPLICIT_ZEROES);
}
#undef BROTLI_IMPLICIT_ZEROES
#undef BROTLI_IBUF_SIZE
#undef BROTLI_IBUF_MASK
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@ -22,7 +22,7 @@
CONTEXT_SIGNED: second-order context model tuned for signed integers.
The context id for the UTF8 context model is calculated as follows. If p1
and p2 are the previous two bytes, we calcualte the context as
and p2 are the previous two bytes, we calculate the context as
context = kContextLookup[p1] | kContextLookup[p2 + 256].

File diff suppressed because it is too large Load Diff

View File

@ -150,6 +150,7 @@ BrotliResult BrotliDecompressBufferStreaming(size_t* available_in,
void BrotliSetCustomDictionary(
size_t size, const uint8_t* dict, BrotliState* s);
/* Escalate internal functions visibility; for testing purposes only. */
void InverseMoveToFrontTransformForTesting(uint8_t* v, int l, BrotliState* s);

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -88,6 +88,7 @@ void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table,
offset[bits] = symbol;
bits++;
});
/* Symbols with code length 0 are placed after all other symbols. */
offset[0] = 17;
/* sort symbols by length, by symbol order within each length */
@ -101,8 +102,8 @@ void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table,
table_size = 1 << BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH;
/* special case code with only one value */
if (offset[BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH] == 0) {
/* Special case: all symbols but one have 0 code length. */
if (offset[0] == 0) {
code.bits = 0;
code.value = (uint16_t)sorted[0];
for (key = 0; key < table_size; ++key) {

View File

@ -53,12 +53,16 @@ int BrotliBuildHuffmanTable(HuffmanCode* root_table,
const uint16_t* const symbol_lists,
uint16_t *count_arg);
/* Builds a simple Huffman table. The num_symbols parameter is to be */
/* interpreted as follows: 0 means 1 symbol, 1 means 2 symbols, 2 means 3 */
/* symbols, 3 means 4 symbols with lengths 2,2,2,2, 4 means 4 symbols with */
/* lengths 1,2,3,3. */
int BrotliBuildSimpleHuffmanTable(HuffmanCode* table,
int root_bits,
uint16_t *symbols,
uint32_t num_symbols);
/* Contains a collection of huffman trees with the same alphabet size. */
/* Contains a collection of Huffman trees with the same alphabet size. */
typedef struct {
HuffmanCode** htrees;
HuffmanCode* codes;

View File

@ -13,7 +13,15 @@
limitations under the License.
*/
/* Macros for branch prediction. */
/* Macros for compiler / platform specific features and build options.
Build options are:
* BROTLI_BUILD_PORTABLE disables dangerous optimizations, like unaligned
read and overlapping memcpy; this reduces decompression speed by 5%
* BROTLI_DEBUG dumps file name and line number when decoder detects stream
or memory error
* BROTLI_DECODE_DEBUG enables asserts and dumps various state information
*/
#ifndef BROTLI_DEC_PORT_H_
#define BROTLI_DEC_PORT_H_
@ -33,6 +41,14 @@
#define __has_feature(x) 0
#endif
#ifdef BROTLI_BUILD_PORTABLE
#define BROTLI_ALIGNED_READ 1
#define BROTLI_SAFE_MEMMOVE 1
#else
#define BROTLI_ALIGNED_READ 0
#define BROTLI_SAFE_MEMMOVE 0
#endif
#define BROTLI_ASAN_BUILD __has_feature(address_sanitizer)
/* Define "PREDICT_TRUE" and "PREDICT_FALSE" macros for capable compilers.
@ -122,7 +138,7 @@ OR:
#define BROTLI_NOINLINE
#endif
#if BROTLI_ASAN_BUILD
#if BROTLI_ASAN_BUILD && !defined(BROTLI_BUILD_PORTABLE)
#define BROTLI_NO_ASAN __attribute__((no_sanitize("address"))) BROTLI_NOINLINE
#else
#define BROTLI_NO_ASAN

View File

@ -25,8 +25,12 @@ extern "C" {
void BrotliStateInit(BrotliState* s) {
s->state = BROTLI_STATE_UNINITED;
s->sub0_state = BROTLI_STATE_SUB0_NONE;
s->sub1_state = BROTLI_STATE_SUB1_NONE;
s->substate_metablock_header = BROTLI_STATE_METABLOCK_HEADER_NONE;
s->substate_tree_group = BROTLI_STATE_TREE_GROUP_NONE;
s->substate_context_map = BROTLI_STATE_CONTEXT_MAP_NONE;
s->substate_uncompressed = BROTLI_STATE_UNCOMPRESSED_NONE;
s->substate_huffman = BROTLI_STATE_HUFFMAN_NONE;
s->substate_decode_uint8 = BROTLI_STATE_DECODE_UINT8_NONE;
s->block_type_trees = NULL;
s->block_len_trees = NULL;
@ -45,10 +49,11 @@ void BrotliStateInit(BrotliState* s) {
s->distance_hgroup.codes = NULL;
s->distance_hgroup.htrees = NULL;
s->custom_dict = NULL;
s->custom_dict_size = 0;
s->input_end = 0;
s->is_last_metablock = 0;
s->window_bits = 0;
s->max_distance = 0;
s->dist_rb[0] = 16;

View File

@ -32,8 +32,8 @@ typedef enum {
BROTLI_STATE_UNINITED,
BROTLI_STATE_BITREADER_WARMUP,
BROTLI_STATE_METABLOCK_BEGIN,
BROTLI_STATE_METABLOCK_HEADER_1,
BROTLI_STATE_METABLOCK_HEADER_2,
BROTLI_STATE_METABLOCK_HEADER,
BROTLI_STATE_CONTEXT_MODES,
BROTLI_STATE_COMMAND_BEGIN,
BROTLI_STATE_COMMAND_INNER,
BROTLI_STATE_UNCOMPRESSED,
@ -46,6 +46,7 @@ typedef enum {
BROTLI_STATE_HUFFMAN_CODE_0,
BROTLI_STATE_HUFFMAN_CODE_1,
BROTLI_STATE_HUFFMAN_CODE_2,
BROTLI_STATE_HUFFMAN_CODE_3,
BROTLI_STATE_CONTEXT_MAP_1,
BROTLI_STATE_CONTEXT_MAP_2,
BROTLI_STATE_TREE_GROUP,
@ -53,23 +54,45 @@ typedef enum {
} BrotliRunningState;
typedef enum {
BROTLI_STATE_SUB0_NONE,
BROTLI_STATE_SUB0_UNCOMPRESSED_SHORT,
BROTLI_STATE_SUB0_UNCOMPRESSED_FILL,
BROTLI_STATE_SUB0_UNCOMPRESSED_COPY,
BROTLI_STATE_SUB0_UNCOMPRESSED_WARMUP,
BROTLI_STATE_SUB0_UNCOMPRESSED_WRITE_1,
BROTLI_STATE_SUB0_UNCOMPRESSED_WRITE_2,
BROTLI_STATE_SUB0_UNCOMPRESSED_WRITE_3,
BROTLI_STATE_SUB0_TREE_GROUP,
BROTLI_STATE_SUB0_CONTEXT_MAP_HUFFMAN,
BROTLI_STATE_SUB0_CONTEXT_MAPS
} BrotliRunningSub0State;
BROTLI_STATE_METABLOCK_HEADER_NONE,
BROTLI_STATE_METABLOCK_HEADER_EMPTY,
BROTLI_STATE_METABLOCK_HEADER_NIBBLES,
BROTLI_STATE_METABLOCK_HEADER_SIZE,
BROTLI_STATE_METABLOCK_HEADER_UNCOMPRESSED,
BROTLI_STATE_METABLOCK_HEADER_RESERVED,
BROTLI_STATE_METABLOCK_HEADER_BYTES,
BROTLI_STATE_METABLOCK_HEADER_METADATA
} BrotliRunningMetablockHeaderState;
typedef enum {
BROTLI_STATE_SUB1_NONE,
BROTLI_STATE_SUB1_HUFFMAN_LENGTH_SYMBOLS
} BrotliRunningSub1State;
BROTLI_STATE_UNCOMPRESSED_NONE,
BROTLI_STATE_UNCOMPRESSED_SHORT,
BROTLI_STATE_UNCOMPRESSED_COPY,
BROTLI_STATE_UNCOMPRESSED_WRITE
} BrotliRunningUncompressedState;
typedef enum {
BROTLI_STATE_TREE_GROUP_NONE,
BROTLI_STATE_TREE_GROUP_LOOP
} BrotliRunningTreeGroupState;
typedef enum {
BROTLI_STATE_CONTEXT_MAP_NONE,
BROTLI_STATE_CONTEXT_MAP_READ_PREFIX,
BROTLI_STATE_CONTEXT_MAP_HUFFMAN,
BROTLI_STATE_CONTEXT_MAP_DECODE
} BrotliRunningContextMapState;
typedef enum {
BROTLI_STATE_HUFFMAN_NONE,
BROTLI_STATE_HUFFMAN_LENGTH_SYMBOLS
} BrotliRunningHuffmanState;
typedef enum {
BROTLI_STATE_DECODE_UINT8_NONE,
BROTLI_STATE_DECODE_UINT8_SHORT,
BROTLI_STATE_DECODE_UINT8_LONG
} BrotliRunningDecodeUint8State;
typedef struct {
BrotliRunningState state;
@ -110,13 +133,15 @@ typedef struct {
int distance_postfix_bits;
int num_direct_distance_codes;
int distance_postfix_mask;
int num_dist_htrees;
uint8_t* dist_context_map;
uint8_t literal_htree_index;
HuffmanCode *literal_htree;
uint8_t literal_htree_index;
uint8_t dist_htree_index;
uint8_t repeat_code_len;
uint8_t prev_code_len;
int copy_length;
int distance_code;
@ -159,15 +184,20 @@ typedef struct {
int custom_dict_size;
/* less used attributes are in the end of this struct */
BrotliRunningSub0State sub0_state; /* State inside function call */
BrotliRunningSub1State sub1_state; /* State inside function call */
/* States inside function calls */
BrotliRunningMetablockHeaderState substate_metablock_header;
BrotliRunningTreeGroupState substate_tree_group;
BrotliRunningContextMapState substate_context_map;
BrotliRunningUncompressedState substate_uncompressed;
BrotliRunningHuffmanState substate_huffman;
BrotliRunningDecodeUint8State substate_decode_uint8;
int input_end;
uint8_t is_last_metablock;
uint8_t is_uncompressed;
uint8_t is_metadata;
uint8_t size_nibbles;
uint32_t window_bits;
/* For CopyUncompressedBlockToOutput */
int nbytes;
int num_literal_htrees;
uint8_t* context_map;
uint8_t* context_modes;

View File

@ -93,6 +93,16 @@ BrotliOutput BrotliFileOutput(FILE* f) {
return out;
}
int BrotliNullOutputFunction(void* data, const uint8_t* buf, size_t count) {
return (int)count;
}
BrotliOutput BrotliNullOutput() {
BrotliOutput out;
out.cb_ = BrotliNullOutputFunction;
out.data_ = NULL;
return out;
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */

View File

@ -93,6 +93,10 @@ BrotliInput BrotliFileInput(FILE* f);
int BrotliFileOutputFunction(void* data, const uint8_t* buf, size_t count);
BrotliOutput BrotliFileOutput(FILE* f);
/* Output callback that does nothing, always consumes the whole input. */
int BrotliNullOutputFunction(void* data, const uint8_t* buf, size_t count);
BrotliOutput BrotliNullOutput();
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

View File

@ -17,6 +17,7 @@ EXPORTS += [
UNIFIED_SOURCES += [
'dec/bit_reader.c',
'dec/decode.c',
'dec/dictionary.c',
'dec/huffman.c',
'dec/state.c',
'dec/streams.c',