Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2025-01-26 23:23:33 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1402350 - Update Brotli library to release 1.0.1. r=glandium

Browse Source
This commit is contained in:
Jonathan Kew 2017-09-28 08:58:54 +01:00
parent c34f5eafdb
commit aec295366b
40 changed files with 1809 additions and 958 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/brotli/README.mozilla
View File

@ -14,4 +14,4 @@ The in-tree copy is updated by running
sh update.sh
from within the modules/brotli directory.
Current version: [commit 46c1a881b41bb638c76247558aa04b1591af3aa7].
Current version: [commit 5b4769990dc14a2bd466d2599c946c5652cba4b2].

2
modules/brotli/common/constants.h
View File

@ -38,6 +38,8 @@
BROTLI_MAX_NDIRECT + \
(BROTLI_MAX_DISTANCE_BITS << \
(BROTLI_MAX_NPOSTFIX + 1)))
/* Distance that is guaranteed to be representable in any stream. */
#define BROTLI_MAX_DISTANCE 0x3FFFFFC
/* 7.1. Context modes and context ID lookup for literals */
/* "context IDs for literals are in the range of 0..63" */

432
modules/brotli/common/dictionary.bin Executable file
View File

File diff suppressed because one or more lines are too long

55
modules/brotli/common/dictionary.c
View File

@ -10,24 +10,8 @@
extern "C" {
#endif
static const BrotliDictionary kBrotliDictionary = {
/* size_bits_by_length */
{
0, 0, 0, 0, 10, 10, 11, 11,
10, 10, 10, 10, 10, 9, 9, 8,
7, 7, 8, 7, 7, 6, 6, 5,
5, 0, 0, 0, 0, 0, 0, 0
},
/* offsets_by_length */
{
0, 0, 0, 0, 0, 4096, 9216, 21504,
35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864,
104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280,
122016, 122784, 122784, 122784, 122784, 122784, 122784, 122784
},
/* data */
#ifndef BROTLI_EXTERNAL_DICTIONARY_DATA
static const uint8_t kBrotliDictionaryData[] =
{
116,105,109,101,100,111,119,110,108,105,102,101,108,101,102,116,98,97,99,107,99,
111,100,101,100,97,116,97,115,104,111,119,111,110,108,121,115,105,116,101,99,105
@ -5875,12 +5859,47 @@ static const BrotliDictionary kBrotliDictionary = {
,164,181,224,164,190,224,164,136,224,164,184,224,164,149,224,165,141,224,164,176
,224,164,191,224,164,175,224,164,164,224,164,190
}
;
#endif /* !BROTLI_EXTERNAL_DICTIONARY_DATA */
static BrotliDictionary kBrotliDictionary = {
/* size_bits_by_length */
{
0, 0, 0, 0, 10, 10, 11, 11,
10, 10, 10, 10, 10, 9, 9, 8,
7, 7, 8, 7, 7, 6, 6, 5,
5, 0, 0, 0, 0, 0, 0, 0
},
/* offsets_by_length */
{
0, 0, 0, 0, 0, 4096, 9216, 21504,
35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864,
104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280,
122016, 122784, 122784, 122784, 122784, 122784, 122784, 122784
},
/* data_size == sizeof(kBrotliDictionaryData) */
122784,
/* data */
#ifdef BROTLI_EXTERNAL_DICTIONARY_DATA
NULL
#else
kBrotliDictionaryData
#endif
};
const BrotliDictionary* BrotliGetDictionary() {
return &kBrotliDictionary;
}
void BrotliSetDictionaryData(const uint8_t* data) {
if (!!data && !kBrotliDictionary.data) {
kBrotliDictionary.data = data;
}
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

24
modules/brotli/common/dictionary.h
View File

@ -27,19 +27,33 @@ typedef struct BrotliDictionary {
* Dictionary consists of words with length of [4..24] bytes.
* Values at [0..3] and [25..31] indices should not be addressed.
*/
uint8_t size_bits_by_length[32];
const uint8_t size_bits_by_length[32];
/* assert(offset[i + 1] == offset[i] + (bits[i] ? (i << bits[i]) : 0)) */
uint32_t offsets_by_length[32];
const uint32_t offsets_by_length[32];
/* assert(data_size == offsets_by_length[31]) */
const size_t data_size;
/* Data array is not bound, and should obey to size_bits_by_length values.
Specified size matches default (RFC 7932) dictionary. */
/* assert(sizeof(data) == offsets_by_length[31]) */
uint8_t data[122784];
Specified size matches default (RFC 7932) dictionary. Its size is
defined by data_size */
const uint8_t* data;
} BrotliDictionary;
BROTLI_COMMON_API extern const BrotliDictionary* BrotliGetDictionary(void);
/**
* Sets dictionary data.
*
* When dictionary data is already set / present, this method is no-op.
*
* Dictionary data MUST be provided before BrotliGetDictionary is invoked.
* This method is used ONLY in multi-client environment (e.g. C + Java),
* to reduce storage by sharing single dictionary between implementations.
*/
BROTLI_COMMON_API void BrotliSetDictionaryData(const uint8_t* data);
#define BROTLI_MIN_DICTIONARY_WORD_LENGTH 4
#define BROTLI_MAX_DICTIONARY_WORD_LENGTH 24

2
modules/brotli/common/version.h
View File

@ -14,6 +14,6 @@
BrotliEncoderVersion methods. */
/* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */
#define BROTLI_VERSION 0x0006000
#define BROTLI_VERSION 0x1000001
#endif /* BROTLI_COMMON_VERSION_H_ */

97
modules/brotli/dec/decode.c
View File

@ -39,6 +39,11 @@ extern "C" {
#define HUFFMAN_TABLE_BITS 8U
#define HUFFMAN_TABLE_MASK 0xff
/* We need the slack region for the following reasons:
- doing up to two 16-byte copies for fast backward copying
- inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */
static const uint32_t kRingBufferWriteAheadSlack = 42;
static const uint8_t kCodeLengthCodeOrder[BROTLI_CODE_LENGTH_CODES] = {
1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15,
};
@ -52,6 +57,17 @@ static const uint8_t kCodeLengthPrefixValue[16] = {
0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5,
};
BROTLI_BOOL BrotliDecoderSetParameter(
BrotliDecoderState* state, BrotliDecoderParameter p, uint32_t value) {
switch (p) {
case BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION:
state->canny_ringbuffer_allocation = !!value ? 0 : 1;
return BROTLI_TRUE;
default: return BROTLI_FALSE;
}
}
BrotliDecoderState* BrotliDecoderCreateInstance(
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
BrotliDecoderState* state = 0;
@ -66,7 +82,6 @@ BrotliDecoderState* BrotliDecoderCreateInstance(
}
BrotliDecoderStateInitWithCustomAllocators(
state, alloc_func, free_func, opaque);
state->error_code = BROTLI_DECODER_NO_ERROR;
return state;
}
@ -1211,7 +1226,9 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE WriteRingBuffer(
BROTLI_LOG_UINT(to_write);
BROTLI_LOG_UINT(num_written);
s->partial_pos_out += num_written;
if (total_out) *total_out = s->partial_pos_out - (size_t)s->custom_dict_size;
if (total_out) {
*total_out = s->partial_pos_out;
}
if (num_written < to_write) {
if (s->ringbuffer_size == (1 << s->window_bits) || force) {
return BROTLI_DECODER_NEEDS_MORE_OUTPUT;
@ -1243,22 +1260,16 @@ static void BROTLI_NOINLINE WrapRingBuffer(BrotliDecoderState* s) {
Last two bytes of ring-buffer are initialized to 0, so context calculation
could be done uniformly for the first two and all other positions.
Custom dictionary, if any, is copied to the end of ring-buffer.
*/
static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
BrotliDecoderState* s) {
/* We need the slack region for the following reasons:
- doing up to two 16-byte copies for fast backward copying
- inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */
static const int kRingBufferWriteAheadSlack = 42;
uint8_t* old_ringbuffer = s->ringbuffer;
if (s->ringbuffer_size == s->new_ringbuffer_size) {
return BROTLI_TRUE;
}
s->ringbuffer = (uint8_t*)BROTLI_ALLOC(s, (size_t)(s->new_ringbuffer_size +
kRingBufferWriteAheadSlack));
s->ringbuffer = (uint8_t*)BROTLI_ALLOC(s, (size_t)(s->new_ringbuffer_size) +
kRingBufferWriteAheadSlack);
if (s->ringbuffer == 0) {
/* Restore previous value. */
s->ringbuffer = old_ringbuffer;
@ -1267,13 +1278,7 @@ static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
s->ringbuffer[s->new_ringbuffer_size - 2] = 0;
s->ringbuffer[s->new_ringbuffer_size - 1] = 0;
if (!old_ringbuffer) {
if (s->custom_dict) {
memcpy(s->ringbuffer, s->custom_dict, (size_t)s->custom_dict_size);
s->partial_pos_out = (size_t)s->custom_dict_size;
s->pos = s->custom_dict_size;
}
} else {
if (!!old_ringbuffer) {
memcpy(s->ringbuffer, old_ringbuffer, (size_t)s->pos);
BROTLI_FREE(s, old_ringbuffer);
}
@ -1362,16 +1367,20 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(
}
if (!s->ringbuffer) {
/* Custom dictionary counts as a "virtual" output. */
output_size = s->custom_dict_size;
output_size = 0;
} else {
output_size = s->pos;
}
output_size += s->meta_block_remaining_len;
min_size = min_size < output_size ? output_size : min_size;
while ((new_ringbuffer_size >> 1) >= min_size) {
new_ringbuffer_size >>= 1;
if (!!s->canny_ringbuffer_allocation) {
/* Reduce ring buffer size to save memory when server is unscrupulous.
In worst case memory usage might be 1.5x bigger for a short period of
ring buffer reallocation.*/
while ((new_ringbuffer_size >> 1) >= min_size) {
new_ringbuffer_size >>= 1;
}
}
s->new_ringbuffer_size = new_ringbuffer_size;
@ -1736,17 +1745,20 @@ CommandPostDecodeLiterals:
/* Apply copy of LZ77 back-reference, or static dictionary reference if
the distance is larger than the max LZ77 distance */
if (s->distance_code > s->max_distance) {
int address = s->distance_code - s->max_distance - 1;
if (i >= BROTLI_MIN_DICTIONARY_WORD_LENGTH &&
i <= BROTLI_MAX_DICTIONARY_WORD_LENGTH) {
int offset = (int)s->dictionary->offsets_by_length[i];
int word_id = s->distance_code - s->max_distance - 1;
uint32_t shift = s->dictionary->size_bits_by_length[i];
int mask = (int)BitMask(shift);
int word_idx = word_id & mask;
int transform_idx = word_id >> shift;
int word_idx = address & mask;
int transform_idx = address >> shift;
/* Compensate double distance-ring-buffer roll. */
s->dist_rb_idx += s->distance_context;
offset += word_idx * i;
if (BROTLI_PREDICT_FALSE(!s->dictionary->data)) {
return BROTLI_FAILURE(BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET);
}
if (transform_idx < kNumTransforms) {
const uint8_t* word = &s->dictionary->data[offset];
int len = i;
@ -1899,6 +1911,10 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
size_t* available_out, uint8_t** next_out, size_t* total_out) {
BrotliDecoderErrorCode result = BROTLI_DECODER_SUCCESS;
BrotliBitReader* br = &s->br;
/* Do not try to process further in a case of unrecoverable error. */
if ((int)s->error_code < 0) {
return BROTLI_DECODER_RESULT_ERROR;
}
if (*available_out && (!next_out || !*next_out)) {
return SaveErrorCode(
s, BROTLI_FAILURE(BROTLI_DECODER_ERROR_INVALID_ARGUMENTS));
@ -1919,7 +1935,13 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
if (result != BROTLI_DECODER_SUCCESS) { /* Error, needs more input/output */
if (result == BROTLI_DECODER_NEEDS_MORE_INPUT) {
if (s->ringbuffer != 0) { /* Pro-actively push output. */
WriteRingBuffer(s, available_out, next_out, total_out, BROTLI_TRUE);
BrotliDecoderErrorCode intermediate_result = WriteRingBuffer(s,
available_out, next_out, total_out, BROTLI_TRUE);
/* WriteRingBuffer checks s->meta_block_remaining_len validity. */
if ((int)intermediate_result < 0) {
result = intermediate_result;
break;
}
}
if (s->buffer_length != 0) { /* Used with internal buffer. */
if (br->avail_in == 0) { /* Successfully finished read transaction. */
@ -1993,11 +2015,6 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
}
/* Maximum distance, see section 9.1. of the spec. */
s->max_backward_distance = (1 << s->window_bits) - BROTLI_WINDOW_GAP;
/* Limit custom dictionary size. */
if (s->custom_dict_size >= s->max_backward_distance) {
s->custom_dict += s->custom_dict_size - s->max_backward_distance;
s->custom_dict_size = s->max_backward_distance;
}
/* Allocate memory for both block_type_trees and block_len_trees. */
s->block_type_trees = (HuffmanCode*)BROTLI_ALLOC(s,
@ -2294,16 +2311,11 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
return SaveErrorCode(s, result);
}
void BrotliDecoderSetCustomDictionary(
BrotliDecoderState* s, size_t size, const uint8_t* dict) {
if (size > (1u << 24)) {
return;
}
s->custom_dict = dict;
s->custom_dict_size = (int)size;
}
BROTLI_BOOL BrotliDecoderHasMoreOutput(const BrotliDecoderState* s) {
/* After unrecoverable error remaining output is considered nonsensical. */
if ((int)s->error_code < 0) {
return BROTLI_FALSE;
}
return TO_BROTLI_BOOL(
s->ringbuffer != 0 && UnwrittenBytes(s, BROTLI_FALSE) != 0);
}
@ -2313,17 +2325,20 @@ const uint8_t* BrotliDecoderTakeOutput(BrotliDecoderState* s, size_t* size) {
size_t available_out = *size ? *size : 1u << 24;
size_t requested_out = available_out;
BrotliDecoderErrorCode status;
if (s->ringbuffer == 0) {
if ((s->ringbuffer == 0) || ((int)s->error_code < 0)) {
*size = 0;
return 0;
}
WrapRingBuffer(s);
status = WriteRingBuffer(s, &available_out, &result, 0, BROTLI_TRUE);
/* Either WriteRingBuffer returns those "success" codes... */
if (status == BROTLI_DECODER_SUCCESS ||
status == BROTLI_DECODER_NEEDS_MORE_OUTPUT) {
*size = requested_out - available_out;
} else {
/* This might happen if previous decoder error code was ignored. */
/* ... or stream is broken. Normally this should be caught by
BrotliDecoderDecompressStream, this is just a safeguard. */
if ((int)status < 0) SaveErrorCode(s, status);
*size = 0;
result = 0;
}

9
modules/brotli/dec/state.c
View File

@ -41,6 +41,8 @@ void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s,
s->memory_manager_opaque = opaque;
}
s->error_code = 0; /* BROTLI_DECODER_NO_ERROR */
BrotliInitBitReader(&s->br);
s->state = BROTLI_STATE_UNINITED;
s->substate_metablock_header = BROTLI_STATE_METABLOCK_HEADER_NONE;
@ -81,11 +83,12 @@ void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s,
s->distance_hgroup.codes = NULL;
s->distance_hgroup.htrees = NULL;
s->custom_dict = NULL;
s->custom_dict_size = 0;
s->is_last_metablock = 0;
s->is_uncompressed = 0;
s->is_metadata = 0;
s->should_wrap_ringbuffer = 0;
s->canny_ringbuffer_allocation = 1;
s->window_bits = 0;
s->max_distance = 0;
s->dist_rb[0] = 16;

7
modules/brotli/dec/state.h
View File

@ -172,7 +172,7 @@ struct BrotliDecoderStateStruct {
uint32_t space;
HuffmanCode table[32];
/* List of of symbol chains. */
/* List of heads of symbol chains. */
uint16_t* symbol_lists;
/* Storage from symbol_lists. */
uint16_t symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 +
@ -197,10 +197,6 @@ struct BrotliDecoderStateStruct {
uint32_t mtf_upper_bound;
uint32_t mtf[64 + 1];
/* For custom dictionaries */
const uint8_t* custom_dict;
int custom_dict_size;
/* less used attributes are in the end of this struct */
/* States inside function calls */
BrotliRunningMetablockHeaderState substate_metablock_header;
@ -215,6 +211,7 @@ struct BrotliDecoderStateStruct {
unsigned int is_uncompressed : 1;
unsigned int is_metadata : 1;
unsigned int should_wrap_ringbuffer : 1;
unsigned int canny_ringbuffer_allocation : 1;
unsigned int size_nibbles : 8;
uint32_t window_bits;

8
modules/brotli/enc/backward_references.c
View File

@ -48,6 +48,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
#define EXPAND_CAT(a, b) CAT(a, b)
#define CAT(a, b) a ## b
#define FN(X) EXPAND_CAT(X, HASHER())
#define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER()))
#define PREFIX() N
#define HASHER() H2
/* NOLINTNEXTLINE(build/include) */
@ -94,6 +96,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
#include "./backward_references_inc.h"
#undef HASHER
#undef PREFIX
#undef EXPORT_FN
#undef FN
#undef CAT
#undef EXPAND_CAT
@ -113,11 +117,11 @@ void BrotliCreateBackwardReferences(const BrotliDictionary* dictionary,
switch (params->hasher.type) {
#define CASE_(N) \
case N: \
CreateBackwardReferencesH ## N(dictionary, \
CreateBackwardReferencesNH ## N(dictionary, \
kStaticDictionaryHash, num_bytes, position, ringbuffer, \
ringbuffer_mask, params, hasher, dist_cache, \
last_insert_len, commands, num_commands, num_literals); \
break;
return;
FOR_GENERIC_HASHERS(CASE_)
#undef CASE_
default:

86
modules/brotli/enc/backward_references_hq.c
View File

@ -300,6 +300,7 @@ static size_t ComputeMinimumCopyLength(const float start_cost,
static uint32_t ComputeDistanceShortcut(const size_t block_start,
const size_t pos,
const size_t max_backward,
const size_t gap,
const ZopfliNode* nodes) {
const size_t clen = ZopfliNodeCopyLength(&nodes[pos]);
const size_t ilen = nodes[pos].insert_length;
@ -311,8 +312,8 @@ static uint32_t ComputeDistanceShortcut(const size_t block_start,
does not update the last distances. */
if (pos == 0) {
return 0;
} else if (dist + clen <= block_start + pos &&
dist <= max_backward &&
} else if (dist + clen <= block_start + pos + gap &&
dist <= max_backward + gap &&
ZopfliNodeDistanceCode(&nodes[pos]) > 0) {
return (uint32_t)pos;
} else {
@ -350,12 +351,12 @@ static void ComputeDistanceCache(const size_t pos,
is eligible. */
static void EvaluateNode(
const size_t block_start, const size_t pos, const size_t max_backward_limit,
const int* starting_dist_cache, const ZopfliCostModel* model,
StartPosQueue* queue, ZopfliNode* nodes) {
const size_t gap, const int* starting_dist_cache,
const ZopfliCostModel* model, StartPosQueue* queue, ZopfliNode* nodes) {
/* Save cost, because ComputeDistanceCache invalidates it. */
float node_cost = nodes[pos].u.cost;
nodes[pos].u.shortcut = ComputeDistanceShortcut(
block_start, pos, max_backward_limit, nodes);
block_start, pos, max_backward_limit, gap, nodes);
if (node_cost <= ZopfliCostModelGetLiteralCosts(model, 0, pos)) {
PosData posdata;
posdata.pos = pos;
@ -385,9 +386,10 @@ static size_t UpdateNodes(
size_t min_len;
size_t result = 0;
size_t k;
size_t gap = 0;
EvaluateNode(block_start, pos, max_backward_limit, starting_dist_cache, model,
queue, nodes);
EvaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache,
model, queue, nodes);
{
const PosData* posdata = StartPosQueueAt(queue, 0);
@ -415,25 +417,31 @@ static size_t UpdateNodes(
const size_t backward =
(size_t)(posdata->distance_cache[idx] + kDistanceCacheOffset[j]);
size_t prev_ix = cur_ix - backward;
if (prev_ix >= cur_ix) {
size_t len = 0;
uint8_t continuation = ringbuffer[cur_ix_masked + best_len];
if (cur_ix_masked + best_len > ringbuffer_mask) {
break;
}
if (BROTLI_PREDICT_FALSE(backward > max_distance + gap)) {
continue;
}
if (BROTLI_PREDICT_FALSE(backward > max_distance)) {
continue;
}
prev_ix &= ringbuffer_mask;
if (backward <= max_distance) {
if (prev_ix >= cur_ix) {
continue;
}
if (cur_ix_masked + best_len > ringbuffer_mask ||
prev_ix + best_len > ringbuffer_mask ||
ringbuffer[cur_ix_masked + best_len] !=
ringbuffer[prev_ix + best_len]) {
prev_ix &= ringbuffer_mask;
if (prev_ix + best_len > ringbuffer_mask ||
continuation != ringbuffer[prev_ix + best_len]) {
continue;
}
len = FindMatchLengthWithLimit(&ringbuffer[prev_ix],
&ringbuffer[cur_ix_masked],
max_len);
} else {
continue;
}
{
const size_t len =
FindMatchLengthWithLimit(&ringbuffer[prev_ix],
&ringbuffer[cur_ix_masked],
max_len);
const float dist_cost = base_cost +
ZopfliCostModelGetDistanceCost(model, j);
size_t l;
@ -464,7 +472,8 @@ static size_t UpdateNodes(
for (j = 0; j < num_matches; ++j) {
BackwardMatch match = matches[j];
size_t dist = match.distance;
BROTLI_BOOL is_dictionary_match = TO_BROTLI_BOOL(dist > max_distance);
BROTLI_BOOL is_dictionary_match =
TO_BROTLI_BOOL(dist > max_distance + gap);
/* We already tried all possible last distance matches, so we can use
normal distance code here. */
size_t dist_code = dist + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
@ -526,11 +535,14 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
const ZopfliNode* nodes,
int* dist_cache,
size_t* last_insert_len,
const BrotliEncoderParams* params,
Command* commands,
size_t* num_literals) {
size_t pos = 0;
uint32_t offset = nodes[0].u.next;
size_t i;
size_t gap = 0;
BROTLI_UNUSED(params);
for (i = 0; offset != BROTLI_UINT32_MAX; i++) {
const ZopfliNode* next = &nodes[pos + offset];
size_t copy_length = ZopfliNodeCopyLength(next);
@ -546,11 +558,11 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
size_t len_code = ZopfliNodeLengthCode(next);
size_t max_distance =
BROTLI_MIN(size_t, block_start + pos, max_backward_limit);
BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance);
BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance + gap);
size_t dist_code = ZopfliNodeDistanceCode(next);
InitCommand(
&commands[i], insert_length, copy_length, len_code, dist_code);
InitCommand(&commands[i], insert_length,
copy_length, (int)len_code - (int)copy_length, dist_code);
if (!is_dictionary && dist_code > 0) {
dist_cache[3] = dist_cache[2];
@ -572,6 +584,7 @@ static size_t ZopfliIterate(size_t num_bytes,
size_t ringbuffer_mask,
const BrotliEncoderParams* params,
const size_t max_backward_limit,
const size_t gap,
const int* dist_cache,
const ZopfliCostModel* model,
const uint32_t* num_matches,
@ -600,8 +613,8 @@ static size_t ZopfliIterate(size_t num_bytes,
while (skip) {
i++;
if (i + 3 >= num_bytes) break;
EvaluateNode(
position, i, max_backward_limit, dist_cache, model, &queue, nodes);
EvaluateNode(position, i, max_backward_limit, gap, dist_cache, model,
&queue, nodes);
cur_match_pos += num_matches[i];
skip--;
}
@ -629,6 +642,7 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
const size_t store_end = num_bytes >= StoreLookaheadH10() ?
position + num_bytes - StoreLookaheadH10() + 1 : position;
size_t i;
size_t gap = 0;
nodes[0].length = 0;
nodes[0].u.cost = 0;
InitZopfliCostModel(m, &model, num_bytes);
@ -640,7 +654,8 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
const size_t pos = position + i;
const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
size_t num_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
ringbuffer_mask, pos, num_bytes - i, max_distance, params, matches);
ringbuffer_mask, pos, num_bytes - i, max_distance, gap, params,
matches);
size_t skip;
if (num_matches > 0 &&
BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) {
@ -662,8 +677,8 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
while (skip) {
i++;
if (i + HashTypeLengthH10() - 1 >= num_bytes) break;
EvaluateNode(
position, i, max_backward_limit, dist_cache, &model, &queue, nodes);
EvaluateNode(position, i, max_backward_limit, gap, dist_cache, &model,
&queue, nodes);
skip--;
}
}
@ -688,7 +703,7 @@ void BrotliCreateZopfliBackwardReferences(
dist_cache, hasher, nodes);
if (BROTLI_IS_OOM(m)) return;
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit, nodes,
dist_cache, last_insert_len, commands, num_literals);
dist_cache, last_insert_len, params, commands, num_literals);
BROTLI_FREE(m, nodes);
}
@ -712,6 +727,7 @@ void BrotliCreateHqZopfliBackwardReferences(
ZopfliCostModel model;
ZopfliNode* nodes;
BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size);
size_t gap = 0;
if (BROTLI_IS_OOM(m)) return;
for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) {
const size_t pos = position + i;
@ -725,14 +741,12 @@ void BrotliCreateHqZopfliBackwardReferences(
cur_match_pos + MAX_NUM_MATCHES_H10);
if (BROTLI_IS_OOM(m)) return;
num_found_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
ringbuffer_mask, pos, max_length, max_distance, params,
ringbuffer_mask, pos, max_length, max_distance, gap, params,
&matches[cur_match_pos]);
cur_match_end = cur_match_pos + num_found_matches;
for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {
assert(BackwardMatchLength(&matches[j]) <
assert(BackwardMatchLength(&matches[j]) <=
BackwardMatchLength(&matches[j + 1]));
assert(matches[j].distance > max_distance ||
matches[j].distance <= matches[j + 1].distance);
}
num_matches[i] = (uint32_t)num_found_matches;
if (num_found_matches > 0) {
@ -774,10 +788,10 @@ void BrotliCreateHqZopfliBackwardReferences(
*last_insert_len = orig_last_insert_len;
memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
*num_commands += ZopfliIterate(num_bytes, position, ringbuffer,
ringbuffer_mask, params, max_backward_limit, dist_cache,
ringbuffer_mask, params, max_backward_limit, gap, dist_cache,
&model, num_matches, matches, nodes);
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit,
nodes, dist_cache, last_insert_len, commands, num_literals);
nodes, dist_cache, last_insert_len, params, commands, num_literals);
}
CleanupZopfliCostModel(m, &model);
BROTLI_FREE(m, nodes);

13
modules/brotli/enc/backward_references_hq.h
View File

@ -83,14 +83,11 @@ BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
const BrotliEncoderParams* params, const size_t max_backward_limit,
const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes);
BROTLI_INTERNAL void BrotliZopfliCreateCommands(const size_t num_bytes,
const size_t block_start,
const size_t max_backward_limit,
const ZopfliNode* nodes,
int* dist_cache,
size_t* last_insert_len,
Command* commands,
size_t* num_literals);
BROTLI_INTERNAL void BrotliZopfliCreateCommands(
const size_t num_bytes, const size_t block_start,
const size_t max_backward_limit, const ZopfliNode* nodes,
int* dist_cache, size_t* last_insert_len, const BrotliEncoderParams* params,
Command* commands, size_t* num_literals);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */

50
modules/brotli/enc/backward_references_inc.h
View File

@ -5,11 +5,11 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN */
/* template parameters: EXPORT_FN, FN */
static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
size_t num_bytes, size_t position,
static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
const BrotliDictionary* dictionary,
const uint16_t* dictionary_hash, size_t num_bytes, size_t position,
const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
@ -27,6 +27,7 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
const size_t random_heuristics_window_size =
LiteralSpreeLengthForSparseSearch(params);
size_t apply_random_heuristics = position + random_heuristics_window_size;
const size_t gap = 0;
/* Minimum score to accept a backward reference. */
const score_t kMinScore = BROTLI_SCORE_BASE + 100;
@ -38,29 +39,29 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
HasherSearchResult sr;
sr.len = 0;
sr.len_x_code = 0;
sr.len_code_delta = 0;
sr.distance = 0;
sr.score = kMinScore;
if (FN(FindLongestMatch)(hasher, dictionary, dictionary_hash,
ringbuffer, ringbuffer_mask, dist_cache,
position, max_length, max_distance, &sr)) {
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
ringbuffer_mask, dist_cache, position,
max_length, max_distance, gap, &sr);
if (sr.score > kMinScore) {
/* Found a match. Let's look for something even better ahead. */
int delayed_backward_references_in_row = 0;
--max_length;
for (;; --max_length) {
const score_t cost_diff_lazy = 175;
BROTLI_BOOL is_match_found;
HasherSearchResult sr2;
sr2.len = params->quality < MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH ?
BROTLI_MIN(size_t, sr.len - 1, max_length) : 0;
sr2.len_x_code = 0;
sr2.len_code_delta = 0;
sr2.distance = 0;
sr2.score = kMinScore;
max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
is_match_found = FN(FindLongestMatch)(hasher, dictionary,
dictionary_hash, ringbuffer, ringbuffer_mask, dist_cache,
position + 1, max_length, max_distance, &sr2);
if (is_match_found && sr2.score >= sr.score + cost_diff_lazy) {
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
ringbuffer_mask, dist_cache, position + 1,
max_length, max_distance, gap, &sr2);
if (sr2.score >= sr.score + cost_diff_lazy) {
/* Ok, let's just write one byte for now and start a match from the
next byte. */
++position;
@ -80,24 +81,33 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
/* The first 16 codes are special short-codes,
and the minimum offset is 1. */
size_t distance_code =
ComputeDistanceCode(sr.distance, max_distance, dist_cache);
if (sr.distance <= max_distance && distance_code > 0) {
ComputeDistanceCode(sr.distance, max_distance + gap, dist_cache);
if ((sr.distance <= (max_distance + gap)) && distance_code > 0) {
dist_cache[3] = dist_cache[2];
dist_cache[2] = dist_cache[1];
dist_cache[1] = dist_cache[0];
dist_cache[0] = (int)sr.distance;
FN(PrepareDistanceCache)(hasher, dist_cache);
}
InitCommand(commands++, insert_length, sr.len, sr.len ^ sr.len_x_code,
InitCommand(commands++, insert_length, sr.len, sr.len_code_delta,
distance_code);
}
*num_literals += insert_length;
insert_length = 0;
/* Put the hash keys into the table, if there are enough bytes left.
Depending on the hasher implementation, it can push all positions
in the given range or only a subset of them. */
FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, position + 2,
BROTLI_MIN(size_t, position + sr.len, store_end));
in the given range or only a subset of them.
Avoid hash poisoning with RLE data. */
{
size_t range_start = position + 2;
size_t range_end = BROTLI_MIN(size_t, position + sr.len, store_end);
if (sr.distance < (sr.len >> 2)) {
range_start = BROTLI_MIN(size_t, range_end, BROTLI_MAX(size_t,
range_start, position + sr.len - (sr.distance << 2)));
}
FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, range_start,
range_end);
}
position += sr.len;
} else {
++insert_length;

6
modules/brotli/enc/block_splitter.c
View File

@ -74,11 +74,9 @@ static void CopyLiteralsToByteArray(const Command* cmds,
}
}
static BROTLI_INLINE unsigned int MyRand(unsigned int* seed) {
static BROTLI_INLINE uint32_t MyRand(uint32_t* seed) {
/* Initial seed should be 7. In this case, loop length is (1 << 29). */
*seed *= 16807U;
if (*seed == 0) {
*seed = 1;
}
return *seed;
}

7
modules/brotli/enc/block_splitter_inc.h
View File

@ -13,7 +13,7 @@ static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
size_t stride,
size_t num_histograms,
HistogramType* histograms) {
unsigned int seed = 7;
uint32_t seed = 7;
size_t block_length = length / num_histograms;
size_t i;
FN(ClearHistograms)(histograms, num_histograms);
@ -29,14 +29,13 @@ static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
}
}
static void FN(RandomSample)(unsigned int* seed,
static void FN(RandomSample)(uint32_t* seed,
const DataType* data,
size_t length,
size_t stride,
HistogramType* sample) {
size_t pos = 0;
if (stride >= length) {
pos = 0;
stride = length;
} else {
pos = MyRand(seed) % (length - stride + 1);
@ -50,7 +49,7 @@ static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
HistogramType* histograms) {
size_t iters =
kIterMulForRefining * length / stride + kMinItersForRefining;
unsigned int seed = 7;
uint32_t seed = 7;
size_t iter;
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms;
for (iter = 0; iter < iters; ++iter) {

11
modules/brotli/enc/brotli_bit_stream.c
View File

@ -1325,17 +1325,6 @@ void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
}
}
void BrotliStoreSyncMetaBlock(size_t* BROTLI_RESTRICT storage_ix,
uint8_t* BROTLI_RESTRICT storage) {
/* Empty metadata meta-block bit pattern:
1 bit: is_last (0)
2 bits: num nibbles (3)
1 bit: reserved (0)
2 bits: metadata length bytes (0) */
BrotliWriteBits(6, 6, storage_ix, storage);
JumpToByteBoundary(storage_ix, storage);
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

4
modules/brotli/enc/brotli_bit_stream.h
View File

@ -96,10 +96,6 @@ BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
BROTLI_BOOL is_final_block, const uint8_t* input, size_t position,
size_t mask, size_t len, size_t* storage_ix, uint8_t* storage);
/* Stores an empty metadata meta-block and syncs to a byte boundary. */
BROTLI_INTERNAL void BrotliStoreSyncMetaBlock(size_t* storage_ix,
uint8_t* storage);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

2
modules/brotli/enc/cluster_inc.h
View File

@ -17,6 +17,8 @@ BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)(
size_t* num_pairs) CODE({
BROTLI_BOOL is_good_pair = BROTLI_FALSE;
HistogramPair p;
p.idx1 = p.idx2 = 0;
p.cost_diff = p.cost_combo = 0;
if (idx1 == idx2) {
return;
}

13
modules/brotli/enc/command.h
View File

@ -114,17 +114,19 @@ typedef struct Command {
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
static BROTLI_INLINE void InitCommand(Command* self, size_t insertlen,
size_t copylen, size_t copylen_code, size_t distance_code) {
size_t copylen, int copylen_code_delta, size_t distance_code) {
/* Don't rely on signed int representation, use honest casts. */
uint32_t delta = (uint8_t)((int8_t)copylen_code_delta);
self->insert_len_ = (uint32_t)insertlen;
self->copy_len_ = (uint32_t)(copylen | ((copylen_code ^ copylen) << 24));
self->copy_len_ = (uint32_t)(copylen | (delta << 24));
/* The distance prefix and extra bits are stored in this Command as if
npostfix and ndirect were 0, they are only recomputed later after the
clustering if needed. */
PrefixEncodeCopyDistance(
distance_code, 0, 0, &self->dist_prefix_, &self->dist_extra_);
GetLengthCode(
insertlen, copylen_code, TO_BROTLI_BOOL(self->dist_prefix_ == 0),
&self->cmd_prefix_);
insertlen, (size_t)((int)copylen + copylen_code_delta),
TO_BROTLI_BOOL(self->dist_prefix_ == 0), &self->cmd_prefix_);
}
static BROTLI_INLINE void InitInsertCommand(Command* self, size_t insertlen) {
@ -167,7 +169,8 @@ static BROTLI_INLINE uint32_t CommandCopyLen(const Command* self) {
}
static BROTLI_INLINE uint32_t CommandCopyLenCode(const Command* self) {
return (self->copy_len_ & 0xFFFFFF) ^ (self->copy_len_ >> 24);
int32_t delta = (int8_t)((uint8_t)(self->copy_len_ >> 24));
return (uint32_t)((int32_t)(self->copy_len_ & 0xFFFFFF) + delta);
}
#if defined(__cplusplus) || defined(c_plusplus)

6
modules/brotli/enc/compress_fragment.c
View File

@ -42,7 +42,7 @@ extern "C" {
static const uint32_t kHashMul32 = 0x1e35a7bd;
static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(p) << 24) * kHashMul32;
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(p) << 24) * kHashMul32;
return (uint32_t)(h >> shift);
}
@ -603,7 +603,7 @@ trawl:
compression we first update "table" with the hashes of some positions
within the last copy. */
{
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 3);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 3);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash = HashBytesAtOffset(input_bytes, 3, shift);
table[prev_hash] = (int)(ip - base_ip - 3);
@ -640,7 +640,7 @@ trawl:
compression we first update "table" with the hashes of some positions
within the last copy. */
{
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 3);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 3);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash = HashBytesAtOffset(input_bytes, 3, shift);
table[prev_hash] = (int)(ip - base_ip - 3);

10
modules/brotli/enc/compress_fragment_two_pass.c
View File

@ -41,7 +41,7 @@ extern "C" {
static const uint32_t kHashMul32 = 0x1e35a7bd;
static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(p) << 16) * kHashMul32;
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(p) << 16) * kHashMul32;
return (uint32_t)(h >> shift);
}
@ -346,7 +346,7 @@ trawl:
/* We could immediately start working at ip now, but to improve
compression we first update "table" with the hashes of some
positions within the last copy. */
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 5);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 5);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash;
table[prev_hash] = (int)(ip - base_ip - 5);
@ -354,7 +354,7 @@ trawl:
table[prev_hash] = (int)(ip - base_ip - 4);
prev_hash = HashBytesAtOffset(input_bytes, 2, shift);
table[prev_hash] = (int)(ip - base_ip - 3);
input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 2);
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 2);
cur_hash = HashBytesAtOffset(input_bytes, 2, shift);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
table[prev_hash] = (int)(ip - base_ip - 2);
@ -386,7 +386,7 @@ trawl:
/* We could immediately start working at ip now, but to improve
compression we first update "table" with the hashes of some
positions within the last copy. */
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 5);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 5);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash;
table[prev_hash] = (int)(ip - base_ip - 5);
@ -394,7 +394,7 @@ trawl:
table[prev_hash] = (int)(ip - base_ip - 4);
prev_hash = HashBytesAtOffset(input_bytes, 2, shift);
table[prev_hash] = (int)(ip - base_ip - 3);
input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 2);
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 2);
cur_hash = HashBytesAtOffset(input_bytes, 2, shift);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
table[prev_hash] = (int)(ip - base_ip - 2);

62
modules/brotli/enc/encode.c
View File

@ -385,8 +385,7 @@ static void ChooseContextMap(int quality,
context values, based on the entropy reduction of histograms over the
first 5 bits of literals. */
static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
size_t start_pos, size_t length, size_t mask, int quality,
size_t size_hint, ContextType* literal_context_mode,
size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
size_t* num_literal_contexts, const uint32_t** literal_context_map) {
static const uint32_t kStaticContextMapComplexUTF8[64] = {
11, 11, 12, 12, /* 0 special */
@ -457,7 +456,6 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
if (entropy[2] > 3.0 || entropy[1] - entropy[2] < 0.2) {
return BROTLI_FALSE;
} else {
*literal_context_mode = CONTEXT_UTF8;
*num_literal_contexts = 13;
*literal_context_map = kStaticContextMapComplexUTF8;
return BROTLI_TRUE;
@ -466,13 +464,12 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
}
static void DecideOverLiteralContextModeling(const uint8_t* input,
size_t start_pos, size_t length, size_t mask, int quality,
size_t size_hint, ContextType* literal_context_mode,
size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
size_t* num_literal_contexts, const uint32_t** literal_context_map) {
if (quality < MIN_QUALITY_FOR_CONTEXT_MODELING || length < 64) {
return;
} else if (ShouldUseComplexStaticContextMap(
input, start_pos, length, mask, quality, size_hint, literal_context_mode,
input, start_pos, length, mask, quality, size_hint,
num_literal_contexts, literal_context_map)) {
/* Context map was already set, nothing else to do. */
} else {
@ -492,7 +489,6 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
prev = lut[literal >> 6] * 3;
}
}
*literal_context_mode = CONTEXT_UTF8;
ChooseContextMap(quality, &bigram_prefix_histo[0], num_literal_contexts,
literal_context_map);
}
@ -596,7 +592,7 @@ static void WriteMetaBlockInternal(MemoryManager* m,
if (!params->disable_literal_context_modeling) {
DecideOverLiteralContextModeling(
data, wrapped_last_flush_pos, bytes, mask, params->quality,
params->size_hint, &literal_context_mode, &num_literal_contexts,
params->size_hint, &num_literal_contexts,
&literal_context_map);
}
BrotliBuildMetaBlockGreedy(m, data, wrapped_last_flush_pos, mask,
@ -832,36 +828,6 @@ static void CopyInputToRingBuffer(BrotliEncoderState* s,
}
}
void BrotliEncoderSetCustomDictionary(BrotliEncoderState* s, size_t size,
const uint8_t* dict) {
size_t max_dict_size = BROTLI_MAX_BACKWARD_LIMIT(s->params.lgwin);
size_t dict_size = size;
MemoryManager* m = &s->memory_manager_;
if (!EnsureInitialized(s)) return;
if (dict_size == 0 ||
s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
return;
}
if (size > max_dict_size) {
dict += size - max_dict_size;
dict_size = max_dict_size;
}
CopyInputToRingBuffer(s, dict_size, dict);
s->last_flush_pos_ = dict_size;
s->last_processed_pos_ = dict_size;
if (dict_size > 0) {
s->prev_byte_ = dict[dict_size - 1];
}
if (dict_size > 1) {
s->prev_byte2_ = dict[dict_size - 2];
}
HasherPrependCustomDictionary(m, &s->hasher_, &s->params, dict_size, dict);
if (BROTLI_IS_OOM(m)) return;
}
/* Marks all input as processed.
Returns true if position wrapping occurs. */
static BROTLI_BOOL UpdateLastProcessedPos(BrotliEncoderState* s) {
@ -1208,7 +1174,8 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
}
BrotliZopfliCreateCommands(block_size, block_start, max_backward_limit,
&nodes[0], dist_cache, &last_insert_len,
&commands[num_commands], &num_literals);
&params, &commands[num_commands],
&num_literals);
num_commands += path_size;
block_start += block_size;
metablock_size += block_size;
@ -1792,23 +1759,6 @@ uint32_t BrotliEncoderVersion(void) {
return BROTLI_VERSION;
}
/* DEPRECATED >>> */
size_t BrotliEncoderInputBlockSize(BrotliEncoderState* s) {
return InputBlockSize(s);
}
void BrotliEncoderCopyInputToRingBuffer(BrotliEncoderState* s,
const size_t input_size,
const uint8_t* input_buffer) {
CopyInputToRingBuffer(s, input_size, input_buffer);
}
BROTLI_BOOL BrotliEncoderWriteData(
BrotliEncoderState* s, const BROTLI_BOOL is_last,
const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output) {
return EncodeData(s, is_last, force_flush, out_size, output);
}
/* <<< DEPRECATED */
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

0
modules/brotli/enc/fast_log.h Executable file → Normal file
View File

10
modules/brotli/enc/find_match_length.h
View File

@ -17,7 +17,7 @@ extern "C" {
#endif
/* Separate implementation for little-endian 64-bit targets, for speed. */
#if defined(__GNUC__) && defined(_LP64) && defined(IS_LITTLE_ENDIAN)
#if defined(__GNUC__) && defined(_LP64) && defined(BROTLI_LITTLE_ENDIAN)
static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
const uint8_t* s2,
@ -25,13 +25,13 @@ static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
size_t matched = 0;
size_t limit2 = (limit >> 3) + 1; /* + 1 is for pre-decrement in while */
while (BROTLI_PREDICT_TRUE(--limit2)) {
if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64(s2) ==
BROTLI_UNALIGNED_LOAD64(s1 + matched))) {
if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64LE(s2) ==
BROTLI_UNALIGNED_LOAD64LE(s1 + matched))) {
s2 += 8;
matched += 8;
} else {
uint64_t x =
BROTLI_UNALIGNED_LOAD64(s2) ^ BROTLI_UNALIGNED_LOAD64(s1 + matched);
uint64_t x = BROTLI_UNALIGNED_LOAD64LE(s2) ^
BROTLI_UNALIGNED_LOAD64LE(s1 + matched);
size_t matching_bits = (size_t)__builtin_ctzll(x);
matched += matching_bits >> 3;
return matched;

38
modules/brotli/enc/hash.h
View File

@ -62,9 +62,9 @@ static const uint64_t kCutoffTransforms =
typedef struct HasherSearchResult {
size_t len;
size_t len_x_code; /* == len ^ len_code */
size_t distance;
score_t score;
int len_code_delta; /* == len_code - len */
} HasherSearchResult;
/* kHashMul32 multiplier has these properties:
@ -173,27 +173,29 @@ static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
backward = max_backward + dist + 1 +
(transform_id << dictionary->size_bits_by_length[len]);
}
if (backward >= BROTLI_MAX_DISTANCE) {
return BROTLI_FALSE;
}
score = BackwardReferenceScore(matchlen, backward);
if (score < out->score) {
return BROTLI_FALSE;
}
out->len = matchlen;
out->len_x_code = len ^ matchlen;
out->len_code_delta = (int)len - (int)matchlen;
out->distance = backward;
out->score = score;
return BROTLI_TRUE;
}
static BROTLI_INLINE BROTLI_BOOL SearchInStaticDictionary(
static BROTLI_INLINE void SearchInStaticDictionary(
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
HasherHandle handle, const uint8_t* data, size_t max_length,
size_t max_backward, HasherSearchResult* out, BROTLI_BOOL shallow) {
size_t key;
size_t i;
BROTLI_BOOL is_match_found = BROTLI_FALSE;
HasherCommon* self = GetHasherCommon(handle);
if (self->dict_num_matches < (self->dict_num_lookups >> 7)) {
return BROTLI_FALSE;
return;
}
key = Hash14(data) << 1;
for (i = 0; i < (shallow ? 1u : 2u); ++i, ++key) {
@ -204,11 +206,9 @@ static BROTLI_INLINE BROTLI_BOOL SearchInStaticDictionary(
dictionary, item, data, max_length, max_backward, out);
if (item_matches) {
self->dict_num_matches++;
is_match_found = BROTLI_TRUE;
}
}
}
return is_match_found;
}
typedef struct BackwardMatch {
@ -420,30 +420,6 @@ static BROTLI_INLINE void HasherSetup(MemoryManager* m, HasherHandle* handle,
}
}
/* Custom LZ77 window. */
static BROTLI_INLINE void HasherPrependCustomDictionary(
MemoryManager* m, HasherHandle* handle, BrotliEncoderParams* params,
const size_t size, const uint8_t* dict) {
size_t overlap;
size_t i;
HasherHandle self;
HasherSetup(m, handle, params, dict, 0, size, BROTLI_FALSE);
if (BROTLI_IS_OOM(m)) return;
self = *handle;
switch (GetHasherCommon(self)->params.type) {
#define PREPEND_(N) \
case N: \
overlap = (StoreLookaheadH ## N()) - 1; \
for (i = 0; i + overlap < size; i++) { \
StoreH ## N(self, dict, ~(size_t)0, i); \
} \
break;
FOR_ALL_HASHERS(PREPEND_)
#undef PREPEND_
default: break;
}
}
static BROTLI_INLINE void InitOrStitchToPreviousBlock(
MemoryManager* m, HasherHandle* handle, const uint8_t* data, size_t mask,
BrotliEncoderParams* params, size_t position, size_t input_size,

19
modules/brotli/enc/hash_forgetful_chain_inc.h
View File

@ -152,24 +152,24 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
Returns 1 when match is found, otherwise 0. */
static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length, const size_t max_backward,
HasherSearchResult* BROTLI_RESTRICT out) {
const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) {
HashForgetfulChain* self = FN(Self)(handle);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
BROTLI_BOOL is_match_found = BROTLI_FALSE;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
const size_t key = FN(HashBytes)(&data[cur_ix_masked]);
const uint8_t tiny_hash = (uint8_t)(key);
out->len = 0;
out->len_x_code = 0;
out->len_code_delta = 0;
/* Try last distance first. */
for (i = 0; i < NUM_LAST_DISTANCES_TO_CHECK; ++i) {
const size_t backward = (size_t)distance_cache[i];
@ -194,7 +194,6 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
@ -234,19 +233,17 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
}
FN(Store)(handle, data, ring_buffer_mask, cur_ix);
}
if (!is_match_found) {
is_match_found = SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
if (out->score == min_score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
}
return is_match_found;
}
#undef BANK_SIZE

21
modules/brotli/enc/hash_longest_match64_inc.h
View File

@ -23,7 +23,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t *data,
const uint64_t mask,
const int shift) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(data) & mask) * kHashMul64Long;
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(data) & mask) * kHashMul64Long;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return (uint32_t)(h >> shift);
@ -156,25 +156,25 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
Returns true when match is found, otherwise false. */
static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t max_length, const size_t max_backward, const size_t gap,
HasherSearchResult* BROTLI_RESTRICT out) {
HasherCommon* common = GetHasherCommon(handle);
HashLongestMatch* self = FN(Self)(handle);
uint16_t* num = FN(Num)(self);
uint32_t* buckets = FN(Buckets)(self);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
BROTLI_BOOL is_match_found = BROTLI_FALSE;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
out->len = 0;
out->len_x_code = 0;
out->len_code_delta = 0;
/* Try last distance first. */
for (i = 0; i < (size_t)common->params.num_last_distances_to_check; ++i) {
const size_t backward = (size_t)distance_cache[i];
@ -209,7 +209,6 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
@ -250,7 +249,6 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
@ -258,12 +256,11 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
bucket[num[key] & self->block_mask_] = (uint32_t)cur_ix;
++num[key];
}
if (!is_match_found) {
is_match_found = SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
if (min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
}
return is_match_found;
}
#undef HashLongestMatch

19
modules/brotli/enc/hash_longest_match_inc.h
View File

@ -149,25 +149,25 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
Returns true when match is found, otherwise false. */
static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t max_length, const size_t max_backward, const size_t gap,
HasherSearchResult* BROTLI_RESTRICT out) {
HasherCommon* common = GetHasherCommon(handle);
HashLongestMatch* self = FN(Self)(handle);
uint16_t* num = FN(Num)(self);
uint32_t* buckets = FN(Buckets)(self);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
BROTLI_BOOL is_match_found = BROTLI_FALSE;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
out->len = 0;
out->len_x_code = 0;
out->len_code_delta = 0;
/* Try last distance first. */
for (i = 0; i < (size_t)common->params.num_last_distances_to_check; ++i) {
const size_t backward = (size_t)distance_cache[i];
@ -202,7 +202,6 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
@ -242,7 +241,6 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
@ -250,12 +248,11 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
bucket[num[key] & self->block_mask_] = (uint32_t)cur_ix;
++num[key];
}
if (!is_match_found) {
is_match_found = SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
if (min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
}
return is_match_found;
}
#undef HashLongestMatch

58
modules/brotli/enc/hash_longest_match_quickly_inc.h
View File

@ -22,7 +22,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
the address in. The HashLongestMatch and HashLongestMatchQuickly
classes have separate, different implementations of hashing. */
static uint32_t FN(HashBytes)(const uint8_t* data) {
const uint64_t h = ((BROTLI_UNALIGNED_LOAD64(data) << (64 - 8 * HASH_LEN)) *
const uint64_t h = ((BROTLI_UNALIGNED_LOAD64LE(data) << (64 - 8 * HASH_LEN)) *
kHashMul64);
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
@ -123,24 +123,24 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
Returns true if match is found, otherwise false. */
static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(
HasherHandle handle, const BrotliDictionary* dictionary,
const uint16_t* dictionary_hash, const uint8_t* BROTLI_RESTRICT data,
const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length, const size_t max_backward,
HasherSearchResult* BROTLI_RESTRICT out) {
const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) {
HashLongestMatchQuickly* self = FN(Self)(handle);
const size_t best_len_in = out->len;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
int compare_char = data[cur_ix_masked + best_len_in];
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = best_len_in;
size_t cached_backward = (size_t)distance_cache[0];
size_t prev_ix = cur_ix - cached_backward;
BROTLI_BOOL is_match_found = BROTLI_FALSE;
out->len_x_code = 0;
out->len_code_delta = 0;
if (prev_ix < cur_ix) {
prev_ix &= (uint32_t)ring_buffer_mask;
if (compare_char == data[prev_ix + best_len]) {
@ -148,17 +148,18 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(
&data[cur_ix_masked],
max_length);
if (len >= 4) {
best_score = BackwardReferenceScoreUsingLastDistance(len);
best_len = len;
out->len = len;
out->distance = cached_backward;
out->score = best_score;
compare_char = data[cur_ix_masked + best_len];
if (BUCKET_SWEEP == 1) {
self->buckets_[key] = (uint32_t)cur_ix;
return BROTLI_TRUE;
} else {
is_match_found = BROTLI_TRUE;
const score_t score = BackwardReferenceScoreUsingLastDistance(len);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = len;
out->distance = cached_backward;
out->score = best_score;
compare_char = data[cur_ix_masked + best_len];
if (BUCKET_SWEEP == 1) {
self->buckets_[key] = (uint32_t)cur_ix;
return;
}
}
}
}
@ -172,19 +173,22 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(
backward = cur_ix - prev_ix;
prev_ix &= (uint32_t)ring_buffer_mask;
if (compare_char != data[prev_ix + best_len_in]) {
return BROTLI_FALSE;
return;
}
if (BROTLI_PREDICT_FALSE(backward == 0 || backward > max_backward)) {
return BROTLI_FALSE;
return;
}
len = FindMatchLengthWithLimit(&data[prev_ix],
&data[cur_ix_masked],
max_length);
if (len >= 4) {
out->len = len;
out->distance = backward;
out->score = BackwardReferenceScore(len, backward);
return BROTLI_TRUE;
const score_t score = BackwardReferenceScore(len, backward);
if (best_score < score) {
out->len = len;
out->distance = backward;
out->score = score;
return;
}
}
} else {
uint32_t *bucket = self->buckets_ + key;
@ -212,18 +216,16 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(
out->distance = backward;
out->score = score;
compare_char = data[cur_ix_masked + best_len];
is_match_found = BROTLI_TRUE;
}
}
}
}
if (USE_DICTIONARY && !is_match_found) {
is_match_found = SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
if (USE_DICTIONARY && min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_TRUE);
}
self->buckets_[key + ((cur_ix >> 3) % BUCKET_SWEEP)] = (uint32_t)cur_ix;
return is_match_found;
}
#undef HASH_MAP_SIZE

14
modules/brotli/enc/hash_to_binary_tree_inc.h
View File

@ -19,8 +19,10 @@
#define BUCKET_SIZE (1 << BUCKET_BITS)
static size_t FN(HashTypeLength)(void) { return 4; }
static size_t FN(StoreLookahead)(void) { return MAX_TREE_COMP_LENGTH; }
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
return MAX_TREE_COMP_LENGTH;
}
static uint32_t FN(HashBytes)(const uint8_t *data) {
uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
@ -199,7 +201,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint8_t* data,
const size_t ring_buffer_mask, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t max_length, const size_t max_backward, const size_t gap,
const BrotliEncoderParams* params, BackwardMatch* matches) {
BackwardMatch* const orig_matches = matches;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
@ -248,8 +250,10 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
for (l = minlen; l <= maxlen; ++l) {
uint32_t dict_id = dict_matches[l];
if (dict_id < kInvalidMatch) {
InitDictionaryBackwardMatch(matches++,
max_backward + (dict_id >> 5) + 1, l, dict_id & 31);
size_t distance = max_backward + gap + (dict_id >> 5) + 1;
if (distance < BROTLI_MAX_DISTANCE) {
InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
}
}
}
}

2
modules/brotli/enc/memory.h
View File

@ -40,7 +40,7 @@ BROTLI_INTERNAL void BrotliInitMemoryManager(
BROTLI_INTERNAL void* BrotliAllocate(MemoryManager* m, size_t n);
#define BROTLI_ALLOC(M, T, N) \
((N) ? ((T*)BrotliAllocate((M), (N) * sizeof(T))) : NULL)
((N) > 0 ? ((T*)BrotliAllocate((M), (N) * sizeof(T))) : NULL)
BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
#define BROTLI_FREE(M, P) { \

64
modules/brotli/enc/port.h
View File

@ -26,36 +26,37 @@
#define __LITTLE_ENDIAN LITTLE_ENDIAN
#endif
/* define the macro IS_LITTLE_ENDIAN
/* define the macro BROTLI_LITTLE_ENDIAN
using the above endian definitions from endian.h if
endian.h was included */
#ifdef __BYTE_ORDER
#if __BYTE_ORDER == __LITTLE_ENDIAN
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
#else
#if defined(__LITTLE_ENDIAN__)
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
#endif /* __BYTE_ORDER */
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
/* Enable little-endian optimization for x64 architecture on Windows. */
#if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
/* Portable handling of unaligned loads, stores, and copies.
On some platforms, like ARM, the copy functions can be more efficient
then a load and a store. */
#if defined(ARCH_PIII) || \
defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC)
#if defined(BROTLI_LITTLE_ENDIAN) && (\
defined(ARCH_PIII) || defined(ARCH_ATHLON) || \
defined(ARCH_K8) || defined(_ARCH_PPC))
/* x86 and x86-64 can perform unaligned loads/stores directly;
modern PowerPC hardware can also do unaligned integer loads and stores;
@ -63,14 +64,12 @@
*/
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
#define BROTLI_UNALIGNED_LOAD64(_p) (*(const uint64_t *)(_p))
#define BROTLI_UNALIGNED_LOAD64LE(_p) (*(const uint64_t *)(_p))
#define BROTLI_UNALIGNED_STORE32(_p, _val) \
(*(uint32_t *)(_p) = (_val))
#define BROTLI_UNALIGNED_STORE64(_p, _val) \
#define BROTLI_UNALIGNED_STORE64LE(_p, _val) \
(*(uint64_t *)(_p) = (_val))
#elif defined(__arm__) && \
#elif defined(BROTLI_LITTLE_ENDIAN) && defined(__arm__) && \
!defined(__ARM_ARCH_5__) && \
!defined(__ARM_ARCH_5T__) && \
!defined(__ARM_ARCH_5TE__) && \
@ -88,16 +87,14 @@
slowly (trip through kernel mode). */
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
#define BROTLI_UNALIGNED_STORE32(_p, _val) \
(*(uint32_t *)(_p) = (_val))
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
}
@ -112,20 +109,47 @@ static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) {
return t;
}
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
#if defined(BROTLI_LITTLE_ENDIAN)
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE32(void *p, uint32_t v) {
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
#else /* BROTLI_LITTLE_ENDIAN */
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
const uint8_t* in = (const uint8_t*)p;
uint64_t value = (uint64_t)(in[0]);
value |= (uint64_t)(in[1]) << 8;
value |= (uint64_t)(in[2]) << 16;
value |= (uint64_t)(in[3]) << 24;
value |= (uint64_t)(in[4]) << 32;
value |= (uint64_t)(in[5]) << 40;
value |= (uint64_t)(in[6]) << 48;
value |= (uint64_t)(in[7]) << 56;
return value;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
uint8_t* out = (uint8_t*)p;
out[0] = (uint8_t)v;
out[1] = (uint8_t)(v >> 8);
out[2] = (uint8_t)(v >> 16);
out[3] = (uint8_t)(v >> 24);
out[4] = (uint8_t)(v >> 32);
out[5] = (uint8_t)(v >> 40);
out[6] = (uint8_t)(v >> 48);
out[7] = (uint8_t)(v >> 56);
}
#endif /* BROTLI_LITTLE_ENDIAN */
#endif
#define TEMPLATE_(T) \

4
modules/brotli/enc/write_bits.h
View File

@ -40,7 +40,7 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits,
uint64_t bits,
size_t * BROTLI_RESTRICT pos,
uint8_t * BROTLI_RESTRICT array) {
#ifdef IS_LITTLE_ENDIAN
#ifdef BROTLI_LITTLE_ENDIAN
/* This branch of the code can write up to 56 bits at a time,
7 bits are lost by being perhaps already in *p and at least
1 bit is needed to initialize the bit-stream ahead (i.e. if 7
@ -54,7 +54,7 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits,
assert((bits >> n_bits) == 0);
assert(n_bits <= 56);
v |= bits << (*pos & 7);
BROTLI_UNALIGNED_STORE64(p, v); /* Set some bits. */
BROTLI_UNALIGNED_STORE64LE(p, v); /* Set some bits. */
*pos += n_bits;
#else
/* implicit & 0xff is assumed for uint8_t arithmetics */

61
modules/brotli/include/brotli/decode.h
View File

@ -84,8 +84,10 @@ typedef enum {
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_1, -14) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR \
\
/* -16..-19 codes are reserved */ \
/* -16..-17 codes are reserved */ \
\
BROTLI_ERROR_CODE(_ERROR_, COMPOUND_DICTIONARY, -18) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_, DICTIONARY_NOT_SET, -19) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_, INVALID_ARGUMENTS, -20) SEPARATOR \
\
/* Memory allocation problems */ \
@ -125,6 +127,29 @@ typedef enum {
*/
#define BROTLI_LAST_ERROR_CODE BROTLI_DECODER_ERROR_UNREACHABLE
/** Options to be used with ::BrotliDecoderSetParameter. */
typedef enum BrotliDecoderParameter {
/**
* Disable "canny" ring buffer allocation strategy.
*
* Ring buffer is allocated according to window size, despite the real size of
* the content.
*/
BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION = 0
} BrotliDecoderParameter;
/**
* Sets the specified parameter to the given decoder instance.
*
* @param state decoder instance
* @param param parameter to set
* @param value new parameter value
* @returns ::BROTLI_FALSE if parameter is unrecognized, or value is invalid
* @returns ::BROTLI_TRUE if value is accepted
*/
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderSetParameter(
BrotliDecoderState* state, BrotliDecoderParameter param, uint32_t value);
/**
* Creates an instance of ::BrotliDecoderState and initializes it.
*
@ -207,9 +232,9 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompress(
* allocation failed, arguments were invalid, etc.;
* use ::BrotliDecoderGetErrorCode to get detailed error code
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT decoding is blocked until
* more output space is provided
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
* more input data is provided
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
* more output space is provided
* @returns ::BROTLI_DECODER_RESULT_SUCCESS decoding is finished, no more
* input might be consumed and no more output will be produced
*/
@ -217,31 +242,6 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStream(
BrotliDecoderState* state, size_t* available_in, const uint8_t** next_in,
size_t* available_out, uint8_t** next_out, size_t* total_out);
/**
* Prepends LZ77 dictionary.
*
* Fills the fresh ::BrotliDecoderState with additional data corpus for LZ77
* backward references.
*
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
* @warning The dictionary must exist in memory until decoding is done and
* is owned by the caller.
*
* Workflow:
* -# Allocate and initialize state with ::BrotliDecoderCreateInstance
* -# Invoke ::BrotliDecoderSetCustomDictionary
* -# Use ::BrotliDecoderDecompressStream
* -# Clean up and free state with ::BrotliDecoderDestroyInstance
*
* @param state decoder instance
* @param size length of @p dict; should be less or equal to 2^24 (16MiB),
* otherwise the dictionary will be ignored
* @param dict "dictionary"; @b MUST be the same as used during compression
*/
BROTLI_DEC_API void BrotliDecoderSetCustomDictionary(
BrotliDecoderState* state, size_t size,
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
/**
* Checks if decoder has more output.
*
@ -303,7 +303,8 @@ BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsUsed(const BrotliDecoderState* state);
* the input and produced all of the output
* @returns ::BROTLI_FALSE otherwise
*/
BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* state);
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsFinished(
const BrotliDecoderState* state);
/**
* Acquires a detailed error code.
@ -316,7 +317,7 @@ BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* state);
* @param state decoder instance
* @returns last saved error code
*/
BrotliDecoderErrorCode BrotliDecoderGetErrorCode(
BROTLI_DEC_API BrotliDecoderErrorCode BrotliDecoderGetErrorCode(
const BrotliDecoderState* state);
/**

52
modules/brotli/include/brotli/encode.h
View File

@ -36,11 +36,6 @@ extern "C" {
/** Maximal value for ::BROTLI_PARAM_QUALITY parameter. */
#define BROTLI_MAX_QUALITY 11
BROTLI_DEPRECATED static const int kBrotliMinWindowBits =
BROTLI_MIN_WINDOW_BITS;
BROTLI_DEPRECATED static const int kBrotliMaxWindowBits =
BROTLI_MAX_WINDOW_BITS;
/** Options for ::BROTLI_PARAM_MODE parameter. */
typedef enum BrotliEncoderMode {
/**
@ -77,7 +72,9 @@ typedef enum BrotliEncoderOperation {
* Actual flush is performed when input stream is depleted and there is enough
* space in output stream. This means that client should repeat
* ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE.
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE. If output is acquired
* via ::BrotliEncoderTakeOutput, then operation should be repeated after
* output buffer is drained.
*
* @warning Until flush is complete, client @b SHOULD @b NOT swap,
* reduce or extend input stream.
@ -91,8 +88,10 @@ typedef enum BrotliEncoderOperation {
*
* Actual finalization is performed when input stream is depleted and there is
* enough space in output stream. This means that client should repeat
* ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE.
* ::BROTLI_OPERATION_FINISH operation until @p available_in becomes @c 0, and
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE. If output is acquired
* via ::BrotliEncoderTakeOutput, then operation should be repeated after
* output buffer is drained.
*
* @warning Until finalization is complete, client @b SHOULD @b NOT swap,
* reduce or extend input stream.
@ -228,43 +227,6 @@ BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance(
*/
BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state);
/* Calculates maximum input size that can be processed at once. */
BROTLI_DEPRECATED BROTLI_ENC_API size_t BrotliEncoderInputBlockSize(
BrotliEncoderState* state);
/* Copies the given input data to the internal ring buffer. */
BROTLI_DEPRECATED BROTLI_ENC_API void BrotliEncoderCopyInputToRingBuffer(
BrotliEncoderState* state, const size_t input_size,
const uint8_t* input_buffer);
/* Processes the accumulated input. */
BROTLI_DEPRECATED BROTLI_ENC_API BROTLI_BOOL BrotliEncoderWriteData(
BrotliEncoderState* state, const BROTLI_BOOL is_last,
const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output);
/**
* Prepends imaginary LZ77 dictionary.
*
* Fills the fresh ::BrotliEncoderState with additional data corpus for LZ77
* backward references.
*
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
*
* Workflow:
* -# Allocate and initialize state with ::BrotliEncoderCreateInstance
* -# Set ::BROTLI_PARAM_LGWIN parameter
* -# Invoke ::BrotliEncoderSetCustomDictionary
* -# Use ::BrotliEncoderCompressStream
* -# Clean up and free state with ::BrotliEncoderDestroyInstance
*
* @param state encoder instance
* @param size length of @p dict; at most "window size" bytes are used
* @param dict "dictionary"; @b MUST use same dictionary during decompression
*/
BROTLI_ENC_API void BrotliEncoderSetCustomDictionary(
BrotliEncoderState* state, size_t size,
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
/**
* Calculates the output size bound for the given @p input_size.
*

4
modules/brotli/moz.build
View File

@ -29,7 +29,7 @@ CFLAGS += ['-DBROTLI_BUILD_PORTABLE']
Library('brotli')
HostProgram('bro')
HostProgram('brotli')
HOST_SOURCES += UNIFIED_SOURCES
@ -51,7 +51,7 @@ HOST_SOURCES += [
'enc/metablock.c',
'enc/static_dict.c',
'enc/utf8_util.c',
'tools/bro.c',
'tools/brotli.c',
]
if CONFIG['HOST_CC_TYPE'] not in ('msvc', 'clang-cl'):

521
modules/brotli/tools/bro.c
View File

@ -1,521 +0,0 @@
/* Copyright 2014 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Example main() function for Brotli library. */
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <brotli/decode.h>
#include <brotli/encode.h>
#if !defined(_WIN32)
#include <unistd.h>
#include <utime.h>
#else
#include <io.h>
#include <share.h>
#include <sys/utime.h>
#define MAKE_BINARY(FILENO) (_setmode((FILENO), _O_BINARY), (FILENO))
#if !defined(__MINGW32__)
#define STDIN_FILENO MAKE_BINARY(_fileno(stdin))
#define STDOUT_FILENO MAKE_BINARY(_fileno(stdout))
#define S_IRUSR S_IREAD
#define S_IWUSR S_IWRITE
#endif
#define fdopen _fdopen
#define unlink _unlink
#define utimbuf _utimbuf
#define utime _utime
#define fopen ms_fopen
#define open ms_open
#define chmod(F, P) (0)
#define chown(F, O, G) (0)
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
#define fseek _fseeki64
#define ftell _ftelli64
#endif
static FILE* ms_fopen(const char *filename, const char *mode) {
FILE* result = 0;
fopen_s(&result, filename, mode);
return result;
}
static int ms_open(const char *filename, int oflag, int pmode) {
int result = -1;
_sopen_s(&result, filename, oflag | O_BINARY, _SH_DENYNO, pmode);
return result;
}
#endif /* WIN32 */
static int ParseQuality(const char* s, int* quality) {
if (s[0] >= '0' && s[0] <= '9') {
*quality = s[0] - '0';
if (s[1] >= '0' && s[1] <= '9') {
*quality = *quality * 10 + s[1] - '0';
return (s[2] == 0) ? 1 : 0;
}
return (s[1] == 0) ? 1 : 0;
}
return 0;
}
static void ParseArgv(int argc, char **argv,
char **input_path,
char **output_path,
char **dictionary_path,
int *force,
int *quality,
int *decompress,
int *repeat,
int *verbose,
int *lgwin,
int *copy_stat) {
int k;
*force = 0;
*input_path = 0;
*output_path = 0;
*repeat = 1;
*verbose = 0;
*lgwin = 22;
*copy_stat = 1;
{
size_t argv0_len = strlen(argv[0]);
*decompress =
argv0_len >= 5 && strcmp(&argv[0][argv0_len - 5], "unbro") == 0;
}
for (k = 1; k < argc; ++k) {
if (!strcmp("--force", argv[k]) ||
!strcmp("-f", argv[k])) {
if (*force != 0) {
goto error;
}
*force = 1;
continue;
} else if (!strcmp("--decompress", argv[k]) ||
!strcmp("--uncompress", argv[k]) ||
!strcmp("-d", argv[k])) {
*decompress = 1;
continue;
} else if (!strcmp("--verbose", argv[k]) ||
!strcmp("-v", argv[k])) {
if (*verbose != 0) {
goto error;
}
*verbose = 1;
continue;
} else if (!strcmp("--no-copy-stat", argv[k])) {
if (*copy_stat == 0) {
goto error;
}
*copy_stat = 0;
continue;
}
if (k < argc - 1) {
if (!strcmp("--input", argv[k]) ||
!strcmp("--in", argv[k]) ||
!strcmp("-i", argv[k])) {
if (*input_path != 0) {
goto error;
}
*input_path = argv[k + 1];
++k;
continue;
} else if (!strcmp("--output", argv[k]) ||
!strcmp("--out", argv[k]) ||
!strcmp("-o", argv[k])) {
if (*output_path != 0) {
goto error;
}
*output_path = argv[k + 1];
++k;
continue;
} else if (!strcmp("--custom-dictionary", argv[k])) {
if (*dictionary_path != 0) {
goto error;
}
*dictionary_path = argv[k + 1];
++k;
continue;
} else if (!strcmp("--quality", argv[k]) ||
!strcmp("-q", argv[k])) {
if (!ParseQuality(argv[k + 1], quality)) {
goto error;
}
++k;
continue;
} else if (!strcmp("--repeat", argv[k]) ||
!strcmp("-r", argv[k])) {
if (!ParseQuality(argv[k + 1], repeat)) {
goto error;
}
++k;
continue;
} else if (!strcmp("--window", argv[k]) ||
!strcmp("-w", argv[k])) {
if (!ParseQuality(argv[k + 1], lgwin)) {
goto error;
}
if (*lgwin < 10 || *lgwin >= 25) {
goto error;
}
++k;
continue;
}
}
goto error;
}
return;
error:
fprintf(stderr,
"Usage: %s [--force] [--quality n] [--decompress]"
" [--input filename] [--output filename] [--repeat iters]"
" [--verbose] [--window n] [--custom-dictionary filename]"
" [--no-copy-stat]\n",
argv[0]);
exit(1);
}
static FILE* OpenInputFile(const char* input_path) {
FILE* f;
if (input_path == 0) {
return fdopen(STDIN_FILENO, "rb");
}
f = fopen(input_path, "rb");
if (f == 0) {
perror("fopen");
exit(1);
}
return f;
}
static FILE *OpenOutputFile(const char *output_path, const int force) {
int fd;
if (output_path == 0) {
return fdopen(STDOUT_FILENO, "wb");
}
fd = open(output_path, O_CREAT | (force ? 0 : O_EXCL) | O_WRONLY | O_TRUNC,
S_IRUSR | S_IWUSR);
if (fd < 0) {
if (!force) {
struct stat statbuf;
if (stat(output_path, &statbuf) == 0) {
fprintf(stderr, "output file exists\n");
exit(1);
}
}
perror("open");
exit(1);
}
return fdopen(fd, "wb");
}
static int64_t FileSize(const char *path) {
FILE *f = fopen(path, "rb");
int64_t retval;
if (f == NULL) {
return -1;
}
if (fseek(f, 0L, SEEK_END) != 0) {
fclose(f);
return -1;
}
retval = ftell(f);
if (fclose(f) != 0) {
return -1;
}
return retval;
}
/* Copy file times and permissions.
TODO: this is a "best effort" implementation; honest cross-platform
fully featured implementation is way too hacky; add more hacks by request. */
static void CopyStat(const char* input_path, const char* output_path) {
struct stat statbuf;
struct utimbuf times;
int res;
if (input_path == 0 || output_path == 0) {
return;
}
if (stat(input_path, &statbuf) != 0) {
return;
}
times.actime = statbuf.st_atime;
times.modtime = statbuf.st_mtime;
utime(output_path, &times);
res = chmod(output_path, statbuf.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO));
if (res != 0)
perror("chmod failed");
res = chown(output_path, (uid_t)-1, statbuf.st_gid);
if (res != 0)
perror("chown failed");
res = chown(output_path, statbuf.st_uid, (gid_t)-1);
if (res != 0)
perror("chown failed");
}
/* Result ownersip is passed to caller.
|*dictionary_size| is set to resulting buffer size. */
static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size) {
static const int kMaxDictionarySize = (1 << 24) - 16;
FILE *f = fopen(path, "rb");
int64_t file_size_64;
uint8_t* buffer;
size_t bytes_read;
if (f == NULL) {
perror("fopen");
exit(1);
}
file_size_64 = FileSize(path);
if (file_size_64 == -1) {
fprintf(stderr, "could not get size of dictionary file");
exit(1);
}
if (file_size_64 > kMaxDictionarySize) {
fprintf(stderr, "dictionary is larger than maximum allowed: %d\n",
kMaxDictionarySize);
exit(1);
}
*dictionary_size = (size_t)file_size_64;
buffer = (uint8_t*)malloc(*dictionary_size);
if (!buffer) {
fprintf(stderr, "could not read dictionary: out of memory\n");
exit(1);
}
bytes_read = fread(buffer, sizeof(uint8_t), *dictionary_size, f);
if (bytes_read != *dictionary_size) {
fprintf(stderr, "could not read dictionary\n");
exit(1);
}
fclose(f);
return buffer;
}
static const size_t kFileBufferSize = 65536;
static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path) {
/* Dictionary should be kept during first rounds of decompression. */
uint8_t* dictionary = NULL;
uint8_t* input;
uint8_t* output;
size_t available_in;
const uint8_t* next_in;
size_t available_out = kFileBufferSize;
uint8_t* next_out;
BrotliDecoderResult result = BROTLI_DECODER_RESULT_ERROR;
BrotliDecoderState* s = BrotliDecoderCreateInstance(NULL, NULL, NULL);
if (!s) {
fprintf(stderr, "out of memory\n");
return 0;
}
if (dictionary_path != NULL) {
size_t dictionary_size = 0;
dictionary = ReadDictionary(dictionary_path, &dictionary_size);
BrotliDecoderSetCustomDictionary(s, dictionary_size, dictionary);
}
input = (uint8_t*)malloc(kFileBufferSize);
output = (uint8_t*)malloc(kFileBufferSize);
if (!input || !output) {
fprintf(stderr, "out of memory\n");
goto end;
}
next_out = output;
result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
while (1) {
if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) {
if (feof(fin)) {
break;
}
available_in = fread(input, 1, kFileBufferSize, fin);
next_in = input;
if (ferror(fin)) {
break;
}
} else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
fwrite(output, 1, kFileBufferSize, fout);
if (ferror(fout)) {
break;
}
available_out = kFileBufferSize;
next_out = output;
} else {
break; /* Error or success. */
}
result = BrotliDecoderDecompressStream(
s, &available_in, &next_in, &available_out, &next_out, 0);
}
if (next_out != output) {
fwrite(output, 1, (size_t)(next_out - output), fout);
}
if ((result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) || ferror(fout)) {
fprintf(stderr, "failed to write output\n");
} else if (result != BROTLI_DECODER_RESULT_SUCCESS) {
/* Error or needs more input. */
fprintf(stderr, "corrupt input\n");
}
end:
free(dictionary);
free(input);
free(output);
BrotliDecoderDestroyInstance(s);
return (result == BROTLI_DECODER_RESULT_SUCCESS) ? 1 : 0;
}
static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,
const char *dictionary_path) {
BrotliEncoderState* s = BrotliEncoderCreateInstance(0, 0, 0);
uint8_t* buffer = (uint8_t*)malloc(kFileBufferSize << 1);
uint8_t* input = buffer;
uint8_t* output = buffer + kFileBufferSize;
size_t available_in = 0;
const uint8_t* next_in = NULL;
size_t available_out = kFileBufferSize;
uint8_t* next_out = output;
int is_eof = 0;
int is_ok = 1;
if (!s || !buffer) {
is_ok = 0;
goto finish;
}
BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)quality);
BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)lgwin);
if (dictionary_path != NULL) {
size_t dictionary_size = 0;
uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size);
BrotliEncoderSetCustomDictionary(s, dictionary_size, dictionary);
free(dictionary);
}
while (1) {
if (available_in == 0 && !is_eof) {
available_in = fread(input, 1, kFileBufferSize, fin);
next_in = input;
if (ferror(fin)) break;
is_eof = feof(fin);
}
if (!BrotliEncoderCompressStream(s,
is_eof ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS,
&available_in, &next_in, &available_out, &next_out, NULL)) {
is_ok = 0;
break;
}
if (available_out != kFileBufferSize) {
size_t out_size = kFileBufferSize - available_out;
fwrite(output, 1, out_size, fout);
if (ferror(fout)) break;
available_out = kFileBufferSize;
next_out = output;
}
if (BrotliEncoderIsFinished(s)) break;
}
finish:
free(buffer);
BrotliEncoderDestroyInstance(s);
if (!is_ok) {
/* Should detect OOM? */
fprintf(stderr, "failed to compress data\n");
return 0;
} else if (ferror(fout)) {
fprintf(stderr, "failed to write output\n");
return 0;
} else if (ferror(fin)) {
fprintf(stderr, "failed to read input\n");
return 0;
}
return 1;
}
int main(int argc, char** argv) {
char *input_path = 0;
char *output_path = 0;
char *dictionary_path = 0;
int force = 0;
int quality = 11;
int decompress = 0;
int repeat = 1;
int verbose = 0;
int lgwin = 0;
int copy_stat = 1;
clock_t clock_start;
int i;
ParseArgv(argc, argv, &input_path, &output_path, &dictionary_path, &force,
&quality, &decompress, &repeat, &verbose, &lgwin, &copy_stat);
clock_start = clock();
for (i = 0; i < repeat; ++i) {
FILE* fin = OpenInputFile(input_path);
FILE* fout = OpenOutputFile(output_path, force || (repeat > 1));
int is_ok = 0;
if (decompress) {
is_ok = Decompress(fin, fout, dictionary_path);
} else {
is_ok = Compress(quality, lgwin, fin, fout, dictionary_path);
}
if (!is_ok) {
unlink(output_path);
exit(1);
}
if (fclose(fout) != 0) {
perror("fclose");
exit(1);
}
/* TOCTOU violation, but otherwise it is impossible to set file times. */
if (copy_stat && (i + 1 == repeat)) {
CopyStat(input_path, output_path);
}
if (fclose(fin) != 0) {
perror("fclose");
exit(1);
}
}
if (verbose) {
clock_t clock_end = clock();
double duration = (double)(clock_end - clock_start) / CLOCKS_PER_SEC;
int64_t uncompressed_size;
double uncompressed_bytes_in_MB;
if (duration < 1e-9) {
duration = 1e-9;
}
uncompressed_size = FileSize(decompress ? output_path : input_path);
if (uncompressed_size == -1) {
fprintf(stderr, "failed to determine uncompressed file size\n");
exit(1);
}
uncompressed_bytes_in_MB =
(double)(repeat * uncompressed_size) / (1024.0 * 1024.0);
if (decompress) {
printf("Brotli decompression speed: ");
} else {
printf("Brotli compression speed: ");
}
printf("%g MB/s\n", uncompressed_bytes_in_MB / duration);
}
return 0;
}

861
modules/brotli/tools/brotli.c Executable file
View File

@ -0,0 +1,861 @@
/* Copyright 2014 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Command line interface for Brotli library. */
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include "../common/constants.h"
#include "../common/version.h"
#include <brotli/decode.h>
#include <brotli/encode.h>
#if !defined(_WIN32)
#include <unistd.h>
#include <utime.h>
#define MAKE_BINARY(FILENO) (FILENO)
#else
#include <io.h>
#include <share.h>
#include <sys/utime.h>
#define MAKE_BINARY(FILENO) (_setmode((FILENO), _O_BINARY), (FILENO))
#if !defined(__MINGW32__)
#define STDIN_FILENO _fileno(stdin)
#define STDOUT_FILENO _fileno(stdout)
#define S_IRUSR S_IREAD
#define S_IWUSR S_IWRITE
#endif
#define fdopen _fdopen
#define isatty _isatty
#define unlink _unlink
#define utimbuf _utimbuf
#define utime _utime
#define fopen ms_fopen
#define open ms_open
#define chmod(F, P) (0)
#define chown(F, O, G) (0)
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
#define fseek _fseeki64
#define ftell _ftelli64
#endif
static FILE* ms_fopen(const char* filename, const char* mode) {
FILE* result = 0;
fopen_s(&result, filename, mode);
return result;
}
static int ms_open(const char* filename, int oflag, int pmode) {
int result = -1;
_sopen_s(&result, filename, oflag | O_BINARY, _SH_DENYNO, pmode);
return result;
}
#endif /* WIN32 */
typedef enum {
COMMAND_COMPRESS,
COMMAND_DECOMPRESS,
COMMAND_HELP,
COMMAND_INVALID,
COMMAND_TEST_INTEGRITY,
COMMAND_NOOP,
COMMAND_VERSION
} Command;
#define DEFAULT_LGWIN 22
#define DEFAULT_SUFFIX ".br"
#define MAX_OPTIONS 20
typedef struct {
/* Parameters */
int quality;
int lgwin;
BROTLI_BOOL force_overwrite;
BROTLI_BOOL junk_source;
BROTLI_BOOL copy_stat;
BROTLI_BOOL verbose;
BROTLI_BOOL write_to_stdout;
BROTLI_BOOL test_integrity;
BROTLI_BOOL decompress;
const char* output_path;
const char* suffix;
int not_input_indices[MAX_OPTIONS];
size_t longest_path_len;
size_t input_count;
/* Inner state */
int argc;
char** argv;
char* modified_path; /* Storage for path with appended / cut suffix */
int iterator;
int ignore;
BROTLI_BOOL iterator_error;
uint8_t* buffer;
uint8_t* input;
uint8_t* output;
const char* current_input_path;
const char* current_output_path;
FILE* fin;
FILE* fout;
} Context;
/* Parse up to 5 decimal digits. */
static BROTLI_BOOL ParseInt(const char* s, int low, int high, int* result) {
int value = 0;
int i;
for (i = 0; i < 5; ++i) {
char c = s[i];
if (c == 0) break;
if (s[i] < '0' || s[i] > '9') return BROTLI_FALSE;
value = (10 * value) + (c - '0');
}
if (i == 0) return BROTLI_FALSE;
if (i > 1 && s[0] == '0') return BROTLI_FALSE;
if (s[i] != 0) return BROTLI_FALSE;
if (value < low || value > high) return BROTLI_FALSE;
*result = value;
return BROTLI_TRUE;
}
/* Returns "base file name" or its tail, if it contains '/' or '\'. */
static const char* FileName(const char* path) {
const char* separator_position = strrchr(path, '/');
if (separator_position) path = separator_position + 1;
separator_position = strrchr(path, '\\');
if (separator_position) path = separator_position + 1;
return path;
}
/* Detect if the program name is a special alias that infers a command type. */
static Command ParseAlias(const char* name) {
/* TODO: cast name to lower case? */
const char* unbrotli = "unbrotli";
size_t unbrotli_len = strlen(unbrotli);
name = FileName(name);
/* Partial comparison. On Windows there could be ".exe" suffix. */
if (strncmp(name, unbrotli, unbrotli_len) == 0) {
char terminator = name[unbrotli_len];
if (terminator == 0 || terminator == '.') return COMMAND_DECOMPRESS;
}
return COMMAND_COMPRESS;
}
static Command ParseParams(Context* params) {
int argc = params->argc;
char** argv = params->argv;
int i;
int next_option_index = 0;
size_t input_count = 0;
size_t longest_path_len = 1;
BROTLI_BOOL command_set = BROTLI_FALSE;
BROTLI_BOOL quality_set = BROTLI_FALSE;
BROTLI_BOOL output_set = BROTLI_FALSE;
BROTLI_BOOL keep_set = BROTLI_FALSE;
BROTLI_BOOL lgwin_set = BROTLI_FALSE;
BROTLI_BOOL suffix_set = BROTLI_FALSE;
BROTLI_BOOL after_dash_dash = BROTLI_FALSE;
Command command = ParseAlias(argv[0]);
for (i = 1; i < argc; ++i) {
const char* arg = argv[i];
/* C99 5.1.2.2.1: "members argv[0] through argv[argc-1] inclusive shall
contain pointers to strings"; NULL and 0-length are not forbidden. */
size_t arg_len = arg ? strlen(arg) : 0;
if (arg_len == 0) {
params->not_input_indices[next_option_index++] = i;
continue;
}
/* Too many options. The expected longest option list is:
"-q 0 -w 10 -o f -D d -S b -d -f -k -n -v --", i.e. 16 items in total.
This check is an additinal guard that is never triggered, but provides an
additional guard for future changes. */
if (next_option_index > (MAX_OPTIONS - 2)) {
return COMMAND_INVALID;
}
/* Input file entry. */
if (after_dash_dash || arg[0] != '-' || arg_len == 1) {
input_count++;
if (longest_path_len < arg_len) longest_path_len = arg_len;
continue;
}
/* Not a file entry. */
params->not_input_indices[next_option_index++] = i;
/* '--' entry stop parsing arguments. */
if (arg_len == 2 && arg[1] == '-') {
after_dash_dash = BROTLI_TRUE;
continue;
}
/* Simple / coalesced options. */
if (arg[1] != '-') {
size_t j;
for (j = 1; j < arg_len; ++j) {
char c = arg[j];
if (c >= '0' && c <= '9') {
if (quality_set) return COMMAND_INVALID;
quality_set = BROTLI_TRUE;
params->quality = c - '0';
continue;
} else if (c == 'c') {
if (output_set) return COMMAND_INVALID;
output_set = BROTLI_TRUE;
params->write_to_stdout = BROTLI_TRUE;
continue;
} else if (c == 'd') {
if (command_set) return COMMAND_INVALID;
command_set = BROTLI_TRUE;
command = COMMAND_DECOMPRESS;
continue;
} else if (c == 'f') {
if (params->force_overwrite) return COMMAND_INVALID;
params->force_overwrite = BROTLI_TRUE;
continue;
} else if (c == 'h') {
/* Don't parse further. */
return COMMAND_HELP;
} else if (c == 'j' || c == 'k') {
if (keep_set) return COMMAND_INVALID;
keep_set = BROTLI_TRUE;
params->junk_source = TO_BROTLI_BOOL(c == 'j');
continue;
} else if (c == 'n') {
if (!params->copy_stat) return COMMAND_INVALID;
params->copy_stat = BROTLI_FALSE;
continue;
} else if (c == 't') {
if (command_set) return COMMAND_INVALID;
command_set = BROTLI_TRUE;
command = COMMAND_TEST_INTEGRITY;
continue;
} else if (c == 'v') {
if (params->verbose) return COMMAND_INVALID;
params->verbose = BROTLI_TRUE;
continue;
} else if (c == 'V') {
/* Don't parse further. */
return COMMAND_VERSION;
} else if (c == 'Z') {
if (quality_set) return COMMAND_INVALID;
quality_set = BROTLI_TRUE;
params->quality = 11;
continue;
}
/* o/q/w/D/S with parameter is expected */
if (c != 'o' && c != 'q' && c != 'w' && c != 'D' && c != 'S') {
return COMMAND_INVALID;
}
if (j + 1 != arg_len) return COMMAND_INVALID;
i++;
if (i == argc || !argv[i] || argv[i][0] == 0) return COMMAND_INVALID;
params->not_input_indices[next_option_index++] = i;
if (c == 'o') {
if (output_set) return COMMAND_INVALID;
params->output_path = argv[i];
} else if (c == 'q') {
if (quality_set) return COMMAND_INVALID;
quality_set = ParseInt(argv[i], BROTLI_MIN_QUALITY,
BROTLI_MAX_QUALITY, &params->quality);
if (!quality_set) return COMMAND_INVALID;
} else if (c == 'w') {
if (lgwin_set) return COMMAND_INVALID;
lgwin_set = ParseInt(argv[i], 0,
BROTLI_MAX_WINDOW_BITS, &params->lgwin);
if (!lgwin_set) return COMMAND_INVALID;
if (params->lgwin != 0 && params->lgwin < BROTLI_MIN_WINDOW_BITS) {
return COMMAND_INVALID;
}
} else if (c == 'S') {
if (suffix_set) return COMMAND_INVALID;
suffix_set = BROTLI_TRUE;
params->suffix = argv[i];
}
}
} else { /* Double-dash. */
arg = &arg[2];
if (strcmp("best", arg) == 0) {
if (quality_set) return COMMAND_INVALID;
quality_set = BROTLI_TRUE;
params->quality = 11;
} else if (strcmp("decompress", arg) == 0) {
if (command_set) return COMMAND_INVALID;
command_set = BROTLI_TRUE;
command = COMMAND_DECOMPRESS;
} else if (strcmp("force", arg) == 0) {
if (params->force_overwrite) return COMMAND_INVALID;
params->force_overwrite = BROTLI_TRUE;
} else if (strcmp("help", arg) == 0) {
/* Don't parse further. */
return COMMAND_HELP;
} else if (strcmp("keep", arg) == 0) {
if (keep_set) return COMMAND_INVALID;
keep_set = BROTLI_TRUE;
params->junk_source = BROTLI_FALSE;
} else if (strcmp("no-copy-stat", arg) == 0) {
if (!params->copy_stat) return COMMAND_INVALID;
params->copy_stat = BROTLI_FALSE;
} else if (strcmp("rm", arg) == 0) {
if (keep_set) return COMMAND_INVALID;
keep_set = BROTLI_TRUE;
params->junk_source = BROTLI_TRUE;
} else if (strcmp("stdout", arg) == 0) {
if (output_set) return COMMAND_INVALID;
output_set = BROTLI_TRUE;
params->write_to_stdout = BROTLI_TRUE;
} else if (strcmp("test", arg) == 0) {
if (command_set) return COMMAND_INVALID;
command_set = BROTLI_TRUE;
command = COMMAND_TEST_INTEGRITY;
} else if (strcmp("verbose", arg) == 0) {
if (params->verbose) return COMMAND_INVALID;
params->verbose = BROTLI_TRUE;
} else if (strcmp("version", arg) == 0) {
/* Don't parse further. */
return COMMAND_VERSION;
} else {
/* key=value */
const char* value = strrchr(arg, '=');
size_t key_len;
if (!value || value[1] == 0) return COMMAND_INVALID;
key_len = (size_t)(value - arg);
value++;
if (strncmp("lgwin", arg, key_len) == 0) {
if (lgwin_set) return COMMAND_INVALID;
lgwin_set = ParseInt(value, 0,
BROTLI_MAX_WINDOW_BITS, &params->lgwin);
if (!lgwin_set) return COMMAND_INVALID;
if (params->lgwin != 0 && params->lgwin < BROTLI_MIN_WINDOW_BITS) {
return COMMAND_INVALID;
}
} else if (strncmp("output", arg, key_len) == 0) {
if (output_set) return COMMAND_INVALID;
params->output_path = value;
} else if (strncmp("quality", arg, key_len) == 0) {
if (quality_set) return COMMAND_INVALID;
quality_set = ParseInt(value, BROTLI_MIN_QUALITY,
BROTLI_MAX_QUALITY, &params->quality);
if (!quality_set) return COMMAND_INVALID;
} else if (strncmp("suffix", arg, key_len) == 0) {
if (suffix_set) return COMMAND_INVALID;
suffix_set = BROTLI_TRUE;
params->suffix = value;
} else {
return COMMAND_INVALID;
}
}
}
}
params->input_count = input_count;
params->longest_path_len = longest_path_len;
params->decompress = (command == COMMAND_DECOMPRESS);
params->test_integrity = (command == COMMAND_TEST_INTEGRITY);
if (input_count > 1 && output_set) return COMMAND_INVALID;
if (params->test_integrity) {
if (params->output_path) return COMMAND_INVALID;
if (params->write_to_stdout) return COMMAND_INVALID;
}
if (strchr(params->suffix, '/') || strchr(params->suffix, '\\')) {
return COMMAND_INVALID;
}
return command;
}
static void PrintVersion(void) {
int major = BROTLI_VERSION >> 24;
int minor = (BROTLI_VERSION >> 12) & 0xFFF;
int patch = BROTLI_VERSION & 0xFFF;
fprintf(stdout, "brotli %d.%d.%d\n", major, minor, patch);
}
static void PrintHelp(const char* name) {
/* String is cut to pieces with length less than 509, to conform C90 spec. */
fprintf(stdout,
"Usage: %s [OPTION]... [FILE]...\n",
name);
fprintf(stdout,
"Options:\n"
" -# compression level (0-9)\n"
" -c, --stdout write on standard output\n"
" -d, --decompress decompress\n"
" -f, --force force output file overwrite\n"
" -h, --help display this help and exit\n");
fprintf(stdout,
" -j, --rm remove source file(s)\n"
" -k, --keep keep source file(s) (default)\n"
" -n, --no-copy-stat do not copy source file(s) attributes\n"
" -o FILE, --output=FILE output file (only if 1 input file)\n");
fprintf(stdout,
" -q NUM, --quality=NUM compression level (%d-%d)\n",
BROTLI_MIN_QUALITY, BROTLI_MAX_QUALITY);
fprintf(stdout,
" -t, --test test compressed file integrity\n"
" -v, --verbose verbose mode\n");
fprintf(stdout,
" -w NUM, --lgwin=NUM set LZ77 window size (0, %d-%d) (default:%d)\n",
BROTLI_MIN_WINDOW_BITS, BROTLI_MAX_WINDOW_BITS, DEFAULT_LGWIN);
fprintf(stdout,
" window size = 2**NUM - 16\n"
" 0 lets compressor choose the optimal value\n");
fprintf(stdout,
" -S SUF, --suffix=SUF output file suffix (default:'%s')\n",
DEFAULT_SUFFIX);
fprintf(stdout,
" -V, --version display version and exit\n"
" -Z, --best use best compression level (11) (default)\n"
"Simple options could be coalesced, i.e. '-9kf' is equivalent to '-9 -k -f'.\n"
"With no FILE, or when FILE is -, read standard input.\n"
"All arguments after '--' are treated as files.\n");
}
static const char* PrintablePath(const char* path) {
return path ? path : "con";
}
static BROTLI_BOOL OpenInputFile(const char* input_path, FILE** f) {
*f = NULL;
if (!input_path) {
*f = fdopen(MAKE_BINARY(STDIN_FILENO), "rb");
return BROTLI_TRUE;
}
*f = fopen(input_path, "rb");
if (!*f) {
fprintf(stderr, "failed to open input file [%s]: %s\n",
PrintablePath(input_path), strerror(errno));
return BROTLI_FALSE;
}
return BROTLI_TRUE;
}
static BROTLI_BOOL OpenOutputFile(const char* output_path, FILE** f,
BROTLI_BOOL force) {
int fd;
*f = NULL;
if (!output_path) {
*f = fdopen(MAKE_BINARY(STDOUT_FILENO), "wb");
return BROTLI_TRUE;
}
fd = open(output_path, O_CREAT | (force ? 0 : O_EXCL) | O_WRONLY | O_TRUNC,
S_IRUSR | S_IWUSR);
if (fd < 0) {
fprintf(stderr, "failed to open output file [%s]: %s\n",
PrintablePath(output_path), strerror(errno));
return BROTLI_FALSE;
}
*f = fdopen(fd, "wb");
if (!*f) {
fprintf(stderr, "failed to open output file [%s]: %s\n",
PrintablePath(output_path), strerror(errno));
return BROTLI_FALSE;
}
return BROTLI_TRUE;
}
/* Copy file times and permissions.
TODO: this is a "best effort" implementation; honest cross-platform
fully featured implementation is way too hacky; add more hacks by request. */
static void CopyStat(const char* input_path, const char* output_path) {
struct stat statbuf;
struct utimbuf times;
int res;
if (input_path == 0 || output_path == 0) {
return;
}
if (stat(input_path, &statbuf) != 0) {
return;
}
times.actime = statbuf.st_atime;
times.modtime = statbuf.st_mtime;
utime(output_path, &times);
res = chmod(output_path, statbuf.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO));
if (res != 0) {
fprintf(stderr, "setting access bits failed for [%s]: %s\n",
PrintablePath(output_path), strerror(errno));
}
res = chown(output_path, (uid_t)-1, statbuf.st_gid);
if (res != 0) {
fprintf(stderr, "setting group failed for [%s]: %s\n",
PrintablePath(output_path), strerror(errno));
}
res = chown(output_path, statbuf.st_uid, (gid_t)-1);
if (res != 0) {
fprintf(stderr, "setting user failed for [%s]: %s\n",
PrintablePath(output_path), strerror(errno));
}
}
static BROTLI_BOOL NextFile(Context* context) {
const char* arg;
size_t arg_len;
/* Iterator points to last used arg; increment to search for the next one. */
context->iterator++;
/* No input path; read from console. */
if (context->input_count == 0) {
if (context->iterator > 1) return BROTLI_FALSE;
context->current_input_path = NULL;
/* Either write to the specified path, or to console. */
context->current_output_path = context->output_path;
return BROTLI_TRUE;
}
/* Skip option arguments. */
while (context->iterator == context->not_input_indices[context->ignore]) {
context->iterator++;
context->ignore++;
}
/* All args are scanned already. */
if (context->iterator >= context->argc) return BROTLI_FALSE;
/* Iterator now points to the input file name. */
arg = context->argv[context->iterator];
arg_len = strlen(arg);
/* Read from console. */
if (arg_len == 1 && arg[0] == '-') {
context->current_input_path = NULL;
context->current_output_path = context->output_path;
return BROTLI_TRUE;
}
context->current_input_path = arg;
context->current_output_path = context->output_path;
if (context->output_path) return BROTLI_TRUE;
if (context->write_to_stdout) return BROTLI_TRUE;
strcpy(context->modified_path, arg);
context->current_output_path = context->modified_path;
/* If output is not specified, input path suffix should match. */
if (context->decompress) {
size_t suffix_len = strlen(context->suffix);
char* name = (char*)FileName(context->modified_path);
char* name_suffix;
size_t name_len = strlen(name);
if (name_len < suffix_len + 1) {
fprintf(stderr, "empty output file name for [%s] input file\n",
PrintablePath(arg));
context->iterator_error = BROTLI_TRUE;
return BROTLI_FALSE;
}
name_suffix = name + name_len - suffix_len;
if (strcmp(context->suffix, name_suffix) != 0) {
fprintf(stderr, "input file [%s] suffix mismatch\n",
PrintablePath(arg));
context->iterator_error = BROTLI_TRUE;
return BROTLI_FALSE;
}
name_suffix[0] = 0;
return BROTLI_TRUE;
} else {
strcpy(context->modified_path + arg_len, context->suffix);
return BROTLI_TRUE;
}
}
static BROTLI_BOOL OpenFiles(Context* context) {
BROTLI_BOOL is_ok = OpenInputFile(context->current_input_path, &context->fin);
if (!context->test_integrity && is_ok) {
is_ok = OpenOutputFile(
context->current_output_path, &context->fout, context->force_overwrite);
}
return is_ok;
}
static BROTLI_BOOL CloseFiles(Context* context, BROTLI_BOOL success) {
BROTLI_BOOL is_ok = BROTLI_TRUE;
if (!context->test_integrity && context->fout) {
if (!success && context->current_output_path) {
unlink(context->current_output_path);
}
if (fclose(context->fout) != 0) {
if (success) {
fprintf(stderr, "fclose failed [%s]: %s\n",
PrintablePath(context->current_output_path), strerror(errno));
}
is_ok = BROTLI_FALSE;
}
/* TOCTOU violation, but otherwise it is impossible to set file times. */
if (success && is_ok && context->copy_stat) {
CopyStat(context->current_input_path, context->current_output_path);
}
}
if (context->fin) {
if (fclose(context->fin) != 0) {
if (is_ok) {
fprintf(stderr, "fclose failed [%s]: %s\n",
PrintablePath(context->current_input_path), strerror(errno));
}
is_ok = BROTLI_FALSE;
}
}
if (success && context->junk_source && context->current_input_path) {
unlink(context->current_input_path);
}
context->fin = NULL;
context->fout = NULL;
return is_ok;
}
static const size_t kFileBufferSize = 1 << 16;
static BROTLI_BOOL DecompressFile(Context* context, BrotliDecoderState* s) {
size_t available_in = 0;
const uint8_t* next_in = NULL;
size_t available_out = kFileBufferSize;
uint8_t* next_out = context->output;
BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
for (;;) {
if (next_out != context->output) {
if (!context->test_integrity) {
size_t out_size = (size_t)(next_out - context->output);
fwrite(context->output, 1, out_size, context->fout);
if (ferror(context->fout)) {
fprintf(stderr, "failed to write output [%s]: %s\n",
PrintablePath(context->current_output_path), strerror(errno));
return BROTLI_FALSE;
}
}
available_out = kFileBufferSize;
next_out = context->output;
}
if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) {
if (feof(context->fin)) {
fprintf(stderr, "corrupt input [%s]\n",
PrintablePath(context->current_input_path));
return BROTLI_FALSE;
}
available_in = fread(context->input, 1, kFileBufferSize, context->fin);
next_in = context->input;
if (ferror(context->fin)) {
fprintf(stderr, "failed to read input [%s]: %s\n",
PrintablePath(context->current_input_path), strerror(errno));
return BROTLI_FALSE;
}
} else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
/* Nothing to do - output is already written. */
} else if (result == BROTLI_DECODER_RESULT_SUCCESS) {
if (available_in != 0 || !feof(context->fin)) {
fprintf(stderr, "corrupt input [%s]\n",
PrintablePath(context->current_input_path));
return BROTLI_FALSE;
}
return BROTLI_TRUE;
} else {
fprintf(stderr, "corrupt input [%s]\n",
PrintablePath(context->current_input_path));
return BROTLI_FALSE;
}
result = BrotliDecoderDecompressStream(
s, &available_in, &next_in, &available_out, &next_out, 0);
}
}
static BROTLI_BOOL DecompressFiles(Context* context) {
while (NextFile(context)) {
BROTLI_BOOL is_ok = BROTLI_TRUE;
BrotliDecoderState* s = BrotliDecoderCreateInstance(NULL, NULL, NULL);
if (!s) {
fprintf(stderr, "out of memory\n");
return BROTLI_FALSE;
}
is_ok = OpenFiles(context);
if (is_ok && !context->current_input_path &&
!context->force_overwrite && isatty(STDIN_FILENO)) {
fprintf(stderr, "Use -h help. Use -f to force input from a terminal.\n");
is_ok = BROTLI_FALSE;
}
if (is_ok) is_ok = DecompressFile(context, s);
BrotliDecoderDestroyInstance(s);
if (!CloseFiles(context, is_ok)) is_ok = BROTLI_FALSE;
if (!is_ok) return BROTLI_FALSE;
}
return BROTLI_TRUE;
}
static BROTLI_BOOL CompressFile(Context* context, BrotliEncoderState* s) {
size_t available_in = 0;
const uint8_t* next_in = NULL;
size_t available_out = kFileBufferSize;
uint8_t* next_out = context->output;
BROTLI_BOOL is_eof = BROTLI_FALSE;
for (;;) {
if (available_in == 0 && !is_eof) {
available_in = fread(context->input, 1, kFileBufferSize, context->fin);
next_in = context->input;
if (ferror(context->fin)) {
fprintf(stderr, "failed to read input [%s]: %s\n",
PrintablePath(context->current_input_path), strerror(errno));
return BROTLI_FALSE;
}
is_eof = feof(context->fin) ? BROTLI_TRUE : BROTLI_FALSE;
}
if (!BrotliEncoderCompressStream(s,
is_eof ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS,
&available_in, &next_in, &available_out, &next_out, NULL)) {
/* Should detect OOM? */
fprintf(stderr, "failed to compress data [%s]\n",
PrintablePath(context->current_input_path));
return BROTLI_FALSE;
}
if (available_out != kFileBufferSize) {
size_t out_size = kFileBufferSize - available_out;
fwrite(context->output, 1, out_size, context->fout);
if (ferror(context->fout)) {
fprintf(stderr, "failed to write output [%s]: %s\n",
PrintablePath(context->current_output_path), strerror(errno));
return BROTLI_FALSE;
}
available_out = kFileBufferSize;
next_out = context->output;
}
if (BrotliEncoderIsFinished(s)) return BROTLI_TRUE;
}
}
static BROTLI_BOOL CompressFiles(Context* context) {
while (NextFile(context)) {
BROTLI_BOOL is_ok = BROTLI_TRUE;
BrotliEncoderState* s = BrotliEncoderCreateInstance(NULL, NULL, NULL);
if (!s) {
fprintf(stderr, "out of memory\n");
return BROTLI_FALSE;
}
BrotliEncoderSetParameter(s,
BROTLI_PARAM_QUALITY, (uint32_t)context->quality);
BrotliEncoderSetParameter(s,
BROTLI_PARAM_LGWIN, (uint32_t)context->lgwin);
is_ok = OpenFiles(context);
if (is_ok && !context->current_output_path &&
!context->force_overwrite && isatty(STDOUT_FILENO)) {
fprintf(stderr, "Use -h help. Use -f to force output to a terminal.\n");
is_ok = BROTLI_FALSE;
}
if (is_ok) is_ok = CompressFile(context, s);
BrotliEncoderDestroyInstance(s);
if (!CloseFiles(context, is_ok)) is_ok = BROTLI_FALSE;
if (!is_ok) return BROTLI_FALSE;
}
return BROTLI_TRUE;
}
int main(int argc, char** argv) {
Command command;
Context context;
BROTLI_BOOL is_ok = BROTLI_TRUE;
int i;
context.quality = 11;
context.lgwin = DEFAULT_LGWIN;
context.force_overwrite = BROTLI_FALSE;
context.junk_source = BROTLI_FALSE;
context.copy_stat = BROTLI_TRUE;
context.test_integrity = BROTLI_FALSE;
context.verbose = BROTLI_FALSE;
context.write_to_stdout = BROTLI_FALSE;
context.decompress = BROTLI_FALSE;
context.output_path = NULL;
context.suffix = DEFAULT_SUFFIX;
for (i = 0; i < MAX_OPTIONS; ++i) context.not_input_indices[i] = 0;
context.longest_path_len = 1;
context.input_count = 0;
context.argc = argc;
context.argv = argv;
context.modified_path = NULL;
context.iterator = 0;
context.ignore = 0;
context.iterator_error = BROTLI_FALSE;
context.buffer = NULL;
context.current_input_path = NULL;
context.current_output_path = NULL;
context.fin = NULL;
context.fout = NULL;
command = ParseParams(&context);
if (command == COMMAND_COMPRESS || command == COMMAND_DECOMPRESS ||
command == COMMAND_TEST_INTEGRITY) {
if (is_ok) {
size_t modified_path_len =
context.longest_path_len + strlen(context.suffix) + 1;
context.modified_path = (char*)malloc(modified_path_len);
context.buffer = (uint8_t*)malloc(kFileBufferSize * 2);
if (!context.modified_path || !context.buffer) {
fprintf(stderr, "out of memory\n");
is_ok = BROTLI_FALSE;
} else {
context.input = context.buffer;
context.output = context.buffer + kFileBufferSize;
}
}
}
if (!is_ok) command = COMMAND_NOOP;
switch (command) {
case COMMAND_NOOP:
break;
case COMMAND_VERSION:
PrintVersion();
break;
case COMMAND_COMPRESS:
is_ok = CompressFiles(&context);
break;
case COMMAND_DECOMPRESS:
case COMMAND_TEST_INTEGRITY:
is_ok = DecompressFiles(&context);
break;
case COMMAND_HELP:
case COMMAND_INVALID:
default:
PrintHelp(FileName(argv[0]));
is_ok = (command == COMMAND_HELP);
break;
}
if (context.iterator_error) is_ok = BROTLI_FALSE;
free(context.modified_path);
free(context.buffer);
if (!is_ok) exit(1);
return 0;
}

107
modules/brotli/tools/brotli.md Executable file
View File

@ -0,0 +1,107 @@
brotli(1) -- brotli, unbrotli - compress or decompress files
================================================================
SYNOPSIS
--------
`brotli` [*OPTION|FILE*]...
`unbrotli` is equivalent to `brotli --decompress`
DESCRIPTION
-----------
`brotli` is a generic-purpose lossless compression algorithm that compresses
data using a combination of a modern variant of the **LZ77** algorithm, Huffman
coding and 2-nd order context modeling, with a compression ratio comparable to
the best currently available general-purpose compression methods. It is similar
in speed with deflate but offers more dense compression.
`brotli` command line syntax similar to `gzip (1)` and `zstd (1)`.
Unlike `gzip (1)`, source files are preserved by default. It is possible to
remove them after processing by using the `--rm` _option_.
Arguments that look like "`--name`" or "`--name=value`" are _options_. Every
_option_ has a short form "`-x`" or "`-x value`". Multiple short form _options_
could be coalesced:
* "`--decompress --stdout --suffix=.b`" works the same as
* "`-d -s -S .b`" and
* "`-dsS .b`"
`brotli` has 3 operation modes:
* default mode is compression;
* `--decompress` option activates decompression mode;
* `--test` option switches to integrity test mode; this option is equivalent to
"`--decompress --stdout`" except that the decompressed data is discarded
instead of being written to standard output.
Every non-option argument is a _file_ entry. If no _files_ are given or _file_
is "`-`", `brotli` reads from standard input. All arguments after "`--`" are
_file_ entries.
Unless `--stdout` or `--output` is specified, _files_ are written to a new file
whose name is derived from the source _file_ name:
* when compressing, a suffix is appended to the source filename to
get the target filename
* when decompressing, a suffix is removed from the source filename to
get the target filename
Default suffix is `.br`, but it could be specified with `--suffix` option.
Conflicting or duplicate _options_ are not allowed.
OPTIONS
-------
* `-#`:
compression level (0-9); bigger values cause denser, but slower compression
* `-c`, `--stdout`:
write on standard output
* `-d`, `--decompress`:
decompress mode
* `-f`, `--force`:
force output file overwrite
* `-h`, `--help`:
display this help and exit
* `-j`, `--rm`:
remove source file(s); `gzip (1)`-like behaviour
* `-k`, `--keep`:
keep source file(s); `zstd (1)`-like behaviour
* `-n`, `--no-copy-stat`:
do not copy source file(s) attributes
* `-o FILE`, `--output=FILE`
output file; valid only if there is a single input entry
* `-q NUM`, `--quality=NUM`:
compression level (0-11); bigger values cause denser, but slower compression
* `-t`, `--test`:
test file integrity mode
* `-v`, `--verbose`:
increase output verbosity
* `-w NUM`, `--lgwin=NUM`:
set LZ77 window size (0, 10-24) (default: 22); window size is
`(2**NUM - 16)`; 0 lets compressor decide over the optimal value; bigger
windows size improve density; decoder might require up to window size
memory to operate
* `-S SUF`, `--suffix=SUF`:
output file suffix (default: `.br`)
* `-V`, `--version`:
display version and exit
* `-Z`, `--best`:
use best compression level (default); same as "`-q 11`"
SEE ALSO
--------
`brotli` file format is defined in
[RFC 7932](https://www.ietf.org/rfc/rfc7932.txt).
`brotli` is open-sourced under the
[MIT License](https://opensource.org/licenses/MIT).
Mailing list: https://groups.google.com/forum/#!forum/brotli
BUGS
----
Report bugs at: https://github.com/google/brotli/issues

4
modules/brotli/update.sh
View File

@ -6,7 +6,7 @@
MY_TEMP_DIR=`mktemp -d -t brotli_update.XXXXXX` || exit 1
git clone https://github.com/google/brotli ${MY_TEMP_DIR}/brotli
git -C ${MY_TEMP_DIR}/brotli checkout v0.6.0
git -C ${MY_TEMP_DIR}/brotli checkout v1.0.1
COMMIT=$(git -C ${MY_TEMP_DIR}/brotli rev-parse HEAD)
perl -p -i -e "s/\[commit [0-9a-f]{40}\]/[commit ${COMMIT}]/" README.mozilla;
@ -15,7 +15,7 @@ DIRS="common dec enc include tools"
for d in $DIRS; do
rm -rf $d
mv ${MY_TEMP_DIR}/brotli/$d $d
mv ${MY_TEMP_DIR}/brotli/c/$d $d
done
rm -rf ${MY_TEMP_DIR}