openharmony/third_party_lz4
1
0
Fork 0
mirror of https://github.com/openharmony/third_party_lz4.git synced 2026-07-01 06:48:22 -04:00
Code Issues Packages Projects Releases Wiki Activity

Signed-off-by: hughes802 <h.hu@huawei.com>

Browse Source
This commit is contained in:
hughes27
2022-08-31 10:53:02 +08:00
parent e93b0eea93
commit f4d5431d5b
142 changed files with 18305 additions and 716 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cirrus.yml
+5
View File
@@ -0,0 +1,5 @@
freebsd_instance:
image_family: freebsd-12-1
task:
script: pkg install -y gmake && gmake test
.gitattributes
+5 -1
View File
@@ -1,17 +1,21 @@
# Set the default behavior
* text eol=lf
# Explicitly declare source files
*.c text eol=lf
*.h text eol=lf
# Denote files that should not be modified.
*.odt binary
*.png binary
# Visual Studio
*.sln text eol=crlf
*.vcxproj* text eol=crlf
*.vcproj* text eol=crlf
*.suo binary
*.rc text eol=crlf
# Windows
*.bat text eol=crlf
*.cmd text eol=crlf
*.cmd text eol=crlf
.gitignore
+3
View File
@@ -26,6 +26,9 @@ _codelite_lz4/
bin/
*.zip
# analyzers
infer-out
# Mac
.DS_Store
*.dSYM
.travis.yml
+25 -9
View File
@@ -10,9 +10,7 @@ matrix:
script:
- make # test library build
- make clean
- make -C tests test-lz4 MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' | tee # test scenario where `stdout` is not the console
- make clean
- CFLAGS=-m32 make -C tests test-lz4-contentSize
- make test MOREFLAGS='-Werror -Wconversion -Wno-sign-conversion' | tee # test scenario where `stdout` is not the console
# Container-based 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
- name: (Precise) benchmark test
@@ -33,9 +31,15 @@ matrix:
script:
- CC=clang MOREFLAGS=-fsanitize=address make -C tests test-frametest test-fuzzer
- name: Custom LZ4_DISTANCE_MAX
- name: Custom LZ4_DISTANCE_MAX ; lz4-wlib (CLI linked to dynamic library); LZ4_USER_MEMORY_FUNCTIONS
script:
- MOREFLAGS=-DLZ4_DISTANCE_MAX=8000 make check
- make clean
- make -C programs lz4-wlib
- make clean
- make -C tests fullbench-wmalloc # test LZ4_USER_MEMORY_FUNCTIONS
- make clean
- CC="c++ -Wno-deprecated" make -C tests fullbench-wmalloc # stricter function signature check
- name: (Precise) g++ and clang CMake test
dist: precise
@@ -47,18 +51,15 @@ matrix:
- make clean travis-install
- make clean clangtest
# 14.04 LTS Server Edition 64 bit
- name: (Trusty) i386 gcc test
dist: trusty
- name: x32 compatibility test
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
script:
- make -C tests test MOREFLAGS=-mx32
# 14.04 LTS Server Edition 64 bit
# presume clang >= v3.9.0
- name: (Trusty) USan test
dist: trusty
@@ -122,6 +123,21 @@ matrix:
- make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static
- make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static
- name: aarch64 real-hw tests
arch: arm64
script:
- make test
- name: PPC64LE real-hw tests
arch: ppc64le
script:
- make test
- name: IBM s390x real-hw tests
arch: s390x
script:
- make test
- name: (Xenial) gcc-5 compilation
dist: xenial
install:
Makefile
+10 -5
View File
@@ -98,7 +98,7 @@ travis-install:
$(MAKE) -j1 install DESTDIR=~/install_test_dir
cmake:
@cd contrib/cmake_unofficial; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
@cd build/cmake; cmake $(CMAKE_PARAMS) CMakeLists.txt; $(MAKE)
endif
@@ -124,8 +124,8 @@ check:
.PHONY: test
test:
$(MAKE) -C $(TESTDIR) $@
$(MAKE) -C $(EXDIR) $@
CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" $(MAKE) -C $(TESTDIR) $@
CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" $(MAKE) -C $(EXDIR) $@
clangtest: CFLAGS ?= -O3
clangtest: CFLAGS += -Werror -Wconversion -Wno-sign-conversion
@@ -142,15 +142,20 @@ clangtest-native: clean
@CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(PRGDIR) native CC=clang
@CFLAGS="-O3 -Werror -Wconversion -Wno-sign-conversion" $(MAKE) -C $(TESTDIR) native CC=clang
usan: CC = clang
usan: CFLAGS = -O3 -g -fsanitize=undefined -fno-sanitize-recover=undefined -fsanitize-recover=pointer-overflow
usan: LDFLAGS = $(CFLAGS)
usan: clean
CC=clang CFLAGS="-O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
usan32: clean
CFLAGS="-m32 -O3 -g -fsanitize=undefined" $(MAKE) test FUZZER_TIME="-T30s" NB_LOOPS=-i1
SCANBUILD ?= scan-build
SCANBUILD_FLAGS += --status-bugs -v --force-analyze-debug-code
.PHONY: staticAnalyze
staticAnalyze: clean
CFLAGS=-g scan-build --status-bugs -v $(MAKE) all
CPPFLAGS=-DLZ4_DEBUG=1 CFLAGS=-g $(SCANBUILD) $(SCANBUILD_FLAGS) $(MAKE) all V=1 DEBUGLEVEL=1
.PHONY: cppcheck
cppcheck:
NEWS
+36
View File
@@ -1,3 +1,39 @@
v1.9.3
perf: highly improved speed in kernel space, by @terrelln
perf: faster speed with Visual Studio, thanks to @wolfpld and @remittor
perf: improved dictionary compression speed, by @felixhandte
perf: fixed LZ4_compress_HC_destSize() ratio, detected by @hsiangkao
perf: reduced stack usage in high compression mode, by @Yanpas
api : LZ4_decompress_safe_partial() supports unknown compressed size, requested by @jfkthame
api : improved LZ4F_compressBound() with automatic flushing, by Christopher Harvie
api : can (de)compress to/from NULL without UBs
api : fix alignment test on 32-bit systems (state initialization)
api : fix LZ4_saveDictHC() in corner case scenario, detected by @IgorKorkin
cli : `-l` legacy format is now compatible with `-m` multiple files, by Filipe Calasans
cli : benchmark mode supports dictionary, by @rkoradi
cli : fix --fast with large argument, detected by @picoHz
build: link to user-defined memory functions with LZ4_USER_MEMORY_FUNCTIONS, suggested by Yuriy Levchenko
build: contrib/cmake_unofficial/ moved to build/cmake/
build: visual/* moved to build/
build: updated meson script, by @neheb
build: tinycc support, by Anton Kochkov
install: Haiku support, by Jerome Duval
doc : updated LZ4 frame format, clarify EndMark
v1.9.2
fix : out-of-bound read in exceptional circumstances when using decompress_partial(), by @terrelln
fix : slim opportunity for out-of-bound write with compress_fast() with a large enough input and when providing an output smaller than recommended (< LZ4_compressBound(inputSize)), by @terrelln
fix : rare data corruption bug with LZ4_compress_destSize(), by @terrelln
fix : data corruption bug when Streaming with an Attached Dict in HC Mode, by @felixhandte
perf: enable LZ4_FAST_DEC_LOOP on aarch64/GCC by default, by @prekageo
perf: improved lz4frame streaming API speed, by @dreambottle
perf: speed up lz4hc on slow patterns when using external dictionary, by @terrelln
api: better in-place decompression and compression support
cli : --list supports multi-frames files, by @gstedman
cli: --version outputs to stdout
cli : add option --best as an alias of -12 , by @Low-power
misc: Integration into oss-fuzz by @cmeister2, expanded list of scenarios by @terrelln
v1.9.1
fix : decompression functions were reading a few bytes beyond input size (introduced in v1.9.0, reported by @ppodolsky and @danlark1)
api : fix : lz4frame initializers compatibility with c++, reported by @degski
README.OpenSource
+2 -2
View File
@@ -3,9 +3,9 @@
"Name": "lz4",
"License": "BSD 2-Clause License",
"License File": "LICENSE",
"Version Number": "1.9.2",
"Version Number": "1.9.3",
"Owner": "chennaidong@huawei.com",
"Upstream URL": "https://github.com/pierrec/lz4",
"Upstream URL": "https://github.com/lz4/lz4",
"Description": "This package provides a streaming interface to LZ4 data streams as well as low level compress and uncompress functions for LZ4 data blocks. The implementation is based on the reference C one."
}
]
README.md
+11 -12
View File
@@ -26,25 +26,13 @@ LZ4 library is provided as open-source software using BSD 2-Clause license.
|Branch |Status |
|------------|---------|
|master | [![Build Status][travisMasterBadge]][travisLink] [![Build status][AppveyorMasterBadge]][AppveyorLink] [![coverity][coverBadge]][coverlink] |
|dev | [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] |
[travisMasterBadge]: https://travis-ci.org/lz4/lz4.svg?branch=master "Continuous Integration test suite"
[travisDevBadge]: https://travis-ci.org/lz4/lz4.svg?branch=dev "Continuous Integration test suite"
[travisLink]: https://travis-ci.org/lz4/lz4
[AppveyorMasterBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=master&svg=true "Windows test suite"
[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=dev&svg=true "Windows test suite"
[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/lz4-1lndh
[coverBadge]: https://scan.coverity.com/projects/4735/badge.svg "Static code analysis of Master branch"
[coverlink]: https://scan.coverity.com/projects/4735
> **Branch Policy:**
> - The "master" branch is considered stable, at all times.
> - The "dev" branch is the one where all contributions must be merged
before being promoted to master.
> + If you plan to propose a patch, please commit into the "dev" branch,
or its own feature branch.
Direct commit to "master" are not permitted.
Benchmarks
-------------------------
@@ -95,6 +83,17 @@ It is compatible with parallel builds (`-j#`).
[redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
[command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html
Building LZ4 - Using vcpkg
You can download and install LZ4 using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager:
git clone https://github.com/Microsoft/vcpkg.git
cd vcpkg
./bootstrap-vcpkg.sh
./vcpkg integrate install
vcpkg install lz4
The LZ4 port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
Documentation
-------------------------
appveyor.yml
+29 -23
View File
@@ -1,22 +1,22 @@
version: 1.0.{build}
environment:
matrix:
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "Win32"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "Win32"
- COMPILER: "gcc"
PLATFORM: "mingw64"
- COMPILER: "gcc"
PLATFORM: "mingw32"
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "Win32"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "Win32"
- COMPILER: "gcc"
PLATFORM: "clang"
@@ -47,10 +47,14 @@ build_script:
make -v &&
echo ----- &&
if not [%PLATFORM%]==[clang] (
make -C programs lz4 && make -C tests fullbench && make -C lib lib
make -C programs lz4 &&
make -C tests fullbench &&
make -C tests fuzzer &&
make -C lib lib V=1
) ELSE (
make -C programs lz4 CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" &&
make -C tests fullbench CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" &&
make -C tests fuzzer CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" &&
make -C lib lib CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"
)
)
@@ -63,39 +67,39 @@ build_script:
COPY lib\lz4hc.h bin\include\ &&
COPY lib\lz4frame.h bin\include\ &&
COPY lib\liblz4.a bin\static\liblz4_static.lib &&
COPY lib\dll\liblz4.* bin\dll\ &&
COPY lib\dll\* bin\dll\ &&
COPY lib\dll\example\Makefile bin\example\ &&
COPY lib\dll\example\fullbench-dll.* bin\example\ &&
COPY lib\dll\example\README.md bin\ &&
COPY programs\lz4.exe bin\lz4.exe
)
- if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
7z.exe a bin\lz4_x64.zip NEWS .\bin\lz4.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include &&
7z.exe a -bb1 bin\lz4_x64.zip NEWS .\bin\lz4.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include &&
appveyor PushArtifact bin\lz4_x64.zip
)
- if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw32] (
7z.exe a bin\lz4_x86.zip NEWS .\bin\lz4.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include &&
7z.exe a -bb1 bin\lz4_x86.zip NEWS .\bin\lz4.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include &&
appveyor PushArtifact bin\lz4_x86.zip
)
- if [%COMPILER%]==[gcc] (COPY tests\fullbench.exe programs\)
- if [%COMPILER%]==[gcc] (COPY tests\*.exe programs\)
- if [%COMPILER%]==[visual] (
ECHO *** &&
ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /p:EnableWholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
msbuild "build\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /p:EnableWholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
ECHO *** &&
ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
msbuild "build\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
ECHO *** &&
ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
msbuild "build\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
ECHO *** &&
ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
COPY visual\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe programs\
msbuild "build\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe programs\
)
test_script:
@@ -110,7 +114,9 @@ test_script:
lz4 -i1b10 lz4.exe &&
lz4 -i1b15 lz4.exe &&
echo ------- lz4 tested ------- &&
fullbench.exe -i1 fullbench.exe
fullbench.exe -i1 fullbench.exe &&
echo trying to launch fuzzer.exe &&
fuzzer.exe -v -T30s
)
artifacts:
build/.gitignore
+16
View File
@@ -0,0 +1,16 @@
# Visual C++
.vs/
*Copy
*.db
*.opensdf
*.sdf
*.suo
*.user
ver*/
VS2010/bin/
VS2017/bin/
ipch
# Fixup for lz4 project directories
!VS2010/lz4
!VS2017/lz4
visual/README.md → build/README.md
+10 -8
View File
@@ -4,7 +4,9 @@ Projects for various integrated development environments (IDE)
#### Included projects
The following projects are included with the lz4 distribution:
- `cmake` - CMake project
- `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015)
- `VS2017` - Visual Studio 2017 project
#### How to compile lz4 with Visual Studio
@@ -12,25 +14,25 @@ The following projects are included with the lz4 distribution:
1. Install Visual Studio e.g. VS 2015 Community Edition (it's free).
2. Download the latest version of lz4 from https://github.com/lz4/lz4/releases
3. Decompress ZIP archive.
4. Go to decompressed directory then to `visual` then `VS2010` and open `lz4.sln`
4. Go to decompressed directory then to `build` then `VS2010` and open `lz4.sln`
5. Visual Studio will ask about converting VS2010 project to VS2015 and you should agree.
6. Change `Debug` to `Release` and if you have 64-bit Windows change also `Win32` to `x64`.
7. Press F7 on keyboard or select `BUILD` from the menu bar and choose `Build Solution`.
8. If compilation will be fine a compiled executable will be in `visual\VS2010\bin\x64_Release\lz4.exe`
8. If compilation will be fine a compiled executable will be in `build\VS2010\bin\x64_Release\lz4.exe`
#### Projects available within lz4.sln
The Visual Studio solution file `lz4.sln` contains many projects that will be compiled to the
`visual\VS2010\bin\$(Platform)_$(Configuration)` directory. For example `lz4` set to `x64` and
`Release` will be compiled to `visual\VS2010\bin\x64_Release\lz4.exe`. The solution file contains the
`build\VS2010\bin\$(Platform)_$(Configuration)` directory. For example `lz4` set to `x64` and
`Release` will be compiled to `build\VS2010\bin\x64_Release\lz4.exe`. The solution file contains the
following projects:
- `lz4` : Command Line Utility, supporting gzip-like arguments
- `datagen` : Synthetic and parametrable data generator, for tests
- `frametest` : Test tool that checks lz4frame integrity on target platform
- `fullbench` : Precisely measure speed for each lz4 inner functions
- `fuzzer` : Test tool, to check lz4 integrity on target platform
- `fuzzer` : Test tool, to check lz4 integrity on target platform
- `liblz4` : A static LZ4 library compiled to `liblz4_static.lib`
- `liblz4-dll` : A dynamic LZ4 library (DLL) compiled to `liblz4.dll` with the import library `liblz4.lib`
- `fullbench-dll` : The fullbench program compiled with the import library; the executable requires LZ4 DLL
@@ -39,8 +41,8 @@ following projects:
#### Using LZ4 DLL with Microsoft Visual C++ project
The header files `lib\lz4.h`, `lib\lz4hc.h`, `lib\lz4frame.h` and the import library
`visual\VS2010\bin\$(Platform)_$(Configuration)\liblz4.lib` are required to compile a
project using Visual C++.
`build\VS2010\bin\$(Platform)_$(Configuration)\liblz4.lib` are required to
compile a project using Visual C++.
1. The path to header files should be added to `Additional Include Directories` that can
be found in Project Properties of Visual Studio IDE in the `C/C++` Property Pages on the `General` page.
@@ -50,4 +52,4 @@ project using Visual C++.
then the directory has to be added to `Linker\General\Additional Library Directories`.
The compiled executable will require LZ4 DLL which is available at
`visual\VS2010\bin\$(Platform)_$(Configuration)\liblz4.dll`.
`build\VS2010\bin\$(Platform)_$(Configuration)\liblz4.dll`.
visual/VS2010/datagen/datagen.vcxproj → build/VS2010/datagen/datagen.vcxproj
View File
visual/VS2010/frametest/frametest.vcxproj → build/VS2010/frametest/frametest.vcxproj
View File
visual/VS2010/fullbench-dll/fullbench-dll.vcxproj → build/VS2010/fullbench-dll/fullbench-dll.vcxproj
View File
visual/VS2010/fullbench/fullbench.vcxproj → build/VS2010/fullbench/fullbench.vcxproj
View File
visual/VS2010/fuzzer/fuzzer.vcxproj → build/VS2010/fuzzer/fuzzer.vcxproj
View File
visual/VS2010/liblz4-dll/liblz4-dll.rc → build/VS2010/liblz4-dll/liblz4-dll.rc
View File
visual/VS2010/liblz4-dll/liblz4-dll.vcxproj → build/VS2010/liblz4-dll/liblz4-dll.vcxproj
View File
visual/VS2010/liblz4/liblz4.vcxproj → build/VS2010/liblz4/liblz4.vcxproj
View File
visual/VS2010/lz4.sln → build/VS2010/lz4.sln
View File
build/VS2010/lz4/lz4.rc
+51
View File
@@ -0,0 +1,51 @@
// Microsoft Visual C++ generated resource script.
//
#include "lz4.h" /* LZ4_VERSION_STRING */
#define APSTUDIO_READONLY_SYMBOLS
#include "verrsrc.h"
#undef APSTUDIO_READONLY_SYMBOLS
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
LANGUAGE 9, 1
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
PRODUCTVERSION LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
#ifdef _DEBUG
FILEFLAGS VS_FF_DEBUG
#else
FILEFLAGS 0x0L
#endif
FILEOS VOS_NT_WINDOWS32
FILETYPE VFT_DLL
FILESUBTYPE VFT2_UNKNOWN
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904B0"
BEGIN
VALUE "CompanyName", "Yann Collet"
VALUE "FileDescription", "Extremely fast compression"
VALUE "FileVersion", LZ4_VERSION_STRING
VALUE "InternalName", "lz4.exe"
VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
VALUE "OriginalFilename", "lz4.exe"
VALUE "ProductName", "LZ4"
VALUE "ProductVersion", LZ4_VERSION_STRING
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x0409, 1200
END
END
#endif
build/VS2010/lz4/lz4.vcxproj
+189
View File
@@ -0,0 +1,189 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{E30329AC-0057-4FE0-8FDA-7F650D398C4C}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>lz4</RootNamespace>
<OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization Condition="'$(EnableWholeProgramOptimization)'=='true'">true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreadedDebug</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>true</EnablePREfast>
<AdditionalOptions>/analyze:stacksize295252 %(AdditionalOptions)</AdditionalOptions>
<RuntimeLibrary Condition="'$(UseStaticCRT)'=='true'">MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\lz4.h" />
<ClInclude Include="..\..\..\lib\lz4frame.h" />
<ClInclude Include="..\..\..\lib\lz4frame_static.h" />
<ClInclude Include="..\..\..\lib\lz4hc.h" />
<ClInclude Include="..\..\..\lib\xxhash.h" />
<ClInclude Include="..\..\..\programs\datagen.h" />
<ClInclude Include="..\..\..\programs\bench.h" />
<ClInclude Include="..\..\..\programs\lz4io.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\lz4.c" />
<ClCompile Include="..\..\..\lib\lz4frame.c" />
<ClCompile Include="..\..\..\lib\lz4hc.c" />
<ClCompile Include="..\..\..\lib\xxhash.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\bench.c" />
<ClCompile Include="..\..\..\programs\lz4cli.c" />
<ClCompile Include="..\..\..\programs\lz4io.c" />
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="lz4.rc" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
visual/VS2017/datagen/datagen.vcxproj → build/VS2017/datagen/datagen.vcxproj
View File
visual/VS2017/frametest/frametest.vcxproj → build/VS2017/frametest/frametest.vcxproj
View File
visual/VS2017/fullbench-dll/fullbench-dll.vcxproj → build/VS2017/fullbench-dll/fullbench-dll.vcxproj
View File
visual/VS2017/fullbench/fullbench.vcxproj → build/VS2017/fullbench/fullbench.vcxproj
View File
visual/VS2017/fuzzer/fuzzer.vcxproj → build/VS2017/fuzzer/fuzzer.vcxproj
View File
visual/VS2017/liblz4-dll/liblz4-dll.rc → build/VS2017/liblz4-dll/liblz4-dll.rc
View File
visual/VS2017/liblz4-dll/liblz4-dll.vcxproj → build/VS2017/liblz4-dll/liblz4-dll.vcxproj
View File
visual/VS2017/liblz4/liblz4.vcxproj → build/VS2017/liblz4/liblz4.vcxproj
View File
visual/VS2017/lz4.sln → build/VS2017/lz4.sln
+10
View File
@@ -19,6 +19,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-
{9800039D-4AAA-43A4-BB78-FEF6F4836927} = {9800039D-4AAA-43A4-BB78-FEF6F4836927}
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{60A3115E-B988-41EE-8815-F4D4F253D866}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
@@ -83,6 +85,14 @@ Global
{13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32
{13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64
{13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64
{60A3115E-B988-41EE-8815-F4D4F253D866}.Debug|Win32.ActiveCfg = Debug|Win32
{60A3115E-B988-41EE-8815-F4D4F253D866}.Debug|Win32.Build.0 = Debug|Win32
{60A3115E-B988-41EE-8815-F4D4F253D866}.Debug|x64.ActiveCfg = Debug|x64
{60A3115E-B988-41EE-8815-F4D4F253D866}.Debug|x64.Build.0 = Debug|x64
{60A3115E-B988-41EE-8815-F4D4F253D866}.Release|Win32.ActiveCfg = Release|Win32
{60A3115E-B988-41EE-8815-F4D4F253D866}.Release|Win32.Build.0 = Release|Win32
{60A3115E-B988-41EE-8815-F4D4F253D866}.Release|x64.ActiveCfg = Release|x64
{60A3115E-B988-41EE-8815-F4D4F253D866}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
build/VS2017/lz4/lz4.rc
+51
View File
@@ -0,0 +1,51 @@
// Microsoft Visual C++ generated resource script.
//
#include "lz4.h" /* LZ4_VERSION_STRING */
#define APSTUDIO_READONLY_SYMBOLS
#include "verrsrc.h"
#undef APSTUDIO_READONLY_SYMBOLS
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
LANGUAGE 9, 1
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
PRODUCTVERSION LZ4_VERSION_MAJOR,LZ4_VERSION_MINOR,LZ4_VERSION_RELEASE,0
FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
#ifdef _DEBUG
FILEFLAGS VS_FF_DEBUG
#else
FILEFLAGS 0x0L
#endif
FILEOS VOS_NT_WINDOWS32
FILETYPE VFT_DLL
FILESUBTYPE VFT2_UNKNOWN
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904B0"
BEGIN
VALUE "CompanyName", "Yann Collet"
VALUE "FileDescription", "Extremely fast compression"
VALUE "FileVersion", LZ4_VERSION_STRING
VALUE "InternalName", "lz4.exe"
VALUE "LegalCopyright", "Copyright (C) 2013-2016, Yann Collet"
VALUE "OriginalFilename", "lz4.exe"
VALUE "ProductName", "LZ4"
VALUE "ProductVersion", LZ4_VERSION_STRING
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x0409, 1200
END
END
#endif
build/VS2017/lz4/lz4.vcxproj
+164
View File
@@ -0,0 +1,164 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<VCProjectVersion>15.0</VCProjectVersion>
<ProjectGuid>{60A3115E-B988-41EE-8815-F4D4F253D866}</ProjectGuid>
<RootNamespace>lz4</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<WholeProgramOptimization>false</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v141</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
<IncludePath>$(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<LinkIncremental>false</LinkIncremental>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<SDLCheck>true</SDLCheck>
<ConformanceMode>true</ConformanceMode>
<TreatWarningAsError>true</TreatWarningAsError>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
<SubSystem>Console</SubSystem>
<OptimizeReferences>false</OptimizeReferences>
<EnableCOMDATFolding>false</EnableCOMDATFolding>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<SDLCheck>true</SDLCheck>
<ConformanceMode>true</ConformanceMode>
</ClCompile>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck>
<ConformanceMode>true</ConformanceMode>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck>
<ConformanceMode>true</ConformanceMode>
</ClCompile>
<Link>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\lz4.c" />
<ClCompile Include="..\..\..\lib\lz4frame.c" />
<ClCompile Include="..\..\..\lib\lz4hc.c" />
<ClCompile Include="..\..\..\lib\xxhash.c" />
<ClCompile Include="..\..\..\programs\bench.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\lz4cli.c" />
<ClCompile Include="..\..\..\programs\lz4io.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\lz4.h" />
<ClInclude Include="..\..\..\lib\lz4frame.h" />
<ClInclude Include="..\..\..\lib\lz4frame_static.h" />
<ClInclude Include="..\..\..\lib\lz4hc.h" />
<ClInclude Include="..\..\..\lib\xxhash.h" />
<ClInclude Include="..\..\..\programs\bench.h" />
<ClInclude Include="..\..\..\programs\datagen.h" />
<ClInclude Include="..\..\..\programs\lz4io.h" />
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="lz4.rc" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
build/cmake/.gitignore
+9
View File
@@ -0,0 +1,9 @@
# cmake artefact
CMakeCache.txt
CMakeFiles
*.cmake
Makefile
liblz4.pc
lz4c
install_manifest.txt
build/cmake/CMakeLists.txt
+235
View File
@@ -0,0 +1,235 @@
# CMake support for LZ4
#
# To the extent possible under law, the author(s) have dedicated all
# copyright and related and neighboring rights to this software to
# the public domain worldwide. This software is distributed without
# any warranty.
#
# For details, see <http://creativecommons.org/publicdomain/zero/1.0/>.
#
# LZ4's CMake support is maintained by Evan Nemerson; when filing
# bugs please mention @nemequ to make sure I see it.
set(LZ4_TOP_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
option(LZ4_BUILD_CLI "Build lz4 program" ON)
option(LZ4_BUILD_LEGACY_LZ4C "Build lz4c progam with legacy argument support" ON)
# Parse version information
file(STRINGS "${LZ4_TOP_SOURCE_DIR}/lib/lz4.h" LZ4_VERSION_MAJOR REGEX "^#define LZ4_VERSION_MAJOR +([0-9]+) +.*$")
string(REGEX REPLACE "^#define LZ4_VERSION_MAJOR +([0-9]+) +.*$" "\\1" LZ4_VERSION_MAJOR "${LZ4_VERSION_MAJOR}")
file(STRINGS "${LZ4_TOP_SOURCE_DIR}/lib/lz4.h" LZ4_VERSION_MINOR REGEX "^#define LZ4_VERSION_MINOR +([0-9]+) +.*$")
string(REGEX REPLACE "^#define LZ4_VERSION_MINOR +([0-9]+) +.*$" "\\1" LZ4_VERSION_MINOR "${LZ4_VERSION_MINOR}")
file(STRINGS "${LZ4_TOP_SOURCE_DIR}/lib/lz4.h" LZ4_VERSION_RELEASE REGEX "^#define LZ4_VERSION_RELEASE +([0-9]+) +.*$")
string(REGEX REPLACE "^#define LZ4_VERSION_RELEASE +([0-9]+) +.*$" "\\1" LZ4_VERSION_RELEASE "${LZ4_VERSION_RELEASE}")
set(LZ4_VERSION_STRING "${LZ4_VERSION_MAJOR}.${LZ4_VERSION_MINOR}.${LZ4_VERSION_RELEASE}")
mark_as_advanced(LZ4_VERSION_STRING LZ4_VERSION_MAJOR LZ4_VERSION_MINOR LZ4_VERSION_RELEASE)
if("${CMAKE_VERSION}" VERSION_LESS "3.0")
project(LZ4 C)
else()
cmake_policy (SET CMP0048 NEW)
project(LZ4
VERSION ${LZ4_VERSION_STRING}
LANGUAGES C)
endif()
cmake_minimum_required (VERSION 2.8.6)
# If LZ4 is being bundled in another project, we don't want to
# install anything. However, we want to let people override this, so
# we'll use the LZ4_BUNDLED_MODE variable to let them do that; just
# set it to OFF in your project before you add_subdirectory(lz4/contrib/cmake_unofficial).
get_directory_property(LZ4_PARENT_DIRECTORY PARENT_DIRECTORY)
if("${LZ4_BUNDLED_MODE}" STREQUAL "")
# Bundled mode hasn't been set one way or the other, set the default
# depending on whether or not we are the top-level project.
if("${LZ4_PARENT_DIRECTORY}" STREQUAL "")
set(LZ4_BUNDLED_MODE OFF)
else()
set(LZ4_BUNDLED_MODE ON)
endif()
endif()
mark_as_advanced(LZ4_BUNDLED_MODE)
# CPack
if(NOT LZ4_BUNDLED_MODE AND NOT CPack_CMake_INCLUDED)
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LZ4 compression library")
set(CPACK_PACKAGE_DESCRIPTION_FILE "${LZ4_TOP_SOURCE_DIR}/README.md")
set(CPACK_RESOURCE_FILE_LICENSE "${LZ4_TOP_SOURCE_DIR}/LICENSE")
set(CPACK_PACKAGE_VERSION_MAJOR ${LZ4_VERSION_MAJOR})
set(CPACK_PACKAGE_VERSION_MINOR ${LZ4_VERSION_MINOR})
set(CPACK_PACKAGE_VERSION_PATCH ${LZ4_VERSION_RELEASE})
include(CPack)
endif(NOT LZ4_BUNDLED_MODE AND NOT CPack_CMake_INCLUDED)
# Allow people to choose whether to build shared or static libraries
# via the BUILD_SHARED_LIBS option unless we are in bundled mode, in
# which case we always use static libraries.
include(CMakeDependentOption)
CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT LZ4_BUNDLED_MODE" OFF)
CMAKE_DEPENDENT_OPTION(BUILD_STATIC_LIBS "Build static libraries" OFF "BUILD_SHARED_LIBS" ON)
if(NOT BUILD_SHARED_LIBS AND NOT BUILD_STATIC_LIBS)
message(FATAL_ERROR "Both BUILD_SHARED_LIBS and BUILD_STATIC_LIBS have been disabled")
endif()
set(LZ4_LIB_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/lib")
set(LZ4_PROG_SOURCE_DIR "${LZ4_TOP_SOURCE_DIR}/programs")
include_directories("${LZ4_LIB_SOURCE_DIR}")
# CLI sources
set(LZ4_SOURCES
"${LZ4_LIB_SOURCE_DIR}/lz4.c"
"${LZ4_LIB_SOURCE_DIR}/lz4hc.c"
"${LZ4_LIB_SOURCE_DIR}/lz4.h"
"${LZ4_LIB_SOURCE_DIR}/lz4hc.h"
"${LZ4_LIB_SOURCE_DIR}/lz4frame.c"
"${LZ4_LIB_SOURCE_DIR}/lz4frame.h"
"${LZ4_LIB_SOURCE_DIR}/xxhash.c")
set(LZ4_CLI_SOURCES
"${LZ4_PROG_SOURCE_DIR}/bench.c"
"${LZ4_PROG_SOURCE_DIR}/lz4cli.c"
"${LZ4_PROG_SOURCE_DIR}/lz4io.c"
"${LZ4_PROG_SOURCE_DIR}/datagen.c")
# Whether to use position independent code for the static library. If
# we're building a shared library this is ignored and PIC is always
# used.
option(LZ4_POSITION_INDEPENDENT_LIB "Use position independent code for static library (if applicable)" ON)
# liblz4
set(LZ4_LIBRARIES_BUILT)
if(BUILD_SHARED_LIBS)
add_library(lz4_shared SHARED ${LZ4_SOURCES})
set_target_properties(lz4_shared PROPERTIES
OUTPUT_NAME lz4
SOVERSION "${LZ4_VERSION_MAJOR}"
VERSION "${LZ4_VERSION_STRING}")
if(MSVC)
target_compile_definitions(lz4_shared PRIVATE
LZ4_DLL_EXPORT=1)
endif()
list(APPEND LZ4_LIBRARIES_BUILT lz4_shared)
endif()
if(BUILD_STATIC_LIBS)
add_library(lz4_static STATIC ${LZ4_SOURCES})
set_target_properties(lz4_static PROPERTIES
OUTPUT_NAME lz4
POSITION_INDEPENDENT_CODE ${LZ4_POSITION_INDEPENDENT_LIB})
list(APPEND LZ4_LIBRARIES_BUILT lz4_static)
endif()
# link to shared whenever possible, to static otherwise
if(BUILD_SHARED_LIBS)
set(LZ4_LINK_LIBRARY lz4_shared)
else()
set(LZ4_LINK_LIBRARY lz4_static)
endif()
# lz4
if (LZ4_BUILD_CLI)
set(LZ4_PROGRAMS_BUILT lz4cli)
add_executable(lz4cli ${LZ4_CLI_SOURCES})
set_target_properties(lz4cli PROPERTIES OUTPUT_NAME lz4)
target_link_libraries(lz4cli ${LZ4_LINK_LIBRARY})
endif()
# lz4c
if (LZ4_BUILD_LEGACY_LZ4C)
list(APPEND LZ4_PROGRAMS_BUILT lz4c)
add_executable(lz4c ${LZ4_CLI_SOURCES})
set_target_properties(lz4c PROPERTIES COMPILE_DEFINITIONS "ENABLE_LZ4C_LEGACY_OPTIONS")
target_link_libraries(lz4c ${LZ4_LINK_LIBRARY})
endif()
# Extra warning flags
include (CheckCCompilerFlag)
foreach (flag
# GCC-style
-Wall
-Wextra
-Wundef
-Wcast-qual
-Wcast-align
-Wshadow
-Wswitch-enum
-Wdeclaration-after-statement
-Wstrict-prototypes
-Wpointer-arith
# MSVC-style
/W4)
# Because https://gcc.gnu.org/wiki/FAQ#wnowarning
string(REGEX REPLACE "\\-Wno\\-(.+)" "-W\\1" flag_to_test "${flag}")
string(REGEX REPLACE "[^a-zA-Z0-9]+" "_" test_name "CFLAG_${flag_to_test}")
check_c_compiler_flag("${ADD_COMPILER_FLAGS_PREPEND} ${flag_to_test}" ${test_name})
if(${test_name})
set(CMAKE_C_FLAGS "${flag} ${CMAKE_C_FLAGS}")
endif()
unset(test_name)
unset(flag_to_test)
endforeach (flag)
if(NOT LZ4_BUNDLED_MODE)
include(GNUInstallDirs)
install(TARGETS ${LZ4_PROGRAMS_BUILT}
BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}"
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(TARGETS ${LZ4_LIBRARIES_BUILT}
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(FILES
"${LZ4_LIB_SOURCE_DIR}/lz4.h"
"${LZ4_LIB_SOURCE_DIR}/lz4frame.h"
"${LZ4_LIB_SOURCE_DIR}/lz4hc.h"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
install(FILES "${LZ4_PROG_SOURCE_DIR}/lz4.1"
DESTINATION "${CMAKE_INSTALL_MANDIR}/man1")
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblz4.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
# install lz4cat and unlz4 symlinks on *nix
if(UNIX AND LZ4_BUILD_CLI)
install(CODE "
foreach(f lz4cat unlz4)
set(dest \"\$ENV{DESTDIR}${CMAKE_INSTALL_FULL_BINDIR}/\${f}\")
message(STATUS \"Symlinking: \${dest} -> lz4\")
execute_process(
COMMAND \"${CMAKE_COMMAND}\" -E create_symlink lz4 \"\${dest}\")
endforeach()
")
# create manpage aliases
foreach(f lz4cat unlz4)
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${f}.1" ".so man1/lz4.1\n")
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${f}.1"
DESTINATION "${CMAKE_INSTALL_MANDIR}/man1")
endforeach()
endif(UNIX AND LZ4_BUILD_CLI)
endif(NOT LZ4_BUNDLED_MODE)
# pkg-config
set(PREFIX "${CMAKE_INSTALL_PREFIX}")
if("${CMAKE_INSTALL_FULL_LIBDIR}" STREQUAL "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
set(LIBDIR "\${prefix}/${CMAKE_INSTALL_LIBDIR}")
else()
set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}")
endif()
if("${CMAKE_INSTALL_FULL_INCLUDEDIR}" STREQUAL "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}")
set(INCLUDEDIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}")
else()
set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
endif()
# for liblz4.pc substitution
set(VERSION ${LZ4_VERSION_STRING})
configure_file(${LZ4_LIB_SOURCE_DIR}/liblz4.pc.in liblz4.pc @ONLY)
contrib/debian/changelog
+10
View File
@@ -0,0 +1,10 @@
liblz4 (1.7.2) unstable; urgency=low
* Changed : moved to versioning; package, cli and library have same version number
* Improved: Small decompression speed boost (+4%)
* Improved: Performance on ARMv6 and ARMv7
* Added : Debianization, by Evgeniy Polyakov
* Makefile: Generates object files (*.o) for faster (re)compilation on low power systems
* Fix : cli : crash on some invalid inputs
-- Yann Collet <Cyan4973@github.com> Sun, 28 Jun 2015 01:00:00 +0000
contrib/debian/compat
+1
View File
@@ -0,0 +1 @@
7
contrib/debian/control
+23
View File
@@ -0,0 +1,23 @@
Source: liblz4
Section: devel
Priority: optional
Maintainer: Evgeniy Polyakov <zbr@ioremap.net>
Build-Depends:
cmake (>= 2.6),
debhelper (>= 7.0.50~),
cdbs
Standards-Version: 3.8.0
Homepage: http://www.lz4.org/
Vcs-Git: git://github.com/lz4/lz4.git
Vcs-Browser: https://github.com/lz4/lz4
Package: liblz4
Architecture: any
Depends: ${shlibs:Depends}, ${misc:Depends}
Description: Extremely Fast Compression algorithm http://www.lz4.org
Package: liblz4-dev
Architecture: any
Depends: ${shlibs:Depends}, ${misc:Depends}
Description: Extremely Fast Compression algorithm http://www.lz4.org
Development files.
contrib/debian/copyright
+9
View File
@@ -0,0 +1,9 @@
Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: liblz4
Upstream-Contact: Yann Collet <Cyan4973@github.com>
Source: https://github.com/lz4/lz4
Files: *
Copyright: (C) 2011-2020 Yann Collet
License: GPL-2+
The full text of license: https://github.com/lz4/lz4/blob/dev/lib/LICENSE
contrib/debian/dirs
+1
View File
@@ -0,0 +1 @@
usr/bin
contrib/debian/docs
View File
contrib/debian/liblz4-dev.install
+2
View File
@@ -0,0 +1,2 @@
usr/include/lz4*
usr/lib/liblz4.so
contrib/debian/liblz4.install
+2
View File
@@ -0,0 +1,2 @@
usr/lib/liblz4.so.*
usr/bin/*
contrib/debian/rules
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/make -f
include /usr/share/cdbs/1/rules/debhelper.mk
include /usr/share/cdbs/1/class/cmake.mk
DEB_CMAKE_EXTRA_FLAGS := -DCMAKE_BUILD_TYPE=RelWithDebInfo ../../build/cmake
contrib/djgpp/LICENSE
+24
View File
@@ -0,0 +1,24 @@
Copyright (c) 2014, lpsantil
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
contrib/djgpp/Makefile
+130
View File
@@ -0,0 +1,130 @@
# Copyright (c) 2015, Louis P. Santillan <lpsantil@gmail.com>
# All rights reserved.
# See LICENSE for licensing details.
DESTDIR ?= /opt/local
# Pulled the code below from lib/Makefile. Might be nicer to derive this somehow without sed
# Version numbers
VERSION ?= 129
RELEASE ?= r$(VERSION)
LIBVER_MAJOR=$(shell sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lib/lz4.h)
LIBVER_MINOR=$(shell sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lib/lz4.h)
LIBVER_PATCH=$(shell sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lib/lz4.h)
LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
######################################################################
CROSS ?= i586-pc-msdosdjgpp
CC = $(CROSS)-gcc
AR = $(CROSS)-ar
LD = $(CROSS)-gcc
CFLAGS ?= -O3 -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\"
LDFLAGS ?= -s
SRC = programs/bench.c programs/lz4io.c programs/lz4cli.c
OBJ = $(SRC:.c=.o)
SDEPS = $(SRC:.c=.d)
IDIR = lib
EDIR = .
EXE = lz4.exe
LNK = lz4
LDIR = lib
LSRC = lib/lz4.c lib/lz4hc.c lib/lz4frame.c lib/xxhash.c
INC = $(LSRC:.c=.h)
LOBJ = $(LSRC:.c=.o)
LSDEPS = $(LSRC:.c=.d)
LIB = $(LDIR)/lib$(LNK).a
# Since LDFLAGS defaults to "-s", probably better to override unless
# you have a default you would like to maintain
ifeq ($(WITH_DEBUG), 1)
CFLAGS += -g
LDFLAGS += -g
endif
# Since LDFLAGS defaults to "-s", probably better to override unless
# you have a default you would like to maintain
ifeq ($(WITH_PROFILING), 1)
CFLAGS += -pg
LDFLAGS += -pg
endif
%.o: %.c $(INC) Makefile
$(CC) $(CFLAGS) -MMD -MP -I$(IDIR) -c $< -o $@
%.exe: %.o $(LIB) Makefile
$(LD) $< -L$(LDIR) -l$(LNK) $(LDFLAGS) $(LIBDEP) -o $@
######################################################################
######################## DO NOT MODIFY BELOW #########################
######################################################################
.PHONY: all install uninstall showconfig gstat gpush
all: $(LIB) $(EXE)
$(LIB): $(LOBJ)
$(AR) -rcs $@ $^
$(EXE): $(LOBJ) $(OBJ)
$(LD) $(LDFLAGS) $(LOBJ) $(OBJ) -o $(EDIR)/$@
clean:
rm -f $(OBJ) $(EXE) $(LOBJ) $(LIB) *.tmp $(SDEPS) $(LSDEPS) $(TSDEPS)
install: $(INC) $(LIB) $(EXE)
mkdir -p $(DESTDIR)/bin $(DESTDIR)/include $(DESTDIR)/lib
rm -f .footprint
echo $(DESTDIR)/bin/$(EXE) >> .footprint
cp -v $(EXE) $(DESTDIR)/bin/
@for T in $(LIB); \
do ( \
echo $(DESTDIR)/$$T >> .footprint; \
cp -v --parents $$T $(DESTDIR) \
); done
@for T in $(INC); \
do ( \
echo $(DESTDIR)/include/`basename -a $$T` >> .footprint; \
cp -v $$T $(DESTDIR)/include/ \
); done
uninstall: .footprint
@for T in $(shell cat .footprint); do rm -v $$T; done
-include $(SDEPS) $(LSDEPS)
showconfig:
@echo "PWD="$(PWD)
@echo "VERSION="$(VERSION)
@echo "RELEASE="$(RELEASE)
@echo "LIBVER_MAJOR="$(LIBVER_MAJOR)
@echo "LIBVER_MINOR="$(LIBVER_MINOR)
@echo "LIBVER_PATCH="$(LIBVER_PATCH)
@echo "LIBVER="$(LIBVER)
@echo "CROSS="$(CROSS)
@echo "CC="$(CC)
@echo "AR="$(AR)
@echo "LD="$(LD)
@echo "DESTDIR="$(DESTDIR)
@echo "CFLAGS="$(CFLAGS)
@echo "LDFLAGS="$(LDFLAGS)
@echo "SRC="$(SRC)
@echo "OBJ="$(OBJ)
@echo "IDIR="$(IDIR)
@echo "INC="$(INC)
@echo "EDIR="$(EDIR)
@echo "EXE="$(EXE)
@echo "LDIR="$(LDIR)
@echo "LSRC="$(LSRC)
@echo "LOBJ="$(LOBJ)
@echo "LNK="$(LNK)
@echo "LIB="$(LIB)
@echo "SDEPS="$(SDEPS)
@echo "LSDEPS="$(LSDEPS)
gstat:
git status
gpush:
git commit
git push
contrib/djgpp/README.MD
+21
View File
@@ -0,0 +1,21 @@
# lz4 for DOS/djgpp
This file details on how to compile lz4.exe, and liblz4.a for use on DOS/djgpp using
Andrew Wu's build-djgpp cross compilers ([GH][0], [Binaries][1]) on OSX, Linux.
## Setup
* Download a djgpp tarball [binaries][1] for your platform.
* Extract and install it (`tar jxvf djgpp-linux64-gcc492.tar.bz2`). Note the path. We'll assume `/home/user/djgpp`.
* Add the `bin` folder to your `PATH`. In bash, do `export PATH=/home/user/djgpp/bin:$PATH`.
* The `Makefile` in `contrib/djgpp/` sets up `CC`, `AR`, `LD` for you. So, `CC=i586-pc-msdosdjgpp-gcc`, `AR=i586-pc-msdosdjgpp-ar`, `LD=i586-pc-msdosdjgpp-gcc`.
## Building LZ4 for DOS
In the base dir of lz4 and with `contrib/djgpp/Makefile`, try:
Try:
* `make -f contrib/djgpp/Makefile`
* `make -f contrib/djgpp/Makefile liblz4.a`
* `make -f contrib/djgpp/Makefile lz4.exe`
* `make -f contrib/djgpp/Makefile DESTDIR=/home/user/dos install`, however it doesn't make much sense on a \*nix.
* You can also do `make -f contrib/djgpp/Makefile uninstall`
[0]: https://github.com/andrewwutw/build-djgpp
[1]: https://github.com/andrewwutw/build-djgpp/releases
contrib/gen_manual/.gitignore
+2
View File
@@ -0,0 +1,2 @@
# build artefact
gen_manual
contrib/gen_manual/Makefile
+76
View File
@@ -0,0 +1,76 @@
# ################################################################
# Copyright (C) Przemyslaw Skibinski 2016-present
# All rights reserved.
#
# BSD license
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# You can contact the author at :
# - LZ4 source repository : https://github.com/Cyan4973/lz4
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# ################################################################
CXXFLAGS ?= -O3
CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment
CXXFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
LZ4API = ../../lib/lz4.h
LZ4MANUAL = ../../doc/lz4_manual.html
LZ4FAPI = ../../lib/lz4frame.h
LZ4FMANUAL = ../../doc/lz4frame_manual.html
LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)`
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
LZ4VER := $(shell echo $(LIBVER_SCRIPT))
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
else
EXT =
endif
.PHONY: default
default: gen_manual
gen_manual: gen_manual.cpp
$(CXX) $(FLAGS) $^ -o $@$(EXT)
$(LZ4MANUAL) : gen_manual $(LZ4API)
echo "Update lz4 manual in /doc"
./gen_manual $(LZ4VER) $(LZ4API) $@
$(LZ4FMANUAL) : gen_manual $(LZ4FAPI)
echo "Update lz4frame manual in /doc"
./gen_manual $(LZ4VER) $(LZ4FAPI) $@
.PHONY: manuals
manuals: gen_manual $(LZ4MANUAL) $(LZ4FMANUAL)
.PHONY: clean
clean:
@$(RM) gen_manual$(EXT)
@echo Cleaning completed
contrib/gen_manual/README.md
+31
View File
@@ -0,0 +1,31 @@
gen_manual - a program for automatic generation of manual from source code
==========================================================================
#### Introduction
This simple C++ program generates a single-page HTML manual from `lz4.h`.
The format of recognized comment blocks is following:
- comments of type `/*!` mean: this is a function declaration; switch comments with declarations
- comments of type `/**` and `/*-` mean: this is a comment; use a `<H2>` header for the first line
- comments of type `/*=` and `/**=` mean: use a `<H3>` header and show also all functions until first empty line
- comments of type `/*X` where `X` is different from above-mentioned are ignored
Moreover:
- `LZ4LIB_API` is removed to improve readability
- `typedef` are detected and included even if uncommented
- comments of type `/**<` and `/*!<` are detected and only function declaration is highlighted (bold)
#### Usage
The program requires 3 parameters:
```
gen_manual [lz4_version] [input_file] [output_html]
```
To compile program and generate lz4 manual we have used:
```
make
./gen_manual.exe 1.7.3 ../../lib/lz4.h lz4_manual.html
```
contrib/gen_manual/gen-lz4-manual.sh
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
LIBVER_MAJOR_SCRIPT=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/lz4.h`
LIBVER_MINOR_SCRIPT=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/lz4.h`
LIBVER_PATCH_SCRIPT=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/lz4.h`
LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT
echo LZ4_VERSION=$LIBVER_SCRIPT
./gen_manual "lz4 $LIBVER_SCRIPT" ../../lib/lz4.h ./lz4_manual.html
./gen_manual "lz4frame $LIBVER_SCRIPT" ../../lib/lz4frame.h ./lz4frame_manual.html
contrib/gen_manual/gen_manual.cpp
+248
View File
@@ -0,0 +1,248 @@
/*
Copyright (c) 2016-present, Przemyslaw Skibinski
All rights reserved.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 homepage : http://www.lz4.org
- LZ4 source repository : https://github.com/lz4/lz4
*/
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
using namespace std;
/* trim string at the beginning and at the end */
void trim(string& s, string characters)
{
size_t p = s.find_first_not_of(characters);
s.erase(0, p);
p = s.find_last_not_of(characters);
if (string::npos != p)
s.erase(p+1);
}
/* trim C++ style comments */
void trim_comments(string &s)
{
size_t spos, epos;
spos = s.find("/*");
epos = s.find("*/");
s = s.substr(spos+3, epos-(spos+3));
}
/* get lines until a given terminator */
vector<string> get_lines(vector<string>& input, int& linenum, string terminator)
{
vector<string> out;
string line;
while ((size_t)linenum < input.size()) {
line = input[linenum];
if (terminator.empty() && line.empty()) { linenum--; break; }
size_t const epos = line.find(terminator);
if (!terminator.empty() && epos!=string::npos) {
out.push_back(line);
break;
}
out.push_back(line);
linenum++;
}
return out;
}
/* print line with LZ4LIB_API removed and C++ comments not bold */
void print_line(stringstream &sout, string line)
{
size_t spos, epos;
if (line.substr(0,11) == "LZ4LIB_API ") line = line.substr(11);
if (line.substr(0,12) == "LZ4FLIB_API ") line = line.substr(12);
spos = line.find("/*");
epos = line.find("*/");
if (spos!=string::npos && epos!=string::npos) {
sout << line.substr(0, spos);
sout << "</b>" << line.substr(spos) << "<b>" << '\n';
} else {
sout << line << '\n';
}
}
int main(int argc, char *argv[]) {
char exclam;
int linenum, chapter = 1;
vector<string> input, lines, comments, chapters;
string line, version;
size_t spos, l;
stringstream sout;
ifstream istream;
ofstream ostream;
if (argc < 4) {
cout << "usage: " << argv[0] << " [lz4_version] [input_file] [output_html]" << endl;
return 1;
}
version = string(argv[1]) + " Manual";
istream.open(argv[2], ifstream::in);
if (!istream.is_open()) {
cout << "Error opening file " << argv[2] << endl;
return 1;
}
ostream.open(argv[3], ifstream::out);
if (!ostream.is_open()) {
cout << "Error opening file " << argv[3] << endl;
return 1;
}
while (getline(istream, line)) {
input.push_back(line);
}
for (linenum=0; (size_t)linenum < input.size(); linenum++) {
line = input[linenum];
/* typedefs are detected and included even if uncommented */
if (line.substr(0,7) == "typedef" && line.find("{")!=string::npos) {
lines = get_lines(input, linenum, "}");
sout << "<pre><b>";
for (l=0; l<lines.size(); l++) {
print_line(sout, lines[l]);
}
sout << "</b></pre><BR>" << endl;
continue;
}
/* comments of type / * * < and / * ! < are detected, and only function declaration is highlighted (bold) */
if ((line.find("/**<")!=string::npos || line.find("/*!<")!=string::npos)
&& line.find("*/")!=string::npos) {
sout << "<pre><b>";
print_line(sout, line);
sout << "</b></pre><BR>" << endl;
continue;
}
spos = line.find("/**=");
if (spos==string::npos) {
spos = line.find("/*!");
if (spos==string::npos)
spos = line.find("/**");
if (spos==string::npos)
spos = line.find("/*-");
if (spos==string::npos)
spos = line.find("/*=");
if (spos==string::npos)
continue;
exclam = line[spos+2];
}
else exclam = '=';
comments = get_lines(input, linenum, "*/");
if (!comments.empty()) comments[0] = line.substr(spos+3);
if (!comments.empty())
comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/"));
for (l=0; l<comments.size(); l++) {
if (comments[l].compare(0, 2, " *") == 0)
comments[l] = comments[l].substr(2);
else if (comments[l].compare(0, 3, " *") == 0)
comments[l] = comments[l].substr(3);
trim(comments[l], "*-=");
}
while (!comments.empty() && comments[comments.size()-1].empty()) comments.pop_back(); // remove empty line at the end
while (!comments.empty() && comments[0].empty()) comments.erase(comments.begin()); // remove empty line at the start
/* comments of type / * ! mean: this is a function declaration; switch comments with declarations */
if (exclam == '!') {
if (!comments.empty()) comments.erase(comments.begin()); /* remove first line like "LZ4_XXX() :" */
linenum++;
lines = get_lines(input, linenum, "");
sout << "<pre><b>";
for (l=0; l<lines.size(); l++) {
print_line(sout, lines[l]);
}
sout << "</b><p>";
for (l=0; l<comments.size(); l++) {
print_line(sout, comments[l]);
}
sout << "</p></pre><BR>" << endl << endl;
} else if (exclam == '=') { /* comments of type / * = and / * * = mean: use a <H3> header and show also all functions until first empty line */
trim(comments[0], " ");
sout << "<h3>" << comments[0] << "</h3><pre>";
for (l=1; l<comments.size(); l++) {
print_line(sout, comments[l]);
}
sout << "</pre><b><pre>";
lines = get_lines(input, ++linenum, "");
for (l=0; l<lines.size(); l++) {
print_line(sout, lines[l]);
}
sout << "</pre></b><BR>" << endl;
} else { /* comments of type / * * and / * - mean: this is a comment; use a <H2> header for the first line */
if (comments.empty()) continue;
trim(comments[0], " ");
sout << "<a name=\"Chapter" << chapter << "\"></a><h2>" << comments[0] << "</h2><pre>";
chapters.push_back(comments[0]);
chapter++;
for (l=1; l<comments.size(); l++) {
print_line(sout, comments[l]);
}
if (comments.size() > 1)
sout << "<BR></pre>" << endl << endl;
else
sout << "</pre>" << endl << endl;
}
}
ostream << "<html>\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=ISO-8859-1\">\n<title>" << version << "</title>\n</head>\n<body>" << endl;
ostream << "<h1>" << version << "</h1>\n";
ostream << "<hr>\n<a name=\"Contents\"></a><h2>Contents</h2>\n<ol>\n";
for (size_t i=0; i<chapters.size(); i++)
ostream << "<li><a href=\"#Chapter" << i+1 << "\">" << chapters[i].c_str() << "</a></li>\n";
ostream << "</ol>\n<hr>\n";
ostream << sout.str();
ostream << "</html>" << endl << "</body>" << endl;
return 0;
}
contrib/meson/README.md
+34
View File
@@ -0,0 +1,34 @@
Meson build system for lz4
==========================
Meson is a build system designed to optimize programmer productivity.
It aims to do this by providing simple, out-of-the-box support for
modern software development tools and practices, such as unit tests,
coverage reports, Valgrind, CCache and the like.
This Meson build system is provided with no guarantee.
## How to build
`cd` to this meson directory (`contrib/meson`)
```sh
meson setup --buildtype=release -Ddefault_library=shared -Dbin_programs=true builddir
cd builddir
ninja # to build
ninja install # to install
```
You might want to install it in staging directory:
```sh
DESTDIR=./staging ninja install
```
To configure build options, use:
```sh
meson configure
```
See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html).
contrib/meson/meson.build
+21
View File
@@ -0,0 +1,21 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
# This is a dummy meson file.
# The intention is that it can be easily moved to the root of the project
# (together with meson_options.txt) and packaged for wrapdb.
project('lz4', ['c'],
license: ['BSD', 'GPLv2'],
default_options : ['c_std=c99',
'buildtype=release'],
version: 'DUMMY',
meson_version: '>=0.47.0')
subdir('meson')
contrib/meson/meson/GetLz4LibraryVersion.py
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python3
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
import re
def find_version_tuple(filepath):
version_file_data = None
with open(filepath) as fd:
version_file_data = fd.read()
patterns = r"""#\s*define\s+LZ4_VERSION_MAJOR\s+([0-9]+).*$
#\s*define\s+LZ4_VERSION_MINOR\s+([0-9]+).*$
#\s*define\s+LZ4_VERSION_RELEASE\s+([0-9]+).*$
"""
regex = re.compile(patterns, re.MULTILINE)
version_match = regex.search(version_file_data)
if version_match:
return version_match.groups()
raise Exception("Unable to find version string.")
def main():
import argparse
parser = argparse.ArgumentParser(description='Print lz4 version from lib/lz4.h')
parser.add_argument('file', help='path to lib/lz4.h')
args = parser.parse_args()
version_tuple = find_version_tuple(args.file)
print('.'.join(version_tuple))
if __name__ == '__main__':
main()
contrib/meson/meson/InstallSymlink.py
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env python3
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
# This file should be synced with https://github.com/lzutao/meson-symlink
import os
import pathlib # since Python 3.4
def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777):
if not install_dir.exists():
install_dir.mkdir(mode=dir_mode, parents=True, exist_ok=True)
if not install_dir.is_dir():
raise NotADirectoryError(install_dir)
new_dst = install_dir.joinpath(dst)
if new_dst.is_symlink() and os.readlink(new_dst) == src:
print('File exists: {!r} -> {!r}'.format(new_dst, src))
return
print('Installing symlink {!r} -> {!r}'.format(new_dst, src))
new_dst.symlink_to(src, target_is_directory=dst_is_dir)
def main():
import argparse
parser = argparse.ArgumentParser(description='Install a symlink',
usage='{0} [-h] [-d] [-m MODE] source dest install_dir\n\n'
'example:\n'
' {0} dash sh /bin'.format(pathlib.Path(__file__).name))
parser.add_argument('source', help='target to link')
parser.add_argument('dest', help='link name')
parser.add_argument('install_dir', help='installation directory')
parser.add_argument('-d', '--isdir',
action='store_true',
help='dest is a directory')
parser.add_argument('-m', '--mode',
help='directory mode on creating if not exist',
default='0o755')
args = parser.parse_args()
dir_mode = int(args.mode, 8)
meson_destdir = os.environ.get('MESON_INSTALL_DESTDIR_PREFIX', default='')
install_dir = pathlib.Path(meson_destdir, args.install_dir)
install_symlink(args.source, args.dest, install_dir, args.isdir, dir_mode)
if __name__ == '__main__':
main()
contrib/meson/meson/contrib/gen_manual/meson.build
+43
View File
@@ -0,0 +1,43 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
lz4_root_dir = '../../../../..'
add_languages('cpp')
cxx = meson.get_compiler('cpp')
gen_manual_includes = include_directories(join_paths(lz4_root_dir, 'contrib/gen_manual'))
gen_manual_cppflags = cxx.get_supported_arguments(['-Wextra', '-Wcast-qual',
'-Wcast-align', '-Wshadow', '-Wstrict-aliasing=1', '-Wswitch-enum',
'-Wno-comment'])
gen_manual = executable('gen_manual',
join_paths(lz4_root_dir, 'contrib/gen_manual/gen_manual.cpp'),
cpp_args: gen_manual_cppflags,
include_directories: gen_manual_includes,
native: true,
install: false)
# Update lz4 manual
lz4_manual_html = custom_target('lz4_manual.html',
output : 'lz4_manual.html',
command : [gen_manual,
lz4_version,
join_paths(meson.current_source_dir(), lz4_root_dir, 'lib/lz4.h'),
'@OUTPUT@'],
install : false)
# Update lz4frame manual
lz4_manual_html = custom_target('lz4frame_manual.html',
output : 'lz4frame_manual.html',
command : [gen_manual,
lz4_version,
join_paths(meson.current_source_dir(), lz4_root_dir, 'lib/lz4frame.h'),
'@OUTPUT@'],
install : false)
contrib/meson/meson/contrib/meson.build
+10
View File
@@ -0,0 +1,10 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
subdir('gen_manual')
contrib/meson/meson/examples/meson.build
+49
View File
@@ -0,0 +1,49 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
lz4_root_dir = '../../../..'
#examples_c_args = ['-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wstrict-prototypes']
printVersion = executable('printVersion',
join_paths(lz4_root_dir, 'examples/printVersion.c'),
dependencies: liblz4_dep,
install: false)
doubleBuffer = executable('doubleBuffer',
join_paths(lz4_root_dir, 'examples/blockStreaming_doubleBuffer.c'),
dependencies: liblz4_dep,
install: false)
dictionaryRandomAccess = executable('dictionaryRandomAccess',
join_paths(lz4_root_dir, 'examples/dictionaryRandomAccess.c'),
dependencies: liblz4_dep,
install: false)
ringBuffer = executable('ringBuffer',
join_paths(lz4_root_dir, 'examples/blockStreaming_ringBuffer.c'),
dependencies: liblz4_dep,
install: false)
ringBufferHC = executable('ringBufferHC',
join_paths(lz4_root_dir, 'examples/HCStreaming_ringBuffer.c'),
dependencies: liblz4_dep,
install: false)
lineCompress = executable('lineCompress',
join_paths(lz4_root_dir, 'examples/blockStreaming_lineByLine.c'),
dependencies: liblz4_dep,
install: false)
frameCompress = executable('frameCompress',
join_paths(lz4_root_dir, 'examples/frameCompress.c'),
dependencies: liblz4_dep,
install: false)
compressFunctions = executable('compressFunctions',
join_paths(lz4_root_dir, 'examples/compress_functions.c'),
dependencies: liblz4_dep,
install: false)
simpleBuffer = executable('simpleBuffer',
join_paths(lz4_root_dir, 'examples/simple_buffer.c'),
dependencies: liblz4_dep,
install: false)
contrib/meson/meson/lib/meson.build
+57
View File
@@ -0,0 +1,57 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
lz4_root_dir = '../../../..'
liblz4_includes = [include_directories(join_paths(lz4_root_dir, 'lib'))]
liblz4_sources = [join_paths(lz4_root_dir, 'lib/lz4.c'),
join_paths(lz4_root_dir, 'lib/lz4frame.c'),
join_paths(lz4_root_dir, 'lib/lz4hc.c'),
join_paths(lz4_root_dir, 'lib/xxhash.c')]
liblz4_c_args = []
liblz4_debug_cflags = []
if use_debug
liblz4_c_args += '-DLZ4_DEBUG=@0@'.format(debug_level)
if [compiler_gcc, compiler_clang].contains(cc_id)
liblz4_debug_cflags = ['-Wextra', '-Wcast-qual', '-Wcast-align', '-Wshadow',
'-Wswitch-enum', '-Wdeclaration-after-statement', '-Wstrict-prototypes',
'-Wundef', '-Wpointer-arith', '-Wstrict-aliasing=1']
endif
endif
liblz4_c_args += cc.get_supported_arguments(liblz4_debug_cflags)
if host_machine_os == os_windows and default_library != 'static'
liblz4_c_args += '-DLZ4_DLL_EXPORT=1'
endif
liblz4 = library('lz4',
liblz4_sources,
include_directories: liblz4_includes,
c_args: liblz4_c_args,
install: true,
version: lz4_libversion)
liblz4_dep = declare_dependency(link_with: liblz4,
include_directories: liblz4_includes)
pkgconfig.generate(liblz4,
name: 'lz4',
filebase: 'liblz4',
description: 'extremely fast lossless compression algorithm library',
version: lz4_libversion,
url: 'http://www.lz4.org/')
install_headers(join_paths(lz4_root_dir, 'lib/lz4.h'),
join_paths(lz4_root_dir, 'lib/lz4hc.h'),
join_paths(lz4_root_dir, 'lib/lz4frame.h'))
if default_library != 'shared'
install_headers(join_paths(lz4_root_dir, 'lib/lz4frame_static.h'))
endif
contrib/meson/meson/meson.build
+117
View File
@@ -0,0 +1,117 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
cc = meson.get_compiler('c')
pkgconfig = import('pkgconfig')
c_std = get_option('c_std')
default_library = get_option('default_library')
host_machine_os = host_machine.system()
os_windows = 'windows'
os_linux = 'linux'
os_darwin = 'darwin'
os_freebsd = 'freebsd'
os_sun = 'sunos'
cc_id = cc.get_id()
compiler_gcc = 'gcc'
compiler_clang = 'clang'
compiler_msvc = 'msvc'
lz4_version = meson.project_version()
lz4_h_file = join_paths(meson.current_source_dir(), '../../../lib/lz4.h')
GetLz4LibraryVersion_py = find_program('GetLz4LibraryVersion.py', native : true)
r = run_command(GetLz4LibraryVersion_py, lz4_h_file)
if r.returncode() == 0
lz4_version = r.stdout().strip()
message('Project version is now: @0@'.format(lz4_version))
else
error('Cannot find project version in @0@'.format(lz4_h_file))
endif
lz4_libversion = lz4_version
# =============================================================================
# Installation directories
# =============================================================================
lz4_prefix = get_option('prefix')
lz4_bindir = get_option('bindir')
lz4_datadir = get_option('datadir')
lz4_mandir = get_option('mandir')
lz4_docdir = join_paths(lz4_datadir, 'doc', meson.project_name())
# =============================================================================
# Project options
# =============================================================================
buildtype = get_option('buildtype')
# Built-in options
use_debug = get_option('debug')
# Custom options
debug_level = get_option('debug_level')
use_backtrace = get_option('backtrace')
bin_programs = get_option('bin_programs')
bin_contrib = get_option('bin_contrib')
bin_tests = get_option('bin_tests')
bin_examples = get_option('bin_examples')
#feature_multi_thread = get_option('multi_thread')
# =============================================================================
# Dependencies
# =============================================================================
#libm_dep = cc.find_library('m', required: bin_tests)
#thread_dep = dependency('threads', required: feature_multi_thread)
#use_multi_thread = thread_dep.found()
# =============================================================================
# Compiler flags
# =============================================================================
add_project_arguments(['-DXXH_NAMESPACE=LZ4_'], language: 'c')
if [compiler_gcc, compiler_clang].contains(cc_id)
common_warning_flags = []
# Should use Meson's own --werror build option
#common_warning_flags += ['-Werror']
if c_std == 'c89' or c_std == 'gnu89'
common_warning_flags += ['-pedantic', '-Wno-long-long', '-Wno-variadic-macros']
elif c_std == 'c99' or c_std == 'gnu99'
common_warning_flags += ['-pedantic']
endif
cc_compile_flags = cc.get_supported_arguments(common_warning_flags)
add_project_arguments(cc_compile_flags, language: 'c')
endif
# =============================================================================
# Subdirs
# =============================================================================
subdir('lib')
if bin_programs
subdir('programs')
endif
if bin_tests
subdir('tests')
endif
if bin_contrib
subdir('contrib')
endif
if bin_examples
subdir('examples')
endif
contrib/meson/meson/programs/meson.build
+52
View File
@@ -0,0 +1,52 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
lz4_root_dir = '../../../..'
lz4_includes = include_directories(join_paths(lz4_root_dir, 'programs'))
lz4_sources = [join_paths(lz4_root_dir, 'programs/bench.c'),
join_paths(lz4_root_dir, 'programs/datagen.c'),
join_paths(lz4_root_dir, 'programs/lz4cli.c'),
join_paths(lz4_root_dir, 'programs/lz4io.c')]
lz4_c_args = []
export_dynamic_on_windows = false
# explicit backtrace enable/disable for Linux & Darwin
if not use_backtrace
lz4_c_args += '-DBACKTRACE_ENABLE=0'
elif use_debug and host_machine_os == os_windows # MinGW target
lz4_c_args += '-DBACKTRACE_ENABLE=1'
export_dynamic_on_windows = true
endif
lz4_deps = [ liblz4_dep ]
lz4 = executable('lz4',
lz4_sources,
include_directories: lz4_includes,
c_args: lz4_c_args,
dependencies: lz4_deps,
export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0
install: true)
# =============================================================================
# Programs and manpages installing
# =============================================================================
install_man(join_paths(lz4_root_dir, 'programs/lz4.1'))
InstallSymlink_py = '../InstallSymlink.py'
lz4_man1_dir = join_paths(lz4_mandir, 'man1')
bin_EXT = host_machine_os == os_windows ? '.exe' : ''
man1_EXT = meson.version().version_compare('>=0.49.0') ? '.1' : '.1.gz'
foreach f : ['lz4c', 'lz4cat', 'unlz4']
meson.add_install_script(InstallSymlink_py, 'lz4' + bin_EXT, f + bin_EXT, lz4_bindir)
meson.add_install_script(InstallSymlink_py, 'lz4' + man1_EXT, f + man1_EXT, lz4_man1_dir)
endforeach
contrib/meson/meson/tests/meson.build
+93
View File
@@ -0,0 +1,93 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
lz4_root_dir = '../../../..'
programs_dir_inc = include_directories(join_paths(lz4_root_dir, 'programs'))
lib_dir_inc = include_directories(join_paths(lz4_root_dir, 'lib'))
# =============================================================================
# Test flags
# =============================================================================
TEST_FILES = join_paths(meson.current_source_dir(), lz4_root_dir, 'tests/COPYING')
FUZZER_TIME = '-T90s'
NB_LOOPS = '-i1'
# =============================================================================
# Executables
# =============================================================================
fullbench_sources = [join_paths(lz4_root_dir, 'tests/fullbench.c')]
fullbench = executable('fullbench',
fullbench_sources,
include_directories: programs_dir_inc,
dependencies: liblz4_dep,
install: false)
fuzzer_sources = [join_paths(lz4_root_dir, 'tests/fuzzer.c')]
fuzzer = executable('fuzzer',
fuzzer_sources,
c_args: ['-D_DEFAULT_SOURCE', '-D_BSD_SOURCE'], # since glibc 2.19
include_directories: programs_dir_inc,
dependencies: liblz4_dep,
install: false)
frametest_sources = [join_paths(lz4_root_dir, 'tests/frametest.c')]
frametest = executable('frametest',
frametest_sources,
include_directories: programs_dir_inc,
dependencies: liblz4_dep,
install: false)
roundTripTest_sources = [join_paths(lz4_root_dir, 'tests/roundTripTest.c')]
roundTripTest = executable('roundTripTest',
roundTripTest_sources,
dependencies: [ liblz4_dep ],
install: false)
datagen_sources = [join_paths(lz4_root_dir, 'tests/datagencli.c')]
datagen = executable('datagen',
datagen_sources,
objects: lz4.extract_objects(join_paths(lz4_root_dir, 'programs/datagen.c')),
include_directories: lz4_includes,
dependencies: [ liblz4_dep ],
install: false)
checkFrame_sources = [join_paths(lz4_root_dir, 'tests/checkFrame.c')]
checkFrame = executable('checkFrame',
checkFrame_sources,
include_directories: programs_dir_inc,
dependencies: [ liblz4_dep ],
install: false)
checkTag_sources = [join_paths(lz4_root_dir, 'tests/checkTag.c')]
checkTag = executable('checkTag',
checkTag_sources,
include_directories: lib_dir_inc,
install: false)
# =============================================================================
# Tests (Use "meson test --list" to list all tests)
# =============================================================================
# XXX: (Need TEST) These timeouts (in seconds) when running on a HDD should be
# at least six times bigger than on a SSD
test('test-fullbench',
fullbench,
args: ['--no-prompt', NB_LOOPS, TEST_FILES],
timeout: 420) # Should enough when running on HDD
test('test-fuzzer',
fuzzer,
args: [FUZZER_TIME],
timeout: 100)
test('test-frametest',
frametest,
args: [FUZZER_TIME],
timeout: 100)
contrib/meson/meson_options.txt
+24
View File
@@ -0,0 +1,24 @@
# #############################################################################
# Copyright (c) 2018-present lzutao <taolzu(at)gmail.com>
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# #############################################################################
# Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
option('debug_level', type: 'integer', min: 0, max: 7, value: 1,
description: 'Enable run-time debug. See lib/lz4hc.c')
option('backtrace', type: 'boolean', value: false,
description: 'Display a stack backtrace when execution generates a runtime exception')
option('bin_programs', type: 'boolean', value: false,
description: 'Enable programs build')
option('bin_tests', type: 'boolean', value: false,
description: 'Enable tests build')
option('bin_contrib', type: 'boolean', value: false,
description: 'Enable contrib build')
option('bin_examples', type: 'boolean', value: false,
description: 'Enable examples build')
contrib/snap/README.md
+29
View File
@@ -0,0 +1,29 @@
Snap Packaging
--------------
This directory contains the config required to generate a snap package
of lz4. Snaps are universal Linux packages that allow you to easily
build your application from any source and ship it to any Linux
distribution by publishing it to https://snapcraft.io/. A key attribute
of a snap package is that it is (ideally) confined such that it
executes within a controlled environmenti with all its dependencies
bundled with it and does not share dependencies with of from any other
package on the system (with a couple of minor exceptions).
The basic anatomy and workflow is:
* ensure snap.snapcraft.yaml is up-to-date e.g. with version info
* build the snap by installing the snapcraft package and running it
* push snap/* changes to the repo (excluding any crud generated by a build of course)
* register yourself as owner of lz4 name in snapstore
* publish new snap to the snap store
* install snap by doing 'snap install lz4' on any Linux distro
* all installed copies of lz4 will be automatically updated to your new version
For more information on Snaps see https://docs.snapcraft.io and https://forum.snapcraft.io/
contrib/snap/snapcraft.yaml
+31
View File
@@ -0,0 +1,31 @@
name: lz4
version: 1.8.4
summary: Extremely Fast Compression algorithm
description: >
LZ4 is lossless compression algorithm, providing compression
speed > 500 MB/s per core, scalable with multi-cores CPU. It features an
extremely fast decoder, with speed in multiple GB/s per core, typically
reaching RAM speed limits on multi-core systems.
.
Speed can be tuned dynamically, selecting an "acceleration" factor which
trades compression ratio for faster speed. On the other end, a high
compression derivative, LZ4_HC, is also provided, trading CPU time for
improved compression ratio. All versions feature the same decompression
speed.
.
LZ4 is also compatible with dictionary compression, and can ingest any
input file as dictionary, including those created by Zstandard Dictionary
Builder. (note: only the final 64KB are used).
.
LZ4 library is provided as open-source software using BSD 2-Clause license.
confinement: strict
grade: stable
apps:
lz4:
command: usr/local/bin/lz4
plugs: [home]
parts:
lz4:
source: ../
plugin: make
doc/lz4_Frame_format.md
+30 -19
View File
@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
### Version
1.6.1 (30/01/2018)
1.6.2 (12/08/2020)
Introduction
@@ -75,7 +75,7 @@ __Frame Descriptor__
3 to 15 Bytes, to be detailed in its own paragraph,
as it is the most important part of the spec.
The combined __Magic Number__ and __Frame Descriptor__ fields are sometimes
The combined _Magic_Number_ and _Frame_Descriptor_ fields are sometimes
called ___LZ4 Frame Header___. Its size varies between 7 and 19 bytes.
__Data Blocks__
@@ -85,14 +85,13 @@ That’s where compressed data is stored.
__EndMark__
The flow of blocks ends when the last data block has a size of “0”.
The size is expressed as a 32-bits value.
The flow of blocks ends when the last data block is followed by
the 32-bit value `0x00000000`.
__Content Checksum__
Content Checksum verify that the full content has been decoded correctly.
The content checksum is the result
of [xxh32() hash function](https://github.com/Cyan4973/xxHash)
_Content_Checksum_ verify that the full content has been decoded correctly.
The content checksum is the result of [xxHash-32 algorithm]
digesting the original (decoded) data as input, and a seed of zero.
Content checksum is only present when its associated flag
is set in the frame descriptor.
@@ -101,7 +100,7 @@ that all blocks were fully transmitted in the correct order and without error,
and also that the encoding/decoding process itself generated no distortion.
Its usage is recommended.
The combined __EndMark__ and __Content Checksum__ fields might sometimes be
The combined _EndMark_ and _Content_Checksum_ fields might sometimes be
referred to as ___LZ4 Frame Footer___. Its size varies between 4 and 8 bytes.
__Frame Concatenation__
@@ -261,38 +260,48 @@ __Block Size__
This field uses 4-bytes, format is little-endian.
The highest bit is “1” if data in the block is uncompressed.
If the highest bit is set (`1`), the block is uncompressed.
The highest bit is “0” if data in the block is compressed by LZ4.
If the highest bit is not set (`0`), the block is LZ4-compressed,
using the [LZ4 block format specification](https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md).
All other bits give the size, in bytes, of the following data block.
All other bits give the size, in bytes, of the data section.
The size does not include the block checksum if present.
Block Size shall never be larger than Block Maximum Size.
Such a thing could potentially happen for non-compressible sources.
In such a case, such data block shall be passed using uncompressed format.
_Block_Size_ shall never be larger than _Block_Maximum_Size_.
Such an outcome could potentially happen for non-compressible sources.
In such a case, such data block must be passed using uncompressed format.
A value of `0x00000000` is invalid, and signifies an _EndMark_ instead.
Note that this is different from a value of `0x80000000` (highest bit set),
which is an uncompressed block of size 0 (empty),
which is valid, and therefore doesn't end a frame.
Note that, if _Block_checksum_ is enabled,
even an empty block must be followed by a 32-bit block checksum.
__Data__
Where the actual data to decode stands.
It might be compressed or not, depending on previous field indications.
When compressed, the data must respect the [LZ4 block format specification](https://github.com/lz4/lz4/blob/master/doc/lz4_Block_format.md).
When compressed, the data must respect the [LZ4 block format specification](https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md).
Note that the block is not necessarily full.
Uncompressed size of data can be any size, up to "Block Maximum Size”,
Note that a block is not necessarily full.
Uncompressed size of data can be any size __up to__ _Block_Maximum_Size_,
so it may contain less data than the maximum block size.
__Block checksum__
Only present if the associated flag is set.
This is a 4-bytes checksum value, in little endian format,
calculated by using the xxHash-32 algorithm on the raw (undecoded) data block,
calculated by using the [xxHash-32 algorithm] on the __raw__ (undecoded) data block,
and a seed of zero.
The intention is to detect data corruption (storage or transmission errors)
before decoding.
Block checksum is cumulative with Content checksum.
_Block_checksum_ can be cumulative with _Content_checksum_.
[xxHash-32 algorithm]: https://github.com/Cyan4973/xxHash/blob/release/doc/xxhash_spec.md
Skippable Frames
@@ -389,6 +398,8 @@ and trigger an error if it does not fit within acceptable range.
Version changes
---------------
1.6.2 : clarifies specification of _EndMark_
1.6.1 : introduced terms "LZ4 Frame Header" and "LZ4 Frame Footer"
1.6.0 : restored Dictionary ID field in Frame header
doc/lz4_manual.html
+83 -46
View File
@@ -1,10 +1,10 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>1.9.2 Manual</title>
<title>1.9.3 Manual</title>
</head>
<body>
<h1>1.9.2 Manual</h1>
<h1>1.9.3 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
@@ -16,7 +16,7 @@
<li><a href="#Chapter6">Streaming Compression Functions</a></li>
<li><a href="#Chapter7">Streaming Decompression Functions</a></li>
<li><a href="#Chapter8">Experimental section</a></li>
<li><a href="#Chapter9">PRIVATE DEFINITIONS</a></li>
<li><a href="#Chapter9">Private Definitions</a></li>
<li><a href="#Chapter10">Obsolete Functions</a></li>
</ol>
<hr>
@@ -117,7 +117,8 @@
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
An acceleration value of "1" is the same as regular LZ4_compress_default()
Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c).
Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c).
Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c).
</p></pre><BR>
<pre><b>int LZ4_sizeofState(void);
@@ -140,31 +141,53 @@ int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int src
New value is necessarily <= input value.
@return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
or 0 if compression fails.
Note : from v1.8.2 to v1.9.1, this function had a bug (fixed un v1.9.2+):
the produced compressed content could, in specific circumstances,
require to be decompressed into a destination buffer larger
by at least 1 byte than the content to decompress.
If an application uses `LZ4_compress_destSize()`,
it's highly recommended to update liblz4 to v1.9.2 or better.
If this can't be done or ensured,
the receiving decompression function should provide
a dstCapacity which is > decompressedSize, by at least 1 byte.
See https://github.com/lz4/lz4/issues/859 for details
</p></pre><BR>
<pre><b>int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
</b><p> Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',
into destination buffer 'dst' of size 'dstCapacity'.
Up to 'targetOutputSize' bytes will be decoded.
The function stops decoding on reaching this objective,
which can boost performance when only the beginning of a block is required.
The function stops decoding on reaching this objective.
This can be useful to boost performance
whenever only the beginning of a block is required.
@return : the number of bytes decoded in `dst` (necessarily <= dstCapacity)
@return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize)
If source stream is detected malformed, function returns a negative result.
Note : @return can be < targetOutputSize, if compressed block contains less data.
Note 1 : @return can be < targetOutputSize, if compressed block contains less data.
Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity,
and expects targetOutputSize <= dstCapacity.
It effectively stops decoding on reaching targetOutputSize,
Note 2 : targetOutputSize must be <= dstCapacity
Note 3 : this function effectively stops decoding on reaching targetOutputSize,
so dstCapacity is kind of redundant.
This is because in a previous version of this function,
decoding operation would not "break" a sequence in the middle.
As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize,
This is because in older versions of this function,
decoding operation would still write complete sequences.
Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize,
it could write more bytes, though only up to dstCapacity.
Some "margin" used to be required for this operation to work properly.
This is no longer necessary.
The function nonetheless keeps its signature, in an effort to not break API.
Thankfully, this is no longer necessary.
The function nonetheless keeps the same signature, in an effort to preserve API compatibility.
Note 4 : If srcSize is the exact size of the block,
then targetOutputSize can be any value,
including larger than the block's decompressed size.
The function will, at most, generate block's decompressed size.
Note 5 : If srcSize is _larger_ than block's compressed size,
then targetOutputSize **MUST** be <= block's decompressed size.
Otherwise, *silent corruption will occur*.
</p></pre><BR>
@@ -423,39 +446,33 @@ int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
</b></pre><BR>
<pre><b>#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) </b>/**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */<b>
</b></pre><BR>
<a name="Chapter9"></a><h2>PRIVATE DEFINITIONS</h2><pre>
<a name="Chapter9"></a><h2>Private Definitions</h2><pre>
Do not use these definitions directly.
They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
Accessing members will expose code to API and/or ABI break in future versions of the library.
Accessing members will expose user code to API and/or ABI break in future versions of the library.
<BR></pre>
<pre><b>typedef struct {
const uint8_t* externalDict;
const LZ4_byte* externalDict;
size_t extDictSize;
const uint8_t* prefixEnd;
const LZ4_byte* prefixEnd;
size_t prefixSize;
} LZ4_streamDecode_t_internal;
</b></pre><BR>
<pre><b>typedef struct {
const unsigned char* externalDict;
const unsigned char* prefixEnd;
size_t extDictSize;
size_t prefixSize;
} LZ4_streamDecode_t_internal;
</b></pre><BR>
<pre><b>#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) </b>/*AS-400*/ )<b>
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
<pre><b>#define LZ4_STREAMSIZE 16416 </b>/* static size, for inter-version compatibility */<b>
#define LZ4_STREAMSIZE_VOIDP (LZ4_STREAMSIZE / sizeof(void*))
union LZ4_stream_u {
unsigned long long table[LZ4_STREAMSIZE_U64];
void* table[LZ4_STREAMSIZE_VOIDP];
LZ4_stream_t_internal internal_donotuse;
} ; </b>/* previously typedef'd to LZ4_stream_t */<b>
</b><p> information structure to track an LZ4 stream.
}; </b>/* previously typedef'd to LZ4_stream_t */<b>
</b><p> Do not use below internal definitions directly !
Declare or allocate an LZ4_stream_t instead.
LZ4_stream_t can also be created using LZ4_createStream(), which is recommended.
The structure definition can be convenient for static allocation
(on stack, or as part of larger structure).
Init this structure with LZ4_initStream() before first use.
note : only use this definition in association with static linking !
this definition is not API/ABI safe, and may change in a future version.
this definition is not API/ABI safe, and may change in future versions.
</p></pre><BR>
@@ -494,18 +511,17 @@ union LZ4_streamDecode_u {
<pre><b>#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
# define LZ4_DEPRECATED(message) </b>/* disable deprecation warnings */<b>
#else
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if defined (__cplusplus) && (__cplusplus >= 201402) </b>/* C++14 or greater */<b>
# define LZ4_DEPRECATED(message) [[deprecated(message)]]
# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (LZ4_GCC_VERSION >= 301)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45))
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# else
# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
# define LZ4_DEPRECATED(message)
# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler")
# define LZ4_DEPRECATED(message) </b>/* disabled */<b>
# endif
#endif </b>/* LZ4_DISABLE_DEPRECATE_WARNINGS */<b>
</b><p>
@@ -520,18 +536,39 @@ union LZ4_streamDecode_u {
</p></pre><BR>
<pre><b></b><p> These functions used to be faster than LZ4_decompress_safe(),
but it has changed, and they are now slower than LZ4_decompress_safe().
<pre><b>LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize);
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
</b><p></p></pre><BR>
<pre><b>LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
</b><p></p></pre><BR>
<pre><b>LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
</b><p></p></pre><BR>
<pre><b>LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead")
int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead")
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead")
int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
</b><p> These functions used to be faster than LZ4_decompress_safe(),
but this is no longer the case. They are now slower.
This is because LZ4_decompress_fast() doesn't know the input size,
and therefore must progress more cautiously in the input buffer to not read beyond the end of block.
and therefore must progress more cautiously into the input buffer to not read beyond the end of block.
On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
The last remaining LZ4_decompress_fast() specificity is that
it can decompress a block without knowing its compressed size.
Such functionality could be achieved in a more secure manner,
by also providing the maximum size of input buffer,
but it would require new prototypes, and adaptation of the implementation to this new use case.
Such functionality can be achieved in a more secure manner
by employing LZ4_decompress_safe_partial().
Parameters:
originalSize : is the uncompressed size to regenerate.
doc/lz4frame_manual.html
+7 -5
View File
@@ -1,10 +1,10 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>1.9.2 Manual</title>
<title>1.9.3 Manual</title>
</head>
<body>
<h1>1.9.2 Manual</h1>
<h1>1.9.3 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
@@ -167,7 +167,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
@return is always the same for a srcSize and prefsPtr.
prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
tech details :
@return includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.
@return if automatic flushing is not enabled, includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.
It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd().
@return doesn't include frame header, as it was already generated by LZ4F_compressBegin().
@@ -299,8 +299,10 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
const LZ4F_decompressOptions_t* dOptPtr);
</b><p> Call this function repetitively to regenerate compressed data from `srcBuffer`.
The function will read up to *srcSizePtr bytes from srcBuffer,
</b><p> Call this function repetitively to regenerate data compressed in `srcBuffer`.
The function requires a valid dctx state.
It will read up to *srcSizePtr bytes from srcBuffer,
and decompress data into dstBuffer, of capacity *dstSizePtr.
The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
examples/.gitignore
+10
View File
@@ -0,0 +1,10 @@
/Makefile.lz4*
/printVersion
/doubleBuffer
/dictionaryRandomAccess
/ringBuffer
/ringBufferHC
/lineCompress
/frameCompress
/simpleBuffer
/*.exe
examples/COPYING
+339
View File
@@ -0,0 +1,339 @@
GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
License is intended to guarantee your freedom to share and change free
software--to make sure the software is free for all its users. This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it. (Some other Free Software Foundation software is covered by
the GNU Lesser General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
this service if you wish), that you receive source code or can get it
if you want it, that you can change the software or use pieces of it
in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid
anyone to deny you these rights or to ask you to surrender the rights.
These restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must give the recipients all the rights that
you have. You must make sure that they, too, receive or can get the
source code. And you must show them these terms so they know their
rights.
We protect your rights with two steps: (1) copyright the software, and
(2) offer you this license which gives you legal permission to copy,
distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain
that everyone understands that there is no warranty for this free
software. If the software is modified by someone else and passed on, we
want its recipients to know that what they have is not the original, so
that any problems introduced by others will not reflect on the original
authors' reputations.
Finally, any free program is threatened constantly by software
patents. We wish to avoid the danger that redistributors of a free
program will individually obtain patent licenses, in effect making the
program proprietary. To prevent this, we have made it clear that any
patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
modification follow.
GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language. (Hereinafter, translation is included without limitation in
the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1. You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
2. You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
b) You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
c) If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.
In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
a) Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
c) Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable. However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.
If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
5. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
6. Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
7. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all. For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
9. The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
10. If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Also add information on how to contact you by electronic and paper mail.
If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:
Gnomovision version 69, Copyright (C) year name of author
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, the commands you use may
be called something other than `show w' and `show c'; they could even be
mouse-clicks or menu items--whatever suits your program.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the program, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
`Gnomovision' (which makes passes at compilers) written by James Hacker.
<signature of Ty Coon>, 1 April 1989
Ty Coon, President of Vice
This General Public License does not permit incorporating your program into
proprietary programs. If your program is a subroutine library, you may
consider it more useful to permit linking proprietary applications with the
library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License.
examples/HCStreaming_ringBuffer.c
+232
View File
@@ -0,0 +1,232 @@
// LZ4 HC streaming API example : ring buffer
// Based on a previous example by Takayuki Matsuoka
/**************************************
* Compiler Options
**************************************/
#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#ifdef __GNUC__
# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
#endif
/**************************************
* Includes
**************************************/
#include "lz4hc.h"
#include "lz4.h"
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
enum {
MESSAGE_MAX_BYTES = 1024,
RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES,
DEC_BUFFER_BYTES = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger to test unsynchronized ring buffers
};
size_t write_int32(FILE* fp, int32_t i) {
return fwrite(&i, sizeof(i), 1, fp);
}
size_t write_bin(FILE* fp, const void* array, int arrayBytes) {
assert(arrayBytes >= 0);
return fwrite(array, 1, (size_t)arrayBytes, fp);
}
size_t read_int32(FILE* fp, int32_t* i) {
return fread(i, sizeof(*i), 1, fp);
}
size_t read_bin(FILE* fp, void* array, int arrayBytes) {
assert(arrayBytes >= 0);
return fread(array, 1, (size_t)arrayBytes, fp);
}
void test_compress(FILE* outFp, FILE* inpFp)
{
LZ4_streamHC_t lz4Stream_body = { 0 };
LZ4_streamHC_t* lz4Stream = &lz4Stream_body;
static char inpBuf[RING_BUFFER_BYTES];
int inpOffset = 0;
for(;;) {
// Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer.
char* const inpPtr = &inpBuf[inpOffset];
const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1;
const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
if (0 == inpBytes) break;
#define CMPBUFSIZE (LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES))
{ char cmpBuf[CMPBUFSIZE];
const int cmpBytes = LZ4_compress_HC_continue(lz4Stream, inpPtr, cmpBuf, inpBytes, CMPBUFSIZE);
if(cmpBytes <= 0) break;
write_int32(outFp, cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
inpOffset += inpBytes;
// Wraparound the ringbuffer offset
if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES)
inpOffset = 0;
}
}
write_int32(outFp, 0);
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
static char decBuf[DEC_BUFFER_BYTES];
int decOffset = 0;
LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
for(;;) {
int cmpBytes = 0;
char cmpBuf[CMPBUFSIZE];
{ const size_t r0 = read_int32(inpFp, &cmpBytes);
size_t r1;
if(r0 != 1 || cmpBytes <= 0)
break;
r1 = read_bin(inpFp, cmpBuf, cmpBytes);
if(r1 != (size_t) cmpBytes)
break;
}
{ char* const decPtr = &decBuf[decOffset];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
if(decBytes <= 0)
break;
decOffset += decBytes;
write_bin(outFp, decPtr, decBytes);
// Wraparound the ringbuffer offset
if(decOffset >= DEC_BUFFER_BYTES - MESSAGE_MAX_BYTES)
decOffset = 0;
}
}
}
// Compare 2 files content
// return 0 if identical
// return ByteNb>0 if different
size_t compare(FILE* f0, FILE* f1)
{
size_t result = 1;
for (;;) {
char b0[65536];
char b1[65536];
const size_t r0 = fread(b0, 1, sizeof(b0), f0);
const size_t r1 = fread(b1, 1, sizeof(b1), f1);
if ((r0==0) && (r1==0)) return 0; // success
if (r0 != r1) {
size_t smallest = r0;
if (r1<r0) smallest = r1;
result += smallest;
break;
}
if (memcmp(b0, b1, r0)) {
unsigned errorPos = 0;
while ((errorPos < r0) && (b0[errorPos]==b1[errorPos])) errorPos++;
result += errorPos;
break;
}
result += sizeof(b0);
}
return result;
}
int main(int argc, const char** argv)
{
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
unsigned fileID = 1;
unsigned pause = 0;
if(argc < 2) {
printf("Please specify input filename\n");
return 0;
}
if (!strcmp(argv[1], "-p")) { pause = 1; fileID = 2; }
snprintf(inpFilename, 256, "%s", argv[fileID]);
snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[fileID], 9);
snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[fileID], 9);
printf("input = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("decoded = [%s]\n", decFilename);
// compress
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
test_compress(outFp, inpFp);
fclose(outFp);
fclose(inpFp);
}
// decompress
{ FILE* const inpFp = fopen(lz4Filename, "rb");
FILE* const outFp = fopen(decFilename, "wb");
test_decompress(outFp, inpFp);
fclose(outFp);
fclose(inpFp);
}
// verify
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const decFp = fopen(decFilename, "rb");
const size_t cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("Verify : OK\n");
} else {
printf("Verify : NG : error at pos %u\n", (unsigned)cmp-1);
}
fclose(decFp);
fclose(inpFp);
}
if (pause) {
int unused;
printf("Press enter to continue ...\n");
unused = getchar(); (void)unused; /* silence static analyzer */
}
return 0;
}
examples/Makefile
+103
View File
@@ -0,0 +1,103 @@
# ##########################################################################
# LZ4 examples - Makefile
# Copyright (C) Yann Collet 2011-2014
#
# GPL v2 License
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# You can contact the author at :
# - LZ4 source repository : https://github.com/lz4/lz4
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# ##########################################################################
# This makefile compile and test
# example programs, using (mostly) LZ4 streaming library,
# kindly provided by Takayuki Matsuoka
# ##########################################################################
CPPFLAGS += -I../lib
CFLAGS ?= -O3
CFLAGS += -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
FLAGS := $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
TESTFILE = Makefile
LZ4DIR := ../lib
LZ4 = ../programs/lz4
include ../Makefile.inc
default: all
all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \
lineCompress frameCompress simpleBuffer
$(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h
$(MAKE) -C $(LZ4DIR) liblz4.a
printVersion: printVersion.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
doubleBuffer: blockStreaming_doubleBuffer.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
dictionaryRandomAccess: dictionaryRandomAccess.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
ringBuffer : blockStreaming_ringBuffer.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
ringBufferHC: HCStreaming_ringBuffer.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
lineCompress: blockStreaming_lineByLine.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
frameCompress: frameCompress.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
compressFunctions: compress_functions.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT) -lrt
simpleBuffer: simple_buffer.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
$(LZ4) :
$(MAKE) -C ../programs lz4
test : all $(LZ4)
@echo "\n=== Print Version ==="
./printVersion$(EXT)
@echo "\n=== Simple compression example ==="
./simpleBuffer$(EXT)
@echo "\n=== Double-buffer ==="
./doubleBuffer$(EXT) $(TESTFILE)
@echo "\n=== Ring Buffer ==="
./ringBuffer$(EXT) $(TESTFILE)
@echo "\n=== Ring Buffer + LZ4 HC ==="
./ringBufferHC$(EXT) $(TESTFILE)
@echo "\n=== Compress line by line ==="
./lineCompress$(EXT) $(TESTFILE)
@echo "\n=== Dictionary Random Access ==="
./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400
@echo "\n=== Frame compression ==="
./frameCompress$(EXT) $(TESTFILE)
$(LZ4) -vt $(TESTFILE).lz4
clean:
@rm -f core *.o *.dec *-0 *-9 *-8192 *.lz4s *.lz4 \
printVersion$(EXT) doubleBuffer$(EXT) dictionaryRandomAccess$(EXT) \
ringBuffer$(EXT) ringBufferHC$(EXT) lineCompress$(EXT) frameCompress$(EXT) \
compressFunctions$(EXT) simpleBuffer$(EXT)
@echo Cleaning completed
examples/README.md
+11
View File
@@ -0,0 +1,11 @@
# LZ4 examples
All examples are GPL-v2 licensed.
## Documents
- [Streaming API Basics](streaming_api_basics.md)
- Examples
- [Double Buffer](blockStreaming_doubleBuffer.md)
- [Line by Line Text Compression](blockStreaming_lineByLine.md)
- [Dictionary Random Access](dictionaryRandomAccess.md)
examples/blockStreaming_doubleBuffer.c
+202
View File
@@ -0,0 +1,202 @@
// LZ4 streaming API example : double buffer
// Copyright : Takayuki Matsuoka
#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif
#include "lz4.h"
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
enum {
BLOCK_BYTES = 1024 * 8,
// BLOCK_BYTES = 1024 * 64,
};
size_t write_int(FILE* fp, int i) {
return fwrite(&i, sizeof(i), 1, fp);
}
size_t write_bin(FILE* fp, const void* array, size_t arrayBytes) {
return fwrite(array, 1, arrayBytes, fp);
}
size_t read_int(FILE* fp, int* i) {
return fread(i, sizeof(*i), 1, fp);
}
size_t read_bin(FILE* fp, void* array, size_t arrayBytes) {
return fread(array, 1, arrayBytes, fp);
}
void test_compress(FILE* outFp, FILE* inpFp)
{
LZ4_stream_t lz4Stream_body;
LZ4_stream_t* lz4Stream = &lz4Stream_body;
char inpBuf[2][BLOCK_BYTES];
int inpBufIndex = 0;
LZ4_initStream(lz4Stream, sizeof (*lz4Stream));
for(;;) {
char* const inpPtr = inpBuf[inpBufIndex];
const int inpBytes = (int) read_bin(inpFp, inpPtr, BLOCK_BYTES);
if(0 == inpBytes) {
break;
}
{
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
const int cmpBytes = LZ4_compress_fast_continue(
lz4Stream, inpPtr, cmpBuf, inpBytes, sizeof(cmpBuf), 1);
if(cmpBytes <= 0) {
break;
}
write_int(outFp, cmpBytes);
write_bin(outFp, cmpBuf, (size_t) cmpBytes);
}
inpBufIndex = (inpBufIndex + 1) % 2;
}
write_int(outFp, 0);
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
LZ4_streamDecode_t lz4StreamDecode_body;
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
char decBuf[2][BLOCK_BYTES];
int decBufIndex = 0;
LZ4_setStreamDecode(lz4StreamDecode, NULL, 0);
for(;;) {
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
int cmpBytes = 0;
{
const size_t readCount0 = read_int(inpFp, &cmpBytes);
if(readCount0 != 1 || cmpBytes <= 0) {
break;
}
const size_t readCount1 = read_bin(inpFp, cmpBuf, (size_t) cmpBytes);
if(readCount1 != (size_t) cmpBytes) {
break;
}
}
{
char* const decPtr = decBuf[decBufIndex];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
if(decBytes <= 0) {
break;
}
write_bin(outFp, decPtr, (size_t) decBytes);
}
decBufIndex = (decBufIndex + 1) % 2;
}
}
int compare(FILE* fp0, FILE* fp1)
{
int result = 0;
while(0 == result) {
char b0[65536];
char b1[65536];
const size_t r0 = read_bin(fp0, b0, sizeof(b0));
const size_t r1 = read_bin(fp1, b1, sizeof(b1));
result = (int) r0 - (int) r1;
if(0 == r0 || 0 == r1) {
break;
}
if(0 == result) {
result = memcmp(b0, b1, r0);
}
}
return result;
}
int main(int argc, char* argv[])
{
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
if(argc < 2) {
printf("Please specify input filename\n");
return 0;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], BLOCK_BYTES);
snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], BLOCK_BYTES);
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("dec = [%s]\n", decFilename);
// compress
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lz4Filename, "wb");
printf("compress : %s -> %s\n", inpFilename, lz4Filename);
test_compress(outFp, inpFp);
printf("compress : done\n");
fclose(outFp);
fclose(inpFp);
}
// decompress
{
FILE* inpFp = fopen(lz4Filename, "rb");
FILE* outFp = fopen(decFilename, "wb");
printf("decompress : %s -> %s\n", lz4Filename, decFilename);
test_decompress(outFp, inpFp);
printf("decompress : done\n");
fclose(outFp);
fclose(inpFp);
}
// verify
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
printf("verify : %s <-> %s\n", inpFilename, decFilename);
const int cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("verify : OK\n");
} else {
printf("verify : NG\n");
}
fclose(decFp);
fclose(inpFp);
}
return 0;
}
examples/blockStreaming_doubleBuffer.md
+100
View File
@@ -0,0 +1,100 @@
# LZ4 Streaming API Example : Double Buffer
by *Takayuki Matsuoka*
`blockStreaming_doubleBuffer.c` is LZ4 Streaming API example which implements double buffer (de)compression.
Please note :
- Firstly, read "LZ4 Streaming API Basics".
- This is relatively advanced application example.
- Output file is not compatible with lz4frame and platform dependent.
## What's the point of this example ?
- Handle huge file in small amount of memory
- Always better compression ratio than Block API
- Uniform block size
## How the compression works
First of all, allocate "Double Buffer" for input and LZ4 compressed data buffer for output.
Double buffer has two pages, "first" page (Page#1) and "second" page (Page#2).
```
Double Buffer
Page#1 Page#2
+---------+---------+
| Block#1 | |
+----+----+---------+
|
v
{Out#1}
Prefix Dependency
+---------+
| |
v |
+---------+----+----+
| Block#1 | Block#2 |
+---------+----+----+
|
v
{Out#2}
External Dictionary Mode
+---------+
| |
| v
+----+----+---------+
| Block#3 | Block#2 |
+----+----+---------+
|
v
{Out#3}
Prefix Dependency
+---------+
| |
v |
+---------+----+----+
| Block#3 | Block#4 |
+---------+----+----+
|
v
{Out#4}
```
Next, read first block to double buffer's first page. And compress it by `LZ4_compress_continue()`.
For the first time, LZ4 doesn't know any previous dependencies,
so it just compress the line without dependencies and generates compressed block {Out#1} to LZ4 compressed data buffer.
After that, write {Out#1} to the file.
Next, read second block to double buffer's second page. And compress it.
This time, LZ4 can use dependency to Block#1 to improve compression ratio.
This dependency is called "Prefix mode".
Next, read third block to double buffer's *first* page, and compress it.
Also this time, LZ4 can use dependency to Block#2.
This dependency is called "External Dictonaly mode".
Continue these procedure to the end of the file.
## How the decompression works
Decompression will do reverse order.
- Read first compressed block.
- Decompress it to the first page and write that page to the file.
- Read second compressed block.
- Decompress it to the second page and write that page to the file.
- Read third compressed block.
- Decompress it to the *first* page and write that page to the file.
Continue these procedure to the end of the compressed file.
examples/blockStreaming_lineByLine.c
+211
View File
@@ -0,0 +1,211 @@
// LZ4 streaming API example : line-by-line logfile compression
// by Takayuki Matsuoka
#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif
#include "lz4.h"
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
static size_t write_uint16(FILE* fp, uint16_t i)
{
return fwrite(&i, sizeof(i), 1, fp);
}
static size_t write_bin(FILE* fp, const void* array, int arrayBytes)
{
return fwrite(array, 1, arrayBytes, fp);
}
static size_t read_uint16(FILE* fp, uint16_t* i)
{
return fread(i, sizeof(*i), 1, fp);
}
static size_t read_bin(FILE* fp, void* array, int arrayBytes)
{
return fread(array, 1, arrayBytes, fp);
}
static void test_compress(
FILE* outFp,
FILE* inpFp,
size_t messageMaxBytes,
size_t ringBufferBytes)
{
LZ4_stream_t* const lz4Stream = LZ4_createStream();
const size_t cmpBufBytes = LZ4_COMPRESSBOUND(messageMaxBytes);
char* const cmpBuf = (char*) malloc(cmpBufBytes);
char* const inpBuf = (char*) malloc(ringBufferBytes);
int inpOffset = 0;
for ( ; ; )
{
char* const inpPtr = &inpBuf[inpOffset];
#if 0
// Read random length data to the ring buffer.
const int randomLength = (rand() % messageMaxBytes) + 1;
const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
if (0 == inpBytes) break;
#else
// Read line to the ring buffer.
int inpBytes = 0;
if (!fgets(inpPtr, (int) messageMaxBytes, inpFp))
break;
inpBytes = (int) strlen(inpPtr);
#endif
{
const int cmpBytes = LZ4_compress_fast_continue(
lz4Stream, inpPtr, cmpBuf, inpBytes, cmpBufBytes, 1);
if (cmpBytes <= 0) break;
write_uint16(outFp, (uint16_t) cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
// Add and wraparound the ringbuffer offset
inpOffset += inpBytes;
if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0;
}
}
write_uint16(outFp, 0);
free(inpBuf);
free(cmpBuf);
LZ4_freeStream(lz4Stream);
}
static void test_decompress(
FILE* outFp,
FILE* inpFp,
size_t messageMaxBytes,
size_t ringBufferBytes)
{
LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode();
char* const cmpBuf = (char*) malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
char* const decBuf = (char*) malloc(ringBufferBytes);
int decOffset = 0;
for ( ; ; )
{
uint16_t cmpBytes = 0;
if (read_uint16(inpFp, &cmpBytes) != 1) break;
if (cmpBytes == 0) break;
if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break;
{
char* const decPtr = &decBuf[decOffset];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes);
if (decBytes <= 0) break;
write_bin(outFp, decPtr, decBytes);
// Add and wraparound the ringbuffer offset
decOffset += decBytes;
if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0;
}
}
free(decBuf);
free(cmpBuf);
LZ4_freeStreamDecode(lz4StreamDecode);
}
static int compare(FILE* f0, FILE* f1)
{
int result = 0;
const size_t tempBufferBytes = 65536;
char* const b0 = (char*) malloc(tempBufferBytes);
char* const b1 = (char*) malloc(tempBufferBytes);
while(0 == result)
{
const size_t r0 = fread(b0, 1, tempBufferBytes, f0);
const size_t r1 = fread(b1, 1, tempBufferBytes, f1);
result = (int) r0 - (int) r1;
if (0 == r0 || 0 == r1) break;
if (0 == result) result = memcmp(b0, b1, r0);
}
free(b1);
free(b0);
return result;
}
int main(int argc, char* argv[])
{
enum {
MESSAGE_MAX_BYTES = 1024,
RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES,
};
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
if (argc < 2)
{
printf("Please specify input filename\n");
return 0;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lz4Filename, 256, "%s.lz4s", argv[1]);
snprintf(decFilename, 256, "%s.lz4s.dec", argv[1]);
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("dec = [%s]\n", decFilename);
// compress
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lz4Filename, "wb");
test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES);
fclose(outFp);
fclose(inpFp);
}
// decompress
{
FILE* inpFp = fopen(lz4Filename, "rb");
FILE* outFp = fopen(decFilename, "wb");
test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES);
fclose(outFp);
fclose(inpFp);
}
// verify
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
const int cmp = compare(inpFp, decFp);
if (0 == cmp)
printf("Verify : OK\n");
else
printf("Verify : NG\n");
fclose(decFp);
fclose(inpFp);
}
return 0;
}
examples/blockStreaming_lineByLine.md
+122
View File
@@ -0,0 +1,122 @@
# LZ4 Streaming API Example : Line by Line Text Compression
by *Takayuki Matsuoka*
`blockStreaming_lineByLine.c` is LZ4 Straming API example which implements line by line incremental (de)compression.
Please note the following restrictions :
- Firstly, read "LZ4 Streaming API Basics".
- This is relatively advanced application example.
- Output file is not compatible with lz4frame and platform dependent.
## What's the point of this example ?
- Line by line incremental (de)compression.
- Handle huge file in small amount of memory
- Generally better compression ratio than Block API
- Non-uniform block size
## How the compression works
First of all, allocate "Ring Buffer" for input and LZ4 compressed data buffer for output.
```
(1)
Ring Buffer
+--------+
| Line#1 |
+---+----+
|
v
{Out#1}
(2)
Prefix Mode Dependency
+----+
| |
v |
+--------+-+------+
| Line#1 | Line#2 |
+--------+---+----+
|
v
{Out#2}
(3)
Prefix Prefix
+----+ +----+
| | | |
v | v |
+--------+-+------+-+------+
| Line#1 | Line#2 | Line#3 |
+--------+--------+---+----+
|
v
{Out#3}
(4)
External Dictionary Mode
+----+ +----+
| | | |
v | v |
------+--------+-+------+-+--------+
| .... | Line#X | Line#X+1 |
------+--------+--------+-----+----+
^ |
| v
| {Out#X+1}
|
Reset
(5)
Prefix
+-----+
| |
v |
------+--------+--------+----------+--+-------+
| .... | Line#X | Line#X+1 | Line#X+2 |
------+--------+--------+----------+-----+----+
^ |
| v
| {Out#X+2}
|
Reset
```
Next (see (1)), read first line to ringbuffer and compress it by `LZ4_compress_continue()`.
For the first time, LZ4 doesn't know any previous dependencies,
so it just compress the line without dependencies and generates compressed line {Out#1} to LZ4 compressed data buffer.
After that, write {Out#1} to the file and forward ringbuffer offset.
Do the same things to second line (see (2)).
But in this time, LZ4 can use dependency to Line#1 to improve compression ratio.
This dependency is called "Prefix mode".
Eventually, we'll reach end of ringbuffer at Line#X (see (4)).
This time, we should reset ringbuffer offset.
After resetting, at Line#X+1 pointer is not adjacent, but LZ4 still maintain its memory.
This is called "External Dictionary Mode".
In Line#X+2 (see (5)), finally LZ4 forget almost all memories but still remains Line#X+1.
This is the same situation as Line#2.
Continue these procedure to the end of text file.
## How the decompression works
Decompression will do reverse order.
- Read compressed line from the file to buffer.
- Decompress it to the ringbuffer.
- Output decompressed plain text line to the file.
- Forward ringbuffer offset. If offset exceedes end of the ringbuffer, reset it.
Continue these procedure to the end of the compressed file.
examples/blockStreaming_ringBuffer.c
+190
View File
@@ -0,0 +1,190 @@
/* LZ4 streaming API example : ring buffer
* Based on sample code from Takayuki Matsuoka */
/**************************************
* Compiler Options
**************************************/
#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif
/**************************************
* Includes
**************************************/
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "lz4.h"
enum {
MESSAGE_MAX_BYTES = 1024,
RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES,
DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES /* Intentionally larger, to test unsynchronized ring buffers */
};
size_t write_int32(FILE* fp, int32_t i) {
return fwrite(&i, sizeof(i), 1, fp);
}
size_t write_bin(FILE* fp, const void* array, int arrayBytes) {
return fwrite(array, 1, arrayBytes, fp);
}
size_t read_int32(FILE* fp, int32_t* i) {
return fread(i, sizeof(*i), 1, fp);
}
size_t read_bin(FILE* fp, void* array, int arrayBytes) {
return fread(array, 1, arrayBytes, fp);
}
void test_compress(FILE* outFp, FILE* inpFp)
{
LZ4_stream_t lz4Stream_body = { { 0 } };
LZ4_stream_t* lz4Stream = &lz4Stream_body;
static char inpBuf[RING_BUFFER_BYTES];
int inpOffset = 0;
for(;;) {
// Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer.
char* const inpPtr = &inpBuf[inpOffset];
const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1;
const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
if (0 == inpBytes) break;
{
#define CMPBUFSIZE (LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES))
char cmpBuf[CMPBUFSIZE];
const int cmpBytes = LZ4_compress_fast_continue(lz4Stream, inpPtr, cmpBuf, inpBytes, CMPBUFSIZE, 0);
if(cmpBytes <= 0) break;
write_int32(outFp, cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
inpOffset += inpBytes;
// Wraparound the ringbuffer offset
if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) inpOffset = 0;
}
}
write_int32(outFp, 0);
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
static char decBuf[DECODE_RING_BUFFER];
int decOffset = 0;
LZ4_streamDecode_t lz4StreamDecode_body = { { 0 } };
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
for(;;) {
int cmpBytes = 0;
char cmpBuf[CMPBUFSIZE];
{ const size_t r0 = read_int32(inpFp, &cmpBytes);
if(r0 != 1 || cmpBytes <= 0) break;
const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes);
if(r1 != (size_t) cmpBytes) break;
}
{ char* const decPtr = &decBuf[decOffset];
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
if(decBytes <= 0) break;
decOffset += decBytes;
write_bin(outFp, decPtr, decBytes);
// Wraparound the ringbuffer offset
if(decOffset >= DECODE_RING_BUFFER - MESSAGE_MAX_BYTES) decOffset = 0;
}
}
}
int compare(FILE* f0, FILE* f1)
{
int result = 0;
while (0 == result) {
char b0[65536];
char b1[65536];
const size_t r0 = fread(b0, 1, sizeof(b0), f0);
const size_t r1 = fread(b1, 1, sizeof(b1), f1);
result = (int) r0 - (int) r1;
if (0 == r0 || 0 == r1) break;
if (0 == result) result = memcmp(b0, b1, r0);
}
return result;
}
int main(int argc, char** argv)
{
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
if (argc < 2) {
printf("Please specify input filename\n");
return 0;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], 0);
snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], 0);
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("dec = [%s]\n", decFilename);
// compress
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
test_compress(outFp, inpFp);
fclose(outFp);
fclose(inpFp);
}
// decompress
{ FILE* const inpFp = fopen(lz4Filename, "rb");
FILE* const outFp = fopen(decFilename, "wb");
test_decompress(outFp, inpFp);
fclose(outFp);
fclose(inpFp);
}
// verify
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const decFp = fopen(decFilename, "rb");
const int cmp = compare(inpFp, decFp);
if (0 == cmp) {
printf("Verify : OK\n");
} else {
printf("Verify : NG\n");
}
fclose(decFp);
fclose(inpFp);
}
return 0;
}
examples/compress_functions.c
+363
View File
@@ -0,0 +1,363 @@
/*
* compress_functions.c
* Copyright : Kyle Harper
* License : Follows same licensing as the lz4.c/lz4.h program at any given time. Currently, BSD 2.
* Description: A program to demonstrate the various compression functions involved in when using LZ4_compress_default(). The idea
* is to show how each step in the call stack can be used directly, if desired. There is also some benchmarking for
* each function to demonstrate the (probably lack of) performance difference when jumping the stack.
* (If you're new to lz4, please read simple_buffer.c to understand the fundamentals)
*
* The call stack (before theoretical compiler optimizations) for LZ4_compress_default is as follows:
* LZ4_compress_default
* LZ4_compress_fast
* LZ4_compress_fast_extState
* LZ4_compress_generic
*
* LZ4_compress_default()
* This is the recommended function for compressing data. It will serve as the baseline for comparison.
* LZ4_compress_fast()
* Despite its name, it's not a "fast" version of compression. It simply decides if HEAPMODE is set and either
* allocates memory on the heap for a struct or creates the struct directly on the stack. Stack access is generally
* faster but this function itself isn't giving that advantage, it's just some logic for compile time.
* LZ4_compress_fast_extState()
* This simply accepts all the pointers and values collected thus far and adds logic to determine how
* LZ4_compress_generic should be invoked; specifically: can the source fit into a single pass as determined by
* LZ4_64Klimit.
* LZ4_compress_generic()
* As the name suggests, this is the generic function that ultimately does most of the heavy lifting. Calling this
* directly can help avoid some test cases and branching which might be useful in some implementation-specific
* situations, but you really need to know what you're doing AND what you're asking lz4 to do! You also need a
* wrapper function because this function isn't exposed with lz4.h.
*
* The call stack for decompression functions is shallow. There are 2 options:
* LZ4_decompress_safe || LZ4_decompress_fast
* LZ4_decompress_generic
*
* LZ4_decompress_safe
* This is the recommended function for decompressing data. It is considered safe because the caller specifies
* both the size of the compresssed buffer to read as well as the maximum size of the output (decompressed) buffer
* instead of just the latter.
* LZ4_decompress_fast
* Again, despite its name it's not a "fast" version of decompression. It simply frees the caller of sending the
* size of the compressed buffer (it will simply be read-to-end, hence it's non-safety).
* LZ4_decompress_generic
* This is the generic function that both of the LZ4_decompress_* functions above end up calling. Calling this
* directly is not advised, period. Furthermore, it is a static inline function in lz4.c, so there isn't a symbol
* exposed for anyone using lz4.h to utilize.
*
* Special Note About Decompression:
* Using the LZ4_decompress_safe() function protects against malicious (user) input. If you are using data from a
* trusted source, or if your program is the producer (P) as well as its consumer (C) in a PC or MPMC setup, you can
* safely use the LZ4_decompress_fast function
*/
/* Since lz4 compiles with c99 and not gnu/std99 we need to enable POSIX linking for time.h structs and functions. */
#if __STDC_VERSION__ >= 199901L
#define _XOPEN_SOURCE 600
#else
#define _XOPEN_SOURCE 500
#endif
#define _POSIX_C_SOURCE 199309L
/* Includes, for Power! */
#define LZ4_DISABLE_DEPRECATE_WARNINGS /* LZ4_decompress_fast */
#include "lz4.h"
#include <stdio.h> /* for printf() */
#include <stdlib.h> /* for exit() */
#include <string.h> /* for atoi() memcmp() */
#include <stdint.h> /* for uint_types */
#include <inttypes.h> /* for PRIu64 */
#include <time.h> /* for clock_gettime() */
#include <locale.h> /* for setlocale() */
/* We need to know what one billion is for clock timing. */
#define BILLION 1000000000L
/* Create a crude set of test IDs so we can switch on them later (Can't switch() on a char[] or char*). */
#define ID__LZ4_COMPRESS_DEFAULT 1
#define ID__LZ4_COMPRESS_FAST 2
#define ID__LZ4_COMPRESS_FAST_EXTSTATE 3
#define ID__LZ4_COMPRESS_GENERIC 4
#define ID__LZ4_DECOMPRESS_SAFE 5
#define ID__LZ4_DECOMPRESS_FAST 6
/*
* Easy show-error-and-bail function.
*/
void run_screaming(const char *message, const int code) {
printf("%s\n", message);
exit(code);
}
/*
* Centralize the usage function to keep main cleaner.
*/
void usage(const char *message) {
printf("Usage: ./argPerformanceTesting <iterations>\n");
run_screaming(message, 1);
return;
}
/*
* Runs the benchmark for LZ4_compress_* based on function_id.
*/
uint64_t bench(
const char *known_good_dst,
const int function_id,
const int iterations,
const char *src,
char *dst,
const size_t src_size,
const size_t max_dst_size,
const size_t comp_size
) {
uint64_t time_taken = 0;
int rv = 0;
const int warm_up = 5000;
struct timespec start, end;
const int acceleration = 1;
LZ4_stream_t state;
// Select the right function to perform the benchmark on. We perform 5000 initial loops to warm the cache and ensure that dst
// remains matching to known_good_dst between successive calls.
switch(function_id) {
case ID__LZ4_COMPRESS_DEFAULT:
printf("Starting benchmark for function: LZ4_compress_default()\n");
for(int junk=0; junk<warm_up; junk++)
rv = LZ4_compress_default(src, dst, src_size, max_dst_size);
if (rv < 1)
run_screaming("Couldn't run LZ4_compress_default()... error code received is in exit code.", rv);
if (memcmp(known_good_dst, dst, max_dst_size) != 0)
run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i=1; i<=iterations; i++)
LZ4_compress_default(src, dst, src_size, max_dst_size);
break;
case ID__LZ4_COMPRESS_FAST:
printf("Starting benchmark for function: LZ4_compress_fast()\n");
for(int junk=0; junk<warm_up; junk++)
rv = LZ4_compress_fast(src, dst, src_size, max_dst_size, acceleration);
if (rv < 1)
run_screaming("Couldn't run LZ4_compress_fast()... error code received is in exit code.", rv);
if (memcmp(known_good_dst, dst, max_dst_size) != 0)
run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i=1; i<=iterations; i++)
LZ4_compress_fast(src, dst, src_size, max_dst_size, acceleration);
break;
case ID__LZ4_COMPRESS_FAST_EXTSTATE:
printf("Starting benchmark for function: LZ4_compress_fast_extState()\n");
for(int junk=0; junk<warm_up; junk++)
rv = LZ4_compress_fast_extState(&state, src, dst, src_size, max_dst_size, acceleration);
if (rv < 1)
run_screaming("Couldn't run LZ4_compress_fast_extState()... error code received is in exit code.", rv);
if (memcmp(known_good_dst, dst, max_dst_size) != 0)
run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i=1; i<=iterations; i++)
LZ4_compress_fast_extState(&state, src, dst, src_size, max_dst_size, acceleration);
break;
// Disabled until LZ4_compress_generic() is exposed in the header.
// case ID__LZ4_COMPRESS_GENERIC:
// printf("Starting benchmark for function: LZ4_compress_generic()\n");
// LZ4_resetStream((LZ4_stream_t*)&state);
// for(int junk=0; junk<warm_up; junk++) {
// LZ4_resetStream((LZ4_stream_t*)&state);
// //rv = LZ4_compress_generic_wrapper(&state, src, dst, src_size, max_dst_size, notLimited, byU16, noDict, noDictIssue, acceleration);
// LZ4_compress_generic_wrapper(&state, src, dst, src_size, max_dst_size, acceleration);
// }
// if (rv < 1)
// run_screaming("Couldn't run LZ4_compress_generic()... error code received is in exit code.", rv);
// if (memcmp(known_good_dst, dst, max_dst_size) != 0)
// run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
// for (int i=1; i<=iterations; i++) {
// LZ4_resetStream((LZ4_stream_t*)&state);
// //LZ4_compress_generic_wrapper(&state, src, dst, src_size, max_dst_size, notLimited, byU16, noDict, noDictIssue, acceleration);
// LZ4_compress_generic_wrapper(&state, src, dst, src_size, max_dst_size, acceleration);
// }
// break;
case ID__LZ4_DECOMPRESS_SAFE:
printf("Starting benchmark for function: LZ4_decompress_safe()\n");
for(int junk=0; junk<warm_up; junk++)
rv = LZ4_decompress_safe(src, dst, comp_size, src_size);
if (rv < 1)
run_screaming("Couldn't run LZ4_decompress_safe()... error code received is in exit code.", rv);
if (memcmp(known_good_dst, dst, src_size) != 0)
run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i=1; i<=iterations; i++)
LZ4_decompress_safe(src, dst, comp_size, src_size);
break;
case ID__LZ4_DECOMPRESS_FAST:
printf("Starting benchmark for function: LZ4_decompress_fast()\n");
for(int junk=0; junk<warm_up; junk++)
rv = LZ4_decompress_fast(src, dst, src_size);
if (rv < 1)
run_screaming("Couldn't run LZ4_decompress_fast()... error code received is in exit code.", rv);
if (memcmp(known_good_dst, dst, src_size) != 0)
run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i=1; i<=iterations; i++)
LZ4_decompress_fast(src, dst, src_size);
break;
default:
run_screaming("The test specified isn't valid. Please check your code.", 1);
break;
}
// Stop timer and return time taken.
clock_gettime(CLOCK_MONOTONIC, &end);
time_taken = BILLION *(end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec;
return time_taken;
}
/*
* main()
* We will demonstrate the use of each function for simplicity sake. Then we will run 2 suites of benchmarking:
* Test suite A) Uses generic Lorem Ipsum text which should be generally compressible insomuch as basic human text is
* compressible for such a small src_size
* Test Suite B) For the sake of testing, see what results we get if the data is drastically easier to compress. IF there are
* indeed losses and IF more compressible data is faster to process, this will exacerbate the findings.
*/
int main(int argc, char **argv) {
// Get and verify options. There's really only 1: How many iterations to run.
int iterations = 1000000;
if (argc > 1)
iterations = atoi(argv[1]);
if (iterations < 1)
usage("Argument 1 (iterations) must be > 0.");
// First we will create 2 sources (char *) of 2000 bytes each. One normal text, the other highly-compressible text.
const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed luctus purus et risus vulputate, et mollis orci ullamcorper. Nulla facilisi. Fusce in ligula sed purus varius aliquet interdum vitae justo. Proin quis diam velit. Nulla varius iaculis auctor. Cras volutpat, justo eu dictum pulvinar, elit sem porttitor metus, et imperdiet metus sapien et ante. Nullam nisi nulla, ornare eu tristique eu, dignissim vitae diam. Nulla sagittis porta libero, a accumsan felis sagittis scelerisque. Integer laoreet eleifend congue. Etiam rhoncus leo vel dolor fermentum, quis luctus nisl iaculis. Praesent a erat sapien. Aliquam semper mi in lorem ultrices ultricies. Lorem ipsum dolor sit amet, consectetur adipiscing elit. In feugiat risus sed enim ultrices, at sodales nulla tristique. Maecenas eget pellentesque justo, sed pellentesque lectus. Fusce sagittis sit amet elit vel varius. Donec sed ligula nec ligula vulputate rutrum sed ut lectus. Etiam congue pharetra leo vitae cursus. Morbi enim ante, porttitor ut varius vel, tincidunt quis justo. Nunc iaculis, risus id ultrices semper, metus est efficitur ligula, vel posuere risus nunc eget purus. Ut lorem turpis, condimentum at sem sed, porta aliquam turpis. In ut sapien a nulla dictum tincidunt quis sit amet lorem. Fusce at est egestas, luctus neque eu, consectetur tortor. Phasellus eleifend ultricies nulla ac lobortis. Morbi maximus quam cursus vehicula iaculis. Maecenas cursus vel justo ut rutrum. Curabitur magna orci, dignissim eget dapibus vitae, finibus id lacus. Praesent rhoncus mattis augue vitae bibendum. Praesent porta mauris non ultrices fermentum. Quisque vulputate ipsum in sodales pulvinar. Aliquam nec mollis felis. Donec vitae augue pulvinar, congue nisl sed, pretium purus. Fusce lobortis mi ac neque scelerisque semper. Pellentesque vel est vitae magna aliquet aliquet. Nam non dolor. Nulla facilisi. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Morbi ac lacinia felis metus.";
const char *hc_src = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
// Set and derive sizes. Since we're using strings, use strlen() + 1 for \0.
const size_t src_size = strlen(src) + 1;
const size_t max_dst_size = LZ4_compressBound(src_size);
int bytes_returned = 0;
// Now build allocations for the data we'll be playing with.
char *dst = calloc(1, max_dst_size);
char *known_good_dst = calloc(1, max_dst_size);
char *known_good_hc_dst = calloc(1, max_dst_size);
if (dst == NULL || known_good_dst == NULL || known_good_hc_dst == NULL)
run_screaming("Couldn't allocate memory for the destination buffers. Sad :(", 1);
// Create known-good buffers to verify our tests with other functions will produce the same results.
bytes_returned = LZ4_compress_default(src, known_good_dst, src_size, max_dst_size);
if (bytes_returned < 1)
run_screaming("Couldn't create a known-good destination buffer for comparison... this is bad.", 1);
const size_t src_comp_size = bytes_returned;
bytes_returned = LZ4_compress_default(hc_src, known_good_hc_dst, src_size, max_dst_size);
if (bytes_returned < 1)
run_screaming("Couldn't create a known-good (highly compressible) destination buffer for comparison... this is bad.", 1);
const size_t hc_src_comp_size = bytes_returned;
/* LZ4_compress_default() */
// This is the default function so we don't need to demonstrate how to use it. See basics.c if you need more basal information.
/* LZ4_compress_fast() */
// Using this function is identical to LZ4_compress_default except we need to specify an "acceleration" value. Defaults to 1.
memset(dst, 0, max_dst_size);
bytes_returned = LZ4_compress_fast(src, dst, src_size, max_dst_size, 1);
if (bytes_returned < 1)
run_screaming("Failed to compress src using LZ4_compress_fast. echo $? for return code.", bytes_returned);
if (memcmp(dst, known_good_dst, bytes_returned) != 0)
run_screaming("According to memcmp(), the value we got in dst from LZ4_compress_fast doesn't match the known-good value. This is bad.", 1);
/* LZ4_compress_fast_extState() */
// Using this function directly requires that we build an LZ4_stream_t struct ourselves. We do NOT have to reset it ourselves.
memset(dst, 0, max_dst_size);
LZ4_stream_t state;
bytes_returned = LZ4_compress_fast_extState(&state, src, dst, src_size, max_dst_size, 1);
if (bytes_returned < 1)
run_screaming("Failed to compress src using LZ4_compress_fast_extState. echo $? for return code.", bytes_returned);
if (memcmp(dst, known_good_dst, bytes_returned) != 0)
run_screaming("According to memcmp(), the value we got in dst from LZ4_compress_fast_extState doesn't match the known-good value. This is bad.", 1);
/* LZ4_compress_generic */
// When you can exactly control the inputs and options of your LZ4 needs, you can use LZ4_compress_generic and fixed (const)
// values for the enum types such as dictionary and limitations. Any other direct-use is probably a bad idea.
//
// That said, the LZ4_compress_generic() function is 'static inline' and does not have a prototype in lz4.h to expose a symbol
// for it. In other words: we can't access it directly. I don't want to submit a PR that modifies lz4.c/h. Yann and others can
// do that if they feel it's worth expanding this example.
//
// I will, however, leave a skeleton of what would be required to use it directly:
/*
memset(dst, 0, max_dst_size);
// LZ4_stream_t state: is already declared above. We can reuse it BUT we have to reset the stream ourselves between each call.
LZ4_resetStream((LZ4_stream_t *)&state);
// Since src size is small we know the following enums will be used: notLimited (0), byU16 (2), noDict (0), noDictIssue (0).
bytes_returned = LZ4_compress_generic(&state, src, dst, src_size, max_dst_size, notLimited, byU16, noDict, noDictIssue, 1);
if (bytes_returned < 1)
run_screaming("Failed to compress src using LZ4_compress_generic. echo $? for return code.", bytes_returned);
if (memcmp(dst, known_good_dst, bytes_returned) != 0)
run_screaming("According to memcmp(), the value we got in dst from LZ4_compress_generic doesn't match the known-good value. This is bad.", 1);
*/
/* Benchmarking */
/* Now we'll run a few rudimentary benchmarks with each function to demonstrate differences in speed based on the function used.
* Remember, we cannot call LZ4_compress_generic() directly (yet) so it's disabled.
*/
// Suite A - Normal Compressibility
char *dst_d = calloc(1, src_size);
memset(dst, 0, max_dst_size);
printf("\nStarting suite A: Normal compressible text.\n");
uint64_t time_taken__default = bench(known_good_dst, ID__LZ4_COMPRESS_DEFAULT, iterations, src, dst, src_size, max_dst_size, src_comp_size);
uint64_t time_taken__fast = bench(known_good_dst, ID__LZ4_COMPRESS_FAST, iterations, src, dst, src_size, max_dst_size, src_comp_size);
uint64_t time_taken__fast_extstate = bench(known_good_dst, ID__LZ4_COMPRESS_FAST_EXTSTATE, iterations, src, dst, src_size, max_dst_size, src_comp_size);
//uint64_t time_taken__generic = bench(known_good_dst, ID__LZ4_COMPRESS_GENERIC, iterations, src, dst, src_size, max_dst_size, src_comp_size);
uint64_t time_taken__decomp_safe = bench(src, ID__LZ4_DECOMPRESS_SAFE, iterations, known_good_dst, dst_d, src_size, max_dst_size, src_comp_size);
uint64_t time_taken__decomp_fast = bench(src, ID__LZ4_DECOMPRESS_FAST, iterations, known_good_dst, dst_d, src_size, max_dst_size, src_comp_size);
// Suite B - Highly Compressible
memset(dst, 0, max_dst_size);
printf("\nStarting suite B: Highly compressible text.\n");
uint64_t time_taken_hc__default = bench(known_good_hc_dst, ID__LZ4_COMPRESS_DEFAULT, iterations, hc_src, dst, src_size, max_dst_size, hc_src_comp_size);
uint64_t time_taken_hc__fast = bench(known_good_hc_dst, ID__LZ4_COMPRESS_FAST, iterations, hc_src, dst, src_size, max_dst_size, hc_src_comp_size);
uint64_t time_taken_hc__fast_extstate = bench(known_good_hc_dst, ID__LZ4_COMPRESS_FAST_EXTSTATE, iterations, hc_src, dst, src_size, max_dst_size, hc_src_comp_size);
//uint64_t time_taken_hc__generic = bench(known_good_hc_dst, ID__LZ4_COMPRESS_GENERIC, iterations, hc_src, dst, src_size, max_dst_size, hc_src_comp_size);
uint64_t time_taken_hc__decomp_safe = bench(hc_src, ID__LZ4_DECOMPRESS_SAFE, iterations, known_good_hc_dst, dst_d, src_size, max_dst_size, hc_src_comp_size);
uint64_t time_taken_hc__decomp_fast = bench(hc_src, ID__LZ4_DECOMPRESS_FAST, iterations, known_good_hc_dst, dst_d, src_size, max_dst_size, hc_src_comp_size);
// Report and leave.
setlocale(LC_ALL, "");
const char *format = "|%-14s|%-30s|%'14.9f|%'16d|%'14d|%'13.2f%%|\n";
const char *header_format = "|%-14s|%-30s|%14s|%16s|%14s|%14s|\n";
const char *separator = "+--------------+------------------------------+--------------+----------------+--------------+--------------+\n";
printf("\n");
printf("%s", separator);
printf(header_format, "Source", "Function Benchmarked", "Total Seconds", "Iterations/sec", "ns/Iteration", "% of default");
printf("%s", separator);
printf(format, "Normal Text", "LZ4_compress_default()", (double)time_taken__default / BILLION, (int)(iterations / ((double)time_taken__default /BILLION)), (int)time_taken__default / iterations, (double)time_taken__default * 100 / time_taken__default);
printf(format, "Normal Text", "LZ4_compress_fast()", (double)time_taken__fast / BILLION, (int)(iterations / ((double)time_taken__fast /BILLION)), (int)time_taken__fast / iterations, (double)time_taken__fast * 100 / time_taken__default);
printf(format, "Normal Text", "LZ4_compress_fast_extState()", (double)time_taken__fast_extstate / BILLION, (int)(iterations / ((double)time_taken__fast_extstate /BILLION)), (int)time_taken__fast_extstate / iterations, (double)time_taken__fast_extstate * 100 / time_taken__default);
//printf(format, "Normal Text", "LZ4_compress_generic()", (double)time_taken__generic / BILLION, (int)(iterations / ((double)time_taken__generic /BILLION)), (int)time_taken__generic / iterations, (double)time_taken__generic * 100 / time_taken__default);
printf(format, "Normal Text", "LZ4_decompress_safe()", (double)time_taken__decomp_safe / BILLION, (int)(iterations / ((double)time_taken__decomp_safe /BILLION)), (int)time_taken__decomp_safe / iterations, (double)time_taken__decomp_safe * 100 / time_taken__default);
printf(format, "Normal Text", "LZ4_decompress_fast()", (double)time_taken__decomp_fast / BILLION, (int)(iterations / ((double)time_taken__decomp_fast /BILLION)), (int)time_taken__decomp_fast / iterations, (double)time_taken__decomp_fast * 100 / time_taken__default);
printf(header_format, "", "", "", "", "", "");
printf(format, "Compressible", "LZ4_compress_default()", (double)time_taken_hc__default / BILLION, (int)(iterations / ((double)time_taken_hc__default /BILLION)), (int)time_taken_hc__default / iterations, (double)time_taken_hc__default * 100 / time_taken_hc__default);
printf(format, "Compressible", "LZ4_compress_fast()", (double)time_taken_hc__fast / BILLION, (int)(iterations / ((double)time_taken_hc__fast /BILLION)), (int)time_taken_hc__fast / iterations, (double)time_taken_hc__fast * 100 / time_taken_hc__default);
printf(format, "Compressible", "LZ4_compress_fast_extState()", (double)time_taken_hc__fast_extstate / BILLION, (int)(iterations / ((double)time_taken_hc__fast_extstate /BILLION)), (int)time_taken_hc__fast_extstate / iterations, (double)time_taken_hc__fast_extstate * 100 / time_taken_hc__default);
//printf(format, "Compressible", "LZ4_compress_generic()", (double)time_taken_hc__generic / BILLION, (int)(iterations / ((double)time_taken_hc__generic /BILLION)), (int)time_taken_hc__generic / iterations, (double)time_taken_hc__generic * 100 / time_taken_hc__default);
printf(format, "Compressible", "LZ4_decompress_safe()", (double)time_taken_hc__decomp_safe / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_safe /BILLION)), (int)time_taken_hc__decomp_safe / iterations, (double)time_taken_hc__decomp_safe * 100 / time_taken_hc__default);
printf(format, "Compressible", "LZ4_decompress_fast()", (double)time_taken_hc__decomp_fast / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_fast /BILLION)), (int)time_taken_hc__decomp_fast / iterations, (double)time_taken_hc__decomp_fast * 100 / time_taken_hc__default);
printf("%s", separator);
printf("\n");
printf("All done, ran %d iterations per test.\n", iterations);
return 0;
}
examples/dictionaryRandomAccess.c
+280
View File
@@ -0,0 +1,280 @@
// LZ4 API example : Dictionary Random Access
#if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif
#include "lz4.h"
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#define MIN(x, y) ((x) < (y) ? (x) : (y))
enum {
BLOCK_BYTES = 1024, /* 1 KiB of uncompressed data in a block */
DICTIONARY_BYTES = 1024, /* Load a 1 KiB dictionary */
MAX_BLOCKS = 1024 /* For simplicity of implementation */
};
/**
* Magic bytes for this test case.
* This is not a great magic number because it is a common word in ASCII.
* However, it is important to have some versioning system in your format.
*/
const char kTestMagic[] = { 'T', 'E', 'S', 'T' };
void write_int(FILE* fp, int i) {
size_t written = fwrite(&i, sizeof(i), 1, fp);
if (written != 1) { exit(10); }
}
void write_bin(FILE* fp, const void* array, size_t arrayBytes) {
size_t written = fwrite(array, 1, arrayBytes, fp);
if (written != arrayBytes) { exit(11); }
}
void read_int(FILE* fp, int* i) {
size_t read = fread(i, sizeof(*i), 1, fp);
if (read != 1) { exit(12); }
}
size_t read_bin(FILE* fp, void* array, size_t arrayBytes) {
size_t read = fread(array, 1, arrayBytes, fp);
if (ferror(fp)) { exit(12); }
return read;
}
void seek_bin(FILE* fp, long offset, int origin) {
if (fseek(fp, offset, origin)) { exit(14); }
}
void test_compress(FILE* outFp, FILE* inpFp, void *dict, int dictSize)
{
LZ4_stream_t lz4Stream_body;
LZ4_stream_t* lz4Stream = &lz4Stream_body;
char inpBuf[BLOCK_BYTES];
int offsets[MAX_BLOCKS];
int *offsetsEnd = offsets;
LZ4_initStream(lz4Stream, sizeof(*lz4Stream));
/* Write header magic */
write_bin(outFp, kTestMagic, sizeof(kTestMagic));
*offsetsEnd++ = sizeof(kTestMagic);
/* Write compressed data blocks. Each block contains BLOCK_BYTES of plain
data except possibly the last. */
for(;;) {
const int inpBytes = (int) read_bin(inpFp, inpBuf, BLOCK_BYTES);
if(0 == inpBytes) {
break;
}
/* Forget previously compressed data and load the dictionary */
LZ4_loadDict(lz4Stream, dict, dictSize);
{
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
const int cmpBytes = LZ4_compress_fast_continue(
lz4Stream, inpBuf, cmpBuf, inpBytes, sizeof(cmpBuf), 1);
if(cmpBytes <= 0) { exit(1); }
write_bin(outFp, cmpBuf, (size_t)cmpBytes);
/* Keep track of the offsets */
*offsetsEnd = *(offsetsEnd - 1) + cmpBytes;
++offsetsEnd;
}
if (offsetsEnd - offsets > MAX_BLOCKS) { exit(2); }
}
/* Write the tailing jump table */
{
int *ptr = offsets;
while (ptr != offsetsEnd) {
write_int(outFp, *ptr++);
}
write_int(outFp, offsetsEnd - offsets);
}
}
void test_decompress(FILE* outFp, FILE* inpFp, void *dict, int dictSize, int offset, int length)
{
LZ4_streamDecode_t lz4StreamDecode_body;
LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
/* The blocks [currentBlock, endBlock) contain the data we want */
int currentBlock = offset / BLOCK_BYTES;
int endBlock = ((offset + length - 1) / BLOCK_BYTES) + 1;
char decBuf[BLOCK_BYTES];
int offsets[MAX_BLOCKS];
/* Special cases */
if (length == 0) { return; }
/* Read the magic bytes */
{
char magic[sizeof(kTestMagic)];
size_t read = read_bin(inpFp, magic, sizeof(magic));
if (read != sizeof(magic)) { exit(1); }
if (memcmp(kTestMagic, magic, sizeof(magic))) { exit(2); }
}
/* Read the offsets tail */
{
int numOffsets;
int block;
int *offsetsPtr = offsets;
seek_bin(inpFp, -4, SEEK_END);
read_int(inpFp, &numOffsets);
if (numOffsets <= endBlock) { exit(3); }
seek_bin(inpFp, -4 * (numOffsets + 1), SEEK_END);
for (block = 0; block <= endBlock; ++block) {
read_int(inpFp, offsetsPtr++);
}
}
/* Seek to the first block to read */
seek_bin(inpFp, offsets[currentBlock], SEEK_SET);
offset = offset % BLOCK_BYTES;
/* Start decoding */
for(; currentBlock < endBlock; ++currentBlock) {
char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
/* The difference in offsets is the size of the block */
int cmpBytes = offsets[currentBlock + 1] - offsets[currentBlock];
{
const size_t read = read_bin(inpFp, cmpBuf, (size_t)cmpBytes);
if(read != (size_t)cmpBytes) { exit(4); }
}
/* Load the dictionary */
LZ4_setStreamDecode(lz4StreamDecode, dict, dictSize);
{
const int decBytes = LZ4_decompress_safe_continue(
lz4StreamDecode, cmpBuf, decBuf, cmpBytes, BLOCK_BYTES);
if(decBytes <= 0) { exit(5); }
{
/* Write out the part of the data we care about */
int blockLength = MIN(length, (decBytes - offset));
write_bin(outFp, decBuf + offset, (size_t)blockLength);
offset = 0;
length -= blockLength;
}
}
}
}
int compare(FILE* fp0, FILE* fp1, int length)
{
int result = 0;
while(0 == result) {
char b0[4096];
char b1[4096];
const size_t r0 = read_bin(fp0, b0, MIN(length, (int)sizeof(b0)));
const size_t r1 = read_bin(fp1, b1, MIN(length, (int)sizeof(b1)));
result = (int) r0 - (int) r1;
if(0 == r0 || 0 == r1) {
break;
}
if(0 == result) {
result = memcmp(b0, b1, r0);
}
length -= r0;
}
return result;
}
int main(int argc, char* argv[])
{
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
char dictFilename[256] = { 0 };
int offset;
int length;
char dict[DICTIONARY_BYTES];
int dictSize;
if(argc < 5) {
printf("Usage: %s input dictionary offset length", argv[0]);
return 0;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], BLOCK_BYTES);
snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], BLOCK_BYTES);
snprintf(dictFilename, 256, "%s", argv[2]);
offset = atoi(argv[3]);
length = atoi(argv[4]);
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("dec = [%s]\n", decFilename);
printf("dict = [%s]\n", dictFilename);
printf("offset = [%d]\n", offset);
printf("length = [%d]\n", length);
/* Load dictionary */
{
FILE* dictFp = fopen(dictFilename, "rb");
dictSize = (int)read_bin(dictFp, dict, DICTIONARY_BYTES);
fclose(dictFp);
}
/* compress */
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lz4Filename, "wb");
printf("compress : %s -> %s\n", inpFilename, lz4Filename);
test_compress(outFp, inpFp, dict, dictSize);
printf("compress : done\n");
fclose(outFp);
fclose(inpFp);
}
/* decompress */
{
FILE* inpFp = fopen(lz4Filename, "rb");
FILE* outFp = fopen(decFilename, "wb");
printf("decompress : %s -> %s\n", lz4Filename, decFilename);
test_decompress(outFp, inpFp, dict, DICTIONARY_BYTES, offset, length);
printf("decompress : done\n");
fclose(outFp);
fclose(inpFp);
}
/* verify */
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
seek_bin(inpFp, offset, SEEK_SET);
printf("verify : %s <-> %s\n", inpFilename, decFilename);
const int cmp = compare(inpFp, decFp, length);
if(0 == cmp) {
printf("verify : OK\n");
} else {
printf("verify : NG\n");
}
fclose(decFp);
fclose(inpFp);
}
return 0;
}
examples/dictionaryRandomAccess.md
+67
View File
@@ -0,0 +1,67 @@
# LZ4 API Example : Dictionary Random Access
`dictionaryRandomAccess.c` is LZ4 API example which implements dictionary compression and random access decompression.
Please note that the output file is not compatible with lz4frame and is platform dependent.
## What's the point of this example ?
- Dictionary based compression for homogenous files.
- Random access to compressed blocks.
## How the compression works
Reads the dictionary from a file, and uses it as the history for each block.
This allows each block to be independent, but maintains compression ratio.
```
Dictionary
+
|
v
+---------+
| Block#1 |
+----+----+
|
v
{Out#1}
Dictionary
+
|
v
+---------+
| Block#2 |
+----+----+
|
v
{Out#2}
```
After writing the magic bytes `TEST` and then the compressed blocks, write out the jump table.
The last 4 bytes is an integer containing the number of blocks in the stream.
If there are `N` blocks, then just before the last 4 bytes is `N + 1` 4 byte integers containing the offsets at the beginning and end of each block.
Let `Offset#K` be the total number of bytes written after writing out `Block#K` *including* the magic bytes for simplicity.
```
+------+---------+ +---------+---+----------+ +----------+-----+
| TEST | Block#1 | ... | Block#N | 4 | Offset#1 | ... | Offset#N | N+1 |
+------+---------+ +---------+---+----------+ +----------+-----+
```
## How the decompression works
Decompression will do reverse order.
- Seek to the last 4 bytes of the file and read the number of offsets.
- Read each offset into an array.
- Seek to the first block containing data we want to read.
We know where to look because we know each block contains a fixed amount of uncompressed data, except possibly the last.
- Decompress it and write what data we need from it to the file.
- Read the next block.
- Decompress it and write that page to the file.
Continue these procedure until all the required data has been read.
examples/frameCompress.c
+401
View File
@@ -0,0 +1,401 @@
/* LZ4frame API example : compress a file
* Modified from an example code by Zbigniew Jędrzejewski-Szmek
*
* This example streams an input file into an output file
* using a bounded memory budget.
* Input is read in chunks of IN_CHUNK_SIZE */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <lz4frame.h>
#define IN_CHUNK_SIZE (16*1024)
static const LZ4F_preferences_t kPrefs = {
{ LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
0, /* compression level; 0 == default */
0, /* autoflush */
0, /* favor decompression speed */
{ 0, 0, 0 }, /* reserved, must be set to 0 */
};
/* safe_fwrite() :
* performs fwrite(), ensure operation success, or immediately exit() */
static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
{
size_t const writtenSize = fwrite(buf, eltSize, nbElt, f);
size_t const expectedSize = eltSize * nbElt;
if (nbElt>0) assert(expectedSize / nbElt == eltSize); /* check overflow */
if (writtenSize < expectedSize) {
if (ferror(f)) /* note : ferror() must follow fwrite */
fprintf(stderr, "Write failed \n");
else
fprintf(stderr, "Write too short \n");
exit(1);
}
}
/* ================================================= */
/* Streaming Compression example */
/* ================================================= */
typedef struct {
int error;
unsigned long long size_in;
unsigned long long size_out;
} compressResult_t;
static compressResult_t
compress_file_internal(FILE* f_in, FILE* f_out,
LZ4F_compressionContext_t ctx,
void* inBuff, size_t inChunkSize,
void* outBuff, size_t outCapacity)
{
compressResult_t result = { 1, 0, 0 }; /* result for an error */
unsigned long long count_in = 0, count_out;
assert(f_in != NULL); assert(f_out != NULL);
assert(ctx != NULL);
assert(outCapacity >= LZ4F_HEADER_SIZE_MAX);
assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs));
/* write frame header */
{ size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs);
if (LZ4F_isError(headerSize)) {
printf("Failed to start compression: error %u \n", (unsigned)headerSize);
return result;
}
count_out = headerSize;
printf("Buffer size is %u bytes, header size %u bytes \n",
(unsigned)outCapacity, (unsigned)headerSize);
safe_fwrite(outBuff, 1, headerSize, f_out);
}
/* stream file */
for (;;) {
size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in);
if (readSize == 0) break; /* nothing left to read from input file */
count_in += readSize;
size_t const compressedSize = LZ4F_compressUpdate(ctx,
outBuff, outCapacity,
inBuff, readSize,
NULL);
if (LZ4F_isError(compressedSize)) {
printf("Compression failed: error %u \n", (unsigned)compressedSize);
return result;
}
printf("Writing %u bytes\n", (unsigned)compressedSize);
safe_fwrite(outBuff, 1, compressedSize, f_out);
count_out += compressedSize;
}
/* flush whatever remains within internal buffers */
{ size_t const compressedSize = LZ4F_compressEnd(ctx,
outBuff, outCapacity,
NULL);
if (LZ4F_isError(compressedSize)) {
printf("Failed to end compression: error %u \n", (unsigned)compressedSize);
return result;
}
printf("Writing %u bytes \n", (unsigned)compressedSize);
safe_fwrite(outBuff, 1, compressedSize, f_out);
count_out += compressedSize;
}
result.size_in = count_in;
result.size_out = count_out;
result.error = 0;
return result;
}
static compressResult_t
compress_file(FILE* f_in, FILE* f_out)
{
assert(f_in != NULL);
assert(f_out != NULL);
/* ressource allocation */
LZ4F_compressionContext_t ctx;
size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
void* const src = malloc(IN_CHUNK_SIZE);
size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs); /* large enough for any input <= IN_CHUNK_SIZE */
void* const outbuff = malloc(outbufCapacity);
compressResult_t result = { 1, 0, 0 }; /* == error (default) */
if (!LZ4F_isError(ctxCreation) && src && outbuff) {
result = compress_file_internal(f_in, f_out,
ctx,
src, IN_CHUNK_SIZE,
outbuff, outbufCapacity);
} else {
printf("error : ressource allocation failed \n");
}
LZ4F_freeCompressionContext(ctx); /* supports free on NULL */
free(src);
free(outbuff);
return result;
}
/* ================================================= */
/* Streaming decompression example */
/* ================================================= */
static size_t get_block_size(const LZ4F_frameInfo_t* info) {
switch (info->blockSizeID) {
case LZ4F_default:
case LZ4F_max64KB: return 1 << 16;
case LZ4F_max256KB: return 1 << 18;
case LZ4F_max1MB: return 1 << 20;
case LZ4F_max4MB: return 1 << 22;
default:
printf("Impossible with expected frame specification (<=v1.6.1)\n");
exit(1);
}
}
/* @return : 1==error, 0==success */
static int
decompress_file_internal(FILE* f_in, FILE* f_out,
LZ4F_dctx* dctx,
void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed,
void* dst, size_t dstCapacity)
{
int firstChunk = 1;
size_t ret = 1;
assert(f_in != NULL); assert(f_out != NULL);
assert(dctx != NULL);
assert(src != NULL); assert(srcCapacity > 0); assert(filled <= srcCapacity); assert(alreadyConsumed <= filled);
assert(dst != NULL); assert(dstCapacity > 0);
/* Decompression */
while (ret != 0) {
/* Load more input */
size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0;
const void* srcPtr = (const char*)src + alreadyConsumed; alreadyConsumed=0;
const void* const srcEnd = (const char*)srcPtr + readSize;
if (readSize == 0 || ferror(f_in)) {
printf("Decompress: not enough input or error reading file\n");
return 1;
}
/* Decompress:
* Continue while there is more input to read (srcPtr != srcEnd)
* and the frame isn't over (ret != 0)
*/
while (srcPtr < srcEnd && ret != 0) {
/* Any data within dst has been flushed at this stage */
size_t dstSize = dstCapacity;
size_t srcSize = (const char*)srcEnd - (const char*)srcPtr;
ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
if (LZ4F_isError(ret)) {
printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
return 1;
}
/* Flush output */
if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out);
/* Update input */
srcPtr = (const char*)srcPtr + srcSize;
}
assert(srcPtr <= srcEnd);
/* Ensure all input data has been consumed.
* It is valid to have multiple frames in the same file,
* but this example only supports one frame.
*/
if (srcPtr < srcEnd) {
printf("Decompress: Trailing data left in file after frame\n");
return 1;
}
}
/* Check that there isn't trailing data in the file after the frame.
* It is valid to have multiple frames in the same file,
* but this example only supports one frame.
*/
{ size_t const readSize = fread(src, 1, 1, f_in);
if (readSize != 0 || !feof(f_in)) {
printf("Decompress: Trailing data left in file after frame\n");
return 1;
} }
return 0;
}
/* @return : 1==error, 0==completed */
static int
decompress_file_allocDst(FILE* f_in, FILE* f_out,
LZ4F_dctx* dctx,
void* src, size_t srcCapacity)
{
assert(f_in != NULL); assert(f_out != NULL);
assert(dctx != NULL);
assert(src != NULL);
assert(srcCapacity >= LZ4F_HEADER_SIZE_MAX); /* ensure LZ4F_getFrameInfo() can read enough data */
/* Read Frame header */
size_t const readSize = fread(src, 1, srcCapacity, f_in);
if (readSize == 0 || ferror(f_in)) {
printf("Decompress: not enough input or error reading file\n");
return 1;
}
LZ4F_frameInfo_t info;
size_t consumedSize = readSize;
{ size_t const fires = LZ4F_getFrameInfo(dctx, &info, src, &consumedSize);
if (LZ4F_isError(fires)) {
printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(fires));
return 1;
} }
/* Allocating enough space for an entire block isn't necessary for
* correctness, but it allows some memcpy's to be elided.
*/
size_t const dstCapacity = get_block_size(&info);
void* const dst = malloc(dstCapacity);
if (!dst) { perror("decompress_file(dst)"); return 1; }
int const decompressionResult = decompress_file_internal(
f_in, f_out,
dctx,
src, srcCapacity, readSize-consumedSize, consumedSize,
dst, dstCapacity);
free(dst);
return decompressionResult;
}
/* @result : 1==error, 0==success */
static int decompress_file(FILE* f_in, FILE* f_out)
{
assert(f_in != NULL); assert(f_out != NULL);
/* Ressource allocation */
void* const src = malloc(IN_CHUNK_SIZE);
if (!src) { perror("decompress_file(src)"); return 1; }
LZ4F_dctx* dctx;
{ size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
if (LZ4F_isError(dctxStatus)) {
printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
} }
int const result = !dctx ? 1 /* error */ :
decompress_file_allocDst(f_in, f_out, dctx, src, IN_CHUNK_SIZE);
free(src);
LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */
return result;
}
int compareFiles(FILE* fp0, FILE* fp1)
{
int result = 0;
while (result==0) {
char b0[1024];
char b1[1024];
size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
result = (r0 != r1);
if (!r0 || !r1) break;
if (!result) result = memcmp(b0, b1, r0);
}
return result;
}
int main(int argc, const char **argv) {
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
if (argc < 2) {
printf("Please specify input filename\n");
return 0;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("dec = [%s]\n", decFilename);
/* compress */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
printf("compress : %s -> %s\n", inpFilename, lz4Filename);
compressResult_t const ret = compress_file(inpFp, outFp);
fclose(outFp);
fclose(inpFp);
if (ret.error) {
printf("compress : failed with code %i\n", ret.error);
return ret.error;
}
printf("%s: %zu → %zu bytes, %.1f%%\n",
inpFilename,
(size_t)ret.size_in, (size_t)ret.size_out, /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */
(double)ret.size_out / ret.size_in * 100);
printf("compress : done\n");
}
/* decompress */
{ FILE* const inpFp = fopen(lz4Filename, "rb");
FILE* const outFp = fopen(decFilename, "wb");
printf("decompress : %s -> %s\n", lz4Filename, decFilename);
int const ret = decompress_file(inpFp, outFp);
fclose(outFp);
fclose(inpFp);
if (ret) {
printf("decompress : failed with code %i\n", ret);
return ret;
}
printf("decompress : done\n");
}
/* verify */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const decFp = fopen(decFilename, "rb");
printf("verify : %s <-> %s\n", inpFilename, decFilename);
int const cmp = compareFiles(inpFp, decFp);
fclose(decFp);
fclose(inpFp);
if (cmp) {
printf("corruption detected : decompressed file differs from original\n");
return cmp;
}
printf("verify : OK\n");
}
return 0;
}
examples/printVersion.c
+13
View File
@@ -0,0 +1,13 @@
// LZ4 trivial example : print Library version number
// by Takayuki Matsuoka
#include <stdio.h>
#include "lz4.h"
int main(int argc, char** argv)
{
(void)argc; (void)argv;
printf("Hello World ! LZ4 Library version = %d\n", LZ4_versionNumber());
return 0;
}
examples/simple_buffer.c
+99
View File
@@ -0,0 +1,99 @@
/*
* simple_buffer.c
* Copyright : Kyle Harper
* License : Follows same licensing as the lz4.c/lz4.h program at any given time. Currently, BSD 2.
* Description: Example program to demonstrate the basic usage of the compress/decompress functions within lz4.c/lz4.h.
* The functions you'll likely want are LZ4_compress_default and LZ4_decompress_safe.
* Both of these are documented in the lz4.h header file; I recommend reading them.
*/
/* Dependencies */
#include <stdio.h> // For printf()
#include <string.h> // For memcmp()
#include <stdlib.h> // For exit()
#include "lz4.h" // This is all that is required to expose the prototypes for basic compression and decompression.
/*
* Simple show-error-and-bail function.
*/
void run_screaming(const char* message, const int code) {
printf("%s \n", message);
exit(code);
}
/*
* main
*/
int main(void) {
/* Introduction */
// Below we will have a Compression and Decompression section to demonstrate.
// There are a few important notes before we start:
// 1) The return codes of LZ4_ functions are important.
// Read lz4.h if you're unsure what a given code means.
// 2) LZ4 uses char* pointers in all LZ4_ functions.
// This is baked into the API and not going to change, for consistency.
// If your program uses different pointer types,
// you may need to do some casting or set the right -Wno compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign).
/* Compression */
// We'll store some text into a variable pointed to by *src to be compressed later.
const char* const src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Lorem ipsum dolor site amat.";
// The compression function needs to know how many bytes exist. Since we're using a string, we can use strlen() + 1 (for \0).
const int src_size = (int)(strlen(src) + 1);
// LZ4 provides a function that will tell you the maximum size of compressed output based on input data via LZ4_compressBound().
const int max_dst_size = LZ4_compressBound(src_size);
// We will use that size for our destination boundary when allocating space.
char* compressed_data = malloc((size_t)max_dst_size);
if (compressed_data == NULL)
run_screaming("Failed to allocate memory for *compressed_data.", 1);
// That's all the information and preparation LZ4 needs to compress *src into *compressed_data.
// Invoke LZ4_compress_default now with our size values and pointers to our memory locations.
// Save the return value for error checking.
const int compressed_data_size = LZ4_compress_default(src, compressed_data, src_size, max_dst_size);
// Check return_value to determine what happened.
if (compressed_data_size <= 0)
run_screaming("A 0 or negative result from LZ4_compress_default() indicates a failure trying to compress the data. ", 1);
if (compressed_data_size > 0)
printf("We successfully compressed some data! Ratio: %.2f\n",
(float) compressed_data_size/src_size);
// Not only does a positive return_value mean success, the value returned == the number of bytes required.
// You can use this to realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept.
compressed_data = (char *)realloc(compressed_data, (size_t)compressed_data_size);
if (compressed_data == NULL)
run_screaming("Failed to re-alloc memory for compressed_data. Sad :(", 1);
/* Decompression */
// Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite!
// The decompression will need to know the compressed size, and an upper bound of the decompressed size.
// In this example, we just re-use this information from previous section,
// but in a real-world scenario, metadata must be transmitted to the decompression side.
// Each implementation is in charge of this part. Oftentimes, it adds some header of its own.
// Sometimes, the metadata can be extracted from the local context.
// First, let's create a *new_src location of size src_size since we know that value.
char* const regen_buffer = malloc(src_size);
if (regen_buffer == NULL)
run_screaming("Failed to allocate memory for *regen_buffer.", 1);
// The LZ4_decompress_safe function needs to know where the compressed data is, how many bytes long it is,
// where the regen_buffer memory location is, and how large regen_buffer (uncompressed) output will be.
// Again, save the return_value.
const int decompressed_size = LZ4_decompress_safe(compressed_data, regen_buffer, compressed_data_size, src_size);
free(compressed_data); /* no longer useful */
if (decompressed_size < 0)
run_screaming("A negative result from LZ4_decompress_safe indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", decompressed_size);
if (decompressed_size >= 0)
printf("We successfully decompressed some data!\n");
// Not only does a positive return value mean success,
// value returned == number of bytes regenerated from compressed_data stream.
if (decompressed_size != src_size)
run_screaming("Decompressed data is different from original! \n", 1);
/* Validation */
// We should be able to compare our original *src with our *new_src and be byte-for-byte identical.
if (memcmp(src, regen_buffer, src_size) != 0)
run_screaming("Validation failed. *src and *new_src are not identical.", 1);
printf("Validation done. The string we ended up with is:\n%s\n", regen_buffer);
return 0;
}
examples/streaming_api_basics.md
+87
View File
@@ -0,0 +1,87 @@
# LZ4 Streaming API Basics
by *Takayuki Matsuoka*
## LZ4 API sets
LZ4 has the following API sets :
- "Auto Framing" API (lz4frame.h) :
This is most recommended API for usual application.
It guarantees interoperability with other LZ4 framing format compliant tools/libraries
such as LZ4 command line utility, node-lz4, etc.
- "Block" API : This is recommended for simple purpose.
It compress single raw memory block to LZ4 memory block and vice versa.
- "Streaming" API : This is designed for complex things.
For example, compress huge stream data in restricted memory environment.
Basically, you should use "Auto Framing" API.
But if you want to write advanced application, it's time to use Block or Streaming APIs.
## What is difference between Block and Streaming API ?
Block API (de)compresses a single contiguous memory block.
In other words, LZ4 library finds redundancy from a single contiguous memory block.
Streaming API does same thing but (de)compresses multiple adjacent contiguous memory blocks.
So LZ4 library could find more redundancy than Block API.
The following figure shows difference between API and block sizes.
In these figures, the original data is split into 4KiBytes contiguous chunks.
```
Original Data
+---------------+---------------+----+----+----+
| 4KiB Chunk A | 4KiB Chunk B | C | D |... |
+---------------+---------------+----+----+----+
Example (1) : Block API, 4KiB Block
+---------------+---------------+----+----+----+
| 4KiB Chunk A | 4KiB Chunk B | C | D |... |
+---------------+---------------+----+----+----+
| Block #1 | Block #2 | #3 | #4 |... |
+---------------+---------------+----+----+----+
(No Dependency)
Example (2) : Block API, 8KiB Block
+---------------+---------------+----+----+----+
| 4KiB Chunk A | 4KiB Chunk B | C | D |... |
+---------------+---------------+----+----+----+
| Block #1 |Block #2 |... |
+--------------------+----------+-------+-+----+
^ | ^ |
| | | |
+--------------+ +----+
Internal Dependency Internal Dependency
Example (3) : Streaming API, 4KiB Block
+---------------+---------------+-----+----+----+
| 4KiB Chunk A | 4KiB Chunk B | C | D |... |
+---------------+---------------+-----+----+----+
| Block #1 | Block #2 | #3 | #4 |... |
+---------------+----+----------+-+---+-+--+----+
^ | ^ | ^ |
| | | | | |
+--------------+ +--------+ +---+
Dependency Dependency Dependency
```
- In example (1), there is no dependency.
All blocks are compressed independently.
- In example (2), naturally 8KiBytes block has internal dependency.
But still block #1 and #2 are compressed independently.
- In example (3), block #2 has dependency to #1,
also #3 has dependency to #2 and #1, #4 has #3, #2 and #1, and so on.
Here, we can observe difference between example (2) and (3).
In (2), there's no dependency between chunk B and C, but (3) has dependency between B and C.
This dependency improves compression ratio.
## Restriction of Streaming API
For efficiency, Streaming API doesn't keep a mirror copy of dependent (de)compressed memory.
This means users should keep these dependent (de)compressed memory explicitly.
Usually, "Dependent memory" is previous adjacent contiguous memory up to 64KiBytes.
LZ4 will not access further memories.
lib/Makefile
+1 -1
View File
@@ -28,7 +28,7 @@
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# You can contact the author at :
# - LZ4 source repository : https://github.com/Cyan4973/lz4
# - LZ4 source repository : https://github.com/lz4/lz4
# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
# ################################################################
lib/README.md
+27 -10
View File
@@ -35,21 +35,22 @@ So it's necessary to include all `*.c` and `*.h` files present in `/lib`.
Definitions which are not guaranteed to remain stable in future versions,
are protected behind macros, such as `LZ4_STATIC_LINKING_ONLY`.
As the name implies, these definitions can only be invoked
As the name strongly implies, these definitions should only be invoked
in the context of static linking ***only***.
Otherwise, dependent application may fail on API or ABI break in the future.
The associated symbols are also not present in dynamic library by default.
The associated symbols are also not exposed by the dynamic library by default.
Should they be nonetheless needed, it's possible to force their publication
by using build macro `LZ4_PUBLISH_STATIC_FUNCTIONS`.
by using build macros `LZ4_PUBLISH_STATIC_FUNCTIONS`
and `LZ4F_PUBLISH_STATIC_FUNCTIONS`.
#### Build macros
The following build macro can be selected at compilation time :
The following build macro can be selected to adjust source code behavior at compilation time :
- `LZ4_FAST_DEC_LOOP` : this triggers the optimized decompression loop.
This loops works great on x86/x64 cpus, and is automatically enabled on this platform.
It's possible to enable or disable it manually, by passing `LZ4_FAST_DEC_LOOP=1` or `0` to the preprocessor.
- `LZ4_FAST_DEC_LOOP` : this triggers a speed optimized decompression loop, more powerful on modern cpus.
This loop works great on `x86`, `x64` and `aarch64` cpus, and is automatically enabled for them.
It's also possible to enable or disable it manually, by passing `LZ4_FAST_DEC_LOOP=1` or `0` to the preprocessor.
For example, with `gcc` : `-DLZ4_FAST_DEC_LOOP=1`,
and with `make` : `CPPFLAGS+=-DLZ4_FAST_DEC_LOOP=1 make lz4`.
@@ -65,8 +66,24 @@ The following build macro can be selected at compilation time :
Should this be a problem, it's generally possible to make the compiler ignore these warnings,
for example with `-Wno-deprecated-declarations` on `gcc`,
or `_CRT_SECURE_NO_WARNINGS` for Visual Studio.
Another method is to define `LZ4_DISABLE_DEPRECATE_WARNINGS`
before including the LZ4 header files.
This build macro offers another project-specific method
by defining `LZ4_DISABLE_DEPRECATE_WARNINGS` before including the LZ4 header files.
- `LZ4_USER_MEMORY_FUNCTIONS` : replace calls to <stdlib>'s `malloc`, `calloc` and `free`
by user-defined functions, which must be called `LZ4_malloc()`, `LZ4_calloc()` and `LZ4_free()`.
User functions must be available at link time.
- `LZ4_FORCE_SW_BITCOUNT` : by default, the compression algorithm tries to determine lengths
by using bitcount instructions, generally implemented as fast single instructions in many cpus.
In case the target cpus doesn't support it, or compiler intrinsic doesn't work, or feature bad performance,
it's possible to use an optimized software path instead.
This is achieved by setting this build macros .
In most cases, it's not expected to be necessary,
but it can be legitimately considered for less common platforms.
- `LZ4_ALIGN_TEST` : alignment test ensures that the memory area
passed as argument to become a compression state is suitably aligned.
This test can be disabled if it proves flaky, by setting this value to 0.
#### Amalgamation
@@ -102,7 +119,7 @@ The compiled executable will require LZ4 DLL which is available at `dll\liblz4.d
#### Miscellaneous
Other files present in the directory are not source code. There are :
Other files present in the directory are not source code. They are :
- `LICENSE` : contains the BSD license text
- `Makefile` : `make` script to compile and install lz4 library (static and dynamic)
lib/lz4.c
+327 -230
View File
File diff suppressed because it is too large Load Diff
lib/lz4.h
+90 -80
View File
@@ -100,7 +100,7 @@ extern "C" {
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
@@ -186,7 +186,8 @@ LZ4LIB_API int LZ4_compressBound(int inputSize);
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
An acceleration value of "1" is the same as regular LZ4_compress_default()
Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c).
Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c).
Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c).
*/
LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
@@ -212,7 +213,18 @@ LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* d
* New value is necessarily <= input value.
* @return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
* or 0 if compression fails.
*/
*
* Note : from v1.8.2 to v1.9.1, this function had a bug (fixed un v1.9.2+):
* the produced compressed content could, in specific circumstances,
* require to be decompressed into a destination buffer larger
* by at least 1 byte than the content to decompress.
* If an application uses `LZ4_compress_destSize()`,
* it's highly recommended to update liblz4 to v1.9.2 or better.
* If this can't be done or ensured,
* the receiving decompression function should provide
* a dstCapacity which is > decompressedSize, by at least 1 byte.
* See https://github.com/lz4/lz4/issues/859 for details
*/
LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
@@ -220,25 +232,35 @@ LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePt
* Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',
* into destination buffer 'dst' of size 'dstCapacity'.
* Up to 'targetOutputSize' bytes will be decoded.
* The function stops decoding on reaching this objective,
* which can boost performance when only the beginning of a block is required.
* The function stops decoding on reaching this objective.
* This can be useful to boost performance
* whenever only the beginning of a block is required.
*
* @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity)
* @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize)
* If source stream is detected malformed, function returns a negative result.
*
* Note : @return can be < targetOutputSize, if compressed block contains less data.
* Note 1 : @return can be < targetOutputSize, if compressed block contains less data.
*
* Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity,
* and expects targetOutputSize <= dstCapacity.
* It effectively stops decoding on reaching targetOutputSize,
* Note 2 : targetOutputSize must be <= dstCapacity
*
* Note 3 : this function effectively stops decoding on reaching targetOutputSize,
* so dstCapacity is kind of redundant.
* This is because in a previous version of this function,
* decoding operation would not "break" a sequence in the middle.
* As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize,
* This is because in older versions of this function,
* decoding operation would still write complete sequences.
* Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize,
* it could write more bytes, though only up to dstCapacity.
* Some "margin" used to be required for this operation to work properly.
* This is no longer necessary.
* The function nonetheless keeps its signature, in an effort to not break API.
* Thankfully, this is no longer necessary.
* The function nonetheless keeps the same signature, in an effort to preserve API compatibility.
*
* Note 4 : If srcSize is the exact size of the block,
* then targetOutputSize can be any value,
* including larger than the block's decompressed size.
* The function will, at most, generate block's decompressed size.
*
* Note 5 : If srcSize is _larger_ than block's compressed size,
* then targetOutputSize **MUST** be <= block's decompressed size.
* Otherwise, *silent corruption will occur*.
*/
LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
@@ -547,74 +569,64 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const
#define LZ4_H_98237428734687
/*-************************************************************
* PRIVATE DEFINITIONS
* Private Definitions
**************************************************************
* Do not use these definitions directly.
* They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
* Accessing members will expose code to API and/or ABI break in future versions of the library.
* Accessing members will expose user code to API and/or ABI break in future versions of the library.
**************************************************************/
#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */
#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
#include <stdint.h>
typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
struct LZ4_stream_t_internal {
uint32_t hashTable[LZ4_HASH_SIZE_U32];
uint32_t currentOffset;
uint16_t dirty;
uint16_t tableType;
const uint8_t* dictionary;
const LZ4_stream_t_internal* dictCtx;
uint32_t dictSize;
};
typedef struct {
const uint8_t* externalDict;
size_t extDictSize;
const uint8_t* prefixEnd;
size_t prefixSize;
} LZ4_streamDecode_t_internal;
# include <stdint.h>
typedef int8_t LZ4_i8;
typedef uint8_t LZ4_byte;
typedef uint16_t LZ4_u16;
typedef uint32_t LZ4_u32;
#else
typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
struct LZ4_stream_t_internal {
unsigned int hashTable[LZ4_HASH_SIZE_U32];
unsigned int currentOffset;
unsigned short dirty;
unsigned short tableType;
const unsigned char* dictionary;
const LZ4_stream_t_internal* dictCtx;
unsigned int dictSize;
};
typedef struct {
const unsigned char* externalDict;
const unsigned char* prefixEnd;
size_t extDictSize;
size_t prefixSize;
} LZ4_streamDecode_t_internal;
typedef signed char LZ4_i8;
typedef unsigned char LZ4_byte;
typedef unsigned short LZ4_u16;
typedef unsigned int LZ4_u32;
#endif
typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
struct LZ4_stream_t_internal {
LZ4_u32 hashTable[LZ4_HASH_SIZE_U32];
LZ4_u32 currentOffset;
LZ4_u32 tableType;
const LZ4_byte* dictionary;
const LZ4_stream_t_internal* dictCtx;
LZ4_u32 dictSize;
};
typedef struct {
const LZ4_byte* externalDict;
size_t extDictSize;
const LZ4_byte* prefixEnd;
size_t prefixSize;
} LZ4_streamDecode_t_internal;
/*! LZ4_stream_t :
* information structure to track an LZ4 stream.
* Do not use below internal definitions directly !
* Declare or allocate an LZ4_stream_t instead.
* LZ4_stream_t can also be created using LZ4_createStream(), which is recommended.
* The structure definition can be convenient for static allocation
* (on stack, or as part of larger structure).
* Init this structure with LZ4_initStream() before first use.
* note : only use this definition in association with static linking !
* this definition is not API/ABI safe, and may change in a future version.
* this definition is not API/ABI safe, and may change in future versions.
*/
#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ )
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
#define LZ4_STREAMSIZE 16416 /* static size, for inter-version compatibility */
#define LZ4_STREAMSIZE_VOIDP (LZ4_STREAMSIZE / sizeof(void*))
union LZ4_stream_u {
unsigned long long table[LZ4_STREAMSIZE_U64];
void* table[LZ4_STREAMSIZE_VOIDP];
LZ4_stream_t_internal internal_donotuse;
} ; /* previously typedef'd to LZ4_stream_t */
}; /* previously typedef'd to LZ4_stream_t */
/*! LZ4_initStream() : v1.9.0+
* An LZ4_stream_t structure must be initialized at least once.
@@ -667,22 +679,21 @@ union LZ4_streamDecode_u {
#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
# define LZ4_DEPRECATED(message) /* disable deprecation warnings */
#else
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define LZ4_DEPRECATED(message) [[deprecated(message)]]
# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (LZ4_GCC_VERSION >= 301)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45))
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# else
# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
# define LZ4_DEPRECATED(message)
# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler")
# define LZ4_DEPRECATED(message) /* disabled */
# endif
#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
/* Obsolete compression functions */
/*! Obsolete compression functions (since v1.7.3) */
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize);
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
@@ -690,11 +701,12 @@ LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_co
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Obsolete decompression functions */
/*! Obsolete decompression functions (since v1.8.0) */
LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
/* Obsolete streaming functions; degraded functionality; do not use!
/* Obsolete streaming functions (since v1.7.0)
* degraded functionality; do not use!
*
* In order to perform streaming compression, these functions depended on data
* that is no longer tracked in the state. They have been preserved as well as
@@ -708,23 +720,22 @@ LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStre
LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer);
LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state);
/* Obsolete streaming decoding functions */
/*! Obsolete streaming decoding functions (since v1.7.0) */
LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
/*! LZ4_decompress_fast() : **unsafe!**
/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) :
* These functions used to be faster than LZ4_decompress_safe(),
* but it has changed, and they are now slower than LZ4_decompress_safe().
* but this is no longer the case. They are now slower.
* This is because LZ4_decompress_fast() doesn't know the input size,
* and therefore must progress more cautiously in the input buffer to not read beyond the end of block.
* and therefore must progress more cautiously into the input buffer to not read beyond the end of block.
* On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
* As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
*
* The last remaining LZ4_decompress_fast() specificity is that
* it can decompress a block without knowing its compressed size.
* Such functionality could be achieved in a more secure manner,
* by also providing the maximum size of input buffer,
* but it would require new prototypes, and adaptation of the implementation to this new use case.
* Such functionality can be achieved in a more secure manner
* by employing LZ4_decompress_safe_partial().
*
* Parameters:
* originalSize : is the uncompressed size to regenerate.
@@ -739,7 +750,6 @@ LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4
* But they may happen if input data is invalid (error or intentional tampering).
* As a consequence, use these functions in trusted environments with trusted data **only**.
*/
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead")
LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead")
lib/lz4frame.c
+93 -54
View File
@@ -71,8 +71,8 @@
* towards another library or solution of their choice
* by modifying below section.
*/
#include <stdlib.h> /* malloc, calloc, free */
#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */
# include <stdlib.h> /* malloc, calloc, free */
# define ALLOC(s) malloc(s)
# define ALLOC_AND_ZERO(s) calloc(1,(s))
# define FREEMEM(p) free(p)
@@ -533,7 +533,7 @@ void LZ4F_freeCDict(LZ4F_CDict* cdict)
* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
* Object can release its memory using LZ4F_freeCompressionContext();
*/
LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version)
{
LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t));
if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
@@ -541,20 +541,18 @@ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_c
cctxPtr->version = version;
cctxPtr->cStage = 0; /* Next stage : init stream */
*LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr;
*LZ4F_compressionContextPtr = cctxPtr;
return LZ4F_OK_NoError;
}
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext)
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr)
{
LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext;
if (cctxPtr != NULL) { /* support free on NULL */
FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */
FREEMEM(cctxPtr->lz4CtxPtr); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */
FREEMEM(cctxPtr->tmpBuff);
FREEMEM(LZ4F_compressionContext);
FREEMEM(cctxPtr);
}
return LZ4F_OK_NoError;
@@ -725,6 +723,9 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,
*/
size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
{
if (preferencesPtr && preferencesPtr->autoFlush) {
return LZ4F_compressBound_internal(srcSize, preferencesPtr, 0);
}
return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1);
}
@@ -747,6 +748,7 @@ static size_t LZ4F_makeBlock(void* dst,
(int)(srcSize), (int)(srcSize-1),
level, cdict);
if (cSize == 0) { /* compression failed */
DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
cSize = (U32)srcSize;
LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
memcpy(cSizePtr+BHSize, src, srcSize);
@@ -989,6 +991,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
BYTE* dstPtr = dstStart;
size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
DEBUGLOG(5,"LZ4F_compressEnd: dstCapacity=%u", (unsigned)dstCapacity);
if (LZ4F_isError(flushSize)) return flushSize;
dstPtr += flushSize;
@@ -1002,6 +1005,7 @@ size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {
U32 const xxh = XXH32_digest(&(cctxPtr->xxh));
if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
DEBUGLOG(5,"Writing 32-bit content checksum");
LZ4F_writeLE32(dstPtr, xxh);
dstPtr+=4; /* content Checksum */
}
@@ -1112,6 +1116,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
size_t frameHeaderSize;
const BYTE* srcPtr = (const BYTE*)src;
DEBUGLOG(5, "LZ4F_decodeHeader");
/* need to decode header to get frameInfo */
if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */
MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
@@ -1132,8 +1137,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize
/* control magic number */
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)
if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) {
DEBUGLOG(4, "frame header error : unknown magic number");
return err0r(LZ4F_ERROR_frameType_unknown);
}
#endif
dctx->frameInfo.frameType = LZ4F_frame;
@@ -1282,15 +1289,20 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
/* LZ4F_updateDict() :
* only used for LZ4F_blockLinked mode */
* only used for LZ4F_blockLinked mode
* Condition : dstPtr != NULL
*/
static void LZ4F_updateDict(LZ4F_dctx* dctx,
const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,
unsigned withinTmp)
{
if (dctx->dictSize==0)
dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */
assert(dstPtr != NULL);
if (dctx->dictSize==0) {
dctx->dict = (const BYTE*)dstPtr; /* priority to prefix mode */
}
assert(dctx->dict != NULL);
if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity, directly within dstBuffer */
if (dctx->dict + dctx->dictSize == dstPtr) { /* prefix mode, everything within dstBuffer */
dctx->dictSize += dstSize;
return;
}
@@ -1304,9 +1316,10 @@ static void LZ4F_updateDict(LZ4F_dctx* dctx,
assert(dstSize < 64 KB); /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */
/* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOut */
/* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOutBuffer */
assert(dctx->tmpOutBuffer != NULL);
if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */
if (withinTmp && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */
/* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */
assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart);
dctx->dictSize += dstSize;
@@ -1378,17 +1391,21 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
const BYTE* const srcEnd = srcStart + *srcSizePtr;
const BYTE* srcPtr = srcStart;
BYTE* const dstStart = (BYTE*)dstBuffer;
BYTE* const dstEnd = dstStart + *dstSizePtr;
BYTE* const dstEnd = dstStart ? dstStart + *dstSizePtr : NULL;
BYTE* dstPtr = dstStart;
const BYTE* selectedIn = NULL;
unsigned doAnotherStage = 1;
size_t nextSrcSizeHint = 1;
DEBUGLOG(5, "LZ4F_decompress : %p,%u => %p,%u",
srcBuffer, (unsigned)*srcSizePtr, dstBuffer, (unsigned)*dstSizePtr);
if (dstBuffer == NULL) assert(*dstSizePtr == 0);
MEM_INIT(&optionsNull, 0, sizeof(optionsNull));
if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;
*srcSizePtr = 0;
*dstSizePtr = 0;
assert(dctx != NULL);
/* behaves as a state machine */
@@ -1398,6 +1415,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
{
case dstage_getFrameHeader:
DEBUGLOG(6, "dstage_getFrameHeader");
if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */
if (LZ4F_isError(hSize)) return hSize;
@@ -1411,6 +1429,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
/* fall-through */
case dstage_storeFrameHeader:
DEBUGLOG(6, "dstage_storeFrameHeader");
{ size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr));
memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
dctx->tmpInSize += sizeToCopy;
@@ -1427,6 +1446,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
break;
case dstage_init:
DEBUGLOG(6, "dstage_init");
if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0);
/* internal buffers allocation */
{ size_t const bufferNeeded = dctx->maxBlockSize
@@ -1480,17 +1500,21 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
} /* if (dctx->dStage == dstage_storeBlockHeader) */
/* decode block header */
{ size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
{ U32 const blockHeader = LZ4F_readLE32(selectedIn);
size_t const nextCBlockSize = blockHeader & 0x7FFFFFFFU;
size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize;
if (nextCBlockSize==0) { /* frameEnd signal, no more block */
if (blockHeader==0) { /* frameEnd signal, no more block */
DEBUGLOG(5, "end of frame");
dctx->dStage = dstage_getSuffix;
break;
}
if (nextCBlockSize > dctx->maxBlockSize)
if (nextCBlockSize > dctx->maxBlockSize) {
return err0r(LZ4F_ERROR_maxBlockSize_invalid);
if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
}
if (blockHeader & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
/* next block is uncompressed */
dctx->tmpInTarget = nextCBlockSize;
DEBUGLOG(5, "next block is uncompressed (size %u)", (U32)nextCBlockSize);
if (dctx->frameInfo.blockChecksumFlag) {
(void)XXH32_reset(&dctx->blockChecksum, 0);
}
@@ -1508,20 +1532,26 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
}
case dstage_copyDirect: /* uncompressed block */
{ size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
memcpy(dstPtr, srcPtr, sizeToCopy);
if (dctx->frameInfo.blockChecksumFlag) {
(void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
}
if (dctx->frameInfo.contentChecksumFlag)
(void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
if (dctx->frameInfo.contentSize)
dctx->frameRemainingSize -= sizeToCopy;
DEBUGLOG(6, "dstage_copyDirect");
{ size_t sizeToCopy;
if (dstPtr == NULL) {
sizeToCopy = 0;
} else {
size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
memcpy(dstPtr, srcPtr, sizeToCopy);
if (dctx->frameInfo.blockChecksumFlag) {
(void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
}
if (dctx->frameInfo.contentChecksumFlag)
(void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
if (dctx->frameInfo.contentSize)
dctx->frameRemainingSize -= sizeToCopy;
/* history management (linked blocks only)*/
if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0);
/* history management (linked blocks only)*/
if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0);
} }
srcPtr += sizeToCopy;
dstPtr += sizeToCopy;
@@ -1534,15 +1564,16 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
break;
}
dctx->tmpInTarget -= sizeToCopy; /* need to copy more */
nextSrcSizeHint = dctx->tmpInTarget +
+(dctx->frameInfo.blockChecksumFlag ? BFSize : 0)
+ BHSize /* next header size */;
doAnotherStage = 0;
break;
}
nextSrcSizeHint = dctx->tmpInTarget +
+(dctx->frameInfo.blockChecksumFlag ? BFSize : 0)
+ BHSize /* next header size */;
doAnotherStage = 0;
break;
/* check block checksum for recently transferred uncompressed block */
case dstage_getBlockChecksum:
DEBUGLOG(6, "dstage_getBlockChecksum");
{ const void* crcSrc;
if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) {
crcSrc = srcPtr;
@@ -1562,8 +1593,12 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
{ U32 const readCRC = LZ4F_readLE32(crcSrc);
U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
if (readCRC != calcCRC)
DEBUGLOG(6, "compare block checksum");
if (readCRC != calcCRC) {
DEBUGLOG(4, "incorrect block checksum: %08X != %08X",
readCRC, calcCRC);
return err0r(LZ4F_ERROR_blockChecksum_invalid);
}
#else
(void)readCRC;
(void)calcCRC;
@@ -1573,6 +1608,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
break;
case dstage_getCBlock:
DEBUGLOG(6, "dstage_getCBlock");
if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) {
dctx->tmpInSize = 0;
dctx->dStage = dstage_storeCBlock;
@@ -1582,7 +1618,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
selectedIn = srcPtr;
srcPtr += dctx->tmpInTarget;
if (0) /* jump over next block */
if (0) /* always jump over next block */
case dstage_storeCBlock:
{ size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize;
size_t const inputLeft = (size_t)(srcEnd-srcPtr);
@@ -1619,6 +1655,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
const char* dict = (const char*)dctx->dict;
size_t dictSize = dctx->dictSize;
int decodedSize;
assert(dstPtr != NULL);
if (dict && dictSize > 1 GB) {
/* the dictSize param is an int, avoid truncation / sign issues */
dict += dictSize - 64 KB;
@@ -1636,8 +1673,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
dctx->frameRemainingSize -= (size_t)decodedSize;
/* dictionary management */
if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
if (dctx->frameInfo.blockMode==LZ4F_blockLinked) {
LZ4F_updateDict(dctx, dstPtr, (size_t)decodedSize, dstStart, 0);
}
dstPtr += decodedSize;
dctx->dStage = dstage_getBlockHeader;
@@ -1684,7 +1722,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
/* fall-through */
case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */
{ size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr));
DEBUGLOG(6, "dstage_flushOut");
if (dstPtr != NULL) {
size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr));
memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);
/* dictionary management */
@@ -1693,16 +1733,15 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
dctx->tmpOutStart += sizeToCopy;
dstPtr += sizeToCopy;
if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */
dctx->dStage = dstage_getBlockHeader; /* get next block */
break;
}
/* could not flush everything : stop there, just request a block header */
doAnotherStage = 0;
nextSrcSizeHint = BHSize;
}
if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */
dctx->dStage = dstage_getBlockHeader; /* get next block */
break;
}
/* could not flush everything : stop there, just request a block header */
doAnotherStage = 0;
nextSrcSizeHint = BHSize;
break;
case dstage_getSuffix:
if (dctx->frameRemainingSize)
@@ -1806,6 +1845,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2);
if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */
&& (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */
&& (dctx->dict != NULL) /* dictionary exists */
&& (!decompressOptionsPtr->stableDst) /* cannot rely on dst data to remain there for next call */
&& ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */
{
@@ -1815,9 +1855,9 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
if (dctx->tmpOutSize > 64 KB) copySize = 0;
if (copySize > preserveSize) copySize = preserveSize;
assert(dctx->tmpOutBuffer != NULL);
if (copySize > 0)
memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
dctx->dict = dctx->tmpOutBuffer;
dctx->dictSize = preserveSize + dctx->tmpOutStart;
@@ -1825,8 +1865,7 @@ size_t LZ4F_decompress(LZ4F_dctx* dctx,
const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize;
size_t const newDictSize = MIN(dctx->dictSize, 64 KB);
if (newDictSize > 0)
memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
dctx->dict = dctx->tmpOutBuffer;
dctx->dictSize = newDictSize;
lib/lz4frame.h
+22 -14
View File
@@ -66,17 +66,22 @@ extern "C" {
*****************************************************************/
/* LZ4_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
* LZ4FLIB_API :
* LZ4FLIB_VISIBILITY :
* Control library symbols visibility.
*/
#ifndef LZ4FLIB_VISIBILITY
# if defined(__GNUC__) && (__GNUC__ >= 4)
# define LZ4FLIB_VISIBILITY __attribute__ ((visibility ("default")))
# else
# define LZ4FLIB_VISIBILITY
# endif
#endif
#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
# define LZ4FLIB_API __declspec(dllexport)
# define LZ4FLIB_API __declspec(dllexport) LZ4FLIB_VISIBILITY
#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
# define LZ4FLIB_API __declspec(dllimport)
#elif defined(__GNUC__) && (__GNUC__ >= 4)
# define LZ4FLIB_API __attribute__ ((__visibility__ ("default")))
# define LZ4FLIB_API __declspec(dllimport) LZ4FLIB_VISIBILITY
#else
# define LZ4FLIB_API
# define LZ4FLIB_API LZ4FLIB_VISIBILITY
#endif
#ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS
@@ -103,7 +108,7 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return
/*-************************************
* Frame compression types
**************************************/
************************************* */
/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */
#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x
@@ -113,7 +118,8 @@ LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return
/* The larger the block size, the (slightly) better the compression ratio,
* though there are diminishing returns.
* Larger blocks also increase memory usage on both compression and decompression sides. */
* Larger blocks also increase memory usage on both compression and decompression sides.
*/
typedef enum {
LZ4F_default=0,
LZ4F_max64KB=4,
@@ -284,7 +290,7 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
* @return is always the same for a srcSize and prefsPtr.
* prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
* tech details :
* @return includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.
* @return if automatic flushing is not enabled, includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes.
* It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd().
* @return doesn't include frame header, as it was already generated by LZ4F_compressBegin().
*/
@@ -376,7 +382,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
* note : Frame header size is variable, but is guaranteed to be
* >= LZ4F_HEADER_SIZE_MIN bytes, and <= LZ4F_HEADER_SIZE_MAX bytes.
*/
size_t LZ4F_headerSize(const void* src, size_t srcSize);
LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize);
/*! LZ4F_getFrameInfo() :
* This function extracts frame parameters (max blockSize, dictID, etc.).
@@ -426,8 +432,10 @@ LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx,
const void* srcBuffer, size_t* srcSizePtr);
/*! LZ4F_decompress() :
* Call this function repetitively to regenerate compressed data from `srcBuffer`.
* The function will read up to *srcSizePtr bytes from srcBuffer,
* Call this function repetitively to regenerate data compressed in `srcBuffer`.
*
* The function requires a valid dctx state.
* It will read up to *srcSizePtr bytes from srcBuffer,
* and decompress data into dstBuffer, of capacity *dstSizePtr.
*
* The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value).
@@ -493,9 +501,9 @@ extern "C" {
* Use at your own risk.
*/
#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS
#define LZ4FLIB_STATIC_API LZ4FLIB_API
# define LZ4FLIB_STATIC_API LZ4FLIB_API
#else
#define LZ4FLIB_STATIC_API
# define LZ4FLIB_STATIC_API
#endif
lib/lz4hc.c
+192 -115
View File
@@ -53,7 +53,7 @@
#include "lz4hc.h"
/*=== Common LZ4 definitions ===*/
/*=== Common definitions ===*/
#if defined(__GNUC__)
# pragma GCC diagnostic ignored "-Wunused-function"
#endif
@@ -61,15 +61,16 @@
# pragma clang diagnostic ignored "-Wunused-function"
#endif
/*=== Enums ===*/
typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive;
#define LZ4_COMMONDEFS_ONLY
#ifndef LZ4_SRC_INCLUDED
#include "lz4.c" /* LZ4_count, constants, mem */
#endif
/*=== Enums ===*/
typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive;
/*=== Constants ===*/
#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
#define LZ4_OPT_NUM (1<<12)
@@ -92,7 +93,7 @@ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)
**************************************/
static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4)
{
MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
MEM_INIT(hc4->hashTable, 0, sizeof(hc4->hashTable));
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
}
@@ -161,8 +162,7 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match,
static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern)
{
size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3;
if (bitsToRotate == 0)
return pattern;
if (bitsToRotate == 0) return pattern;
return LZ4HC_rotl32(pattern, (int)bitsToRotate);
}
@@ -172,7 +172,8 @@ static unsigned
LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32)
{
const BYTE* const iStart = ip;
reg_t const pattern = (sizeof(pattern)==8) ? (reg_t)pattern32 + (((reg_t)pattern32) << 32) : pattern32;
reg_t const pattern = (sizeof(pattern)==8) ?
(reg_t)pattern32 + (((reg_t)pattern32) << (sizeof(pattern)*4)) : pattern32;
while (likely(ip < iEnd-(sizeof(pattern)-1))) {
reg_t const diff = LZ4_read_ARCH(ip) ^ pattern;
@@ -270,7 +271,7 @@ LZ4HC_InsertAndGetWiderMatch (
DEBUGLOG(7, "First match at index %u / %u (lowestMatchIndex)",
matchIndex, lowestMatchIndex);
while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) {
while ((matchIndex>=lowestMatchIndex) && (nbAttempts>0)) {
int matchLength=0;
nbAttempts--;
assert(matchIndex < ipIndex);
@@ -389,8 +390,8 @@ LZ4HC_InsertAndGetWiderMatch (
if (lookBackLength==0) { /* no back possible */
size_t const maxML = MIN(currentSegmentLength, srcPatternLength);
if ((size_t)longest < maxML) {
assert(base + matchIndex < ip);
if (ip - (base+matchIndex) > LZ4_DISTANCE_MAX) break;
assert(base + matchIndex != ip);
if ((size_t)(ip - base) - matchIndex > LZ4_DISTANCE_MAX) break;
assert(maxML < 2 GB);
longest = (int)maxML;
*matchpos = base + matchIndex; /* virtual pos, relative to ip, to retrieve offset */
@@ -410,7 +411,7 @@ LZ4HC_InsertAndGetWiderMatch (
} /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */
if ( dict == usingDictCtxHc
&& nbAttempts
&& nbAttempts > 0
&& ipIndex - lowestMatchIndex < LZ4_DISTANCE_MAX) {
size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->base);
U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)];
@@ -460,74 +461,90 @@ int LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index tabl
* @return : 0 if ok,
* 1 if buffer issue detected */
LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
const BYTE** ip,
BYTE** op,
const BYTE** anchor,
const BYTE** _ip,
BYTE** _op,
const BYTE** _anchor,
int matchLength,
const BYTE* const match,
limitedOutput_directive limit,
BYTE* oend)
{
#define ip (*_ip)
#define op (*_op)
#define anchor (*_anchor)
size_t length;
BYTE* const token = (*op)++;
BYTE* const token = op++;
#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6)
static const BYTE* start = NULL;
static U32 totalCost = 0;
U32 const pos = (start==NULL) ? 0 : (U32)(*anchor - start);
U32 const ll = (U32)(*ip - *anchor);
U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start);
U32 const ll = (U32)(ip - anchor);
U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0;
U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
if (start==NULL) start = *anchor; /* only works for single segment */
if (start==NULL) start = anchor; /* only works for single segment */
/* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */
DEBUGLOG(6, "pos:%7u -- literals:%3u, match:%4i, offset:%5u, cost:%3u + %u",
DEBUGLOG(6, "pos:%7u -- literals:%4u, match:%4i, offset:%5u, cost:%4u + %5u",
pos,
(U32)(*ip - *anchor), matchLength, (U32)(*ip-match),
(U32)(ip - anchor), matchLength, (U32)(ip-match),
cost, totalCost);
totalCost += cost;
#endif
/* Encode Literal length */
length = (size_t)(*ip - *anchor);
if ((limit) && ((*op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
length = (size_t)(ip - anchor);
LZ4_STATIC_ASSERT(notLimited == 0);
/* Check output limit */
if (limit && ((op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) {
DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)",
(int)length, (int)(oend - op));
return 1;
}
if (length >= RUN_MASK) {
size_t len = length - RUN_MASK;
*token = (RUN_MASK << ML_BITS);
for(; len >= 255 ; len -= 255) *(*op)++ = 255;
*(*op)++ = (BYTE)len;
for(; len >= 255 ; len -= 255) *op++ = 255;
*op++ = (BYTE)len;
} else {
*token = (BYTE)(length << ML_BITS);
}
/* Copy Literals */
LZ4_wildCopy8(*op, *anchor, (*op) + length);
*op += length;
LZ4_wildCopy8(op, anchor, op + length);
op += length;
/* Encode Offset */
assert( (*ip - match) <= LZ4_DISTANCE_MAX ); /* note : consider providing offset as a value, rather than as a pointer difference */
LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
assert( (ip - match) <= LZ4_DISTANCE_MAX ); /* note : consider providing offset as a value, rather than as a pointer difference */
LZ4_writeLE16(op, (U16)(ip - match)); op += 2;
/* Encode MatchLength */
assert(matchLength >= MINMATCH);
length = (size_t)matchLength - MINMATCH;
if ((limit) && (*op + (length / 255) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
if (limit && (op + (length / 255) + (1 + LASTLITERALS) > oend)) {
DEBUGLOG(6, "Not enough room to write match length");
return 1; /* Check output limit */
}
if (length >= ML_MASK) {
*token += ML_MASK;
length -= ML_MASK;
for(; length >= 510 ; length -= 510) { *(*op)++ = 255; *(*op)++ = 255; }
if (length >= 255) { length -= 255; *(*op)++ = 255; }
*(*op)++ = (BYTE)length;
for(; length >= 510 ; length -= 510) { *op++ = 255; *op++ = 255; }
if (length >= 255) { length -= 255; *op++ = 255; }
*op++ = (BYTE)length;
} else {
*token += (BYTE)(length);
}
/* Prepare next loop */
*ip += matchLength;
*anchor = *ip;
ip += matchLength;
anchor = ip;
return 0;
}
#undef ip
#undef op
#undef anchor
LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
LZ4HC_CCtx_internal* const ctx,
@@ -535,7 +552,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
char* const dest,
int* srcSizePtr,
int const maxOutputSize,
unsigned maxNbAttempts,
int maxNbAttempts,
const limitedOutput_directive limit,
const dictCtx_directive dict
)
@@ -565,7 +582,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
/* init */
*srcSizePtr = 0;
if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* Main Loop */
while (ip <= mflimit) {
@@ -637,7 +654,11 @@ _Search3:
if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, limit, oend)) goto _dest_overflow;
ip = start2;
optr = op;
if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) goto _dest_overflow;
if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml2, ref2, limit, oend)) {
ml = ml2;
ref = ref2;
goto _dest_overflow;
}
continue;
}
@@ -709,17 +730,18 @@ _Search3:
_last_literals:
/* Encode Last Literals */
{ size_t lastRunSize = (size_t)(iend - anchor); /* literals */
size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
size_t const totalSize = 1 + litLength + lastRunSize;
size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255;
size_t const totalSize = 1 + llAdd + lastRunSize;
if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */
if (limit && (op + totalSize > oend)) {
if (limit == limitedOutput) return 0; /* Check output limit */
if (limit == limitedOutput) return 0;
/* adapt lastRunSize to fill 'dest' */
lastRunSize = (size_t)(oend - op) - 1;
litLength = (lastRunSize + 255 - RUN_MASK) / 255;
lastRunSize -= litLength;
lastRunSize = (size_t)(oend - op) - 1 /*token*/;
llAdd = (lastRunSize + 256 - RUN_MASK) / 256;
lastRunSize -= llAdd;
}
ip = anchor + lastRunSize;
DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize);
ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */
if (lastRunSize >= RUN_MASK) {
size_t accumulator = lastRunSize - RUN_MASK;
@@ -739,9 +761,25 @@ _last_literals:
_dest_overflow:
if (limit == fillOutput) {
/* Assumption : ip, anchor, ml and ref must be set correctly */
size_t const ll = (size_t)(ip - anchor);
size_t const ll_addbytes = (ll + 240) / 255;
size_t const ll_totalCost = 1 + ll_addbytes + ll;
BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */
DEBUGLOG(6, "Last sequence overflowing");
op = optr; /* restore correct out pointer */
if (op + ll_totalCost <= maxLitPos) {
/* ll validated; now adjust match length */
size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost));
size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255);
assert(maxMlSize < INT_MAX); assert(ml >= 0);
if ((size_t)ml > maxMlSize) ml = (int)maxMlSize;
if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ml >= MFLIMIT) {
LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ref, notLimited, oend);
} }
goto _last_literals;
}
/* compression failed */
return 0;
}
@@ -752,7 +790,7 @@ static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,
int const nbSearches, size_t sufficient_len,
const limitedOutput_directive limit, int const fullUpdate,
const dictCtx_directive dict,
HCfavor_e favorDecSpeed);
const HCfavor_e favorDecSpeed);
LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
@@ -769,7 +807,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
typedef enum { lz4hc, lz4opt } lz4hc_strat_e;
typedef struct {
lz4hc_strat_e strat;
U32 nbSearches;
int nbSearches;
U32 targetLength;
} cParams_t;
static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = {
@@ -788,7 +826,8 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
{ lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */
};
DEBUGLOG(4, "LZ4HC_compress_generic(ctx=%p, src=%p, srcSize=%d)", ctx, src, *srcSizePtr);
DEBUGLOG(4, "LZ4HC_compress_generic(ctx=%p, src=%p, srcSize=%d, limit=%d)",
ctx, src, *srcSizePtr, limit);
if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */
if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */
@@ -808,7 +847,7 @@ LZ4_FORCE_INLINE int LZ4HC_compress_generic_internal (
assert(cParam.strat == lz4opt);
result = LZ4HC_compress_optimal(ctx,
src, dst, srcSizePtr, dstCapacity,
(int)cParam.nbSearches, cParam.targetLength, limit,
cParam.nbSearches, cParam.targetLength, limit,
cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */
dict, favor);
}
@@ -881,27 +920,22 @@ LZ4HC_compress_generic (
int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); }
#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :
* it reports an aligment of 8-bytes,
* while actually aligning LZ4_streamHC_t on 4 bytes. */
static size_t LZ4_streamHC_t_alignment(void)
{
struct { char c; LZ4_streamHC_t t; } t_a;
return sizeof(t_a) - sizeof(t_a.t);
}
#if LZ4_ALIGN_TEST
typedef struct { char c; LZ4_streamHC_t t; } t_a;
return sizeof(t_a) - sizeof(LZ4_streamHC_t);
#else
return 1; /* effectively disabled */
#endif
}
/* state is presumed correctly initialized,
* in which case its size and alignment have already been validate */
int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel)
{
LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse;
#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :
* it reports an aligment of 8-bytes,
* while actually aligning LZ4_streamHC_t on 4 bytes. */
assert(((size_t)state & (LZ4_streamHC_t_alignment() - 1)) == 0); /* check alignment */
#endif
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
if (!LZ4_isAligned(state, LZ4_streamHC_t_alignment())) return 0;
LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel);
LZ4HC_init_internal (ctx, (const BYTE*)src);
if (dstCapacity < LZ4_compressBound(srcSize))
@@ -950,10 +984,11 @@ int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* s
/* allocation */
LZ4_streamHC_t* LZ4_createStreamHC(void)
{
LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t));
if (LZ4_streamHCPtr==NULL) return NULL;
LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); /* full initialization, malloc'ed buffer can be full of garbage */
return LZ4_streamHCPtr;
LZ4_streamHC_t* const state =
(LZ4_streamHC_t*)ALLOC_AND_ZERO(sizeof(LZ4_streamHC_t));
if (state == NULL) return NULL;
LZ4_setCompressionLevel(state, LZ4HC_CLEVEL_DEFAULT);
return state;
}
int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr)
@@ -968,22 +1003,16 @@ int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr)
LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size)
{
LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer;
if (buffer == NULL) return NULL;
if (size < sizeof(LZ4_streamHC_t)) return NULL;
#ifndef _MSC_VER /* for some reason, Visual fails the aligment test on 32-bit x86 :
* it reports an aligment of 8-bytes,
* while actually aligning LZ4_streamHC_t on 4 bytes. */
if (((size_t)buffer) & (LZ4_streamHC_t_alignment() - 1)) return NULL; /* alignment check */
#endif
/* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= LZ4_STREAMHCSIZE);
DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", LZ4_streamHCPtr, (unsigned)size);
/* end-base will trigger a clearTable on starting compression */
LZ4_streamHCPtr->internal_donotuse.end = (const BYTE *)(ptrdiff_t)-1;
LZ4_streamHCPtr->internal_donotuse.base = NULL;
LZ4_streamHCPtr->internal_donotuse.dictCtx = NULL;
LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = 0;
LZ4_streamHCPtr->internal_donotuse.dirty = 0;
DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", buffer, (unsigned)size);
/* check conditions */
if (buffer == NULL) return NULL;
if (size < sizeof(LZ4_streamHC_t)) return NULL;
if (!LZ4_isAligned(buffer, LZ4_streamHC_t_alignment())) return NULL;
/* init */
{ LZ4HC_CCtx_internal* const hcstate = &(LZ4_streamHCPtr->internal_donotuse);
MEM_INIT(hcstate, 0, sizeof(*hcstate)); }
LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT);
return LZ4_streamHCPtr;
}
@@ -1028,7 +1057,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr,
const char* dictionary, int dictSize)
{
LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
DEBUGLOG(4, "LZ4_loadDictHC(%p, %p, %d)", LZ4_streamHCPtr, dictionary, dictSize);
DEBUGLOG(4, "LZ4_loadDictHC(ctx:%p, dict:%p, dictSize:%d)", LZ4_streamHCPtr, dictionary, dictSize);
assert(LZ4_streamHCPtr != NULL);
if (dictSize > 64 KB) {
dictionary += (size_t)dictSize - 64 KB;
@@ -1069,14 +1098,15 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl
ctxPtr->dictCtx = NULL;
}
static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
const char* src, char* dst,
int* srcSizePtr, int dstCapacity,
limitedOutput_directive limit)
static int
LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
const char* src, char* dst,
int* srcSizePtr, int dstCapacity,
limitedOutput_directive limit)
{
LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
DEBUGLOG(4, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d)",
LZ4_streamHCPtr, src, *srcSizePtr);
DEBUGLOG(5, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d, limit=%d)",
LZ4_streamHCPtr, src, *srcSizePtr, limit);
assert(ctxPtr != NULL);
/* auto-init if forgotten */
if (ctxPtr->base == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src);
@@ -1100,8 +1130,7 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
}
}
} }
return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit);
}
@@ -1121,23 +1150,30 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
/* dictionary saving */
/* LZ4_saveDictHC :
* save history content
* into a user-provided buffer
* which is then used to continue compression
*/
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
{
LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
DEBUGLOG(4, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
assert(prefixSize >= 0);
if (dictSize > 64 KB) dictSize = 64 KB;
if (dictSize < 4) dictSize = 0;
if (dictSize > prefixSize) dictSize = prefixSize;
memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
if (safeBuffer == NULL) assert(dictSize == 0);
if (dictSize > 0)
memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
{ U32 const endIndex = (U32)(streamPtr->end - streamPtr->base);
streamPtr->end = (const BYTE*)safeBuffer + dictSize;
streamPtr->base = streamPtr->end - endIndex;
streamPtr->dictLimit = endIndex - (U32)dictSize;
streamPtr->lowLimit = endIndex - (U32)dictSize;
if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit;
if (streamPtr->nextToUpdate < streamPtr->dictLimit)
streamPtr->nextToUpdate = streamPtr->dictLimit;
}
return dictSize;
}
@@ -1287,8 +1323,13 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
const dictCtx_directive dict,
const HCfavor_e favorDecSpeed)
{
int retval = 0;
#define TRAILING_LITERALS 3
#ifdef LZ4HC_HEAPMODE
LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)ALLOC(sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS));
#else
LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */
#endif
const BYTE* ip = (const BYTE*) source;
const BYTE* anchor = ip;
@@ -1298,15 +1339,19 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
BYTE* op = (BYTE*) dst;
BYTE* opSaved = (BYTE*) dst;
BYTE* oend = op + dstCapacity;
int ovml = MINMATCH; /* overflow - last sequence */
const BYTE* ovref = NULL;
/* init */
#ifdef LZ4HC_HEAPMODE
if (opt == NULL) goto _return_label;
#endif
DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity);
*srcSizePtr = 0;
if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
/* Main Loop */
assert(ip - anchor < LZ4_MAX_INPUT_SIZE);
while (ip <= mflimit) {
int const llen = (int)(ip - anchor);
int best_mlen, best_off;
@@ -1320,8 +1365,11 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
int const firstML = firstMatch.len;
const BYTE* const matchPos = ip - firstMatch.off;
opSaved = op;
if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */
if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, matchPos, limit, oend) ) { /* updates ip, op and anchor */
ovml = firstML;
ovref = matchPos;
goto _dest_overflow;
}
continue;
}
@@ -1463,7 +1511,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
best_off = opt[last_match_pos].off;
cur = last_match_pos - best_mlen;
encode: /* cur, last_match_pos, best_mlen, best_off must be set */
encode: /* cur, last_match_pos, best_mlen, best_off must be set */
assert(cur < LZ4_OPT_NUM);
assert(last_match_pos >= 1); /* == 1 when only one candidate */
DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos);
@@ -1493,25 +1541,31 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
assert(ml >= MINMATCH);
assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX));
opSaved = op;
if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */
if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, ip - offset, limit, oend) ) { /* updates ip, op and anchor */
ovml = ml;
ovref = ip - offset;
goto _dest_overflow;
} }
} } }
} /* while (ip <= mflimit) */
_last_literals:
_last_literals:
/* Encode Last Literals */
{ size_t lastRunSize = (size_t)(iend - anchor); /* literals */
size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
size_t const totalSize = 1 + litLength + lastRunSize;
size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255;
size_t const totalSize = 1 + llAdd + lastRunSize;
if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */
if (limit && (op + totalSize > oend)) {
if (limit == limitedOutput) return 0; /* Check output limit */
if (limit == limitedOutput) { /* Check output limit */
retval = 0;
goto _return_label;
}
/* adapt lastRunSize to fill 'dst' */
lastRunSize = (size_t)(oend - op) - 1;
litLength = (lastRunSize + 255 - RUN_MASK) / 255;
lastRunSize -= litLength;
lastRunSize = (size_t)(oend - op) - 1 /*token*/;
llAdd = (lastRunSize + 256 - RUN_MASK) / 256;
lastRunSize -= llAdd;
}
ip = anchor + lastRunSize;
DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize);
ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */
if (lastRunSize >= RUN_MASK) {
size_t accumulator = lastRunSize - RUN_MASK;
@@ -1527,12 +1581,35 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
/* End */
*srcSizePtr = (int) (((const char*)ip) - source);
return (int) ((char*)op-dst);
retval = (int) ((char*)op-dst);
goto _return_label;
_dest_overflow:
if (limit == fillOutput) {
op = opSaved; /* restore correct out pointer */
goto _last_literals;
}
return 0;
}
_dest_overflow:
if (limit == fillOutput) {
/* Assumption : ip, anchor, ovml and ovref must be set correctly */
size_t const ll = (size_t)(ip - anchor);
size_t const ll_addbytes = (ll + 240) / 255;
size_t const ll_totalCost = 1 + ll_addbytes + ll;
BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */
DEBUGLOG(6, "Last sequence overflowing (only %i bytes remaining)", (int)(oend-1-opSaved));
op = opSaved; /* restore correct out pointer */
if (op + ll_totalCost <= maxLitPos) {
/* ll validated; now adjust match length */
size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost));
size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255);
assert(maxMlSize < INT_MAX); assert(ovml >= 0);
if ((size_t)ovml > maxMlSize) ovml = (int)maxMlSize;
if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ovml >= MFLIMIT) {
DEBUGLOG(6, "Space to end : %i + ml (%i)", (int)((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1), ovml);
DEBUGLOG(6, "Before : ip = %p, anchor = %p", ip, anchor);
LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ovml, ovref, notLimited, oend);
DEBUGLOG(6, "After : ip = %p, anchor = %p", ip, anchor);
} }
goto _last_literals;
}
_return_label:
#ifdef LZ4HC_HEAPMODE
FREEMEM(opt);
#endif
return retval;
}
lib/lz4hc.h
+15 -40
View File
@@ -198,57 +198,32 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1)
#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
#include <stdint.h>
typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
struct LZ4HC_CCtx_internal
{
uint32_t hashTable[LZ4HC_HASHTABLESIZE];
uint16_t chainTable[LZ4HC_MAXD];
const uint8_t* end; /* next block here to continue on current prefix */
const uint8_t* base; /* All index relative to this position */
const uint8_t* dictBase; /* alternate base for extDict */
uint32_t dictLimit; /* below that point, need extDict */
uint32_t lowLimit; /* below that point, no more dict */
uint32_t nextToUpdate; /* index from which to continue dictionary update */
short compressionLevel;
int8_t favorDecSpeed; /* favor decompression speed if this flag set,
otherwise, favor compression ratio */
int8_t dirty; /* stream has to be fully reset if this flag is set */
LZ4_u32 hashTable[LZ4HC_HASHTABLESIZE];
LZ4_u16 chainTable[LZ4HC_MAXD];
const LZ4_byte* end; /* next block here to continue on current prefix */
const LZ4_byte* base; /* All index relative to this position */
const LZ4_byte* dictBase; /* alternate base for extDict */
LZ4_u32 dictLimit; /* below that point, need extDict */
LZ4_u32 lowLimit; /* below that point, no more dict */
LZ4_u32 nextToUpdate; /* index from which to continue dictionary update */
short compressionLevel;
LZ4_i8 favorDecSpeed; /* favor decompression speed if this flag set,
otherwise, favor compression ratio */
LZ4_i8 dirty; /* stream has to be fully reset if this flag is set */
const LZ4HC_CCtx_internal* dictCtx;
};
#else
typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
struct LZ4HC_CCtx_internal
{
unsigned int hashTable[LZ4HC_HASHTABLESIZE];
unsigned short chainTable[LZ4HC_MAXD];
const unsigned char* end; /* next block here to continue on current prefix */
const unsigned char* base; /* All index relative to this position */
const unsigned char* dictBase; /* alternate base for extDict */
unsigned int dictLimit; /* below that point, need extDict */
unsigned int lowLimit; /* below that point, no more dict */
unsigned int nextToUpdate; /* index from which to continue dictionary update */
short compressionLevel;
char favorDecSpeed; /* favor decompression speed if this flag set,
otherwise, favor compression ratio */
char dirty; /* stream has to be fully reset if this flag is set */
const LZ4HC_CCtx_internal* dictCtx;
};
#endif
/* Do not use these definitions directly !
* Declare or allocate an LZ4_streamHC_t instead.
*/
#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56 + ((sizeof(void*)==16) ? 56 : 0) /* AS400*/ ) /* 262200 or 262256*/
#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
#define LZ4_STREAMHCSIZE 262200 /* static size, for inter-version compatibility */
#define LZ4_STREAMHCSIZE_VOIDP (LZ4_STREAMHCSIZE / sizeof(void*))
union LZ4_streamHC_u {
size_t table[LZ4_STREAMHCSIZE_SIZET];
void* table[LZ4_STREAMHCSIZE_VOIDP];
LZ4HC_CCtx_internal internal_donotuse;
}; /* previously typedef'd to LZ4_streamHC_t */
ossfuzz/Makefile
+12 -8
View File
@@ -26,7 +26,7 @@
# ##########################################################################
LZ4DIR := ../lib
LIB_FUZZING_ENGINE ?= standaloneengine.o
LIB_FUZZING_ENGINE ?=
DEBUGLEVEL?= 1
DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
@@ -47,6 +47,7 @@ FUZZERS := \
round_trip_frame_fuzzer \
decompress_frame_fuzzer
.PHONY: all
all: $(FUZZERS)
# Include a rule to build the static library if calling this target
@@ -58,17 +59,20 @@ $(LZ4DIR)/liblz4.a:
$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) $< -o $@
# Generic rule for generating fuzzers
%_fuzzer: %_fuzzer.o lz4_helpers.o $(LZ4DIR)/liblz4.a
# Compile the standalone code just in case. The OSS-Fuzz code might
# override the LIB_FUZZING_ENGINE value to "-fsanitize=fuzzer"
$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) standaloneengine.c -o standaloneengine.o
# Now compile the actual fuzzer.
ifeq ($(LIB_FUZZING_ENGINE),)
LIB_FUZZING_DEPS := standaloneengine.o
else
LIB_FUZZING_DEPS :=
endif
%_fuzzer: %_fuzzer.o lz4_helpers.o fuzz_data_producer.o $(LZ4DIR)/liblz4.a $(LIB_FUZZING_DEPS)
$(CXX) $(LZ4_CXXFLAGS) $(LZ4_CPPFLAGS) $(LDFLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT)
%_fuzzer_clean:
$(RM) $*_fuzzer $*_fuzzer.o standaloneengine.o
.PHONY: clean
clean: compress_fuzzer_clean decompress_fuzzer_clean
clean: compress_fuzzer_clean decompress_fuzzer_clean \
compress_frame_fuzzer_clean compress_hc_fuzzer_clean \
decompress_frame_fuzzer_clean round_trip_frame_fuzzer_clean \
round_trip_fuzzer_clean round_trip_hc_fuzzer_clean round_trip_stream_fuzzer_clean
$(MAKE) -C $(LZ4DIR) clean
ossfuzz/compress_frame_fuzzer.c
+11 -5
View File
@@ -13,18 +13,23 @@
#include "lz4.h"
#include "lz4frame.h"
#include "lz4_helpers.h"
#include "fuzz_data_producer.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
uint32_t seed = FUZZ_seed(&data, &size);
LZ4F_preferences_t const prefs = FUZZ_randomPreferences(&seed);
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
LZ4F_preferences_t const prefs = FUZZ_dataProducer_preferences(producer);
size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const compressBound = LZ4F_compressFrameBound(size, &prefs);
size_t const dstCapacity = FUZZ_rand32(&seed, 0, compressBound);
size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, compressBound);
char* const dst = (char*)malloc(dstCapacity);
char* const rt = (char*)malloc(size);
FUZZ_ASSERT(dst);
FUZZ_ASSERT(rt);
FUZZ_ASSERT(dst!=NULL);
FUZZ_ASSERT(rt!=NULL);
/* If compression succeeds it must round trip correctly. */
size_t const dstSize =
@@ -37,6 +42,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
free(dst);
free(rt);
FUZZ_dataProducer_free(producer);
return 0;
}
ossfuzz/compress_fuzzer.c
+9 -2
View File
@@ -10,12 +10,18 @@
#include <string.h>
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
#include "lz4.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
uint32_t seed = FUZZ_seed(&data, &size);
size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size));
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const compressBound = LZ4_compressBound(size);
size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, compressBound);
char* const dst = (char*)malloc(dstCapacity);
char* const rt = (char*)malloc(size);
@@ -46,6 +52,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
free(dst);
free(rt);
FUZZ_dataProducer_free(producer);
return 0;
}
ossfuzz/compress_hc_fuzzer.c
+10 -3
View File
@@ -10,16 +10,22 @@
#include <string.h>
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
#include "lz4.h"
#include "lz4hc.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
uint32_t seed = FUZZ_seed(&data, &size);
size_t const dstCapacity = FUZZ_rand32(&seed, 0, LZ4_compressBound(size));
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size_t const levelSeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, size);
int const level = FUZZ_getRange_from_uint32(levelSeed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
char* const dst = (char*)malloc(dstCapacity);
char* const rt = (char*)malloc(size);
int const level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
FUZZ_ASSERT(dst);
FUZZ_ASSERT(rt);
@@ -52,6 +58,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
free(dst);
free(rt);
FUZZ_dataProducer_free(producer);
return 0;
}
ossfuzz/decompress_frame_fuzzer.c
+11 -3
View File
@@ -9,6 +9,7 @@
#include <string.h>
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
#include "lz4.h"
#define LZ4F_STATIC_LINKING_ONLY
#include "lz4frame.h"
@@ -29,11 +30,17 @@ static void decompress(LZ4F_dctx* dctx, void* dst, size_t dstCapacity,
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size_t const dictSizeSeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
uint32_t seed = FUZZ_seed(&data, &size);
size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size);
size_t const dstCapacity = FUZZ_getRange_from_uint32(
dstCapacitySeed, 0, 4 * size);
size_t const largeDictSize = 64 * 1024;
size_t const dictSize = FUZZ_rand32(&seed, 0, largeDictSize);
size_t const dictSize = FUZZ_getRange_from_uint32(
dictSizeSeed, 0, largeDictSize);
char* const dst = (char*)malloc(dstCapacity);
char* const dict = (char*)malloc(dictSize);
LZ4F_decompressOptions_t opts;
@@ -62,6 +69,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
LZ4F_freeDecompressionContext(dctx);
free(dst);
free(dict);
FUZZ_dataProducer_free(producer);
return 0;
}
ossfuzz/decompress_fuzzer.c
+6 -2
View File
@@ -9,13 +9,16 @@
#include <string.h>
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
#include "lz4.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
uint32_t seed = FUZZ_seed(&data, &size);
size_t const dstCapacity = FUZZ_rand32(&seed, 0, 4 * size);
size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, 4 * size);
size_t const smallDictSize = size + 1;
size_t const largeDictSize = 64 * 1024 - 1;
size_t const dictSize = MAX(smallDictSize, largeDictSize);
@@ -53,6 +56,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
dstCapacity, dstCapacity);
free(dst);
free(dict);
FUZZ_dataProducer_free(producer);
return 0;
}

Some files were not shown because too many files have changed in this diff Show More