Compare commits

...
Author SHA1 Message Date
Tim Makram Ghatas f0b078bcd1 add export command for c2 profiles 2024-04-02 22:28:47 +02:00
Ronan Kervella 9f4988e12a Merge pull request #1622 from BishopFox/bugfix_merge_issue
re-add deleted commands from bad merge for c2profiles and implant staging
2024-04-02 10:11:07 -07:00
Tim Makram Ghatas 124c12bebd re-add deleted commands from bad merge 2024-04-02 18:06:18 +02:00
Joe 142bf83ffd Merge pull request #1620 from BishopFox/77cedependabot/github_actions/actions/configure-pages-5
Bump actions/configure-pages from 4 to 5
2024-04-01 06:49:08 -07:00
dependabot[bot] b290474e5d Bump actions/configure-pages from 4 to 5
Bumps [actions/configure-pages](https://github.com/actions/configure-pages) from 4 to 5.
- [Release notes](https://github.com/actions/configure-pages/releases)
- [Commits](https://github.com/actions/configure-pages/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/configure-pages
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-04-01 03:28:25 +00:00
Joe c18fc567a3 Merge pull request #1617 from BishopFox/fix-msf-stager
Fix hang when calling generate stager
2024-03-27 07:36:32 -07:00
rkervella 5bd1f4727c Fix hang when calling generate stager 2024-03-26 15:55:35 -07:00
Joe 619d4994da Merge pull request #1612 from BishopFox/c64bdependabot/go_modules/golang.org/x/crypto-0.21.0
Bump golang.org/x/crypto from 0.17.0 to 0.21.0
2024-03-26 06:55:33 -07:00
dependabot[bot] 1666d6f5ad Bump golang.org/x/crypto from 0.17.0 to 0.21.0
Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.17.0 to 0.21.0.
- [Commits](https://github.com/golang/crypto/compare/v0.17.0...v0.21.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-03-25 03:44:43 +00:00
Joe 6443aaba3d Merge pull request #1597 from BishopFox/48d6dependabot/go_modules/google.golang.org/protobuf-1.33.0
Bump google.golang.org/protobuf from 1.31.0 to 1.33.0
2024-03-19 10:21:19 -07:00
dependabot[bot] e14619fffc Bump google.golang.org/protobuf from 1.31.0 to 1.33.0
Bumps google.golang.org/protobuf from 1.31.0 to 1.33.0.

---
updated-dependencies:
- dependency-name: google.golang.org/protobuf
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-03-19 15:46:19 +00:00
Joe 07ec53e148 Merge pull request #1599 from BishopFox/88cbdependabot/go_modules/github.com/jackc/pgx/v5-5.5.4
Bump github.com/jackc/pgx/v5 from 5.3.1 to 5.5.4
2024-03-19 08:44:52 -07:00
dependabot[bot] 1c9d11f967 Bump github.com/jackc/pgx/v5 from 5.3.1 to 5.5.4
Bumps [github.com/jackc/pgx/v5](https://github.com/jackc/pgx) from 5.3.1 to 5.5.4.
- [Changelog](https://github.com/jackc/pgx/blob/master/CHANGELOG.md)
- [Commits](https://github.com/jackc/pgx/compare/v5.3.1...v5.5.4)

---
updated-dependencies:
- dependency-name: github.com/jackc/pgx/v5
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-03-14 22:01:41 +00:00
Ronan Kervella 85b949a7df Merge pull request #1545 from precurse-bf/ifns
Adding Linux namespace support to ifconfig
2024-03-14 14:57:23 -07:00
Andrew Klaus 2c0d46278a Adding namespace support to Linux ifconfig 2024-03-14 12:01:25 -06:00
Ronan Kervella 74653968df Merge pull request #1596 from precurse-bf/mount_linux_rebase
Adding mount command for Windows and Linux
2024-03-14 09:44:16 -07:00
Andrew Klaus 49b55c7763 Testing fix for Windows implant 2024-03-14 10:07:24 -06:00
Andrew Klaus d625c8c2b7 Updating protobufs 2024-03-13 14:55:04 -06:00
Andrew Klaus 9f33e8103a Adding linux mount command 2024-03-13 14:55:04 -06:00
Raf abd375f501 Adding the mount command and the implementation for Windows implants 2024-03-13 14:24:19 -06:00
moloch-- 715c37addf Remove prelude operator support, since product is discontinued 2024-03-07 16:07:47 -08:00
Joe 754ce39076 Merge pull request #1592 from BishopFox/go/v1.22.1
Go 1.22.1
2024-03-07 16:47:15 -07:00
moloch-- 580f98419d Go 1.22.1 2024-03-07 15:10:30 -08:00
moloch-- c8a7948671 Fix link in docs 2024-02-29 10:14:02 -08:00
moloch-- 9da3112c04 Fix link in docs 2024-02-29 10:12:23 -08:00
Ronan Kervella 12ca498386 Merge pull request #1586 from xl4sh/fix/loopvar
Fix using duplicated envelope id bug
2024-02-29 08:33:38 -08:00
xl4sh 2f9c1857bb Fix using duplicated envelope id bug 2024-02-29 13:43:40 +09:00
Joe ac2ee6dfff Merge pull request #1585 from gatariee/master
Fix typo in `make-token` when no credentials are provided.
2024-02-27 07:54:34 -07:00
gatari bfde6a4dc8 Update make-token.go
Signed-off-by: gatari <[email protected]>
2024-02-27 21:06:50 +08:00
Ronan Kervella dc93d054a9 Merge pull request #1572 from BishopFox/v1.6.0/remove-legacy-DLL-extension-support
Removing support for legacy DLL extensions
2024-02-15 15:41:19 -08:00
Ronan Kervella 173366ba7d Merge branch 'master' into v1.6.0/remove-legacy-DLL-extension-support 2024-02-15 14:17:09 -08:00
Joe 4bfca79860 Merge pull request #1579 from BishopFox/ps_elastic
Add elastic binaries to knownSecurityTools
2024-02-06 17:06:50 -06:00
Raf 59b2996862 Removing support for legacy DLL extensions 2024-01-30 08:42:34 -05:00
402 changed files with 13680 additions and 7080 deletions

No files matched your search

+2 -2
View File
@@ -42,10 +42,10 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Go 1.21
- name: Go 1.22
uses: actions/setup-go@v5
with:
go-version: "^1.21"
go-version: "^1.22"
- name: Check Out Code
uses: actions/checkout@v4
+2 -2
View File
@@ -16,10 +16,10 @@ jobs:
steps:
# Setup go environment
- name: Go 1.21
- name: Go 1.22
uses: actions/setup-go@v5
with:
go-version: "^1.21"
go-version: "^1.22"
- name: Checkout repository
uses: actions/checkout@v4
+1 -1
View File
@@ -63,7 +63,7 @@ jobs:
node-version: "lts/*"
- name: Setup Pages
uses: actions/configure-pages@v4
uses: actions/configure-pages@v5
- name: Restore cache
uses: actions/cache@v3
+6 -6
View File
@@ -23,10 +23,10 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
- name: Go 1.21
- name: Go 1.22
uses: actions/setup-go@v5
with:
go-version: "^1.21"
go-version: "^1.22"
- name: OS Packages
run: |
@@ -57,10 +57,10 @@ jobs:
runs-on: macos-latest
timeout-minutes: 90
steps:
- name: Go 1.21
- name: Go 1.22
uses: actions/setup-go@v5
with:
go-version: "^1.21"
go-version: "^1.22"
- name: Mingw
run: brew install mingw-w64
@@ -88,10 +88,10 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
- name: Go 1.21
- name: Go 1.22
uses: actions/setup-go@v5
with:
go-version: "^1.21"
go-version: "^1.22"
- name: Check Out Code
uses: actions/checkout@v4
+1 -1
View File
@@ -9,7 +9,7 @@
# STAGE: base
## Compiles Sliver for use
FROM --platform=linux/amd64 golang:1.21.4 as base
FROM --platform=linux/amd64 golang:1.22.1 as base
### Base packages
RUN apt-get update --fix-missing && apt-get -y install \
+163
View File
@@ -123,6 +123,169 @@ func ImportC2ProfileCmd(cmd *cobra.Command, con *console.SliverClient, args []st
}
}
func ExportC2ProfileCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
filepath, _ := cmd.Flags().GetString("file")
if filepath == "" {
con.PrintErrorf("Missing file path\n")
return
}
profileName, _ := cmd.Flags().GetString("name")
if profileName == "" {
con.PrintErrorf("Invalid c2 profile name\n")
return
}
if profileName == constants.DefaultC2Profile {
httpC2Profiles, err := con.Rpc.GetHTTPC2Profiles(context.Background(), &commonpb.Empty{})
if err != nil {
con.PrintErrorf("failed to fetch HTTP C2 profiles: %s", err.Error())
return
}
if len(httpC2Profiles.Configs) != 1 {
profileName = selectC2Profile(httpC2Profiles.Configs)
}
}
profile, err := con.Rpc.GetHTTPC2ProfileByName(context.Background(), &clientpb.C2ProfileReq{Name: profileName})
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
jsonProfile, err := C2ConfigToJSON(profileName, profile)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
err = os.WriteFile(filepath, jsonProfile, 0644)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
con.Println(profileName, "C2 profile exported to ", filepath)
}
// convert protobuf to json
func C2ConfigToJSON(profileName string, profile *clientpb.HTTPC2Config) ([]byte, error) {
implantConfig := assets.HTTPC2ImplantConfig{
UserAgent: profile.ImplantConfig.UserAgent,
ChromeBaseVersion: int(profile.ImplantConfig.ChromeBaseVersion),
MacOSVersion: profile.ImplantConfig.MacOSVersion,
NonceQueryArgChars: profile.ImplantConfig.NonceQueryArgChars,
MaxFiles: int(profile.ImplantConfig.MaxFiles),
MinFiles: int(profile.ImplantConfig.MinFiles),
MaxPaths: int(profile.ImplantConfig.MaxPaths),
MinPaths: int(profile.ImplantConfig.MinFiles),
StagerFileExt: profile.ImplantConfig.StagerFileExtension,
PollFileExt: profile.ImplantConfig.PollFileExtension,
StartSessionFileExt: profile.ImplantConfig.StartSessionFileExtension,
SessionFileExt: profile.ImplantConfig.SessionFileExtension,
CloseFileExt: profile.ImplantConfig.CloseFileExtension,
}
var headers []assets.NameValueProbability
for _, header := range profile.ImplantConfig.Headers {
headers = append(headers, assets.NameValueProbability{
Name: header.Name,
Value: header.Value,
Probability: int(header.Probability),
})
}
implantConfig.Headers = headers
var urlParameters []assets.NameValueProbability
for _, urlParameter := range profile.ImplantConfig.ExtraURLParameters {
urlParameters = append(urlParameters, assets.NameValueProbability{
Name: urlParameter.Name,
Value: urlParameter.Value,
Probability: int(urlParameter.Probability),
})
}
implantConfig.URLParameters = urlParameters
var (
stagerFiles []string
pollFiles []string
sessionFiles []string
closeFiles []string
stagerPaths []string
pollPaths []string
sessionPaths []string
closePaths []string
)
for _, pathSegment := range profile.ImplantConfig.PathSegments {
if pathSegment.IsFile {
switch pathSegment.SegmentType {
case 0:
pollFiles = append(pollFiles, pathSegment.Value)
case 1:
sessionFiles = append(sessionFiles, pathSegment.Value)
case 2:
closeFiles = append(closeFiles, pathSegment.Value)
case 3:
stagerFiles = append(stagerFiles, pathSegment.Value)
}
} else {
switch pathSegment.SegmentType {
case 0:
pollPaths = append(pollPaths, pathSegment.Value)
case 1:
sessionPaths = append(sessionPaths, pathSegment.Value)
case 2:
closePaths = append(closePaths, pathSegment.Value)
case 3:
stagerPaths = append(stagerPaths, pathSegment.Value)
}
}
}
implantConfig.PollFiles = pollFiles
implantConfig.SessionFiles = sessionFiles
implantConfig.CloseFiles = closeFiles
implantConfig.StagerFiles = stagerFiles
implantConfig.PollPaths = pollPaths
implantConfig.SessionPaths = sessionPaths
implantConfig.ClosePaths = closePaths
implantConfig.StagerPaths = stagerPaths
var serverHeaders []assets.NameValueProbability
for _, header := range profile.ServerConfig.Headers {
serverHeaders = append(serverHeaders, assets.NameValueProbability{
Name: header.Name,
Value: header.Value,
Probability: int(header.Probability),
})
}
var serverCookies []string
for _, cookie := range profile.ServerConfig.Cookies {
serverCookies = append(serverCookies, cookie.Name)
}
serverConfig := assets.HTTPC2ServerConfig{
RandomVersionHeaders: profile.ServerConfig.RandomVersionHeaders,
Headers: serverHeaders,
Cookies: serverCookies,
}
config := assets.HTTPC2Config{
ImplantConfig: implantConfig,
ServerConfig: serverConfig,
}
jsonConfig, err := json.Marshal(config)
if err != nil {
return nil, err
}
return jsonConfig, nil
}
// convert json to protobuf
func C2ConfigToProtobuf(profileName string, config *assets.HTTPC2Config) *clientpb.HTTPC2Config {
+67
View File
@@ -0,0 +1,67 @@
package c2profiles
import (
"github.com/bishopfox/sliver/client/command/flags"
"github.com/bishopfox/sliver/client/command/generate"
"github.com/bishopfox/sliver/client/command/help"
"github.com/bishopfox/sliver/client/console"
consts "github.com/bishopfox/sliver/client/constants"
"github.com/rsteube/carapace"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
// Commands returns the “ command and its subcommands.
func Commands(con *console.SliverClient) []*cobra.Command {
importC2ProfileCmd := &cobra.Command{
Use: consts.ImportC2ProfileStr,
Short: "Import HTTP C2 profile",
Long: help.GetHelpFor([]string{consts.ImportC2ProfileStr}),
Run: func(cmd *cobra.Command, args []string) {
ImportC2ProfileCmd(cmd, con, args)
},
}
flags.Bind(consts.ImportC2ProfileStr, false, importC2ProfileCmd, func(f *pflag.FlagSet) {
f.StringP("name", "n", consts.DefaultC2Profile, "HTTP C2 Profile name")
f.StringP("file", "f", "", "Path to C2 configuration file to import")
f.BoolP("overwrite", "o", false, "Overwrite profile if it exists")
})
exportC2ProfileCmd := &cobra.Command{
Use: consts.ExportC2ProfileStr,
Short: "Export HTTP C2 profile",
Long: help.GetHelpFor([]string{consts.ExportC2ProfileStr}),
Run: func(cmd *cobra.Command, args []string) {
ExportC2ProfileCmd(cmd, con, args)
},
}
flags.Bind(consts.ExportC2ProfileStr, false, exportC2ProfileCmd, func(f *pflag.FlagSet) {
f.StringP("file", "f", "", "Path to file to export C2 configuration to")
f.StringP("name", "n", consts.DefaultC2Profile, "HTTP C2 Profile name")
})
C2ProfileCmd := &cobra.Command{
Use: consts.C2ProfileStr,
Short: "Display C2 profile details",
Long: help.GetHelpFor([]string{consts.C2ProfileStr}),
Run: func(cmd *cobra.Command, args []string) {
C2ProfileCmd(cmd, con, args)
},
GroupID: consts.NetworkHelpGroup,
}
flags.Bind(consts.C2ProfileStr, true, C2ProfileCmd, func(f *pflag.FlagSet) {
f.StringP("name", "n", consts.DefaultC2Profile, "HTTP C2 Profile to display")
})
flags.BindFlagCompletions(C2ProfileCmd, func(comp *carapace.ActionMap) {
(*comp)["name"] = generate.HTTPC2Completer(con)
})
C2ProfileCmd.AddCommand(importC2ProfileCmd)
C2ProfileCmd.AddCommand(exportC2ProfileCmd)
return []*cobra.Command{
C2ProfileCmd,
}
}
+1 -3
View File
@@ -28,8 +28,6 @@ import (
"github.com/spf13/pflag"
)
const defaultTimeout = 60
// Bind is a convenience function to bind flags to a given command.
// name - The name of the flag set (can be empty).
// cmd - The command to which the flags should be bound.
@@ -111,7 +109,7 @@ func makeBind(cmd *cobra.Command, con *client.SliverClient) func(group string, c
//
// @group - Name of the group under which the command should be shown. Preferably use a string in the constants package.
// @ cmds - A list of functions returning a list of root commands to bind. See any package's `commands.go` file and function.
type commandBinder func(group string, cmds ...func(con *client.SliverClient) []*cobra.Command)
// type commandBinder func(group string, cmds ...func(con *client.SliverClient) []*cobra.Command)
// [ Core ]
// [ Sessions ]
+10 -10
View File
@@ -487,16 +487,10 @@ func runExtensionCmd(cmd *cobra.Command, con *console.SliverClient, args []strin
entryPoint = loadedExtensions[extName].Entrypoint // should exist at this point
} else {
// Regular DLL
// extArgs := strings.Join(args, " ")
//legacy case - single string arg
if len(ext.Arguments) == 1 && ext.Arguments[0].Type == "string" {
extensionArgs = []byte(strings.Join(args, " "))
} else {
extensionArgs, err = getExtArgs(cmd, args, binPath, ext)
if err != nil {
con.PrintErrorf("ext args error: %s\n", err)
return
}
extensionArgs, err = getExtArgs(cmd, args, binPath, ext)
if err != nil {
con.PrintErrorf("ext args error: %s\n", err)
return
}
extName = ext.CommandName
entryPoint = ext.Entrypoint
@@ -558,6 +552,12 @@ func getExtArgs(cmd *cobra.Command, args []string, binPath string, ext *ExtComma
// Parse BOF arguments from grumble
missingRequiredArgs := make([]string, 0)
// If we have an extension that expects a single string, but more than one has been parsed, combine them
if len(ext.Arguments) == 1 && strings.Contains(ext.Arguments[0].Type, "string") {
// The loop below will only read the first element of args because ext.Arguments is 1
args[0] = strings.Join(args, " ")
}
for _, arg := range ext.Arguments {
// If we don't have any positional words left to consume,
// add the remaining required extension arguments in the
+14
View File
@@ -230,6 +230,19 @@ func Commands(con *console.SliverClient) []*cobra.Command {
carapace.Gen(memfilesRmCmd).PositionalCompletion(carapace.ActionValues().Usage("memfile file descriptor"))
mountCmd := &cobra.Command{
Use: consts.MountStr,
Short: "Get information on mounted filesystems",
Long: help.GetHelpFor([]string{consts.MountStr}),
Run: func(cmd *cobra.Command, args []string) {
MountCmd(cmd, con, args)
},
GroupID: consts.FilesystemHelpGroup,
}
flags.Bind("", false, mountCmd, func(f *pflag.FlagSet) {
f.Int64P("timeout", "t", flags.DefaultTimeout, "grpc timeout in seconds")
})
grepCmd := &cobra.Command{
Use: consts.GrepStr,
Short: "Search for strings that match a regex within a file or directory",
@@ -324,6 +337,7 @@ func Commands(con *console.SliverClient) []*cobra.Command {
downloadCmd,
uploadCmd,
memfilesCmd,
mountCmd,
grepCmd,
headCmd,
tailCmd,
+165
View File
@@ -0,0 +1,165 @@
/*
Sliver Implant Framework
Copyright (C) 2024 Bishop Fox
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package filesystem
import (
"context"
"fmt"
"math"
"google.golang.org/protobuf/proto"
"github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra"
"github.com/bishopfox/sliver/client/command/settings"
"github.com/bishopfox/sliver/client/console"
"github.com/bishopfox/sliver/protobuf/clientpb"
"github.com/bishopfox/sliver/protobuf/sliverpb"
)
// Drive mappings for Windows
var driveTypeMap = map[string]string{
"0": "Unknown",
"1": "Root Path invalid (no volume mounted for path)",
"2": "Removable",
"3": "Fixed disk",
"4": "Remote / network drive",
"5": "CD-ROM",
"6": "RAM disk",
}
// MountCmd - Print information about mounted filesystems
func MountCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
session, beacon := con.ActiveTarget.GetInteractive()
if session == nil && beacon == nil {
return
}
mount, err := con.Rpc.Mount(context.Background(), &sliverpb.MountReq{
Request: con.ActiveTarget.Request(cmd),
})
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
os := getOS(session, beacon)
if mount.Response != nil && mount.Response.Async {
con.AddBeaconCallback(mount.Response.TaskID, func(task *clientpb.BeaconTask) {
err = proto.Unmarshal(task.Response, mount)
if err != nil {
con.PrintErrorf("Failed to decode response %s\n", err)
return
}
PrintMount(os, mount, con)
})
con.PrintAsyncResponse(mount.Response)
} else {
PrintMount(os, mount, con)
}
}
func getOS(session *clientpb.Session, beacon *clientpb.Beacon) string {
if session != nil {
return session.OS
} else if beacon != nil {
return beacon.OS
}
return ""
}
func reduceSpaceMetric(numberOfBytes float64) string {
units := []string{"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
if numberOfBytes < 1 {
return fmt.Sprintf("0 %s", units[0])
}
base := 1024.0
exp := math.Floor(math.Log(numberOfBytes) / math.Log(base))
index := int(math.Min(exp, float64(len(units)-1)))
divisor := math.Pow(base, float64(index))
value := numberOfBytes / divisor
return fmt.Sprintf("%.2f %s", value, units[index])
}
// PrintMount - Print a table containing information on mounted filesystems
func PrintMount(os string, mount *sliverpb.Mount, con *console.SliverClient) {
if mount.Response != nil && mount.Response.Err != "" {
con.PrintErrorf("%s\n", mount.Response.Err)
return
}
tw := table.NewWriter()
tw.SetStyle(settings.GetTableStyle(con))
switch os {
case "windows":
tw.AppendHeader(table.Row{"Volume", "Volume Type", "Mount Point", "Label", "Filesystem", "Used Space", "Free Space", "Total Space"})
case "darwin":
fallthrough
case "linux":
fallthrough
default:
tw.AppendHeader(table.Row{"Source", "Mount Point", "Mount Root", "Filesystem Type", "Options", "Total Space"})
}
for _, mountPoint := range mount.Info {
row := mountRow(os, mountPoint)
tw.AppendRow(row)
}
settings.PaginateTable(tw, 0, false, false, con)
}
func mountRow(os string, mountInfo *sliverpb.MountInfo) table.Row {
var row table.Row
switch os {
case "windows":
// Translate VolumeType
volType, ok := driveTypeMap[mountInfo.VolumeType]
if !ok {
volType = driveTypeMap["0"]
}
row = table.Row{mountInfo.VolumeName,
volType,
mountInfo.MountPoint,
mountInfo.Label,
mountInfo.FileSystem,
reduceSpaceMetric(float64(mountInfo.UsedSpace)),
reduceSpaceMetric(float64(mountInfo.FreeSpace)),
reduceSpaceMetric(float64(mountInfo.TotalSpace)),
}
case "darwin":
fallthrough
case "linux":
fallthrough
default:
row = table.Row{mountInfo.VolumeName,
mountInfo.MountPoint,
mountInfo.Label,
mountInfo.VolumeType,
mountInfo.MountOptions,
reduceSpaceMetric(float64(mountInfo.TotalSpace)),
}
}
return row
}
+12
View File
@@ -4,6 +4,7 @@ import (
"github.com/bishopfox/sliver/client/command/flags"
"github.com/bishopfox/sliver/client/command/help"
"github.com/bishopfox/sliver/client/console"
"github.com/bishopfox/sliver/client/constants"
consts "github.com/bishopfox/sliver/client/constants"
"github.com/rsteube/carapace"
"github.com/spf13/cobra"
@@ -269,6 +270,16 @@ func Commands(con *console.SliverClient) []*cobra.Command {
carapace.Gen(implantsRmCmd).PositionalCompletion(ImplantBuildNameCompleter(con))
implantBuildsCmd.AddCommand(implantsRmCmd)
implantsStageCmd := &cobra.Command{
Use: consts.StageStr,
Short: "Serve a previously generated build",
Long: help.GetHelpFor([]string{consts.ImplantBuildsStr, consts.StageStr}),
Run: func(cmd *cobra.Command, args []string) {
ImplantsStageCmd(cmd, con, args)
},
}
implantBuildsCmd.AddCommand(implantsStageCmd)
canariesCmd := &cobra.Command{
Use: consts.CanariesStr,
Short: "List previously generated canaries",
@@ -321,6 +332,7 @@ func coreImplantFlags(name string, cmd *cobra.Command) {
f.Int64P("reconnect", "j", DefaultReconnect, "attempt to reconnect every n second(s)")
f.Int64P("poll-timeout", "P", DefaultPollTimeout, "long poll request timeout")
f.Uint32P("max-errors", "k", DefaultMaxErrors, "max number of connection errors")
f.StringP("c2profile", "C", constants.DefaultC2Profile, "HTTP C2 profile to use")
// Limits
f.StringP("limit-datetime", "w", "", "limit execution to before datetime")
+4
View File
@@ -78,6 +78,7 @@ var (
consts.PsExecStr: psExecHelp,
consts.BackdoorStr: backdoorHelp,
consts.SpawnDllStr: spawnDllHelp,
consts.MountStr: mountHelp,
consts.WebsitesStr: websitesHelp,
consts.ScreenshotStr: screenshotHelp,
@@ -380,6 +381,9 @@ On Windows, escaping is disabled. Instead, '\\' is treated as path separator.`
[[.Bold]]About:[[.Normal]] (Windows Only) Executes the .NET assembly in a child process.
`
mountHelp = `[[.Bold]]Command:[[.Normal]] mount
[[.Bold]]About:[[.Normal]] Displays information about mounted drives on the system, including mount point, space metrics, and filesystem.`
executeShellcodeHelp = `[[.Bold]]Command:[[.Normal]] execute-shellcode [local path to raw shellcode]
[[.Bold]]About:[[.Normal]] Executes the given shellcode in the implant's process.
@@ -1,4 +0,0 @@
Prelude Operator
=================
Connection code for [Prelude Operator](https://www.prelude.org/)
@@ -1,45 +0,0 @@
package operator
import (
"github.com/rsteube/carapace"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/bishopfox/sliver/client/command/flags"
"github.com/bishopfox/sliver/client/command/help"
"github.com/bishopfox/sliver/client/console"
consts "github.com/bishopfox/sliver/client/constants"
)
// Commands returns the “ command and its subcommands.
func Commands(con *console.SliverClient) []*cobra.Command {
operatorCmd := &cobra.Command{
Use: consts.PreludeOperatorStr,
Short: "Manage connection to Prelude's Operator",
Long: help.GetHelpFor([]string{consts.PreludeOperatorStr}),
GroupID: consts.GenericHelpGroup,
Run: func(cmd *cobra.Command, args []string) {
OperatorCmd(cmd, con, args)
},
}
operatorConnectCmd := &cobra.Command{
Use: consts.ConnectStr,
Short: "Connect with Prelude's Operator",
Long: help.GetHelpFor([]string{consts.PreludeOperatorStr, consts.ConnectStr}),
Run: func(cmd *cobra.Command, args []string) {
ConnectCmd(cmd, con, args)
},
Args: cobra.ExactArgs(1),
}
operatorCmd.AddCommand(operatorConnectCmd)
flags.Bind("operator", false, operatorConnectCmd, func(f *pflag.FlagSet) {
f.BoolP("skip-existing", "s", false, "Do not add existing sessions as Operator Agents")
f.StringP("aes-key", "a", "abcdefghijklmnopqrstuvwxyz012345", "AES key for communication encryption")
f.StringP("range", "r", "sliver", "Agents range")
})
carapace.Gen(operatorConnectCmd).PositionalCompletion(
carapace.ActionValues().Usage("connection string to the Operator Host (e.g. 127.0.0.1:1234)"))
return []*cobra.Command{operatorCmd}
}
@@ -1,84 +0,0 @@
package operator
/*
Sliver Implant Framework
Copyright (C) 2021 Bishop Fox
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import (
"context"
"github.com/spf13/cobra"
"github.com/bishopfox/sliver/client/console"
"github.com/bishopfox/sliver/client/prelude"
"github.com/bishopfox/sliver/protobuf/clientpb"
"github.com/bishopfox/sliver/protobuf/commonpb"
)
func ConnectCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
url := args[0]
aesKey, _ := cmd.Flags().GetString("aes-key")
agentRange, _ := cmd.Flags().GetString("range")
skipExisting, _ := cmd.Flags().GetBool("skip-existing")
config := &prelude.OperatorConfig{
Range: agentRange,
OperatorURL: url,
RPC: con.Rpc,
AESKey: aesKey,
}
implantMapper := prelude.InitImplantMapper(config)
con.PrintInfof("Connected to Operator at %s\n", url)
if !skipExisting {
sessions, err := con.Rpc.GetSessions(context.Background(), &commonpb.Empty{})
if err != nil {
con.PrintErrorf("Could not get session list: %s", err)
return
}
if len(sessions.Sessions) > 0 {
con.PrintInfof("Adding existing sessions ...\n")
for _, session := range sessions.Sessions {
if !session.IsDead {
err = implantMapper.AddImplant(session, nil)
if err != nil {
con.PrintErrorf("Could not add session %s to implant mapper: %s", session.Name, err)
}
}
}
con.PrintInfof("Done !\n")
}
beacons, err := con.Rpc.GetBeacons(context.Background(), &commonpb.Empty{})
if err != nil {
con.PrintErrorf("Could not get beacon list: %s", err)
return
}
if len(beacons.Beacons) > 0 {
con.PrintInfof("Adding existing beacons ...\n")
for _, beacon := range beacons.Beacons {
err = implantMapper.AddImplant(beacon, func(taskID string, cb func(task *clientpb.BeaconTask)) {
con.AddBeaconCallback(taskID, cb)
})
if err != nil {
con.PrintErrorf("Could not add beacon %s to implant mapper: %s", beacon.Name, err)
}
}
con.PrintInfof("Done !\n")
}
}
}
@@ -1,34 +0,0 @@
package operator
/*
Sliver Implant Framework
Copyright (C) 2021 Bishop Fox
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import (
"github.com/spf13/cobra"
"github.com/bishopfox/sliver/client/console"
"github.com/bishopfox/sliver/client/prelude"
)
func OperatorCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
if prelude.ImplantMapper != nil {
con.PrintInfof("Connected to Operator at %s\n", prelude.ImplantMapper.GetConfig().OperatorURL)
return
}
con.PrintInfof("Not connected to any Operator server. Use `operator connect` to connect to one.")
}
+1 -1
View File
@@ -58,7 +58,7 @@ func MakeTokenCmd(cmd *cobra.Command, con *console.SliverClient, args []string)
}
if username == "" || password == "" {
con.PrintErrorf("Pou must provide a username and password\n")
con.PrintErrorf("You must provide a username and password\n")
return
}
+2 -2
View File
@@ -25,6 +25,7 @@ import (
"github.com/bishopfox/sliver/client/command/armory"
"github.com/bishopfox/sliver/client/command/beacons"
"github.com/bishopfox/sliver/client/command/builders"
"github.com/bishopfox/sliver/client/command/c2profiles"
"github.com/bishopfox/sliver/client/command/crack"
"github.com/bishopfox/sliver/client/command/creds"
"github.com/bishopfox/sliver/client/command/exit"
@@ -36,7 +37,6 @@ import (
"github.com/bishopfox/sliver/client/command/loot"
"github.com/bishopfox/sliver/client/command/monitor"
"github.com/bishopfox/sliver/client/command/operators"
operator "github.com/bishopfox/sliver/client/command/prelude-operator"
"github.com/bishopfox/sliver/client/command/reaction"
"github.com/bishopfox/sliver/client/command/sessions"
"github.com/bishopfox/sliver/client/command/settings"
@@ -91,7 +91,6 @@ func ServerCommands(con *client.SliverClient, serverCmds func() []*cobra.Command
armory.Commands,
update.Commands,
operators.Commands,
operator.Commands,
creds.Commands,
crack.Commands,
)
@@ -101,6 +100,7 @@ func ServerCommands(con *client.SliverClient, serverCmds func() []*cobra.Command
jobs.Commands,
websites.Commands,
wireguard.Commands,
c2profiles.Commands,
)
// Payloads
+2
View File
@@ -145,6 +145,7 @@ const (
TrafficEncodersStr = "traffic-encoders"
C2ProfileStr = "c2profiles"
ImportC2ProfileStr = "import"
ExportC2ProfileStr = "export"
// Generic.
@@ -223,6 +224,7 @@ const (
ChmodStr = "chmod"
ChownStr = "chown"
ChtimesStr = "chtimes"
MountStr = "mount"
MemfilesStr = "memfiles"
+1 -1
View File
@@ -1,6 +1,6 @@
Sliver keeps an audit log of every command and its arguments executed by the server (including commands executed by operators in multiplayer mode), as well as most events (such as a new session or beacon connecting to the server). The audit log's intended use is for after-action analysis; providing a detailed history of the entire engagement, including which commands were executed on which hosts when. This will include any commands executed by any operator on any session. Note some console commands only perform actions on the "client-side" and may not appear in the audit log, but will still appear in the client's command history. Additionally, interactive commands (e.g., `shell`) may not appear in the logs aside from the initial usage of the `shell` command.
By default the audit log is located on the server at: `~/.sliver/logs/audit.json`. However, this can be changed by modifying the [`SLIVER_ROOT_DIR`](/docs?name=Environment-Variables#assets) environment variable.
By default the audit log is located on the server at: `~/.sliver/logs/audit.json`. However, this can be changed by modifying the [`SLIVER_ROOT_DIR`](/docs?name=Environment+Variables) environment variable.
#### Parsing Audit Logs
@@ -4,7 +4,7 @@ You'll want to compile from a MacOS or Linux machine, compiling from native Wind
From scratch without Docker, requirements for compiling:
- Go v1.20 or later, though we recommend Go v1.21
- Go v1.21 or later, though we recommend Go v1.22. When compiling v1.5.x use Go v1.20.7
- `make`, `sed`, `tar`, `curl`, `zip`, `cut` commands; most of these are installed by default but you may need to install `make`, `curl`, and `zip` depending on your distribution. On MacOS you may need to install XCode and accompanying cli tools.
**IMPORTANT:** The Sliver Makefile requires version information from the git repository, so you must `git clone` the repository. Using GitHub's "download zip" feature may omit the `.git` directory and result in broken builds.
+1 -1
View File
@@ -67,7 +67,7 @@ The priority of retrieval is the following:
#### NTLM/Kerberos Proxy Authentication
You can use [advanced options](/docs?name=C2-Advanced-Options) to enable the use of the `wininet` HTTP library, which supports NTLM/Kerberos authentication (Windows only). Using this library tends to be a little less stable (we have to covert Go calls to native DLL calls) and is generally more susceptible to introspection by security products as these functions are well-known and easy to hook. However, if you need NTLM/Kerberos authentication you don't have much of a choice.
You can use [advanced options](/docs?name=C2+Advanced+Options) to enable the use of the `wininet` HTTP library, which supports NTLM/Kerberos authentication (Windows only). Using this library tends to be a little less stable (we have to covert Go calls to native DLL calls) and is generally more susceptible to introspection by security products as these functions are well-known and easy to hook. However, if you need NTLM/Kerberos authentication you don't have much of a choice.
## Start the Listener
+2 -2
View File
@@ -20,8 +20,8 @@ set -e
# Creates the static go asset archives
GO_VER="1.21.4"
GARBLE_VER="1.21.4b"
GO_VER="1.22.1"
GARBLE_VER="1.22.1"
SGN_VER="0.0.3"
BLOAT_FILES="AUTHORS CONTRIBUTORS PATENTS VERSION favicon.ico robots.txt SECURITY.md CONTRIBUTING.md LICENSE README.md ./doc ./test ./api ./misc"
+8 -7
View File
@@ -21,6 +21,7 @@ require (
github.com/fatih/color v1.15.0
github.com/glebarez/sqlite v1.10.0
github.com/gofrs/uuid v4.4.0+incompatible
github.com/golang/protobuf v1.5.3
github.com/google/uuid v1.4.0
github.com/gorilla/mux v1.8.1
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
@@ -45,16 +46,16 @@ require (
github.com/things-go/go-socks5 v0.0.3
github.com/xlab/treeprint v1.2.0
github.com/yiya1989/sshkrb5 v0.0.1
golang.org/x/crypto v0.17.0
golang.org/x/crypto v0.21.0
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
golang.org/x/net v0.17.0
golang.org/x/sys v0.15.0
golang.org/x/term v0.15.0
golang.org/x/net v0.21.0
golang.org/x/sys v0.18.0
golang.org/x/term v0.18.0
golang.org/x/text v0.14.0
golang.zx2c4.com/wireguard v0.0.0-20231022001213-2e0774f246fb
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220208144051-fde48d68ee68
google.golang.org/grpc v1.59.0
google.golang.org/protobuf v1.31.0
google.golang.org/protobuf v1.33.0
gopkg.in/AlecAivazis/survey.v1 v1.8.8
gorm.io/driver/mysql v1.5.1
gorm.io/driver/postgres v1.5.2
@@ -109,7 +110,6 @@ require (
github.com/gobwas/ws v1.3.0 // indirect
github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c // indirect
@@ -122,7 +122,8 @@ require (
github.com/insomniacslk/dhcp v0.0.0-20230908212754-65c27093e38a // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx/v5 v5.3.1 // indirect
github.com/jackc/pgx/v5 v5.5.4 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
+14 -14
View File
@@ -226,8 +226,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU=
github.com/jackc/pgx/v5 v5.3.1/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8=
github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8=
github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw=
@@ -352,8 +354,6 @@ github.com/reeflective/carapace v0.25.2-0.20230602202234-e8d757e458ca h1:tD797h1
github.com/reeflective/carapace v0.25.2-0.20230602202234-e8d757e458ca/go.mod h1:jkLt41Ne2TD2xPuMdX/2O05Smhy8vMgG7O2TYvC0yOc=
github.com/reeflective/console v0.1.6 h1:BhhvQU/m8QOpaIRzrXfwcbtkGQJX9jVR5qIqAy/Mcuw=
github.com/reeflective/console v0.1.6/go.mod h1:7owTBE9k2lg2QpVw7g4DrK1HxEgr/5DQCmA3O2Znoek=
github.com/reeflective/readline v1.0.11 h1:4+aiebj7a89hTRJOMM98H+md1Kxu+v1XkfdCs0n6odQ=
github.com/reeflective/readline v1.0.11/go.mod h1:mcD0HxNVJVteVwDm9caXKg52nQACVyfh8EyuBmgVlzY=
github.com/reeflective/readline v1.0.13 h1:TeJmYw9B7VRPZWfNExr9QHxL1m0iSicyqBSQIRn39Ss=
github.com/reeflective/readline v1.0.13/go.mod h1:3iOe/qyb2jEy0KqLrNlb/CojBVqxga9ACqz/VU22H6A=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
@@ -450,8 +450,8 @@ golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
@@ -480,8 +480,8 @@ golang.org/x/net v0.0.0-20211111083644-e5c967477495/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
@@ -529,12 +529,12 @@ golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -591,8 +591,8 @@ google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/AlecAivazis/survey.v1 v1.8.8 h1:5UtTowJZTz1j7NxVzDGKTz6Lm9IWm8DDF6b7a2wq9VY=
gopkg.in/AlecAivazis/survey.v1 v1.8.8/go.mod h1:CaHjv79TCgAvXMSFJSVgonHXYWxnhzI3eoHtnX5UgUo=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+2 -2
View File
@@ -18,9 +18,11 @@ require (
github.com/things-go/go-socks5 v0.0.3-0.20210722055343-24af464efe43
github.com/yiya1989/sshkrb5 v0.0.0-20201110125252-a1455b75a35e
golang.org/x/crypto v0.13.0
golang.org/x/net v0.15.0
golang.org/x/sys v0.14.0
golang.zx2c4.com/wireguard v0.0.0-20231022001213-2e0774f246fb
google.golang.org/protobuf v1.31.0
gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259
)
require (
@@ -37,7 +39,6 @@ require (
github.com/lxn/win v0.0.0-20210218163916-a377121e959e // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/net v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.13.0 // indirect
@@ -48,5 +49,4 @@ require (
gopkg.in/jcmturner/gokrb5.v7 v7.5.0 // indirect
gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 // indirect
)
+24 -1
View File
@@ -32,6 +32,7 @@ import (
"unsafe"
"github.com/bishopfox/sliver/implant/sliver/procdump"
"github.com/bishopfox/sliver/implant/sliver/mount"
"github.com/bishopfox/sliver/implant/sliver/taskrunner"
"github.com/bishopfox/sliver/protobuf/commonpb"
"github.com/bishopfox/sliver/protobuf/sliverpb"
@@ -57,7 +58,7 @@ var (
sliverpb.MsgMvReq: mvHandler,
sliverpb.MsgCpReq: cpHandler,
sliverpb.MsgTaskReq: taskHandler,
sliverpb.MsgIfconfigReq: ifconfigHandler,
sliverpb.MsgIfconfigReq: ifconfigLinuxHandler,
sliverpb.MsgExecuteReq: executeHandler,
sliverpb.MsgEnvReq: getEnvHandler,
sliverpb.MsgSetEnvReq: setEnvHandler,
@@ -71,6 +72,7 @@ var (
sliverpb.MsgReconfigureReq: reconfigureHandler,
sliverpb.MsgSSHCommandReq: runSSHCommandHandler,
sliverpb.MsgProcessDumpReq: dumpHandler,
sliverpb.MsgMountReq: mountHandler,
sliverpb.MsgGrepReq: grepHandler,
// Wasm Extensions - Note that execution can be done via a tunnel handler
@@ -124,6 +126,27 @@ func dumpHandler(data []byte, resp RPCResponse) {
resp(data, err)
}
func mountHandler(data []byte, resp RPCResponse) {
mountReq := &sliverpb.MountReq{}
err := proto.Unmarshal(data, mountReq)
if err != nil {
return
}
mountData, err := mount.GetMountInformation()
mountResp := &sliverpb.Mount{
Info: mountData,
Response: &commonpb.Response{},
}
if err != nil {
mountResp.Response.Err = err.Error()
}
data, err = proto.Marshal(mountResp)
resp(data, err)
}
func taskHandler(data []byte, resp RPCResponse) {
var err error
task := &sliverpb.TaskReq{}
@@ -35,6 +35,7 @@ import (
"syscall"
"github.com/bishopfox/sliver/implant/sliver/extension"
"github.com/bishopfox/sliver/implant/sliver/mount"
"github.com/bishopfox/sliver/implant/sliver/priv"
"github.com/bishopfox/sliver/implant/sliver/procdump"
"github.com/bishopfox/sliver/implant/sliver/ps"
@@ -92,6 +93,7 @@ var (
sliverpb.MsgServicesReq: servicesListHandler,
sliverpb.MsgServiceDetailReq: serviceDetailHandler,
sliverpb.MsgStartServiceByNameReq: startServiceByNameHandler,
sliverpb.MsgMountReq: mountHandler,
// Generic
sliverpb.MsgPing: pingHandler,
@@ -872,6 +874,27 @@ func serviceDetailHandler(data []byte, resp RPCResponse) {
resp(data, err)
}
func mountHandler(data []byte, resp RPCResponse) {
mountReq := &sliverpb.MountReq{}
err := proto.Unmarshal(data, mountReq)
if err != nil {
return
}
mountData, err := mount.GetMountInformation()
mountResp := &sliverpb.Mount{
Info: mountData,
Response: &commonpb.Response{},
}
if err != nil {
mountResp.Response.Err = err.Error()
}
data, err = proto.Marshal(mountResp)
resp(data, err)
}
// Extensions
func registerExtensionHandler(data []byte, resp RPCResponse) {
+167 -4
View File
@@ -23,10 +23,21 @@ import (
"log"
// {{end}}
"fmt"
"io/ioutil"
"net"
"os"
"path/filepath"
"runtime"
"strconv"
"syscall"
"github.com/bishopfox/sliver/implant/sliver/ps"
"github.com/bishopfox/sliver/protobuf/commonpb"
"github.com/bishopfox/sliver/protobuf/sliverpb"
"google.golang.org/protobuf/proto"
"golang.org/x/sys/unix"
)
func psHandler(data []byte, resp RPCResponse) {
@@ -51,10 +62,10 @@ func psHandler(data []byte, resp RPCResponse) {
for _, proc := range procs {
p := &commonpb.Process{
Pid: int32(proc.Pid()),
Ppid: int32(proc.PPid()),
Executable: proc.Executable(),
Owner: proc.Owner(),
Pid: int32(proc.Pid()),
Ppid: int32(proc.PPid()),
Executable: proc.Executable(),
Owner: proc.Owner(),
Architecture: proc.Architecture(),
}
p.CmdLine = proc.(*ps.UnixProcess).CmdLine()
@@ -63,3 +74,155 @@ func psHandler(data []byte, resp RPCResponse) {
data, err = proto.Marshal(psList)
resp(data, err)
}
func getFdFromPath(path string) (int, error) {
fd, err := unix.Open(path, unix.O_RDONLY|unix.O_CLOEXEC, 0)
if err != nil {
return -1, err
}
return fd, nil
}
func getUniqueFd(fd int) string {
// Returns the unique namespace ID
var s unix.Stat_t
err := unix.Fstat(fd, &s)
if err != nil {
return "Unknown"
}
return fmt.Sprintf("NS(%d:%d)", s.Dev, s.Ino)
}
func ifconfigLinuxHandler(_ []byte, resp RPCResponse) {
interfaces := ifconfigLinux()
// {{if .Config.Debug}}
log.Printf("network interfaces: %#v", interfaces)
// {{end}}
data, err := proto.Marshal(interfaces)
resp(data, err)
}
func nsLinuxIfconfig(interfaces *sliverpb.Ifconfig) {
namespacesFound := make(map[uint64]string)
procDir := "/proc"
procContents, err := ioutil.ReadDir(procDir)
if err != nil {
//{{if .Config.Debug}}
log.Printf("error reading /proc: %v", err)
//{{end}}
return
}
for _, entry := range procContents {
if !entry.IsDir() {
continue
}
match, _ := filepath.Match("[1-9]*", entry.Name())
if match {
// Check if /proc/PID/net/ns exists
checkPath := filepath.Join(procDir, entry.Name(), "/ns/net")
if _, err := os.Stat(checkPath); !os.IsNotExist(err) {
// path for /proc/PID/ns/net exists
// inode used to track unique namespaces
var inode uint64
fileinfo, err := os.Stat(checkPath)
if err != nil {
//{{if .Config.Debug}}
log.Printf("error : %v", err)
//{{end}}
continue
}
inode = fileinfo.Sys().(*syscall.Stat_t).Ino
// Track unique namespaces
namespacesFound[inode] = checkPath
}
}
}
// Lock the OS Thread so we don't accidentally switch namespaces
runtime.LockOSThread()
defer runtime.UnlockOSThread()
// Save the current network namespace
pidPath := strconv.Itoa(os.Getpid())
tidPath := strconv.Itoa(unix.Gettid())
origns, _ := getFdFromPath(filepath.Join(procDir, pidPath, "/task", tidPath, "/ns/net"))
defer unix.Close(origns)
// We only need to use the path value
for _, nsPath := range namespacesFound {
nsFd, err := unix.Open(nsPath, unix.O_RDONLY|unix.O_CLOEXEC, 0)
if err != nil {
continue
}
// Ignore origin namespace
if getUniqueFd(nsFd) == getUniqueFd(origns) {
continue
}
err = unix.Setns(nsFd, unix.CLONE_NEWNET)
if err != nil {
// Failed to enter namespace
continue
}
ifaces, _ := net.Interfaces()
// {{if .Config.Debug}}
log.Printf("Interfaces: %v\n", ifaces)
// {{end}}
ifconfigParseInterfaces(ifaces, interfaces, nsPath)
}
// Switch back to the original namespace
unix.Setns(origns, unix.CLONE_NEWNET)
}
func ifconfigLinux() *sliverpb.Ifconfig {
netInterfaces, err := net.Interfaces()
if err != nil {
return nil
}
interfaces := &sliverpb.Ifconfig{
NetInterfaces: []*sliverpb.NetInterface{},
}
ifconfigParseInterfaces(netInterfaces, interfaces)
nsLinuxIfconfig(interfaces)
return interfaces
}
func ifconfigParseInterfaces(netInterfaces []net.Interface, interfaces *sliverpb.Ifconfig, namespacePath ...string) {
// Append namespace ID if passed in
var appendNsId = ""
if len(namespacePath) > 0 {
appendNsId = fmt.Sprintf(" NS(%v)",namespacePath[0])
}
for _, iface := range netInterfaces {
netIface := &sliverpb.NetInterface{
Index: int32(iface.Index),
Name: iface.Name + appendNsId,
}
if iface.HardwareAddr != nil {
netIface.MAC = iface.HardwareAddr.String()
}
addresses, err := iface.Addrs()
if err == nil {
for _, address := range addresses {
netIface.IPAddresses = append(netIface.IPAddresses, address.String())
}
}
interfaces.NetInterfaces = append(interfaces.NetInterfaces, netIface)
}
}
+56
View File
@@ -0,0 +1,56 @@
// +linux
package mount
import (
"bufio"
"os"
"strings"
"syscall"
"github.com/bishopfox/sliver/protobuf/sliverpb"
)
func GetMountInformation() ([]*sliverpb.MountInfo, error) {
mountInfo := make([]*sliverpb.MountInfo, 0)
file, err := os.Open("/proc/self/mountinfo")
if err != nil {
return mountInfo, err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
// Extract fields according to the /proc/self/mountinfo format
// https://man7.org/linux/man-pages/man5/proc.5.html
mountRoot := fields[3]
mountPoint := fields[4]
mountOptions := fields[5]
mountType := fields[len(fields)-3]
mountSource := fields[len(fields)-2]
// Get mount information using statfs
var stat syscall.Statfs_t
err := syscall.Statfs(mountPoint, &stat)
if err != nil {
continue
}
var mountData sliverpb.MountInfo
mountData.Label = mountRoot
mountData.MountPoint = mountPoint
mountData.VolumeType = mountType
mountData.VolumeName = mountSource
mountData.MountOptions = mountOptions
mountData.TotalSpace = stat.Blocks * uint64(stat.Bsize)
mountInfo = append(mountInfo, &mountData)
}
return mountInfo, nil
}
+279
View File
@@ -0,0 +1,279 @@
//go:build windows
// +build windows
/*
Sliver Implant Framework
Copyright (C) 2023 Bishop Fox
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package mount
import (
"errors"
"fmt"
"strconv"
"strings"
"unicode/utf16"
"unsafe"
"github.com/bishopfox/sliver/protobuf/sliverpb"
"golang.org/x/sys/windows"
)
const (
pathLength = windows.MAX_PATH + 1
volumePathsLength = 32769 // 32K + 1 for a null terminator
universalNameInfoLevel = 1
remoteNameInfoLevel = 2
remoteNameMaxLength = 1024
)
type UniversalNameInfo struct {
UniversalName [remoteNameMaxLength]uint16
}
func findAllVolumes() ([]string, error) {
var volumes []string
volumeGuid := make([]uint16, pathLength)
handle, err := windows.FindFirstVolume(&volumeGuid[0], pathLength)
defer windows.FindVolumeClose(handle)
if err != nil {
return volumes, err
}
volumes = append(volumes, windows.UTF16ToString(volumeGuid))
// Keep gathering volumes until we cannot find anymore
for {
err = windows.FindNextVolume(handle, &volumeGuid[0], pathLength)
if err != nil {
if err == windows.ERROR_NO_MORE_FILES {
// Then we are done
break
} else {
return volumes, err
}
}
volumes = append(volumes, windows.UTF16ToString(volumeGuid))
}
return volumes, nil
}
func findMountPointsForVolume(volumeName string) ([]string, error) {
const nullByte uint16 = 0
var volumeMounts []string
mountsForVolume := make([]uint16, volumePathsLength)
var returnedInformationLength uint32 = 0
volNameUint, err := windows.UTF16PtrFromString(volumeName)
if err != nil {
return volumeMounts, err
}
err = windows.GetVolumePathNamesForVolumeName(volNameUint, &mountsForVolume[0], volumePathsLength, &returnedInformationLength)
if err != nil {
if err == windows.ERROR_MORE_DATA {
// Then we need to make the buffer bigger and try again
// We will try one more time
mountsForVolume := make([]uint16, returnedInformationLength)
err = windows.GetVolumePathNamesForVolumeName(volNameUint, &mountsForVolume[0], volumePathsLength, &returnedInformationLength)
if err != nil {
return volumeMounts, err
}
} else {
return volumeMounts, err
}
}
/*
Because the buffer contains multiple strings separated by a null terminator, we will read the
buffer in chunks and separate the strings
*/
volumeMounts = splitStringBuffer(mountsForVolume)
return volumeMounts, nil
}
func splitStringBuffer(buffer []uint16) []string {
bufferString := string(utf16.Decode(buffer))
return strings.Split(strings.TrimRight(bufferString, "\x00"), "\x00")
}
func getAllDrives() ([]string, error) {
var logicalDrives []string
// Get necessary buffer size
n, err := windows.GetLogicalDriveStrings(0, nil)
driveStrings := make([]uint16, n)
_, err = windows.GetLogicalDriveStrings(n, &driveStrings[0])
if err != nil {
return logicalDrives, err
}
logicalDrives = splitStringBuffer(driveStrings)
return logicalDrives, nil
}
func getDriveType(driveSpec string) string {
driveUTF16, err := windows.UTF16PtrFromString(driveSpec)
if err != nil {
return ""
}
// Convert type to string (client will handle translation)
driveTypeValue := strconv.FormatUint(uint64(windows.GetDriveType(driveUTF16)), 10)
return driveTypeValue
}
func getUniversalName(driveSpec string) (string, error) {
mpr := windows.NewLazyDLL("mpr.dll")
wNetGetUniversalNameWProc := mpr.NewProc("WNetGetUniversalNameW")
driveUTF16, err := windows.UTF16PtrFromString(driveSpec)
if err != nil {
return "", err
}
var universalNameBuffer UniversalNameInfo
var bufferSize = uint32(unsafe.Sizeof(universalNameBuffer))
returnCode, _, err := wNetGetUniversalNameWProc.Call(
uintptr(unsafe.Pointer(driveUTF16)),
uintptr(universalNameInfoLevel),
uintptr(unsafe.Pointer(&universalNameBuffer)),
uintptr(unsafe.Pointer(&bufferSize)),
)
if returnCode != 0 {
// Then there was an error
return "", fmt.Errorf("%v", windows.Errno(returnCode).Error())
}
bufferStrings := splitStringBuffer(universalNameBuffer.UniversalName[:])
// There is some junk returned at the beginning of the structure. The actual UNC path starts after the first null terminator.
if len(bufferStrings) > 1 {
return bufferStrings[1], nil
} else {
return "", errors.New("No valid data returned")
}
}
func getDiskStats(driveSpec string) (uint64, uint64, uint64, error) {
var freeSpace uint64 = 0
var usedSpace uint64 = 0
var totalSpace uint64 = 0
driveUTF16, err := windows.UTF16PtrFromString(driveSpec)
if err != nil {
return freeSpace, usedSpace, totalSpace, err
}
err = windows.GetDiskFreeSpaceEx(driveUTF16, nil, &totalSpace, &freeSpace)
if err != nil {
return freeSpace, usedSpace, totalSpace, err
}
usedSpace = totalSpace - freeSpace
return freeSpace, usedSpace, totalSpace, nil
}
func getDriveInfo(driveSpec string) (string, string) {
// Get label and filesystem type
driveUTF16, err := windows.UTF16PtrFromString(driveSpec)
if err != nil {
return "", ""
}
volumeNameBuffer := make([]uint16, pathLength)
fileSystemNameBuffer := make([]uint16, pathLength)
err = windows.GetVolumeInformation(driveUTF16, &volumeNameBuffer[0], pathLength, nil, nil, nil, &fileSystemNameBuffer[0], pathLength)
if err != nil {
return "", ""
}
return windows.UTF16ToString(volumeNameBuffer), windows.UTF16ToString(fileSystemNameBuffer)
}
func constructDriveVolumeMap() (map[string]string, error) {
driveVolumeMap := make(map[string]string)
volumes, err := findAllVolumes()
if err != nil {
return driveVolumeMap, err
}
for _, volume := range volumes {
mountPoints, err := findMountPointsForVolume(volume)
if err != nil {
return driveVolumeMap, err
}
for _, mountPoint := range mountPoints {
driveVolumeMap[mountPoint] = volume
}
}
return driveVolumeMap, nil
}
func GetMountInformation() ([]*sliverpb.MountInfo, error) {
mountInfo := make([]*sliverpb.MountInfo, 0)
logicalDrives, err := getAllDrives()
if err != nil {
return mountInfo, err
}
driveVolumeMap, err := constructDriveVolumeMap()
if err != nil {
return mountInfo, err
}
for _, drive := range logicalDrives {
var mountData sliverpb.MountInfo
mountData.MountPoint = drive
mountData.VolumeType = getDriveType(drive)
// Drive type of 4 is "Remote"
// As per https://cs.opensource.google/go/x/sys/+/refs/tags/v0.18.0:windows/syscall_windows.go;l=33
if mountData.VolumeType == "4" {
// Then this is a network drive, so let's figure out the UNC path
networkPath, err := getUniversalName(drive)
if err != nil {
return mountInfo, err
}
mountData.VolumeName = networkPath
} else {
mountData.VolumeName = driveVolumeMap[drive]
}
mountData.Label, mountData.FileSystem = getDriveInfo(drive)
free, used, total, err := getDiskStats(drive)
if err != nil {
return mountInfo, err
}
mountData.UsedSpace = used
mountData.FreeSpace = free
mountData.TotalSpace = total
mountInfo = append(mountInfo, &mountData)
}
return mountInfo, nil
}
+1
View File
@@ -598,6 +598,7 @@ func sessionMainLoop(connection *transports.Connection) error {
rportfwdHandlers := handlers.GetRportFwdHandlers()
for envelope := range connection.Recv {
envelope := envelope
if _, ok := specialHandlers[envelope.Type]; ok {
// {{if .Config.Debug}}
log.Printf("[recv] specialHandler %d", envelope.Type)
File diff suppressed because it is too large. Load diff
+1
View File
@@ -177,6 +177,7 @@ service SliverRPC {
rpc MemfilesList(sliverpb.MemfilesListReq) returns (sliverpb.Ls);
rpc MemfilesAdd(sliverpb.MemfilesAddReq) returns (sliverpb.MemfilesAdd);
rpc MemfilesRm(sliverpb.MemfilesRmReq) returns (sliverpb.MemfilesRm);
rpc Mount(sliverpb.MountReq) returns (sliverpb.Mount);
rpc ProcessDump(sliverpb.ProcessDumpReq) returns (sliverpb.ProcessDump);
rpc RunAs(sliverpb.RunAsReq) returns (sliverpb.RunAs);
rpc Impersonate(sliverpb.ImpersonateReq) returns (sliverpb.Impersonate);
+36
View File
@@ -158,6 +158,7 @@ type SliverRPCClient interface {
MemfilesList(ctx context.Context, in *sliverpb.MemfilesListReq, opts ...grpc.CallOption) (*sliverpb.Ls, error)
MemfilesAdd(ctx context.Context, in *sliverpb.MemfilesAddReq, opts ...grpc.CallOption) (*sliverpb.MemfilesAdd, error)
MemfilesRm(ctx context.Context, in *sliverpb.MemfilesRmReq, opts ...grpc.CallOption) (*sliverpb.MemfilesRm, error)
Mount(ctx context.Context, in *sliverpb.MountReq, opts ...grpc.CallOption) (*sliverpb.Mount, error)
ProcessDump(ctx context.Context, in *sliverpb.ProcessDumpReq, opts ...grpc.CallOption) (*sliverpb.ProcessDump, error)
RunAs(ctx context.Context, in *sliverpb.RunAsReq, opts ...grpc.CallOption) (*sliverpb.RunAs, error)
Impersonate(ctx context.Context, in *sliverpb.ImpersonateReq, opts ...grpc.CallOption) (*sliverpb.Impersonate, error)
@@ -1332,6 +1333,15 @@ func (c *sliverRPCClient) MemfilesRm(ctx context.Context, in *sliverpb.MemfilesR
return out, nil
}
func (c *sliverRPCClient) Mount(ctx context.Context, in *sliverpb.MountReq, opts ...grpc.CallOption) (*sliverpb.Mount, error) {
out := new(sliverpb.Mount)
err := c.cc.Invoke(ctx, "/rpcpb.SliverRPC/Mount", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *sliverRPCClient) ProcessDump(ctx context.Context, in *sliverpb.ProcessDumpReq, opts ...grpc.CallOption) (*sliverpb.ProcessDump, error) {
out := new(sliverpb.ProcessDump)
err := c.cc.Invoke(ctx, "/rpcpb.SliverRPC/ProcessDump", in, out, opts...)
@@ -2139,6 +2149,7 @@ type SliverRPCServer interface {
MemfilesList(context.Context, *sliverpb.MemfilesListReq) (*sliverpb.Ls, error)
MemfilesAdd(context.Context, *sliverpb.MemfilesAddReq) (*sliverpb.MemfilesAdd, error)
MemfilesRm(context.Context, *sliverpb.MemfilesRmReq) (*sliverpb.MemfilesRm, error)
Mount(context.Context, *sliverpb.MountReq) (*sliverpb.Mount, error)
ProcessDump(context.Context, *sliverpb.ProcessDumpReq) (*sliverpb.ProcessDump, error)
RunAs(context.Context, *sliverpb.RunAsReq) (*sliverpb.RunAs, error)
Impersonate(context.Context, *sliverpb.ImpersonateReq) (*sliverpb.Impersonate, error)
@@ -2561,6 +2572,9 @@ func (UnimplementedSliverRPCServer) MemfilesAdd(context.Context, *sliverpb.Memfi
func (UnimplementedSliverRPCServer) MemfilesRm(context.Context, *sliverpb.MemfilesRmReq) (*sliverpb.MemfilesRm, error) {
return nil, status.Errorf(codes.Unimplemented, "method MemfilesRm not implemented")
}
func (UnimplementedSliverRPCServer) Mount(context.Context, *sliverpb.MountReq) (*sliverpb.Mount, error) {
return nil, status.Errorf(codes.Unimplemented, "method Mount not implemented")
}
func (UnimplementedSliverRPCServer) ProcessDump(context.Context, *sliverpb.ProcessDumpReq) (*sliverpb.ProcessDump, error) {
return nil, status.Errorf(codes.Unimplemented, "method ProcessDump not implemented")
}
@@ -4823,6 +4837,24 @@ func _SliverRPC_MemfilesRm_Handler(srv interface{}, ctx context.Context, dec fun
return interceptor(ctx, in, info, handler)
}
func _SliverRPC_Mount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(sliverpb.MountReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SliverRPCServer).Mount(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/rpcpb.SliverRPC/Mount",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SliverRPCServer).Mount(ctx, req.(*sliverpb.MountReq))
}
return interceptor(ctx, in, info, handler)
}
func _SliverRPC_ProcessDump_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(sliverpb.ProcessDumpReq)
if err := dec(in); err != nil {
@@ -6495,6 +6527,10 @@ var SliverRPC_ServiceDesc = grpc.ServiceDesc{
MethodName: "MemfilesRm",
Handler: _SliverRPC_MemfilesRm_Handler,
},
{
MethodName: "Mount",
Handler: _SliverRPC_Mount_Handler,
},
{
MethodName: "ProcessDump",
Handler: _SliverRPC_ProcessDump_Handler,
+7
View File
@@ -349,6 +349,9 @@ const (
MsgStartServiceByNameReq
MsgRegistryReadHiveReq
// MsgMountReq - Request filesystem mounts
MsgMountReq
)
// Constants to replace enums
@@ -595,8 +598,12 @@ func MsgNumber(request proto.Message) uint32 {
case *GrepReq:
return MsgGrepReq
case *MountReq:
return MsgMountReq
case *MemfilesListReq:
return MsgMemfilesListReq
case *MemfilesAddReq:
return MsgMemfilesAddReq
case *MemfilesAdd:
File diff suppressed because it is too large. Load diff
+23
View File
@@ -327,6 +327,28 @@ message Grep {
commonpb.Response Response = 9;
}
message MountReq {
commonpb.Request Request = 9;
}
message MountInfo {
string VolumeName = 1;
string VolumeType = 2;
string MountPoint = 3;
string Label = 4;
string FileSystem = 5;
uint64 UsedSpace = 6;
uint64 FreeSpace = 7;
uint64 TotalSpace = 8;
string MountOptions = 9;
}
message Mount {
repeated MountInfo Info = 1;
commonpb.Response Response = 9;
}
message ProcessDumpReq {
int32 Pid = 1;
int32 Timeout = 2;
@@ -340,6 +362,7 @@ message ProcessDump {
commonpb.Response Response = 9;
}
message RunAsReq {
string Username = 1;
string ProcessName = 2;
+8 -8
View File
@@ -222,10 +222,10 @@ func VenomPayload(config VenomConfig) ([]byte, error) {
func venomCmd(args []string) ([]byte, error) {
msfLog.Printf("%s %v", venomBin, args)
cmd := exec.Command(venomBin, args...)
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
cmd.Stdout = stdout
cmd.Stderr = stderr
err := cmd.Run()
msfLog.Println(cmd.String())
if err != nil {
@@ -240,10 +240,10 @@ func venomCmd(args []string) ([]byte, error) {
// consoleCmd - Execute a msfvenom command
func consoleCmd(args []string) ([]byte, error) {
cmd := exec.Command(consoleBin, args...)
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
cmd.Stdout = stdout
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
msfLog.Printf("--- stdout ---\n%s\n", stdout.String())
+10
View File
@@ -239,3 +239,13 @@ func (rpc *Server) Grep(ctx context.Context, req *sliverpb.GrepReq) (*sliverpb.G
}
return resp, nil
}
// Mount - Get information on mounted filesystems
func (rpc *Server) Mount(ctx context.Context, req *sliverpb.MountReq) (*sliverpb.Mount, error) {
resp := &sliverpb.Mount{Response: &commonpb.Response{}}
err := rpc.GenericHandler(req, resp)
if err != nil {
return nil, err
}
return resp, nil
}
+112
View File
@@ -1,3 +1,115 @@
# 5.5.4 (March 4, 2024)
Fix CVE-2024-27304
SQL injection can occur if an attacker can cause a single query or bind message to exceed 4 GB in size. An integer
overflow in the calculated message size can cause the one large message to be sent as multiple messages under the
attacker's control.
Thanks to Paul Gerste for reporting this issue.
* Fix behavior of CollectRows to return empty slice if Rows are empty (Felix)
* Fix simple protocol encoding of json.RawMessage
* Fix *Pipeline.getResults should close pipeline on error
* Fix panic in TryFindUnderlyingTypeScanPlan (David Kurman)
* Fix deallocation of invalidated cached statements in a transaction
* Handle invalid sslkey file
* Fix scan float4 into sql.Scanner
* Fix pgtype.Bits not making copy of data from read buffer. This would cause the data to be corrupted by future reads.
# 5.5.3 (February 3, 2024)
* Fix: prepared statement already exists
* Improve CopyFrom auto-conversion of text-ish values
* Add ltree type support (Florent Viel)
* Make some properties of Batch and QueuedQuery public (Pavlo Golub)
* Add AppendRows function (Edoardo Spadolini)
* Optimize convert UUID [16]byte to string (Kirill Malikov)
* Fix: LargeObject Read and Write of more than ~1GB at a time (Mitar)
# 5.5.2 (January 13, 2024)
* Allow NamedArgs to start with underscore
* pgproto3: Maximum message body length support (jeremy.spriet)
* Upgrade golang.org/x/crypto to v0.17.0
* Add snake_case support to RowToStructByName (Tikhon Fedulov)
* Fix: update description cache after exec prepare (James Hartig)
* Fix: pipeline checks if it is closed (James Hartig and Ryan Fowler)
* Fix: normalize timeout / context errors during TLS startup (Samuel Stauffer)
* Add OnPgError for easier centralized error handling (James Hartig)
# 5.5.1 (December 9, 2023)
* Add CopyFromFunc helper function. (robford)
* Add PgConn.Deallocate method that uses PostgreSQL protocol Close message.
* pgx uses new PgConn.Deallocate method. This allows deallocating statements to work in a failed transaction. This fixes a case where the prepared statement map could become invalid.
* Fix: Prefer driver.Valuer over json.Marshaler for json fields. (Jacopo)
* Fix: simple protocol SQL sanitizer previously panicked if an invalid $0 placeholder was used. This now returns an error instead. (maksymnevajdev)
* Add pgtype.Numeric.ScanScientific (Eshton Robateau)
# 5.5.0 (November 4, 2023)
* Add CollectExactlyOneRow. (Julien GOTTELAND)
* Add OpenDBFromPool to create *database/sql.DB from *pgxpool.Pool. (Lev Zakharov)
* Prepare can automatically choose statement name based on sql. This makes it easier to explicitly manage prepared statements.
* Statement cache now uses deterministic, stable statement names.
* database/sql prepared statement names are deterministically generated.
* Fix: SendBatch wasn't respecting context cancellation.
* Fix: Timeout error from pipeline is now normalized.
* Fix: database/sql encoding json.RawMessage to []byte.
* CancelRequest: Wait for the cancel request to be acknowledged by the server. This should improve PgBouncer compatibility. (Anton Levakin)
* stdlib: Use Ping instead of CheckConn in ResetSession
* Add json.Marshaler and json.Unmarshaler for Float4, Float8 (Kirill Mironov)
# 5.4.3 (August 5, 2023)
* Fix: QCharArrayOID was defined with the wrong OID (Christoph Engelbert)
* Fix: connect_timeout for sslmode=allow|prefer (smaher-edb)
* Fix: pgxpool: background health check cannot overflow pool
* Fix: Check for nil in defer when sending batch (recover properly from panic)
* Fix: json scan of non-string pointer to pointer
* Fix: zeronull.Timestamptz should use pgtype.Timestamptz
* Fix: NewConnsCount was not correctly counting connections created by Acquire directly. (James Hartig)
* RowTo(AddrOf)StructByPos ignores fields with "-" db tag
* Optimization: improve text format numeric parsing (horpto)
# 5.4.2 (July 11, 2023)
* Fix: RowScanner errors are fatal to Rows
* Fix: Enable failover efforts when pg_hba.conf disallows non-ssl connections (Brandon Kauffman)
* Hstore text codec internal improvements (Evan Jones)
* Fix: Stop timers for background reader when not in use. Fixes memory leak when closing connections (Adrian-Stefan Mares)
* Fix: Stop background reader as soon as possible.
* Add PgConn.SyncConn(). This combined with the above fix makes it safe to directly use the underlying net.Conn.
# 5.4.1 (June 18, 2023)
* Fix: concurrency bug with pgtypeDefaultMap and simple protocol (Lev Zakharov)
* Add TxOptions.BeginQuery to allow overriding the default BEGIN query
# 5.4.0 (June 14, 2023)
* Replace platform specific syscalls for non-blocking IO with more traditional goroutines and deadlines. This returns to the v4 approach with some additional improvements and fixes. This restores the ability to use a pgx.Conn over an ssh.Conn as well as other non-TCP or Unix socket connections. In addition, it is a significantly simpler implementation that is less likely to have cross platform issues.
* Optimization: The default type registrations are now shared among all connections. This saves about 100KB of memory per connection. `pgtype.Type` and `pgtype.Codec` values are now required to be immutable after registration. This was already necessary in most cases but wasn't documented until now. (Lev Zakharov)
* Fix: Ensure pgxpool.Pool.QueryRow.Scan releases connection on panic
* CancelRequest: don't try to read the reply (Nicola Murino)
* Fix: correctly handle bool type aliases (Wichert Akkerman)
* Fix: pgconn.CancelRequest: Fix unix sockets: don't use RemoteAddr()
* Fix: pgx.Conn memory leak with prepared statement caching (Evan Jones)
* Add BeforeClose to pgxpool.Pool (Evan Cordell)
* Fix: various hstore fixes and optimizations (Evan Jones)
* Fix: RowToStructByPos with embedded unexported struct
* Support different bool string representations (Lev Zakharov)
* Fix: error when using BatchResults.Exec on a select that returns an error after some rows.
* Fix: pipelineBatchResults.Exec() not returning error from ResultReader
* Fix: pipeline batch results not closing pipeline when error occurs while reading directly from results instead of using
a callback.
* Fix: scanning a table type into a struct
* Fix: scan array of record to pointer to slice of struct
* Fix: handle null for json (Cemre Mengu)
* Batch Query callback is called even when there is an error
* Add RowTo(AddrOf)StructByNameLax (Audi P. Risa P)
# 5.3.1 (February 27, 2023)
* Fix: Support v4 and v5 stdlib in same program (Tomáš Procházka)
+2 -16
View File
@@ -79,20 +79,11 @@ echo "listen_addresses = '127.0.0.1'" >> .testdb/$POSTGRESQL_DATA_DIR/postgresql
echo "port = $PGPORT" >> .testdb/$POSTGRESQL_DATA_DIR/postgresql.conf
cat testsetup/postgresql_ssl.conf >> .testdb/$POSTGRESQL_DATA_DIR/postgresql.conf
cp testsetup/pg_hba.conf .testdb/$POSTGRESQL_DATA_DIR/pg_hba.conf
cp testsetup/ca.cnf .testdb
cp testsetup/localhost.cnf .testdb
cp testsetup/pgx_sslcert.cnf .testdb
cd .testdb
# Generate a CA public / private key pair.
openssl genrsa -out ca.key 4096
openssl req -x509 -config ca.cnf -new -nodes -key ca.key -sha256 -days 365 -subj '/O=pgx-test-root' -out ca.pem
# Generate the certificate for localhost (the server).
openssl genrsa -out localhost.key 2048
openssl req -new -config localhost.cnf -key localhost.key -out localhost.csr
openssl x509 -req -in localhost.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out localhost.crt -days 364 -sha256 -extfile localhost.cnf -extensions v3_req
# Generate CA, server, and encrypted client certificates.
go run ../testsetup/generate_certs.go
# Copy certificates to server directory and set permissions.
cp ca.pem $POSTGRESQL_DATA_DIR/root.crt
@@ -100,11 +91,6 @@ cp localhost.key $POSTGRESQL_DATA_DIR/server.key
chmod 600 $POSTGRESQL_DATA_DIR/server.key
cp localhost.crt $POSTGRESQL_DATA_DIR/server.crt
# Generate the certificate for client authentication.
openssl genrsa -des3 -out pgx_sslcert.key -passout pass:certpw 2048
openssl req -new -config pgx_sslcert.cnf -key pgx_sslcert.key -passin pass:certpw -out pgx_sslcert.csr
openssl x509 -req -in pgx_sslcert.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out pgx_sslcert.crt -days 363 -sha256 -extfile pgx_sslcert.cnf -extensions v3_req
cd ..
```
+19 -2
View File
@@ -1,5 +1,5 @@
[![Go Reference](https://pkg.go.dev/badge/github.com/jackc/pgx/v5.svg)](https://pkg.go.dev/github.com/jackc/pgx/v5)
![Build Status](https://github.com/jackc/pgx/actions/workflows/ci.yml/badge.svg)
[![Build Status](https://github.com/jackc/pgx/actions/workflows/ci.yml/badge.svg)](https://github.com/jackc/pgx/actions/workflows/ci.yml)
# pgx - PostgreSQL Driver and Toolkit
@@ -86,9 +86,13 @@ It is also possible to use the `database/sql` interface and convert a connection
See CONTRIBUTING.md for setup instructions.
## Architecture
See the presentation at Golang Estonia, [PGX Top to Bottom](https://www.youtube.com/watch?v=sXMSWhcHCf8) for a description of pgx architecture.
## Supported Go and PostgreSQL Versions
pgx supports the same versions of Go and PostgreSQL that are supported by their respective teams. For [Go](https://golang.org/doc/devel/release.html#policy) that is the two most recent major releases and for [PostgreSQL](https://www.postgresql.org/support/versioning/) the major releases in the last 5 years. This means pgx supports Go 1.19 and higher and PostgreSQL 11 and higher. pgx also is tested against the latest version of [CockroachDB](https://www.cockroachlabs.com/product/).
pgx supports the same versions of Go and PostgreSQL that are supported by their respective teams. For [Go](https://golang.org/doc/devel/release.html#policy) that is the two most recent major releases and for [PostgreSQL](https://www.postgresql.org/support/versioning/) the major releases in the last 5 years. This means pgx supports Go 1.20 and higher and PostgreSQL 12 and higher. pgx also is tested against the latest version of [CockroachDB](https://www.cockroachlabs.com/product/).
## Version Policy
@@ -116,6 +120,7 @@ pgerrcode contains constants for the PostgreSQL error codes.
* [github.com/jackc/pgx-gofrs-uuid](https://github.com/jackc/pgx-gofrs-uuid)
* [github.com/jackc/pgx-shopspring-decimal](https://github.com/jackc/pgx-shopspring-decimal)
* [github.com/twpayne/pgx-geos](https://github.com/twpayne/pgx-geos) ([PostGIS](https://postgis.net/) and [GEOS](https://libgeos.org/) via [go-geos](https://github.com/twpayne/go-geos))
* [github.com/vgarvardt/pgx-google-uuid](https://github.com/vgarvardt/pgx-google-uuid)
@@ -132,13 +137,25 @@ These adapters can be used with the tracelog package.
* [github.com/jackc/pgx-logrus](https://github.com/jackc/pgx-logrus)
* [github.com/jackc/pgx-zap](https://github.com/jackc/pgx-zap)
* [github.com/jackc/pgx-zerolog](https://github.com/jackc/pgx-zerolog)
* [github.com/mcosta74/pgx-slog](https://github.com/mcosta74/pgx-slog)
* [github.com/kataras/pgx-golog](https://github.com/kataras/pgx-golog)
## 3rd Party Libraries with PGX Support
### [github.com/pashagolub/pgxmock](https://github.com/pashagolub/pgxmock)
pgxmock is a mock library implementing pgx interfaces.
pgxmock has one and only purpose - to simulate pgx behavior in tests, without needing a real database connection.
### [github.com/georgysavva/scany](https://github.com/georgysavva/scany)
Library for scanning data from a database into Go structs and more.
### [github.com/vingarcia/ksql](https://github.com/vingarcia/ksql)
A carefully designed SQL client for making using SQL easier,
more productive, and less error-prone on Golang.
### [https://github.com/otan/gopgkrb5](https://github.com/otan/gopgkrb5)
Adds GSSAPI / Kerberos authentication support.
+35 -37
View File
@@ -10,8 +10,8 @@ import (
// QueuedQuery is a query that has been queued for execution via a Batch.
type QueuedQuery struct {
query string
arguments []any
SQL string
Arguments []any
fn batchItemFunc
sd *pgconn.StatementDescription
}
@@ -21,13 +21,10 @@ type batchItemFunc func(br BatchResults) error
// Query sets fn to be called when the response to qq is received.
func (qq *QueuedQuery) Query(fn func(rows Rows) error) {
qq.fn = func(br BatchResults) error {
rows, err := br.Query()
if err != nil {
return err
}
rows, _ := br.Query()
defer rows.Close()
err = fn(rows)
err := fn(rows)
if err != nil {
return err
}
@@ -60,22 +57,24 @@ func (qq *QueuedQuery) Exec(fn func(ct pgconn.CommandTag) error) {
// Batch queries are a way of bundling multiple queries together to avoid
// unnecessary network round trips. A Batch must only be sent once.
type Batch struct {
queuedQueries []*QueuedQuery
QueuedQueries []*QueuedQuery
}
// Queue queues a query to batch b. query can be an SQL query or the name of a prepared statement.
// The only pgx option argument that is supported is QueryRewriter. Queries are executed using the
// connection's DefaultQueryExecMode.
func (b *Batch) Queue(query string, arguments ...any) *QueuedQuery {
qq := &QueuedQuery{
query: query,
arguments: arguments,
SQL: query,
Arguments: arguments,
}
b.queuedQueries = append(b.queuedQueries, qq)
b.QueuedQueries = append(b.QueuedQueries, qq)
return qq
}
// Len returns number of queries that have been queued so far.
func (b *Batch) Len() int {
return len(b.queuedQueries)
return len(b.QueuedQueries)
}
type BatchResults interface {
@@ -142,7 +141,10 @@ func (br *batchResults) Exec() (pgconn.CommandTag, error) {
}
commandTag, err := br.mrr.ResultReader().Close()
br.err = err
if err != nil {
br.err = err
br.mrr.Close()
}
if br.conn.batchTracer != nil {
br.conn.batchTracer.TraceBatchQuery(br.ctx, br.conn, TraceBatchQueryData{
@@ -225,10 +227,10 @@ func (br *batchResults) Close() error {
}
// Read and run fn for all remaining items
for br.err == nil && !br.closed && br.b != nil && br.qqIdx < len(br.b.queuedQueries) {
if br.b.queuedQueries[br.qqIdx].fn != nil {
err := br.b.queuedQueries[br.qqIdx].fn(br)
if err != nil && br.err == nil {
for br.err == nil && !br.closed && br.b != nil && br.qqIdx < len(br.b.QueuedQueries) {
if br.b.QueuedQueries[br.qqIdx].fn != nil {
err := br.b.QueuedQueries[br.qqIdx].fn(br)
if err != nil {
br.err = err
}
} else {
@@ -251,10 +253,10 @@ func (br *batchResults) earlyError() error {
}
func (br *batchResults) nextQueryAndArgs() (query string, args []any, ok bool) {
if br.b != nil && br.qqIdx < len(br.b.queuedQueries) {
bi := br.b.queuedQueries[br.qqIdx]
query = bi.query
args = bi.arguments
if br.b != nil && br.qqIdx < len(br.b.QueuedQueries) {
bi := br.b.QueuedQueries[br.qqIdx]
query = bi.SQL
args = bi.Arguments
ok = true
br.qqIdx++
}
@@ -290,7 +292,7 @@ func (br *pipelineBatchResults) Exec() (pgconn.CommandTag, error) {
results, err := br.pipeline.GetResults()
if err != nil {
br.err = err
return pgconn.CommandTag{}, err
return pgconn.CommandTag{}, br.err
}
var commandTag pgconn.CommandTag
switch results := results.(type) {
@@ -309,7 +311,7 @@ func (br *pipelineBatchResults) Exec() (pgconn.CommandTag, error) {
})
}
return commandTag, err
return commandTag, br.err
}
// Query reads the results from the next query in the batch as if the query has been sent with Query.
@@ -384,24 +386,20 @@ func (br *pipelineBatchResults) Close() error {
}
}()
if br.err != nil {
return br.err
}
if br.lastRows != nil && br.lastRows.err != nil {
if br.err == nil && br.lastRows != nil && br.lastRows.err != nil {
br.err = br.lastRows.err
return br.err
}
if br.closed {
return nil
return br.err
}
// Read and run fn for all remaining items
for br.err == nil && !br.closed && br.b != nil && br.qqIdx < len(br.b.queuedQueries) {
if br.b.queuedQueries[br.qqIdx].fn != nil {
err := br.b.queuedQueries[br.qqIdx].fn(br)
if err != nil && br.err == nil {
for br.err == nil && !br.closed && br.b != nil && br.qqIdx < len(br.b.QueuedQueries) {
if br.b.QueuedQueries[br.qqIdx].fn != nil {
err := br.b.QueuedQueries[br.qqIdx].fn(br)
if err != nil {
br.err = err
}
} else {
@@ -424,10 +422,10 @@ func (br *pipelineBatchResults) earlyError() error {
}
func (br *pipelineBatchResults) nextQueryAndArgs() (query string, args []any, ok bool) {
if br.b != nil && br.qqIdx < len(br.b.queuedQueries) {
bi := br.b.queuedQueries[br.qqIdx]
query = bi.query
args = bi.arguments
if br.b != nil && br.qqIdx < len(br.b.QueuedQueries) {
bi := br.b.QueuedQueries[br.qqIdx]
query = bi.SQL
args = bi.Arguments
ok = true
br.qqIdx++
}
+140 -83
View File
@@ -2,6 +2,8 @@ package pgx
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"strconv"
@@ -35,7 +37,7 @@ type ConnConfig struct {
// DefaultQueryExecMode controls the default mode for executing queries. By default pgx uses the extended protocol
// and automatically prepares and caches prepared statements. However, this may be incompatible with proxies such as
// PGBouncer. In this case it may be preferrable to use QueryExecModeExec or QueryExecModeSimpleProtocol. The same
// PGBouncer. In this case it may be preferable to use QueryExecModeExec or QueryExecModeSimpleProtocol. The same
// functionality can be controlled on a per query basis by passing a QueryExecMode as the first query argument.
DefaultQueryExecMode QueryExecMode
@@ -99,8 +101,12 @@ func (ident Identifier) Sanitize() string {
return strings.Join(parts, ".")
}
// ErrNoRows occurs when rows are expected but none are returned.
var ErrNoRows = errors.New("no rows in result set")
var (
// ErrNoRows occurs when rows are expected but none are returned.
ErrNoRows = errors.New("no rows in result set")
// ErrTooManyRows occurs when more rows than expected are returned.
ErrTooManyRows = errors.New("too many rows in result set")
)
var errDisabledStatementCache = fmt.Errorf("cannot use QueryExecModeCacheStatement with disabled statement cache")
var errDisabledDescriptionCache = fmt.Errorf("cannot use QueryExecModeCacheDescribe with disabled description cache")
@@ -178,7 +184,7 @@ func ParseConfigWithOptions(connString string, options ParseConfigOptions) (*Con
case "simple_protocol":
defaultQueryExecMode = QueryExecModeSimpleProtocol
default:
return nil, fmt.Errorf("invalid default_query_exec_mode: %v", err)
return nil, fmt.Errorf("invalid default_query_exec_mode: %s", s)
}
}
@@ -194,7 +200,7 @@ func ParseConfigWithOptions(connString string, options ParseConfigOptions) (*Con
return connConfig, nil
}
// ParseConfig creates a ConnConfig from a connection string. ParseConfig handles all options that pgconn.ParseConfig
// ParseConfig creates a ConnConfig from a connection string. ParseConfig handles all options that [pgconn.ParseConfig]
// does. In addition, it accepts the following options:
//
// - default_query_exec_mode.
@@ -269,7 +275,7 @@ func connect(ctx context.Context, config *ConnConfig) (c *Conn, err error) {
return c, nil
}
// Close closes a connection. It is safe to call Close on a already closed
// Close closes a connection. It is safe to call Close on an already closed
// connection.
func (c *Conn) Close(ctx context.Context) error {
if c.IsClosed() {
@@ -280,12 +286,15 @@ func (c *Conn) Close(ctx context.Context) error {
return err
}
// Prepare creates a prepared statement with name and sql. sql can contain placeholders
// for bound parameters. These placeholders are referenced positional as $1, $2, etc.
// Prepare creates a prepared statement with name and sql. sql can contain placeholders for bound parameters. These
// placeholders are referenced positionally as $1, $2, etc. name can be used instead of sql with Query, QueryRow, and
// Exec to execute the statement. It can also be used with Batch.Queue.
//
// Prepare is idempotent; i.e. it is safe to call Prepare multiple times with the same
// name and sql arguments. This allows a code path to Prepare and Query/Exec without
// concern for if the statement has already been prepared.
// The underlying PostgreSQL identifier for the prepared statement will be name if name != sql or a digest of sql if
// name == sql.
//
// Prepare is idempotent; i.e. it is safe to call Prepare multiple times with the same name and sql arguments. This
// allows a code path to Prepare and Query/Exec without concern for if the statement has already been prepared.
func (c *Conn) Prepare(ctx context.Context, name, sql string) (sd *pgconn.StatementDescription, err error) {
if c.prepareTracer != nil {
ctx = c.prepareTracer.TracePrepareStart(ctx, c, TracePrepareStartData{Name: name, SQL: sql})
@@ -307,23 +316,48 @@ func (c *Conn) Prepare(ctx context.Context, name, sql string) (sd *pgconn.Statem
}()
}
sd, err = c.pgConn.Prepare(ctx, name, sql, nil)
var psName, psKey string
if name == sql {
digest := sha256.Sum256([]byte(sql))
psName = "stmt_" + hex.EncodeToString(digest[0:24])
psKey = sql
} else {
psName = name
psKey = name
}
sd, err = c.pgConn.Prepare(ctx, psName, sql, nil)
if err != nil {
return nil, err
}
if name != "" {
c.preparedStatements[name] = sd
if psKey != "" {
c.preparedStatements[psKey] = sd
}
return sd, nil
}
// Deallocate released a prepared statement
// Deallocate releases a prepared statement. Calling Deallocate on a non-existent prepared statement will succeed.
func (c *Conn) Deallocate(ctx context.Context, name string) error {
delete(c.preparedStatements, name)
_, err := c.pgConn.Exec(ctx, "deallocate "+quoteIdentifier(name)).ReadAll()
return err
var psName string
sd := c.preparedStatements[name]
if sd != nil {
psName = sd.Name
} else {
psName = name
}
err := c.pgConn.Deallocate(ctx, psName)
if err != nil {
return err
}
if sd != nil {
delete(c.preparedStatements, name)
}
return nil
}
// DeallocateAll releases all previously prepared statements from the server and client, where it also resets the statement and description cache.
@@ -382,11 +416,9 @@ func quoteIdentifier(s string) string {
return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
}
// Ping executes an empty sql statement against the *Conn
// If the sql returns without error, the database Ping is considered successful, otherwise, the error is returned.
// Ping delegates to the underlying *pgconn.PgConn.Ping.
func (c *Conn) Ping(ctx context.Context) error {
_, err := c.Exec(ctx, ";")
return err
return c.pgConn.Ping(ctx)
}
// PgConn returns the underlying *pgconn.PgConn. This is an escape hatch method that allows lower level access to the
@@ -443,7 +475,7 @@ optionLoop:
if queryRewriter != nil {
sql, arguments, err = queryRewriter.RewriteQuery(ctx, c, sql, arguments)
if err != nil {
return pgconn.CommandTag{}, fmt.Errorf("rewrite query failed: %v", err)
return pgconn.CommandTag{}, fmt.Errorf("rewrite query failed: %w", err)
}
}
@@ -463,7 +495,7 @@ optionLoop:
}
sd := c.statementCache.Get(sql)
if sd == nil {
sd, err = c.Prepare(ctx, stmtcache.NextStatementName(), sql)
sd, err = c.Prepare(ctx, stmtcache.StatementName(sql), sql)
if err != nil {
return pgconn.CommandTag{}, err
}
@@ -481,6 +513,7 @@ optionLoop:
if err != nil {
return pgconn.CommandTag{}, err
}
c.descriptionCache.Put(sd)
}
return c.execParams(ctx, sd, arguments)
@@ -509,7 +542,7 @@ func (c *Conn) execSimpleProtocol(ctx context.Context, sql string, arguments []a
mrr := c.pgConn.Exec(ctx, sql)
for mrr.NextResult() {
commandTag, err = mrr.ResultReader().Close()
commandTag, _ = mrr.ResultReader().Close()
}
err = mrr.Close()
return commandTag, err
@@ -575,30 +608,35 @@ type QueryExecMode int32
const (
_ QueryExecMode = iota
// Automatically prepare and cache statements. This uses the extended protocol. Queries are executed in a single
// round trip after the statement is cached. This is the default.
// Automatically prepare and cache statements. This uses the extended protocol. Queries are executed in a single round
// trip after the statement is cached. This is the default. If the database schema is modified or the search_path is
// changed after a statement is cached then the first execution of a previously cached query may fail. e.g. If the
// number of columns returned by a "SELECT *" changes or the type of a column is changed.
QueryExecModeCacheStatement
// Cache statement descriptions (i.e. argument and result types) and assume they do not change. This uses the
// extended protocol. Queries are executed in a single round trip after the description is cached. If the database
// schema is modified or the search_path is changed this may result in undetected result decoding errors.
// Cache statement descriptions (i.e. argument and result types) and assume they do not change. This uses the extended
// protocol. Queries are executed in a single round trip after the description is cached. If the database schema is
// modified or the search_path is changed after a statement is cached then the first execution of a previously cached
// query may fail. e.g. If the number of columns returned by a "SELECT *" changes or the type of a column is changed.
QueryExecModeCacheDescribe
// Get the statement description on every execution. This uses the extended protocol. Queries require two round trips
// to execute. It does not use prepared statements (allowing usage with most connection poolers) and is safe even
// when the the database schema is modified concurrently.
// to execute. It does not use named prepared statements. But it does use the unnamed prepared statement to get the
// statement description on the first round trip and then uses it to execute the query on the second round trip. This
// may cause problems with connection poolers that switch the underlying connection between round trips. It is safe
// even when the the database schema is modified concurrently.
QueryExecModeDescribeExec
// Assume the PostgreSQL query parameter types based on the Go type of the arguments. This uses the extended protocol
// with text formatted parameters and results. Queries are executed in a single round trip. Type mappings can be
// registered with pgtype.Map.RegisterDefaultPgType. Queries will be rejected that have arguments that are
// unregistered or ambigious. e.g. A map[string]string may have the PostgreSQL type json or hstore. Modes that know
// unregistered or ambiguous. e.g. A map[string]string may have the PostgreSQL type json or hstore. Modes that know
// the PostgreSQL type can use a map[string]string directly as an argument. This mode cannot.
QueryExecModeExec
// Use the simple protocol. Assume the PostgreSQL query parameter types based on the Go type of the arguments.
// Queries are executed in a single round trip. Type mappings can be registered with
// pgtype.Map.RegisterDefaultPgType. Queries will be rejected that have arguments that are unregistered or ambigious.
// pgtype.Map.RegisterDefaultPgType. Queries will be rejected that have arguments that are unregistered or ambiguous.
// e.g. A map[string]string may have the PostgreSQL type json or hstore. Modes that know the PostgreSQL type can use
// a map[string]string directly as an argument. This mode cannot.
//
@@ -648,6 +686,9 @@ type QueryRewriter interface {
// returned Rows even if an error is returned. The error will be the available in rows.Err() after rows are closed. It
// is allowed to ignore the error returned from Query and handle it in Rows.
//
// It is possible for a call of FieldDescriptions on the returned Rows to return nil even if the Query call did not
// return an error.
//
// It is possible for a query to return one or more rows before encountering an error. In most cases the rows should be
// collected before processing rather than processed while receiving each row. This avoids the possibility of the
// application processing rows from a query that the server rejected. The CollectRows function is useful here.
@@ -702,7 +743,7 @@ optionLoop:
sql, args, err = queryRewriter.RewriteQuery(ctx, c, sql, args)
if err != nil {
rows := c.getRows(ctx, originalSQL, originalArgs)
err = fmt.Errorf("rewrite query failed: %v", err)
err = fmt.Errorf("rewrite query failed: %w", err)
rows.fatal(err)
return rows, err
}
@@ -812,7 +853,7 @@ func (c *Conn) getStatementDescription(
}
sd = c.statementCache.Get(sql)
if sd == nil {
sd, err = c.Prepare(ctx, stmtcache.NextStatementName(), sql)
sd, err = c.Prepare(ctx, stmtcache.StatementName(sql), sql)
if err != nil {
return nil, err
}
@@ -862,15 +903,14 @@ func (c *Conn) SendBatch(ctx context.Context, b *Batch) (br BatchResults) {
return &batchResults{ctx: ctx, conn: c, err: err}
}
mode := c.config.DefaultQueryExecMode
for _, bi := range b.queuedQueries {
for _, bi := range b.QueuedQueries {
var queryRewriter QueryRewriter
sql := bi.query
arguments := bi.arguments
sql := bi.SQL
arguments := bi.Arguments
optionLoop:
for len(arguments) > 0 {
// Update Batch.Queue function comment when additional options are implemented
switch arg := arguments[0].(type) {
case QueryRewriter:
queryRewriter = arg
@@ -884,21 +924,23 @@ func (c *Conn) SendBatch(ctx context.Context, b *Batch) (br BatchResults) {
var err error
sql, arguments, err = queryRewriter.RewriteQuery(ctx, c, sql, arguments)
if err != nil {
return &batchResults{ctx: ctx, conn: c, err: fmt.Errorf("rewrite query failed: %v", err)}
return &batchResults{ctx: ctx, conn: c, err: fmt.Errorf("rewrite query failed: %w", err)}
}
}
bi.query = sql
bi.arguments = arguments
bi.SQL = sql
bi.Arguments = arguments
}
// TODO: changing mode per batch? Update Batch.Queue function comment when implemented
mode := c.config.DefaultQueryExecMode
if mode == QueryExecModeSimpleProtocol {
return c.sendBatchQueryExecModeSimpleProtocol(ctx, b)
}
// All other modes use extended protocol and thus can use prepared statements.
for _, bi := range b.queuedQueries {
if sd, ok := c.preparedStatements[bi.query]; ok {
for _, bi := range b.QueuedQueries {
if sd, ok := c.preparedStatements[bi.SQL]; ok {
bi.sd = sd
}
}
@@ -919,11 +961,11 @@ func (c *Conn) SendBatch(ctx context.Context, b *Batch) (br BatchResults) {
func (c *Conn) sendBatchQueryExecModeSimpleProtocol(ctx context.Context, b *Batch) *batchResults {
var sb strings.Builder
for i, bi := range b.queuedQueries {
for i, bi := range b.QueuedQueries {
if i > 0 {
sb.WriteByte(';')
}
sql, err := c.sanitizeForSimpleQuery(bi.query, bi.arguments...)
sql, err := c.sanitizeForSimpleQuery(bi.SQL, bi.Arguments...)
if err != nil {
return &batchResults{ctx: ctx, conn: c, err: err}
}
@@ -942,21 +984,21 @@ func (c *Conn) sendBatchQueryExecModeSimpleProtocol(ctx context.Context, b *Batc
func (c *Conn) sendBatchQueryExecModeExec(ctx context.Context, b *Batch) *batchResults {
batch := &pgconn.Batch{}
for _, bi := range b.queuedQueries {
for _, bi := range b.QueuedQueries {
sd := bi.sd
if sd != nil {
err := c.eqb.Build(c.typeMap, sd, bi.arguments)
err := c.eqb.Build(c.typeMap, sd, bi.Arguments)
if err != nil {
return &batchResults{ctx: ctx, conn: c, err: err}
}
batch.ExecPrepared(sd.Name, c.eqb.ParamValues, c.eqb.ParamFormats, c.eqb.ResultFormats)
} else {
err := c.eqb.Build(c.typeMap, nil, bi.arguments)
err := c.eqb.Build(c.typeMap, nil, bi.Arguments)
if err != nil {
return &batchResults{ctx: ctx, conn: c, err: err}
}
batch.ExecParams(bi.query, c.eqb.ParamValues, nil, c.eqb.ParamFormats, c.eqb.ResultFormats)
batch.ExecParams(bi.SQL, c.eqb.ParamValues, nil, c.eqb.ParamFormats, c.eqb.ResultFormats)
}
}
@@ -975,24 +1017,24 @@ func (c *Conn) sendBatchQueryExecModeExec(ctx context.Context, b *Batch) *batchR
func (c *Conn) sendBatchQueryExecModeCacheStatement(ctx context.Context, b *Batch) (pbr *pipelineBatchResults) {
if c.statementCache == nil {
return &pipelineBatchResults{ctx: ctx, conn: c, err: errDisabledStatementCache}
return &pipelineBatchResults{ctx: ctx, conn: c, err: errDisabledStatementCache, closed: true}
}
distinctNewQueries := []*pgconn.StatementDescription{}
distinctNewQueriesIdxMap := make(map[string]int)
for _, bi := range b.queuedQueries {
for _, bi := range b.QueuedQueries {
if bi.sd == nil {
sd := c.statementCache.Get(bi.query)
sd := c.statementCache.Get(bi.SQL)
if sd != nil {
bi.sd = sd
} else {
if idx, present := distinctNewQueriesIdxMap[bi.query]; present {
if idx, present := distinctNewQueriesIdxMap[bi.SQL]; present {
bi.sd = distinctNewQueries[idx]
} else {
sd = &pgconn.StatementDescription{
Name: stmtcache.NextStatementName(),
SQL: bi.query,
Name: stmtcache.StatementName(bi.SQL),
SQL: bi.SQL,
}
distinctNewQueriesIdxMap[sd.SQL] = len(distinctNewQueries)
distinctNewQueries = append(distinctNewQueries, sd)
@@ -1007,23 +1049,23 @@ func (c *Conn) sendBatchQueryExecModeCacheStatement(ctx context.Context, b *Batc
func (c *Conn) sendBatchQueryExecModeCacheDescribe(ctx context.Context, b *Batch) (pbr *pipelineBatchResults) {
if c.descriptionCache == nil {
return &pipelineBatchResults{ctx: ctx, conn: c, err: errDisabledDescriptionCache}
return &pipelineBatchResults{ctx: ctx, conn: c, err: errDisabledDescriptionCache, closed: true}
}
distinctNewQueries := []*pgconn.StatementDescription{}
distinctNewQueriesIdxMap := make(map[string]int)
for _, bi := range b.queuedQueries {
for _, bi := range b.QueuedQueries {
if bi.sd == nil {
sd := c.descriptionCache.Get(bi.query)
sd := c.descriptionCache.Get(bi.SQL)
if sd != nil {
bi.sd = sd
} else {
if idx, present := distinctNewQueriesIdxMap[bi.query]; present {
if idx, present := distinctNewQueriesIdxMap[bi.SQL]; present {
bi.sd = distinctNewQueries[idx]
} else {
sd = &pgconn.StatementDescription{
SQL: bi.query,
SQL: bi.SQL,
}
distinctNewQueriesIdxMap[sd.SQL] = len(distinctNewQueries)
distinctNewQueries = append(distinctNewQueries, sd)
@@ -1040,13 +1082,13 @@ func (c *Conn) sendBatchQueryExecModeDescribeExec(ctx context.Context, b *Batch)
distinctNewQueries := []*pgconn.StatementDescription{}
distinctNewQueriesIdxMap := make(map[string]int)
for _, bi := range b.queuedQueries {
for _, bi := range b.QueuedQueries {
if bi.sd == nil {
if idx, present := distinctNewQueriesIdxMap[bi.query]; present {
if idx, present := distinctNewQueriesIdxMap[bi.SQL]; present {
bi.sd = distinctNewQueries[idx]
} else {
sd := &pgconn.StatementDescription{
SQL: bi.query,
SQL: bi.SQL,
}
distinctNewQueriesIdxMap[sd.SQL] = len(distinctNewQueries)
distinctNewQueries = append(distinctNewQueries, sd)
@@ -1059,9 +1101,9 @@ func (c *Conn) sendBatchQueryExecModeDescribeExec(ctx context.Context, b *Batch)
}
func (c *Conn) sendBatchExtendedWithDescription(ctx context.Context, b *Batch, distinctNewQueries []*pgconn.StatementDescription, sdCache stmtcache.Cache) (pbr *pipelineBatchResults) {
pipeline := c.pgConn.StartPipeline(context.Background())
pipeline := c.pgConn.StartPipeline(ctx)
defer func() {
if pbr.err != nil {
if pbr != nil && pbr.err != nil {
pipeline.Close()
}
}()
@@ -1074,18 +1116,18 @@ func (c *Conn) sendBatchExtendedWithDescription(ctx context.Context, b *Batch, d
err := pipeline.Sync()
if err != nil {
return &pipelineBatchResults{ctx: ctx, conn: c, err: err}
return &pipelineBatchResults{ctx: ctx, conn: c, err: err, closed: true}
}
for _, sd := range distinctNewQueries {
results, err := pipeline.GetResults()
if err != nil {
return &pipelineBatchResults{ctx: ctx, conn: c, err: err}
return &pipelineBatchResults{ctx: ctx, conn: c, err: err, closed: true}
}
resultSD, ok := results.(*pgconn.StatementDescription)
if !ok {
return &pipelineBatchResults{ctx: ctx, conn: c, err: fmt.Errorf("expected statement description, got %T", results)}
return &pipelineBatchResults{ctx: ctx, conn: c, err: fmt.Errorf("expected statement description, got %T", results), closed: true}
}
// Fill in the previously empty / pending statement descriptions.
@@ -1095,12 +1137,12 @@ func (c *Conn) sendBatchExtendedWithDescription(ctx context.Context, b *Batch, d
results, err := pipeline.GetResults()
if err != nil {
return &pipelineBatchResults{ctx: ctx, conn: c, err: err}
return &pipelineBatchResults{ctx: ctx, conn: c, err: err, closed: true}
}
_, ok := results.(*pgconn.PipelineSync)
if !ok {
return &pipelineBatchResults{ctx: ctx, conn: c, err: fmt.Errorf("expected sync, got %T", results)}
return &pipelineBatchResults{ctx: ctx, conn: c, err: fmt.Errorf("expected sync, got %T", results), closed: true}
}
}
@@ -1112,12 +1154,12 @@ func (c *Conn) sendBatchExtendedWithDescription(ctx context.Context, b *Batch, d
}
// Queue the queries.
for _, bi := range b.queuedQueries {
err := c.eqb.Build(c.typeMap, bi.sd, bi.arguments)
for _, bi := range b.QueuedQueries {
err := c.eqb.Build(c.typeMap, bi.sd, bi.Arguments)
if err != nil {
// we wrap the error so we the user can understand which query failed inside the batch
err = fmt.Errorf("error building query %s: %w", bi.query, err)
return &pipelineBatchResults{ctx: ctx, conn: c, err: err}
err = fmt.Errorf("error building query %s: %w", bi.SQL, err)
return &pipelineBatchResults{ctx: ctx, conn: c, err: err, closed: true}
}
if bi.sd.Name == "" {
@@ -1129,7 +1171,7 @@ func (c *Conn) sendBatchExtendedWithDescription(ctx context.Context, b *Batch, d
err := pipeline.Sync()
if err != nil {
return &pipelineBatchResults{ctx: ctx, conn: c, err: err}
return &pipelineBatchResults{ctx: ctx, conn: c, err: err, closed: true}
}
return &pipelineBatchResults{
@@ -1161,7 +1203,15 @@ func (c *Conn) sanitizeForSimpleQuery(sql string, args ...any) (string, error) {
return sanitize.SanitizeSQL(sql, valueArgs...)
}
// LoadType inspects the database for typeName and produces a pgtype.Type suitable for registration.
// LoadType inspects the database for typeName and produces a pgtype.Type suitable for registration. typeName must be
// the name of a type where the underlying type(s) is already understood by pgx. It is for derived types. In particular,
// typeName must be one of the following:
// - An array type name of a type that is already registered. e.g. "_foo" when "foo" is registered.
// - A composite type name where all field types are already registered.
// - A domain type name where the base type is already registered.
// - An enum type name.
// - A range type name where the element type is already registered.
// - A multirange type name where the element type is already registered.
func (c *Conn) LoadType(ctx context.Context, typeName string) (*pgtype.Type, error) {
var oid uint32
@@ -1282,7 +1332,9 @@ func (c *Conn) getCompositeFields(ctx context.Context, oid uint32) ([]pgtype.Com
var fieldOID uint32
rows, _ := c.Query(ctx, `select attname, atttypid
from pg_attribute
where attrelid=$1 and not attisdropped
where attrelid=$1
and not attisdropped
and attnum > 0
order by attnum`,
typrelid,
)
@@ -1302,17 +1354,17 @@ order by attnum`,
}
func (c *Conn) deallocateInvalidatedCachedStatements(ctx context.Context) error {
if c.pgConn.TxStatus() != 'I' {
if txStatus := c.pgConn.TxStatus(); txStatus != 'I' && txStatus != 'T' {
return nil
}
if c.descriptionCache != nil {
c.descriptionCache.HandleInvalidated()
c.descriptionCache.RemoveInvalidated()
}
var invalidatedStatements []*pgconn.StatementDescription
if c.statementCache != nil {
invalidatedStatements = c.statementCache.HandleInvalidated()
invalidatedStatements = c.statementCache.GetInvalidated()
}
if len(invalidatedStatements) == 0 {
@@ -1336,5 +1388,10 @@ func (c *Conn) deallocateInvalidatedCachedStatements(ctx context.Context) error
return fmt.Errorf("failed to deallocate cached statement(s): %w", err)
}
c.statementCache.RemoveInvalidated()
for _, sd := range invalidatedStatements {
delete(c.preparedStatements, sd.Name)
}
return nil
}
+27
View File
@@ -64,6 +64,33 @@ func (cts *copyFromSlice) Err() error {
return cts.err
}
// CopyFromFunc returns a CopyFromSource interface that relies on nxtf for values.
// nxtf returns rows until it either signals an 'end of data' by returning row=nil and err=nil,
// or it returns an error. If nxtf returns an error, the copy is aborted.
func CopyFromFunc(nxtf func() (row []any, err error)) CopyFromSource {
return &copyFromFunc{next: nxtf}
}
type copyFromFunc struct {
next func() ([]any, error)
valueRow []any
err error
}
func (g *copyFromFunc) Next() bool {
g.valueRow, g.err = g.next()
// only return true if valueRow exists and no error
return g.valueRow != nil && g.err == nil
}
func (g *copyFromFunc) Values() ([]any, error) {
return g.valueRow, g.err
}
func (g *copyFromFunc) Err() error {
return g.err
}
// CopyFromSource is the interface used by *Conn.CopyFrom as the source for copy data.
type CopyFromSource interface {
// Next returns true if there is another row and makes the next row data
+5 -5
View File
@@ -7,17 +7,17 @@ details.
Establishing a Connection
The primary way of establishing a connection is with `pgx.Connect`.
The primary way of establishing a connection is with [pgx.Connect]:
conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL"))
The database connection string can be in URL or DSN format. Both PostgreSQL settings and pgx settings can be specified
here. In addition, a config struct can be created by `ParseConfig` and modified before establishing the connection with
`ConnectConfig` to configure settings such as tracing that cannot be configured with a connection string.
here. In addition, a config struct can be created by [ParseConfig] and modified before establishing the connection with
[ConnectConfig] to configure settings such as tracing that cannot be configured with a connection string.
Connection Pool
`*pgx.Conn` represents a single connection to the database and is not concurrency safe. Use package
[*pgx.Conn] represents a single connection to the database and is not concurrency safe. Use package
github.com/jackc/pgx/v5/pgxpool for a concurrency safe connection pool.
Query Interface
@@ -187,7 +187,7 @@ implemented on top of pgconn. The Conn.PgConn() method can be used to access thi
PgBouncer
By default pgx automatically uses prepared statements. Prepared statements are incompaptible with PgBouncer. This can be
By default pgx automatically uses prepared statements. Prepared statements are incompatible with PgBouncer. This can be
disabled by setting a different QueryExecMode in ConnConfig.DefaultQueryExecMode.
*/
package pgx
+1 -1
View File
@@ -36,7 +36,7 @@ func (eqb *ExtendedQueryBuilder) Build(m *pgtype.Map, sd *pgconn.StatementDescri
for i := range args {
err := eqb.appendParam(m, sd.ParamOIDs[i], -1, args[i])
if err != nil {
err = fmt.Errorf("failed to encode args[%d]: %v", i, err)
err = fmt.Errorf("failed to encode args[%d]: %w", i, err)
return err
}
}
-70
View File
@@ -1,70 +0,0 @@
package nbconn
import (
"sync"
)
const minBufferQueueLen = 8
type bufferQueue struct {
lock sync.Mutex
queue []*[]byte
r, w int
}
func (bq *bufferQueue) pushBack(buf *[]byte) {
bq.lock.Lock()
defer bq.lock.Unlock()
if bq.w >= len(bq.queue) {
bq.growQueue()
}
bq.queue[bq.w] = buf
bq.w++
}
func (bq *bufferQueue) pushFront(buf *[]byte) {
bq.lock.Lock()
defer bq.lock.Unlock()
if bq.w >= len(bq.queue) {
bq.growQueue()
}
copy(bq.queue[bq.r+1:bq.w+1], bq.queue[bq.r:bq.w])
bq.queue[bq.r] = buf
bq.w++
}
func (bq *bufferQueue) popFront() *[]byte {
bq.lock.Lock()
defer bq.lock.Unlock()
if bq.r == bq.w {
return nil
}
buf := bq.queue[bq.r]
bq.queue[bq.r] = nil // Clear reference so it can be garbage collected.
bq.r++
if bq.r == bq.w {
bq.r = 0
bq.w = 0
if len(bq.queue) > minBufferQueueLen {
bq.queue = make([]*[]byte, minBufferQueueLen)
}
}
return buf
}
func (bq *bufferQueue) growQueue() {
desiredLen := (len(bq.queue) + 1) * 3 / 2
if desiredLen < minBufferQueueLen {
desiredLen = minBufferQueueLen
}
newQueue := make([]*[]byte, desiredLen)
copy(newQueue, bq.queue)
bq.queue = newQueue
}
-520
View File
@@ -1,520 +0,0 @@
// Package nbconn implements a non-blocking net.Conn wrapper.
//
// It is designed to solve three problems.
//
// The first is resolving the deadlock that can occur when both sides of a connection are blocked writing because all
// buffers between are full. See https://github.com/jackc/pgconn/issues/27 for discussion.
//
// The second is the inability to use a write deadline with a TLS.Conn without killing the connection.
//
// The third is to efficiently check if a connection has been closed via a non-blocking read.
package nbconn
import (
"crypto/tls"
"errors"
"net"
"os"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/jackc/pgx/v5/internal/iobufpool"
)
var errClosed = errors.New("closed")
var ErrWouldBlock = new(wouldBlockError)
const fakeNonblockingWriteWaitDuration = 100 * time.Millisecond
const minNonblockingReadWaitDuration = time.Microsecond
const maxNonblockingReadWaitDuration = 100 * time.Millisecond
// NonBlockingDeadline is a magic value that when passed to Set[Read]Deadline places the connection in non-blocking read
// mode.
var NonBlockingDeadline = time.Date(1900, 1, 1, 0, 0, 0, 608536336, time.UTC)
// disableSetDeadlineDeadline is a magic value that when passed to Set[Read|Write]Deadline causes those methods to
// ignore all future calls.
var disableSetDeadlineDeadline = time.Date(1900, 1, 1, 0, 0, 0, 968549727, time.UTC)
// wouldBlockError implements net.Error so tls.Conn will recognize ErrWouldBlock as a temporary error.
type wouldBlockError struct{}
func (*wouldBlockError) Error() string {
return "would block"
}
func (*wouldBlockError) Timeout() bool { return true }
func (*wouldBlockError) Temporary() bool { return true }
// Conn is a net.Conn where Write never blocks and always succeeds. Flush or Read must be called to actually write to
// the underlying connection.
type Conn interface {
net.Conn
// Flush flushes any buffered writes.
Flush() error
// BufferReadUntilBlock reads and buffers any successfully read bytes until the read would block.
BufferReadUntilBlock() error
}
// NetConn is a non-blocking net.Conn wrapper. It implements net.Conn.
type NetConn struct {
// 64 bit fields accessed with atomics must be at beginning of struct to guarantee alignment for certain 32-bit
// architectures. See BUGS section of https://pkg.go.dev/sync/atomic and https://github.com/jackc/pgx/issues/1288 and
// https://github.com/jackc/pgx/issues/1307. Only access with atomics
closed int64 // 0 = not closed, 1 = closed
conn net.Conn
rawConn syscall.RawConn
readQueue bufferQueue
writeQueue bufferQueue
readFlushLock sync.Mutex
// non-blocking writes with syscall.RawConn are done with a callback function. By using these fields instead of the
// callback functions closure to pass the buf argument and receive the n and err results we avoid some allocations.
nonblockWriteFunc func(fd uintptr) (done bool)
nonblockWriteBuf []byte
nonblockWriteErr error
nonblockWriteN int
// non-blocking reads with syscall.RawConn are done with a callback function. By using these fields instead of the
// callback functions closure to pass the buf argument and receive the n and err results we avoid some allocations.
nonblockReadFunc func(fd uintptr) (done bool)
nonblockReadBuf []byte
nonblockReadErr error
nonblockReadN int
readDeadlineLock sync.Mutex
readDeadline time.Time
readNonblocking bool
fakeNonBlockingShortReadCount int
fakeNonblockingReadWaitDuration time.Duration
writeDeadlineLock sync.Mutex
writeDeadline time.Time
}
func NewNetConn(conn net.Conn, fakeNonBlockingIO bool) *NetConn {
nc := &NetConn{
conn: conn,
fakeNonblockingReadWaitDuration: maxNonblockingReadWaitDuration,
}
if !fakeNonBlockingIO {
if sc, ok := conn.(syscall.Conn); ok {
if rawConn, err := sc.SyscallConn(); err == nil {
nc.rawConn = rawConn
}
}
}
return nc
}
// Read implements io.Reader.
func (c *NetConn) Read(b []byte) (n int, err error) {
if c.isClosed() {
return 0, errClosed
}
c.readFlushLock.Lock()
defer c.readFlushLock.Unlock()
err = c.flush()
if err != nil {
return 0, err
}
for n < len(b) {
buf := c.readQueue.popFront()
if buf == nil {
break
}
copiedN := copy(b[n:], *buf)
if copiedN < len(*buf) {
*buf = (*buf)[copiedN:]
c.readQueue.pushFront(buf)
} else {
iobufpool.Put(buf)
}
n += copiedN
}
// If any bytes were already buffered return them without trying to do a Read. Otherwise, when the caller is trying to
// Read up to len(b) bytes but all available bytes have already been buffered the underlying Read would block.
if n > 0 {
return n, nil
}
var readNonblocking bool
c.readDeadlineLock.Lock()
readNonblocking = c.readNonblocking
c.readDeadlineLock.Unlock()
var readN int
if readNonblocking {
readN, err = c.nonblockingRead(b[n:])
} else {
readN, err = c.conn.Read(b[n:])
}
n += readN
return n, err
}
// Write implements io.Writer. It never blocks due to buffering all writes. It will only return an error if the Conn is
// closed. Call Flush to actually write to the underlying connection.
func (c *NetConn) Write(b []byte) (n int, err error) {
if c.isClosed() {
return 0, errClosed
}
buf := iobufpool.Get(len(b))
copy(*buf, b)
c.writeQueue.pushBack(buf)
return len(b), nil
}
func (c *NetConn) Close() (err error) {
swapped := atomic.CompareAndSwapInt64(&c.closed, 0, 1)
if !swapped {
return errClosed
}
defer func() {
closeErr := c.conn.Close()
if err == nil {
err = closeErr
}
}()
c.readFlushLock.Lock()
defer c.readFlushLock.Unlock()
err = c.flush()
if err != nil {
return err
}
return nil
}
func (c *NetConn) LocalAddr() net.Addr {
return c.conn.LocalAddr()
}
func (c *NetConn) RemoteAddr() net.Addr {
return c.conn.RemoteAddr()
}
// SetDeadline is the equivalent of calling SetReadDealine(t) and SetWriteDeadline(t).
func (c *NetConn) SetDeadline(t time.Time) error {
err := c.SetReadDeadline(t)
if err != nil {
return err
}
return c.SetWriteDeadline(t)
}
// SetReadDeadline sets the read deadline as t. If t == NonBlockingDeadline then future reads will be non-blocking.
func (c *NetConn) SetReadDeadline(t time.Time) error {
if c.isClosed() {
return errClosed
}
c.readDeadlineLock.Lock()
defer c.readDeadlineLock.Unlock()
if c.readDeadline == disableSetDeadlineDeadline {
return nil
}
if t == disableSetDeadlineDeadline {
c.readDeadline = t
return nil
}
if t == NonBlockingDeadline {
c.readNonblocking = true
t = time.Time{}
} else {
c.readNonblocking = false
}
c.readDeadline = t
return c.conn.SetReadDeadline(t)
}
func (c *NetConn) SetWriteDeadline(t time.Time) error {
if c.isClosed() {
return errClosed
}
c.writeDeadlineLock.Lock()
defer c.writeDeadlineLock.Unlock()
if c.writeDeadline == disableSetDeadlineDeadline {
return nil
}
if t == disableSetDeadlineDeadline {
c.writeDeadline = t
return nil
}
c.writeDeadline = t
return c.conn.SetWriteDeadline(t)
}
func (c *NetConn) Flush() error {
if c.isClosed() {
return errClosed
}
c.readFlushLock.Lock()
defer c.readFlushLock.Unlock()
return c.flush()
}
// flush does the actual work of flushing the writeQueue. readFlushLock must already be held.
func (c *NetConn) flush() error {
var stopChan chan struct{}
var errChan chan error
defer func() {
if stopChan != nil {
select {
case stopChan <- struct{}{}:
case <-errChan:
}
}
}()
for buf := c.writeQueue.popFront(); buf != nil; buf = c.writeQueue.popFront() {
remainingBuf := *buf
for len(remainingBuf) > 0 {
n, err := c.nonblockingWrite(remainingBuf)
remainingBuf = remainingBuf[n:]
if err != nil {
if !errors.Is(err, ErrWouldBlock) {
*buf = (*buf)[:len(remainingBuf)]
copy(*buf, remainingBuf)
c.writeQueue.pushFront(buf)
return err
}
// Writing was blocked. Reading might unblock it.
if stopChan == nil {
stopChan, errChan = c.bufferNonblockingRead()
}
select {
case err := <-errChan:
stopChan = nil
return err
default:
}
}
}
iobufpool.Put(buf)
}
return nil
}
func (c *NetConn) BufferReadUntilBlock() error {
for {
buf := iobufpool.Get(8 * 1024)
n, err := c.nonblockingRead(*buf)
if n > 0 {
*buf = (*buf)[:n]
c.readQueue.pushBack(buf)
} else if n == 0 {
iobufpool.Put(buf)
}
if err != nil {
if errors.Is(err, ErrWouldBlock) {
return nil
} else {
return err
}
}
}
}
func (c *NetConn) bufferNonblockingRead() (stopChan chan struct{}, errChan chan error) {
stopChan = make(chan struct{})
errChan = make(chan error, 1)
go func() {
for {
err := c.BufferReadUntilBlock()
if err != nil {
errChan <- err
return
}
select {
case <-stopChan:
return
default:
}
}
}()
return stopChan, errChan
}
func (c *NetConn) isClosed() bool {
closed := atomic.LoadInt64(&c.closed)
return closed == 1
}
func (c *NetConn) nonblockingWrite(b []byte) (n int, err error) {
if c.rawConn == nil {
return c.fakeNonblockingWrite(b)
} else {
return c.realNonblockingWrite(b)
}
}
func (c *NetConn) fakeNonblockingWrite(b []byte) (n int, err error) {
c.writeDeadlineLock.Lock()
defer c.writeDeadlineLock.Unlock()
deadline := time.Now().Add(fakeNonblockingWriteWaitDuration)
if c.writeDeadline.IsZero() || deadline.Before(c.writeDeadline) {
err = c.conn.SetWriteDeadline(deadline)
if err != nil {
return 0, err
}
defer func() {
// Ignoring error resetting deadline as there is nothing that can reasonably be done if it fails.
c.conn.SetWriteDeadline(c.writeDeadline)
if err != nil {
if errors.Is(err, os.ErrDeadlineExceeded) {
err = ErrWouldBlock
}
}
}()
}
return c.conn.Write(b)
}
func (c *NetConn) nonblockingRead(b []byte) (n int, err error) {
if c.rawConn == nil {
return c.fakeNonblockingRead(b)
} else {
return c.realNonblockingRead(b)
}
}
func (c *NetConn) fakeNonblockingRead(b []byte) (n int, err error) {
c.readDeadlineLock.Lock()
defer c.readDeadlineLock.Unlock()
// The first 5 reads only read 1 byte at a time. This should give us 4 chances to read when we are sure the bytes are
// already in Go or the OS's receive buffer.
if c.fakeNonBlockingShortReadCount < 5 && len(b) > 0 && c.fakeNonblockingReadWaitDuration < minNonblockingReadWaitDuration {
b = b[:1]
}
startTime := time.Now()
deadline := startTime.Add(c.fakeNonblockingReadWaitDuration)
if c.readDeadline.IsZero() || deadline.Before(c.readDeadline) {
err = c.conn.SetReadDeadline(deadline)
if err != nil {
return 0, err
}
defer func() {
// If the read was successful and the wait duration is not already the minimum
if err == nil && c.fakeNonblockingReadWaitDuration > minNonblockingReadWaitDuration {
endTime := time.Now()
if n > 0 && c.fakeNonBlockingShortReadCount < 5 {
c.fakeNonBlockingShortReadCount++
}
// The wait duration should be 2x the fastest read that has occurred. This should give reasonable assurance that
// a Read deadline will not block a read before it has a chance to read data already in Go or the OS's receive
// buffer.
proposedWait := endTime.Sub(startTime) * 2
if proposedWait < minNonblockingReadWaitDuration {
proposedWait = minNonblockingReadWaitDuration
}
if proposedWait < c.fakeNonblockingReadWaitDuration {
c.fakeNonblockingReadWaitDuration = proposedWait
}
}
// Ignoring error resetting deadline as there is nothing that can reasonably be done if it fails.
c.conn.SetReadDeadline(c.readDeadline)
if err != nil {
if errors.Is(err, os.ErrDeadlineExceeded) {
err = ErrWouldBlock
}
}
}()
}
return c.conn.Read(b)
}
// syscall.Conn is interface
// TLSClient establishes a TLS connection as a client over conn using config.
//
// To avoid the first Read on the returned *TLSConn also triggering a Write due to the TLS handshake and thereby
// potentially causing a read and write deadlines to behave unexpectedly, Handshake is called explicitly before the
// *TLSConn is returned.
func TLSClient(conn *NetConn, config *tls.Config) (*TLSConn, error) {
tc := tls.Client(conn, config)
err := tc.Handshake()
if err != nil {
return nil, err
}
// Ensure last written part of Handshake is actually sent.
err = conn.Flush()
if err != nil {
return nil, err
}
return &TLSConn{
tlsConn: tc,
nbConn: conn,
}, nil
}
// TLSConn is a TLS wrapper around a *Conn. It works around a temporary write error (such as a timeout) being fatal to a
// tls.Conn.
type TLSConn struct {
tlsConn *tls.Conn
nbConn *NetConn
}
func (tc *TLSConn) Read(b []byte) (n int, err error) { return tc.tlsConn.Read(b) }
func (tc *TLSConn) Write(b []byte) (n int, err error) { return tc.tlsConn.Write(b) }
func (tc *TLSConn) BufferReadUntilBlock() error { return tc.nbConn.BufferReadUntilBlock() }
func (tc *TLSConn) Flush() error { return tc.nbConn.Flush() }
func (tc *TLSConn) LocalAddr() net.Addr { return tc.tlsConn.LocalAddr() }
func (tc *TLSConn) RemoteAddr() net.Addr { return tc.tlsConn.RemoteAddr() }
func (tc *TLSConn) Close() error {
// tls.Conn.closeNotify() sets a 5 second deadline to avoid blocking, sends a TLS alert close notification, and then
// sets the deadline to now. This causes NetConn's Close not to be able to flush the write buffer. Instead we set our
// own 5 second deadline then make all set deadlines no-op.
tc.tlsConn.SetDeadline(time.Now().Add(time.Second * 5))
tc.tlsConn.SetDeadline(disableSetDeadlineDeadline)
return tc.tlsConn.Close()
}
func (tc *TLSConn) SetDeadline(t time.Time) error { return tc.tlsConn.SetDeadline(t) }
func (tc *TLSConn) SetReadDeadline(t time.Time) error { return tc.tlsConn.SetReadDeadline(t) }
func (tc *TLSConn) SetWriteDeadline(t time.Time) error { return tc.tlsConn.SetWriteDeadline(t) }
@@ -1,11 +0,0 @@
//go:build !unix
package nbconn
func (c *NetConn) realNonblockingWrite(b []byte) (n int, err error) {
return c.fakeNonblockingWrite(b)
}
func (c *NetConn) realNonblockingRead(b []byte) (n int, err error) {
return c.fakeNonblockingRead(b)
}
@@ -1,81 +0,0 @@
//go:build unix
package nbconn
import (
"errors"
"io"
"syscall"
)
// realNonblockingWrite does a non-blocking write. readFlushLock must already be held.
func (c *NetConn) realNonblockingWrite(b []byte) (n int, err error) {
if c.nonblockWriteFunc == nil {
c.nonblockWriteFunc = func(fd uintptr) (done bool) {
c.nonblockWriteN, c.nonblockWriteErr = syscall.Write(int(fd), c.nonblockWriteBuf)
return true
}
}
c.nonblockWriteBuf = b
c.nonblockWriteN = 0
c.nonblockWriteErr = nil
err = c.rawConn.Write(c.nonblockWriteFunc)
n = c.nonblockWriteN
c.nonblockWriteBuf = nil // ensure that no reference to b is kept.
if err == nil && c.nonblockWriteErr != nil {
if errors.Is(c.nonblockWriteErr, syscall.EWOULDBLOCK) {
err = ErrWouldBlock
} else {
err = c.nonblockWriteErr
}
}
if err != nil {
// n may be -1 when an error occurs.
if n < 0 {
n = 0
}
return n, err
}
return n, nil
}
func (c *NetConn) realNonblockingRead(b []byte) (n int, err error) {
if c.nonblockReadFunc == nil {
c.nonblockReadFunc = func(fd uintptr) (done bool) {
c.nonblockReadN, c.nonblockReadErr = syscall.Read(int(fd), c.nonblockReadBuf)
return true
}
}
c.nonblockReadBuf = b
c.nonblockReadN = 0
c.nonblockReadErr = nil
err = c.rawConn.Read(c.nonblockReadFunc)
n = c.nonblockReadN
c.nonblockReadBuf = nil // ensure that no reference to b is kept.
if err == nil && c.nonblockReadErr != nil {
if errors.Is(c.nonblockReadErr, syscall.EWOULDBLOCK) {
err = ErrWouldBlock
} else {
err = c.nonblockReadErr
}
}
if err != nil {
// n may be -1 when an error occurs.
if n < 0 {
n = 0
}
return n, err
}
// syscall read did not return an error and 0 bytes were read means EOF.
if n == 0 {
return 0, io.EOF
}
return n, nil
}
+9
View File
@@ -35,6 +35,11 @@ func (q *Query) Sanitize(args ...any) (string, error) {
str = part
case int:
argIdx := part - 1
if argIdx < 0 {
return "", fmt.Errorf("first sql argument must be > 0")
}
if argIdx >= len(args) {
return "", fmt.Errorf("insufficient arguments")
}
@@ -58,6 +63,10 @@ func (q *Query) Sanitize(args ...any) (string, error) {
return "", fmt.Errorf("invalid arg type: %T", arg)
}
argUse[argIdx] = true
// Prevent SQL injection via Line Comment Creation
// https://github.com/jackc/pgx/security/advisories/GHSA-m7wr-2xf7-cm9p
str = "(" + str + ")"
default:
return "", fmt.Errorf("invalid Part type: %T", part)
}
+18 -4
View File
@@ -34,7 +34,8 @@ func (c *LRUCache) Get(key string) *pgconn.StatementDescription {
}
// Put stores sd in the cache. Put panics if sd.SQL is "". Put does nothing if sd.SQL already exists in the cache.
// Put stores sd in the cache. Put panics if sd.SQL is "". Put does nothing if sd.SQL already exists in the cache or
// sd.SQL has been invalidated and HandleInvalidated has not been called yet.
func (c *LRUCache) Put(sd *pgconn.StatementDescription) {
if sd.SQL == "" {
panic("cannot store statement description with empty SQL")
@@ -44,6 +45,13 @@ func (c *LRUCache) Put(sd *pgconn.StatementDescription) {
return
}
// The statement may have been invalidated but not yet handled. Do not readd it to the cache.
for _, invalidSD := range c.invalidStmts {
if invalidSD.SQL == sd.SQL {
return
}
}
if c.l.Len() == c.cap {
c.invalidateOldest()
}
@@ -73,10 +81,16 @@ func (c *LRUCache) InvalidateAll() {
c.l = list.New()
}
func (c *LRUCache) HandleInvalidated() []*pgconn.StatementDescription {
invalidStmts := c.invalidStmts
// GetInvalidated returns a slice of all statement descriptions invalidated since the last call to RemoveInvalidated.
func (c *LRUCache) GetInvalidated() []*pgconn.StatementDescription {
return c.invalidStmts
}
// RemoveInvalidated removes all invalidated statement descriptions. No other calls to Cache must be made between a
// call to GetInvalidated and RemoveInvalidated or RemoveInvalidated may remove statement descriptions that were
// never seen by the call to GetInvalidated.
func (c *LRUCache) RemoveInvalidated() {
c.invalidStmts = nil
return invalidStmts
}
// Len returns the number of cached prepared statement descriptions.
+14 -26
View File
@@ -2,18 +2,17 @@
package stmtcache
import (
"strconv"
"sync/atomic"
"crypto/sha256"
"encoding/hex"
"github.com/jackc/pgx/v5/pgconn"
)
var stmtCounter int64
// NextStatementName returns a statement name that will be unique for the lifetime of the program.
func NextStatementName() string {
n := atomic.AddInt64(&stmtCounter, 1)
return "stmtcache_" + strconv.FormatInt(n, 10)
// StatementName returns a statement name that will be stable for sql across multiple connections and program
// executions.
func StatementName(sql string) string {
digest := sha256.Sum256([]byte(sql))
return "stmtcache_" + hex.EncodeToString(digest[0:24])
}
// Cache caches statement descriptions.
@@ -30,8 +29,13 @@ type Cache interface {
// InvalidateAll invalidates all statement descriptions.
InvalidateAll()
// HandleInvalidated returns a slice of all statement descriptions invalidated since the last call to HandleInvalidated.
HandleInvalidated() []*pgconn.StatementDescription
// GetInvalidated returns a slice of all statement descriptions invalidated since the last call to RemoveInvalidated.
GetInvalidated() []*pgconn.StatementDescription
// RemoveInvalidated removes all invalidated statement descriptions. No other calls to Cache must be made between a
// call to GetInvalidated and RemoveInvalidated or RemoveInvalidated may remove statement descriptions that were
// never seen by the call to GetInvalidated.
RemoveInvalidated()
// Len returns the number of cached prepared statement descriptions.
Len() int
@@ -39,19 +43,3 @@ type Cache interface {
// Cap returns the maximum number of cached prepared statement descriptions.
Cap() int
}
func IsStatementInvalid(err error) bool {
pgErr, ok := err.(*pgconn.PgError)
if !ok {
return false
}
// https://github.com/jackc/pgx/issues/1162
//
// We used to look for the message "cached plan must not change result type". However, that message can be localized.
// Unfortunately, error code "0A000" - "FEATURE NOT SUPPORTED" is used for many different errors and the only way to
// tell the difference is by the message. But all that happens is we clear a statement that we otherwise wouldn't
// have so it should be safe.
possibleInvalidCachedPlanError := pgErr.Code == "0A000"
return possibleInvalidCachedPlanError
}
+9 -3
View File
@@ -54,10 +54,16 @@ func (c *UnlimitedCache) InvalidateAll() {
c.m = make(map[string]*pgconn.StatementDescription)
}
func (c *UnlimitedCache) HandleInvalidated() []*pgconn.StatementDescription {
invalidStmts := c.invalidStmts
// GetInvalidated returns a slice of all statement descriptions invalidated since the last call to RemoveInvalidated.
func (c *UnlimitedCache) GetInvalidated() []*pgconn.StatementDescription {
return c.invalidStmts
}
// RemoveInvalidated removes all invalidated statement descriptions. No other calls to Cache must be made between a
// call to GetInvalidated and RemoveInvalidated or RemoveInvalidated may remove statement descriptions that were
// never seen by the call to GetInvalidated.
func (c *UnlimitedCache) RemoveInvalidated() {
c.invalidStmts = nil
return invalidStmts
}
// Len returns the number of cached prepared statement descriptions.
+55 -18
View File
@@ -6,6 +6,11 @@ import (
"io"
)
// The PostgreSQL wire protocol has a limit of 1 GB - 1 per message. See definition of
// PQ_LARGE_MESSAGE_LIMIT in the PostgreSQL source code. To allow for the other data
// in the message,maxLargeObjectMessageLength should be no larger than 1 GB - 1 KB.
var maxLargeObjectMessageLength = 1024*1024*1024 - 1024
// LargeObjects is a structure used to access the large objects API. It is only valid within the transaction where it
// was created.
//
@@ -68,32 +73,64 @@ type LargeObject struct {
// Write writes p to the large object and returns the number of bytes written and an error if not all of p was written.
func (o *LargeObject) Write(p []byte) (int, error) {
var n int
err := o.tx.QueryRow(o.ctx, "select lowrite($1, $2)", o.fd, p).Scan(&n)
if err != nil {
return n, err
nTotal := 0
for {
expected := len(p) - nTotal
if expected == 0 {
break
} else if expected > maxLargeObjectMessageLength {
expected = maxLargeObjectMessageLength
}
var n int
err := o.tx.QueryRow(o.ctx, "select lowrite($1, $2)", o.fd, p[nTotal:nTotal+expected]).Scan(&n)
if err != nil {
return nTotal, err
}
if n < 0 {
return nTotal, errors.New("failed to write to large object")
}
nTotal += n
if n < expected {
return nTotal, errors.New("short write to large object")
} else if n > expected {
return nTotal, errors.New("invalid write to large object")
}
}
if n < 0 {
return 0, errors.New("failed to write to large object")
}
return n, nil
return nTotal, nil
}
// Read reads up to len(p) bytes into p returning the number of bytes read.
func (o *LargeObject) Read(p []byte) (int, error) {
var res []byte
err := o.tx.QueryRow(o.ctx, "select loread($1, $2)", o.fd, len(p)).Scan(&res)
copy(p, res)
if err != nil {
return len(res), err
nTotal := 0
for {
expected := len(p) - nTotal
if expected == 0 {
break
} else if expected > maxLargeObjectMessageLength {
expected = maxLargeObjectMessageLength
}
var res []byte
err := o.tx.QueryRow(o.ctx, "select loread($1, $2)", o.fd, expected).Scan(&res)
copy(p[nTotal:], res)
nTotal += len(res)
if err != nil {
return nTotal, err
}
if len(res) < expected {
return nTotal, io.EOF
} else if len(res) > expected {
return nTotal, errors.New("invalid read of large object")
}
}
if len(res) < len(p) {
err = io.EOF
}
return len(res), err
return nTotal, nil
}
// Seek moves the current location pointer to the new location specified by offset.
+4 -1
View File
@@ -14,6 +14,9 @@ import (
//
// conn.Query(ctx, "select * from widgets where foo = @foo and bar = @bar", pgx.NamedArgs{"foo": 1, "bar": 2})
// conn.Query(ctx, "select * from widgets where foo = $1 and bar = $2", 1, 2)
//
// Named placeholders are case sensitive and must start with a letter or underscore. Subsequent characters can be
// letters, numbers, or underscores.
type NamedArgs map[string]any
// RewriteQuery implements the QueryRewriter interface.
@@ -80,7 +83,7 @@ func rawState(l *sqlLexer) stateFn {
return doubleQuoteState
case '@':
nextRune, _ := utf8.DecodeRuneInString(l.src[l.pos:])
if isLetter(nextRune) {
if isLetter(nextRune) || nextRune == '_' {
if l.pos-l.start > 0 {
l.parts = append(l.parts, l.src[l.start:l.pos-width])
}
+4 -4
View File
@@ -42,12 +42,12 @@ func (c *PgConn) scramAuth(serverAuthMechanisms []string) error {
Data: sc.clientFirstMessage(),
}
c.frontend.Send(saslInitialResponse)
err = c.frontend.Flush()
err = c.flushWithPotentialWriteReadDeadlock()
if err != nil {
return err
}
// Receive server-first-message payload in a AuthenticationSASLContinue.
// Receive server-first-message payload in an AuthenticationSASLContinue.
saslContinue, err := c.rxSASLContinue()
if err != nil {
return err
@@ -62,12 +62,12 @@ func (c *PgConn) scramAuth(serverAuthMechanisms []string) error {
Data: []byte(sc.clientFinalMessage()),
}
c.frontend.Send(saslResponse)
err = c.frontend.Flush()
err = c.flushWithPotentialWriteReadDeadlock()
if err != nil {
return err
}
// Receive server-final-message payload in a AuthenticationSASLFinal.
// Receive server-final-message payload in an AuthenticationSASLFinal.
saslFinal, err := c.rxSASLFinal()
if err != nil {
return err
+28 -13
View File
@@ -26,7 +26,7 @@ type AfterConnectFunc func(ctx context.Context, pgconn *PgConn) error
type ValidateConnectFunc func(ctx context.Context, pgconn *PgConn) error
type GetSSLPasswordFunc func(ctx context.Context) string
// Config is the settings used to establish a connection to a PostgreSQL server. It must be created by ParseConfig. A
// Config is the settings used to establish a connection to a PostgreSQL server. It must be created by [ParseConfig]. A
// manually initialized Config will cause ConnectConfig to panic.
type Config struct {
Host string // host (e.g. localhost) or absolute path to unix domain socket directory (e.g. /private/tmp)
@@ -60,6 +60,11 @@ type Config struct {
// OnNotification is a callback function called when a notification from the LISTEN/NOTIFY system is received.
OnNotification NotificationHandler
// OnPgError is a callback function called when a Postgres error is received by the server. The default handler will close
// the connection on any FATAL errors. If you override this handler you should call the previously set handler or ensure
// that you close on FATAL errors by returning false.
OnPgError PgErrorHandler
createdByParseConfig bool // Used to enforce created by ParseConfig rule.
}
@@ -232,12 +237,12 @@ func ParseConfigWithOptions(connString string, options ParseConfigOptions) (*Con
if strings.HasPrefix(connString, "postgres://") || strings.HasPrefix(connString, "postgresql://") {
connStringSettings, err = parseURLSettings(connString)
if err != nil {
return nil, &parseConfigError{connString: connString, msg: "failed to parse as URL", err: err}
return nil, &ParseConfigError{ConnString: connString, msg: "failed to parse as URL", err: err}
}
} else {
connStringSettings, err = parseDSNSettings(connString)
if err != nil {
return nil, &parseConfigError{connString: connString, msg: "failed to parse as DSN", err: err}
return nil, &ParseConfigError{ConnString: connString, msg: "failed to parse as DSN", err: err}
}
}
}
@@ -246,7 +251,7 @@ func ParseConfigWithOptions(connString string, options ParseConfigOptions) (*Con
if service, present := settings["service"]; present {
serviceSettings, err := parseServiceSettings(settings["servicefile"], service)
if err != nil {
return nil, &parseConfigError{connString: connString, msg: "failed to read service", err: err}
return nil, &ParseConfigError{ConnString: connString, msg: "failed to read service", err: err}
}
settings = mergeSettings(defaultSettings, envSettings, serviceSettings, connStringSettings)
@@ -261,12 +266,19 @@ func ParseConfigWithOptions(connString string, options ParseConfigOptions) (*Con
BuildFrontend: func(r io.Reader, w io.Writer) *pgproto3.Frontend {
return pgproto3.NewFrontend(r, w)
},
OnPgError: func(_ *PgConn, pgErr *PgError) bool {
// we want to automatically close any fatal errors
if strings.EqualFold(pgErr.Severity, "FATAL") {
return false
}
return true
},
}
if connectTimeoutSetting, present := settings["connect_timeout"]; present {
connectTimeout, err := parseConnectTimeoutSetting(connectTimeoutSetting)
if err != nil {
return nil, &parseConfigError{connString: connString, msg: "invalid connect_timeout", err: err}
return nil, &ParseConfigError{ConnString: connString, msg: "invalid connect_timeout", err: err}
}
config.ConnectTimeout = connectTimeout
config.DialFunc = makeConnectTimeoutDialFunc(connectTimeout)
@@ -328,7 +340,7 @@ func ParseConfigWithOptions(connString string, options ParseConfigOptions) (*Con
port, err := parsePort(portStr)
if err != nil {
return nil, &parseConfigError{connString: connString, msg: "invalid port", err: err}
return nil, &ParseConfigError{ConnString: connString, msg: "invalid port", err: err}
}
var tlsConfigs []*tls.Config
@@ -340,7 +352,7 @@ func ParseConfigWithOptions(connString string, options ParseConfigOptions) (*Con
var err error
tlsConfigs, err = configTLS(settings, host, options)
if err != nil {
return nil, &parseConfigError{connString: connString, msg: "failed to configure TLS", err: err}
return nil, &ParseConfigError{ConnString: connString, msg: "failed to configure TLS", err: err}
}
}
@@ -384,7 +396,7 @@ func ParseConfigWithOptions(connString string, options ParseConfigOptions) (*Con
case "any":
// do nothing
default:
return nil, &parseConfigError{connString: connString, msg: fmt.Sprintf("unknown target_session_attrs value: %v", tsa)}
return nil, &ParseConfigError{ConnString: connString, msg: fmt.Sprintf("unknown target_session_attrs value: %v", tsa)}
}
return config, nil
@@ -709,6 +721,9 @@ func configTLS(settings map[string]string, thisHost string, parseConfigOptions P
return nil, fmt.Errorf("unable to read sslkey: %w", err)
}
block, _ := pem.Decode(buf)
if block == nil {
return nil, errors.New("failed to decode sslkey")
}
var pemKey []byte
var decryptedKey []byte
var decryptedError error
@@ -809,7 +824,7 @@ func makeConnectTimeoutDialFunc(timeout time.Duration) DialFunc {
return d.DialContext
}
// ValidateConnectTargetSessionAttrsReadWrite is an ValidateConnectFunc that implements libpq compatible
// ValidateConnectTargetSessionAttrsReadWrite is a ValidateConnectFunc that implements libpq compatible
// target_session_attrs=read-write.
func ValidateConnectTargetSessionAttrsReadWrite(ctx context.Context, pgConn *PgConn) error {
result := pgConn.ExecParams(ctx, "show transaction_read_only", nil, nil, nil, nil).Read()
@@ -824,7 +839,7 @@ func ValidateConnectTargetSessionAttrsReadWrite(ctx context.Context, pgConn *PgC
return nil
}
// ValidateConnectTargetSessionAttrsReadOnly is an ValidateConnectFunc that implements libpq compatible
// ValidateConnectTargetSessionAttrsReadOnly is a ValidateConnectFunc that implements libpq compatible
// target_session_attrs=read-only.
func ValidateConnectTargetSessionAttrsReadOnly(ctx context.Context, pgConn *PgConn) error {
result := pgConn.ExecParams(ctx, "show transaction_read_only", nil, nil, nil, nil).Read()
@@ -839,7 +854,7 @@ func ValidateConnectTargetSessionAttrsReadOnly(ctx context.Context, pgConn *PgCo
return nil
}
// ValidateConnectTargetSessionAttrsStandby is an ValidateConnectFunc that implements libpq compatible
// ValidateConnectTargetSessionAttrsStandby is a ValidateConnectFunc that implements libpq compatible
// target_session_attrs=standby.
func ValidateConnectTargetSessionAttrsStandby(ctx context.Context, pgConn *PgConn) error {
result := pgConn.ExecParams(ctx, "select pg_is_in_recovery()", nil, nil, nil, nil).Read()
@@ -854,7 +869,7 @@ func ValidateConnectTargetSessionAttrsStandby(ctx context.Context, pgConn *PgCon
return nil
}
// ValidateConnectTargetSessionAttrsPrimary is an ValidateConnectFunc that implements libpq compatible
// ValidateConnectTargetSessionAttrsPrimary is a ValidateConnectFunc that implements libpq compatible
// target_session_attrs=primary.
func ValidateConnectTargetSessionAttrsPrimary(ctx context.Context, pgConn *PgConn) error {
result := pgConn.ExecParams(ctx, "select pg_is_in_recovery()", nil, nil, nil, nil).Read()
@@ -869,7 +884,7 @@ func ValidateConnectTargetSessionAttrsPrimary(ctx context.Context, pgConn *PgCon
return nil
}
// ValidateConnectTargetSessionAttrsPreferStandby is an ValidateConnectFunc that implements libpq compatible
// ValidateConnectTargetSessionAttrsPreferStandby is a ValidateConnectFunc that implements libpq compatible
// target_session_attrs=prefer-standby.
func ValidateConnectTargetSessionAttrsPreferStandby(ctx context.Context, pgConn *PgConn) error {
result := pgConn.ExecParams(ctx, "select pg_is_in_recovery()", nil, nil, nil, nil).Read()
+18 -12
View File
@@ -57,22 +57,23 @@ func (pe *PgError) SQLState() string {
return pe.Code
}
type connectError struct {
config *Config
// ConnectError is the error returned when a connection attempt fails.
type ConnectError struct {
Config *Config // The configuration that was used in the connection attempt.
msg string
err error
}
func (e *connectError) Error() string {
func (e *ConnectError) Error() string {
sb := &strings.Builder{}
fmt.Fprintf(sb, "failed to connect to `host=%s user=%s database=%s`: %s", e.config.Host, e.config.User, e.config.Database, e.msg)
fmt.Fprintf(sb, "failed to connect to `host=%s user=%s database=%s`: %s", e.Config.Host, e.Config.User, e.Config.Database, e.msg)
if e.err != nil {
fmt.Fprintf(sb, " (%s)", e.err.Error())
}
return sb.String()
}
func (e *connectError) Unwrap() error {
func (e *ConnectError) Unwrap() error {
return e.err
}
@@ -88,33 +89,38 @@ func (e *connLockError) Error() string {
return e.status
}
type parseConfigError struct {
connString string
// ParseConfigError is the error returned when a connection string cannot be parsed.
type ParseConfigError struct {
ConnString string // The connection string that could not be parsed.
msg string
err error
}
func (e *parseConfigError) Error() string {
connString := redactPW(e.connString)
func (e *ParseConfigError) Error() string {
// Now that ParseConfigError is public and ConnString is available to the developer, perhaps it would be better only
// return a static string. That would ensure that the error message cannot leak a password. The ConnString field would
// allow access to the original string if desired and Unwrap would allow access to the underlying error.
connString := redactPW(e.ConnString)
if e.err == nil {
return fmt.Sprintf("cannot parse `%s`: %s", connString, e.msg)
}
return fmt.Sprintf("cannot parse `%s`: %s (%s)", connString, e.msg, e.err.Error())
}
func (e *parseConfigError) Unwrap() error {
func (e *ParseConfigError) Unwrap() error {
return e.err
}
func normalizeTimeoutError(ctx context.Context, err error) error {
if err, ok := err.(net.Error); ok && err.Timeout() {
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
if ctx.Err() == context.Canceled {
// Since the timeout was caused by a context cancellation, the actual error is context.Canceled not the timeout error.
return context.Canceled
} else if ctx.Err() == context.DeadlineExceeded {
return &errTimeout{err: ctx.Err()}
} else {
return &errTimeout{err: err}
return &errTimeout{err: netErr}
}
}
return err
+139
View File
@@ -0,0 +1,139 @@
// Package bgreader provides a io.Reader that can optionally buffer reads in the background.
package bgreader
import (
"io"
"sync"
"github.com/jackc/pgx/v5/internal/iobufpool"
)
const (
StatusStopped = iota
StatusRunning
StatusStopping
)
// BGReader is an io.Reader that can optionally buffer reads in the background. It is safe for concurrent use.
type BGReader struct {
r io.Reader
cond *sync.Cond
status int32
readResults []readResult
}
type readResult struct {
buf *[]byte
err error
}
// Start starts the backgrounder reader. If the background reader is already running this is a no-op. The background
// reader will stop automatically when the underlying reader returns an error.
func (r *BGReader) Start() {
r.cond.L.Lock()
defer r.cond.L.Unlock()
switch r.status {
case StatusStopped:
r.status = StatusRunning
go r.bgRead()
case StatusRunning:
// no-op
case StatusStopping:
r.status = StatusRunning
}
}
// Stop tells the background reader to stop after the in progress Read returns. It is safe to call Stop when the
// background reader is not running.
func (r *BGReader) Stop() {
r.cond.L.Lock()
defer r.cond.L.Unlock()
switch r.status {
case StatusStopped:
// no-op
case StatusRunning:
r.status = StatusStopping
case StatusStopping:
// no-op
}
}
// Status returns the current status of the background reader.
func (r *BGReader) Status() int32 {
r.cond.L.Lock()
defer r.cond.L.Unlock()
return r.status
}
func (r *BGReader) bgRead() {
keepReading := true
for keepReading {
buf := iobufpool.Get(8192)
n, err := r.r.Read(*buf)
*buf = (*buf)[:n]
r.cond.L.Lock()
r.readResults = append(r.readResults, readResult{buf: buf, err: err})
if r.status == StatusStopping || err != nil {
r.status = StatusStopped
keepReading = false
}
r.cond.L.Unlock()
r.cond.Broadcast()
}
}
// Read implements the io.Reader interface.
func (r *BGReader) Read(p []byte) (int, error) {
r.cond.L.Lock()
defer r.cond.L.Unlock()
if len(r.readResults) > 0 {
return r.readFromReadResults(p)
}
// There are no unread background read results and the background reader is stopped.
if r.status == StatusStopped {
return r.r.Read(p)
}
// Wait for results from the background reader
for len(r.readResults) == 0 {
r.cond.Wait()
}
return r.readFromReadResults(p)
}
// readBackgroundResults reads a result previously read by the background reader. r.cond.L must be held.
func (r *BGReader) readFromReadResults(p []byte) (int, error) {
buf := r.readResults[0].buf
var err error
n := copy(p, *buf)
if n == len(*buf) {
err = r.readResults[0].err
iobufpool.Put(buf)
if len(r.readResults) == 1 {
r.readResults = nil
} else {
r.readResults = r.readResults[1:]
}
} else {
*buf = (*buf)[n:]
r.readResults[0].buf = buf
}
return n, err
}
func New(r io.Reader) *BGReader {
return &BGReader{
r: r,
cond: &sync.Cond{
L: &sync.Mutex{},
},
}
}
+1 -1
View File
@@ -63,7 +63,7 @@ func (c *PgConn) gssAuth() error {
Data: nextData,
}
c.frontend.Send(gssResponse)
err = c.frontend.Flush()
err = c.flushWithPotentialWriteReadDeadlock()
if err != nil {
return err
}
+424 -148
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,6 +1,6 @@
# pgproto3
Package pgproto3 is a encoder and decoder of the PostgreSQL wire protocol version 3.
Package pgproto3 is an encoder and decoder of the PostgreSQL wire protocol version 3.
pgproto3 can be used as a foundation for PostgreSQL drivers, proxies, mock servers, load balancers and more.
@@ -35,11 +35,10 @@ func (dst *AuthenticationCleartextPassword) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *AuthenticationCleartextPassword) Encode(dst []byte) []byte {
dst = append(dst, 'R')
dst = pgio.AppendInt32(dst, 8)
func (src *AuthenticationCleartextPassword) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'R')
dst = pgio.AppendUint32(dst, AuthTypeCleartextPassword)
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -4
View File
@@ -27,11 +27,10 @@ func (a *AuthenticationGSS) Decode(src []byte) error {
return nil
}
func (a *AuthenticationGSS) Encode(dst []byte) []byte {
dst = append(dst, 'R')
dst = pgio.AppendInt32(dst, 4)
func (a *AuthenticationGSS) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'R')
dst = pgio.AppendUint32(dst, AuthTypeGSS)
return dst
return finishMessage(dst, sp)
}
func (a *AuthenticationGSS) MarshalJSON() ([]byte, error) {
+3 -4
View File
@@ -31,12 +31,11 @@ func (a *AuthenticationGSSContinue) Decode(src []byte) error {
return nil
}
func (a *AuthenticationGSSContinue) Encode(dst []byte) []byte {
dst = append(dst, 'R')
dst = pgio.AppendInt32(dst, int32(len(a.Data))+8)
func (a *AuthenticationGSSContinue) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'R')
dst = pgio.AppendUint32(dst, AuthTypeGSSCont)
dst = append(dst, a.Data...)
return dst
return finishMessage(dst, sp)
}
func (a *AuthenticationGSSContinue) MarshalJSON() ([]byte, error) {
+3 -4
View File
@@ -38,12 +38,11 @@ func (dst *AuthenticationMD5Password) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *AuthenticationMD5Password) Encode(dst []byte) []byte {
dst = append(dst, 'R')
dst = pgio.AppendInt32(dst, 12)
func (src *AuthenticationMD5Password) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'R')
dst = pgio.AppendUint32(dst, AuthTypeMD5Password)
dst = append(dst, src.Salt[:]...)
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -4
View File
@@ -35,11 +35,10 @@ func (dst *AuthenticationOk) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *AuthenticationOk) Encode(dst []byte) []byte {
dst = append(dst, 'R')
dst = pgio.AppendInt32(dst, 8)
func (src *AuthenticationOk) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'R')
dst = pgio.AppendUint32(dst, AuthTypeOk)
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -7
View File
@@ -47,10 +47,8 @@ func (dst *AuthenticationSASL) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *AuthenticationSASL) Encode(dst []byte) []byte {
dst = append(dst, 'R')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *AuthenticationSASL) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'R')
dst = pgio.AppendUint32(dst, AuthTypeSASL)
for _, s := range src.AuthMechanisms {
@@ -59,9 +57,7 @@ func (src *AuthenticationSASL) Encode(dst []byte) []byte {
}
dst = append(dst, 0)
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -9
View File
@@ -38,17 +38,11 @@ func (dst *AuthenticationSASLContinue) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *AuthenticationSASLContinue) Encode(dst []byte) []byte {
dst = append(dst, 'R')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *AuthenticationSASLContinue) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'R')
dst = pgio.AppendUint32(dst, AuthTypeSASLContinue)
dst = append(dst, src.Data...)
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -9
View File
@@ -38,17 +38,11 @@ func (dst *AuthenticationSASLFinal) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *AuthenticationSASLFinal) Encode(dst []byte) []byte {
dst = append(dst, 'R')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *AuthenticationSASLFinal) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'R')
dst = pgio.AppendUint32(dst, AuthTypeSASLFinal)
dst = append(dst, src.Data...)
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Unmarshaler.
+34 -4
View File
@@ -16,7 +16,8 @@ type Backend struct {
// before it is actually transmitted (i.e. before Flush).
tracer *tracer
wbuf []byte
wbuf []byte
encodeError error
// Frontend message flyweights
bind Bind
@@ -38,6 +39,7 @@ type Backend struct {
terminate Terminate
bodyLen int
maxBodyLen int // maxBodyLen is the maximum length of a message body in octets. If a message body exceeds this length, Receive will return an error.
msgType byte
partialMsg bool
authType uint32
@@ -54,11 +56,21 @@ func NewBackend(r io.Reader, w io.Writer) *Backend {
return &Backend{cr: cr, w: w}
}
// Send sends a message to the frontend (i.e. the client). The message is not guaranteed to be written until Flush is
// called.
// Send sends a message to the frontend (i.e. the client). The message is buffered until Flush is called. Any error
// encountered will be returned from Flush.
func (b *Backend) Send(msg BackendMessage) {
if b.encodeError != nil {
return
}
prevLen := len(b.wbuf)
b.wbuf = msg.Encode(b.wbuf)
newBuf, err := msg.Encode(b.wbuf)
if err != nil {
b.encodeError = err
return
}
b.wbuf = newBuf
if b.tracer != nil {
b.tracer.traceMessage('B', int32(len(b.wbuf)-prevLen), msg)
}
@@ -66,6 +78,12 @@ func (b *Backend) Send(msg BackendMessage) {
// Flush writes any pending messages to the frontend (i.e. the client).
func (b *Backend) Flush() error {
if err := b.encodeError; err != nil {
b.encodeError = nil
b.wbuf = b.wbuf[:0]
return &writeError{err: err, safeToRetry: true}
}
n, err := b.w.Write(b.wbuf)
const maxLen = 1024
@@ -158,6 +176,9 @@ func (b *Backend) Receive() (FrontendMessage, error) {
b.msgType = header[0]
b.bodyLen = int(binary.BigEndian.Uint32(header[1:])) - 4
if b.maxBodyLen > 0 && b.bodyLen > b.maxBodyLen {
return nil, &ExceededMaxBodyLenErr{b.maxBodyLen, b.bodyLen}
}
b.partialMsg = true
}
@@ -260,3 +281,12 @@ func (b *Backend) SetAuthType(authType uint32) error {
return nil
}
// SetMaxBodyLen sets the maximum length of a message body in octets. If a message body exceeds this length, Receive will return
// an error. This is useful for protecting against malicious clients that send large messages with the intent of
// causing memory exhaustion.
// The default value is 0.
// If maxBodyLen is 0, then no maximum is enforced.
func (b *Backend) SetMaxBodyLen(maxBodyLen int) {
b.maxBodyLen = maxBodyLen
}
+3 -4
View File
@@ -29,12 +29,11 @@ func (dst *BackendKeyData) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *BackendKeyData) Encode(dst []byte) []byte {
dst = append(dst, 'K')
dst = pgio.AppendUint32(dst, 12)
func (src *BackendKeyData) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'K')
dst = pgio.AppendUint32(dst, src.ProcessID)
dst = pgio.AppendUint32(dst, src.SecretKey)
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+14 -7
View File
@@ -5,7 +5,9 @@ import (
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"math"
"github.com/jackc/pgx/v5/internal/pgio"
)
@@ -108,21 +110,25 @@ func (dst *Bind) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *Bind) Encode(dst []byte) []byte {
dst = append(dst, 'B')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *Bind) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'B')
dst = append(dst, src.DestinationPortal...)
dst = append(dst, 0)
dst = append(dst, src.PreparedStatement...)
dst = append(dst, 0)
if len(src.ParameterFormatCodes) > math.MaxUint16 {
return nil, errors.New("too many parameter format codes")
}
dst = pgio.AppendUint16(dst, uint16(len(src.ParameterFormatCodes)))
for _, fc := range src.ParameterFormatCodes {
dst = pgio.AppendInt16(dst, fc)
}
if len(src.Parameters) > math.MaxUint16 {
return nil, errors.New("too many parameters")
}
dst = pgio.AppendUint16(dst, uint16(len(src.Parameters)))
for _, p := range src.Parameters {
if p == nil {
@@ -134,14 +140,15 @@ func (src *Bind) Encode(dst []byte) []byte {
dst = append(dst, p...)
}
if len(src.ResultFormatCodes) > math.MaxUint16 {
return nil, errors.New("too many result format codes")
}
dst = pgio.AppendUint16(dst, uint16(len(src.ResultFormatCodes)))
for _, fc := range src.ResultFormatCodes {
dst = pgio.AppendInt16(dst, fc)
}
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+2 -2
View File
@@ -20,8 +20,8 @@ func (dst *BindComplete) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *BindComplete) Encode(dst []byte) []byte {
return append(dst, '2', 0, 0, 0, 4)
func (src *BindComplete) Encode(dst []byte) ([]byte, error) {
return append(dst, '2', 0, 0, 0, 4), nil
}
// MarshalJSON implements encoding/json.Marshaler.
+2 -2
View File
@@ -36,12 +36,12 @@ func (dst *CancelRequest) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 4 byte message length.
func (src *CancelRequest) Encode(dst []byte) []byte {
func (src *CancelRequest) Encode(dst []byte) ([]byte, error) {
dst = pgio.AppendInt32(dst, 16)
dst = pgio.AppendInt32(dst, cancelRequestCode)
dst = pgio.AppendUint32(dst, src.ProcessID)
dst = pgio.AppendUint32(dst, src.SecretKey)
return dst
return dst, nil
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -11
View File
@@ -4,8 +4,6 @@ import (
"bytes"
"encoding/json"
"errors"
"github.com/jackc/pgx/v5/internal/pgio"
)
type Close struct {
@@ -37,18 +35,12 @@ func (dst *Close) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *Close) Encode(dst []byte) []byte {
dst = append(dst, 'C')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *Close) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'C')
dst = append(dst, src.ObjectType)
dst = append(dst, src.Name...)
dst = append(dst, 0)
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+2 -2
View File
@@ -20,8 +20,8 @@ func (dst *CloseComplete) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *CloseComplete) Encode(dst []byte) []byte {
return append(dst, '3', 0, 0, 0, 4)
func (src *CloseComplete) Encode(dst []byte) ([]byte, error) {
return append(dst, '3', 0, 0, 0, 4), nil
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -11
View File
@@ -3,8 +3,6 @@ package pgproto3
import (
"bytes"
"encoding/json"
"github.com/jackc/pgx/v5/internal/pgio"
)
type CommandComplete struct {
@@ -31,17 +29,11 @@ func (dst *CommandComplete) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *CommandComplete) Encode(dst []byte) []byte {
dst = append(dst, 'C')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *CommandComplete) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'C')
dst = append(dst, src.CommandTag...)
dst = append(dst, 0)
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+7 -7
View File
@@ -5,6 +5,7 @@ import (
"encoding/binary"
"encoding/json"
"errors"
"math"
"github.com/jackc/pgx/v5/internal/pgio"
)
@@ -44,19 +45,18 @@ func (dst *CopyBothResponse) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *CopyBothResponse) Encode(dst []byte) []byte {
dst = append(dst, 'W')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *CopyBothResponse) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'W')
dst = append(dst, src.OverallFormat)
if len(src.ColumnFormatCodes) > math.MaxUint16 {
return nil, errors.New("too many column format codes")
}
dst = pgio.AppendUint16(dst, uint16(len(src.ColumnFormatCodes)))
for _, fc := range src.ColumnFormatCodes {
dst = pgio.AppendUint16(dst, fc)
}
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -6
View File
@@ -3,8 +3,6 @@ package pgproto3
import (
"encoding/hex"
"encoding/json"
"github.com/jackc/pgx/v5/internal/pgio"
)
type CopyData struct {
@@ -25,11 +23,10 @@ func (dst *CopyData) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *CopyData) Encode(dst []byte) []byte {
dst = append(dst, 'd')
dst = pgio.AppendInt32(dst, int32(4+len(src.Data)))
func (src *CopyData) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'd')
dst = append(dst, src.Data...)
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+2 -2
View File
@@ -24,8 +24,8 @@ func (dst *CopyDone) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *CopyDone) Encode(dst []byte) []byte {
return append(dst, 'c', 0, 0, 0, 4)
func (src *CopyDone) Encode(dst []byte) ([]byte, error) {
return append(dst, 'c', 0, 0, 0, 4), nil
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -11
View File
@@ -3,8 +3,6 @@ package pgproto3
import (
"bytes"
"encoding/json"
"github.com/jackc/pgx/v5/internal/pgio"
)
type CopyFail struct {
@@ -28,17 +26,11 @@ func (dst *CopyFail) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *CopyFail) Encode(dst []byte) []byte {
dst = append(dst, 'f')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *CopyFail) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'f')
dst = append(dst, src.Message...)
dst = append(dst, 0)
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+7 -7
View File
@@ -5,6 +5,7 @@ import (
"encoding/binary"
"encoding/json"
"errors"
"math"
"github.com/jackc/pgx/v5/internal/pgio"
)
@@ -44,20 +45,19 @@ func (dst *CopyInResponse) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *CopyInResponse) Encode(dst []byte) []byte {
dst = append(dst, 'G')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *CopyInResponse) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'G')
dst = append(dst, src.OverallFormat)
if len(src.ColumnFormatCodes) > math.MaxUint16 {
return nil, errors.New("too many column format codes")
}
dst = pgio.AppendUint16(dst, uint16(len(src.ColumnFormatCodes)))
for _, fc := range src.ColumnFormatCodes {
dst = pgio.AppendUint16(dst, fc)
}
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+7 -7
View File
@@ -5,6 +5,7 @@ import (
"encoding/binary"
"encoding/json"
"errors"
"math"
"github.com/jackc/pgx/v5/internal/pgio"
)
@@ -43,21 +44,20 @@ func (dst *CopyOutResponse) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *CopyOutResponse) Encode(dst []byte) []byte {
dst = append(dst, 'H')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *CopyOutResponse) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'H')
dst = append(dst, src.OverallFormat)
if len(src.ColumnFormatCodes) > math.MaxUint16 {
return nil, errors.New("too many column format codes")
}
dst = pgio.AppendUint16(dst, uint16(len(src.ColumnFormatCodes)))
for _, fc := range src.ColumnFormatCodes {
dst = pgio.AppendUint16(dst, fc)
}
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+8 -7
View File
@@ -4,6 +4,8 @@ import (
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"math"
"github.com/jackc/pgx/v5/internal/pgio"
)
@@ -63,11 +65,12 @@ func (dst *DataRow) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *DataRow) Encode(dst []byte) []byte {
dst = append(dst, 'D')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *DataRow) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'D')
if len(src.Values) > math.MaxUint16 {
return nil, errors.New("too many values")
}
dst = pgio.AppendUint16(dst, uint16(len(src.Values)))
for _, v := range src.Values {
if v == nil {
@@ -79,9 +82,7 @@ func (src *DataRow) Encode(dst []byte) []byte {
dst = append(dst, v...)
}
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -11
View File
@@ -4,8 +4,6 @@ import (
"bytes"
"encoding/json"
"errors"
"github.com/jackc/pgx/v5/internal/pgio"
)
type Describe struct {
@@ -37,18 +35,12 @@ func (dst *Describe) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *Describe) Encode(dst []byte) []byte {
dst = append(dst, 'D')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *Describe) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'D')
dst = append(dst, src.ObjectType)
dst = append(dst, src.Name...)
dst = append(dst, 0)
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+2 -2
View File
@@ -1,7 +1,7 @@
// Package pgproto3 is a encoder and decoder of the PostgreSQL wire protocol version 3.
// Package pgproto3 is an encoder and decoder of the PostgreSQL wire protocol version 3.
//
// The primary interfaces are Frontend and Backend. They correspond to a client and server respectively. Messages are
// sent with Send (or a specialized Send variant). Messages are automatically bufferred to minimize small writes. Call
// sent with Send (or a specialized Send variant). Messages are automatically buffered to minimize small writes. Call
// Flush to ensure a message has actually been sent.
//
// The Trace method of Frontend and Backend can be used to examine the wire-level message traffic. It outputs in a
+2 -2
View File
@@ -20,8 +20,8 @@ func (dst *EmptyQueryResponse) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *EmptyQueryResponse) Encode(dst []byte) []byte {
return append(dst, 'I', 0, 0, 0, 4)
func (src *EmptyQueryResponse) Encode(dst []byte) ([]byte, error) {
return append(dst, 'I', 0, 0, 0, 4), nil
}
// MarshalJSON implements encoding/json.Marshaler.
+64 -71
View File
@@ -2,7 +2,6 @@ package pgproto3
import (
"bytes"
"encoding/binary"
"encoding/json"
"strconv"
)
@@ -111,119 +110,113 @@ func (dst *ErrorResponse) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *ErrorResponse) Encode(dst []byte) []byte {
return append(dst, src.marshalBinary('E')...)
func (src *ErrorResponse) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'E')
dst = src.appendFields(dst)
return finishMessage(dst, sp)
}
func (src *ErrorResponse) marshalBinary(typeByte byte) []byte {
var bigEndian BigEndianBuf
buf := &bytes.Buffer{}
buf.WriteByte(typeByte)
buf.Write(bigEndian.Uint32(0))
func (src *ErrorResponse) appendFields(dst []byte) []byte {
if src.Severity != "" {
buf.WriteByte('S')
buf.WriteString(src.Severity)
buf.WriteByte(0)
dst = append(dst, 'S')
dst = append(dst, src.Severity...)
dst = append(dst, 0)
}
if src.SeverityUnlocalized != "" {
buf.WriteByte('V')
buf.WriteString(src.SeverityUnlocalized)
buf.WriteByte(0)
dst = append(dst, 'V')
dst = append(dst, src.SeverityUnlocalized...)
dst = append(dst, 0)
}
if src.Code != "" {
buf.WriteByte('C')
buf.WriteString(src.Code)
buf.WriteByte(0)
dst = append(dst, 'C')
dst = append(dst, src.Code...)
dst = append(dst, 0)
}
if src.Message != "" {
buf.WriteByte('M')
buf.WriteString(src.Message)
buf.WriteByte(0)
dst = append(dst, 'M')
dst = append(dst, src.Message...)
dst = append(dst, 0)
}
if src.Detail != "" {
buf.WriteByte('D')
buf.WriteString(src.Detail)
buf.WriteByte(0)
dst = append(dst, 'D')
dst = append(dst, src.Detail...)
dst = append(dst, 0)
}
if src.Hint != "" {
buf.WriteByte('H')
buf.WriteString(src.Hint)
buf.WriteByte(0)
dst = append(dst, 'H')
dst = append(dst, src.Hint...)
dst = append(dst, 0)
}
if src.Position != 0 {
buf.WriteByte('P')
buf.WriteString(strconv.Itoa(int(src.Position)))
buf.WriteByte(0)
dst = append(dst, 'P')
dst = append(dst, strconv.Itoa(int(src.Position))...)
dst = append(dst, 0)
}
if src.InternalPosition != 0 {
buf.WriteByte('p')
buf.WriteString(strconv.Itoa(int(src.InternalPosition)))
buf.WriteByte(0)
dst = append(dst, 'p')
dst = append(dst, strconv.Itoa(int(src.InternalPosition))...)
dst = append(dst, 0)
}
if src.InternalQuery != "" {
buf.WriteByte('q')
buf.WriteString(src.InternalQuery)
buf.WriteByte(0)
dst = append(dst, 'q')
dst = append(dst, src.InternalQuery...)
dst = append(dst, 0)
}
if src.Where != "" {
buf.WriteByte('W')
buf.WriteString(src.Where)
buf.WriteByte(0)
dst = append(dst, 'W')
dst = append(dst, src.Where...)
dst = append(dst, 0)
}
if src.SchemaName != "" {
buf.WriteByte('s')
buf.WriteString(src.SchemaName)
buf.WriteByte(0)
dst = append(dst, 's')
dst = append(dst, src.SchemaName...)
dst = append(dst, 0)
}
if src.TableName != "" {
buf.WriteByte('t')
buf.WriteString(src.TableName)
buf.WriteByte(0)
dst = append(dst, 't')
dst = append(dst, src.TableName...)
dst = append(dst, 0)
}
if src.ColumnName != "" {
buf.WriteByte('c')
buf.WriteString(src.ColumnName)
buf.WriteByte(0)
dst = append(dst, 'c')
dst = append(dst, src.ColumnName...)
dst = append(dst, 0)
}
if src.DataTypeName != "" {
buf.WriteByte('d')
buf.WriteString(src.DataTypeName)
buf.WriteByte(0)
dst = append(dst, 'd')
dst = append(dst, src.DataTypeName...)
dst = append(dst, 0)
}
if src.ConstraintName != "" {
buf.WriteByte('n')
buf.WriteString(src.ConstraintName)
buf.WriteByte(0)
dst = append(dst, 'n')
dst = append(dst, src.ConstraintName...)
dst = append(dst, 0)
}
if src.File != "" {
buf.WriteByte('F')
buf.WriteString(src.File)
buf.WriteByte(0)
dst = append(dst, 'F')
dst = append(dst, src.File...)
dst = append(dst, 0)
}
if src.Line != 0 {
buf.WriteByte('L')
buf.WriteString(strconv.Itoa(int(src.Line)))
buf.WriteByte(0)
dst = append(dst, 'L')
dst = append(dst, strconv.Itoa(int(src.Line))...)
dst = append(dst, 0)
}
if src.Routine != "" {
buf.WriteByte('R')
buf.WriteString(src.Routine)
buf.WriteByte(0)
dst = append(dst, 'R')
dst = append(dst, src.Routine...)
dst = append(dst, 0)
}
for k, v := range src.UnknownFields {
buf.WriteByte(k)
buf.WriteString(v)
buf.WriteByte(0)
dst = append(dst, k)
dst = append(dst, v...)
dst = append(dst, 0)
}
buf.WriteByte(0)
dst = append(dst, 0)
binary.BigEndian.PutUint32(buf.Bytes()[1:5], uint32(buf.Len()-1))
return buf.Bytes()
return dst
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -10
View File
@@ -36,19 +36,12 @@ func (dst *Execute) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *Execute) Encode(dst []byte) []byte {
dst = append(dst, 'E')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *Execute) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'E')
dst = append(dst, src.Portal...)
dst = append(dst, 0)
dst = pgio.AppendUint32(dst, src.MaxRows)
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+2 -2
View File
@@ -20,8 +20,8 @@ func (dst *Flush) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *Flush) Encode(dst []byte) []byte {
return append(dst, 'H', 0, 0, 0, 4)
func (src *Flush) Encode(dst []byte) ([]byte, error) {
return append(dst, 'H', 0, 0, 0, 4), nil
}
// MarshalJSON implements encoding/json.Marshaler.
+116 -25
View File
@@ -18,7 +18,8 @@ type Frontend struct {
// idle. Setting and unsetting tracer provides equivalent functionality to PQtrace and PQuntrace in libpq.
tracer *tracer
wbuf []byte
wbuf []byte
encodeError error
// Backend message flyweights
authenticationOk AuthenticationOk
@@ -64,16 +65,26 @@ func NewFrontend(r io.Reader, w io.Writer) *Frontend {
return &Frontend{cr: cr, w: w}
}
// Send sends a message to the backend (i.e. the server). The message is not guaranteed to be written until Flush is
// called.
// Send sends a message to the backend (i.e. the server). The message is buffered until Flush is called. Any error
// encountered will be returned from Flush.
//
// Send can work with any FrontendMessage. Some commonly used message types such as Bind have specialized send methods
// such as SendBind. These methods should be preferred when the type of message is known up front (e.g. when building an
// extended query protocol query) as they may be faster due to knowing the type of msg rather than it being hidden
// behind an interface.
func (f *Frontend) Send(msg FrontendMessage) {
if f.encodeError != nil {
return
}
prevLen := len(f.wbuf)
f.wbuf = msg.Encode(f.wbuf)
newBuf, err := msg.Encode(f.wbuf)
if err != nil {
f.encodeError = err
return
}
f.wbuf = newBuf
if f.tracer != nil {
f.tracer.traceMessage('F', int32(len(f.wbuf)-prevLen), msg)
}
@@ -81,6 +92,12 @@ func (f *Frontend) Send(msg FrontendMessage) {
// Flush writes any pending messages to the backend (i.e. the server).
func (f *Frontend) Flush() error {
if err := f.encodeError; err != nil {
f.encodeError = nil
f.wbuf = f.wbuf[:0]
return &writeError{err: err, safeToRetry: true}
}
if len(f.wbuf) == 0 {
return nil
}
@@ -116,71 +133,141 @@ func (f *Frontend) Untrace() {
f.tracer = nil
}
// SendBind sends a Bind message to the backend (i.e. the server). The message is not guaranteed to be written until
// Flush is called.
// SendBind sends a Bind message to the backend (i.e. the server). The message is buffered until Flush is called. Any
// error encountered will be returned from Flush.
func (f *Frontend) SendBind(msg *Bind) {
if f.encodeError != nil {
return
}
prevLen := len(f.wbuf)
f.wbuf = msg.Encode(f.wbuf)
newBuf, err := msg.Encode(f.wbuf)
if err != nil {
f.encodeError = err
return
}
f.wbuf = newBuf
if f.tracer != nil {
f.tracer.traceBind('F', int32(len(f.wbuf)-prevLen), msg)
}
}
// SendParse sends a Parse message to the backend (i.e. the server). The message is not guaranteed to be written until
// Flush is called.
// SendParse sends a Parse message to the backend (i.e. the server). The message is buffered until Flush is called. Any
// error encountered will be returned from Flush.
func (f *Frontend) SendParse(msg *Parse) {
if f.encodeError != nil {
return
}
prevLen := len(f.wbuf)
f.wbuf = msg.Encode(f.wbuf)
newBuf, err := msg.Encode(f.wbuf)
if err != nil {
f.encodeError = err
return
}
f.wbuf = newBuf
if f.tracer != nil {
f.tracer.traceParse('F', int32(len(f.wbuf)-prevLen), msg)
}
}
// SendClose sends a Close message to the backend (i.e. the server). The message is not guaranteed to be written until
// Flush is called.
// SendClose sends a Close message to the backend (i.e. the server). The message is buffered until Flush is called. Any
// error encountered will be returned from Flush.
func (f *Frontend) SendClose(msg *Close) {
if f.encodeError != nil {
return
}
prevLen := len(f.wbuf)
f.wbuf = msg.Encode(f.wbuf)
newBuf, err := msg.Encode(f.wbuf)
if err != nil {
f.encodeError = err
return
}
f.wbuf = newBuf
if f.tracer != nil {
f.tracer.traceClose('F', int32(len(f.wbuf)-prevLen), msg)
}
}
// SendDescribe sends a Describe message to the backend (i.e. the server). The message is not guaranteed to be written until
// Flush is called.
// SendDescribe sends a Describe message to the backend (i.e. the server). The message is buffered until Flush is
// called. Any error encountered will be returned from Flush.
func (f *Frontend) SendDescribe(msg *Describe) {
if f.encodeError != nil {
return
}
prevLen := len(f.wbuf)
f.wbuf = msg.Encode(f.wbuf)
newBuf, err := msg.Encode(f.wbuf)
if err != nil {
f.encodeError = err
return
}
f.wbuf = newBuf
if f.tracer != nil {
f.tracer.traceDescribe('F', int32(len(f.wbuf)-prevLen), msg)
}
}
// SendExecute sends a Execute message to the backend (i.e. the server). The message is not guaranteed to be written until
// Flush is called.
// SendExecute sends an Execute message to the backend (i.e. the server). The message is buffered until Flush is called.
// Any error encountered will be returned from Flush.
func (f *Frontend) SendExecute(msg *Execute) {
if f.encodeError != nil {
return
}
prevLen := len(f.wbuf)
f.wbuf = msg.Encode(f.wbuf)
newBuf, err := msg.Encode(f.wbuf)
if err != nil {
f.encodeError = err
return
}
f.wbuf = newBuf
if f.tracer != nil {
f.tracer.TraceQueryute('F', int32(len(f.wbuf)-prevLen), msg)
}
}
// SendSync sends a Sync message to the backend (i.e. the server). The message is not guaranteed to be written until
// Flush is called.
// SendSync sends a Sync message to the backend (i.e. the server). The message is buffered until Flush is called. Any
// error encountered will be returned from Flush.
func (f *Frontend) SendSync(msg *Sync) {
if f.encodeError != nil {
return
}
prevLen := len(f.wbuf)
f.wbuf = msg.Encode(f.wbuf)
newBuf, err := msg.Encode(f.wbuf)
if err != nil {
f.encodeError = err
return
}
f.wbuf = newBuf
if f.tracer != nil {
f.tracer.traceSync('F', int32(len(f.wbuf)-prevLen), msg)
}
}
// SendQuery sends a Query message to the backend (i.e. the server). The message is not guaranteed to be written until
// Flush is called.
// SendQuery sends a Query message to the backend (i.e. the server). The message is buffered until Flush is called. Any
// error encountered will be returned from Flush.
func (f *Frontend) SendQuery(msg *Query) {
if f.encodeError != nil {
return
}
prevLen := len(f.wbuf)
f.wbuf = msg.Encode(f.wbuf)
newBuf, err := msg.Encode(f.wbuf)
if err != nil {
f.encodeError = err
return
}
f.wbuf = newBuf
if f.tracer != nil {
f.tracer.traceQuery('F', int32(len(f.wbuf)-prevLen), msg)
}
@@ -361,3 +448,7 @@ func (f *Frontend) findAuthenticationMessageType(src []byte) (BackendMessage, er
func (f *Frontend) GetAuthType() uint32 {
return f.authType
}
func (f *Frontend) ReadBufferLen() int {
return f.cr.wp - f.cr.rp
}
+13 -6
View File
@@ -2,6 +2,8 @@ package pgproto3
import (
"encoding/binary"
"errors"
"math"
"github.com/jackc/pgx/v5/internal/pgio"
)
@@ -71,15 +73,21 @@ func (dst *FunctionCall) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *FunctionCall) Encode(dst []byte) []byte {
dst = append(dst, 'F')
sp := len(dst)
dst = pgio.AppendUint32(dst, 0) // Unknown length, set it at the end
func (src *FunctionCall) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'F')
dst = pgio.AppendUint32(dst, src.Function)
if len(src.ArgFormatCodes) > math.MaxUint16 {
return nil, errors.New("too many arg format codes")
}
dst = pgio.AppendUint16(dst, uint16(len(src.ArgFormatCodes)))
for _, argFormatCode := range src.ArgFormatCodes {
dst = pgio.AppendUint16(dst, argFormatCode)
}
if len(src.Arguments) > math.MaxUint16 {
return nil, errors.New("too many arguments")
}
dst = pgio.AppendUint16(dst, uint16(len(src.Arguments)))
for _, argument := range src.Arguments {
if argument == nil {
@@ -90,6 +98,5 @@ func (src *FunctionCall) Encode(dst []byte) []byte {
}
}
dst = pgio.AppendUint16(dst, src.ResultFormatCode)
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
+3 -7
View File
@@ -39,10 +39,8 @@ func (dst *FunctionCallResponse) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length.
func (src *FunctionCallResponse) Encode(dst []byte) []byte {
dst = append(dst, 'V')
sp := len(dst)
dst = pgio.AppendInt32(dst, -1)
func (src *FunctionCallResponse) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'V')
if src.Result == nil {
dst = pgio.AppendInt32(dst, -1)
@@ -51,9 +49,7 @@ func (src *FunctionCallResponse) Encode(dst []byte) []byte {
dst = append(dst, src.Result...)
}
pgio.SetInt32(dst[sp:], int32(len(dst[sp:])))
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
+2 -2
View File
@@ -31,10 +31,10 @@ func (dst *GSSEncRequest) Decode(src []byte) error {
}
// Encode encodes src into dst. dst will include the 4 byte message length.
func (src *GSSEncRequest) Encode(dst []byte) []byte {
func (src *GSSEncRequest) Encode(dst []byte) ([]byte, error) {
dst = pgio.AppendInt32(dst, 8)
dst = pgio.AppendInt32(dst, gssEncReqNumber)
return dst
return dst, nil
}
// MarshalJSON implements encoding/json.Marshaler.
+3 -6
View File
@@ -2,8 +2,6 @@ package pgproto3
import (
"encoding/json"
"github.com/jackc/pgx/v5/internal/pgio"
)
type GSSResponse struct {
@@ -18,11 +16,10 @@ func (g *GSSResponse) Decode(data []byte) error {
return nil
}
func (g *GSSResponse) Encode(dst []byte) []byte {
dst = append(dst, 'p')
dst = pgio.AppendInt32(dst, int32(4+len(g.Data)))
func (g *GSSResponse) Encode(dst []byte) ([]byte, error) {
dst, sp := beginMessage(dst, 'p')
dst = append(dst, g.Data...)
return dst
return finishMessage(dst, sp)
}
// MarshalJSON implements encoding/json.Marshaler.
Loaded 100 of 402 files, more files were not shown because too many files have changed in this diff. Show more