Compare commits

...
Author SHA1 Message Date
Ronan Kervella 3faabd36c1 Merge branch 'master' into fix_missing_extension 2025-02-13 15:18:07 -08:00
Tim Makram Ghatas 2d9a46543d Keep track of file extension in implant config and automatically append correct extension when saving builds 2025-02-09 17:46:33 +01:00
Joe 2f486be19c Merge pull request #1872 from BishopFox/go/v1.23.5
Update to Go v1.23.5 and Garble v1.23.5
2025-02-05 10:56:03 -06:00
moloch-- 85868e5b8f Update to Go v1.23.5 and Garble v1.23.5 2025-01-29 11:28:42 -07:00
Joe 31c70938bd Merge pull request #1858 from BishopFox/48b9dependabot/go_modules/golang.org/x/net-0.33.0
Bump golang.org/x/net from 0.26.0 to 0.33.0
2025-01-21 11:07:27 -06:00
dependabot[bot] 63668f57b5 Bump golang.org/x/net from 0.26.0 to 0.33.0
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.26.0 to 0.33.0.
- [Commits](https://github.com/golang/net/compare/v0.26.0...v0.33.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <[email protected]>
2025-01-20 18:01:38 +00:00
Joe aefe6b31e9 Merge pull request #1854 from BishopFox/74eedependabot/go_modules/golang.org/x/crypto-0.32.0
Bump golang.org/x/crypto from 0.29.0 to 0.32.0
2025-01-20 12:00:00 -06:00
Joe f854af1134 Merge pull request #1855 from BishopFox/abide_generate_name
Abide generate name
2025-01-20 11:59:44 -06:00
Joe a9ef42a416 Merge branch 'master' into abide_generate_name 2025-01-20 11:03:14 -06:00
Joe 67c6140e6a Merge pull request #1857 from BishopFox/v1.6.0/download-refactor
Simplifying Implant Download Code
2025-01-20 11:02:33 -06:00
Raf 59352177eb Simplifying file downloads to address issue #1812 2025-01-17 12:30:49 -05:00
c2biz c9a624831a fix input bug to AllowedName in GenerateExternal() 2025-01-15 09:23:02 -05:00
c2biz ed64f2e0d2 fix bug ignoring supplied name for generate functions 2025-01-15 08:40:45 -05:00
dependabot[bot] 5f18609ed7 Bump golang.org/x/crypto from 0.29.0 to 0.32.0
Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.29.0 to 0.32.0.
- [Commits](https://github.com/golang/crypto/compare/v0.29.0...v0.32.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2025-01-13 03:30:28 +00:00
Timothy Makram Ghatas 53c61cb024 Merge pull request #1844 from w4lle-code/fix_c2profile_import_overwrite_url_parameters
fix to import overwrite c2Profile to url_parameters
2025-01-07 16:25:50 +01:00
Timothy Makram Ghatas 3d70247f8b Update httpclient.go - use updated uri for debug logs
Signed-off-by: Timothy Makram Ghatas <[email protected]>
2025-01-07 15:33:29 +01:00
Timothy Makram Ghatas 52b1c7b93b Merge branch 'master' into fix_c2profile_import_overwrite_url_parameters 2025-01-07 14:57:24 +01:00
Timothy Makram Ghatas daeb463f77 Update httpclient.go
add check for method in implant httpclient

Signed-off-by: Timothy Makram Ghatas <[email protected]>
2025-01-07 14:46:32 +01:00
Joe 0c75f70a1d Merge pull request #1847 from BishopFox/404ddependabot/npm_and_yarn/docs/sliver-docs/next-14.2.22
Bump next from 14.2.20 to 14.2.22 in /docs/sliver-docs
2025-01-06 13:41:51 -06:00
AndresFelipeGualdron 674df112a4 adding method of url parameter in template 2025-01-04 20:12:10 -05:00
AndresFelipeGualdron 26efc776c7 fix to import no overwrite c2Profile 2025-01-03 22:32:53 -05:00
dependabot[bot] f221dc95cf Bump next from 14.2.20 to 14.2.22 in /docs/sliver-docs
Bumps [next](https://github.com/vercel/next.js) from 14.2.20 to 14.2.22.
- [Release notes](https://github.com/vercel/next.js/releases)
- [Changelog](https://github.com/vercel/next.js/blob/canary/release.js)
- [Commits](https://github.com/vercel/next.js/compare/v14.2.20...v14.2.22)

---
updated-dependencies:
- dependency-name: next
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <[email protected]>
2025-01-04 02:59:04 +00:00
AndresFelipeGualdron 187fc6acd2 fix to import overwrite c2Profile to url_parameters 2025-01-03 12:01:42 -05:00
Joe 9ecb68f6a3 Merge pull request #1841 from BishopFox/fix_c2profile_display
fix broken pr for http headers and add additional constraints for c2p…
2025-01-02 20:42:29 -06:00
Tim Makram Ghatas f0113655ce fix broken pr for http headers and add additional constraints for c2profile import 2025-01-02 14:53:31 +01:00
Joe 31265c626b Merge pull request #1836 from BishopFox/b3f8dependabot/npm_and_yarn/docs/sliver-docs/next-14.2.20
Bump next from 14.2.12 to 14.2.20 in /docs/sliver-docs
2024-12-18 15:26:49 -06:00
dependabot[bot] e2603ac3ae Bump next from 14.2.12 to 14.2.20 in /docs/sliver-docs
Bumps [next](https://github.com/vercel/next.js) from 14.2.12 to 14.2.20.
- [Release notes](https://github.com/vercel/next.js/releases)
- [Changelog](https://github.com/vercel/next.js/blob/canary/release.js)
- [Commits](https://github.com/vercel/next.js/compare/v14.2.12...v14.2.20)

---
updated-dependencies:
- dependency-name: next
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-12-18 19:02:18 +00:00
Ronan Kervella 7acbed9637 Merge pull request #1724 from BishopFox/4ddedependabot/go_modules/gorm.io/driver/postgres-1.5.9
Bump gorm.io/driver/postgres from 1.5.7 to 1.5.9
2024-12-03 10:33:02 -10:00
Ronan Kervella 20786fa6fb Merge branch 'master' into 4ddedependabot/go_modules/gorm.io/driver/postgres-1.5.9
Signed-off-by: Ronan Kervella <[email protected]>
2024-12-03 10:34:33 -08:00
Ronan Kervella 03c750a8b5 Merge pull request #1741 from BishopFox/378ddependabot/go_modules/gorm.io/driver/mysql-1.5.7
Bump gorm.io/driver/mysql from 1.5.6 to 1.5.7
2024-12-03 10:30:21 -08:00
Ronan Kervella 4da8c6053f Merge branch 'master' into 4ddedependabot/go_modules/gorm.io/driver/postgres-1.5.9 2024-12-02 16:12:23 -08:00
Ronan Kervella a43061c61b Merge branch 'master' into 378ddependabot/go_modules/gorm.io/driver/mysql-1.5.7 2024-12-02 15:40:49 -08:00
Ronan Kervella 11dc6bac4d Merge pull request #1824 from BishopFox/b42adependabot/go_modules/github.com/ncruces/go-sqlite3-0.20.3
Bump github.com/ncruces/go-sqlite3 from 0.15.0 to 0.20.3
2024-12-02 15:39:58 -08:00
dependabot[bot] 7d5a886397 Bump github.com/ncruces/go-sqlite3 from 0.15.0 to 0.20.3
Bumps [github.com/ncruces/go-sqlite3](https://github.com/ncruces/go-sqlite3) from 0.15.0 to 0.20.3.
- [Release notes](https://github.com/ncruces/go-sqlite3/releases)
- [Commits](https://github.com/ncruces/go-sqlite3/compare/v0.15.0...v0.20.3)

---
updated-dependencies:
- dependency-name: github.com/ncruces/go-sqlite3
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-12-02 03:42:38 +00:00
Joe 77aef8179d Merge branch 'master' into 4ddedependabot/go_modules/gorm.io/driver/postgres-1.5.9 2024-11-25 10:28:39 -06:00
Joe 69e7c8ce71 Merge pull request #1811 from rwincey/c2_profile_headers_fix
Fixed implant custom HTTP header code
2024-11-11 14:40:00 -06:00
b0yd 82e294e3ef Gofmt 2024-11-06 15:40:48 +00:00
b0yd ccf3701011 Fixed implant custom HTTP header code 2024-11-06 15:26:40 +00:00
Joe e4b64e1050 Merge pull request #1800 from toasted-time-traveller/fix-wording
Added a missing word and a missing period.
2024-10-30 12:19:52 -05:00
Joe 0b8f702aa0 Merge pull request #1787 from An00bRektn/master
Update link in Community Guides section of docs
2024-10-30 12:19:14 -05:00
Ronan Kervella 8caf90536b Merge branch 'master' into master 2024-10-30 08:27:08 -07:00
toasted-time-traveller 50e7d89576 Added the missing word 'at' to the note section and a period to the 'Feedback' section in README 2024-10-23 03:38:28 +02:00
Joe fe5e9382d7 Merge pull request #1796 from AD2011/patch-1
Correct Windows compile command
2024-10-17 10:40:53 -05:00
Abir Dey 934c01f08d Correct Windows compile command
Signed-off-by: Abir Dey <[email protected]>
2024-10-17 14:17:28 +05:30
An00bRektn 4ced15e528 Changed link to point to updated location 2024-10-04 17:14:39 -05:00
Joe 3c151ed172 Merge pull request #1784 from BishopFox/docs/community-guides
Add detecting sliver c2 with zeek to community guides
2024-09-29 17:04:22 -05:00
Ronan Kervella 2a849312cc Merge branch 'master' into 378ddependabot/go_modules/gorm.io/driver/mysql-1.5.7 2024-07-31 13:21:26 -07:00
dependabot[bot] 1bf020c594 Bump gorm.io/driver/mysql from 1.5.6 to 1.5.7
Bumps [gorm.io/driver/mysql](https://github.com/go-gorm/mysql) from 1.5.6 to 1.5.7.
- [Commits](https://github.com/go-gorm/mysql/compare/v1.5.6...v1.5.7)

---
updated-dependencies:
- dependency-name: gorm.io/driver/mysql
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-07-18 18:08:17 +00:00
Ronan Kervella c3b0909be0 Merge branch 'master' into 4ddedependabot/go_modules/gorm.io/driver/postgres-1.5.9 2024-06-28 11:10:57 -07:00
dependabot[bot] 4a149f073a Bump gorm.io/driver/postgres from 1.5.7 to 1.5.9
Bumps [gorm.io/driver/postgres](https://github.com/go-gorm/postgres) from 1.5.7 to 1.5.9.
- [Commits](https://github.com/go-gorm/postgres/compare/v1.5.7...v1.5.9)

---
updated-dependencies:
- dependency-name: gorm.io/driver/postgres
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-06-17 03:11:56 +00:00
299 changed files with 34129 additions and 16609 deletions

No files matched your search

+1 -1
View File
@@ -9,7 +9,7 @@
# STAGE: base
## Compiles Sliver for use
FROM golang:1.22.5 AS base
FROM golang:1.23.5 AS base
### Base packages
RUN apt-get update --fix-missing && apt-get -y install \
+2 -2
View File
@@ -8,7 +8,7 @@ The server and client support MacOS, Windows, and Linux. Implants are supported
# v1.6.0 / `master`
**NOTE:** You are looking the latest master branch of Sliver v1.6.0; new PRs should target this branch. However, this branch is NOT RECOMMENDED for production use yet. Please use release tagged versions for the best experience.
**NOTE:** You are looking at the latest master branch of Sliver v1.6.0; new PRs should target this branch. However, this branch is NOT RECOMMENDED for production use yet. Please use release tagged versions for the best experience.
For PRs containing bug fixes specific to Sliver v1.5, please target the [`v1.5.x/master` branch](https://github.com/BishopFox/sliver/tree/v1.5.x/master).
@@ -47,7 +47,7 @@ See the [wiki](https://sliver.sh/docs?name=Compile+from+Source).
### Feedback
Please take a moment and fill out [our survey](https://forms.gle/SwVsHFNh24ChG58C6)
Please take a moment and fill out [our survey](https://forms.gle/SwVsHFNh24ChG58C6).
### License - GPLv3
+1 -1
View File
@@ -35,7 +35,7 @@ type NameValueProbability struct {
Name string `json:"name"`
Value string `json:"value"`
Probability int `json:"probability"`
Methods []string
Method string `json:"method"`
}
// HTTPC2ImplantConfig - Implant configuration options
+28 -10
View File
@@ -292,6 +292,7 @@ func C2ConfigToJSON(profileName string, profile *clientpb.HTTPC2Config) (*assets
Name: header.Name,
Value: header.Value,
Probability: int(header.Probability),
Method: header.Method,
})
}
implantConfig.Headers = headers
@@ -358,6 +359,7 @@ func C2ConfigToJSON(profileName string, profile *clientpb.HTTPC2Config) (*assets
Name: header.Name,
Value: header.Value,
Probability: int(header.Probability),
Method: header.Method,
})
}
@@ -453,6 +455,24 @@ func C2ConfigToProtobuf(profileName string, config *assets.HTTPC2Config) *client
})
}
for _, clientHeader := range config.ImplantConfig.Headers {
httpC2Headers = append(httpC2Headers, &clientpb.HTTPC2Header{
Method: clientHeader.Method,
Name: clientHeader.Name,
Value: clientHeader.Value,
Probability: int32(clientHeader.Probability),
})
}
for _, urlParameter := range config.ImplantConfig.URLParameters {
httpC2UrlParameters = append(httpC2UrlParameters, &clientpb.HTTPC2URLParameter{
Method: urlParameter.Method,
Name: urlParameter.Name,
Value: urlParameter.Value,
Probability: int32(urlParameter.Probability),
})
}
implantConfig := &clientpb.HTTPC2ImplantConfig{
UserAgent: config.ImplantConfig.UserAgent,
ChromeBaseVersion: int32(config.ImplantConfig.ChromeBaseVersion),
@@ -475,14 +495,12 @@ func C2ConfigToProtobuf(profileName string, config *assets.HTTPC2Config) *client
// Server Config
serverHeaders := []*clientpb.HTTPC2Header{}
for _, serverHeader := range config.ServerConfig.Headers {
for _, method := range serverHeader.Methods {
serverHeaders = append(serverHeaders, &clientpb.HTTPC2Header{
Method: method,
Name: serverHeader.Name,
Value: serverHeader.Value,
Probability: int32(serverHeader.Probability),
})
}
serverHeaders = append(serverHeaders, &clientpb.HTTPC2Header{
Method: serverHeader.Method,
Name: serverHeader.Name,
Value: serverHeader.Value,
Probability: int32(serverHeader.Probability),
})
}
serverCookies := []*clientpb.HTTPC2Cookie{}
@@ -525,7 +543,7 @@ func PrintC2Profiles(profile *clientpb.HTTPC2Config, con *console.SliverClient)
var serverHeaders []string
for _, header := range profile.ServerConfig.Headers {
serverHeaders = append(serverHeaders, header.Value)
serverHeaders = append(serverHeaders, header.Name)
}
tw.AppendRow(table.Row{
"Server Headers",
@@ -550,7 +568,7 @@ func PrintC2Profiles(profile *clientpb.HTTPC2Config, con *console.SliverClient)
var clientHeaders []string
for _, header := range profile.ImplantConfig.Headers {
clientHeaders = append(clientHeaders, header.Value)
clientHeaders = append(clientHeaders, header.Name)
}
tw.AppendRow(table.Row{
"Client Headers",
+7 -3
View File
@@ -23,6 +23,7 @@ import (
"encoding/hex"
"fmt"
"os"
"strings"
"github.com/alecthomas/chroma/formatters"
"github.com/alecthomas/chroma/lexers"
@@ -72,16 +73,16 @@ func CatCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
con.PrintErrorf("Failed to decode response %s\n", err)
return
}
PrintCat(download, cmd, con)
PrintCat(filePath, download, cmd, con)
})
con.PrintAsyncResponse(download.Response)
} else {
PrintCat(download, cmd, con)
PrintCat(filePath, download, cmd, con)
}
}
// PrintCat - Print the download to stdout.
func PrintCat(download *sliverpb.Download, cmd *cobra.Command, con *console.SliverClient) {
func PrintCat(originalFileName string, download *sliverpb.Download, cmd *cobra.Command, con *console.SliverClient) {
var (
lootDownload bool = true
err error
@@ -108,6 +109,9 @@ func PrintCat(download *sliverpb.Download, cmd *cobra.Command, con *console.Sliv
con.Printf("\n")
}
}
if !strings.Contains(download.Path, originalFileName) {
con.PrintInfof("Supplied pattern %s matched file %s\n\n", originalFileName, download.Path)
}
if color, _ := cmd.Flags().GetBool("colorize-output"); color {
if err = colorize(download); err != nil {
con.Println(string(download.Data))
+2 -2
View File
@@ -295,7 +295,7 @@ func Commands(con *console.SliverClient) []*cobra.Command {
f.StringP("file-type", "F", "", "force a specific file type (binary/text) if looting (optional)")
f.Int64P("timeout", "t", flags.DefaultTimeout, "grpc timeout in seconds")
f.Int64P("bytes", "b", 0, "Grab the first number of bytes from the file")
f.Int64P("lines", "l", 0, "Grab the first number of lines from the file")
f.Int64P("lines", "l", 10, "Grab the first number of lines from the file")
})
carapace.Gen(headCmd).PositionalCompletion(carapace.ActionValues().Usage("path to the file to print"))
@@ -322,7 +322,7 @@ func Commands(con *console.SliverClient) []*cobra.Command {
f.StringP("file-type", "F", "", "force a specific file type (binary/text) if looting (optional)")
f.Int64P("timeout", "t", flags.DefaultTimeout, "grpc timeout in seconds")
f.Int64P("bytes", "b", 0, "Grab the last number of bytes from the file")
f.Int64P("lines", "l", 0, "Grab the last number of lines from the file")
f.Int64P("lines", "l", 10, "Grab the last number of lines from the file")
})
carapace.Gen(tailCmd).PositionalCompletion(carapace.ActionValues().Usage("path to the file to print"))
+10 -4
View File
@@ -97,9 +97,7 @@ func prettifyDownloadName(path string) string {
filteredString = multipleUnderscoreRegex.ReplaceAllString(filteredString, "_")
// If there is an underscore at the front of the filename, strip that off
if strings.HasPrefix(filteredString, "_") {
filteredString = filteredString[1:]
}
filteredString, _ = strings.CutPrefix(filteredString, "_")
return filteredString
}
@@ -118,7 +116,15 @@ func HandleDownloadResponse(download *sliverpb.Download, cmd *cobra.Command, arg
}
}
remotePath := args[0]
// Use download.Path because a glob matching a single file on the remote will not have the
// correct file name - the filename will contain the globs if we use the path from the user
// On non-Windows systems, filepath.Base will not see backslashes, so we will replace them
// on systems that do not use backslashes as path separators
remotePath := download.Path
if strings.Contains(download.Path, "\\") && string(os.PathSeparator) != "\\" {
remotePath = strings.ReplaceAll(download.Path, "\\", "/")
}
var localPath string
if len(args) == 1 {
localPath = "."
+3 -6
View File
@@ -63,7 +63,7 @@ func HeadCmd(cmd *cobra.Command, con *console.SliverClient, args []string, head
} else {
operationName = "bytes"
}
} else if cmd.Flags().Changed("lines") {
} else {
fetchBytes = false
fetchSize, _ = cmd.Flags().GetInt64("lines")
if fetchSize < 0 {
@@ -76,9 +76,6 @@ func HeadCmd(cmd *cobra.Command, con *console.SliverClient, args []string, head
} else {
operationName = "lines"
}
} else {
con.PrintErrorf("A number of bytes or a number of lines must be specified.")
return
}
ctrl := make(chan bool)
@@ -118,10 +115,10 @@ func HeadCmd(cmd *cobra.Command, con *console.SliverClient, args []string, head
con.PrintErrorf("Failed to decode response %s\n", err)
return
}
PrintCat(download, cmd, con)
PrintCat(filePath, download, cmd, con)
})
con.PrintAsyncResponse(download.Response)
} else {
PrintCat(download, cmd, con)
PrintCat(filePath, download, cmd, con)
}
}
+1 -1
View File
@@ -32,7 +32,7 @@ func GenerateBeaconCmd(cmd *cobra.Command, con *console.SliverClient, args []str
save, _ = os.Getwd()
}
if external, _ := cmd.Flags().GetBool("external-builder"); !external {
compile(config, save, con)
compile(name, config, save, con)
} else {
externalBuild(name, config, save, con)
}
+3 -2
View File
@@ -98,7 +98,7 @@ func GenerateCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
save, _ = os.Getwd()
}
if external, _ := cmd.Flags().GetBool("external-builder"); !external {
compile(config, save, con)
compile(name, config, save, con)
} else {
_, err := externalBuild(name, config, save, con)
if err != nil {
@@ -905,7 +905,7 @@ func externalBuild(name string, config *clientpb.ImplantConfig, save string, con
return nil, nil
}
func compile(config *clientpb.ImplantConfig, save string, con *console.SliverClient) (*commonpb.File, error) {
func compile(name string, config *clientpb.ImplantConfig, save string, con *console.SliverClient) (*commonpb.File, error) {
if config.IsBeacon {
interval := time.Duration(config.BeaconInterval)
con.PrintInfof("Generating new %s/%s beacon implant binary (%v)\n", config.GOOS, config.GOARCH, interval)
@@ -923,6 +923,7 @@ func compile(config *clientpb.ImplantConfig, save string, con *console.SliverCli
con.SpinUntil("Compiling, please wait ...", ctrl)
generated, err := con.Rpc.Generate(context.Background(), &clientpb.GenerateReq{
Name: name,
Config: config,
})
ctrl <- true
+1 -1
View File
@@ -46,7 +46,7 @@ func ProfilesGenerateCmd(cmd *cobra.Command, con *console.SliverClient, args []s
if SGNDisabled, _ := cmd.Flags().GetBool("disable-sgn"); SGNDisabled {
profile.Config.SGNEnabled = !SGNDisabled
}
_, err := compile(profile.Config, save, con)
_, err := compile(name, profile.Config, save, con)
if err != nil {
return
}
+44 -44
View File
@@ -19,7 +19,7 @@
"framer-motion": "^10.16.16",
"fuse.js": "^7.0.0",
"lunr": "^2.3.9",
"next": "^14.2.12",
"next": "^14.2.22",
"next-themes": "^0.2.1",
"react": "18.2.0",
"react-dom": "18.2.0",
@@ -383,9 +383,9 @@
}
},
"node_modules/@next/env": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.12.tgz",
"integrity": "sha512-3fP29GIetdwVIfIRyLKM7KrvJaqepv+6pVodEbx0P5CaMLYBtx+7eEg8JYO5L9sveJO87z9eCReceZLi0hxO1Q=="
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.22.tgz",
"integrity": "sha512-EQ6y1QeNQglNmNIXvwP/Bb+lf7n9WtgcWvtoFsHquVLCJUuxRs+6SfZ5EK0/EqkkLex4RrDySvKgKNN7PXip7Q=="
},
"node_modules/@next/eslint-plugin-next": {
"version": "14.0.4",
@@ -397,9 +397,9 @@
}
},
"node_modules/@next/swc-darwin-arm64": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.12.tgz",
"integrity": "sha512-crHJ9UoinXeFbHYNok6VZqjKnd8rTd7K3Z2zpyzF1ch7vVNKmhjv/V7EHxep3ILoN8JB9AdRn/EtVVyG9AkCXw==",
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.22.tgz",
"integrity": "sha512-HUaLiehovgnqY4TMBZJ3pDaOsTE1spIXeR10pWgdQVPYqDGQmHJBj3h3V6yC0uuo/RoY2GC0YBFRkOX3dI9WVQ==",
"cpu": [
"arm64"
],
@@ -412,9 +412,9 @@
}
},
"node_modules/@next/swc-darwin-x64": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.12.tgz",
"integrity": "sha512-JbEaGbWq18BuNBO+lCtKfxl563Uw9oy2TodnN2ioX00u7V1uzrsSUcg3Ep9ce+P0Z9es+JmsvL2/rLphz+Frcw==",
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.22.tgz",
"integrity": "sha512-ApVDANousaAGrosWvxoGdLT0uvLBUC+srqOcpXuyfglA40cP2LBFaGmBjhgpxYk5z4xmunzqQvcIgXawTzo2uQ==",
"cpu": [
"x64"
],
@@ -427,9 +427,9 @@
}
},
"node_modules/@next/swc-linux-arm64-gnu": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.12.tgz",
"integrity": "sha512-qBy7OiXOqZrdp88QEl2H4fWalMGnSCrr1agT/AVDndlyw2YJQA89f3ttR/AkEIP9EkBXXeGl6cC72/EZT5r6rw==",
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.22.tgz",
"integrity": "sha512-3O2J99Bk9aM+d4CGn9eEayJXHuH9QLx0BctvWyuUGtJ3/mH6lkfAPRI4FidmHMBQBB4UcvLMfNf8vF0NZT7iKw==",
"cpu": [
"arm64"
],
@@ -442,9 +442,9 @@
}
},
"node_modules/@next/swc-linux-arm64-musl": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.12.tgz",
"integrity": "sha512-EfD9L7o9biaQxjwP1uWXnk3vYZi64NVcKUN83hpVkKocB7ogJfyH2r7o1pPnMtir6gHZiGCeHKagJ0yrNSLNHw==",
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.22.tgz",
"integrity": "sha512-H/hqfRz75yy60y5Eg7DxYfbmHMjv60Dsa6IWHzpJSz4MRkZNy5eDnEW9wyts9bkxwbOVZNPHeb3NkqanP+nGPg==",
"cpu": [
"arm64"
],
@@ -457,9 +457,9 @@
}
},
"node_modules/@next/swc-linux-x64-gnu": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.12.tgz",
"integrity": "sha512-iQ+n2pxklJew9IpE47hE/VgjmljlHqtcD5UhZVeHICTPbLyrgPehaKf2wLRNjYH75udroBNCgrSSVSVpAbNoYw==",
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.22.tgz",
"integrity": "sha512-LckLwlCLcGR1hlI5eiJymR8zSHPsuruuwaZ3H2uudr25+Dpzo6cRFjp/3OR5UYJt8LSwlXv9mmY4oI2QynwpqQ==",
"cpu": [
"x64"
],
@@ -472,9 +472,9 @@
}
},
"node_modules/@next/swc-linux-x64-musl": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.12.tgz",
"integrity": "sha512-rFkUkNwcQ0ODn7cxvcVdpHlcOpYxMeyMfkJuzaT74xjAa5v4fxP4xDk5OoYmPi8QNLDs3UgZPMSBmpBuv9zKWA==",
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.22.tgz",
"integrity": "sha512-qGUutzmh0PoFU0fCSu0XYpOfT7ydBZgDfcETIeft46abPqP+dmePhwRGLhFKwZWxNWQCPprH26TjaTxM0Nv8mw==",
"cpu": [
"x64"
],
@@ -487,9 +487,9 @@
}
},
"node_modules/@next/swc-win32-arm64-msvc": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.12.tgz",
"integrity": "sha512-PQFYUvwtHs/u0K85SG4sAdDXYIPXpETf9mcEjWc0R4JmjgMKSDwIU/qfZdavtP6MPNiMjuKGXHCtyhR/M5zo8g==",
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.22.tgz",
"integrity": "sha512-K6MwucMWmIvMb9GlvT0haYsfIPxfQD8yXqxwFy4uLFMeXIb2TcVYQimxkaFZv86I7sn1NOZnpOaVk5eaxThGIw==",
"cpu": [
"arm64"
],
@@ -502,9 +502,9 @@
}
},
"node_modules/@next/swc-win32-ia32-msvc": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.12.tgz",
"integrity": "sha512-FAj2hMlcbeCV546eU2tEv41dcJb4NeqFlSXU/xL/0ehXywHnNpaYajOUvn3P8wru5WyQe6cTZ8fvckj/2XN4Vw==",
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.22.tgz",
"integrity": "sha512-5IhDDTPEbzPR31ZzqHe90LnNe7BlJUZvC4sA1thPJV6oN5WmtWjZ0bOYfNsyZx00FJt7gggNs6SrsX0UEIcIpA==",
"cpu": [
"ia32"
],
@@ -517,9 +517,9 @@
}
},
"node_modules/@next/swc-win32-x64-msvc": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.12.tgz",
"integrity": "sha512-yu8QvV53sBzoIVRHsxCHqeuS8jYq6Lrmdh0briivuh+Brsp6xjg80MAozUsBTAV9KNmY08KlX0KYTWz1lbPzEg==",
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.22.tgz",
"integrity": "sha512-nvRaB1PyG4scn9/qNzlkwEwLzuoPH3Gjp7Q/pLuwUgOTt1oPMlnCI3A3rgkt+eZnU71emOiEv/mR201HoURPGg==",
"cpu": [
"x64"
],
@@ -6588,11 +6588,11 @@
"dev": true
},
"node_modules/next": {
"version": "14.2.12",
"resolved": "https://registry.npmjs.org/next/-/next-14.2.12.tgz",
"integrity": "sha512-cDOtUSIeoOvt1skKNihdExWMTybx3exnvbFbb9ecZDIxlvIbREQzt9A5Km3Zn3PfU+IFjyYGsHS+lN9VInAGKA==",
"version": "14.2.22",
"resolved": "https://registry.npmjs.org/next/-/next-14.2.22.tgz",
"integrity": "sha512-Ps2caobQ9hlEhscLPiPm3J3SYhfwfpMqzsoCMZGWxt9jBRK9hoBZj2A37i8joKhsyth2EuVKDVJCTF5/H4iEDw==",
"dependencies": {
"@next/env": "14.2.12",
"@next/env": "14.2.22",
"@swc/helpers": "0.5.5",
"busboy": "1.6.0",
"caniuse-lite": "^1.0.30001579",
@@ -6607,15 +6607,15 @@
"node": ">=18.17.0"
},
"optionalDependencies": {
"@next/swc-darwin-arm64": "14.2.12",
"@next/swc-darwin-x64": "14.2.12",
"@next/swc-linux-arm64-gnu": "14.2.12",
"@next/swc-linux-arm64-musl": "14.2.12",
"@next/swc-linux-x64-gnu": "14.2.12",
"@next/swc-linux-x64-musl": "14.2.12",
"@next/swc-win32-arm64-msvc": "14.2.12",
"@next/swc-win32-ia32-msvc": "14.2.12",
"@next/swc-win32-x64-msvc": "14.2.12"
"@next/swc-darwin-arm64": "14.2.22",
"@next/swc-darwin-x64": "14.2.22",
"@next/swc-linux-arm64-gnu": "14.2.22",
"@next/swc-linux-arm64-musl": "14.2.22",
"@next/swc-linux-x64-gnu": "14.2.22",
"@next/swc-linux-x64-musl": "14.2.22",
"@next/swc-win32-arm64-msvc": "14.2.22",
"@next/swc-win32-ia32-msvc": "14.2.22",
"@next/swc-win32-x64-msvc": "14.2.22"
},
"peerDependencies": {
"@opentelemetry/api": "^1.1.0",
+1 -1
View File
@@ -21,7 +21,7 @@
"framer-motion": "^10.16.16",
"fuse.js": "^7.0.0",
"lunr": "^2.3.9",
"next": "^14.2.12",
"next": "^14.2.22",
"next-themes": "^0.2.1",
"react": "18.2.0",
"react-dom": "18.2.0",
@@ -15,7 +15,7 @@
### Blogs
- https://bishopfox.com/blog/passing-the-osep-exam-using-sliver
- https://notateamserver.xyz/sliver-101/
- https://notateamserver.xyz/blog/sliver-101/
- https://dominicbreuker.com/post/learning_sliver_c2_01_installation/
- https://vk9-sec.com/how-to-set-up-use-c2-sliver/
- https://0x90-1.gitbook.io/sliver/
@@ -120,7 +120,7 @@ $ make macos
$ make macos-arm64
$ make linux
$ make linux-arm64
$ make windows
$ make windows-amd64
```
### Docker Build
@@ -140,4 +140,4 @@ The Docker build includes mingw and Metasploit, so it can take a while to build
- Kali/Ubuntu/Debian `sudo apt install mingw-w64`
- MacOS `brew install mingw-w64`
If all you have is a Windows machine, the easiest way to build Sliver is using [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) and following the Linux/cross-compile instructions above. To cross-compile a native Windows binary use `make windows` and copy it to your Windows file system (i.e. `/mnt/c/Users/foo/Desktop`) and run it using a terminal that supports ANSI sequences such as the [Windows Terminal](https://github.com/microsoft/terminal).
If all you have is a Windows machine, the easiest way to build Sliver is using [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) and following the Linux/cross-compile instructions above. To cross-compile a native Windows binary use `make windows-amd64` and copy it to your Windows file system (i.e. `/mnt/c/Users/foo/Desktop`) and run it using a terminal that supports ANSI sequences such as the [Windows Terminal](https://github.com/microsoft/terminal).
+2 -2
View File
@@ -20,8 +20,8 @@ set -e
# Creates the static go asset archives
GO_VER="1.22.5"
GARBLE_VER="1.22.5"
GO_VER="1.23.5"
GARBLE_VER="1.23.5"
ZIG_VER="0.13.0"
SGN_VER="0.0.3"
+11 -11
View File
@@ -34,7 +34,7 @@ require (
github.com/miekg/dns v1.1.59
github.com/moloch--/asciicast v0.1.0
github.com/moloch--/memmod v0.0.0-20230225130813-fd77d905589e
github.com/ncruces/go-sqlite3 v0.15.0
github.com/ncruces/go-sqlite3 v0.20.3
github.com/reeflective/console v0.1.15
github.com/reeflective/readline v1.0.14
github.com/rsteube/carapace v0.46.3-0.20231214181515-27e49f3c3b69
@@ -42,24 +42,24 @@ require (
github.com/spf13/cobra v1.8.1
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.9.0
github.com/tetratelabs/wazero v1.8.0
github.com/tetratelabs/wazero v1.8.2
github.com/things-go/go-socks5 v0.0.5
github.com/ulikunitz/xz v0.5.12
github.com/xlab/treeprint v1.2.0
github.com/yiya1989/sshkrb5 v0.0.1
golang.org/x/crypto v0.25.0
golang.org/x/crypto v0.32.0
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
golang.org/x/net v0.26.0
golang.org/x/sys v0.22.0
golang.org/x/term v0.22.0
golang.org/x/text v0.16.0
golang.org/x/net v0.33.0
golang.org/x/sys v0.29.0
golang.org/x/term v0.28.0
golang.org/x/text v0.21.0
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6
google.golang.org/grpc v1.64.1
google.golang.org/protobuf v1.34.1
gopkg.in/AlecAivazis/survey.v1 v1.8.8
gorm.io/driver/mysql v1.5.6
gorm.io/driver/postgres v1.5.7
gorm.io/driver/mysql v1.5.7
gorm.io/driver/postgres v1.5.9
gorm.io/driver/sqlite v1.5.5
gorm.io/gorm v1.25.10
gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3
@@ -127,7 +127,7 @@ require (
github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx/v5 v5.5.4 // indirect
github.com/jackc/pgx/v5 v5.5.5 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
@@ -175,7 +175,7 @@ require (
go4.org/mem v0.0.0-20220726221520-4f986261bf13 // indirect
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
+22 -22
View File
@@ -215,8 +215,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8=
github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
@@ -303,8 +303,8 @@ github.com/moloch--/asciicast v0.1.0 h1:eBOJwuFKSk447s/kPs9MWsc4kAl5HmuKIDLDYD6/
github.com/moloch--/asciicast v0.1.0/go.mod h1:OckO16UDLgxVLclrCnbocL1ix15Br/8Xv/caBoYq98o=
github.com/moloch--/memmod v0.0.0-20230225130813-fd77d905589e h1:IkFCPlAa0iTiLxck+NqAwBx8JDlnHYm4orOQBbs4BDQ=
github.com/moloch--/memmod v0.0.0-20230225130813-fd77d905589e/go.mod h1:eYeI6cQ5YHhHt9i0BBW0zc1DaQnb4ZMXsSPuEV/V5Og=
github.com/ncruces/go-sqlite3 v0.15.0 h1:C+SIrcYsAIR5GUYWmCnif6x81n6BS9y75vYcQynuGNU=
github.com/ncruces/go-sqlite3 v0.15.0/go.mod h1:kHHYmFmK4G2VFFoIovEg9BEQ8BP+D81y4ESHXnzJV/w=
github.com/ncruces/go-sqlite3 v0.20.3 h1:+4G4uEqOeusF0yRuQVUl9fuoEebUolwQSnBUjYBLYIw=
github.com/ncruces/go-sqlite3 v0.20.3/go.mod h1:ojLIAB243gtz68Eo283Ps+k9PyR3dvzS+9/RgId4+AA=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
@@ -387,8 +387,8 @@ github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA=
github.com/tc-hib/winres v0.2.1/go.mod h1:C/JaNhH3KBvhNKVbvdlDWkbMDO9H4fKKDaN7/07SSuk=
github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g=
github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
github.com/tetratelabs/wazero v1.8.2 h1:yIgLR/b2bN31bjxwXHD8a3d+BogigR952csSDdLYEv4=
github.com/tetratelabs/wazero v1.8.2/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
github.com/thedevsaddam/gojsonq/v2 v2.5.2 h1:CoMVaYyKFsVj6TjU6APqAhAvC07hTI6IQen8PHzHYY0=
github.com/thedevsaddam/gojsonq/v2 v2.5.2/go.mod h1:bv6Xa7kWy82uT0LnXPE2SzGqTj33TAEeR560MdJkiXs=
github.com/things-go/go-socks5 v0.0.5 h1:qvKaGcBkfDrUL33SchHN93srAmYGzb4CxSM2DPYufe8=
@@ -427,8 +427,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
@@ -455,8 +455,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
@@ -467,8 +467,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -498,18 +498,18 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -578,10 +578,10 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/mysql v1.5.6 h1:Ld4mkIickM+EliaQZQx3uOJDJHtrd70MxAUqWqlx3Y8=
gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM=
gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA=
gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8=
gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI=
gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E=
gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE=
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+177 -115
View File
@@ -472,105 +472,197 @@ func pwdHandler(data []byte, resp RPCResponse) {
resp(data, err)
}
func prepareDownload(path string, filter string, recurse bool, maxBytes int64, maxLines int64) ([]byte, bool, int, int, error) {
func readSingleFile(path string, maxBytes, maxLines int64) ([]byte, error) {
fileHandle, err := os.Open(path)
if err != nil {
return nil, err
}
defer fileHandle.Close()
/*
Combine the path and filter to see if the user wants
to download a single file
Made a bit of a design decision - this function is going to go for accuracy.
To that end, if maxLines is specified and those lines are all blank, that is
what the user will get back. The other approach would be to skip blank lines
but that would not be accurate. So if you head or tail a file that starts or
ends with blank lines, you are going to get those blank lines. :)
*/
var rawData []byte
var err error
fileInfo, err := os.Stat(path + filter)
if err != nil && os.IsNotExist(err) {
// Then the file does not exist
return nil, false, 0, 1, err
// If maxBytes is negative, seek to that many bytes from the end of the file
if maxBytes < 0 {
_, err = fileHandle.Seek(maxBytes, io.SeekEnd)
if err != nil {
return nil, err
}
}
if err == nil && !fileInfo.IsDir() {
// Then this is a single file
fileHandle, err := os.Open(path + filter)
if err != nil {
// Then we could not read the file
return nil, false, 0, 1, err
}
defer fileHandle.Close()
reader := bufio.NewReader(fileHandle)
lines := []string{}
var bytesRead int64 = 0
if maxBytes != 0 {
var readFirst bool = maxBytes > 0
if readFirst {
rawData = make([]byte, maxBytes)
_, err = fileHandle.Read(rawData)
} else {
rawData = make([]byte, maxBytes*-1)
var bytesToRead int64 = 0
if fileInfo.Size()+maxBytes < 0 {
bytesToRead = 0
} else {
bytesToRead = fileInfo.Size() + maxBytes
}
_, err = fileHandle.ReadAt(rawData, bytesToRead)
}
} else if maxLines != 0 {
var linesRead int64 = 0
var lines []string
var readFirst bool = true
if maxLines < 0 {
maxLines *= -1
readFirst = false
}
fileScanner := bufio.NewScanner(fileHandle)
for fileScanner.Scan() {
lines = append(lines, fileScanner.Text())
linesRead += 1
if linesRead == maxLines && readFirst {
break
}
}
err = fileScanner.Err()
if err == nil {
if readFirst {
rawData = []byte(strings.Join(lines, "\n"))
} else {
linePosition := int64(len(lines)) - maxLines
if linePosition < 0 {
linePosition = 0
}
rawData = []byte(strings.Join(lines[linePosition:], "\n"))
}
}
} else {
// Read the entire file
rawData = make([]byte, fileInfo.Size())
_, err = fileHandle.Read(rawData)
}
for {
// Read a single line
line, err := reader.ReadBytes('\n')
if err != nil && err != io.EOF {
// Then we could not read the file
return nil, false, 0, 1, err
// We hit an error trying to read the file
return nil, err
}
if maxBytes > 0 && bytesRead+int64(len(line)) > maxBytes {
// If we save this line, then we will have read too many bytes.
// Truncate the line
remainingBytes := maxBytes - bytesRead
line = line[:remainingBytes]
}
lines = append(lines, string(line))
bytesRead += int64(len(line))
// If this is the end of the file, then we are done reading the file
if err == io.EOF {
break
}
// If we have read the maximum number of bytes we are allowed to read, we are done reading the file
if maxBytes > 0 && bytesRead >= maxBytes {
break
}
}
// If maxLines is negative, slice the last maxLines * -1 lines
if maxLines < 0 {
// Determine where in the line buffer we should be for negative lines
startIndex := int64(len(lines)) + maxLines
if startIndex < 0 {
// Make sure we do not go out of bounds
startIndex = 0
}
lines = lines[startIndex:]
} else if maxLines > 0 && int64(len(lines)) > maxLines {
lines = lines[:maxLines]
}
// Join the lines
combinedFileData := strings.Join(lines, "")
return []byte(combinedFileData), nil
}
func readMultipleFiles(path string, filter string, recurse bool) *sliverpb.Download {
var downloadData bytes.Buffer
var downloadResponse *sliverpb.Download = &sliverpb.Download{
Path: path + filter,
Exists: true,
IsDir: true,
ReadFiles: 0,
UnreadableFiles: 1,
}
readFiles, unreadableFiles, err := compressDir(path, filter, recurse, &downloadData)
// {{if .Config.Debug}}
log.Printf("error creating the archive: %v", err)
// {{end}}
downloadResponse.ReadFiles = int32(readFiles)
downloadResponse.UnreadableFiles = int32(unreadableFiles)
if err != nil {
downloadResponse.Response = &commonpb.Response{
Err: fmt.Sprintf("%v", err),
}
return downloadResponse
}
gzipData := bytes.NewBuffer([]byte{})
gzipWrite(gzipData, downloadData.Bytes())
downloadResponse.Data = gzipData.Bytes()
downloadResponse.Encoder = "gzip"
downloadResponse.Response = &commonpb.Response{}
return downloadResponse
}
// func prepareDownload(path string, filter string, recurse bool, maxBytes int64, maxLines int64) ([]byte, bool, int, int, error) {
func prepareDownload(path string, filter string, recurse bool, restrictedToFiles bool, maxBytes int64, maxLines int64) *sliverpb.Download {
var err error
// Default response
var downloadResponse *sliverpb.Download = &sliverpb.Download{
Path: path + filter,
Exists: false,
IsDir: false,
ReadFiles: 0,
UnreadableFiles: 1,
Response: &commonpb.Response{},
}
// Check to see how many files or dirs match path+filter
matches, err := filepath.Glob(path + filter)
if err != nil {
// If we got here, then there is something wrong with the supplied pattern
downloadResponse.Response = &commonpb.Response{
Err: fmt.Sprintf("%v", err),
}
return downloadResponse
}
if len(matches) == 0 {
// Then nothing matches the pattern and there is nothing to download
downloadResponse.Response = &commonpb.Response{
Err: "no files match pattern",
}
return downloadResponse
} else if len(matches) == 1 {
fileInfo, err := os.Stat(matches[0])
if err != nil {
downloadResponse.Response = &commonpb.Response{
Err: fmt.Sprintf("%v", err),
}
return downloadResponse
}
if !fileInfo.IsDir() {
// If we are here, the user requested a single file
fileData, err := readSingleFile(matches[0], maxBytes, maxLines)
//{{if .Config.Debug}}
log.Printf("error while preparing download for %s: %v", matches[0], err)
//{{end}}
if err != nil {
downloadResponse.Response = &commonpb.Response{
Err: fmt.Sprintf("%v", err),
}
return downloadResponse
}
gzipData := bytes.NewBuffer([]byte{})
gzipWrite(gzipData, fileData)
downloadResponse.Path = matches[0]
downloadResponse.Data = gzipData.Bytes()
downloadResponse.Encoder = "gzip"
downloadResponse.Exists = true
downloadResponse.ReadFiles = 1
downloadResponse.UnreadableFiles = 0
return downloadResponse
} else {
return rawData, false, 1, 0, nil
if restrictedToFiles {
downloadResponse.Response = &commonpb.Response{
Err: "multiple files match pattern, command is restricted to one file",
}
return downloadResponse
}
downloadResponse = readMultipleFiles(path, filter, recurse)
return downloadResponse
}
}
// If we are here, then the user wants multiple files (a directory or part of a directory)
var downloadData bytes.Buffer
readFiles, unreadableFiles, err := compressDir(path, filter, recurse, &downloadData)
return downloadData.Bytes(), true, readFiles, unreadableFiles, err
if restrictedToFiles {
downloadResponse.Response = &commonpb.Response{
Err: "multiple files match pattern, command is restricted to one file",
}
return downloadResponse
}
downloadResponse = readMultipleFiles(path, filter, recurse)
return downloadResponse
}
// Send a file back to the hive
func downloadHandler(data []byte, resp RPCResponse) {
var rawData []byte
/*
A flag for whether this is a directory - used if
this download is being looted
*/
var isDir bool
var download *sliverpb.Download
downloadReq := &sliverpb.DownloadReq{}
@@ -585,8 +677,7 @@ func downloadHandler(data []byte, resp RPCResponse) {
target, _ := filepath.Abs(downloadReq.Path)
if pathIsDirectory(target) {
// Even if the implant is running on Windows, Go can deal with "/" as a path separator
target += "/"
target += string(os.PathSeparator)
if downloadReq.RestrictedToFile {
/*
The user has asked to perform a download operation that should only be allowed on
@@ -609,36 +700,7 @@ func downloadHandler(data []byte, resp RPCResponse) {
path, filter := determineDirPathFilter(target)
rawData, isDir, readFiles, unreadableFiles, err := prepareDownload(path, filter, downloadReq.Recurse, downloadReq.MaxBytes, downloadReq.MaxLines)
if err != nil {
if isDir {
// {{if .Config.Debug}}
log.Printf("error creating the archive: %v", err)
// {{end}}
} else {
//{{if .Config.Debug}}
log.Printf("error while preparing download for %s: %v", target, err)
//{{end}}
}
download = &sliverpb.Download{Path: target, Exists: false, ReadFiles: int32(readFiles), UnreadableFiles: int32(unreadableFiles)}
download.Response = &commonpb.Response{
Err: fmt.Sprintf("%v", err),
}
} else {
gzipData := bytes.NewBuffer([]byte{})
gzipWrite(gzipData, rawData)
download = &sliverpb.Download{
Path: target,
Data: gzipData.Bytes(),
Encoder: "gzip",
Exists: true,
IsDir: isDir,
ReadFiles: int32(readFiles),
UnreadableFiles: int32(unreadableFiles),
Response: &commonpb.Response{},
}
}
download = prepareDownload(path, filter, downloadReq.Recurse, downloadReq.RestrictedToFile, downloadReq.MaxBytes, downloadReq.MaxLines)
data, _ = proto.Marshal(download)
resp(data, err)
@@ -238,7 +238,7 @@ func (s *SliverHTTPClient) newHTTPRequest(method string, uri *url.URL, body io.R
Name string
Value string
Probability string
Methods []string
Method string
}
// HTTP C2 Profile headers
@@ -248,20 +248,15 @@ func (s *SliverHTTPClient) newHTTPRequest(method string, uri *url.URL, body io.R
Name: "{{$header.Name}}",
Value: "{{$header.Value}}",
Probability: "{{$header.Probability}}",
Methods: []string{
// {{range $method := $header.Methods}}
"{{$method}}",
// {{end}}
},
Method: "{{$header.Method}}",
},
// {{end}}
}
for _, header := range extraHeaders {
// Empty array means all methods (backwards compatibility)
if len(header.Methods) > 0 {
if !contains(header.Methods, method) {
continue
}
if len(header.Method) > 0 && header.Method != method {
continue
}
// {{if .Config.Debug}}
log.Printf("Rolling to add HTTP header '%s: %s' (%s)", header.Name, header.Value, header.Probability)
@@ -278,11 +273,19 @@ func (s *SliverHTTPClient) newHTTPRequest(method string, uri *url.URL, body io.R
extraURLParams := []nameValueProbability{
// {{range $param := .HTTPC2ImplantConfig.ExtraURLParameters}}
{Name: "{{$param.Name}}", Value: "{{$param.Value}}", Probability: "{{$param.Probability}}"},
{
Name: "{{$param.Name}}",
Value: "{{$param.Value}}",
Probability: "{{$param.Probability}}",
Method: "{{$param.Method}}",
},
// {{end}}
}
queryParams := req.URL.Query()
for _, param := range extraURLParams {
if len(param.Method)>0 && param.Method != method {
continue
}
probability, _ := strconv.Atoi(param.Probability)
if 0 < probability {
roll := insecureRand.Intn(99) + 1
@@ -431,7 +434,7 @@ func (s *SliverHTTPClient) ReadEnvelope() (*pb.Envelope, error) {
s.NonceQueryArgument(uri, nonce)
req := s.newHTTPRequest(http.MethodGet, uri, nil)
// {{if .Config.Debug}}
log.Printf("[http] GET -> %s", uri)
log.Printf("[http] GET -> %s", req.URL)
// {{end}}
resp, rawRespData, err := s.DoPoll(req)
if err != nil {
File diff suppressed because it is too large. Load diff
+1
View File
@@ -187,6 +187,7 @@ message ImplantConfig {
bool NetGoEnabled = 151;
bool TrafficEncodersEnabled = 152;
repeated string TrafficEncoders = 153;
string Extension = 154;
repeated commonpb.File Assets = 200;
}
+58 -149
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.1
// protoc v4.25.3
// protoc-gen-go v1.36.3
// protoc v5.29.3
// source: commonpb/common.proto
package commonpb
@@ -21,18 +21,16 @@ const (
)
type Empty struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
state protoimpl.MessageState `protogen:"open.v1"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *Empty) Reset() {
*x = Empty{}
if protoimpl.UnsafeEnabled {
mi := &file_commonpb_common_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
mi := &file_commonpb_common_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *Empty) String() string {
@@ -43,7 +41,7 @@ func (*Empty) ProtoMessage() {}
func (x *Empty) ProtoReflect() protoreflect.Message {
mi := &file_commonpb_common_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -60,23 +58,20 @@ func (*Empty) Descriptor() ([]byte, []int) {
// Request - Common fields used in all gRPC requests
type Request struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
state protoimpl.MessageState `protogen:"open.v1"`
Async bool `protobuf:"varint,1,opt,name=Async,proto3" json:"Async,omitempty"`
Timeout int64 `protobuf:"varint,2,opt,name=Timeout,proto3" json:"Timeout,omitempty"`
BeaconID string `protobuf:"bytes,8,opt,name=BeaconID,proto3" json:"BeaconID,omitempty"`
SessionID string `protobuf:"bytes,9,opt,name=SessionID,proto3" json:"SessionID,omitempty"`
unknownFields protoimpl.UnknownFields
Async bool `protobuf:"varint,1,opt,name=Async,proto3" json:"Async,omitempty"`
Timeout int64 `protobuf:"varint,2,opt,name=Timeout,proto3" json:"Timeout,omitempty"`
BeaconID string `protobuf:"bytes,8,opt,name=BeaconID,proto3" json:"BeaconID,omitempty"`
SessionID string `protobuf:"bytes,9,opt,name=SessionID,proto3" json:"SessionID,omitempty"`
sizeCache protoimpl.SizeCache
}
func (x *Request) Reset() {
*x = Request{}
if protoimpl.UnsafeEnabled {
mi := &file_commonpb_common_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
mi := &file_commonpb_common_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *Request) String() string {
@@ -87,7 +82,7 @@ func (*Request) ProtoMessage() {}
func (x *Request) ProtoReflect() protoreflect.Message {
mi := &file_commonpb_common_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -135,23 +130,20 @@ func (x *Request) GetSessionID() string {
// only used when the implant needs to return an error to the server.
// Client<->Server comms should use normal gRPC error handling.
type Response struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
state protoimpl.MessageState `protogen:"open.v1"`
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
Async bool `protobuf:"varint,2,opt,name=Async,proto3" json:"Async,omitempty"`
BeaconID string `protobuf:"bytes,8,opt,name=BeaconID,proto3" json:"BeaconID,omitempty"`
TaskID string `protobuf:"bytes,9,opt,name=TaskID,proto3" json:"TaskID,omitempty"`
unknownFields protoimpl.UnknownFields
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
Async bool `protobuf:"varint,2,opt,name=Async,proto3" json:"Async,omitempty"`
BeaconID string `protobuf:"bytes,8,opt,name=BeaconID,proto3" json:"BeaconID,omitempty"`
TaskID string `protobuf:"bytes,9,opt,name=TaskID,proto3" json:"TaskID,omitempty"`
sizeCache protoimpl.SizeCache
}
func (x *Response) Reset() {
*x = Response{}
if protoimpl.UnsafeEnabled {
mi := &file_commonpb_common_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
mi := &file_commonpb_common_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *Response) String() string {
@@ -162,7 +154,7 @@ func (*Response) ProtoMessage() {}
func (x *Response) ProtoReflect() protoreflect.Message {
mi := &file_commonpb_common_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -207,21 +199,18 @@ func (x *Response) GetTaskID() string {
// File - A basic file data type
type File struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"`
unknownFields protoimpl.UnknownFields
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"`
sizeCache protoimpl.SizeCache
}
func (x *File) Reset() {
*x = File{}
if protoimpl.UnsafeEnabled {
mi := &file_commonpb_common_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
mi := &file_commonpb_common_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *File) String() string {
@@ -232,7 +221,7 @@ func (*File) ProtoMessage() {}
func (x *File) ProtoReflect() protoreflect.Message {
mi := &file_commonpb_common_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -263,26 +252,23 @@ func (x *File) GetData() []byte {
// Process - A basic process data type
type Process struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
state protoimpl.MessageState `protogen:"open.v1"`
Pid int32 `protobuf:"varint,1,opt,name=Pid,proto3" json:"Pid,omitempty"`
Ppid int32 `protobuf:"varint,2,opt,name=Ppid,proto3" json:"Ppid,omitempty"`
Executable string `protobuf:"bytes,3,opt,name=Executable,proto3" json:"Executable,omitempty"`
Owner string `protobuf:"bytes,4,opt,name=Owner,proto3" json:"Owner,omitempty"`
Architecture string `protobuf:"bytes,7,opt,name=Architecture,proto3" json:"Architecture,omitempty"`
SessionID int32 `protobuf:"varint,5,opt,name=SessionID,proto3" json:"SessionID,omitempty"`
CmdLine []string `protobuf:"bytes,6,rep,name=CmdLine,proto3" json:"CmdLine,omitempty"`
unknownFields protoimpl.UnknownFields
Pid int32 `protobuf:"varint,1,opt,name=Pid,proto3" json:"Pid,omitempty"`
Ppid int32 `protobuf:"varint,2,opt,name=Ppid,proto3" json:"Ppid,omitempty"`
Executable string `protobuf:"bytes,3,opt,name=Executable,proto3" json:"Executable,omitempty"`
Owner string `protobuf:"bytes,4,opt,name=Owner,proto3" json:"Owner,omitempty"`
Architecture string `protobuf:"bytes,7,opt,name=Architecture,proto3" json:"Architecture,omitempty"`
SessionID int32 `protobuf:"varint,5,opt,name=SessionID,proto3" json:"SessionID,omitempty"`
CmdLine []string `protobuf:"bytes,6,rep,name=CmdLine,proto3" json:"CmdLine,omitempty"`
sizeCache protoimpl.SizeCache
}
func (x *Process) Reset() {
*x = Process{}
if protoimpl.UnsafeEnabled {
mi := &file_commonpb_common_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
mi := &file_commonpb_common_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *Process) String() string {
@@ -293,7 +279,7 @@ func (*Process) ProtoMessage() {}
func (x *Process) ProtoReflect() protoreflect.Message {
mi := &file_commonpb_common_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -359,21 +345,18 @@ func (x *Process) GetCmdLine() []string {
// EnvVar - Environment variable K/V
type EnvVar struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
state protoimpl.MessageState `protogen:"open.v1"`
Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"`
Value string `protobuf:"bytes,2,opt,name=Value,proto3" json:"Value,omitempty"`
unknownFields protoimpl.UnknownFields
Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"`
Value string `protobuf:"bytes,2,opt,name=Value,proto3" json:"Value,omitempty"`
sizeCache protoimpl.SizeCache
}
func (x *EnvVar) Reset() {
*x = EnvVar{}
if protoimpl.UnsafeEnabled {
mi := &file_commonpb_common_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
mi := &file_commonpb_common_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *EnvVar) String() string {
@@ -384,7 +367,7 @@ func (*EnvVar) ProtoMessage() {}
func (x *EnvVar) ProtoReflect() protoreflect.Message {
mi := &file_commonpb_common_proto_msgTypes[5]
if protoimpl.UnsafeEnabled && x != nil {
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -470,7 +453,7 @@ func file_commonpb_common_proto_rawDescGZIP() []byte {
}
var file_commonpb_common_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
var file_commonpb_common_proto_goTypes = []interface{}{
var file_commonpb_common_proto_goTypes = []any{
(*Empty)(nil), // 0: commonpb.Empty
(*Request)(nil), // 1: commonpb.Request
(*Response)(nil), // 2: commonpb.Response
@@ -491,80 +474,6 @@ func file_commonpb_common_proto_init() {
if File_commonpb_common_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_commonpb_common_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Empty); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_commonpb_common_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Request); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_commonpb_common_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Response); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_commonpb_common_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*File); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_commonpb_common_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Process); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_commonpb_common_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*EnvVar); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
+15 -32
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.1
// protoc v4.25.3
// protoc-gen-go v1.36.3
// protoc v5.29.3
// source: dnspb/dns.proto
package dnspb
@@ -92,25 +92,22 @@ func (DNSMessageType) EnumDescriptor() ([]byte, []int) {
//
// [Type TOTP]: ID field is used for the TOTP code
type DNSMessage struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
state protoimpl.MessageState `protogen:"open.v1"`
Type DNSMessageType `protobuf:"varint,1,opt,name=Type,proto3,enum=dnspb.DNSMessageType" json:"Type,omitempty"`
ID uint32 `protobuf:"varint,2,opt,name=ID,proto3" json:"ID,omitempty"` // 8 bit message id + 24 bit dns session ID
Start uint32 `protobuf:"varint,3,opt,name=Start,proto3" json:"Start,omitempty"` // Bytes start at
Stop uint32 `protobuf:"varint,4,opt,name=Stop,proto3" json:"Stop,omitempty"` // Bytes stop at
Size uint32 `protobuf:"varint,5,opt,name=Size,proto3" json:"Size,omitempty"` // Total size
Data []byte `protobuf:"bytes,6,opt,name=Data,proto3" json:"Data,omitempty"` // Actual data
unknownFields protoimpl.UnknownFields
Type DNSMessageType `protobuf:"varint,1,opt,name=Type,proto3,enum=dnspb.DNSMessageType" json:"Type,omitempty"`
ID uint32 `protobuf:"varint,2,opt,name=ID,proto3" json:"ID,omitempty"` // 8 bit message id + 24 bit dns session ID
Start uint32 `protobuf:"varint,3,opt,name=Start,proto3" json:"Start,omitempty"` // Bytes start at
Stop uint32 `protobuf:"varint,4,opt,name=Stop,proto3" json:"Stop,omitempty"` // Bytes stop at
Size uint32 `protobuf:"varint,5,opt,name=Size,proto3" json:"Size,omitempty"` // Total size
Data []byte `protobuf:"bytes,6,opt,name=Data,proto3" json:"Data,omitempty"` // Actual data
sizeCache protoimpl.SizeCache
}
func (x *DNSMessage) Reset() {
*x = DNSMessage{}
if protoimpl.UnsafeEnabled {
mi := &file_dnspb_dns_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
mi := &file_dnspb_dns_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *DNSMessage) String() string {
@@ -121,7 +118,7 @@ func (*DNSMessage) ProtoMessage() {}
func (x *DNSMessage) ProtoReflect() protoreflect.Message {
mi := &file_dnspb_dns_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
@@ -221,7 +218,7 @@ func file_dnspb_dns_proto_rawDescGZIP() []byte {
var file_dnspb_dns_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_dnspb_dns_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_dnspb_dns_proto_goTypes = []interface{}{
var file_dnspb_dns_proto_goTypes = []any{
(DNSMessageType)(0), // 0: dnspb.DNSMessageType
(*DNSMessage)(nil), // 1: dnspb.DNSMessage
}
@@ -239,20 +236,6 @@ func file_dnspb_dns_proto_init() {
if File_dnspb_dns_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_dnspb_dns_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DNSMessage); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
+3 -3
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.1
// protoc v4.25.3
// protoc-gen-go v1.36.3
// protoc v5.29.3
// source: rpcpb/services.proto
package rpcpb
@@ -724,7 +724,7 @@ var file_rpcpb_services_proto_rawDesc = []byte{
0x70, 0x63, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var file_rpcpb_services_proto_goTypes = []interface{}{
var file_rpcpb_services_proto_goTypes = []any{
(*commonpb.Empty)(nil), // 0: commonpb.Empty
(*clientpb.ClientLogData)(nil), // 1: clientpb.ClientLogData
(*sliverpb.KillReq)(nil), // 2: sliverpb.KillReq
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+1 -4
View File
@@ -241,10 +241,7 @@ func (b *Builder) handleBuildEvent(event *clientpb.Event) {
return
}
fileName := filepath.Base(extConfig.Build.Name)
if extConfig.Config.GOOS == "windows" {
fileName += ".exe"
}
fileName := filepath.Base(extConfig.Build.Name) + extConfig.Config.Extension
builderLog.Infof("Uploading '%s' to server ...", extConfig.Build.Name)
_, err = b.rpc.GenerateExternalSaveBuild(context.Background(), &clientpb.ExternalImplantBinary{
+2
View File
@@ -60,6 +60,8 @@ var (
ErrDuplicateStageExt = errors.New("stager extension is already used in another C2 profile")
ErrDuplicateStartSessionExt = errors.New("start session extension is already used in another C2 profile")
ErrDuplicateC2ProfileName = errors.New("C2 Profile name is already in use")
ErrMissingC2ProfileName = errors.New("C2 Profile name is required")
ErrC2ProfileNotFound = errors.New("C2 Profile does not exist")
ErrUserAgentIllegalCharacters = errors.New("user agent cannot contain the ` character")
fileNameExp = regexp.MustCompile(`[^a-zA-Z0-9\\._-]+`)
+34
View File
@@ -470,6 +470,20 @@ func HTTPC2ConfigUpdate(newConf *clientpb.HTTPC2Config, oldConf *clientpb.HTTPC2
return err.Error
}
err = Session().Where(&models.HttpC2Header{
HttpC2ImplantConfigID: &clientID,
}).Delete(&models.HttpC2Header{})
if err.Error != nil {
return err.Error
}
err = Session().Where(&models.HttpC2URLParameter{
HttpC2ImplantConfigID: clientID,
}).Delete(&models.HttpC2URLParameter{})
if err.Error != nil {
return err.Error
}
err = Session().Where(&models.ImplantConfig{
ID: clientID,
}).Updates(c2Config.ImplantConfig)
@@ -487,6 +501,26 @@ func HTTPC2ConfigUpdate(newConf *clientpb.HTTPC2Config, oldConf *clientpb.HTTPC2
}
}
for _, header := range c2Config.ImplantConfig.Headers {
header.HttpC2ImplantConfigID = &clientID
err = Session().Clauses(clause.OnConflict{
UpdateAll: true,
}).Create(&header)
if err.Error != nil {
return err.Error
}
}
for _, urlParameter := range c2Config.ImplantConfig.ExtraURLParameters {
urlParameter.HttpC2ImplantConfigID = clientID
err = Session().Clauses(clause.OnConflict{
UpdateAll: true,
}).Create(&urlParameter)
if err.Error != nil {
return err.Error
}
}
serverID, _ := uuid.FromString(oldConf.ServerConfig.ID)
err = Session().Where(&models.HttpC2Cookie{
+4 -2
View File
@@ -136,8 +136,7 @@ func ImplantBuildFromProtobuf(ib *clientpb.ImplantBuild) *ImplantBuild {
WGImplantPrivKey: ib.WGImplantPrivKey,
WGServerPubKey: ib.WGServerPubKey,
Stage: ib.Stage,
}
Stage: ib.Stage}
return &build
}
@@ -207,6 +206,7 @@ type ImplantConfig struct {
NetGoEnabled bool
TrafficEncodersEnabled bool
Assets []EncoderAsset
Extension string
}
// BeforeCreate - GORM hook
@@ -284,6 +284,7 @@ func (ic *ImplantConfig) ToProtobuf() *clientpb.ImplantConfig {
IncludeNamePipe: ic.IncludeNamePipe,
IncludeWG: ic.IncludeWG,
IncludeTCP: ic.IncludeTCP,
Extension: ic.Extension,
}
if ic.ImplantProfileID != nil {
@@ -505,6 +506,7 @@ func ImplantConfigFromProtobuf(pbConfig *clientpb.ImplantConfig) *ImplantConfig
Name: pbAsset.Name,
})
}
cfg.Extension = pbConfig.Extension
return &cfg
}
+16
View File
@@ -33,6 +33,22 @@ func SliverExternal(name string, config *clientpb.ImplantConfig) (*clientpb.Exte
config.IncludeNamePipe = models.IsC2Enabled([]string{"namedpipe"}, config.C2)
config.IncludeTCP = models.IsC2Enabled([]string{"tcppivot"}, config.C2)
// set file extension for external builds
if config.IsSharedLib {
switch config.GOOS {
case WINDOWS:
config.Extension = ".dll"
case DARWIN:
config.Extension = ".dylib"
case LINUX:
config.Extension = ".so"
}
} else {
if config.GOOS == WINDOWS {
config.Extension = ".exe"
}
}
build, err := GenerateConfig(name, config)
if err != nil {
return nil, err
+7 -5
View File
@@ -59,16 +59,15 @@ func (rpc *Server) SaveHTTPC2Profile(ctx context.Context, req *clientpb.HTTPC2Co
return nil, err
}
profileName := ""
if req.Overwrite {
profileName = req.C2Config.Name
if req.Overwrite && req.C2Config.Name == "" {
return nil, configs.ErrMissingC2ProfileName
}
err = db.SearchStageExtensions(req.C2Config.ImplantConfig.StagerFileExtension, profileName)
err = db.SearchStageExtensions(req.C2Config.ImplantConfig.StagerFileExtension, req.C2Config.Name)
if err != nil {
return nil, err
}
err = db.SearchStartSessionExtensions(req.C2Config.ImplantConfig.StartSessionFileExtension, profileName)
err = db.SearchStartSessionExtensions(req.C2Config.ImplantConfig.StartSessionFileExtension, req.C2Config.Name)
if err != nil {
return nil, err
}
@@ -82,6 +81,9 @@ func (rpc *Server) SaveHTTPC2Profile(ctx context.Context, req *clientpb.HTTPC2Co
}
if req.Overwrite {
if httpC2Config.Name == "" {
return nil, configs.ErrC2ProfileNotFound
}
err = db.HTTPC2ConfigUpdate(req.C2Config, httpC2Config)
if err != nil {
log.Printf("Error:\n%s", err)
+9 -3
View File
@@ -68,7 +68,7 @@ func (rpc *Server) Generate(ctx context.Context, req *clientpb.GenerateReq) (*cl
if err != nil {
return nil, err
}
} else if err := util.AllowedName(name); err != nil {
} else if err := util.AllowedName(req.Name); err != nil {
return nil, err
} else {
name = req.Name
@@ -154,6 +154,11 @@ func (rpc *Server) Regenerate(ctx context.Context, req *clientpb.RegenerateReq)
return nil, status.Error(codes.InvalidArgument, "invalid implant name")
}
config, err := db.ImplantConfigByID(build.ImplantConfigID)
if err != nil {
return nil, err
}
fileData, err := generate.ImplantFileFromBuild(build)
if err != nil {
return nil, err
@@ -161,7 +166,7 @@ func (rpc *Server) Regenerate(ctx context.Context, req *clientpb.RegenerateReq)
return &clientpb.Generate{
File: &commonpb.File{
Name: build.Name,
Name: build.Name + config.Extension,
Data: fileData,
},
}, nil
@@ -340,7 +345,7 @@ func (rpc *Server) GenerateExternal(ctx context.Context, req *clientpb.ExternalG
if err != nil {
return nil, err
}
} else if err := util.AllowedName(name); err != nil {
} else if err := util.AllowedName(req.Name); err != nil {
return nil, err
} else {
name = req.Name
@@ -384,6 +389,7 @@ func (rpc *Server) GenerateExternalSaveBuild(ctx context.Context, req *clientpb.
rcpGenLog.Errorf("Failed to write implant binary to temp file: %s", err)
return nil, status.Error(codes.Internal, "Failed to write implant binary to temp file")
}
rpcLog.Infof("Saving external build '%s' from %s", req.Name, tmpFile.Name())
err = generate.ImplantBuildSave(implantBuild, implantConfig, tmpFile.Name())
if err != nil {
+7
View File
@@ -1,3 +1,10 @@
# 5.5.5 (March 9, 2024)
Use spaces instead of parentheses for SQL sanitization.
This still solves the problem of negative numbers creating a line comment, but this avoids breaking edge cases such as
`set foo to $1` where the substitution is taking place in a location where an arbitrary expression is not allowed.
# 5.5.4 (March 4, 2024)
Fix CVE-2024-27304
+1 -1
View File
@@ -66,7 +66,7 @@ func (q *Query) Sanitize(args ...any) (string, error) {
// Prevent SQL injection via Line Comment Creation
// https://github.com/jackc/pgx/security/advisories/GHSA-m7wr-2xf7-cm9p
str = "(" + str + ")"
str = " " + str + " "
default:
return "", fmt.Errorf("invalid Part type: %T", part)
}
+36 -38
View File
@@ -1,4 +1,4 @@
# Go bindings to SQLite using Wazero
# Go bindings to SQLite using wazero
[![Go Reference](https://pkg.go.dev/badge/image)](https://pkg.go.dev/github.com/ncruces/go-sqlite3)
[![Go Report](https://goreportcard.com/badge/github.com/ncruces/go-sqlite3)](https://goreportcard.com/report/github.com/ncruces/go-sqlite3)
@@ -10,7 +10,21 @@ as well as direct access to most of the [C SQLite API](https://sqlite.org/cintro
It wraps a [Wasm](https://webassembly.org/) [build](embed/) of SQLite,
and uses [wazero](https://wazero.io/) as the runtime.\
Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies [^1].
Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies.
### Getting started
Using the [`database/sql`](https://pkg.go.dev/database/sql) driver:
```go
import "database/sql"
import _ "github.com/ncruces/go-sqlite3/driver"
import _ "github.com/ncruces/go-sqlite3/embed"
var version string
db, _ := sql.Open("sqlite3", "file:demo.db")
db.QueryRow(`SELECT sqlite_version()`).Scan(&version)
```
### Packages
@@ -27,37 +41,6 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run
- [`github.com/ncruces/go-sqlite3/gormlite`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/gormlite)
provides a [GORM](https://gorm.io) driver.
### Extensions
- [`github.com/ncruces/go-sqlite3/ext/array`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/array)
provides the [`array`](https://sqlite.org/carray.html) table-valued function.
- [`github.com/ncruces/go-sqlite3/ext/blobio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/blobio)
simplifies [incremental BLOB I/O](https://sqlite.org/c3ref/blob_open.html).
- [`github.com/ncruces/go-sqlite3/ext/csv`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/csv)
reads [comma-separated values](https://sqlite.org/csv.html).
- [`github.com/ncruces/go-sqlite3/ext/fileio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/fileio)
reads, writes and lists files.
- [`github.com/ncruces/go-sqlite3/ext/hash`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/hash)
provides cryptographic hash functions.
- [`github.com/ncruces/go-sqlite3/ext/lines`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/lines)
reads data [line-by-line](https://github.com/asg017/sqlite-lines).
- [`github.com/ncruces/go-sqlite3/ext/pivot`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/pivot)
creates [pivot tables](https://github.com/jakethaw/pivot_vtab).
- [`github.com/ncruces/go-sqlite3/ext/statement`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/statement)
creates [parameterized views](https://github.com/0x09/sqlite-statement-vtab).
- [`github.com/ncruces/go-sqlite3/ext/stats`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/stats)
provides [statistics](https://www.oreilly.com/library/view/sql-in-a/9780596155322/ch04s02.html) functions.
- [`github.com/ncruces/go-sqlite3/ext/unicode`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/unicode)
provides [Unicode aware](https://sqlite.org/src/dir/ext/icu) functions.
- [`github.com/ncruces/go-sqlite3/ext/zorder`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/zorder)
maps multidimensional data to one dimension.
- [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb)
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
### Advanced features
- [incremental BLOB I/O](https://sqlite.org/c3ref/blob_open.html)
@@ -70,7 +53,11 @@ Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ run
- [math functions](https://sqlite.org/lang_mathfunc.html)
- [full-text search](https://sqlite.org/fts5.html)
- [geospatial search](https://sqlite.org/geopoly.html)
- [Unicode support](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/unicode)
- [statistics functions](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/stats)
- [encryption at rest](vfs/adiantum/README.md)
- [many extensions](ext/README.md)
- [custom VFSes](vfs/README.md#custom-vfses)
- [and more…](embed/README.md)
### Caveats
@@ -87,8 +74,11 @@ This project aims for [high test coverage](https://github.com/ncruces/go-sqlite3
It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and
[wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing.
Every commit is [tested](.github/workflows/test.yml) on
Linux (amd64/arm64/386/riscv64), macOS (amd64/arm64), Windows, FreeBSD and illumos.
Every commit is [tested](https://github.com/ncruces/go-sqlite3/wiki/Test-matrix) on
Linux (amd64/arm64/386/riscv64/ppc64le/s390x), macOS (amd64/arm64),
Windows (amd64), FreeBSD (amd64), OpenBSD (amd64), NetBSD (amd64),
DragonFly BSD (amd64), illumos (amd64), and Solaris (amd64).
The Go VFS is tested by running SQLite's
[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c).
@@ -100,12 +90,20 @@ Perfomance of the [`database/sql`](https://pkg.go.dev/database/sql) driver is
The Wasm and VFS layers are also tested by running SQLite's
[speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c).
### FAQ, issues, new features
For questions, please see [Discussions](https://github.com/ncruces/go-sqlite3/discussions/categories/q-a).
Also, post there if you used this driver for something interesting
([_"Show and tell"_](https://github.com/ncruces/go-sqlite3/discussions/categories/show-and-tell)),
have an [idea](https://github.com/ncruces/go-sqlite3/discussions/categories/ideas)…
The [Issue](https://github.com/ncruces/go-sqlite3/issues) tracker is for bugs we want fixed,
and features we're working on, planning to work on, or asking for help with.
### Alternatives
- [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite)
- [`crawshaw.io/sqlite`](https://pkg.go.dev/crawshaw.io/sqlite)
- [`github.com/mattn/go-sqlite3`](https://pkg.go.dev/github.com/mattn/go-sqlite3)
- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite)
[^1]: anything else you find in `go.mod` is either a test dependency,
or needed by one of the extensions.
+32 -21
View File
@@ -21,6 +21,8 @@ type Blob struct {
bytes int64
offset int64
handle uint32
bufptr uint32
buflen int64
}
var _ io.ReadWriteSeeker = &Blob{}
@@ -29,7 +31,6 @@ var _ io.ReadWriteSeeker = &Blob{}
//
// https://sqlite.org/c3ref/blob_open.html
func (c *Conn) OpenBlob(db, table, column string, row int64, write bool) (*Blob, error) {
c.checkInterrupt()
defer c.arena.mark()()
blobPtr := c.arena.new(ptrlen)
dbPtr := c.arena.string(db)
@@ -41,6 +42,7 @@ func (c *Conn) OpenBlob(db, table, column string, row int64, write bool) (*Blob,
flags = 1
}
c.checkInterrupt(c.handle)
r := c.call("sqlite3_blob_open", uint64(c.handle),
uint64(dbPtr), uint64(tablePtr), uint64(columnPtr),
uint64(row), flags, uint64(blobPtr))
@@ -66,7 +68,7 @@ func (b *Blob) Close() error {
}
r := b.c.call("sqlite3_blob_close", uint64(b.handle))
b.c.free(b.bufptr)
b.handle = 0
return b.c.error(r)
}
@@ -86,17 +88,18 @@ func (b *Blob) Read(p []byte) (n int, err error) {
return 0, io.EOF
}
avail := b.bytes - b.offset
want := int64(len(p))
avail := b.bytes - b.offset
if want > avail {
want = avail
}
defer b.c.arena.mark()()
ptr := b.c.arena.new(uint64(want))
if want > b.buflen {
b.bufptr = b.c.realloc(b.bufptr, uint64(want))
b.buflen = want
}
r := b.c.call("sqlite3_blob_read", uint64(b.handle),
uint64(ptr), uint64(want), uint64(b.offset))
uint64(b.bufptr), uint64(want), uint64(b.offset))
err = b.c.error(r)
if err != nil {
return 0, err
@@ -106,7 +109,7 @@ func (b *Blob) Read(p []byte) (n int, err error) {
err = io.EOF
}
copy(p, util.View(b.c.mod, ptr, uint64(want)))
copy(p, util.View(b.c.mod, b.bufptr, uint64(want)))
return int(want), err
}
@@ -123,19 +126,20 @@ func (b *Blob) WriteTo(w io.Writer) (n int64, err error) {
if want > avail {
want = avail
}
defer b.c.arena.mark()()
ptr := b.c.arena.new(uint64(want))
if want > b.buflen {
b.bufptr = b.c.realloc(b.bufptr, uint64(want))
b.buflen = want
}
for want > 0 {
r := b.c.call("sqlite3_blob_read", uint64(b.handle),
uint64(ptr), uint64(want), uint64(b.offset))
uint64(b.bufptr), uint64(want), uint64(b.offset))
err = b.c.error(r)
if err != nil {
return n, err
}
mem := util.View(b.c.mod, ptr, uint64(want))
mem := util.View(b.c.mod, b.bufptr, uint64(want))
m, err := w.Write(mem[:want])
b.offset += int64(m)
n += int64(m)
@@ -143,6 +147,7 @@ func (b *Blob) WriteTo(w io.Writer) (n int64, err error) {
return n, err
}
if int64(m) != want {
// notest // Write misbehaving
return n, io.ErrShortWrite
}
@@ -158,11 +163,15 @@ func (b *Blob) WriteTo(w io.Writer) (n int64, err error) {
//
// https://sqlite.org/c3ref/blob_write.html
func (b *Blob) Write(p []byte) (n int, err error) {
defer b.c.arena.mark()()
ptr := b.c.arena.bytes(p)
want := int64(len(p))
if want > b.buflen {
b.bufptr = b.c.realloc(b.bufptr, uint64(want))
b.buflen = want
}
util.WriteBytes(b.c.mod, b.bufptr, p)
r := b.c.call("sqlite3_blob_write", uint64(b.handle),
uint64(ptr), uint64(len(p)), uint64(b.offset))
uint64(b.bufptr), uint64(want), uint64(b.offset))
err = b.c.error(r)
if err != nil {
return 0, err
@@ -186,16 +195,17 @@ func (b *Blob) ReadFrom(r io.Reader) (n int64, err error) {
if want < 1 {
want = 1
}
defer b.c.arena.mark()()
ptr := b.c.arena.new(uint64(want))
if want > b.buflen {
b.bufptr = b.c.realloc(b.bufptr, uint64(want))
b.buflen = want
}
for {
mem := util.View(b.c.mod, ptr, uint64(want))
mem := util.View(b.c.mod, b.bufptr, uint64(want))
m, err := r.Read(mem[:want])
if m > 0 {
r := b.c.call("sqlite3_blob_write", uint64(b.handle),
uint64(ptr), uint64(m), uint64(b.offset))
uint64(b.bufptr), uint64(m), uint64(b.offset))
err := b.c.error(r)
if err != nil {
return n, err
@@ -243,6 +253,7 @@ func (b *Blob) Seek(offset int64, whence int) (int64, error) {
//
// https://sqlite.org/c3ref/blob_reopen.html
func (b *Blob) Reopen(row int64) error {
b.c.checkInterrupt(b.c.handle)
err := b.c.error(b.c.call("sqlite3_blob_reopen", uint64(b.handle), uint64(row)))
b.bytes = int64(b.c.call("sqlite3_blob_bytes", uint64(b.handle)))
b.offset = 0
+234 -15
View File
@@ -2,9 +2,13 @@ package sqlite3
import (
"context"
"fmt"
"strconv"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/vfs"
)
// Config makes configuration changes to a database connection.
@@ -14,8 +18,19 @@ import (
//
// https://sqlite.org/c3ref/db_config.html
func (c *Conn) Config(op DBConfig, arg ...bool) (bool, error) {
if op < DBCONFIG_ENABLE_FKEY || op > DBCONFIG_REVERSE_SCANORDER {
return false, MISUSE
}
// We need to call sqlite3_db_config, a variadic function.
// We only support the `int int*` variants.
// The int is a three-valued bool: -1 queries, 0/1 sets false/true.
// The int* points to where new state will be written to.
// The vararg is a pointer to an array containing these arguments:
// an int and an int* pointing to that int.
defer c.arena.mark()()
argsPtr := c.arena.new(2 * ptrlen)
argsPtr := c.arena.new(intlen + ptrlen)
var flag int
switch {
@@ -56,6 +71,112 @@ func logCallback(ctx context.Context, mod api.Module, _, iCode, zMsg uint32) {
}
}
// Log writes a message into the error log established by [Conn.ConfigLog].
//
// https://sqlite.org/c3ref/log.html
func (c *Conn) Log(code ExtendedErrorCode, format string, a ...any) {
if c.log != nil {
c.log(code, fmt.Sprintf(format, a...))
}
}
// FileControl allows low-level control of database files.
// Only a subset of opcodes are supported.
//
// https://sqlite.org/c3ref/file_control.html
func (c *Conn) FileControl(schema string, op FcntlOpcode, arg ...any) (any, error) {
defer c.arena.mark()()
ptr := c.arena.new(max(ptrlen, intlen))
var schemaPtr uint32
if schema != "" {
schemaPtr = c.arena.string(schema)
}
var rc uint64
var res any
switch op {
default:
return nil, MISUSE
case FCNTL_RESET_CACHE:
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), 0)
case FCNTL_PERSIST_WAL, FCNTL_POWERSAFE_OVERWRITE:
var flag int
switch {
case len(arg) == 0:
flag = -1
case arg[0]:
flag = 1
}
util.WriteUint32(c.mod, ptr, uint32(flag))
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
res = util.ReadUint32(c.mod, ptr) != 0
case FCNTL_CHUNK_SIZE:
util.WriteUint32(c.mod, ptr, uint32(arg[0].(int)))
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
case FCNTL_RESERVE_BYTES:
bytes := -1
if len(arg) > 0 {
bytes = arg[0].(int)
}
util.WriteUint32(c.mod, ptr, uint32(bytes))
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
res = int(util.ReadUint32(c.mod, ptr))
case FCNTL_DATA_VERSION:
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
res = util.ReadUint32(c.mod, ptr)
case FCNTL_LOCKSTATE:
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
res = vfs.LockLevel(util.ReadUint32(c.mod, ptr))
case FCNTL_VFS_POINTER:
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
if rc == _OK {
const zNameOffset = 16
ptr = util.ReadUint32(c.mod, ptr)
ptr = util.ReadUint32(c.mod, ptr+zNameOffset)
name := util.ReadString(c.mod, ptr, _MAX_NAME)
res = vfs.Find(name)
}
case FCNTL_FILE_POINTER, FCNTL_JOURNAL_POINTER:
rc = c.call("sqlite3_file_control",
uint64(c.handle), uint64(schemaPtr),
uint64(op), uint64(ptr))
if rc == _OK {
const fileHandleOffset = 4
ptr = util.ReadUint32(c.mod, ptr)
ptr = util.ReadUint32(c.mod, ptr+fileHandleOffset)
res = util.GetHandle(c.ctx, ptr)
}
}
if err := c.error(rc); err != nil {
return nil, err
}
return res, nil
}
// Limit allows the size of various constructs to be
// limited on a connection by connection basis.
//
@@ -68,7 +189,7 @@ func (c *Conn) Limit(id LimitCategory, value int) int {
// SetAuthorizer registers an authorizer callback with the database connection.
//
// https://sqlite.org/c3ref/set_authorizer.html
func (c *Conn) SetAuthorizer(cb func(action AuthorizerActionCode, name3rd, name4th, schema, nameInner string) AuthorizerReturnCode) error {
func (c *Conn) SetAuthorizer(cb func(action AuthorizerActionCode, name3rd, name4th, schema, inner string) AuthorizerReturnCode) error {
var enable uint64
if cb != nil {
enable = 1
@@ -82,9 +203,9 @@ func (c *Conn) SetAuthorizer(cb func(action AuthorizerActionCode, name3rd, name4
}
func authorizerCallback(ctx context.Context, mod api.Module, pDB uint32, action AuthorizerActionCode, zName3rd, zName4th, zSchema, zNameInner uint32) (rc AuthorizerReturnCode) {
func authorizerCallback(ctx context.Context, mod api.Module, pDB uint32, action AuthorizerActionCode, zName3rd, zName4th, zSchema, zInner uint32) (rc AuthorizerReturnCode) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.authorizer != nil {
var name3rd, name4th, schema, nameInner string
var name3rd, name4th, schema, inner string
if zName3rd != 0 {
name3rd = util.ReadString(mod, zName3rd, _MAX_NAME)
}
@@ -94,18 +215,56 @@ func authorizerCallback(ctx context.Context, mod api.Module, pDB uint32, action
if zSchema != 0 {
schema = util.ReadString(mod, zSchema, _MAX_NAME)
}
if zNameInner != 0 {
nameInner = util.ReadString(mod, zNameInner, _MAX_NAME)
if zInner != 0 {
inner = util.ReadString(mod, zInner, _MAX_NAME)
}
rc = c.authorizer(action, name3rd, name4th, schema, nameInner)
rc = c.authorizer(action, name3rd, name4th, schema, inner)
}
return rc
}
// WalCheckpoint checkpoints a WAL database.
// Trace registers a trace callback function against the database connection.
//
// https://sqlite.org/c3ref/trace_v2.html
func (c *Conn) Trace(mask TraceEvent, cb func(evt TraceEvent, arg1 any, arg2 any) error) error {
r := c.call("sqlite3_trace_go", uint64(c.handle), uint64(mask))
if err := c.error(r); err != nil {
return err
}
c.trace = cb
return nil
}
func traceCallback(ctx context.Context, mod api.Module, evt TraceEvent, pDB, pArg1, pArg2 uint32) (rc uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.trace != nil {
var arg1, arg2 any
if evt == TRACE_CLOSE {
arg1 = c
} else {
for _, s := range c.stmts {
if pArg1 == s.handle {
arg1 = s
switch evt {
case TRACE_STMT:
arg2 = s.SQL()
case TRACE_PROFILE:
arg2 = int64(util.ReadUint64(mod, pArg2))
}
break
}
}
}
if arg1 != nil {
_, rc = errorCode(c.trace(evt, arg1, arg2), ERROR)
}
}
return rc
}
// WALCheckpoint checkpoints a WAL database.
//
// https://sqlite.org/c3ref/wal_checkpoint_v2.html
func (c *Conn) WalCheckpoint(schema string, mode CheckpointMode) (nLog, nCkpt int, err error) {
func (c *Conn) WALCheckpoint(schema string, mode CheckpointMode) (nLog, nCkpt int, err error) {
defer c.arena.mark()()
nLogPtr := c.arena.new(ptrlen)
nCkptPtr := c.arena.new(ptrlen)
@@ -118,19 +277,19 @@ func (c *Conn) WalCheckpoint(schema string, mode CheckpointMode) (nLog, nCkpt in
return nLog, nCkpt, c.error(r)
}
// WalAutoCheckpoint configures WAL auto-checkpoints.
// WALAutoCheckpoint configures WAL auto-checkpoints.
//
// https://sqlite.org/c3ref/wal_autocheckpoint.html
func (c *Conn) WalAutoCheckpoint(pages int) error {
func (c *Conn) WALAutoCheckpoint(pages int) error {
r := c.call("sqlite3_wal_autocheckpoint", uint64(c.handle), uint64(pages))
return c.error(r)
}
// WalHook registers a callback function to be invoked
// WALHook registers a callback function to be invoked
// each time data is committed to a database in WAL mode.
//
// https://sqlite.org/c3ref/wal_hook.html
func (c *Conn) WalHook(cb func(db *Conn, schema string, pages int) error) {
func (c *Conn) WALHook(cb func(db *Conn, schema string, pages int) error) {
var enable uint64
if cb != nil {
enable = 1
@@ -152,7 +311,10 @@ func walCallback(ctx context.Context, mod api.Module, _, pDB, zSchema uint32, pa
//
// https://sqlite.org/c3ref/autovacuum_pages.html
func (c *Conn) AutoVacuumPages(cb func(schema string, dbPages, freePages, bytesPerPage uint) uint) error {
funcPtr := util.AddHandle(c.ctx, cb)
var funcPtr uint32
if cb != nil {
funcPtr = util.AddHandle(c.ctx, cb)
}
r := c.call("sqlite3_autovacuum_pages_go", uint64(c.handle), uint64(funcPtr))
return c.error(r)
}
@@ -162,3 +324,60 @@ func autoVacuumCallback(ctx context.Context, mod api.Module, pApp, zSchema, nDbP
schema := util.ReadString(mod, zSchema, _MAX_NAME)
return uint32(fn(schema, uint(nDbPage), uint(nFreePage), uint(nBytePerPage)))
}
// SoftHeapLimit imposes a soft limit on heap size.
//
// https://sqlite.org/c3ref/hard_heap_limit64.html
func (c *Conn) SoftHeapLimit(n int64) int64 {
return int64(c.call("sqlite3_soft_heap_limit64", uint64(n)))
}
// HardHeapLimit imposes a hard limit on heap size.
//
// https://sqlite.org/c3ref/hard_heap_limit64.html
func (c *Conn) HardHeapLimit(n int64) int64 {
return int64(c.call("sqlite3_hard_heap_limit64", uint64(n)))
}
// EnableChecksums enables checksums on a database.
//
// https://sqlite.org/cksumvfs.html
func (c *Conn) EnableChecksums(schema string) error {
r, err := c.FileControl(schema, FCNTL_RESERVE_BYTES)
if err != nil {
return err
}
if r == 8 {
// Correct value, enabled.
return nil
}
if r == 0 {
// Default value, enable.
_, err = c.FileControl(schema, FCNTL_RESERVE_BYTES, 8)
if err != nil {
return err
}
r, err = c.FileControl(schema, FCNTL_RESERVE_BYTES)
if err != nil {
return err
}
}
if r != 8 {
// Invalid value.
return util.ErrorString("sqlite3: reserve bytes must be 8, is: " + strconv.Itoa(r.(int)))
}
// VACUUM the database.
if schema != "" {
err = c.Exec(`VACUUM ` + QuoteIdentifier(schema))
} else {
err = c.Exec(`VACUUM`)
}
if err != nil {
return err
}
// Checkpoint the WAL.
_, _, err = c.WALCheckpoint(schema, CHECKPOINT_RESTART)
return err
}
+166 -45
View File
@@ -8,9 +8,10 @@ import (
"strings"
"time"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/tetratelabs/wazero/api"
)
// Conn is a database connection handle.
@@ -22,27 +23,36 @@ type Conn struct {
interrupt context.Context
pending *Stmt
busy func(int) bool
stmts []*Stmt
timer *time.Timer
busy func(context.Context, int) bool
log func(xErrorCode, string)
collation func(*Conn, string)
wal func(*Conn, string, int) error
trace func(TraceEvent, any, any) error
authorizer func(AuthorizerActionCode, string, string, string, string) AuthorizerReturnCode
update func(AuthorizerActionCode, string, string, int64)
commit func() bool
rollback func()
wal func(*Conn, string, int) error
arena arena
handle uint32
}
// Open calls [OpenFlags] with [OPEN_READWRITE], [OPEN_CREATE], [OPEN_URI] and [OPEN_NOFOLLOW].
// Open calls [OpenFlags] with [OPEN_READWRITE], [OPEN_CREATE] and [OPEN_URI].
func Open(filename string) (*Conn, error) {
return newConn(filename, OPEN_READWRITE|OPEN_CREATE|OPEN_URI|OPEN_NOFOLLOW)
return newConn(context.Background(), filename, OPEN_READWRITE|OPEN_CREATE|OPEN_URI)
}
// OpenContext is like [Open] but includes a context,
// which is used to interrupt the process of opening the connectiton.
func OpenContext(ctx context.Context, filename string) (*Conn, error) {
return newConn(ctx, filename, OPEN_READWRITE|OPEN_CREATE|OPEN_URI)
}
// OpenFlags opens an SQLite database file as specified by the filename argument.
//
// If none of the required flags is used, a combination of [OPEN_READWRITE] and [OPEN_CREATE] is used.
// If none of the required flags are used, a combination of [OPEN_READWRITE] and [OPEN_CREATE] is used.
// If a URI filename is used, PRAGMA statements to execute can be specified using "_pragma":
//
// sqlite3.Open("file:demo.db?_pragma=busy_timeout(10000)")
@@ -52,26 +62,37 @@ func OpenFlags(filename string, flags OpenFlag) (*Conn, error) {
if flags&(OPEN_READONLY|OPEN_READWRITE|OPEN_CREATE) == 0 {
flags |= OPEN_READWRITE | OPEN_CREATE
}
return newConn(filename, flags)
return newConn(context.Background(), filename, flags)
}
type connKey struct{}
func newConn(filename string, flags OpenFlag) (conn *Conn, err error) {
sqlite, err := instantiateSQLite()
func newConn(ctx context.Context, filename string, flags OpenFlag) (res *Conn, _ error) {
err := ctx.Err()
if err != nil {
return nil, err
}
c := &Conn{interrupt: ctx}
c.sqlite, err = instantiateSQLite()
if err != nil {
return nil, err
}
defer func() {
if conn == nil {
sqlite.close()
if res == nil {
c.Close()
c.sqlite.close()
} else {
c.interrupt = context.Background()
}
}()
c := &Conn{sqlite: sqlite}
c.arena = c.newArena(1024)
c.ctx = context.WithValue(c.ctx, connKey{}, c)
c.arena = c.newArena(1024)
c.handle, err = c.openDB(filename, flags)
if err == nil {
err = initExtensions(c)
}
if err != nil {
return nil, err
}
@@ -92,6 +113,7 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (uint32, error) {
return 0, err
}
c.call("sqlite3_progress_handler_go", uint64(handle), 100)
if flags|OPEN_URI != 0 && strings.HasPrefix(filename, "file:") {
var pragmas strings.Builder
if _, after, ok := strings.Cut(filename, "?"); ok {
@@ -103,6 +125,7 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (uint32, error) {
}
}
if pragmas.Len() != 0 {
c.checkInterrupt(handle)
pragmaPtr := c.arena.string(pragmas.String())
r := c.call("sqlite3_exec", uint64(handle), uint64(pragmaPtr), 0, 0, 0)
if err := c.sqlite.error(r, handle, pragmas.String()); err != nil {
@@ -112,7 +135,6 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (uint32, error) {
}
}
}
c.call("sqlite3_progress_handler_go", uint64(handle), 100)
return handle, nil
}
@@ -154,10 +176,10 @@ func (c *Conn) Close() error {
//
// https://sqlite.org/c3ref/exec.html
func (c *Conn) Exec(sql string) error {
c.checkInterrupt()
defer c.arena.mark()()
sqlPtr := c.arena.string(sql)
c.checkInterrupt(c.handle)
r := c.call("sqlite3_exec", uint64(c.handle), uint64(sqlPtr), 0, 0, 0)
return c.error(r, sql)
}
@@ -183,6 +205,7 @@ func (c *Conn) PrepareFlags(sql string, flags PrepareFlag) (stmt *Stmt, tail str
tailPtr := c.arena.new(ptrlen)
sqlPtr := c.arena.string(sql)
c.checkInterrupt(c.handle)
r := c.call("sqlite3_prepare_v3", uint64(c.handle),
uint64(sqlPtr), uint64(len(sql)+1), uint64(flags),
uint64(stmtPtr), uint64(tailPtr))
@@ -199,6 +222,7 @@ func (c *Conn) PrepareFlags(sql string, flags PrepareFlag) (stmt *Stmt, tail str
if stmt.handle == 0 {
return nil, "", nil
}
c.stmts = append(c.stmts, stmt)
return stmt, tail, nil
}
@@ -224,9 +248,8 @@ func (c *Conn) Filename(schema string) *vfs.Filename {
defer c.arena.mark()()
ptr = c.arena.string(schema)
}
r := c.call("sqlite3_db_filename", uint64(c.handle), uint64(ptr))
return vfs.OpenFilename(c.ctx, c.mod, uint32(r), vfs.OPEN_MAIN_DB)
return vfs.GetFilename(c.ctx, c.mod, uint32(r), vfs.OPEN_MAIN_DB)
}
// ReadOnly determines if a database is read-only.
@@ -295,8 +318,7 @@ func (c *Conn) ReleaseMemory() error {
return c.error(r)
}
// GetInterrupt gets the context set with [Conn.SetInterrupt],
// or nil if none was set.
// GetInterrupt gets the context set with [Conn.SetInterrupt].
func (c *Conn) GetInterrupt() context.Context {
return c.interrupt
}
@@ -316,40 +338,43 @@ func (c *Conn) GetInterrupt() context.Context {
//
// https://sqlite.org/c3ref/interrupt.html
func (c *Conn) SetInterrupt(ctx context.Context) (old context.Context) {
// Is it the same context?
if ctx == c.interrupt {
return ctx
old = c.interrupt
c.interrupt = ctx
if ctx == old || ctx.Done() == old.Done() {
return old
}
// A busy SQL statement prevents SQLite from ignoring an interrupt
// that comes before any other statements are started.
if c.pending == nil {
c.pending, _, _ = c.Prepare(`WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x FROM c) SELECT x FROM c`)
defer c.arena.mark()()
stmtPtr := c.arena.new(ptrlen)
loopPtr := c.arena.string(`WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x FROM c) SELECT x FROM c`)
c.call("sqlite3_prepare_v3", uint64(c.handle), uint64(loopPtr), math.MaxUint64,
uint64(PREPARE_PERSISTENT), uint64(stmtPtr), 0)
c.pending = &Stmt{c: c}
c.pending.handle = util.ReadUint32(c.mod, stmtPtr)
}
old = c.interrupt
c.interrupt = ctx
if old != nil && old.Done() != nil && (ctx == nil || ctx.Err() == nil) {
if old.Done() != nil && ctx.Err() == nil {
c.pending.Reset()
}
if ctx != nil && ctx.Done() != nil {
if ctx.Done() != nil {
c.pending.Step()
}
return old
}
func (c *Conn) checkInterrupt() {
if c.interrupt != nil && c.interrupt.Err() != nil {
c.call("sqlite3_interrupt", uint64(c.handle))
func (c *Conn) checkInterrupt(handle uint32) {
if c.interrupt.Err() != nil {
c.call("sqlite3_interrupt", uint64(handle))
}
}
func progressCallback(ctx context.Context, mod api.Module, pDB uint32) (interrupt uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.interrupt != nil {
if c.interrupt.Err() != nil {
interrupt = 1
}
func progressCallback(ctx context.Context, mod api.Module, _ uint32) (interrupt uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.interrupt.Err() != nil {
interrupt = 1
}
return interrupt
}
@@ -363,10 +388,47 @@ func (c *Conn) BusyTimeout(timeout time.Duration) error {
return c.error(r)
}
func timeoutCallback(ctx context.Context, mod api.Module, count, tmout int32) (retry uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.interrupt.Err() == nil {
const delays = "\x01\x02\x05\x0a\x0f\x14\x19\x19\x19\x32\x32\x64"
const totals = "\x00\x01\x03\x08\x12\x21\x35\x4e\x67\x80\xb2\xe4"
const ndelay = int32(len(delays) - 1)
var delay, prior int32
if count <= ndelay {
delay = int32(delays[count])
prior = int32(totals[count])
} else {
delay = int32(delays[ndelay])
prior = int32(totals[ndelay]) + delay*(count-ndelay)
}
if delay = min(delay, tmout-prior); delay > 0 {
delay := time.Duration(delay) * time.Millisecond
if c.interrupt.Done() == nil {
time.Sleep(delay)
return 1
}
if c.timer == nil {
c.timer = time.NewTimer(delay)
} else {
c.timer.Reset(delay)
}
select {
case <-c.interrupt.Done():
c.timer.Stop()
case <-c.timer.C:
return 1
}
}
}
return 0
}
// BusyHandler registers a callback to handle [BUSY] errors.
//
// https://sqlite.org/c3ref/busy_handler.html
func (c *Conn) BusyHandler(cb func(count int) (retry bool)) error {
func (c *Conn) BusyHandler(cb func(ctx context.Context, count int) (retry bool)) error {
var enable uint64
if cb != nil {
enable = 1
@@ -381,22 +443,81 @@ func (c *Conn) BusyHandler(cb func(count int) (retry bool)) error {
func busyCallback(ctx context.Context, mod api.Module, pDB uint32, count int32) (retry uint32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.busy != nil {
if c.busy(int(count)) {
interrupt := c.interrupt
if interrupt == nil {
interrupt = context.Background()
}
if interrupt.Err() == nil && c.busy(interrupt, int(count)) {
retry = 1
}
}
return retry
}
// Status retrieves runtime status information about a database connection.
//
// https://sqlite.org/c3ref/db_status.html
func (c *Conn) Status(op DBStatus, reset bool) (current, highwater int, err error) {
defer c.arena.mark()()
hiPtr := c.arena.new(intlen)
curPtr := c.arena.new(intlen)
var i uint64
if reset {
i = 1
}
r := c.call("sqlite3_db_status", uint64(c.handle),
uint64(op), uint64(curPtr), uint64(hiPtr), i)
if err = c.error(r); err == nil {
current = int(util.ReadUint32(c.mod, curPtr))
highwater = int(util.ReadUint32(c.mod, hiPtr))
}
return
}
// TableColumnMetadata extracts metadata about a column of a table.
//
// https://sqlite.org/c3ref/table_column_metadata.html
func (c *Conn) TableColumnMetadata(schema, table, column string) (declType, collSeq string, notNull, primaryKey, autoInc bool, err error) {
defer c.arena.mark()()
var schemaPtr, columnPtr uint32
declTypePtr := c.arena.new(ptrlen)
collSeqPtr := c.arena.new(ptrlen)
notNullPtr := c.arena.new(ptrlen)
autoIncPtr := c.arena.new(ptrlen)
primaryKeyPtr := c.arena.new(ptrlen)
if schema != "" {
schemaPtr = c.arena.string(schema)
}
tablePtr := c.arena.string(table)
if column != "" {
columnPtr = c.arena.string(column)
}
r := c.call("sqlite3_table_column_metadata", uint64(c.handle),
uint64(schemaPtr), uint64(tablePtr), uint64(columnPtr),
uint64(declTypePtr), uint64(collSeqPtr),
uint64(notNullPtr), uint64(primaryKeyPtr), uint64(autoIncPtr))
if err = c.error(r); err == nil && column != "" {
declType = util.ReadString(c.mod, util.ReadUint32(c.mod, declTypePtr), _MAX_NAME)
collSeq = util.ReadString(c.mod, util.ReadUint32(c.mod, collSeqPtr), _MAX_NAME)
notNull = util.ReadUint32(c.mod, notNullPtr) != 0
autoInc = util.ReadUint32(c.mod, autoIncPtr) != 0
primaryKey = util.ReadUint32(c.mod, primaryKeyPtr) != 0
}
return
}
func (c *Conn) error(rc uint64, sql ...string) error {
return c.sqlite.error(rc, c.handle, sql...)
}
// DriverConn is implemented by the SQLite [database/sql] driver connection.
//
// It can be used to access SQLite features like [online backup].
//
// [online backup]: https://sqlite.org/backup.html
type DriverConn interface {
Raw() *Conn
func (c *Conn) stmtsIter(yield func(*Stmt) bool) {
for _, s := range c.stmts {
if !yield(s) {
break
}
}
}
+11
View File
@@ -0,0 +1,11 @@
//go:build go1.23
package sqlite3
import "iter"
// Stmts returns an iterator for the prepared statements
// associated with the database connection.
//
// https://sqlite.org/c3ref/next_stmt.html
func (c *Conn) Stmts() iter.Seq[*Stmt] { return c.stmtsIter }
+9
View File
@@ -0,0 +1,9 @@
//go:build !go1.23
package sqlite3
// Stmts returns an iterator for the prepared statements
// associated with the database connection.
//
// https://sqlite.org/c3ref/next_stmt.html
func (c *Conn) Stmts() func(func(*Stmt) bool) { return c.stmtsIter }
+65 -14
View File
@@ -7,15 +7,13 @@ const (
_ROW = 100 /* sqlite3_step() has another row ready */
_DONE = 101 /* sqlite3_step() has finished executing */
_UTF8 = 1
_MAX_NAME = 1e6 // Self-imposed limit for most NUL terminated strings.
_MAX_LENGTH = 1e9
_MAX_SQL_LENGTH = 1e9
_MAX_ALLOCATION_SIZE = 0x7ffffeff
_MAX_FUNCTION_ARG = 100
_MAX_NAME = 1e6 // Self-imposed limit for most NUL terminated strings.
_MAX_LENGTH = 1e9
_MAX_SQL_LENGTH = 1e9
_MAX_FUNCTION_ARG = 100
ptrlen = 4
intlen = 4
)
// ErrorCode is a result code that [Error.Code] might return.
@@ -109,7 +107,7 @@ const (
CANTOPEN_ISDIR ExtendedErrorCode = xErrorCode(CANTOPEN) | (2 << 8)
CANTOPEN_FULLPATH ExtendedErrorCode = xErrorCode(CANTOPEN) | (3 << 8)
CANTOPEN_CONVPATH ExtendedErrorCode = xErrorCode(CANTOPEN) | (4 << 8)
CANTOPEN_DIRTYWAL ExtendedErrorCode = xErrorCode(CANTOPEN) | (5 << 8) /* Not Used */
// CANTOPEN_DIRTYWAL ExtendedErrorCode = xErrorCode(CANTOPEN) | (5 << 8) /* Not Used */
CANTOPEN_SYMLINK ExtendedErrorCode = xErrorCode(CANTOPEN) | (6 << 8)
CORRUPT_VTAB ExtendedErrorCode = xErrorCode(CORRUPT) | (1 << 8)
CORRUPT_SEQUENCE ExtendedErrorCode = xErrorCode(CORRUPT) | (2 << 8)
@@ -177,11 +175,12 @@ const (
type FunctionFlag uint32
const (
DETERMINISTIC FunctionFlag = 0x000000800
DIRECTONLY FunctionFlag = 0x000080000
SUBTYPE FunctionFlag = 0x000100000
INNOCUOUS FunctionFlag = 0x000200000
RESULT_SUBTYPE FunctionFlag = 0x001000000
DETERMINISTIC FunctionFlag = 0x000000800
DIRECTONLY FunctionFlag = 0x000080000
INNOCUOUS FunctionFlag = 0x000200000
SELFORDER1 FunctionFlag = 0x002000000
// SUBTYPE FunctionFlag = 0x000100000
// RESULT_SUBTYPE FunctionFlag = 0x001000000
)
// StmtStatus name counter values associated with the [Stmt.Status] method.
@@ -201,6 +200,27 @@ const (
STMTSTATUS_MEMUSED StmtStatus = 99
)
// DBStatus are the available "verbs" that can be passed to the [Conn.Status] method.
//
// https://sqlite.org/c3ref/c_dbstatus_options.html
type DBStatus uint32
const (
DBSTATUS_LOOKASIDE_USED DBStatus = 0
DBSTATUS_CACHE_USED DBStatus = 1
DBSTATUS_SCHEMA_USED DBStatus = 2
DBSTATUS_STMT_USED DBStatus = 3
DBSTATUS_LOOKASIDE_HIT DBStatus = 4
DBSTATUS_LOOKASIDE_MISS_SIZE DBStatus = 5
DBSTATUS_LOOKASIDE_MISS_FULL DBStatus = 6
DBSTATUS_CACHE_HIT DBStatus = 7
DBSTATUS_CACHE_MISS DBStatus = 8
DBSTATUS_CACHE_WRITE DBStatus = 9
DBSTATUS_DEFERRED_FKS DBStatus = 10
DBSTATUS_CACHE_USED_SHARED DBStatus = 11
DBSTATUS_CACHE_SPILL DBStatus = 12
)
// DBConfig are the available database connection configuration options.
//
// https://sqlite.org/c3ref/c_dbconfig_defensive.html
@@ -227,6 +247,25 @@ const (
DBCONFIG_TRUSTED_SCHEMA DBConfig = 1017
DBCONFIG_STMT_SCANSTATUS DBConfig = 1018
DBCONFIG_REVERSE_SCANORDER DBConfig = 1019
// DBCONFIG_MAX DBConfig = 1019
)
// FcntlOpcode are the available opcodes for [Conn.FileControl].
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html
type FcntlOpcode uint32
const (
FCNTL_LOCKSTATE FcntlOpcode = 1
FCNTL_CHUNK_SIZE FcntlOpcode = 6
FCNTL_FILE_POINTER FcntlOpcode = 7
FCNTL_PERSIST_WAL FcntlOpcode = 10
FCNTL_POWERSAFE_OVERWRITE FcntlOpcode = 13
FCNTL_VFS_POINTER FcntlOpcode = 27
FCNTL_JOURNAL_POINTER FcntlOpcode = 28
FCNTL_DATA_VERSION FcntlOpcode = 35
FCNTL_RESERVE_BYTES FcntlOpcode = 38
FCNTL_RESET_CACHE FcntlOpcode = 42
)
// LimitCategory are the available run-time limit categories.
@@ -289,8 +328,8 @@ const (
AUTH_DROP_VTABLE AuthorizerActionCode = 30 /* Table Name Module Name */
AUTH_FUNCTION AuthorizerActionCode = 31 /* NULL Function Name */
AUTH_SAVEPOINT AuthorizerActionCode = 32 /* Operation Savepoint Name */
AUTH_COPY AuthorizerActionCode = 0 /* No longer used */
AUTH_RECURSIVE AuthorizerActionCode = 33 /* NULL NULL */
// AUTH_COPY AuthorizerActionCode = 0 /* No longer used */
)
// AuthorizerReturnCode are the integer codes
@@ -328,6 +367,18 @@ const (
TXN_WRITE TxnState = 2
)
// TraceEvent identify classes of events that can be monitored with [Conn.Trace].
//
// https://sqlite.org/c3ref/c_trace.html
type TraceEvent uint32
const (
TRACE_STMT TraceEvent = 0x01
TRACE_PROFILE TraceEvent = 0x02
TRACE_ROW TraceEvent = 0x04
TRACE_CLOSE TraceEvent = 0x08
)
// Datatype is a fundamental datatype of SQLite.
//
// https://sqlite.org/c3ref/c_blob.html
+14 -15
View File
@@ -84,19 +84,18 @@ func (ctx Context) ResultFloat(value float64) {
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultText(value string) {
ptr := ctx.c.newString(value)
ctx.c.call("sqlite3_result_text64",
uint64(ctx.handle), uint64(ptr), uint64(len(value)),
uint64(ctx.c.freer), _UTF8)
ctx.c.call("sqlite3_result_text_go",
uint64(ctx.handle), uint64(ptr), uint64(len(value)))
}
// ResultRawText sets the text result of the function to a []byte.
// Returning a nil slice is the same as calling [Context.ResultNull].
//
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultRawText(value []byte) {
ptr := ctx.c.newBytes(value)
ctx.c.call("sqlite3_result_text64",
uint64(ctx.handle), uint64(ptr), uint64(len(value)),
uint64(ctx.c.freer), _UTF8)
ctx.c.call("sqlite3_result_text_go",
uint64(ctx.handle), uint64(ptr), uint64(len(value)))
}
// ResultBlob sets the result of the function to a []byte.
@@ -105,9 +104,8 @@ func (ctx Context) ResultRawText(value []byte) {
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultBlob(value []byte) {
ptr := ctx.c.newBytes(value)
ctx.c.call("sqlite3_result_blob64",
uint64(ctx.handle), uint64(ptr), uint64(len(value)),
uint64(ctx.c.freer))
ctx.c.call("sqlite3_result_blob_go",
uint64(ctx.handle), uint64(ptr), uint64(len(value)))
}
// ResultZeroBlob sets the result of the function to a zero-filled, length n BLOB.
@@ -130,7 +128,8 @@ func (ctx Context) ResultNull() {
//
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultTime(value time.Time, format TimeFormat) {
if format == TimeFormatDefault {
switch format {
case TimeFormatDefault, TimeFormatAuto, time.RFC3339Nano:
ctx.resultRFC3339Nano(value)
return
}
@@ -153,9 +152,8 @@ func (ctx Context) resultRFC3339Nano(value time.Time) {
buf := util.View(ctx.c.mod, ptr, maxlen)
buf = value.AppendFormat(buf[:0], time.RFC3339Nano)
ctx.c.call("sqlite3_result_text64",
uint64(ctx.handle), uint64(ptr), uint64(len(buf)),
uint64(ctx.c.freer), _UTF8)
ctx.c.call("sqlite3_result_text_go",
uint64(ctx.handle), uint64(ptr), uint64(len(buf)))
}
// ResultPointer sets the result of the function to NULL, just like [Context.ResultNull],
@@ -165,7 +163,8 @@ func (ctx Context) resultRFC3339Nano(value time.Time) {
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultPointer(ptr any) {
valPtr := util.AddHandle(ctx.c.ctx, ptr)
ctx.c.call("sqlite3_result_pointer_go", uint64(valPtr))
ctx.c.call("sqlite3_result_pointer_go",
uint64(ctx.handle), uint64(valPtr))
}
// ResultJSON sets the result of the function to the JSON encoding of value.
@@ -175,7 +174,7 @@ func (ctx Context) ResultJSON(value any) {
data, err := json.Marshal(value)
if err != nil {
ctx.ResultError(err)
return
return // notest
}
ctx.ResultRawText(data)
}
+214 -91
View File
@@ -8,21 +8,50 @@
//
// The data source name for "sqlite3" databases can be a filename or a "file:" [URI].
//
// # Default transaction mode
//
// The [TRANSACTION] mode can be specified using "_txlock":
//
// sql.Open("sqlite3", "file:demo.db?_txlock=immediate")
//
// Possible values are: "deferred", "immediate", "exclusive".
// A [read-only] transaction is always "deferred", regardless of "_txlock".
// Possible values are: "deferred" (the default), "immediate", "exclusive".
// Regardless of "_txlock":
// - a [linearizable] transaction is always "exclusive";
// - a [serializable] transaction is always "immediate";
// - a [read-only] transaction is always "deferred".
//
// # Working with time
//
// The time encoding/decoding format can be specified using "_timefmt":
//
// sql.Open("sqlite3", "file:demo.db?_timefmt=sqlite")
//
// Possible values are: "auto" (the default), "sqlite", "rfc3339";
// "auto" encodes as RFC 3339 and decodes any [format] supported by SQLite;
// "sqlite" encodes as SQLite and decodes any [format] supported by SQLite;
// "rfc3339" encodes and decodes RFC 3339 only.
// - "auto" encodes as RFC 3339 and decodes any [format] supported by SQLite;
// - "sqlite" encodes as SQLite and decodes any [format] supported by SQLite;
// - "rfc3339" encodes and decodes RFC 3339 only.
//
// If you encode as RFC 3339 (the default),
// consider using the TIME [collating sequence] to produce a time-ordered sequence.
//
// To scan values in other formats, [sqlite3.TimeFormat.Scanner] may be helpful.
// To bind values in other formats, [sqlite3.TimeFormat.Encode] them before binding.
//
// When using a custom time struct, you'll have to implement
// [database/sql/driver.Valuer] and [database/sql.Scanner].
//
// The Value method should ideally encode to a time [format] supported by SQLite.
// This ensures SQL date and time functions work as they should,
// and that your schema works with other SQLite tools.
// [sqlite3.TimeFormat.Encode] may help.
//
// The Scan method needs to take into account that the value it receives can be of differing types.
// It can already be a [time.Time], if the driver decoded the value according to "_timefmt" rules.
// Or it can be a: string, int64, float64, []byte, or nil,
// depending on the column type and what whoever wrote the value.
// [sqlite3.TimeFormat.Decode] may help.
//
// # Setting PRAGMAs
//
// [PRAGMA] statements can be specified using "_pragma":
//
@@ -31,13 +60,17 @@
// If no PRAGMAs are specified, a busy timeout of 1 minute is set.
//
// Order matters:
// busy timeout and locking mode should be the first PRAGMAs set, in that order.
// encryption keys, busy timeout and locking mode should be the first PRAGMAs set,
// in that order.
//
// [URI]: https://sqlite.org/uri.html
// [PRAGMA]: https://sqlite.org/pragma.html
// [format]: https://sqlite.org/lang_datefunc.html#time_values
// [TRANSACTION]: https://sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions
// [linearizable]: https://pkg.go.dev/database/sql#TxOptions
// [serializable]: https://pkg.go.dev/database/sql#TxOptions
// [read-only]: https://pkg.go.dev/database/sql#TxOptions
// [format]: https://sqlite.org/lang_datefunc.html#time_values
// [collating sequence]: https://sqlite.org/datatype3.html#collating_sequences
package driver
import (
@@ -69,11 +102,22 @@ func init() {
// Open opens the SQLite database specified by dataSourceName as a [database/sql.DB].
//
// The init function is called by the driver on new connections.
// Open accepts zero, one, or two callbacks (nil callbacks are ignored).
// The first callback is called when the driver opens a new connection.
// The second callback is called before the driver closes a connection.
// The [sqlite3.Conn] can be used to execute queries, register functions, etc.
// Any error returned closes the connection and is returned to [database/sql].
func Open(dataSourceName string, init func(*sqlite3.Conn) error) (*sql.DB, error) {
c, err := (&SQLite{Init: init}).OpenConnector(dataSourceName)
func Open(dataSourceName string, fn ...func(*sqlite3.Conn) error) (*sql.DB, error) {
var drv SQLite
if len(fn) > 2 {
return nil, sqlite3.MISUSE
}
if len(fn) > 1 {
drv.term = fn[1]
}
if len(fn) > 0 {
drv.init = fn[0]
}
c, err := drv.OpenConnector(dataSourceName)
if err != nil {
return nil, err
}
@@ -82,12 +126,15 @@ func Open(dataSourceName string, init func(*sqlite3.Conn) error) (*sql.DB, error
// SQLite implements [database/sql/driver.Driver].
type SQLite struct {
// Init function is called by the driver on new connections.
// The [sqlite3.Conn] can be used to execute queries, register functions, etc.
// Any error returned closes the connection and is returned to [database/sql].
Init func(*sqlite3.Conn) error
init func(*sqlite3.Conn) error
term func(*sqlite3.Conn) error
}
var (
// Ensure these interfaces are implemented:
_ driver.DriverContext = &SQLite{}
)
// Open implements [database/sql/driver.Driver].
func (d *SQLite) Open(name string) (driver.Conn, error) {
c, err := d.newConnector(name)
@@ -119,10 +166,8 @@ func (d *SQLite) newConnector(name string) (*connector, error) {
}
switch txlock {
case "":
c.txBegin = "BEGIN"
case "deferred", "immediate", "exclusive":
c.txBegin = "BEGIN " + txlock
case "", "deferred", "concurrent", "immediate", "exclusive":
c.txLock = txlock
default:
return nil, fmt.Errorf("sqlite3: invalid _txlock: %s", txlock)
}
@@ -147,7 +192,7 @@ func (d *SQLite) newConnector(name string) (*connector, error) {
type connector struct {
driver *SQLite
name string
txBegin string
txLock string
tmRead sqlite3.TimeFormat
tmWrite sqlite3.TimeFormat
pragmas bool
@@ -157,19 +202,19 @@ func (n *connector) Driver() driver.Driver {
return n.driver
}
func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) {
func (n *connector) Connect(ctx context.Context) (res driver.Conn, err error) {
c := &conn{
txBegin: n.txBegin,
txLock: n.txLock,
tmRead: n.tmRead,
tmWrite: n.tmWrite,
}
c.Conn, err = sqlite3.Open(n.name)
c.Conn, err = sqlite3.OpenContext(ctx, n.name)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
if res == nil {
c.Close()
}
}()
@@ -178,22 +223,23 @@ func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) {
defer c.Conn.SetInterrupt(old)
if !n.pragmas {
err = c.Conn.BusyTimeout(60 * time.Second)
err = c.Conn.BusyTimeout(time.Minute)
if err != nil {
return nil, err
}
}
if n.driver.Init != nil {
err = n.driver.Init(c.Conn)
if n.driver.init != nil {
err = n.driver.init(c.Conn)
if err != nil {
return nil, err
}
}
if n.pragmas || n.driver.Init != nil {
if n.pragmas || n.driver.init != nil {
s, _, err := c.Conn.Prepare(`PRAGMA query_only`)
if err != nil {
return nil, err
}
defer s.Close()
if s.Step() && s.ColumnBool(0) {
c.readOnly = '1'
} else {
@@ -204,57 +250,93 @@ func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) {
return nil, err
}
}
if n.driver.term != nil {
err = c.Conn.Trace(sqlite3.TRACE_CLOSE, func(sqlite3.TraceEvent, any, any) error {
return n.driver.term(c.Conn)
})
if err != nil {
return nil, err
}
}
return c, nil
}
// Conn is implemented by the SQLite [database/sql] driver connections.
//
// It can be used to access SQLite features like [online backup]:
//
// db, err := driver.Open("temp.db")
// if err != nil {
// log.Fatal(err)
// }
// defer db.Close()
//
// conn, err := db.Conn(context.TODO())
// if err != nil {
// log.Fatal(err)
// }
//
// err = conn.Raw(func(driverConn any) error {
// conn := driverConn.(driver.Conn)
// return conn.Raw().Backup("main", "backup.db")
// })
// if err != nil {
// log.Fatal(err)
// }
//
// [online backup]: https://sqlite.org/backup.html
type Conn interface {
Raw() *sqlite3.Conn
driver.Conn
}
type conn struct {
*sqlite3.Conn
txBegin string
txCommit string
txRollback string
tmRead sqlite3.TimeFormat
tmWrite sqlite3.TimeFormat
readOnly byte
txLock string
txReset string
tmRead sqlite3.TimeFormat
tmWrite sqlite3.TimeFormat
readOnly byte
}
var (
// Ensure these interfaces are implemented:
_ Conn = &conn{}
_ driver.ConnBeginTx = &conn{}
_ driver.ConnPrepareContext = &conn{}
_ driver.ExecerContext = &conn{}
_ driver.ConnBeginTx = &conn{}
_ sqlite3.DriverConn = &conn{}
)
func (c *conn) Raw() *sqlite3.Conn {
return c.Conn
}
// Deprecated: use BeginTx instead.
func (c *conn) Begin() (driver.Tx, error) {
// notest
return c.BeginTx(context.Background(), driver.TxOptions{})
}
func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) {
txBegin := c.txBegin
c.txCommit = `COMMIT`
c.txRollback = `ROLLBACK`
if opts.ReadOnly {
txBegin = `
BEGIN deferred;
PRAGMA query_only=on`
c.txRollback = `
ROLLBACK;
PRAGMA query_only=` + string(c.readOnly)
c.txCommit = c.txRollback
}
var txLock string
switch opts.Isolation {
default:
return nil, util.IsolationErr
case
driver.IsolationLevel(sql.LevelDefault),
driver.IsolationLevel(sql.LevelSerializable):
break
case driver.IsolationLevel(sql.LevelLinearizable):
txLock = "exclusive"
case driver.IsolationLevel(sql.LevelSerializable):
txLock = "immediate"
case driver.IsolationLevel(sql.LevelDefault):
if !opts.ReadOnly {
txLock = c.txLock
}
}
c.txReset = ``
txBegin := `BEGIN ` + txLock
if opts.ReadOnly {
txBegin += ` ; PRAGMA query_only=on`
c.txReset = `; PRAGMA query_only=` + string(c.readOnly)
}
old := c.Conn.SetInterrupt(ctx)
@@ -268,7 +350,7 @@ func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, e
}
func (c *conn) Commit() error {
err := c.Conn.Exec(c.txCommit)
err := c.Conn.Exec(`COMMIT` + c.txReset)
if err != nil && !c.Conn.GetAutocommit() {
c.Rollback()
}
@@ -276,16 +358,17 @@ func (c *conn) Commit() error {
}
func (c *conn) Rollback() error {
err := c.Conn.Exec(c.txRollback)
err := c.Conn.Exec(`ROLLBACK` + c.txReset)
if errors.Is(err, sqlite3.INTERRUPT) {
old := c.Conn.SetInterrupt(context.Background())
defer c.Conn.SetInterrupt(old)
err = c.Conn.Exec(c.txRollback)
err = c.Conn.Exec(`ROLLBACK` + c.txReset)
}
return err
}
func (c *conn) Prepare(query string) (driver.Stmt, error) {
// notest
return c.PrepareContext(context.Background(), query)
}
@@ -301,7 +384,7 @@ func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, e
s.Close()
return nil, util.TailErr
}
return &stmt{Stmt: s, tmRead: c.tmRead, tmWrite: c.tmWrite}, nil
return &stmt{Stmt: s, tmRead: c.tmRead, tmWrite: c.tmWrite, inputs: -2}, nil
}
func (c *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
@@ -328,6 +411,8 @@ func (c *conn) ExecContext(ctx context.Context, query string, args []driver.Name
}
func (c *conn) CheckNamedValue(arg *driver.NamedValue) error {
// Fast path: short circuit argument verification.
// Arguments will be rejected by conn.ExecContext.
return nil
}
@@ -335,6 +420,7 @@ type stmt struct {
*sqlite3.Stmt
tmWrite sqlite3.TimeFormat
tmRead sqlite3.TimeFormat
inputs int
}
var (
@@ -345,22 +431,29 @@ var (
)
func (s *stmt) NumInput() int {
if s.inputs >= -1 {
return s.inputs
}
n := s.Stmt.BindCount()
for i := 1; i <= n; i++ {
if s.Stmt.BindName(i) != "" {
s.inputs = -1
return -1
}
}
s.inputs = n
return n
}
// Deprecated: use ExecContext instead.
func (s *stmt) Exec(args []driver.Value) (driver.Result, error) {
// notest
return s.ExecContext(context.Background(), namedValues(args))
}
// Deprecated: use QueryContext instead.
func (s *stmt) Query(args []driver.Value) (driver.Rows, error) {
// notest
return s.QueryContext(context.Background(), namedValues(args))
}
@@ -374,6 +467,7 @@ func (s *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (drive
defer s.Stmt.Conn().SetInterrupt(old)
err = s.Stmt.Exec()
s.Stmt.ClearBindings()
if err != nil {
return nil, err
}
@@ -389,19 +483,14 @@ func (s *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driv
return &rows{ctx: ctx, stmt: s}, nil
}
func (s *stmt) setupBindings(args []driver.NamedValue) error {
err := s.Stmt.ClearBindings()
if err != nil {
return err
}
func (s *stmt) setupBindings(args []driver.NamedValue) (err error) {
var ids [3]int
for _, arg := range args {
ids := ids[:0]
if arg.Name == "" {
ids = append(ids, arg.Ordinal)
} else {
for _, prefix := range []string{":", "@", "$"} {
for _, prefix := range [...]string{":", "@", "$"} {
if id := s.Stmt.BindIndex(prefix + arg.Name); id != 0 {
ids = append(ids, id)
}
@@ -435,9 +524,9 @@ func (s *stmt) setupBindings(args []driver.NamedValue) error {
default:
panic(util.AssertErr())
}
}
if err != nil {
return err
if err != nil {
return err
}
}
}
return nil
@@ -491,8 +580,15 @@ type rows struct {
*stmt
names []string
types []string
nulls []bool
}
var (
// Ensure these interfaces are implemented:
_ driver.RowsColumnTypeDatabaseTypeName = &rows{}
_ driver.RowsColumnTypeNullable = &rows{}
)
func (r *rows) Close() error {
r.Stmt.ClearBindings()
return r.Stmt.Reset()
@@ -501,27 +597,48 @@ func (r *rows) Close() error {
func (r *rows) Columns() []string {
if r.names == nil {
count := r.Stmt.ColumnCount()
r.names = make([]string, count)
for i := range r.names {
r.names[i] = r.Stmt.ColumnName(i)
names := make([]string, count)
for i := range names {
names[i] = r.Stmt.ColumnName(i)
}
r.names = names
}
return r.names
}
func (r *rows) loadTypes() {
if r.nulls == nil {
count := r.Stmt.ColumnCount()
nulls := make([]bool, count)
types := make([]string, count)
for i := range nulls {
if col := r.Stmt.ColumnOriginName(i); col != "" {
types[i], _, nulls[i], _, _, _ = r.Stmt.Conn().TableColumnMetadata(
r.Stmt.ColumnDatabaseName(i),
r.Stmt.ColumnTableName(i),
col)
}
}
r.nulls = nulls
r.types = types
}
}
func (r *rows) declType(index int) string {
if r.types == nil {
count := r.Stmt.ColumnCount()
r.types = make([]string, count)
for i := range r.types {
r.types[i] = strings.ToUpper(r.Stmt.ColumnDeclType(i))
types := make([]string, count)
for i := range types {
types[i] = strings.ToUpper(r.Stmt.ColumnDeclType(i))
}
r.types = types
}
return r.types[index]
}
func (r *rows) ColumnTypeDatabaseTypeName(index int) string {
decltype := r.declType(index)
r.loadTypes()
decltype := r.types[index]
if len := len(decltype); len > 0 && decltype[len-1] == ')' {
if i := strings.LastIndexByte(decltype, '('); i >= 0 {
decltype = decltype[:i]
@@ -530,6 +647,14 @@ func (r *rows) ColumnTypeDatabaseTypeName(index int) string {
return strings.TrimSpace(decltype)
}
func (r *rows) ColumnTypeNullable(index int) (nullable, ok bool) {
r.loadTypes()
if r.nulls[index] {
return false, true
}
return true, false
}
func (r *rows) Next(dest []driver.Value) error {
old := r.Stmt.Conn().SetInterrupt(r.ctx)
defer r.Stmt.Conn().SetInterrupt(old)
@@ -546,31 +671,29 @@ func (r *rows) Next(dest []driver.Value) error {
for i := range dest {
if t, ok := r.decodeTime(i, dest[i]); ok {
dest[i] = t
continue
}
if s, ok := dest[i].(string); ok {
t, ok := maybeTime(s)
if ok {
dest[i] = t
}
}
}
return err
}
func (r *rows) decodeTime(i int, v any) (_ time.Time, _ bool) {
if r.tmRead == sqlite3.TimeFormatDefault {
func (r *rows) decodeTime(i int, v any) (_ time.Time, ok bool) {
switch v := v.(type) {
case int64, float64:
// could be a time value
case string:
if r.tmWrite != "" && r.tmWrite != time.RFC3339 && r.tmWrite != time.RFC3339Nano {
break
}
t, ok := maybeTime(v)
if ok {
return t, true
}
default:
return
}
switch r.declType(i) {
case "DATE", "TIME", "DATETIME", "TIMESTAMP":
// maybe
default:
return
}
switch v.(type) {
case int64, float64, string:
// maybe
// could be a time value
default:
return
}
+17 -4
View File
@@ -16,12 +16,25 @@ func Savepoint(tx *sql.Tx) sqlite3.Savepoint {
return ctx.Savepoint
}
// A saveptCtx is never canceled, has no values, and has no deadline.
type saveptCtx struct{ sqlite3.Savepoint }
func (*saveptCtx) Deadline() (deadline time.Time, ok bool) { return }
func (*saveptCtx) Deadline() (deadline time.Time, ok bool) {
// notest
return
}
func (*saveptCtx) Done() <-chan struct{} { return nil }
func (*saveptCtx) Done() <-chan struct{} {
// notest
return nil
}
func (*saveptCtx) Err() error { return nil }
func (*saveptCtx) Err() error {
// notest
return nil
}
func (*saveptCtx) Value(key any) any { return nil }
func (*saveptCtx) Value(key any) any {
// notest
return nil
}
+19 -4
View File
@@ -1,6 +1,6 @@
# Embeddable Wasm build of SQLite
This folder includes an embeddable Wasm build of SQLite 3.45.3 for use with
This folder includes an embeddable Wasm build of SQLite 3.47.1 for use with
[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3).
The following optional features are compiled in:
@@ -9,18 +9,33 @@ The following optional features are compiled in:
- [JSON](https://sqlite.org/json1.html)
- [R*Tree](https://sqlite.org/rtree.html)
- [GeoPoly](https://sqlite.org/geopoly.html)
- [Spellfix1](https://sqlite.org/spellfix1.html)
- [soundex](https://sqlite.org/lang_corefunc.html#soundex)
- [stat4](https://sqlite.org/compile.html#enable_stat4)
- [base64](https://github.com/sqlite/sqlite/blob/master/ext/misc/base64.c)
- [decimal](https://github.com/sqlite/sqlite/blob/master/ext/misc/decimal.c)
- [ieee754](https://github.com/sqlite/sqlite/blob/master/ext/misc/ieee754.c)
- [regexp](https://github.com/sqlite/sqlite/blob/master/ext/misc/regexp.c)
- [series](https://github.com/sqlite/sqlite/blob/master/ext/misc/series.c)
- [uint](https://github.com/sqlite/sqlite/blob/master/ext/misc/uint.c)
- [uuid](https://github.com/sqlite/sqlite/blob/master/ext/misc/uuid.c)
- [time](../sqlite3/time.c)
See the [configuration options](../sqlite3/sqlite_cfg.h),
See the [configuration options](../sqlite3/sqlite_opt.h),
and [patches](../sqlite3) applied.
Built using [`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk),
and [`binaryen`](https://github.com/WebAssembly/binaryen).
and [`binaryen`](https://github.com/WebAssembly/binaryen).
The build is easily reproducible, and verifiable, using
[Artifact Attestations](https://github.com/ncruces/go-sqlite3/attestations).
### Customizing the build
You can use your own custom build of SQLite.
Examples of custom builds of SQLite are:
- [`github.com/ncruces/go-sqlite3/embed/bcw2`](https://github.com/ncruces/go-sqlite3/tree/main/embed/bcw2)
built from a branch supporting [`BEGIN CONCURRENT`](https://sqlite.org/src/doc/begin-concurrent/doc/begin_concurrent.md)
and [Wal2](https://sqlite.org/cgi/src/doc/wal2/doc/wal2.md).
- [`github.com/asg017/sqlite-vec-go-bindings/ncruces`](https://github.com/asg017/sqlite-vec-go-bindings)
which includes the [`sqlite-vec`](https://github.com/asg017/sqlite-vec) vector search extension.
+9 -7
View File
@@ -4,25 +4,27 @@ set -euo pipefail
cd -P -- "$(dirname -- "$0")"
ROOT=../
BINARYEN="$ROOT/tools/binaryen-version_117/bin"
WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
BINARYEN="$ROOT/tools/binaryen/bin"
WASI_SDK="$ROOT/tools/wasi-sdk/bin"
"$WASI_SDK/clang" --target=wasm32-wasi -std=c17 -flto -g0 -O2 \
-Wall -Wextra -Wno-unused-parameter \
trap 'rm -f sqlite3.tmp' EXIT
"$WASI_SDK/clang" --target=wasm32-wasi -std=c23 -g0 -O2 \
-Wall -Wextra -Wno-unused-parameter -Wno-unused-function \
-o sqlite3.wasm "$ROOT/sqlite3/main.c" \
-I"$ROOT/sqlite3" \
-mexec-model=reactor \
-msimd128 -mmutable-globals \
-matomics -msimd128 -mmutable-globals -mmultivalue \
-mbulk-memory -mreference-types \
-mnontrapping-fptoint -msign-ext \
-fno-stack-protector -fno-stack-clash-protection \
-Wl,--initial-memory=327680 \
-Wl,--stack-first \
-Wl,--import-undefined \
-Wl,--initial-memory=327680 \
-D_HAVE_SQLITE_CONFIG_H \
-DSQLITE_CUSTOM_INCLUDE=sqlite_opt.h \
$(awk '{print "-Wl,--export="$0}' exports.txt)
trap 'rm -f sqlite3.tmp' EXIT
"$BINARYEN/wasm-ctor-eval" -g -c _initialize sqlite3.wasm -o sqlite3.tmp
"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \
sqlite3.tmp -o sqlite3.wasm \
+20 -8
View File
@@ -1,7 +1,4 @@
aligned_alloc
free
malloc
malloc_destructor
sqlite3_anycollseq_init
sqlite3_autovacuum_pages_go
sqlite3_backup_finish
@@ -9,7 +6,7 @@ sqlite3_backup_init
sqlite3_backup_pagecount
sqlite3_backup_remaining
sqlite3_backup_step
sqlite3_bind_blob64
sqlite3_bind_blob_go
sqlite3_bind_double
sqlite3_bind_int64
sqlite3_bind_null
@@ -17,7 +14,7 @@ sqlite3_bind_parameter_count
sqlite3_bind_parameter_index
sqlite3_bind_parameter_name
sqlite3_bind_pointer_go
sqlite3_bind_text64
sqlite3_bind_text_go
sqlite3_bind_value
sqlite3_bind_zeroblob64
sqlite3_blob_bytes
@@ -36,10 +33,13 @@ sqlite3_collation_needed_go
sqlite3_column_blob
sqlite3_column_bytes
sqlite3_column_count
sqlite3_column_database_name
sqlite3_column_decltype
sqlite3_column_double
sqlite3_column_int64
sqlite3_column_name
sqlite3_column_origin_name
sqlite3_column_table_name
sqlite3_column_text
sqlite3_column_type
sqlite3_column_value
@@ -51,33 +51,42 @@ sqlite3_create_collation_go
sqlite3_create_function_go
sqlite3_create_module_go
sqlite3_create_window_function_go
sqlite3_data_count
sqlite3_database_file_object
sqlite3_db_cacheflush
sqlite3_db_config
sqlite3_db_filename
sqlite3_db_name
sqlite3_db_readonly
sqlite3_db_release_memory
sqlite3_db_status
sqlite3_declare_vtab
sqlite3_errcode
sqlite3_errmsg
sqlite3_error_offset
sqlite3_errstr
sqlite3_exec
sqlite3_expanded_sql
sqlite3_file_control
sqlite3_filename_database
sqlite3_filename_journal
sqlite3_filename_wal
sqlite3_finalize
sqlite3_free
sqlite3_get_autocommit
sqlite3_get_auxdata
sqlite3_hard_heap_limit64
sqlite3_interrupt
sqlite3_last_insert_rowid
sqlite3_limit
sqlite3_malloc64
sqlite3_open_v2
sqlite3_overload_function
sqlite3_prepare_v3
sqlite3_progress_handler_go
sqlite3_realloc64
sqlite3_reset
sqlite3_result_blob64
sqlite3_result_blob_go
sqlite3_result_double
sqlite3_result_error
sqlite3_result_error_code
@@ -86,27 +95,30 @@ sqlite3_result_error_toobig
sqlite3_result_int64
sqlite3_result_null
sqlite3_result_pointer_go
sqlite3_result_text64
sqlite3_result_text_go
sqlite3_result_value
sqlite3_result_zeroblob64
sqlite3_rollback_hook_go
sqlite3_set_authorizer_go
sqlite3_set_auxdata_go
sqlite3_set_last_insert_rowid
sqlite3_soft_heap_limit64
sqlite3_step
sqlite3_stmt_busy
sqlite3_stmt_readonly
sqlite3_stmt_status
sqlite3_table_column_metadata
sqlite3_total_changes64
sqlite3_trace_go
sqlite3_txn_state
sqlite3_update_hook_go
sqlite3_uri_key
sqlite3_uri_parameter
sqlite3_value_blob
sqlite3_value_bytes
sqlite3_value_double
sqlite3_value_dup
sqlite3_value_free
sqlite3_value_frombind
sqlite3_value_int64
sqlite3_value_nochange
sqlite3_value_numeric_type
Binary file not shown.
+10
View File
@@ -106,6 +106,11 @@ func (e ErrorCode) Temporary() bool {
return e == BUSY
}
// ExtendedCode returns the extended error code for this error.
func (e ErrorCode) ExtendedCode() ExtendedErrorCode {
return ExtendedErrorCode(e)
}
// Error implements the error interface.
func (e ExtendedErrorCode) Error() string {
return util.ErrorCodeString(uint32(e))
@@ -136,6 +141,11 @@ func (e ExtendedErrorCode) Timeout() bool {
return e == BUSY_TIMEOUT
}
// Code returns the primary error code for this error.
func (e ExtendedErrorCode) Code() ErrorCode {
return ErrorCode(e)
}
func errorCode(err error, def ErrorCode) (msg string, code uint32) {
switch code := err.(type) {
case nil:
+25 -9
View File
@@ -4,8 +4,9 @@ import (
"context"
"sync"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
// CollationNeeded registers a callback to be invoked
@@ -31,17 +32,25 @@ func (c *Conn) CollationNeeded(cb func(db *Conn, name string)) error {
//
// This can be used to load schemas that contain
// one or more unknown collating sequences.
func (c *Conn) AnyCollationNeeded() {
c.call("sqlite3_anycollseq_init", uint64(c.handle), 0, 0)
func (c Conn) AnyCollationNeeded() error {
r := c.call("sqlite3_anycollseq_init", uint64(c.handle), 0, 0)
if err := c.error(r); err != nil {
return err
}
c.collation = nil
return nil
}
// CreateCollation defines a new collating sequence.
//
// https://sqlite.org/c3ref/create_collation.html
func (c *Conn) CreateCollation(name string, fn func(a, b []byte) int) error {
var funcPtr uint32
defer c.arena.mark()()
namePtr := c.arena.string(name)
funcPtr := util.AddHandle(c.ctx, fn)
if fn != nil {
funcPtr = util.AddHandle(c.ctx, fn)
}
r := c.call("sqlite3_create_collation_go",
uint64(c.handle), uint64(namePtr), uint64(funcPtr))
return c.error(r)
@@ -51,9 +60,12 @@ func (c *Conn) CreateCollation(name string, fn func(a, b []byte) int) error {
//
// https://sqlite.org/c3ref/create_function.html
func (c *Conn) CreateFunction(name string, nArg int, flag FunctionFlag, fn ScalarFunction) error {
var funcPtr uint32
defer c.arena.mark()()
namePtr := c.arena.string(name)
funcPtr := util.AddHandle(c.ctx, fn)
if fn != nil {
funcPtr = util.AddHandle(c.ctx, fn)
}
r := c.call("sqlite3_create_function_go",
uint64(c.handle), uint64(namePtr), uint64(nArg),
uint64(flag), uint64(funcPtr))
@@ -70,10 +82,13 @@ type ScalarFunction func(ctx Context, arg ...Value)
//
// https://sqlite.org/c3ref/create_function.html
func (c *Conn) CreateWindowFunction(name string, nArg int, flag FunctionFlag, fn func() AggregateFunction) error {
var funcPtr uint32
defer c.arena.mark()()
call := "sqlite3_create_aggregate_function_go"
namePtr := c.arena.string(name)
funcPtr := util.AddHandle(c.ctx, fn)
if fn != nil {
funcPtr = util.AddHandle(c.ctx, fn)
}
call := "sqlite3_create_aggregate_function_go"
if _, ok := fn().(WindowFunction); ok {
call = "sqlite3_create_window_function_go"
}
@@ -183,11 +198,12 @@ func callbackAggregate(db *Conn, pAgg, pApp uint32) (AggregateFunction, uint32)
// We need to create the aggregate.
fn := util.GetHandle(db.ctx, pApp).(func() AggregateFunction)()
handle := util.AddHandle(db.ctx, fn)
if pAgg != 0 {
handle := util.AddHandle(db.ctx, fn)
util.WriteUint32(db.mod, pAgg, handle)
return fn, handle
}
return fn, handle
return fn, 0
}
func callbackArgs(db *Conn, arg []Value, pArg uint32) {
+11 -5
View File
@@ -1,8 +1,14 @@
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+24
View File
@@ -0,0 +1,24 @@
//go:build !(unix || windows) || sqlite3_nosys
package alloc
import "github.com/tetratelabs/wazero/experimental"
func NewMemory(cap, max uint64) experimental.LinearMemory {
return &sliceMemory{make([]byte, 0, cap)}
}
type sliceMemory struct {
buf []byte
}
func (b *sliceMemory) Free() {}
func (b *sliceMemory) Reallocate(size uint64) []byte {
if cap := uint64(cap(b.buf)); size > cap {
b.buf = append(b.buf[:cap], make([]byte, size-cap)...)
} else {
b.buf = b.buf[:size]
}
return b.buf
}
@@ -1,6 +1,6 @@
//go:build unix
//go:build unix && !sqlite3_nosys
package util
package alloc
import (
"math"
@@ -9,30 +9,24 @@ import (
"golang.org/x/sys/unix"
)
func mmappedAllocator(cap, max uint64) experimental.LinearMemory {
func NewMemory(_, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
max = (max + rnd) &^ rnd
cap = (cap + rnd) &^ rnd
if max > math.MaxInt {
// This ensures int(max) overflows to a negative value,
// and unix.Mmap returns EINVAL.
max = math.MaxUint64
}
// Reserve max bytes of address space, to ensure we won't need to move it.
// A protected, private, anonymous mapping should not commit memory.
b, err := unix.Mmap(-1, 0, int(max), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON)
if err != nil {
panic(err)
}
// Commit the initial cap bytes of memory.
err = unix.Mprotect(b[:cap], unix.PROT_READ|unix.PROT_WRITE)
if err != nil {
unix.Munmap(b)
panic(err)
}
return &mmappedMemory{buf: b[:cap]}
return &mmappedMemory{buf: b[:0]}
}
// The slice covers the entire mmapped memory:
@@ -43,7 +37,9 @@ type mmappedMemory struct {
}
func (m *mmappedMemory) Reallocate(size uint64) []byte {
if com := uint64(len(m.buf)); com < size {
com := uint64(len(m.buf))
res := uint64(cap(m.buf))
if com < size && size <= res {
// Round up to the page size.
rnd := uint64(unix.Getpagesize() - 1)
new := (size + rnd) &^ rnd
@@ -51,7 +47,7 @@ func (m *mmappedMemory) Reallocate(size uint64) []byte {
// Commit additional memory up to new bytes.
err := unix.Mprotect(m.buf[com:new], unix.PROT_READ|unix.PROT_WRITE)
if err != nil {
panic(err)
return nil
}
// Update committed memory.
+76
View File
@@ -0,0 +1,76 @@
//go:build !sqlite3_nosys
package alloc
import (
"math"
"reflect"
"unsafe"
"github.com/tetratelabs/wazero/experimental"
"golang.org/x/sys/windows"
)
func NewMemory(_, max uint64) experimental.LinearMemory {
// Round up to the page size.
rnd := uint64(windows.Getpagesize() - 1)
max = (max + rnd) &^ rnd
if max > math.MaxInt {
// This ensures uintptr(max) overflows to a large value,
// and windows.VirtualAlloc returns an error.
max = math.MaxUint64
}
// Reserve max bytes of address space, to ensure we won't need to move it.
// This does not commit memory.
r, err := windows.VirtualAlloc(0, uintptr(max), windows.MEM_RESERVE, windows.PAGE_READWRITE)
if err != nil {
panic(err)
}
mem := virtualMemory{addr: r}
// SliceHeader, although deprecated, avoids a go vet warning.
sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem.buf))
sh.Cap = int(max)
sh.Data = r
return &mem
}
// The slice covers the entire mmapped memory:
// - len(buf) is the already committed memory,
// - cap(buf) is the reserved address space.
type virtualMemory struct {
buf []byte
addr uintptr
}
func (m *virtualMemory) Reallocate(size uint64) []byte {
com := uint64(len(m.buf))
res := uint64(cap(m.buf))
if com < size && size <= res {
// Round up to the page size.
rnd := uint64(windows.Getpagesize() - 1)
new := (size + rnd) &^ rnd
// Commit additional memory up to new bytes.
_, err := windows.VirtualAlloc(m.addr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE)
if err != nil {
return nil
}
// Update committed memory.
m.buf = m.buf[:new]
}
// Limit returned capacity because bytes beyond
// len(m.buf) have not yet been committed.
return m.buf[:size:len(m.buf)]
}
func (m *virtualMemory) Free() {
err := windows.VirtualFree(m.addr, 0, windows.MEM_RELEASE)
if err != nil {
panic(err)
}
m.addr = 0
}
-22
View File
@@ -1,22 +0,0 @@
package util
import "strings"
func ParseBool(s string) (b, ok bool) {
if len(s) == 0 {
return false, false
}
if s[0] == '0' {
return false, true
}
if '1' <= s[0] && s[0] <= '9' {
return true, true
}
switch strings.ToLower(s) {
case "true", "yes", "on":
return true, true
case "false", "no", "off":
return false, true
}
return false, false
}
+1 -1
View File
@@ -1,6 +1,6 @@
package util
// https://sqlite.com/matrix/rescode.html
// https://sqlite.com/rescode.html
const (
OK = 0 /* Successful result */
+10
View File
@@ -104,3 +104,13 @@ func ErrorCodeString(rc uint32) string {
}
return "sqlite3: unknown error"
}
type ErrorJoiner []error
func (j *ErrorJoiner) Join(errs ...error) {
for _, err := range errs {
if err != nil {
*j = append(*j, err)
}
}
}
+12
View File
@@ -26,6 +26,7 @@ func ExportFuncVI[T0 i32](mod wazero.HostModuleBuilder, name string, fn func(con
type funcVII[T0, T1 i32] func(context.Context, api.Module, T0, T1)
func (fn funcVII[T0, T1]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[1] // prevent bounds check on every slice access
fn(ctx, mod, T0(stack[0]), T1(stack[1]))
}
@@ -39,6 +40,7 @@ func ExportFuncVII[T0, T1 i32](mod wazero.HostModuleBuilder, name string, fn fun
type funcVIII[T0, T1, T2 i32] func(context.Context, api.Module, T0, T1, T2)
func (fn funcVIII[T0, T1, T2]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[2] // prevent bounds check on every slice access
fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]))
}
@@ -52,6 +54,7 @@ func ExportFuncVIII[T0, T1, T2 i32](mod wazero.HostModuleBuilder, name string, f
type funcVIIII[T0, T1, T2, T3 i32] func(context.Context, api.Module, T0, T1, T2, T3)
func (fn funcVIIII[T0, T1, T2, T3]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[3] // prevent bounds check on every slice access
fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]))
}
@@ -65,6 +68,7 @@ func ExportFuncVIIII[T0, T1, T2, T3 i32](mod wazero.HostModuleBuilder, name stri
type funcVIIIII[T0, T1, T2, T3, T4 i32] func(context.Context, api.Module, T0, T1, T2, T3, T4)
func (fn funcVIIIII[T0, T1, T2, T3, T4]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[4] // prevent bounds check on every slice access
fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]), T4(stack[4]))
}
@@ -78,6 +82,7 @@ func ExportFuncVIIIII[T0, T1, T2, T3, T4 i32](mod wazero.HostModuleBuilder, name
type funcVIIIIJ[T0, T1, T2, T3 i32, T4 i64] func(context.Context, api.Module, T0, T1, T2, T3, T4)
func (fn funcVIIIIJ[T0, T1, T2, T3, T4]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[4] // prevent bounds check on every slice access
fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]), T4(stack[4]))
}
@@ -104,6 +109,7 @@ func ExportFuncII[TR, T0 i32](mod wazero.HostModuleBuilder, name string, fn func
type funcIII[TR, T0, T1 i32] func(context.Context, api.Module, T0, T1) TR
func (fn funcIII[TR, T0, T1]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[1] // prevent bounds check on every slice access
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1])))
}
@@ -117,6 +123,7 @@ func ExportFuncIII[TR, T0, T1 i32](mod wazero.HostModuleBuilder, name string, fn
type funcIIII[TR, T0, T1, T2 i32] func(context.Context, api.Module, T0, T1, T2) TR
func (fn funcIIII[TR, T0, T1, T2]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[2] // prevent bounds check on every slice access
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2])))
}
@@ -130,6 +137,7 @@ func ExportFuncIIII[TR, T0, T1, T2 i32](mod wazero.HostModuleBuilder, name strin
type funcIIIII[TR, T0, T1, T2, T3 i32] func(context.Context, api.Module, T0, T1, T2, T3) TR
func (fn funcIIIII[TR, T0, T1, T2, T3]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[3] // prevent bounds check on every slice access
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3])))
}
@@ -143,6 +151,7 @@ func ExportFuncIIIII[TR, T0, T1, T2, T3 i32](mod wazero.HostModuleBuilder, name
type funcIIIIII[TR, T0, T1, T2, T3, T4 i32] func(context.Context, api.Module, T0, T1, T2, T3, T4) TR
func (fn funcIIIIII[TR, T0, T1, T2, T3, T4]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[4] // prevent bounds check on every slice access
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]), T4(stack[4])))
}
@@ -156,6 +165,7 @@ func ExportFuncIIIIII[TR, T0, T1, T2, T3, T4 i32](mod wazero.HostModuleBuilder,
type funcIIIIIII[TR, T0, T1, T2, T3, T4, T5 i32] func(context.Context, api.Module, T0, T1, T2, T3, T4, T5) TR
func (fn funcIIIIIII[TR, T0, T1, T2, T3, T4, T5]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[5] // prevent bounds check on every slice access
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]), T4(stack[4]), T5(stack[5])))
}
@@ -169,6 +179,7 @@ func ExportFuncIIIIIII[TR, T0, T1, T2, T3, T4, T5 i32](mod wazero.HostModuleBuil
type funcIIIIJ[TR, T0, T1, T2 i32, T3 i64] func(context.Context, api.Module, T0, T1, T2, T3) TR
func (fn funcIIIIJ[TR, T0, T1, T2, T3]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[3] // prevent bounds check on every slice access
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3])))
}
@@ -182,6 +193,7 @@ func ExportFuncIIIIJ[TR, T0, T1, T2 i32, T3 i64](mod wazero.HostModuleBuilder, n
type funcIIJ[TR, T0 i32, T1 i64] func(context.Context, api.Module, T0, T1) TR
func (fn funcIIJ[TR, T0, T1]) Call(ctx context.Context, mod api.Module, stack []uint64) {
_ = stack[1] // prevent bounds check on every slice access
stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1])))
}
+7 -2
View File
@@ -35,17 +35,22 @@ func DelHandle(ctx context.Context, id uint32) error {
s := ctx.Value(moduleKey{}).(*moduleState)
a := s.handles[^id]
s.handles[^id] = nil
s.holes++
if l := uint32(len(s.handles)); l == ^id {
s.handles = s.handles[:l-1]
} else {
s.holes++
}
if c, ok := a.(io.Closer); ok {
return c.Close()
}
return nil
}
func AddHandle(ctx context.Context, a any) (id uint32) {
func AddHandle(ctx context.Context, a any) uint32 {
if a == nil {
panic(NilErr)
}
s := ctx.Value(moduleKey{}).(*moduleState)
// Find an empty slot.
+1 -1
View File
@@ -26,7 +26,7 @@ func (j JSON) Scan(value any) error {
buf = v.AppendFormat(buf, time.RFC3339Nano)
buf = append(buf, '"')
case nil:
buf = append(buf, "null"...)
buf = []byte("null")
default:
panic(AssertErr())
}
+29
View File
@@ -0,0 +1,29 @@
package util
import "math"
func abs(n int) int {
if n < 0 {
return -n
}
return n
}
func GCD(m, n int) int {
for n != 0 {
m, n = n, m%n
}
return abs(m)
}
func LCM(m, n int) int {
if n == 0 {
return 0
}
return abs(n) * (abs(m) / GCD(m, n))
}
// https://developer.nvidia.com/blog/lerp-faster-cuda/
func Lerp(v0, v1, t float64) float64 {
return math.FMA(t, v1, math.FMA(-t, v0, v0))
}
+15 -25
View File
@@ -1,4 +1,4 @@
//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys)
//go:build unix && !sqlite3_nosys
package util
@@ -8,15 +8,9 @@ import (
"unsafe"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"golang.org/x/sys/unix"
)
func withMmappedAllocator(ctx context.Context) context.Context {
return experimental.WithMemoryAllocator(ctx,
experimental.MemoryAllocatorFunc(mmappedAllocator))
}
type mmapState struct {
regions []*MappedRegion
}
@@ -31,7 +25,7 @@ func (s *mmapState) new(ctx context.Context, mod api.Module, size int32) *Mapped
// Allocate page aligned memmory.
alloc := mod.ExportedFunction("aligned_alloc")
stack := [2]uint64{
stack := [...]uint64{
uint64(unix.Getpagesize()),
uint64(size),
}
@@ -45,7 +39,7 @@ func (s *mmapState) new(ctx context.Context, mod api.Module, size int32) *Mapped
// Save the newly allocated region.
ptr := uint32(stack[0])
buf := View(mod, ptr, uint64(size))
addr := uintptr(unsafe.Pointer(&buf[0]))
addr := unsafe.Pointer(&buf[0])
s.regions = append(s.regions, &MappedRegion{
Ptr: ptr,
addr: addr,
@@ -55,16 +49,16 @@ func (s *mmapState) new(ctx context.Context, mod api.Module, size int32) *Mapped
}
type MappedRegion struct {
addr uintptr
addr unsafe.Pointer
Ptr uint32
size int32
used bool
}
func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, prot int) (*MappedRegion, error) {
func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, readOnly bool) (*MappedRegion, error) {
s := ctx.Value(moduleKey{}).(*moduleState)
r := s.new(ctx, mod, size)
err := r.mmap(f, offset, prot)
err := r.mmap(f, offset, readOnly)
if err != nil {
return nil, err
}
@@ -75,23 +69,19 @@ func (r *MappedRegion) Unmap() error {
// We can't munmap the region, otherwise it could be remaped.
// Instead, convert it to a protected, private, anonymous mapping.
// If successful, it can be reused for a subsequent mmap.
_, err := mmap(r.addr, uintptr(r.size),
unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON|unix.MAP_FIXED,
-1, 0)
_, err := unix.MmapPtr(-1, 0, r.addr, uintptr(r.size),
unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_FIXED|unix.MAP_ANON)
r.used = err != nil
return err
}
func (r *MappedRegion) mmap(f *os.File, offset int64, prot int) error {
_, err := mmap(r.addr, uintptr(r.size),
prot, unix.MAP_SHARED|unix.MAP_FIXED,
int(f.Fd()), offset)
func (r *MappedRegion) mmap(f *os.File, offset int64, readOnly bool) error {
prot := unix.PROT_READ
if !readOnly {
prot |= unix.PROT_WRITE
}
_, err := unix.MmapPtr(int(f.Fd()), offset, r.addr, uintptr(r.size),
prot, unix.MAP_SHARED|unix.MAP_FIXED)
r.used = err == nil
return err
}
// We need the low level mmap for MAP_FIXED to work.
// Bind the syscall version hoping that it is more stable.
//go:linkname mmap syscall.mmap
func mmap(addr, length uintptr, prot, flag, fd int, pos int64) (*byte, error)
+1 -7
View File
@@ -1,11 +1,5 @@
//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys
//go:build !unix || sqlite3_nosys
package util
import "context"
type mmapState struct{}
func withMmappedAllocator(ctx context.Context) context.Context {
return ctx
}
+53
View File
@@ -0,0 +1,53 @@
//go:build !sqlite3_nosys
package util
import (
"context"
"os"
"reflect"
"unsafe"
"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/windows"
)
type MappedRegion struct {
windows.Handle
Data []byte
addr uintptr
}
func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32) (*MappedRegion, error) {
h, err := windows.CreateFileMapping(windows.Handle(f.Fd()), nil, windows.PAGE_READWRITE, 0, 0, nil)
if h == 0 {
return nil, err
}
a, err := windows.MapViewOfFile(h, windows.FILE_MAP_WRITE,
uint32(offset>>32), uint32(offset), uintptr(size))
if a == 0 {
windows.CloseHandle(h)
return nil, err
}
res := &MappedRegion{Handle: h, addr: a}
// SliceHeader, although deprecated, avoids a go vet warning.
sh := (*reflect.SliceHeader)(unsafe.Pointer(&res.Data))
sh.Len = int(size)
sh.Cap = int(size)
sh.Data = a
return res, nil
}
func (r *MappedRegion) Unmap() error {
if r.Data == nil {
return nil
}
err := windows.UnmapViewOfFile(r.addr)
if err != nil {
return err
}
r.Data = nil
return windows.CloseHandle(r.Handle)
}
+3 -1
View File
@@ -4,6 +4,8 @@ import (
"context"
"github.com/tetratelabs/wazero/experimental"
"github.com/ncruces/go-sqlite3/internal/alloc"
)
type moduleKey struct{}
@@ -14,7 +16,7 @@ type moduleState struct {
func NewContext(ctx context.Context) context.Context {
state := new(moduleState)
ctx = withMmappedAllocator(ctx)
ctx = experimental.WithMemoryAllocator(ctx, experimental.MemoryAllocatorFunc(alloc.NewMemory))
ctx = experimental.WithCloseNotifier(ctx, state)
ctx = context.WithValue(ctx, moduleKey{}, state)
return ctx
+12
View File
@@ -0,0 +1,12 @@
package util
type Set[E comparable] map[E]struct{}
func (s Set[E]) Add(v E) {
s[v] = struct{}{}
}
func (s Set[E]) Contains(v E) bool {
_, ok := s[v]
return ok
}
+2 -1
View File
@@ -5,7 +5,8 @@ import "github.com/ncruces/go-sqlite3/internal/util"
// JSON returns a value that can be used as an argument to
// [database/sql.DB.Exec], [database/sql.Row.Scan] and similar methods to
// store value as JSON, or decode JSON into value.
// JSON should NOT be used with [BindJSON] or [ResultJSON].
// JSON should NOT be used with [Stmt.BindJSON], [Stmt.ColumnJSON],
// [Value.JSON], or [Context.ResultJSON].
func JSON(value any) any {
return util.JSON{Value: value}
}
+2 -1
View File
@@ -4,7 +4,8 @@ import "github.com/ncruces/go-sqlite3/internal/util"
// Pointer returns a pointer to a value that can be used as an argument to
// [database/sql.DB.Exec] and similar methods.
// Pointer should NOT be used with [BindPointer] or [ResultPointer].
// Pointer should NOT be used with [Stmt.BindPointer],
// [Value.Pointer], or [Context.ResultPointer].
//
// https://sqlite.org/bindptr.html
func Pointer[T any](value T) any {
+39 -11
View File
@@ -3,6 +3,7 @@ package sqlite3
import (
"bytes"
"math"
"reflect"
"strconv"
"strings"
"time"
@@ -13,6 +14,9 @@ import (
// Quote escapes and quotes a value
// making it safe to embed in SQL text.
// Strings with embedded NUL characters are truncated.
//
// https://sqlite.org/lang_corefunc.html#quote
func Quote(value any) string {
switch v := value.(type) {
case nil:
@@ -42,8 +46,8 @@ func Quote(value any) string {
return "'" + v.Format(time.RFC3339Nano) + "'"
case string:
if strings.IndexByte(v, 0) >= 0 {
break
if i := strings.IndexByte(v, 0); i >= 0 {
v = v[:i]
}
buf := make([]byte, 2+len(v)+strings.Count(v, "'"))
@@ -57,13 +61,13 @@ func Quote(value any) string {
buf[i] = b
i += 1
}
buf[i] = '\''
buf[len(buf)-1] = '\''
return unsafe.String(&buf[0], len(buf))
case []byte:
buf := make([]byte, 3+2*len(v))
buf[0] = 'x'
buf[1] = '\''
buf[0] = 'x'
i := 2
for _, b := range v {
const hex = "0123456789ABCDEF"
@@ -71,26 +75,50 @@ func Quote(value any) string {
buf[i+1] = hex[b%16]
i += 2
}
buf[i] = '\''
buf[len(buf)-1] = '\''
return unsafe.String(&buf[0], len(buf))
case ZeroBlob:
if v > ZeroBlob(1e9-3)/2 {
break
}
buf := bytes.Repeat([]byte("0"), int(3+2*int64(v)))
buf[0] = 'x'
buf[1] = '\''
buf[0] = 'x'
buf[len(buf)-1] = '\''
return unsafe.String(&buf[0], len(buf))
}
v := reflect.ValueOf(value)
k := v.Kind()
if k == reflect.Interface || k == reflect.Pointer {
if v.IsNil() {
return "NULL"
}
v = v.Elem()
k = v.Kind()
}
switch {
case v.CanInt():
return strconv.FormatInt(v.Int(), 10)
case v.CanUint():
return strconv.FormatUint(v.Uint(), 10)
case v.CanFloat():
return Quote(v.Float())
case k == reflect.Bool:
return Quote(v.Bool())
case k == reflect.String:
return Quote(v.String())
case (k == reflect.Slice || k == reflect.Array && v.CanAddr()) &&
v.Type().Elem().Kind() == reflect.Uint8:
return Quote(v.Bytes())
}
panic(util.ValueErr)
}
// QuoteIdentifier escapes and quotes an identifier
// making it safe to embed in SQL text.
// Strings with embedded NUL characters panic.
func QuoteIdentifier(id string) string {
if strings.IndexByte(id, 0) >= 0 {
panic(util.ValueErr)
@@ -107,6 +135,6 @@ func QuoteIdentifier(id string) string {
buf[i] = b
i += 1
}
buf[i] = '"'
buf[len(buf)-1] = '"'
return unsafe.String(&buf[0], len(buf))
}
+30
View File
@@ -0,0 +1,30 @@
package sqlite3
import "sync"
var (
// +checklocks:extRegistryMtx
extRegistry []func(*Conn) error
extRegistryMtx sync.RWMutex
)
// AutoExtension causes the entryPoint function to be invoked
// for each new database connection that is created.
//
// https://sqlite.org/c3ref/auto_extension.html
func AutoExtension(entryPoint func(*Conn) error) {
extRegistryMtx.Lock()
defer extRegistryMtx.Unlock()
extRegistry = append(extRegistry, entryPoint)
}
func initExtensions(c *Conn) error {
extRegistryMtx.RLock()
defer extRegistryMtx.RUnlock()
for _, f := range extRegistry {
if err := f(c); err != nil {
return err
}
}
return nil
}
+45 -26
View File
@@ -9,10 +9,12 @@ import (
"sync"
"unsafe"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/vfs"
)
// Configure SQLite Wasm.
@@ -28,6 +30,14 @@ var (
RuntimeConfig wazero.RuntimeConfig
)
// Initialize decodes and compiles the SQLite Wasm binary.
// This is called implicitly when the first connection is openned,
// but is potentially slow, so you may want to call it at a more convenient time.
func Initialize() error {
instance.once.Do(compileSQLite)
return instance.err
}
var instance struct {
runtime wazero.Runtime
compiled wazero.CompiledModule
@@ -36,12 +46,19 @@ var instance struct {
}
func compileSQLite() {
if RuntimeConfig == nil {
RuntimeConfig = wazero.NewRuntimeConfig()
}
ctx := context.Background()
instance.runtime = wazero.NewRuntimeWithConfig(ctx, RuntimeConfig)
cfg := RuntimeConfig
if cfg == nil {
cfg = wazero.NewRuntimeConfig()
if bits.UintSize >= 64 {
cfg = cfg.WithMemoryLimitPages(4096) // 256MB
} else {
cfg = cfg.WithMemoryLimitPages(512) // 32MB
}
}
cfg = cfg.WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads)
instance.runtime = wazero.NewRuntimeWithConfig(ctx, cfg)
env := instance.runtime.NewHostModuleBuilder("env")
env = vfs.ExportHostFunctions(env)
@@ -74,14 +91,12 @@ type sqlite struct {
id [32]*byte
mask uint32
}
stack [8]uint64
freer uint32
stack [9]uint64
}
func instantiateSQLite() (sqlt *sqlite, err error) {
instance.once.Do(compileSQLite)
if instance.err != nil {
return nil, instance.err
if err := Initialize(); err != nil {
return nil, err
}
sqlt = new(sqlite)
@@ -92,14 +107,7 @@ func instantiateSQLite() (sqlt *sqlite, err error) {
if err != nil {
return nil, err
}
global := sqlt.mod.ExportedGlobal("malloc_destructor")
if global == nil {
return nil, util.BadBinaryErr
}
sqlt.freer = util.ReadUint32(sqlt.mod, uint32(global.Get()))
if sqlt.freer == 0 {
if sqlt.getfn("sqlite3_progress_handler_go") == nil {
return nil, util.BadBinaryErr
}
return sqlt, nil
@@ -129,7 +137,7 @@ func (sqlt *sqlite) error(rc uint64, handle uint32, sql ...string) error {
err.msg = util.ReadString(sqlt.mod, uint32(r), _MAX_LENGTH)
}
if sql != nil {
if len(sql) != 0 {
if r := sqlt.call("sqlite3_error_offset", uint64(handle)); r != math.MaxUint32 {
err.sql = sql[0][r:]
}
@@ -186,14 +194,19 @@ func (sqlt *sqlite) free(ptr uint32) {
if ptr == 0 {
return
}
sqlt.call("free", uint64(ptr))
sqlt.call("sqlite3_free", uint64(ptr))
}
func (sqlt *sqlite) new(size uint64) uint32 {
if size > _MAX_ALLOCATION_SIZE {
ptr := uint32(sqlt.call("sqlite3_malloc64", size))
if ptr == 0 && size != 0 {
panic(util.OOMErr)
}
ptr := uint32(sqlt.call("malloc", size))
return ptr
}
func (sqlt *sqlite) realloc(ptr uint32, size uint64) uint32 {
ptr = uint32(sqlt.call("sqlite3_realloc64", uint64(ptr), size))
if ptr == 0 && size != 0 {
panic(util.OOMErr)
}
@@ -204,7 +217,11 @@ func (sqlt *sqlite) newBytes(b []byte) uint32 {
if (*[0]byte)(b) == nil {
return 0
}
ptr := sqlt.new(uint64(len(b)))
size := len(b)
if size == 0 {
size = 1
}
ptr := sqlt.new(uint64(size))
util.WriteBytes(sqlt.mod, ptr, b)
return ptr
}
@@ -289,12 +306,14 @@ func (a *arena) string(s string) uint32 {
}
func exportCallbacks(env wazero.HostModuleBuilder) wazero.HostModuleBuilder {
util.ExportFuncIII(env, "go_busy_handler", busyCallback)
util.ExportFuncII(env, "go_progress_handler", progressCallback)
util.ExportFuncIII(env, "go_busy_timeout", timeoutCallback)
util.ExportFuncIII(env, "go_busy_handler", busyCallback)
util.ExportFuncII(env, "go_commit_hook", commitCallback)
util.ExportFuncVI(env, "go_rollback_hook", rollbackCallback)
util.ExportFuncVIIIIJ(env, "go_update_hook", updateCallback)
util.ExportFuncIIIII(env, "go_wal_hook", walCallback)
util.ExportFuncIIIII(env, "go_trace", traceCallback)
util.ExportFuncIIIIII(env, "go_autovacuum_pages", autoVacuumCallback)
util.ExportFuncIIIIIII(env, "go_authorizer", authorizerCallback)
util.ExportFuncVIII(env, "go_log", logCallback)
+116 -35
View File
@@ -15,6 +15,7 @@ import (
type Stmt struct {
c *Conn
err error
sql string
handle uint32
}
@@ -29,6 +30,16 @@ func (s *Stmt) Close() error {
}
r := s.c.call("sqlite3_finalize", uint64(s.handle))
stmts := s.c.stmts
for i := range stmts {
if s == stmts[i] {
l := len(stmts) - 1
stmts[i] = stmts[l]
stmts[l] = nil
s.c.stmts = stmts[:l]
break
}
}
s.handle = 0
return s.c.error(r)
@@ -41,6 +52,24 @@ func (s *Stmt) Conn() *Conn {
return s.c
}
// SQL returns the SQL text used to create the prepared statement.
//
// https://sqlite.org/c3ref/expanded_sql.html
func (s *Stmt) SQL() string {
return s.sql
}
// ExpandedSQL returns the SQL text of the prepared statement
// with bound parameters expanded.
//
// https://sqlite.org/c3ref/expanded_sql.html
func (s *Stmt) ExpandedSQL() string {
r := s.c.call("sqlite3_expanded_sql", uint64(s.handle))
sql := util.ReadString(s.c.mod, uint32(r), _MAX_SQL_LENGTH)
s.c.free(uint32(r))
return sql
}
// ReadOnly returns true if and only if the statement
// makes no direct changes to the content of the database file.
//
@@ -77,7 +106,7 @@ func (s *Stmt) Busy() bool {
//
// https://sqlite.org/c3ref/step.html
func (s *Stmt) Step() bool {
s.c.checkInterrupt()
s.c.checkInterrupt(s.c.handle)
r := s.c.call("sqlite3_step", uint64(s.handle))
switch r {
case _ROW:
@@ -218,15 +247,15 @@ func (s *Stmt) BindText(param int, value string) error {
return TOOBIG
}
ptr := s.c.newString(value)
r := s.c.call("sqlite3_bind_text64",
r := s.c.call("sqlite3_bind_text_go",
uint64(s.handle), uint64(param),
uint64(ptr), uint64(len(value)),
uint64(s.c.freer), _UTF8)
uint64(ptr), uint64(len(value)))
return s.c.error(r)
}
// BindRawText binds a []byte to the prepared statement as text.
// The leftmost SQL parameter has an index of 1.
// Binding a nil slice is the same as calling [Stmt.BindNull].
//
// https://sqlite.org/c3ref/bind_blob.html
func (s *Stmt) BindRawText(param int, value []byte) error {
@@ -234,10 +263,9 @@ func (s *Stmt) BindRawText(param int, value []byte) error {
return TOOBIG
}
ptr := s.c.newBytes(value)
r := s.c.call("sqlite3_bind_text64",
r := s.c.call("sqlite3_bind_text_go",
uint64(s.handle), uint64(param),
uint64(ptr), uint64(len(value)),
uint64(s.c.freer), _UTF8)
uint64(ptr), uint64(len(value)))
return s.c.error(r)
}
@@ -251,10 +279,9 @@ func (s *Stmt) BindBlob(param int, value []byte) error {
return TOOBIG
}
ptr := s.c.newBytes(value)
r := s.c.call("sqlite3_bind_blob64",
r := s.c.call("sqlite3_bind_blob_go",
uint64(s.handle), uint64(param),
uint64(ptr), uint64(len(value)),
uint64(s.c.freer))
uint64(ptr), uint64(len(value)))
return s.c.error(r)
}
@@ -283,7 +310,8 @@ func (s *Stmt) BindNull(param int) error {
//
// https://sqlite.org/c3ref/bind_blob.html
func (s *Stmt) BindTime(param int, value time.Time, format TimeFormat) error {
if format == TimeFormatDefault {
switch format {
case TimeFormatDefault, TimeFormatAuto, time.RFC3339Nano:
return s.bindRFC3339Nano(param, value)
}
switch v := format.Encode(value).(type) {
@@ -306,10 +334,9 @@ func (s *Stmt) bindRFC3339Nano(param int, value time.Time) error {
buf := util.View(s.c.mod, ptr, maxlen)
buf = value.AppendFormat(buf[:0], time.RFC3339Nano)
r := s.c.call("sqlite3_bind_text64",
r := s.c.call("sqlite3_bind_text_go",
uint64(s.handle), uint64(param),
uint64(ptr), uint64(len(buf)),
uint64(s.c.freer), _UTF8)
uint64(ptr), uint64(len(buf)))
return s.c.error(r)
}
@@ -351,6 +378,15 @@ func (s *Stmt) BindValue(param int, value Value) error {
return s.c.error(r)
}
// DataCount resets the number of columns in a result set.
//
// https://sqlite.org/c3ref/data_count.html
func (s *Stmt) DataCount() int {
r := s.c.call("sqlite3_data_count",
uint64(s.handle))
return int(int32(r))
}
// ColumnCount returns the number of columns in a result set.
//
// https://sqlite.org/c3ref/column_count.html
@@ -367,12 +403,10 @@ func (s *Stmt) ColumnCount() int {
func (s *Stmt) ColumnName(col int) string {
r := s.c.call("sqlite3_column_name",
uint64(s.handle), uint64(col))
ptr := uint32(r)
if ptr == 0 {
if r == 0 {
panic(util.OOMErr)
}
return util.ReadString(s.c.mod, ptr, _MAX_NAME)
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnType returns the initial [Datatype] of the result column.
@@ -398,15 +432,57 @@ func (s *Stmt) ColumnDeclType(col int) string {
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnDatabaseName returns the name of the database
// that is the origin of a particular result column.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_database_name.html
func (s *Stmt) ColumnDatabaseName(col int) string {
r := s.c.call("sqlite3_column_database_name",
uint64(s.handle), uint64(col))
if r == 0 {
return ""
}
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnTableName returns the name of the table
// that is the origin of a particular result column.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_database_name.html
func (s *Stmt) ColumnTableName(col int) string {
r := s.c.call("sqlite3_column_table_name",
uint64(s.handle), uint64(col))
if r == 0 {
return ""
}
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnOriginName returns the name of the table column
// that is the origin of a particular result column.
// The leftmost column of the result set has the index 0.
//
// https://sqlite.org/c3ref/column_database_name.html
func (s *Stmt) ColumnOriginName(col int) string {
r := s.c.call("sqlite3_column_origin_name",
uint64(s.handle), uint64(col))
if r == 0 {
return ""
}
return util.ReadString(s.c.mod, uint32(r), _MAX_NAME)
}
// ColumnBool returns the value of the result column as a bool.
// The leftmost column of the result set has the index 0.
// SQLite does not have a separate boolean storage class.
// Instead, boolean values are retrieved as integers,
// Instead, boolean values are retrieved as numbers,
// with 0 converted to false and any other value to true.
//
// https://sqlite.org/c3ref/column_blob.html
func (s *Stmt) ColumnBool(col int) bool {
return s.ColumnInt64(col) != 0
return s.ColumnFloat(col) != 0
}
// ColumnInt returns the value of the result column as an int.
@@ -524,7 +600,7 @@ func (s *Stmt) ColumnJSON(col int, ptr any) error {
var data []byte
switch s.ColumnType(col) {
case NULL:
data = append(data, "null"...)
data = []byte("null")
case TEXT:
data = s.ColumnRawText(col)
case BLOB:
@@ -565,7 +641,7 @@ func (s *Stmt) Columns(dest []any) error {
defer s.c.arena.mark()()
count := uint64(len(dest))
typePtr := s.c.arena.new(count)
dataPtr := s.c.arena.new(8 * count)
dataPtr := s.c.arena.new(count * 8)
r := s.c.call("sqlite3_columns_go",
uint64(s.handle), count, uint64(typePtr), uint64(dataPtr))
@@ -574,26 +650,31 @@ func (s *Stmt) Columns(dest []any) error {
}
types := util.View(s.c.mod, typePtr, count)
// Avoid bounds checks on types below.
if len(types) != len(dest) {
panic(util.AssertErr())
}
for i := range dest {
switch types[i] {
case byte(INTEGER):
dest[i] = int64(util.ReadUint64(s.c.mod, dataPtr+8*uint32(i)))
continue
dest[i] = int64(util.ReadUint64(s.c.mod, dataPtr))
case byte(FLOAT):
dest[i] = util.ReadFloat64(s.c.mod, dataPtr+8*uint32(i))
continue
dest[i] = util.ReadFloat64(s.c.mod, dataPtr)
case byte(NULL):
dest[i] = nil
continue
}
ptr := util.ReadUint32(s.c.mod, dataPtr+8*uint32(i)+0)
len := util.ReadUint32(s.c.mod, dataPtr+8*uint32(i)+4)
buf := util.View(s.c.mod, ptr, uint64(len))
if types[i] == byte(TEXT) {
dest[i] = string(buf)
} else {
dest[i] = buf
default:
ptr := util.ReadUint32(s.c.mod, dataPtr+0)
len := util.ReadUint32(s.c.mod, dataPtr+4)
buf := util.View(s.c.mod, ptr, uint64(len))
if types[i] == byte(TEXT) {
dest[i] = string(buf)
} else {
dest[i] = buf
}
}
dataPtr += 8
}
return nil
}
+9 -6
View File
@@ -101,7 +101,7 @@ func (f TimeFormat) Encode(t time.Time) any {
return t.UnixMicro()
case TimeFormatUnixNano:
return t.UnixNano()
// Special formats
// Special formats.
case TimeFormatDefault, TimeFormatAuto:
f = time.RFC3339Nano
// SQLite assumes UTC if unspecified.
@@ -138,8 +138,11 @@ func (f TimeFormat) Encode(t time.Time) any {
//
// https://sqlite.org/lang_datefunc.html
func (f TimeFormat) Decode(v any) (time.Time, error) {
if t, ok := v.(time.Time); ok {
return t, nil
}
switch f {
// Numeric formats
// Numeric formats.
case TimeFormatJulianDay:
switch v := v.(type) {
case string:
@@ -183,7 +186,7 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
case float64:
return time.UnixMilli(int64(math.Floor(v))).UTC(), nil
case int64:
return time.UnixMilli(int64(v)).UTC(), nil
return time.UnixMilli(v).UTC(), nil
default:
return time.Time{}, util.TimeErr
}
@@ -200,7 +203,7 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
case float64:
return time.UnixMicro(int64(math.Floor(v))).UTC(), nil
case int64:
return time.UnixMicro(int64(v)).UTC(), nil
return time.UnixMicro(v).UTC(), nil
default:
return time.Time{}, util.TimeErr
}
@@ -217,12 +220,12 @@ func (f TimeFormat) Decode(v any) (time.Time, error) {
case float64:
return time.Unix(0, int64(math.Floor(v))).UTC(), nil
case int64:
return time.Unix(0, int64(v)).UTC(), nil
return time.Unix(0, v).UTC(), nil
default:
return time.Time{}, util.TimeErr
}
// Special formats
// Special formats.
case TimeFormatAuto:
switch s := v.(type) {
case string:
+35 -19
View File
@@ -3,14 +3,14 @@ package sqlite3
import (
"context"
"errors"
"fmt"
"math/rand"
"runtime"
"strconv"
"strings"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
// Txn is an in-progress database transaction.
@@ -32,6 +32,19 @@ func (c *Conn) Begin() Txn {
return Txn{c}
}
// BeginConcurrent starts a concurrent transaction.
//
// Experimental: requires a custom build of SQLite.
//
// https://sqlite.org/cgi/src/doc/begin-concurrent/doc/begin_concurrent.md
func (c *Conn) BeginConcurrent() (Txn, error) {
err := c.Exec(`BEGIN CONCURRENT`)
if err != nil {
return Txn{}, err
}
return Txn{c}, nil
}
// BeginImmediate starts an immediate transaction.
//
// https://sqlite.org/lang_transaction.html
@@ -123,23 +136,21 @@ type Savepoint struct {
//
// https://sqlite.org/lang_savepoint.html
func (c *Conn) Savepoint() Savepoint {
// Names can be reused; this makes catching bugs more likely.
name := saveptName() + "_" + strconv.Itoa(int(rand.Int31()))
name := callerName()
if name == "" {
name = "sqlite3.Savepoint"
}
// Names can be reused, but this makes catching bugs more likely.
name = QuoteIdentifier(name + "_" + strconv.Itoa(int(rand.Int31())))
err := c.txnExecInterrupted(fmt.Sprintf("SAVEPOINT %q;", name))
err := c.txnExecInterrupted(`SAVEPOINT ` + name)
if err != nil {
panic(err)
}
return Savepoint{c: c, name: name}
}
func saveptName() (name string) {
defer func() {
if name == "" {
name = "sqlite3.Savepoint"
}
}()
func callerName() (name string) {
var pc [8]uintptr
n := runtime.Callers(3, pc[:])
if n <= 0 {
@@ -176,7 +187,7 @@ func (s Savepoint) Release(errp *error) {
if s.c.GetAutocommit() { // There is nothing to commit.
return
}
*errp = s.c.Exec(fmt.Sprintf("RELEASE %q;", s.name))
*errp = s.c.Exec(`RELEASE ` + s.name)
if *errp == nil {
return
}
@@ -188,10 +199,7 @@ func (s Savepoint) Release(errp *error) {
return
}
// ROLLBACK and RELEASE even if interrupted.
err := s.c.txnExecInterrupted(fmt.Sprintf(`
ROLLBACK TO %[1]q;
RELEASE %[1]q;
`, s.name))
err := s.c.txnExecInterrupted(`ROLLBACK TO ` + s.name + `; RELEASE ` + s.name)
if err != nil {
panic(err)
}
@@ -204,7 +212,7 @@ func (s Savepoint) Release(errp *error) {
// https://sqlite.org/lang_transaction.html
func (s Savepoint) Rollback() error {
// ROLLBACK even if interrupted.
return s.c.txnExecInterrupted(fmt.Sprintf("ROLLBACK TO %q;", s.name))
return s.c.txnExecInterrupted(`ROLLBACK TO ` + s.name)
}
func (c *Conn) txnExecInterrupted(sql string) error {
@@ -217,7 +225,7 @@ func (c *Conn) txnExecInterrupted(sql string) error {
return err
}
// TxnState starts a deferred transaction.
// TxnState determines the transaction state of a database.
//
// https://sqlite.org/c3ref/txn_state.html
func (c *Conn) TxnState(schema string) TxnState {
@@ -292,3 +300,11 @@ func updateCallback(ctx context.Context, mod api.Module, pDB uint32, action Auth
c.update(action, schema, table, int64(rowid))
}
}
// CacheFlush flushes caches to disk mid-transaction.
//
// https://sqlite.org/c3ref/db_cacheflush.html
func (c *Conn) CacheFlush() error {
r := c.call("sqlite3_db_cacheflush", uint64(c.handle))
return c.error(r)
}
+1 -1
View File
@@ -15,7 +15,7 @@ func OpenFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
if name == "" {
return nil, &os.PathError{Op: "open", Path: name, Err: ENOENT}
}
r, e := syscallOpen(name, flag, uint32(perm.Perm()))
r, e := syscallOpen(name, flag|O_CLOEXEC, uint32(perm.Perm()))
if e != nil {
return nil, &os.PathError{Op: "open", Path: name, Err: e}
}
+1 -1
View File
@@ -1,2 +1,2 @@
// Package osutil implements operating system utility functions.
// Package osutil implements operating system utilities.
package osutil
+9
View File
@@ -0,0 +1,9 @@
# SQLite utility functions
This package implements assorted SQLite utilities
useful to extension writers.
It also wraps a [parser](https://github.com/marcobambini/sqlite-createtable-parser)
for the [`CREATE`](https://sqlite.org/lang_createtable.html) and
[`ALTER TABLE`](https://sqlite.org/lang_altertable.html) commands,
created by [Marco Bambini](https://github.com/marcobambini).
+65
View File
@@ -0,0 +1,65 @@
package sql3util
import "strings"
// NamedArg splits an named arg into a key and value,
// around an equals sign.
// Spaces are trimmed around both key and value.
func NamedArg(arg string) (key, val string) {
key, val, _ = strings.Cut(arg, "=")
key = strings.TrimSpace(key)
val = strings.TrimSpace(val)
return
}
// Unquote unquotes a string.
//
// https://sqlite.org/lang_keywords.html
func Unquote(val string) string {
if len(val) < 2 {
return val
}
fst := val[0]
lst := val[len(val)-1]
rst := val[1 : len(val)-1]
if fst == '[' && lst == ']' {
return rst
}
if fst != lst {
return val
}
var old, new string
switch fst {
default:
return val
case '`':
old, new = "``", "`"
case '"':
old, new = `""`, `"`
case '\'':
old, new = `''`, `'`
}
return strings.ReplaceAll(rst, old, new)
}
// ParseBool parses a boolean.
//
// https://sqlite.org/pragma.html#syntax
func ParseBool(s string) (b, ok bool) {
if len(s) == 0 {
return false, false
}
if s[0] == '0' {
return false, true
}
if '1' <= s[0] && s[0] <= '9' {
return true, true
}
switch strings.ToLower(s) {
case "true", "yes", "on":
return true, true
case "false", "no", "off":
return false, true
}
return false, false
}
+61
View File
@@ -0,0 +1,61 @@
package sql3util
const (
_NONE = iota
_MEMORY
_SYNTAX
_UNSUPPORTEDSQL
)
type ConflictClause uint32
const (
CONFLICT_NONE ConflictClause = iota
CONFLICT_ROLLBACK
CONFLICT_ABORT
CONFLICT_FAIL
CONFLICT_IGNORE
CONFLICT_REPLACE
)
type OrderClause uint32
const (
ORDER_NONE OrderClause = iota
ORDER_ASC
ORDER_DESC
)
type FKAction uint32
const (
FKACTION_NONE FKAction = iota
FKACTION_SETNULL
FKACTION_SETDEFAULT
FKACTION_CASCADE
FKACTION_RESTRICT
FKACTION_NOACTION
)
type FKDefType uint32
const (
DEFTYPE_NONE FKDefType = iota
DEFTYPE_DEFERRABLE
DEFTYPE_DEFERRABLE_INITIALLY_DEFERRED
DEFTYPE_DEFERRABLE_INITIALLY_IMMEDIATE
DEFTYPE_NOTDEFERRABLE
DEFTYPE_NOTDEFERRABLE_INITIALLY_DEFERRED
DEFTYPE_NOTDEFERRABLE_INITIALLY_IMMEDIATE
)
type StatementType uint32
const (
CREATE_UNKNOWN StatementType = iota
CREATE_TABLE
ALTER_RENAME_TABLE
ALTER_RENAME_COLUMN
ALTER_ADD_COLUMN
ALTER_DROP_COLUMN
)
+210
View File
@@ -0,0 +1,210 @@
package sql3util
import (
"context"
_ "embed"
"sync"
"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
const (
errp = 4
sqlp = 8
)
var (
//go:embed parse/sql3parse_table.wasm
binary []byte
once sync.Once
runtime wazero.Runtime
compiled wazero.CompiledModule
)
// ParseTable parses a [CREATE] or [ALTER TABLE] command.
//
// [CREATE]: https://sqlite.org/lang_createtable.html
// [ALTER TABLE]: https://sqlite.org/lang_altertable.html
func ParseTable(sql string) (_ *Table, err error) {
once.Do(func() {
ctx := context.Background()
cfg := wazero.NewRuntimeConfigInterpreter()
runtime = wazero.NewRuntimeWithConfig(ctx, cfg)
compiled, err = runtime.CompileModule(ctx, binary)
})
if err != nil {
return nil, err
}
ctx := context.Background()
mod, err := runtime.InstantiateModule(ctx, compiled, wazero.NewModuleConfig().WithName(""))
if err != nil {
return nil, err
}
defer mod.Close(ctx)
if buf, ok := mod.Memory().Read(sqlp, uint32(len(sql))); ok {
copy(buf, sql)
}
stack := [...]uint64{sqlp, uint64(len(sql)), errp}
err = mod.ExportedFunction("sql3parse_table").CallWithStack(ctx, stack[:])
if err != nil {
return nil, err
}
c, _ := mod.Memory().ReadUint32Le(errp)
switch c {
case _MEMORY:
panic(util.OOMErr)
case _SYNTAX:
return nil, util.ErrorString("sql3parse: invalid syntax")
case _UNSUPPORTEDSQL:
return nil, util.ErrorString("sql3parse: unsupported SQL")
}
var tab Table
tab.load(mod, uint32(stack[0]), sql)
return &tab, nil
}
// Table holds metadata about a table.
type Table struct {
Name string
Schema string
Comment string
IsTemporary bool
IsIfNotExists bool
IsWithoutRowID bool
IsStrict bool
Columns []Column
Type StatementType
CurrentName string
NewName string
}
func (t *Table) load(mod api.Module, ptr uint32, sql string) {
t.Name = loadString(mod, ptr+0, sql)
t.Schema = loadString(mod, ptr+8, sql)
t.Comment = loadString(mod, ptr+16, sql)
t.IsTemporary = loadBool(mod, ptr+24)
t.IsIfNotExists = loadBool(mod, ptr+25)
t.IsWithoutRowID = loadBool(mod, ptr+26)
t.IsStrict = loadBool(mod, ptr+27)
t.Columns = loadSlice(mod, ptr+28, func(ptr uint32, res *Column) {
p, _ := mod.Memory().ReadUint32Le(ptr)
res.load(mod, p, sql)
})
t.Type = loadEnum[StatementType](mod, ptr+44)
t.CurrentName = loadString(mod, ptr+48, sql)
t.NewName = loadString(mod, ptr+56, sql)
}
// Column holds metadata about a column.
type Column struct {
Name string
Type string
Length string
ConstraintName string
Comment string
IsPrimaryKey bool
IsAutoIncrement bool
IsNotNull bool
IsUnique bool
PKOrder OrderClause
PKConflictClause ConflictClause
NotNullConflictClause ConflictClause
UniqueConflictClause ConflictClause
CheckExpr string
DefaultExpr string
CollateName string
ForeignKeyClause *ForeignKey
}
func (c *Column) load(mod api.Module, ptr uint32, sql string) {
c.Name = loadString(mod, ptr+0, sql)
c.Type = loadString(mod, ptr+8, sql)
c.Length = loadString(mod, ptr+16, sql)
c.ConstraintName = loadString(mod, ptr+24, sql)
c.Comment = loadString(mod, ptr+32, sql)
c.IsPrimaryKey = loadBool(mod, ptr+40)
c.IsAutoIncrement = loadBool(mod, ptr+41)
c.IsNotNull = loadBool(mod, ptr+42)
c.IsUnique = loadBool(mod, ptr+43)
c.PKOrder = loadEnum[OrderClause](mod, ptr+44)
c.PKConflictClause = loadEnum[ConflictClause](mod, ptr+48)
c.NotNullConflictClause = loadEnum[ConflictClause](mod, ptr+52)
c.UniqueConflictClause = loadEnum[ConflictClause](mod, ptr+56)
c.CheckExpr = loadString(mod, ptr+60, sql)
c.DefaultExpr = loadString(mod, ptr+68, sql)
c.CollateName = loadString(mod, ptr+76, sql)
if ptr, _ := mod.Memory().ReadUint32Le(ptr + 84); ptr != 0 {
c.ForeignKeyClause = &ForeignKey{}
c.ForeignKeyClause.load(mod, ptr, sql)
}
}
type ForeignKey struct {
Table string
Columns []string
OnDelete FKAction
OnUpdate FKAction
Match string
Deferrable FKDefType
}
func (f *ForeignKey) load(mod api.Module, ptr uint32, sql string) {
f.Table = loadString(mod, ptr+0, sql)
f.Columns = loadSlice(mod, ptr+8, func(ptr uint32, res *string) {
*res = loadString(mod, ptr, sql)
})
f.OnDelete = loadEnum[FKAction](mod, ptr+16)
f.OnUpdate = loadEnum[FKAction](mod, ptr+20)
f.Match = loadString(mod, ptr+24, sql)
f.Deferrable = loadEnum[FKDefType](mod, ptr+32)
}
func loadString(mod api.Module, ptr uint32, sql string) string {
off, _ := mod.Memory().ReadUint32Le(ptr + 0)
if off == 0 {
return ""
}
len, _ := mod.Memory().ReadUint32Le(ptr + 4)
return sql[off-sqlp : off+len-sqlp]
}
func loadSlice[T any](mod api.Module, ptr uint32, fn func(uint32, *T)) []T {
ref, _ := mod.Memory().ReadUint32Le(ptr + 4)
if ref == 0 {
return nil
}
len, _ := mod.Memory().ReadUint32Le(ptr + 0)
res := make([]T, len)
for i := range res {
fn(ref, &res[i])
ref += 4
}
return res
}
func loadEnum[T ~uint32](mod api.Module, ptr uint32) T {
val, _ := mod.Memory().ReadUint32Le(ptr)
return T(val)
}
func loadBool(mod api.Module, ptr uint32) bool {
val, _ := mod.Memory().ReadByte(ptr)
return val != 0
}
Binary file not shown.
+9
View File
@@ -0,0 +1,9 @@
// Package sql3util implements SQLite utilities.
package sql3util
// ValidPageSize returns true if s is a valid page size.
//
// https://sqlite.org/fileformat.html#pages
func ValidPageSize(s int) bool {
return 512 <= s && s <= 65536 && s&(s-1) == 0
}
+11 -3
View File
@@ -68,12 +68,12 @@ func (v Value) NumericType() Datatype {
// Bool returns the value as a bool.
// SQLite does not have a separate boolean storage class.
// Instead, boolean values are retrieved as integers,
// Instead, boolean values are retrieved as numbers,
// with 0 converted to false and any other value to true.
//
// https://sqlite.org/c3ref/value_blob.html
func (v Value) Bool() bool {
return v.Int64() != 0
return v.Float() != 0
}
// Int returns the value as an int.
@@ -177,7 +177,7 @@ func (v Value) JSON(ptr any) error {
var data []byte
switch v.Type() {
case NULL:
data = append(data, "null"...)
data = []byte("null")
case TEXT:
data = v.RawText()
case BLOB:
@@ -201,6 +201,14 @@ func (v Value) NoChange() bool {
return r != 0
}
// FromBind returns true if value originated from a bound parameter.
//
// https://sqlite.org/c3ref/value_blob.html
func (v Value) FromBind() bool {
r := v.c.call("sqlite3_value_frombind", v.protected())
return r != 0
}
// InFirst returns the first element
// on the right-hand side of an IN constraint.
//
+70 -34
View File
@@ -2,78 +2,114 @@
This package implements the SQLite [OS Interface](https://sqlite.org/vfs.html) (aka VFS).
It replaces the default SQLite VFS with a **pure Go** implementation.
It also exposes [interfaces](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#VFS)
that should allow you to implement your own custom VFSes.
It replaces the default SQLite VFS with a **pure Go** implementation,
and exposes [interfaces](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#VFS)
that should allow you to implement your own [custom VFSes](#custom-vfses).
Since it is a from scratch reimplementation,
there are naturally some ways it deviates from the original.
The main differences are [file locking](#file-locking) and [WAL mode](write-ahead-logging) support.
The main differences are [file locking](#file-locking) and [WAL mode](#write-ahead-logging) support.
### File Locking
POSIX advisory locks, which SQLite uses on Unix, are
[broken by design](https://github.com/sqlite/sqlite/blob/b74eb0/src/os_unix.c#L1073-L1161).
On Linux and macOS, this module uses
Instead, on Linux and macOS, this package uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
OFD locks are fully compatible with POSIX advisory locks.
On BSD Unixes, this module uses
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2).
On BSD, these locks are fully compatible with POSIX advisory locks.
However, concurrency is reduced with BSD locks
(`BEGIN IMMEDIATE` behaves the same as `BEGIN EXCLUSIVE`).
This package can also use
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2),
albeit with reduced concurrency (`BEGIN IMMEDIATE` behaves like `BEGIN EXCLUSIVE`).
BSD locks are the default on BSD and illumos,
but you can opt into them with the `sqlite3_flock` build tag.
On Windows, this module uses `LockFileEx` and `UnlockFileEx`,
On Windows, this package uses `LockFileEx` and `UnlockFileEx`,
like SQLite.
On all other platforms, file locking is not supported, and you must use
You can also opt into a cross-platform locking implementation
with the `sqlite3_dotlk` build tag.
Otherwise, file locking is not supported, and you must use
[`nolock=1`](https://sqlite.org/uri.html#urinolock)
(or [`immutable=1`](https://sqlite.org/uri.html#uriimmutable))
to open database files.\
to open database files.
To use the [`database/sql`](https://pkg.go.dev/database/sql) driver
with `nolock=1` you must disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
You can use [`vfs.SupportsFileLocking`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsFileLocking)
to check if your platform supports file locking.
to check if your build supports file locking.
### Write-Ahead Logging
On 64-bit Linux and macOS, this module uses `mmap` to implement
On Unix, this package may use `mmap` to implement
[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
like SQLite.
To allow `mmap` to work, each connection needs to reserve up to 4GB of address space.\
To limit the amount of address space each connection needs,
use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go).
With [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2)
a WAL database can only be accessed by a single proccess.
Other processes that attempt to access a database locked with BSD locks,
will fail with the [`SQLITE_PROTOCOL`](https://sqlite.org/rescode.html#protocol) error code.
On Windows and BSD, [WAL](https://sqlite.org/wal.html) support is
[limited](https://sqlite.org/wal.html#noshm).
`EXCLUSIVE` locking mode can be set to create, read, and write WAL databases.\
On Windows, this package may use `MapViewOfFile`, like SQLite.
You can also opt into a cross-platform, in-process, memory sharing implementation
with the `sqlite3_dotlk` build tag.
Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm),
and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases.
To use `EXCLUSIVE` locking mode with the
[`database/sql`](https://pkg.go.dev/database/sql) driver
you must disable connection pooling by calling
[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns).
On all other platforms, where file locking is not supported, WAL mode does not work.
You can use [`vfs.SupportsSharedMemory`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsSharedMemory)
to check if your platform supports shared memory.
to check if your build supports shared memory.
### Batch-Atomic Write
On 64-bit Linux, this module supports [batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
with the F2FS filesystem.
On Linux, this package may support
[batch-atomic writes](https://sqlite.org/cgi/src/technote/714)
on the F2FS filesystem.
### Build tags
### Checksums
This package can be [configured](https://pkg.go.dev/github.com/ncruces/go-sqlite3#Conn.EnableChecksums)
to add an 8-byte checksum to the end of every page in an SQLite database.
The checksum is added as each page is written
and verified as each page is read.\
The checksum is intended to help detect database corruption
caused by random bit-flips in the mass storage device.
The implementation is compatible with SQLite's
[Checksum VFS Shim](https://sqlite.org/cksumvfs.html).
### Build Tags
The VFS can be customized with a few build tags:
- `sqlite3_flock` forces the use of BSD locks; it can be used on macOS to test the BSD locking implementation.
- `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys);
disables locking _and_ shared memory on all platforms.
- `sqlite3_noshm` disables shared memory on all platforms.
- `sqlite3_flock` forces the use of BSD locks.
- `sqlite3_dotlk` forces the use of dot-file locks.
- `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys).
> [!IMPORTANT]
> The default configuration of this package is compatible with the standard
> [Unix and Windows SQLite VFSes](https://sqlite.org/vfs.html#multiple_vfses);
> `sqlite3_flock` builds are compatible with the
> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style);
> `sqlite3_dotlk` builds are compatible with the
> [`unix-dotfile` VFS](https://sqlite.org/compile.html#enable_locking_style).
> If incompatible file locking is used, accessing databases concurrently with
> _other_ SQLite libraries will eventually corrupt data.
### Custom VFSes
- [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb)
implements an in-memory VFS.
- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs)
implements a VFS for immutable databases.
- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum)
wraps a VFS to offer encryption at rest.
- [`github.com/ncruces/go-sqlite3/vfs/xts`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/xts)
wraps a VFS to offer encryption at rest.
+51 -33
View File
@@ -49,6 +49,13 @@ type File interface {
DeviceCharacteristics() DeviceCharacteristic
}
// FileUnwrap should be implemented by a File
// that wraps another File implementation.
type FileUnwrap interface {
File
Unwrap() File
}
// FileLockState extends File to implement the
// SQLITE_FCNTL_LOCKSTATE file control opcode.
//
@@ -58,6 +65,26 @@ type FileLockState interface {
LockState() LockLevel
}
// FilePersistentWAL extends File to implement the
// SQLITE_FCNTL_PERSIST_WAL file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpersistwal
type FilePersistentWAL interface {
File
PersistentWAL() bool
SetPersistentWAL(bool)
}
// FilePowersafeOverwrite extends File to implement the
// SQLITE_FCNTL_POWERSAFE_OVERWRITE file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpowersafeoverwrite
type FilePowersafeOverwrite interface {
File
PowersafeOverwrite() bool
SetPowersafeOverwrite(bool)
}
// FileChunkSize extends File to implement the
// SQLITE_FCNTL_CHUNK_SIZE file control opcode.
//
@@ -94,26 +121,6 @@ type FileOverwrite interface {
Overwrite() error
}
// FilePersistentWAL extends File to implement the
// SQLITE_FCNTL_PERSIST_WAL file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpersistwal
type FilePersistentWAL interface {
File
PersistentWAL() bool
SetPersistentWAL(bool)
}
// FilePowersafeOverwrite extends File to implement the
// SQLITE_FCNTL_POWERSAFE_OVERWRITE file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpowersafeoverwrite
type FilePowersafeOverwrite interface {
File
PowersafeOverwrite() bool
SetPowersafeOverwrite(bool)
}
// FileCommitPhaseTwo extends File to implement the
// SQLITE_FCNTL_COMMIT_PHASETWO file control opcode.
//
@@ -135,15 +142,6 @@ type FileBatchAtomicWrite interface {
RollbackAtomicWrite() error
}
// FilePragma extends File to implement the
// SQLITE_FCNTL_PRAGMA file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpragma
type FilePragma interface {
File
Pragma(name, value string) (string, error)
}
// FileCheckpoint extends File to implement the
// SQLITE_FCNTL_CKPT_START and SQLITE_FCNTL_CKPT_DONE
// file control opcodes.
@@ -151,8 +149,17 @@ type FilePragma interface {
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlckptstart
type FileCheckpoint interface {
File
CheckpointDone() error
CheckpointStart() error
CheckpointStart()
CheckpointDone()
}
// FilePragma extends File to implement the
// SQLITE_FCNTL_PRAGMA file control opcode.
//
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpragma
type FilePragma interface {
File
Pragma(name, value string) (string, error)
}
// FileSharedMemory extends File to possibly implement
@@ -168,8 +175,19 @@ type FileSharedMemory interface {
// SharedMemory is a shared-memory WAL-index implementation.
// Use [NewSharedMemory] to create a shared-memory.
type SharedMemory interface {
shmMap(context.Context, api.Module, int32, int32, bool) (uint32, error)
shmLock(int32, int32, _ShmFlag) error
shmMap(context.Context, api.Module, int32, int32, bool) (uint32, _ErrorCode)
shmLock(int32, int32, _ShmFlag) _ErrorCode
shmUnmap(bool)
shmBarrier()
io.Closer
}
type blockingSharedMemory interface {
SharedMemory
shmEnableBlocking(block bool)
}
type fileControl interface {
File
fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg uint32) _ErrorCode
}
+157
View File
@@ -0,0 +1,157 @@
package vfs
import (
"bytes"
"context"
_ "embed"
"encoding/binary"
"strconv"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/util/sql3util"
)
func cksmWrapFile(name *Filename, flags OpenFlag, file File) File {
// Checksum only main databases and WALs.
if flags&(OPEN_MAIN_DB|OPEN_WAL) == 0 {
return file
}
cksm := cksmFile{File: file}
if flags&OPEN_WAL != 0 {
main, _ := name.DatabaseFile().(cksmFile)
cksm.cksmFlags = main.cksmFlags
} else {
cksm.cksmFlags = new(cksmFlags)
cksm.isDB = true
}
return cksm
}
type cksmFile struct {
File
*cksmFlags
isDB bool
}
type cksmFlags struct {
computeCksm bool
verifyCksm bool
inCkpt bool
pageSize int
}
func (c cksmFile) ReadAt(p []byte, off int64) (n int, err error) {
n, err = c.File.ReadAt(p, off)
// SQLite is reading the header of a database file.
if c.isDB && off == 0 && len(p) >= 100 &&
bytes.HasPrefix(p, []byte("SQLite format 3\000")) {
c.init(p)
}
// Verify checksums.
if c.verifyCksm && !c.inCkpt && len(p) == c.pageSize {
cksm1 := cksmCompute(p[:len(p)-8])
cksm2 := *(*[8]byte)(p[len(p)-8:])
if cksm1 != cksm2 {
return 0, _IOERR_DATA
}
}
return n, err
}
func (c cksmFile) WriteAt(p []byte, off int64) (n int, err error) {
// SQLite is writing the first page of a database file.
if c.isDB && off == 0 && len(p) >= 100 &&
bytes.HasPrefix(p, []byte("SQLite format 3\000")) {
c.init(p)
}
// Compute checksums.
if c.computeCksm && !c.inCkpt && len(p) == c.pageSize {
*(*[8]byte)(p[len(p)-8:]) = cksmCompute(p[:len(p)-8])
}
return c.File.WriteAt(p, off)
}
func (c cksmFile) Pragma(name string, value string) (string, error) {
switch name {
case "checksum_verification":
b, ok := sql3util.ParseBool(value)
if ok {
c.verifyCksm = b && c.computeCksm
}
if !c.verifyCksm {
return "0", nil
}
return "1", nil
case "page_size":
if c.computeCksm {
// Do not allow page size changes on a checksum database.
return strconv.Itoa(c.pageSize), nil
}
}
return "", _NOTFOUND
}
func (c cksmFile) DeviceCharacteristics() DeviceCharacteristic {
res := c.File.DeviceCharacteristics()
if c.verifyCksm {
res &^= IOCAP_SUBPAGE_READ
}
return res
}
func (c cksmFile) fileControl(ctx context.Context, mod api.Module, op _FcntlOpcode, pArg uint32) _ErrorCode {
switch op {
case _FCNTL_CKPT_START:
c.inCkpt = true
case _FCNTL_CKPT_DONE:
c.inCkpt = false
}
if rc := vfsFileControlImpl(ctx, mod, c, op, pArg); rc != _NOTFOUND {
return rc
}
return vfsFileControlImpl(ctx, mod, c.File, op, pArg)
}
func (f *cksmFlags) init(header []byte) {
f.pageSize = 256 * int(binary.LittleEndian.Uint16(header[16:18]))
if r := header[20] == 8; r != f.computeCksm {
f.computeCksm = r
f.verifyCksm = r
}
}
func cksmCompute(a []byte) (cksm [8]byte) {
var s1, s2 uint32
for len(a) >= 8 {
s1 += binary.LittleEndian.Uint32(a[0:4]) + s2
s2 += binary.LittleEndian.Uint32(a[4:8]) + s1
a = a[8:]
}
if len(a) != 0 {
panic(util.AssertErr())
}
binary.LittleEndian.PutUint32(cksm[0:4], s1)
binary.LittleEndian.PutUint32(cksm[4:8], s2)
return
}
func (c cksmFile) SharedMemory() SharedMemory {
if f, ok := c.File.(FileSharedMemory); ok {
return f.SharedMemory()
}
return nil
}
func (c cksmFile) Unwrap() File {
return c.File
}
+8
View File
@@ -47,9 +47,12 @@ const (
_IOERR_SHMMAP _ErrorCode = util.IOERR_SHMMAP
_IOERR_SEEK _ErrorCode = util.IOERR_SEEK
_IOERR_DELETE_NOENT _ErrorCode = util.IOERR_DELETE_NOENT
_IOERR_GETTEMPPATH _ErrorCode = util.IOERR_GETTEMPPATH
_IOERR_BEGIN_ATOMIC _ErrorCode = util.IOERR_BEGIN_ATOMIC
_IOERR_COMMIT_ATOMIC _ErrorCode = util.IOERR_COMMIT_ATOMIC
_IOERR_ROLLBACK_ATOMIC _ErrorCode = util.IOERR_ROLLBACK_ATOMIC
_IOERR_DATA _ErrorCode = util.IOERR_DATA
_BUSY_SNAPSHOT _ErrorCode = util.BUSY_SNAPSHOT
_CANTOPEN_FULLPATH _ErrorCode = util.CANTOPEN_FULLPATH
_CANTOPEN_ISDIR _ErrorCode = util.CANTOPEN_ISDIR
_READONLY_CANTINIT _ErrorCode = util.READONLY_CANTINIT
@@ -174,6 +177,7 @@ const (
IOCAP_POWERSAFE_OVERWRITE DeviceCharacteristic = 0x00001000
IOCAP_IMMUTABLE DeviceCharacteristic = 0x00002000
IOCAP_BATCH_ATOMIC DeviceCharacteristic = 0x00004000
IOCAP_SUBPAGE_READ DeviceCharacteristic = 0x00008000
)
// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html
@@ -231,4 +235,8 @@ const (
_SHM_LOCK _ShmFlag = 2
_SHM_SHARED _ShmFlag = 4
_SHM_EXCLUSIVE _ShmFlag = 8
_SHM_NLOCK = 8
_SHM_BASE = 120
_SHM_DMS = _SHM_BASE + _SHM_NLOCK
)
+22 -16
View File
@@ -19,25 +19,26 @@ func (vfsOS) FullPathname(path string) (string, error) {
if err != nil {
return "", err
}
fi, err := os.Lstat(path)
return path, testSymlinks(filepath.Dir(path))
}
func testSymlinks(path string) error {
p, err := filepath.EvalSymlinks(path)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return path, nil
}
return "", err
return err
}
if fi.Mode()&fs.ModeSymlink != 0 {
err = _OK_SYMLINK
if p != path {
return _OK_SYMLINK
}
return path, err
return nil
}
func (vfsOS) Delete(path string, syncDir bool) error {
err := os.Remove(path)
if errors.Is(err, fs.ErrNotExist) {
return _IOERR_DELETE_NOENT
}
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return _IOERR_DELETE_NOENT
}
return err
}
if runtime.GOOS != "windows" && syncDir {
@@ -69,11 +70,12 @@ func (vfsOS) Access(name string, flags AccessFlag) (bool, error) {
}
func (vfsOS) Open(name string, flags OpenFlag) (File, OpenFlag, error) {
// notest // OpenFilename is called instead
return nil, 0, _CANTOPEN
}
func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error) {
var oflags int
oflags := _O_NOFOLLOW
if flags&OPEN_EXCLUSIVE != 0 {
oflags |= os.O_EXCL
}
@@ -95,6 +97,9 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error
f, err = osutil.OpenFile(name.String(), oflags, 0666)
}
if err != nil {
if name == nil {
return nil, flags, _IOERR_GETTEMPPATH
}
if errors.Is(err, syscall.EISDIR) {
return nil, flags, _CANTOPEN_ISDIR
}
@@ -146,6 +151,7 @@ func (f *vfsFile) Close() error {
if f.shm != nil {
f.shm.Close()
}
f.Unlock(LOCK_NONE)
return f.File.Close()
}
@@ -181,7 +187,7 @@ func (f *vfsFile) SectorSize() int {
}
func (f *vfsFile) DeviceCharacteristics() DeviceCharacteristic {
var res DeviceCharacteristic
res := IOCAP_SUBPAGE_READ
if osBatchAtomic(f.File) {
res |= IOCAP_BATCH_ATOMIC
}
@@ -201,10 +207,10 @@ func (f *vfsFile) HasMoved() (bool, error) {
return false, err
}
pi, err := os.Stat(f.Name())
if errors.Is(err, fs.ErrNotExist) {
return true, nil
}
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return true, nil
}
return false, err
}
return !os.SameFile(fi, pi), nil
+8 -3
View File
@@ -4,8 +4,9 @@ import (
"context"
"net/url"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/tetratelabs/wazero/api"
"github.com/ncruces/go-sqlite3/internal/util"
)
// Filename is used by SQLite to pass filenames
@@ -20,8 +21,8 @@ type Filename struct {
stack [2]uint64
}
// OpenFilename is an internal API users should not call directly.
func OpenFilename(ctx context.Context, mod api.Module, id uint32, flags OpenFlag) *Filename {
// GetFilename is an internal API users should not call directly.
func GetFilename(ctx context.Context, mod api.Module, id uint32, flags OpenFlag) *Filename {
if id == 0 {
return nil
}
@@ -66,6 +67,10 @@ func (n *Filename) path(method string) string {
if n == nil || n.zPath == 0 {
return ""
}
if n.flags&(OPEN_MAIN_DB|OPEN_MAIN_JOURNAL|OPEN_WAL) == 0 {
return ""
}
n.stack[0] = uint64(n.zPath)
fn := n.mod.ExportedFunction(method)
if err := fn.CallWithStack(n.ctx, n.stack[:]); err != nil {
+2 -15
View File
@@ -1,4 +1,4 @@
//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys
//go:build ((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk
package vfs
@@ -75,20 +75,7 @@ func (f *vfsFile) Lock(lock LockLevel) error {
if f.lock <= LOCK_NONE || f.lock >= LOCK_EXCLUSIVE {
panic(util.AssertErr())
}
reserved := f.lock == LOCK_RESERVED
// A PENDING lock is needed before acquiring an EXCLUSIVE lock.
if f.lock < LOCK_PENDING {
// If we're already RESERVED, we can block indefinitely,
// since only new readers may briefly hold the PENDING lock.
if rc := osGetPendingLock(f.File, reserved /* block */); rc != _OK {
return rc
}
f.lock = LOCK_PENDING
}
// We already have PENDING, so we're just waiting for readers to leave.
// If we were RESERVED, we can wait for a little while, before invoking
// the busy handler; we will only do this once.
if rc := osGetExclusiveLock(f.File, reserved /* wait */); rc != _OK {
if rc := osGetExclusiveLock(f.File, &f.lock); rc != _OK {
return rc
}
f.lock = LOCK_EXCLUSIVE
+1 -1
View File
@@ -1,4 +1,4 @@
//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) || sqlite3_nosys
//go:build !(((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk)
package vfs
+55 -16
View File
@@ -1,33 +1,72 @@
//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys
//go:build ((freebsd || openbsd || netbsd || dragonfly || illumos) && !(sqlite3_dotlk || sqlite3_nosys)) || sqlite3_flock
package vfs
import (
"os"
"time"
"golang.org/x/sys/unix"
)
func osUnlock(file *os.File, start, len int64) _ErrorCode {
if start == 0 && len == 0 {
err := unix.Flock(int(file.Fd()), unix.LOCK_UN)
if err != nil {
return _IOERR_UNLOCK
}
func osGetSharedLock(file *os.File) _ErrorCode {
return osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
}
func osGetReservedLock(file *os.File) _ErrorCode {
rc := osLock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK)
if rc == _BUSY {
// The documentation states that a lock is upgraded by
// releasing the previous lock, then acquiring the new lock.
// Going over the source code of various BSDs, though,
// with LOCK_NB, the lock is not released,
// and EAGAIN is returned holding the shared lock.
// Still, if we're already in a transaction, we want to abort it,
// so return BUSY_SNAPSHOT here. If there's no transaction active,
// SQLite will change this back to SQLITE_BUSY,
// and invoke the busy handler if appropriate.
return _BUSY_SNAPSHOT
}
return rc
}
func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
if *state >= LOCK_RESERVED {
return _OK
}
return osGetReservedLock(file)
}
func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode {
rc := osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
if rc == _BUSY {
// The documentation states that a lock is downgraded by
// releasing the previous lock then acquiring the new lock.
// Going over the source code of various BSDs, though,
// with LOCK_SH|LOCK_NB this should never happen.
// Return IOERR_RDLOCK, as BUSY would cause an assert to fail.
return _IOERR_RDLOCK
}
return _OK
}
func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode {
err := unix.Flock(int(file.Fd()), unix.LOCK_UN)
if err != nil {
return _IOERR_UNLOCK
}
return _OK
}
func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
// Test the RESERVED lock with fcntl(F_GETLK).
// This only works on systems where fcntl and flock are compatible.
// However, SQLite only calls this while holding a shared lock,
// so the difference is immaterial.
lock, rc := osTestLock(file, _RESERVED_BYTE, 1)
return lock == unix.F_WRLCK, rc
}
func osLock(file *os.File, how int, def _ErrorCode) _ErrorCode {
err := unix.Flock(int(file.Fd()), how)
return osLockErrorCode(err, def)
}
func osReadLock(file *os.File, _ /*start*/, _ /*len*/ int64, _ /*timeout*/ time.Duration) _ErrorCode {
return osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK)
}
func osWriteLock(file *os.File, _ /*start*/, _ /*len*/ int64, _ /*timeout*/ time.Duration) _ErrorCode {
return osLock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK)
}
+4 -2
View File
@@ -1,10 +1,11 @@
//go:build !sqlite3_flock && !sqlite3_nosys
//go:build !(sqlite3_flock || sqlite3_nosys)
package vfs
import (
"io"
"os"
"runtime"
"time"
"golang.org/x/sys/unix"
@@ -68,7 +69,7 @@ func osUnlock(file *os.File, start, len int64) _ErrorCode {
}
func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode {
lock := flocktimeout_t{fl: unix.Flock_t{
lock := &flocktimeout_t{fl: unix.Flock_t{
Type: typ,
Start: start,
Len: len,
@@ -82,6 +83,7 @@ func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, d
default:
lock.timeout = unix.NsecToTimespec(int64(timeout / time.Nanosecond))
err = unix.FcntlFlock(file.Fd(), _F_OFD_SETLKWTIMEOUT, &lock.fl)
runtime.KeepAlive(lock)
}
return osLockErrorCode(err, def)
}
+144
View File
@@ -0,0 +1,144 @@
//go:build sqlite3_dotlk
package vfs
import (
"errors"
"io/fs"
"os"
"sync"
)
var (
// +checklocks:vfsDotLocksMtx
vfsDotLocks = map[string]*vfsDotLocker{}
vfsDotLocksMtx sync.Mutex
)
type vfsDotLocker struct {
shared int // +checklocks:vfsDotLocksMtx
pending *os.File // +checklocks:vfsDotLocksMtx
reserved *os.File // +checklocks:vfsDotLocksMtx
}
func osGetSharedLock(file *os.File) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
f, err := os.OpenFile(name+".lock", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
f.Close()
if errors.Is(err, fs.ErrExist) {
return _BUSY // Another process has the lock.
}
if err != nil {
return _IOERR_LOCK
}
locker = &vfsDotLocker{}
vfsDotLocks[name] = locker
}
if locker.pending != nil {
return _BUSY
}
locker.shared++
return _OK
}
func osGetReservedLock(file *os.File) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_LOCK
}
if locker.reserved != nil && locker.reserved != file {
return _BUSY
}
locker.reserved = file
return _OK
}
func osGetExclusiveLock(file *os.File, _ *LockLevel) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_LOCK
}
if locker.pending != nil && locker.pending != file {
return _BUSY
}
locker.pending = file
if locker.shared > 1 {
return _BUSY
}
return _OK
}
func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_UNLOCK
}
if locker.reserved == file {
locker.reserved = nil
}
if locker.pending == file {
locker.pending = nil
}
return _OK
}
func osReleaseLock(file *os.File, state LockLevel) _ErrorCode {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return _IOERR_UNLOCK
}
if locker.shared == 1 {
err := os.Remove(name + ".lock")
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return _IOERR_UNLOCK
}
delete(vfsDotLocks, name)
}
if locker.reserved == file {
locker.reserved = nil
}
if locker.pending == file {
locker.pending = nil
}
locker.shared--
return _OK
}
func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
vfsDotLocksMtx.Lock()
defer vfsDotLocksMtx.Unlock()
name := file.Name()
locker := vfsDotLocks[name]
if locker == nil {
return false, _OK
}
return locker.reserved != nil, _OK
}
+4 -1
View File
@@ -9,13 +9,16 @@ import (
)
const (
// https://godbolt.org/z/1PcK5vea3
_F2FS_IOC_START_ATOMIC_WRITE = 62721
_F2FS_IOC_COMMIT_ATOMIC_WRITE = 62722
_F2FS_IOC_ABORT_ATOMIC_WRITE = 62725
_F2FS_IOC_GET_FEATURES = 2147808524
_F2FS_IOC_GET_FEATURES = 2147808524 // -2147158772
_F2FS_FEATURE_ATOMIC_WRITE = 4
)
// notest
func osBatchAtomic(file *os.File) bool {
flags, err := unix.IoctlGetInt(int(file.Fd()), _F2FS_IOC_GET_FEATURES)
return err == nil && flags&_F2FS_FEATURE_ATOMIC_WRITE != 0
+4 -3
View File
@@ -1,4 +1,4 @@
//go:build !sqlite3_nosys
//go:build !(sqlite3_flock || sqlite3_nosys)
package vfs
@@ -53,10 +53,11 @@ func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, d
if errno, _ := err.(unix.Errno); errno != unix.EAGAIN {
break
}
if timeout < time.Since(before) {
if time.Since(before) > timeout {
break
}
osSleep(time.Duration(rand.Int63n(int64(time.Millisecond))))
const sleepIncrement = 1024*1024 - 1 // power of two, ~1ms
time.Sleep(time.Duration(rand.Int63() & sleepIncrement))
}
}
return osLockErrorCode(err, def)
+59
View File
@@ -0,0 +1,59 @@
//go:build (linux || darwin) && !(sqlite3_flock || sqlite3_dotlk || sqlite3_nosys)
package vfs
import (
"os"
"time"
"golang.org/x/sys/unix"
)
func osGetSharedLock(file *os.File) _ErrorCode {
// Test the PENDING lock before acquiring a new SHARED lock.
if lock, _ := osTestLock(file, _PENDING_BYTE, 1); lock == unix.F_WRLCK {
return _BUSY
}
// Acquire the SHARED lock.
return osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0)
}
func osGetReservedLock(file *os.File) _ErrorCode {
// Acquire the RESERVED lock.
return osWriteLock(file, _RESERVED_BYTE, 1, 0)
}
func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode {
if *state == LOCK_RESERVED {
// A PENDING lock is needed before acquiring an EXCLUSIVE lock.
if rc := osWriteLock(file, _PENDING_BYTE, 1, -1); rc != _OK {
return rc
}
*state = LOCK_PENDING
}
// Acquire the EXCLUSIVE lock.
return osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, time.Millisecond)
}
func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode {
if state >= LOCK_EXCLUSIVE {
// Downgrade to a SHARED lock.
if rc := osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK {
// notest // this should never happen
return _IOERR_RDLOCK
}
}
// Release the PENDING and RESERVED locks.
return osUnlock(file, _PENDING_BYTE, 2)
}
func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode {
// Release all locks.
return osUnlock(file, 0, 0)
}
func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
// Test the RESERVED lock.
lock, rc := osTestLock(file, _RESERVED_BYTE, 1)
return lock == unix.F_WRLCK, rc
}
@@ -7,6 +7,8 @@ import (
"os"
)
const _O_NOFOLLOW = 0
func osAccess(path string, flags AccessFlag) error {
fi, err := os.Stat(path)
if err != nil {
@@ -34,3 +36,12 @@ func osAccess(path string, flags AccessFlag) error {
}
return nil
}
func osSetMode(file *os.File, modeof string) error {
fi, err := os.Stat(modeof)
if err != nil {
return err
}
file.Chmod(fi.Mode())
return nil
}
Loaded 100 of 299 files, more files were not shown because too many files have changed in this diff. Show more